diff --git a/.github/actions/multi-gpu/aggregate_test_summary.py b/.github/actions/multi-gpu/aggregate_test_summary.py new file mode 100755 index 000000000000..6e1bec25f2f7 --- /dev/null +++ b/.github/actions/multi-gpu/aggregate_test_summary.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Aggregate per-shard JUnit XML reports from the multi-GPU pytest lane. + +Reads ``RUNTIME_DIR`` (the host launcher's work-queue root, holding +``queue/done//``) and the ``tests/test-reports-*.xml`` JUnit files, +then prints a per-file + per-shard table to stdout and, when +``GITHUB_STEP_SUMMARY`` is set, writes the same content as markdown so it +renders at the top of the run page. +""" + +import glob +import os +import pathlib +import xml.etree.ElementTree as ET + +runtime_dir = os.environ.get("RUNTIME_DIR", "") +done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None + +# Map slug -> shard from done//; slug uses "__" for "/". +slug_to_shard = {} +if done_root and done_root.is_dir(): + for shard_dir in done_root.iterdir(): + if not shard_dir.is_dir(): + continue + shard = shard_dir.name # e.g. "cuda-1" + for entry in shard_dir.iterdir(): + slug_to_shard[entry.name] = shard + + +def unslug(name: str) -> str: + # report file pattern: test-reports-.xml + if name.startswith("test-reports-"): + name = name[len("test-reports-") :] + if name.endswith(".xml"): + name = name[:-4] + return name # this is the slug; the file path is slug.replace("__", "/") + + +# Per-file rows from JUnit XMLs. +rows = [] +for path in sorted(glob.glob("tests/test-reports-*.xml")): + slug = unslug(os.path.basename(path)) + file_path = slug.replace("__", "/") + shard = slug_to_shard.get(slug, "?") + try: + tree = ET.parse(path) + root = tree.getroot() + except ET.ParseError: + rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0)) + continue + tests = failures = errors = skipped = 0 + total_time = 0.0 + for ts in root.iter("testsuite"): + tests += int(ts.get("tests", 0)) + failures += int(ts.get("failures", 0)) + errors += int(ts.get("errors", 0)) + skipped += int(ts.get("skipped", 0)) + total_time += float(ts.get("time", 0) or 0) + passed = tests - failures - errors - skipped + status = "PASS" + if failures + errors > 0: + status = "FAIL" + elif tests == 0: + status = "EMPTY" + rows.append((file_path, shard, status, passed, tests, skipped, failures + errors, total_time)) + +# Per-shard aggregate. +shards = {} +for fp, sh, st, p, t, sk, f, tm in rows: + if sh not in shards: + shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} + s = shards[sh] + s["files"] += 1 + s["passed"] += p + s["tests"] += t + s["skipped"] += sk + s["failed"] += f + s["time"] += tm + + +# ---------- Output: plain text to stdout ---------- +def fmt_row(cols, widths): + return " | ".join(c.ljust(w) for c, w in zip(cols, widths)) + + +out = [] +out.append("PER-FILE RESULTS") +out.append("=" * 110) +hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"] +widths = [60, 7, 8, 12, 5, 5, 8] +out.append(fmt_row(hdr, widths)) +out.append("-" * 110) +for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths)) +out.append("") +out.append("PER-SHARD AGGREGATE") +out.append("=" * 80) +shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"] +shard_widths = [10, 7, 12, 9, 8, 14] +out.append(fmt_row(shard_hdr, shard_widths)) +out.append("-" * 80) +grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0} +for sh in sorted(shards): + s = shards[sh] + for k in grand: + grand[k] += s[k] + out.append( + fmt_row( + [ + sh, + str(s["files"]), + f"{s['passed']}/{s['tests']}", + str(s["skipped"]), + str(s["failed"]), + f"{s['time']:.1f}", + ], + shard_widths, + ) + ) +out.append("-" * 80) +out.append( + fmt_row( + [ + "TOTAL", + str(grand["files"]), + f"{grand['passed']}/{grand['tests']}", + str(grand["skipped"]), + str(grand["failed"]), + f"{grand['time']:.1f}", + ], + shard_widths, + ) +) +print("\n".join(out)) + +# ---------- Output: markdown to $GITHUB_STEP_SUMMARY ---------- +summary_path = os.environ.get("GITHUB_STEP_SUMMARY") +if summary_path: + md = [ + "## Multi-GPU CI: Aggregated Test Summary", + "", + "### Per-shard totals", + "", + "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |", + "|---|---:|---:|---:|---:|---:|", + ] + for sh in sorted(shards): + s = shards[sh] + md.append( + f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |" + ) + md.append( + f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | " + f"**{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |" + ) + md.append("") + md.append("### Per-file results") + md.append("") + md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |") + md.append("|---|---|---|---:|---:|---:|---:|") + for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])): + status_md = { + "PASS": ":white_check_mark: PASS", + "FAIL": ":x: FAIL", + "EMPTY": ":white_circle: EMPTY", + "PARSE-ERROR": ":warning: PARSE-ERROR", + }.get(st, st) + md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |") + md.append("") + with open(summary_path, "a") as fh: + fh.write("\n".join(md) + "\n") diff --git a/.github/actions/multi-gpu/multi_gpu_host_launcher.sh b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh new file mode 100755 index 000000000000..ad623c61c649 --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_host_launcher.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash +# Runs ON THE HOST runner — the counterpart to multi_gpu_shard_runner.sh, which +# runs INSIDE the container. Launches ONE container that hosts all N pytest +# shards as parallel subshells, then reconciles the shared work queue. +# +# Invoked by .github/workflows/test-multi-gpu-pytest.yaml's "Run shards in +# parallel" step. Reads from the environment (set by that step): +# IMAGE_TAG — per-commit CI image to ``docker run`` +# INCLUDE_FILES — comma-sep test basenames (conftest include filter) +# PATHS — comma-sep relative test paths (seed the work queue) +# plus the runner built-ins GITHUB_RUN_ID / GITHUB_RUN_ATTEMPT (container name) +# and GITHUB_ENV (exports MGPU_RUNTIME_DIR for the downstream summary step). +# +# 1-docker N-shard rationale: ONE container hosts all shards (each pinned to its +# own non-default cuda:N via ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES, pulling +# from the shared queue). Collapsing the previous N-container layout into one +# removes cross-container races on the workspace mount (``_isaac_sim`` symlink, +# ``/mgpu`` queue, ``/dev/dri`` cap-add) and ~30s of docker init per shard, at +# the cost of sharing /isaac-sim writable subtrees; per-shard HOME (under /tmp, +# set in the shard runner) keeps user-level caches isolated. +# +# Scaling: shard count is derived inside the container from ``nvidia-smi -L`` +# (truth — torch.cuda.device_count() under-counts MIG slices on the same parent +# GPU), so this works unchanged on any GPU count/topology. CUDA_VISIBLE_DEVICES +# is set to the comma-separated MIG-UUID list only on MIG-mode hosts; discrete +# runners rely on ``--gpus all``. + +set +e # keep going on errors so the reconciler below always runs and reports +host_uid="$(id -u)" +host_gid="$(id -g)" +host_user="$(id -un)" +# ``/isaac-sim`` is the Kit install root inside the container; pytest +# invocations under ``./isaaclab.sh -p`` resolve the bundled Python as +# ``./_isaac_sim/python.sh``. Mounting the workspace at ``/workspace/isaaclab`` +# brings the host's CWD into the container, so we plant a host-side symlink +# whose target resolves only inside the container. Single creator (this step) +# — no concurrent creators. +rm -f _isaac_sim && ln -s /isaac-sim _isaac_sim + +# unique per-run temp dir (RUNNER_TEMP if set, else /tmp; XXXXXX -> random suffix) +runtime_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/mgpu-runtime.XXXXXX")" +mkdir -p "$runtime_dir" + +# Directory-based work queue (atomic os.rename pattern; conftest claims entries +# via os.rename(queue/X, inflight//X) — POSIX rename is atomic on the +# same filesystem, so two shards racing on the same entry are kernel-serialized: +# one wins, the other gets ENOENT and moves on). Each pending test path is +# materialized as an empty file in queue/ with '/' slugged to '__' so it's a +# flat filename. +# +# Subdirs: +# queue/ — pending claims +# inflight// — claimed by this shard, test in flight +# done// — test exited cleanly +# +# At job-end the reconciler below asserts queue/ + inflight/ are empty — if +# either is non-empty the job FAILS (silent-drop signal: never claimed OR +# claimed but crashed mid-test). +queue_root="$runtime_dir/queue" +mkdir -p "$queue_root/queue" "$queue_root/inflight" "$queue_root/done" +IFS=',' read -ra _paths <<< "$PATHS" # split the comma-separated PATHS into the _paths array +for path in "${_paths[@]}"; do + [ -n "$path" ] || continue # skip empty entries (PATHS has a trailing comma) + slug="${path//\//__}" # flatten to a filename: replace every '/' with '__' + : > "$queue_root/queue/$slug" # create an empty marker file (':' = no-op, '>' creates it) +done +echo "::notice::seeded work queue with $(ls -1 "$queue_root/queue" | wc -l) entries" + +# Per-shard log dir mounted from the container so we can grouped-print the +# per-shard stream after the run completes. +logs_dir="$runtime_dir/logs" +mkdir -p "$logs_dir" + +# MIG detection: only override CUDA_VISIBLE_DEVICES on MIG hosts. nvidia-smi -L +# is authoritative — torch.cuda.device_count() under-counts MIG slices on the +# same parent GPU. +cvd_args=() +if nvidia-smi -L | grep -q "^ MIG "; then + # read each MIG slice's UUID into the MIGS array (one element per line) + mapfile -t MIGS < <(nvidia-smi -L | awk -F'UUID: ' '/^ MIG /{print $2}' | tr -d ')') + MIG_LIST=$(IFS=,; echo "${MIGS[*]}") # join the array into a comma-separated string + cvd_args=(-e "CUDA_VISIBLE_DEVICES=${MIG_LIST}") # stays empty on discrete hosts -> no flag added + echo "::notice::MIG mode: ${#MIGS[@]} slices — overriding CUDA_VISIBLE_DEVICES" +else + echo "::notice::discrete GPU mode — relying on --gpus all" +fi + +# "${cvd_args[@]}" expands to the CUDA_VISIBLE_DEVICES flag on MIG hosts, nothing otherwise. +docker run --rm --gpus all --network=host \ + --entrypoint bash \ + --user "${host_uid}:${host_gid}" \ + --name "isaac-lab-mgpu-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + -v "$PWD:/workspace/isaaclab:rw" \ + -v "$queue_root:/mgpu:rw" \ + -v "$logs_dir:/shard-logs:rw" \ + -e USER="${host_user}" \ + -e LOGNAME="${host_user}" \ + -e OMNI_KIT_ACCEPT_EULA=yes \ + -e ACCEPT_EULA=Y \ + -e OMNI_KIT_DISABLE_CUP=1 \ + -e ISAAC_SIM_HEADLESS=1 \ + -e ISAAC_SIM_LOW_MEMORY=1 \ + -e PYTHONUNBUFFERED=1 \ + -e PYTHONIOENCODING=utf-8 \ + -e ISAACLAB_TEST_QUEUE=/mgpu \ + -e TEST_INCLUDE_FILES="$INCLUDE_FILES" \ + -e ISAACLAB_PIN_KIT_GPU=1 \ + -e HOME=/tmp/mgpu-base-home \ + -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase \ + "${cvd_args[@]}" \ + -v "$PWD/.github/actions/multi-gpu/multi_gpu_shard_runner.sh:/multi_gpu_shard_runner.sh:ro" \ + "$IMAGE_TAG" \ + /multi_gpu_shard_runner.sh +docker_rc=$? # $? = exit code of the docker run above + +# Grouped re-print of per-shard logs (preserved across the host). +for log in "$logs_dir"/cuda-*.log; do + cuda=$(basename "$log" .log | sed s/cuda-//) # extract N from "cuda-N.log" + echo "::group::shard cuda:${cuda} log" + cat "$log" + echo "::endgroup::" +done + +# Reconciler — silent-drop guard. Anything left in queue/ was never claimed by +# any shard (work-queue starvation). Anything still in inflight// was +# claimed but never moved to done/ (shard crashed mid-test). Either is a +# silent-drop signal that the docker exit alone can't surface. Fail the job and +# name the orphans. +echo "::group::work-queue reconciler" +unclaimed=$(ls -1 "$queue_root/queue" 2>/dev/null | wc -l) # files still pending (never claimed) +if [ "$unclaimed" -gt 0 ]; then + echo "::error::${unclaimed} test(s) never claimed by any shard:" + ls -1 "$queue_root/queue" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent for the log +fi +orphans=0 +for shard_dir in "$queue_root/inflight"/*/; do # each inflight// dir (the /*/ matches dirs only) + [ -d "$shard_dir" ] || continue # skip if the glob matched nothing + count=$(ls -1 "$shard_dir" 2>/dev/null | wc -l) + if [ "$count" -gt 0 ]; then + echo "::error::$(basename "$shard_dir") claimed but never finished ${count} test(s):" + ls -1 "$shard_dir" | sed 's|__|/|g; s/^/ /' # decode '__' back to '/', indent + orphans=$((orphans + count)) # $(( )) = integer arithmetic + fi +done +done_total=$(find "$queue_root/done" -type f 2>/dev/null | wc -l) +echo "::notice::reconciler: done=${done_total} unclaimed=${unclaimed} orphans=${orphans}" +echo "::endgroup::" + +# Treat reconciler signals as fatal even if all shards exited 0. +if [ "$unclaimed" -gt 0 ] || [ "$orphans" -gt 0 ]; then + echo "::error::silent-drop detected — see reconciler group above" + [ "$docker_rc" -eq 0 ] && docker_rc=2 # clean docker exit but drops found -> force a non-zero code +fi + +# Export the runtime directory so the downstream "Aggregated test summary" step +# can locate the done// work distribution and the per-shard logs. +echo "MGPU_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV" # export to later workflow steps via GitHub's env file + +exit "$docker_rc" diff --git a/.github/actions/multi-gpu/multi_gpu_shard_runner.sh b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh new file mode 100755 index 000000000000..902b51f6001c --- /dev/null +++ b/.github/actions/multi-gpu/multi_gpu_shard_runner.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Runs INSIDE the 1-docker container hosting the multi-GPU lane's pytest +# shards. Mounted by the CI workflow (.github/workflows/test-multi-gpu- +# pytest.yaml) so the logic lives in exactly one place and changes are +# version-controlled + shellcheck-able. Lives under ``.github/actions/multi-gpu/`` +# so it sits next to the workflow that consumes it rather than alongside ``tools/``. +# +# Container expectations (set by the host launcher): +# --gpus all +# --user $host_uid:$host_gid +# -v :/workspace/isaaclab:rw +# -v :/mgpu:rw (subdirs queue/, inflight/, done/) +# -v :/shard-logs:rw (per-shard tee target) +# -e HOME=/tmp/mgpu-base-home (writable tmpfs for pip --user) +# -e PYTHONUSERBASE=/tmp/mgpu-pyuserbase (shared site-packages across shards) +# -e ISAACLAB_TEST_QUEUE=/mgpu (conftest queue root) +# -e TEST_INCLUDE_FILES="" +# -e ISAACLAB_PIN_KIT_GPU=1 (Kit overrides for the multi-GPU race) +# -e CUDA_VISIBLE_DEVICES="" (MIG hosts only; discrete = --gpus all) +# +# Behavior: +# 1. Materializes HOME + PYTHONUSERBASE dirs (tmpfs, world-writable) +# 2. Installs pytest deps (junitparser et al.) into the shared PYTHONUSERBASE +# 3. Derives shard count from nvidia-smi -L (authoritative; torch.cuda.device_count +# under-counts MIG-on-same-parent unless CUDA_VISIBLE_DEVICES enumerates each) +# 4. Cross-checks torch against the nvidia-smi count and caps shards to what torch +# can address (guards against CUDA_VISIBLE_DEVICES misconfig on a MIG host) +# 5. Fans out 1 pytest subshell per non-default cuda:N with per-shard HOME + +# ISAACLAB_SIM_DEVICE + ISAACLAB_TEST_DEVICES; each shard tees its stdout to +# /shard-logs/cuda-N.log for the host's grouped re-print after the run +# 6. Waits on every shard before aggregating exit codes — a fast failure doesn't +# tear down still-running siblings + +set +e # keep going on errors; per-shard exit codes are aggregated at the end +cd /workspace/isaaclab +unset HUB__ARGS__DETECT_ONLY DISPLAY # clear vars that would force Kit into detect-only / headed (X11) mode + +# Container-level HOME + PYTHONUSERBASE for pip --user installs. The image +# runs as --user $host_uid:$host_gid with no matching /etc/passwd entry, so +# HOME defaults to /root which the user cannot write. /tmp/* is on tmpfs +# (1777, world-writable). +# +# PYTHONUSERBASE is the key for the 1-docker shape: pip --user writes to +# ${PYTHONUSERBASE}/lib/python3.12/site-packages, and every Python invocation +# that sees the same env var imports from there. Per-shard subshells below +# override HOME (so .cache / .nvidia-omniverse are isolated) but inherit +# PYTHONUSERBASE so junitparser et al. resolve everywhere. +mkdir -p /tmp/mgpu-base-home /tmp/mgpu-pyuserbase + +# Pytest deps (same as run-tests action). junitparser is imported at +# tools/conftest.py load time, so it must be present first. +./isaaclab.sh -p -m pip install pytest pytest-mock junitparser flatdict flaky "coverage>=7.6.1" + +# Shard count from nvidia-smi -L (truth; torch under-counts MIG). +MIG_COUNT=$(nvidia-smi -L | grep -c "^ MIG ") # grep -c = count of matching lines (MIG slices) +GPU_COUNT=$(nvidia-smi -L | grep -c "^GPU ") # count of whole GPUs +if [ "$MIG_COUNT" -gt 0 ]; then + DEV_COUNT=$MIG_COUNT + echo "::notice::container: MIG mode, $MIG_COUNT slices" +else + DEV_COUNT=$GPU_COUNT + echo "::notice::container: discrete mode, $GPU_COUNT GPUs" +fi +if [ "$DEV_COUNT" -lt 2 ]; then + echo "::error::Need at least 2 visible devices; found $DEV_COUNT" + exit 1 +fi + +# Cross-check with torch and cap shard count to what torch can actually +# address. Guards against a CUDA_VISIBLE_DEVICES misconfig silently fanning +# out shards that crash on device access. +TORCH_COUNT=$(/isaac-sim/python.sh -c "import torch; print(torch.cuda.device_count())") +echo "container: torch sees $TORCH_COUNT cuda devices (cross-check vs $DEV_COUNT)" +if [ "$TORCH_COUNT" -lt "$DEV_COUNT" ]; then + echo "::warning::torch sees fewer devices than nvidia-smi — capping shards to $TORCH_COUNT" + DEV_COUNT=$TORCH_COUNT +fi + +# Fan out 1 pytest subshell per non-default cuda:N. Each gets its own HOME +# (per-shard isolation for .cache, .local/share, etc.) and per-shard +# ISAACLAB_SIM_DEVICE / ISAACLAB_TEST_DEVICES. +declare -A pids # associative array: shard index -> background PID +for ((cuda = 1; cuda < DEV_COUNT; cuda++)); do # C-style loop; start at 1 to skip cuda:0 (single-GPU CI covers it) + zeros="" + for ((i = 0; i <= cuda; i++)); do zeros+="0"; done # build the leading zeros of the device mask + runtime_devices="${zeros}1" # e.g. cuda:2 -> "0001": only this GPU active in the ISAACLAB_TEST_DEVICES mask + + shard_home="/tmp/isaaclab-ci-home-${cuda}" + mkdir -p "${shard_home}/.cache" "${shard_home}/.local/share" \ + "${shard_home}/.nvidia-omniverse/config" \ + "${shard_home}/.nvidia-omniverse/logs" + + shard_log="/shard-logs/cuda-${cuda}.log" + + # each shard runs in its own subshell, backgrounded with '&' below, so all shards run in parallel + ( + export HOME="$shard_home" + export XDG_CACHE_HOME="${HOME}/.cache" + export XDG_DATA_HOME="${HOME}/.local/share" + export ISAACLAB_TEST_DEVICES="$runtime_devices" + export ISAACLAB_SIM_DEVICE="cuda:${cuda}" + + # Full pytest output captures to $shard_log; live stdout is filtered + # down to high-signal lines (test boundaries, failures, summary stats, + # tracebacks). Kit init chatter, plugin registration, omni.usd Transfer + # logs, etc. only end up in the shard logfile, NOT on the live workflow + # log. The workflow's grouped re-print at end of run still shows the + # full $shard_log under a collapsible ``::group::shard cuda:N log``. + # (tee = full output to the log file; stdbuf -oL = flush per line so the + # filtered grep/sed stream appears live, not in delayed chunks.) + ./isaaclab.sh -p -m pytest \ + --ignore=tools/conftest.py \ + --ignore=source/isaaclab/test/install_ci \ + tools -v 2>&1 \ + | tee "$shard_log" \ + | stdbuf -oL grep -aE \ + '🚀|^source/.*::.* (PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)|^(Total|Passing|Failing|Crashed|Startup Hang|Timeout|Total Wall Time|Total Test Time|Passing Percentage):|^~~~~|^=+ |^E +|^ +File |Traceback|^FAILED|^ERROR ' \ + | stdbuf -oL sed "s/^/[cuda:${cuda}] /" + + exit "${PIPESTATUS[0]}" # exit with pytest's code, not tee/grep/sed's (PIPESTATUS[0] = first pipe stage) + ) & + pids[$cuda]=$! # $! = PID of the subshell just backgrounded + echo "::notice::launched shard cuda:${cuda} (pid ${pids[$cuda]}, runtime_devices=$runtime_devices)" +done + +# Wait for every shard before aggregating exits — a fast failure must not +# tear down still-running siblings. +declare -A results # associative array: shard index -> exit code +for cuda in "${!pids[@]}"; do # "${!pids[@]}" = the array's keys (the shard indices) + wait "${pids[$cuda]}" # block until that shard's PID exits + results[$cuda]=$? # $? = the waited shard's exit code +done + +fail=0 +for cuda in "${!results[@]}"; do + rc="${results[$cuda]}" + echo "shard cuda:${cuda} exited $rc" + [ "$rc" -eq 0 ] || fail=1 # any non-zero shard makes the whole run fail +done +exit $fail diff --git a/.github/workflows/test-multi-gpu-pytest.yaml b/.github/workflows/test-multi-gpu-pytest.yaml new file mode 100644 index 000000000000..e9bbeb07832b --- /dev/null +++ b/.github/workflows/test-multi-gpu-pytest.yaml @@ -0,0 +1,188 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Multi-GPU unit-test workflow +# +# Runs the non-default-GPU subset of unit tests across the multi-GPU runner +# pool's GPUs in parallel. Uses the same ECR-pulled isaac-lab image as the +# single-GPU CI; only diffs are the runner label and the env vars that pin +# Kit and the test parametrize to the shard's non-default GPU. +# +# Adding a new test to multi-GPU coverage: give its device parametrize a +# non-default-capable scope — argless ``isaaclab.test.utils.test_devices()`` +# (cpu + cuda:0 + non-default GPUs) or any ``"..X"`` mask. The workflow +# auto-discovers it. + +name: Multi-GPU pytest + +on: + pull_request: + paths: + - "source/isaaclab/isaaclab/test/utils/**" + - "source/isaaclab/isaaclab/app/app_launcher.py" + - "source/**/test/**/test_*.py" + - ".github/workflows/test-multi-gpu-pytest.yaml" + - ".github/actions/run-package-tests/**" + - ".github/actions/run-tests/**" + - ".github/actions/ecr-build-push-pull/**" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CI_IMAGE_TAG: isaac-lab-ci:${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || github.ref_name }}-${{ github.sha }} + +jobs: + config: + name: Load Config + runs-on: ubuntu-latest + outputs: + isaacsim_image_name: ${{ steps.load.outputs.isaacsim_image_name }} + isaacsim_image_tag: ${{ steps.load.outputs.isaacsim_image_tag }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + sparse-checkout: .github/workflows/config.yaml + sparse-checkout-cone-mode: false + - id: load + run: | + set -euo pipefail + f=.github/workflows/config.yaml + echo "isaacsim_image_name=$(yq -r .isaacsim_image_name "$f")" >> "$GITHUB_OUTPUT" + echo "isaacsim_image_tag=$(yq -r .isaacsim_image_tag "$f")" >> "$GITHUB_OUTPUT" + + build: + name: Build / cache image + needs: [config] + # Must run on the SAME pool as the test job. The ECR cache repo is resolved + # per runner pool (single-GPU `gpu` runners -> gitci-docker-cache; multi-GPU + # runners -> multigpu-docker-cache). If this built on `[self-hosted, gpu]` + # the image would land in gitci-docker-cache, which the multi-GPU test job + # cannot see, so it would rebuild from scratch on the scarce multi-GPU + # runner. Building here populates multigpu-docker-cache so the test job's + # pull hits. + runs-on: [self-hosted, linux, x64, multi-gpu] + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + # Pre-populates the ECR exact-commit tag from deps-cache (registry-side + # alias). Without this prior step, run-package-tests' internal + # ecr-build-push-pull hits exact-cache-miss + deps-cache-hit and leaves + # no local image, causing `docker run` to fail with `pull access + # denied`. Mirrors the build → test split in build.yaml. + - uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + test-multi-gpu-pytest: + name: Multi-GPU unit tests + needs: [config, build] + if: needs.build.result == 'success' + # The ``multi-gpu`` label is the multi-GPU pool. Do NOT add ``gpu`` here: + # in this fleet ``gpu`` tags single-GPU runners, so requiring it routes the + # job onto a 1-GPU box and the shard runner aborts with "Need at least 2 + # visible devices; found 1". + runs-on: [self-hosted, linux, x64, multi-gpu] + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 1 + lfs: true + + - name: Discover opt-in test files + # Auto-discovery: any test_*.py with a non-default-capable scope — an + # argless ``test_devices()`` (defaults to "11X") or any + # ``test_devices("..X")`` mask, whose trailing ``X`` spans the + # non-default GPUs — is in scope. Adding a test to multi-GPU CI needs + # no workflow edit; opting a file out is just narrowing its scope (drop + # the ``X``, e.g. "110"), so the workflow stays opt-in/opt-out-free. + # + # Within a discovered file, tests that are NOT parametrized over the + # ``device`` argument are deselected at collection time by the + # ``test_runner.selector`` plugin (which the runner injects on every + # shard): single-GPU CI already covers them on ``cuda:0`` and re-running on + # every non-default shard adds wall-time without surfacing any new failure mode. + id: discover + run: | + # File-level opt-out: a test file can exclude itself from multi-GPU CI + # by declaring a module-level ``MULTI_GPU_SKIP_REASON = "..."`` line. + # Used for files with known Kit/Isaac-Sim concurrency issues; the file + # still runs in single-GPU CI. Excluded files are reported as a notice + # below for visibility. No workflow edit needed to add/remove a file. + mapfile -t candidates < <(grep -rlE 'test_devices\(\)|test_devices\("[^"]*X"\)' source/ --include='test_*.py' | sort -u) + discovered=() + skipped=() + for f in "${candidates[@]}"; do + if grep -q '^MULTI_GPU_SKIP_REASON' "$f"; then + skipped+=("$f") + else + discovered+=("$f") + fi + done + for f in "${skipped[@]}"; do + reason=$(grep -m1 '^MULTI_GPU_SKIP_REASON' "$f" | sed -E 's/^MULTI_GPU_SKIP_REASON[[:space:]]*=[[:space:]]*//; s/^"//; s/"$//') + echo "::notice::multi-GPU skipped: $f — $reason" + done + if [ ${#discovered[@]} -eq 0 ]; then + echo "::error::No opt-in tests discovered (grep for test_devices with an X scope)" + exit 1 + fi + basenames=$(printf '%s\n' "${discovered[@]}" | xargs -n1 basename | sort -u | paste -sd,) + echo "include=$basenames" >> "$GITHUB_OUTPUT" + # Full relative paths too, to seed the shared work queue (the run step + # needs runnable paths, not just basenames). + echo "paths=$(printf '%s,' "${discovered[@]}")" >> "$GITHUB_OUTPUT" + echo "::notice::Discovered ${#discovered[@]} opt-in test files" + printf ' %s\n' "${discovered[@]}" + + - name: Pull image from ECR + # Pulls the per-commit image the build job pushed. ecr-build-push-pull + # handles ECR auth via the EC2 IAM role, and on exact-cache-hit (which + # the build job's deps-cache-hit registry-tag created for this SHA) + # it pulls the image locally and tags it as ``$CI_IMAGE_TAG`` for our + # parallel ``docker run`` block below. + uses: ./.github/actions/ecr-build-push-pull + env: + NGC_API_KEY: ${{ secrets.NGC_API_KEY }} + with: + image-tag: ${{ env.CI_IMAGE_TAG }} + isaacsim-base-image: ${{ needs.config.outputs.isaacsim_image_name }} + isaacsim-version: ${{ needs.config.outputs.isaacsim_image_tag }} + dockerfile-path: docker/Dockerfile.base + cache-tag: cache-base + + - name: Run shards in parallel on local GPUs (1-docker N-shard) + # ONE container hosts all N pytest shards as parallel subshells, each + # pinned to its own non-default cuda:N and pulling from a shared work + # queue; the host reconciles the queue afterward and exports + # MGPU_RUNTIME_DIR for the summary step. Full rationale + the + # 1-docker-N-shard tradeoffs live in the script header. + env: + IMAGE_TAG: ${{ env.CI_IMAGE_TAG }} + INCLUDE_FILES: ${{ steps.discover.outputs.include }} + PATHS: ${{ steps.discover.outputs.paths }} + run: bash .github/actions/multi-gpu/multi_gpu_host_launcher.sh + + - name: Aggregated test summary + # Per-shard + per-file pass/total/walltime, plus a combined table, also + # written to $GITHUB_STEP_SUMMARY. Runs on success or failure. + if: always() + env: + RUNTIME_DIR: ${{ env.MGPU_RUNTIME_DIR }} + run: python3 .github/actions/multi-gpu/aggregate_test_summary.py diff --git a/pyproject.toml b/pyproject.toml index a4a31197858c..bd4fe1a2f64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -196,7 +196,7 @@ ignore-words-list = "haa,slq,collapsable,buss,reacher,thirdparty,segway" markers = [ "isaacsim_ci: mark test to run in isaacsim ci", - "device_split: re-invoke this file once per device (CPU and GPU) in CI due to process-global device locks (e.g., ovphysx<=0.3.7 gap G5)", + "device_isolated: this file's backend holds a process-global device lock (e.g. ovphysx<=0.3.7 gap G5); the CI runner runs it once per device instead of mixing devices in one process", ] # Add pypi.nvidia.com so that `uv pip install isaaclab[isaacsim]` works without --extra-index-url. diff --git a/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst new file mode 100644 index 000000000000..89e774bd26d7 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-fix-build-sim-context-device.rst @@ -0,0 +1,11 @@ +Fixed +^^^^^ + +* Fixed :func:`isaaclab.sim.build_simulation_context` silently ignoring the + ``device`` kwarg when ``sim_cfg`` is also provided. Most test callers pass + both kwargs together; the helper now applies the explicit ``device`` over + ``sim_cfg.device`` so the caller's choice wins. Without this, warp kernel + launches in :mod:`isaaclab_newton.assets.articulation` raised device + mismatch errors on non-default GPUs (``env_ids`` allocated on the test's + device while the articulation's resolved device came from the untouched + ``sim_cfg`` default ``cuda:0``). diff --git a/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst new file mode 100644 index 000000000000..a78313ab14d9 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-mgpu-pin-kit-resources.rst @@ -0,0 +1,13 @@ +Added +^^^^^ + +* Added ``ISAACLAB_PIN_KIT_GPU`` env var for :class:`~isaaclab.app.AppLauncher`. + When set to a truthy value, appends ``--/renderer/multiGpu/enabled=False``, + ``--/renderer/multiGpu/autoEnable=False``, ``--/renderer/multiGpu/maxGpuCount=1`` + and ``--/physics/fabricUseGPUInterop=false`` to the Kit command line so each + Kit process touches only its assigned GPU (rather than enumerating every + visible GPU at startup). Used by the multi-GPU CI workflow to prevent the + shared GPU-interop context across sibling shards that surfaces as + ``[Error] [omni.physx.plugin] Stage X already attached`` and + ``SimulationApp.close`` hangs (see https://github.com/isaac-sim/IsaacLab/issues/3475). + Off by default; single-GPU and user-facing rendering paths are unchanged. diff --git a/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst new file mode 100644 index 000000000000..5f66a3c858a0 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-multi-gpu-ci.minor.rst @@ -0,0 +1,16 @@ +Added +^^^^^ + +* Added :func:`isaaclab.test.utils.test_devices` to parametrize unit tests over + a device set resolved as ``scope ∩ runtime``: ``scope`` is the call-site mask + of devices a test is valid on (default ``"11X"`` ⇒ cpu + cuda:0 + the + non-default GPUs), and the runtime is the ``ISAACLAB_TEST_DEVICES`` env var of + devices a run may use (default ``"110"`` ⇒ cpu + cuda:0). A trailing ``X`` + includes the remaining devices. Single-GPU CI is unchanged; multi-GPU CI sets + the runtime to one non-default GPU per shard. + +* Added ``ISAACLAB_SIM_DEVICE`` env var honored by + :class:`isaaclab.app.AppLauncher` as the implicit-default device when + the caller doesn't pass ``device=``. Lets the multi-GPU CI workflow + boot Kit on a non-default GPU without editing every test's + :class:`~isaaclab.app.AppLauncher` call site. diff --git a/source/isaaclab/isaaclab/app/app_launcher.py b/source/isaaclab/isaaclab/app/app_launcher.py index 511b46a3f816..6304ec38e2c7 100644 --- a/source/isaaclab/isaaclab/app/app_launcher.py +++ b/source/isaaclab/isaaclab/app/app_launcher.py @@ -1003,6 +1003,17 @@ def _resolve_device_settings(self, launcher_args: dict): device = launcher_args.get("device", AppLauncher._APPLAUNCHER_CFG_INFO["device"][1]) device_explicitly_passed = launcher_args.pop("device_explicit", False) + # When the caller didn't pin a device, allow ISAACLAB_SIM_DEVICE to + # override the default. Used by the multi-GPU CI workflow to boot + # Kit with active_gpu=1 without editing every test's AppLauncher() + # call site; Kit's active_gpu is process-global and locked after + # SimulationApp init, so per-test selection cannot retarget it. + if not device_explicitly_passed: + env_device = os.environ.get("ISAACLAB_SIM_DEVICE") + if env_device: + device = env_device + launcher_args["device"] = env_device + if self._xr and not device_explicitly_passed: # If no device is specified, default to the CPU device if we are running in XR device = "cpu" @@ -1060,6 +1071,32 @@ def _resolve_device_settings(self, launcher_args: dict): launcher_args["physics_gpu"] = self.device_id launcher_args["active_gpu"] = self.device_id + # Pin Kit's renderer to a single GPU when ``ISAACLAB_PIN_KIT_GPU`` is + # truthy. The default ``apps/isaaclab.python.headless.kit`` sets + # ``renderer.multiGpu.enabled = true`` + ``renderer.multiGpu.autoEnable + # = true``, so each Kit process enumerates every visible GPU at + # startup. Under concurrent multi-GPU CI shards (``--gpus all`` per + # container, one Kit per non-default cuda device), that produces a + # shared GPU-interop context across sibling processes -- surfacing as + # ``[Error] [omni.physx.plugin] Stage X already attached`` mid-test and + # ``SimulationApp.close`` hanging in teardown (see + # https://github.com/isaac-sim/IsaacLab/issues/3475). The mitigation is + # to set ``renderer.multiGpu.enabled = false`` + ``maxGpuCount = 1`` so + # each Kit only touches its assigned GPU. + if os.environ.get("ISAACLAB_PIN_KIT_GPU", "0").lower() not in {"", "0", "false", "no", "off"}: + sys.argv.append("--/renderer/multiGpu/enabled=False") + sys.argv.append("--/renderer/multiGpu/autoEnable=False") + sys.argv.append("--/renderer/multiGpu/maxGpuCount=1") + # Also disable the fabric GPU-interop path. The renderer multiGpu + # flags above mitigate the startup-time enumeration race; this + # mitigates the runtime GPU-interop race on top. Safe for the + # multi-GPU CI lane: it covers physics / scene / utility tests, + # not rendering. + sys.argv.append("--/physics/fabricUseGPUInterop=false") + logger.info( + "ISAACLAB_PIN_KIT_GPU enabled: pinning Kit renderer to a single GPU + disabling fabric GPU-interop" + ) + # Defer importing torch until after SimulationApp starts. Importing # torch can import NumPy/OpenBLAS, whose at-fork handlers can crash # Kit's platform-info fork during startup. diff --git a/source/isaaclab/isaaclab/test/utils/__init__.py b/source/isaaclab/isaaclab/test/utils/__init__.py new file mode 100644 index 000000000000..9e4bb3d6c875 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Test-time helpers for Isaac Lab. + +Exposes :func:`test_devices` for selecting the device list to parametrize tests +over. The set is ``scope ∩ runtime``: ``scope`` is the call-site argument (the +devices the test is valid on), the runtime is the ``ISAACLAB_TEST_DEVICES`` env +var (the devices the run may use). +""" + +from .devices import test_devices + +__all__ = ["test_devices"] diff --git a/source/isaaclab/isaaclab/test/utils/devices.py b/source/isaaclab/isaaclab/test/utils/devices.py new file mode 100644 index 000000000000..c54cabfdfbd5 --- /dev/null +++ b/source/isaaclab/isaaclab/test/utils/devices.py @@ -0,0 +1,169 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Device selection for parametrizing tests over cpu / cuda devices. + +Intended use:: + + from isaaclab.test.utils import test_devices + + @pytest.mark.parametrize("device", test_devices()) # cpu + cuda:0 + a non-default GPU + def test_foo(device): ... + +Two masks, two roles +-------------------- +A test runs on ``scope ∩ runtime``: + +* **scope** — the call-site mask: the devices the *test* is valid on. The + author owns this; defaults to ``"11X"`` (device-agnostic). +* **runtime** — the ``ISAACLAB_TEST_DEVICES`` env var: the devices the *run* + may use. The operator / CI owns this; defaults to ``"110"`` (cpu + cuda:0). + The multi-GPU workflow sets it to one device per shard, matching + ``ISAACLAB_SIM_DEVICE`` (AppLauncher's boot device — not read here). + +A test author never names the shard's GPU; the operator never inspects a +test's device support. The helper intersects the two. + +Mask grammar +------------ +Positions left to right: ``0`` = cpu, ``1`` = cuda:0 (the default GPU), +``2`` = cuda:1, ``3`` = cuda:2, ... Each position is ``0`` (exclude) or ``1`` +(include). A trailing ``X`` means "include all the remaining devices". + +cpu, cuda:0, and a non-default GPU stay distinct by position — running on +cuda:0 says nothing about cuda:1+, which is the whole reason this exists. + +Common masks +------------ +====== =================================================================== +Mask Meaning +====== =================================================================== +``11X`` cpu + cuda:0 + every non-default GPU (default scope; device-agnostic) +``110`` cpu + cuda:0 (pure-math / backend tests: cuda:0 == cuda:N) +``00X`` non-default GPUs only (validates non-default-device behavior) +``100`` cpu only (pure logic) +====== =================================================================== + +Worked example — a ``scope="11X"`` (default) test: + +* single-GPU CI (runtime unset ⇒ ``"110"``) ⇒ ``[cpu, cuda:0]``. +* a multi-GPU shard (runtime ``"0001"``, one device) ⇒ ``[cuda:2]``. + +So argless tests run on cpu + cuda:0 in single-GPU CI and on exactly one +non-default GPU per shard in multi-GPU CI (run-once is a property of the +one-device-per-shard runtime plus round-robin file assignment). + +An empty result means the test is cleanly skipped for this run (e.g. a +``"00X"`` test on a single-GPU host, or a ``"110"`` test on a non-default-GPU +shard). But if the run *explicitly* set ``ISAACLAB_TEST_DEVICES`` to devices +the host does not have, that is a misconfigured run and raises instead of +skipping everything and reporting a vacuous green. + +Local runs +---------- +Set the runtime from the shell to opt a run into non-default GPUs:: + + ISAACLAB_TEST_DEVICES=0001 ISAACLAB_SIM_DEVICE=cuda:2 \\ + ./isaaclab.sh -p -m pytest path/to/test.py +""" + +from __future__ import annotations + +import os + +import torch + +_RUNTIME_DEVICES_ENV_VAR = "ISAACLAB_TEST_DEVICES" +"""Env var naming the run's devices: the devices a run may use (see module docstring).""" + +_DEFAULT_RUNTIME = "110" +"""Runtime devices when :data:`_RUNTIME_DEVICES_ENV_VAR` is unset: cpu + cuda:0, +i.e. the historical single-GPU device set, so non-default GPUs are opt-in per run.""" + + +def test_devices(scope: str = "11X", *, skip: dict[str, str] | None = None) -> list: + """Resolve the device list to parametrize a test over, as ``scope ∩ runtime``. + + ``scope`` is this argument; the runtime comes from the + ``ISAACLAB_TEST_DEVICES`` env var (see the module docstring for the grammar + and the scope / runtime split). + + Args: + scope: Device mask (e.g. ``"11X"``) the test is valid on. Defaults to + ``"11X"`` (cpu + cuda:0 + every non-default GPU): the device-agnostic + common case, so most call sites can use ``test_devices()`` with no + argument. + skip: Optional ``{device: reason}``; in-scope devices listed here are + wrapped in :func:`pytest.mark.skip` so they collect as SKIPPED with + the reason instead of being dropped. Use to gate a device variant + that is known broken while keeping it visible. + + Returns: + Ordered device entries (``"cpu"`` / ``"cuda:N"`` strings, or + :func:`pytest.param` for skipped ones) for the second argument to + :func:`pytest.mark.parametrize`. Empty means the test is skipped for + this run. + + Raises: + ValueError: When the run explicitly set ``ISAACLAB_TEST_DEVICES`` to + devices that are not available on this host (a misconfigured run that + would otherwise skip everything and pass vacuously). A scope that + merely does not intersect this run's devices skips, it does not raise. + """ + requested = os.environ.get(_RUNTIME_DEVICES_ENV_VAR) + available = _list_available_devices() + runtime_keep = _expand(requested or _DEFAULT_RUNTIME, len(available)) + # A run that explicitly named devices the host cannot provide (e.g. a shard + # pinned to cuda:2 on a 2-GPU host) is misconfigured: fail loudly rather than + # skip every test and report a vacuous green. A scope that simply does not + # intersect this run's devices (a "110" test on a non-default-GPU shard) is + # not an error — it skips. + if requested is not None and not any(runtime_keep): + raise ValueError( + f"{_RUNTIME_DEVICES_ENV_VAR}={requested!r} names no device available on this host (available: {available})" + ) + scope_keep = _expand(scope, len(available)) + devices = [ + device for device, in_scope, in_runtime in zip(available, scope_keep, runtime_keep) if in_scope and in_runtime + ] + if not skip: + return devices + import pytest + + return [ + pytest.param(device, marks=pytest.mark.skip(reason=skip[device])) if device in skip else device + for device in devices + ] + + +def _expand(mask: str, count: int) -> list[bool]: + """Expand a mask to ``count`` include-flags; a trailing ``X`` fills the rest with ``True``. + + Args: + mask: A mask string (positions plus an optional trailing ``X``). + count: The number of available devices to expand to. + + Returns: + A list of ``count`` booleans, one per device position. + """ + body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False) + return ([char == "1" for char in body] + [fill] * count)[:count] + + +def _list_available_devices() -> list[str]: + """Return the host's visible devices in mask order: ``cpu`` then ``cuda:0, cuda:1, ...``. + + Returns: + Ordered list of device strings as torch addresses them. + """ + devices = ["cpu"] + # torch.cuda (not warp) is deliberate: this runs at pytest collection time, + # before AppLauncher boots Kit. torch.cuda.device_count() enumerates without + # creating a CUDA context, whereas warp.get_cuda_devices() initializes the + # warp runtime — doing that at collection, ahead of Kit's device setup, + # risks the non-default-GPU init-order fragility this suite targets (#5132). + if torch.cuda.is_available(): + devices.extend(f"cuda:{i}" for i in range(torch.cuda.device_count())) + return devices diff --git a/source/isaaclab/test/sim/test_newton_model_utils.py b/source/isaaclab/test/sim/test_newton_model_utils.py index cc157074fd42..5cfedaab3354 100644 --- a/source/isaaclab/test/sim/test_newton_model_utils.py +++ b/source/isaaclab/test/sim/test_newton_model_utils.py @@ -27,6 +27,7 @@ _scatter_shape_color_rows_kernel, replace_newton_shape_colors, ) +from isaaclab.test.utils import test_devices _WARNING_MESSAGE = "Newton shape color replacement is enabled; this workaround will be deprecated in a future release." @@ -177,7 +178,7 @@ def _run_scatter_shape_color_rows_kernel( pytest.param((-0.25, 1.75, 0.5), id="oob_clamps_pow"), ], ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_scatter_shape_color_rows_kernel(device: str, linear_rgb: tuple[float, float, float]): """Packed RGB per case; the two parametrized cases jointly cover every ``_linear_channel_to_srgb_warp`` branch.""" after = _run_scatter_shape_color_rows_kernel([linear_rgb], device=device) @@ -284,7 +285,7 @@ def test_replace_newton_shape_colors_warning(): replace_newton_shape_colors(model) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPatch, device: str): """Setting ``ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS`` to ``0`` disables the workaround.""" monkeypatch.setenv("ISAACLAB_REPLACE_NEWTON_SHAPE_COLORS", "0") @@ -315,7 +316,7 @@ def test_replace_newton_shape_colors_env_var_switch(monkeypatch: pytest.MonkeyPa assert not any(issubclass(w.category, FutureWarning) for w in recorded) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_invalid_prim(device: str): """Invalid prim path leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -330,7 +331,7 @@ def test_replace_newton_shape_colors_invalid_prim(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_guide_purpose(device: str): """Guide-purpose mesh leaves ``shape_color`` unchanged.""" stage = Usd.Stage.CreateInMemory() @@ -349,7 +350,7 @@ def test_replace_newton_shape_colors_guide_purpose(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_no_material_binding(device: str): """No material: ``displayColor`` or unbound gray as linear RGB, then sRGB OETF into ``shape_color``.""" stage = Usd.Stage.CreateInMemory() @@ -381,7 +382,7 @@ def test_replace_newton_shape_colors_no_material_binding(device: str): @pytest.mark.parametrize(("diffuse_color_constant", "diffuse_tint"), _OMNIPBR_ALBEDO_INPUT_CASES) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_omnipbr_binding( device: str, diffuse_color_constant: tuple[float, float, float] | None, @@ -405,7 +406,7 @@ def test_replace_newton_shape_colors_omnipbr_binding( ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_neutral_material(device: str): """Bound ``UsdPreviewSurface`` material leaves ``shape_color`` unchanged.""" stage, mesh_path = _make_preview_surface_bound_mesh_stage() @@ -418,7 +419,7 @@ def test_replace_newton_shape_colors_neutral_material(device: str): assert torch.allclose(wp.to_torch(shape_color), before) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_respects_binding_strength(device: str): """Parent stronger-than-descendants binding overrides direct child binding.""" # Scene graph (``ComputeBoundMaterial`` on the mesh yields ParentMat / green, not ChildMat / red): @@ -470,7 +471,7 @@ def test_replace_newton_shape_colors_respects_binding_strength(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_replace_newton_shape_colors_instanced(device: str): """Instance-proxy labels deduplicate via canonical prototype paths in the per-key cache.""" stage = Usd.Stage.CreateInMemory() diff --git a/source/isaaclab/test/sim/test_simulation_context.py b/source/isaaclab/test/sim/test_simulation_context.py index 6ea578a85e30..c96883b839a0 100644 --- a/source/isaaclab/test/sim/test_simulation_context.py +++ b/source/isaaclab/test/sim/test_simulation_context.py @@ -6,6 +6,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -43,7 +44,7 @@ def test_setup_teardown(): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_init(device): """Test the simulation context initialization.""" from isaaclab.sim.spawners.materials import RigidBodyMaterialCfg diff --git a/source/isaaclab/test/sim/test_views_xform_prim.py b/source/isaaclab/test/sim/test_views_xform_prim.py index 64cd86a7466f..827ed20d3261 100644 --- a/source/isaaclab/test/sim/test_views_xform_prim.py +++ b/source/isaaclab/test/sim/test_views_xform_prim.py @@ -11,6 +11,7 @@ """ from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices simulation_app = AppLauncher(headless=True).app @@ -101,7 +102,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_toggle(device): """Test toggling visibility multiple times.""" if device == "cuda" and not torch.cuda.is_available(): @@ -129,7 +130,7 @@ def test_visibility_toggle(device): assert vis[0] and not vis[1] and vis[2] -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_visibility_parent_inheritance(device): """Making a parent invisible hides all children.""" if device == "cuda" and not torch.cuda.is_available(): @@ -155,7 +156,7 @@ def test_visibility_parent_inheritance(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_prim_ordering_follows_creation_order(device): """Prims are returned in USD creation order (DFS), not alphabetical.""" if device == "cuda" and not torch.cuda.is_available(): @@ -181,7 +182,7 @@ def test_prim_ordering_follows_creation_order(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_standardize_transform_op(device): """FrameView standardizes a prim with xformOp:transform to translate/orient/scale.""" if device == "cuda" and not torch.cuda.is_available(): @@ -209,7 +210,7 @@ def test_standardize_transform_op(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_nested_hierarchy_world_poses(device): """World pose of nested child == sum of parent + child translations.""" if device == "cuda" and not torch.cuda.is_available(): @@ -271,7 +272,7 @@ def test_compare_get_world_poses_with_isaacsim(): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("device", test_devices()) def test_with_franka_robots(device): """Verify FrameView works with real Franka robot USD assets.""" if device == "cuda" and not torch.cuda.is_available(): diff --git a/source/isaaclab/test/utils/test_device_selection.py b/source/isaaclab/test/utils/test_device_selection.py new file mode 100644 index 000000000000..ff09b244e20a --- /dev/null +++ b/source/isaaclab/test/utils/test_device_selection.py @@ -0,0 +1,148 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for :func:`isaaclab.test.utils.test_devices` (the device-selection helper). + +These tests mock the host's device list, so they need no GPU and run on the +single-GPU CI lane. The helper is imported under an alias (``resolve_devices``) +for two reasons: pytest would otherwise collect the ``test_``-prefixed function +as a test case, and the multi-GPU workflow's auto-discovery greps for +``test_devices(...)`` call sites — neither should fire on this file. +""" + +import pytest + +from isaaclab.test.utils import devices as devices_mod +from isaaclab.test.utils.devices import test_devices as resolve_devices + +# Representative hosts, in mask order (cpu first, then cuda:0, cuda:1, ...). +SINGLE_GPU = ["cpu", "cuda:0"] +MULTI_GPU = ["cpu", "cuda:0", "cuda:1", "cuda:2"] + + +@pytest.fixture +def host(monkeypatch): + """Return a setter that pins the available device list and the runtime env var.""" + + def _set(available: list[str], runtime: str | None = None) -> None: + monkeypatch.setattr(devices_mod, "_list_available_devices", lambda: available) + if runtime is None: + monkeypatch.delenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, raising=False) + else: + monkeypatch.setenv(devices_mod._RUNTIME_DEVICES_ENV_VAR, runtime) + + return _set + + +# --------------------------------------------------------------------------- +# scope ∩ runtime resolution +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "available, runtime, scope, expected", + [ + # single-GPU CI lane: runtime unset -> default "110" (cpu + cuda:0). + (SINGLE_GPU, None, "11X", ["cpu", "cuda:0"]), # argless default is device-agnostic + (SINGLE_GPU, None, "110", ["cpu", "cuda:0"]), # pure-math scope + (SINGLE_GPU, None, "100", ["cpu"]), # cpu-only scope + (SINGLE_GPU, None, "00X", []), # non-default-only -> nothing here, skip + # multi-GPU shard: runtime pins exactly one non-default GPU. + (MULTI_GPU, "001", "11X", ["cuda:1"]), # cuda:1 shard + (MULTI_GPU, "0001", "11X", ["cuda:2"]), # cuda:2 shard + (MULTI_GPU, "0001", "00X", ["cuda:2"]), # non-default regression on its shard + (MULTI_GPU, "0001", "110", []), # math skips the shard (cuda:0 != this shard) + (MULTI_GPU, "0001", "100", []), # cpu test skips the shard + # a runtime that lists several GPUs hands back all in-scope ones. + (MULTI_GPU, "111", "11X", ["cpu", "cuda:0", "cuda:1"]), + ], +) +def test_resolves_scope_intersect_runtime(host, available, runtime, scope, expected): + host(available, runtime) + assert resolve_devices(scope) == expected + + +def test_argless_equals_default_scope(host): + # The common case: argless must equal the explicit default mask everywhere. + for available, runtime in [(SINGLE_GPU, None), (MULTI_GPU, "0001"), (MULTI_GPU, "001")]: + host(available, runtime) + assert resolve_devices() == resolve_devices("11X") + + +def test_argless_runs_once_on_one_non_default_gpu(host): + # Run-once property: with a one-device-per-shard runtime, an argless test + # resolves to exactly one non-default GPU. + host(MULTI_GPU, "0001") + result = resolve_devices() + assert result == ["cuda:2"] + assert all(d not in ("cpu", "cuda:0") for d in result) + + +# --------------------------------------------------------------------------- +# skip vs raise: legitimate skips never raise; only a missing runtime device does +# --------------------------------------------------------------------------- + + +def test_unset_runtime_never_raises(host): + # On a dev box / single-GPU lane (runtime unset), an out-of-scope result is a + # silent skip, not an error. + host(SINGLE_GPU, None) + assert resolve_devices("00X") == [] + + +def test_runtime_naming_absent_device_raises(host): + # A run that explicitly asked for cuda:2 on a host that only has cuda:0/cuda:1 + # is misconfigured -> fail loudly instead of a vacuous green. + host(["cpu", "cuda:0", "cuda:1"], "0001") + with pytest.raises(ValueError, match="no device available"): + resolve_devices("11X") + + +def test_in_range_runtime_with_empty_scope_does_not_raise(host): + # The runtime device exists (cuda:2 is present), the scope just doesn't include + # it -> skip, not raise. This is what keeps "110"/"100" tests green on a shard. + host(MULTI_GPU, "0001") + assert resolve_devices("110") == [] # would raise if the guard keyed off scope + + +# --------------------------------------------------------------------------- +# skip= : gate a specific device visibly +# --------------------------------------------------------------------------- + + +def test_skip_wraps_named_device_as_skipped_param(host): + host(SINGLE_GPU, None) + result = resolve_devices("11X", skip={"cuda:0": "known broken"}) + assert result[0] == "cpu" # untouched devices stay plain strings + param = result[1] + assert param.values == ("cuda:0",) + assert param.marks[0].name == "skip" + assert param.marks[0].kwargs["reason"] == "known broken" + + +def test_skip_ignores_out_of_result_devices(host): + # Skipping a device that isn't in the resolved set is a no-op (no stray params). + host(SINGLE_GPU, None) + assert resolve_devices("11X", skip={"cuda:3": "n/a"}) == ["cpu", "cuda:0"] + + +# --------------------------------------------------------------------------- +# _expand mask grammar +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "mask, count, expected", + [ + ("110", 2, [True, True]), # exact length, no wildcard + ("100", 3, [True, False, False]), # short mask padded with False + ("11X", 4, [True, True, True, True]), # trailing X fills the rest True + ("00X", 4, [False, False, True, True]), # X spans the non-default GPUs + ("11X", 2, [True, True]), # X with nothing left to fill + ("0001", 3, [False, False, False]), # mask longer than host -> truncated + ], +) +def test_expand(mask, count, expected): + assert devices_mod._expand(mask, count) == expected diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index a2d570d9d6ef..6062c25d0400 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -5,10 +5,11 @@ import pytest import torch +from isaaclab.test.utils import test_devices from isaaclab.utils.datasets import EpisodeData -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_is_empty(device): """Test checking whether the episode is empty.""" episode = EpisodeData() @@ -18,7 +19,7 @@ def test_is_empty(device): assert not episode.is_empty() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_tensors(device): """Test appending tensor data to the episode.""" dummy_data_0 = torch.tensor([0], device=device) @@ -55,7 +56,7 @@ def test_add_tensors(device): assert torch.equal(second_data, expected_added_data) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_add_dict_tensors(device): """Test appending dict data to the episode.""" dummy_dict_data_0 = { @@ -102,7 +103,7 @@ def test_add_dict_tensors(device): assert torch.equal(key_1_1_data, torch.tensor([[2], [5]], device=device)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_initial_state(device): """Test getting the initial state of the episode.""" dummy_initial_state = torch.tensor([1, 2, 3], device=device) @@ -114,7 +115,7 @@ def test_get_initial_state(device): assert torch.equal(initial_state, dummy_initial_state.unsqueeze(0)) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices("110")) def test_get_next_action(device): """Test getting next actions.""" # dummy actions diff --git a/source/isaaclab/test/utils/test_math.py b/source/isaaclab/test/utils/test_math.py index c2e5b3550816..4fad6881e56e 100644 --- a/source/isaaclab/test/utils/test_math.py +++ b/source/isaaclab/test/utils/test_math.py @@ -13,6 +13,7 @@ import torch.utils.benchmark as benchmark import isaaclab.utils.math as math_utils +from isaaclab.test.utils import test_devices DECIMAL_PRECISION = 5 """Precision of the test. @@ -22,7 +23,7 @@ """ -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_scale_unscale_transform(device, size): """Test scale_transform and unscale_transform.""" @@ -59,7 +60,7 @@ def test_scale_unscale_transform(device, size): torch.testing.assert_close(output_unscale_offset, inputs) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_saturate(device, size): "Test saturate of a tensor of differed shapes and device." @@ -81,7 +82,7 @@ def test_saturate(device, size): assert torch.all(torch.less_equal(output_per_batch, upper_per_batch)).item() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((5, 4, 3), (10, 2))) def test_normalize(device, size): """Test normalize of a tensor along its last dimension and check the norm of that dimension is close to 1.0.""" @@ -93,7 +94,7 @@ def test_normalize(device, size): torch.testing.assert_close(norm, torch.ones(size[0:-1], device=device)) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_copysign(device): """Test copysign by copying a sign from both a negative and positive value and verify that the new sign is the same. @@ -123,7 +124,7 @@ def test_copysign(device): torch.testing.assert_close(expected_value_neg_dim1_neg, value_neg_dim1_neg) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_is_identity_pose(device): """Test is_identity_pose method.""" # Single row identity pose (xyzw format) @@ -147,7 +148,7 @@ def test_is_identity_pose(device): assert math_utils.is_identity_pose(identity_pos, identity_rot) is False -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat(device): """Test axis_angle_from_quat method.""" # Quaternions of the form (2,4) and (2,2,4) in xyzw format @@ -171,7 +172,7 @@ def test_axis_angle_from_quat(device): torch.testing.assert_close(math_utils.axis_angle_from_quat(quat), angle) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_axis_angle_from_quat_approximation(device): """Test the Taylor approximation from axis_angle_from_quat method. @@ -197,7 +198,7 @@ def test_axis_angle_from_quat_approximation(device): torch.testing.assert_close(axis_angle_computed, axis_angle_expected) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_magnitude(device): """Test quat_error_magnitude method.""" # No rotation (xyzw format) @@ -236,7 +237,7 @@ def test_quat_error_magnitude(device): torch.testing.assert_close(q12_diff, expected_diff) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_unique(device): """Test quat_unique method.""" # Define test cases @@ -254,7 +255,7 @@ def test_quat_unique(device): torch.testing.assert_close(pos_real_quats[~non_pos_indices], quats[~non_pos_indices]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_mul_with_quat_unique(device): """Test quat_mul method with different quaternions. @@ -287,7 +288,7 @@ def test_quat_mul_with_quat_unique(device): torch.testing.assert_close(quat_result_3, quat_result_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_error_mag_with_quat_unique(device): """Test quat_error_magnitude method with positive real quaternions.""" @@ -310,7 +311,7 @@ def test_quat_error_mag_with_quat_unique(device): torch.testing.assert_close(error_4, error_1) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_convention_converter(device): """Test convert_camera_frame_orientation_convention to and from ros, opengl, and world conventions.""" # Quaternions in xyzw format (converted from original wxyz test values) @@ -348,7 +349,7 @@ def test_convention_converter(device): ) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("size", ((10, 4), (5, 3, 4))) def test_convert_quat(device, size): """Test convert_quat from "xyzw" to "wxyz" and back to "xyzw" and verify the correct rolling of the tensor. @@ -383,7 +384,7 @@ def test_convert_quat(device, size): math_utils.convert_quat(quat, to="xwyz") -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_conjugate(device): """Test quat_conjugate by checking the sign of the imaginary part changes but the magnitudes stay the same.""" @@ -397,7 +398,7 @@ def test_quat_conjugate(device): torch.testing.assert_close(expected_real, value[..., 3]) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", (1, 10)) @pytest.mark.parametrize( "euler_angles", @@ -428,7 +429,7 @@ def test_quat_from_euler_xyz(device, num_envs, euler_angles): torch.testing.assert_close(expected_quat, quat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_wrap_to_pi(device): """Test wrap_to_pi method.""" # No wrapping needed @@ -464,7 +465,7 @@ def test_wrap_to_pi(device): torch.testing.assert_close(wrapped_angle, expected_angle) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("shape", ((3,), (1024, 3))) def test_skew_symmetric_matrix(device, shape): """Test skew_symmetric_matrix.""" @@ -494,7 +495,7 @@ def test_skew_symmetric_matrix(device, shape): torch.testing.assert_close(vec_rand_resized[:, 0], mat_value[:, 2, 1]) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_orthogonalize_perspective_depth(device): """Test for converting perspective depth to orthogonal depth.""" # Create a sample perspective depth image (N, H, W) @@ -515,7 +516,7 @@ def test_orthogonalize_perspective_depth(device): torch.testing.assert_close(orthogonal_depth, expected_orthogonal_depth) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_combine_frame_transform(device): """Test combine_frame_transforms function.""" # create random poses @@ -540,7 +541,7 @@ def test_combine_frame_transform(device): torch.testing.assert_close(pose01, torch.cat((pos01, quat01), dim=-1)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4]) def test_interpolate_poses(device, seed): """Test interpolate_poses function. @@ -608,7 +609,7 @@ def test_pose_inv(): np.testing.assert_array_almost_equal(result, expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_to_and_from_angle_axis(device): """Test that axis_angle_from_quat against scipy and that quat_from_angle_axis are the inverse of each other.""" n = 1024 @@ -627,7 +628,7 @@ def test_quat_to_and_from_angle_axis(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus(device): """Test quat_box_minus method. @@ -645,7 +646,7 @@ def test_quat_box_minus(device): torch.testing.assert_close(expected_diff, axis_diff, atol=1e-06, rtol=1e-06) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_box_minus_and_quat_box_plus(device): """Test consistency of quat_box_plus and quat_box_minus. @@ -687,7 +688,7 @@ def test_quat_box_minus_and_quat_box_plus(device): torch.testing.assert_close(delta_result, delta_angle, atol=1e-04, rtol=1e-04) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t12_inputs", ["True", "False"]) @pytest.mark.parametrize("q12_inputs", ["True", "False"]) def test_combine_frame_transforms(device, t12_inputs, q12_inputs): @@ -726,7 +727,7 @@ def test_combine_frame_transforms(device, t12_inputs, q12_inputs): torch.testing.assert_close(math_utils.quat_unique(expected_quat), math_utils.quat_unique(quat_value)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("t02_inputs", ["True", "False"]) @pytest.mark.parametrize("q02_inputs", ["True", "False"]) def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): @@ -766,7 +767,7 @@ def test_subtract_frame_transforms(device, t02_inputs, q02_inputs): torch.testing.assert_close(math_utils.quat_unique(q02_expected), math_utils.quat_unique(q02_compare)) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("rot_error_type", ("quat", "axis_angle")) def test_compute_pose_error(device, rot_error_type): """Test compute_pose_error for different rot_error_type.""" @@ -794,7 +795,7 @@ def test_compute_pose_error(device, rot_error_type): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_twist_transform(device): """Test rigid_body_twist_transform method. @@ -822,7 +823,7 @@ def test_rigid_body_twist_transform(device): torch.testing.assert_close(w_AA_, w_AA) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_yaw_quat(device): """ Test for yaw_quat methods. @@ -844,7 +845,7 @@ def test_yaw_quat(device): torch.testing.assert_close(result, expected_output) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_slerp(device): """Test quat_slerp function. @@ -874,7 +875,7 @@ def test_quat_slerp(device): np.testing.assert_array_almost_equal(result.cpu(), expected, decimal=DECIMAL_PRECISION) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_matrix_from_quat(device): """test matrix_from_quat against scipy.""" # prepare random quaternions and vectors @@ -893,7 +894,7 @@ def test_matrix_from_quat(device): torch.testing.assert_close(q_rand, q_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "euler_angles", [ @@ -926,7 +927,7 @@ def test_matrix_from_euler(device, euler_angles, convention): torch.testing.assert_close(expected_mag, mat_value) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply(device): """Test for quat_apply against scipy.""" # prepare random quaternions and vectors @@ -943,7 +944,7 @@ def test_quat_apply(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_apply_inverse(device): """Test for quat_apply against scipy.""" @@ -963,7 +964,7 @@ def test_quat_apply_inverse(device): torch.testing.assert_close(scipy_result.to(device=device), apply_result, atol=2e-4, rtol=2e-4) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_quat_inv(device): """Test for quat_inv method. @@ -1040,7 +1041,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor return a - b + c # check that implementation produces the same result as the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors q_rand = math_utils.random_orientation(num=1024, device=device) v_rand = math_utils.sample_uniform(-1000, 1000, (1024, 3), device=device) @@ -1064,7 +1065,7 @@ def einsum_quat_rotate_inverse(q: torch.Tensor, v: torch.Tensor) -> torch.Tensor torch.testing.assert_close(einsum_result_inv, new_result_inv, atol=1e-3, rtol=1e-3) # check the performance of the new implementation - for device in ["cpu", "cuda:0"]: + for device in test_devices("110"): # prepare random quaternions and vectors # new implementation supports batched inputs q_shape = (1024, 2, 5, 4) @@ -1315,7 +1316,7 @@ def test_euler_xyz_from_quat(): torch.testing.assert_close(output, wrapped) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): """Camera above target on +Z axis with Z-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, 5.0]], device=device) @@ -1327,7 +1328,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_z(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): """Camera at +Y looking at origin with Y-up should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 5.0, 0.0]], device=device) @@ -1339,7 +1340,7 @@ def test_create_rotation_matrix_from_view_lookat_along_up_axis_y(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): """Camera below target looking up (-Z alignment with Z-up) should return a valid orthonormal frame.""" eyes = torch.tensor([[0.0, 0.0, -5.0]], device=device) @@ -1351,7 +1352,7 @@ def test_create_rotation_matrix_from_view_lookat_along_negative_up_axis(device): torch.testing.assert_close(torch.linalg.det(R), torch.ones(1, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): """When eyes == targets the forward direction is undefined; all entries of the row are NaN.""" eyes = torch.tensor([[1.0, 2.0, 3.0]], device=device) @@ -1360,7 +1361,7 @@ def test_create_rotation_matrix_from_view_zero_forward_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_batched_partial_failure(device): """Mixed batch with one degenerate row should produce NaN in that row and a valid rotation in the other.""" eyes = torch.tensor([[1.0, 2.0, 3.0], [0.0, 0.0, 5.0]], device=device) @@ -1371,7 +1372,7 @@ def test_create_rotation_matrix_from_view_batched_partial_failure(device): torch.testing.assert_close(torch.linalg.det(R[1]), torch.tensor(1.0, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_unit_norm_on_valid_input(device): """quat_from_matrix should produce unit quaternions for any valid rotation matrix.""" n = 100 @@ -1382,7 +1383,7 @@ def test_quat_from_matrix_unit_norm_on_valid_input(device): torch.testing.assert_close(norms, torch.ones(n, device=device), atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_singular_matrix_returns_nan(device): """quat_from_matrix on a singular (non-rotation) matrix should signal NaN, not garbage.""" singular = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device) @@ -1390,7 +1391,7 @@ def test_quat_from_matrix_singular_matrix_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_standard(device): """Sanity: off-axis eye produces an orthonormal frame whose z-axis points from target back to eye.""" eyes = torch.tensor([[3.0, 0.0, 4.0]], device=device) @@ -1404,7 +1405,7 @@ def test_create_rotation_matrix_from_view_standard(device): torch.testing.assert_close(R[:, :, 2], expected_z, atol=1e-5, rtol=1e-5) -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): """Non-finite input (NaN or Inf in eyes/targets) should produce NaN rows.""" eyes = torch.tensor([[float("nan"), 0.0, 0.0]], device=device) @@ -1413,7 +1414,7 @@ def test_create_rotation_matrix_from_view_non_finite_returns_nan(device): assert torch.isnan(R).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_reflection_returns_nan(device): """A reflection matrix (det = -1) is not a proper rotation; the safeguard should signal NaN.""" reflection = torch.diag(torch.tensor([1.0, 1.0, -1.0], device=device)).unsqueeze(0) @@ -1421,7 +1422,7 @@ def test_quat_from_matrix_reflection_returns_nan(device): assert torch.isnan(q).all() -@pytest.mark.parametrize("device", ("cpu", "cuda:0")) +@pytest.mark.parametrize("device", test_devices()) def test_quat_from_matrix_non_orthonormal_returns_nan(device): """A non-orthonormal matrix (1% scale error on one axis) is not a valid rotation; expect NaN.""" R = torch.diag(torch.tensor([1.01, 1.0, 1.0], device=device)).unsqueeze(0) diff --git a/source/isaaclab/test/utils/test_modifiers.py b/source/isaaclab/test/utils/test_modifiers.py index 6e0e39820fdf..b14622e90550 100644 --- a/source/isaaclab/test/utils/test_modifiers.py +++ b/source/isaaclab/test/utils/test_modifiers.py @@ -9,6 +9,7 @@ import torch import isaaclab.utils.modifiers as modifiers +from isaaclab.test.utils import test_devices from isaaclab.utils.configclass import configclass @@ -142,7 +143,7 @@ def test_torch_relu_modifier(): assert torch.allclose(output, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_digital_filter(device): """Test digital filter modifier.""" # create test data @@ -178,7 +179,7 @@ def test_digital_filter(device): torch.testing.assert_close(processed_data, test_cfg.result) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_integral(device): """Test integral modifier.""" # create test data diff --git a/source/isaaclab/test/utils/test_noise.py b/source/isaaclab/test/utils/test_noise.py index 45f9d2d61384..c1035453b5a9 100644 --- a/source/isaaclab/test/utils/test_noise.py +++ b/source/isaaclab/test/utils/test_noise.py @@ -7,9 +7,10 @@ import torch import isaaclab.utils.noise as noise +from isaaclab.test.utils import test_devices -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_gaussian_noise(device, noise_device, op): @@ -42,7 +43,7 @@ def test_gaussian_noise(device, noise_device, op): torch.testing.assert_close(noise_cfg.mean, mean_result, atol=1e-2, rtol=1e-2) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_uniform_noise(device, noise_device, op): @@ -77,7 +78,7 @@ def test_uniform_noise(device, noise_device, op): assert all(torch.ge(noise_cfg.n_max + 1e-5, max_result).tolist()) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices("110")) @pytest.mark.parametrize("noise_device", ["cpu", "cuda:0"]) @pytest.mark.parametrize("op", ["add", "scale", "abs"]) def test_constant_noise(device, noise_device, op): diff --git a/source/isaaclab/test/utils/test_wrench_composer.py b/source/isaaclab/test/utils/test_wrench_composer.py index b711aaab44a6..2b13cf23a9f2 100644 --- a/source/isaaclab/test/utils/test_wrench_composer.py +++ b/source/isaaclab/test/utils/test_wrench_composer.py @@ -9,6 +9,7 @@ import warp as wp from isaaclab.test.mock_interfaces.assets import MockRigidObjectCollection +from isaaclab.test.utils import test_devices from isaaclab.utils.wrench_composer import WrenchComposer @@ -97,7 +98,7 @@ def random_unit_quaternion_np(rng: np.random.Generator, shape: tuple) -> np.ndar return q -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): @@ -136,7 +137,7 @@ def test_wrench_composer_add_force(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_force_np, hand_calculated_composed_force_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int): @@ -175,7 +176,7 @@ def test_wrench_composer_add_torque(device: str, num_envs: int, num_bodies: int) assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): @@ -234,7 +235,7 @@ def test_add_forces_at_positions(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -280,7 +281,7 @@ def test_add_torques_at_position(device: str, num_envs: int, num_bodies: int): assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodies: int): @@ -343,7 +344,7 @@ def test_add_forces_and_torques_at_position(device: str, num_envs: int, num_bodi assert np.allclose(composed_torque_np, hand_calculated_composed_torque_np, atol=1, rtol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100, 1000]) @pytest.mark.parametrize("num_bodies", [1, 3, 5, 10]) def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): @@ -392,7 +393,7 @@ def test_wrench_composer_reset(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -434,7 +435,7 @@ def test_global_forces_with_rotation(device: str, num_envs: int, num_bodies: int ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: int): @@ -476,7 +477,7 @@ def test_global_torques_with_rotation(device: str, num_envs: int, num_bodies: in ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies: int): @@ -540,7 +541,7 @@ def test_global_forces_at_global_position(device: str, num_envs: int, num_bodies ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_local_vs_global_identity_quaternion(device: str): """Test that local and global give same result with identity quaternion and zero position.""" rng = np.random.default_rng(seed=13) @@ -582,7 +583,7 @@ def test_local_vs_global_identity_quaternion(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_90_degree_rotation_global_force(device: str): """Test global force with a known 90-degree rotation for easy verification.""" num_envs, num_bodies = 1, 1 @@ -615,7 +616,7 @@ def test_90_degree_rotation_global_force(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composition_mixed_local_and_global(device: str): """Test that local and global forces can be composed together correctly.""" rng = np.random.default_rng(seed=14) @@ -660,7 +661,7 @@ def test_composition_mixed_local_and_global(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 50]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: int): @@ -705,7 +706,7 @@ def test_local_forces_at_local_position(device: str, num_envs: int, num_bodies: assert np.allclose(composed_torque_np, expected_torques, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_link_origin_no_torque(device: str): """Test that a global force applied at the link origin produces no torque.""" rng = np.random.default_rng(seed=16) @@ -753,7 +754,7 @@ def test_global_force_at_link_origin_no_torque(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): @@ -819,7 +820,7 @@ def test_add_raw_buffers_from(device: str, num_envs: int, num_bodies: int): ), "add_raw_buffers_from torque mismatch vs direct accumulation" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_add_raw_buffers_from_inactive_is_noop(device: str): """Test that add_raw_buffers_from is a no-op when the source composer is inactive.""" num_envs, num_bodies = 4, 2 @@ -853,7 +854,7 @@ def test_add_raw_buffers_from_inactive_is_noop(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): @@ -908,7 +909,7 @@ def test_add_forces_mask(device: str, num_envs: int, num_bodies: int): ), f"Mask vs index torque mismatch (envs={num_envs}, bodies={num_bodies})" -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("num_envs", [1, 10, 100]) @pytest.mark.parametrize("num_bodies", [1, 3, 5]) def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): @@ -963,7 +964,7 @@ def test_add_forces_mask_global(device: str, num_envs: int, num_bodies: int): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_overwrites_previous_add(device: str): """Test that set_forces_and_torques_index clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -992,7 +993,7 @@ def test_set_forces_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_index clears only the targeted environments.""" num_envs, num_bodies = 4, 3 @@ -1063,7 +1064,7 @@ def test_set_forces_clears_targeted_envs_only(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_partial_reset_zeros_only_specified_envs(device: str): """Test that partial reset zeros only the specified environments and leaves others intact.""" num_envs, num_bodies = 8, 3 @@ -1114,7 +1115,7 @@ def test_partial_reset_zeros_only_specified_envs(device: str): assert composer._dirty -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_full_reset_clears_active_flag(device: str): """Test that full reset (no args) clears the _active flag.""" num_envs, num_bodies = 4, 2 @@ -1138,7 +1139,7 @@ def test_full_reset_clears_active_flag(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_force_emits_deprecation_warning(device: str): """Test that accessing composed_force emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1158,7 +1159,7 @@ def test_composed_force_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_force_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_composed_torque_emits_deprecation_warning(device: str): """Test that accessing composed_torque emits a DeprecationWarning.""" num_envs, num_bodies = 2, 1 @@ -1177,7 +1178,7 @@ def test_composed_torque_emits_deprecation_warning(device: str): assert np.allclose(result.warp.numpy(), composer.out_torque_b.warp.numpy(), atol=1e-7) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_add_forces_and_torques_emits_warning(device: str): """Test that the deprecated add_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1202,7 +1203,7 @@ def test_deprecated_add_forces_and_torques_emits_warning(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_overwrites_previous_add(device: str): """Test that set_forces_and_torques_mask clears previously accumulated values.""" num_envs, num_bodies = 4, 2 @@ -1231,7 +1232,7 @@ def test_set_forces_mask_overwrites_previous_add(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_clears_targeted_envs_only(device: str): """Test that set_forces_and_torques_mask clears only the masked environments.""" num_envs, num_bodies = 4, 3 @@ -1301,7 +1302,7 @@ def test_set_forces_mask_clears_targeted_envs_only(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_forces_mask_matches_set_forces_index(device: str): """Test that set_forces_and_torques_mask produces the same result as the index variant.""" num_envs, num_bodies = 6, 3 @@ -1351,7 +1352,7 @@ def test_set_forces_mask_matches_set_forces_index(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_force_b_triggers_lazy_composition(device: str): """Test that accessing out_force_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1378,7 +1379,7 @@ def test_out_force_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_torque_b_triggers_lazy_composition(device: str): """Test that accessing out_torque_b without explicit compose_to_body_frame still returns correct results.""" num_envs, num_bodies = 4, 2 @@ -1405,7 +1406,7 @@ def test_out_torque_b_triggers_lazy_composition(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_lazy_composition_tracks_dirty_flag(device: str): """Test that the dirty flag is correctly managed through add/compose/add cycles.""" num_envs, num_bodies = 2, 1 @@ -1442,7 +1443,7 @@ def test_lazy_composition_tracks_dirty_flag(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), expected, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_compose_is_idempotent(device: str): """Calling compose_to_body_frame twice without intervening writes produces the same result.""" rng = np.random.default_rng(seed=456) @@ -1493,7 +1494,7 @@ def test_compose_is_idempotent(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_with_com_offset(device: str): """Test that torque correction uses CoM position, not link position, when they differ.""" num_envs, num_bodies = 2, 1 @@ -1548,7 +1549,7 @@ def test_global_force_with_com_offset(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_global_force_at_com_no_torque_with_com_offset(device: str): """Test that a global force at CoM position produces zero torque even with CoM offset.""" num_envs, num_bodies = 2, 1 @@ -1594,7 +1595,7 @@ def test_global_force_at_com_no_torque_with_com_offset(device: str): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_com_offset_with_rotation(device: str): """Test torque correction with both CoM offset and non-identity rotation.""" num_envs, num_bodies = 1, 1 @@ -1652,7 +1653,7 @@ def test_com_offset_with_rotation(device: str): # ============================================================================ -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_emits_warning(device: str): """Test that the deprecated set_forces_and_torques wrapper emits a warning and works.""" num_envs, num_bodies = 4, 2 @@ -1672,7 +1673,7 @@ def test_deprecated_set_forces_and_torques_emits_warning(device: str): assert np.allclose(composer.out_force_b.warp.numpy(), forces_np, atol=1e-4, rtol=1e-5) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_deprecated_set_forces_and_torques_clears_previous(device: str): """Test that deprecated set_forces_and_torques actually replaces previous values.""" num_envs, num_bodies = 4, 2 diff --git a/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst new file mode 100644 index 000000000000..c53da690a8d8 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/jichuanh-multi-gpu-ci.rst @@ -0,0 +1,5 @@ +Fixed +^^^^^ + +* Fixed Newton physics failing to initialize on non-default CUDA devices + (``cuda:1`` and higher). diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py index e8f3933f692f..25973385f057 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_manager.py @@ -978,6 +978,15 @@ def start_simulation(cls) -> None: cls._cl_inject_sites_fallback() device = PhysicsManager._device + # Pin torch + Warp to the target device before any Warp/Newton + # allocations. Without this, mujoco_warp's collision pipeline later + # allocates on cuda:1 against a primary CUDA context that was never + # made current, returning null pointers (issue #5132). + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) logger.info(f"Finalizing model on device: {device}") cls._builder.up_axis = Axis.from_string(cls._up_axis) # Forward pending extended attribute requests to builder and clear them @@ -1258,6 +1267,16 @@ def initialize_solver(cls) -> None: if cfg is None: return + # Pin torch + Warp to the target device before solver build and + # collision-pipeline init. Mirrors the guard in :meth:`start_simulation` + # (issue #5132); idempotent if already pinned. + device = PhysicsManager._device + if device and "cuda" in device: + import torch + + torch.cuda.set_device(device) + wp.set_device(device) + with Timer(name="newton_initialize_solver", msg="Initialize solver took:"): NewtonManager._num_substeps = cfg.num_substeps # type: ignore[union-attr] NewtonManager._collision_decimation = cfg.collision_decimation # type: ignore[union-attr] @@ -1331,7 +1350,7 @@ def _capture_or_defer_graph(cls) -> None: with Timer(name="newton_cuda_graph", msg="CUDA graph took:"): if cls._usdrt_stage is None: simulate = cls._simulate_full if cls._is_all_graphable() else cls._simulate_physics_only - with wp.ScopedCapture() as capture: + with wp.ScopedCapture(device=device) as capture: simulate() NewtonManager._graph = capture.graph logger.info("Newton CUDA graph captured (standard Warp mode)") diff --git a/source/isaaclab_newton/test/assets/test_articulation.py b/source/isaaclab_newton/test/assets/test_articulation.py index e21f520c4324..40540c0ac655 100644 --- a/source/isaaclab_newton/test/assets/test_articulation.py +++ b/source/isaaclab_newton/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -479,7 +480,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -529,7 +530,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_articulations", [2, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -569,7 +570,7 @@ def test_gravity_vec_w_tracks_model_gravity(sim, num_articulations, device, add_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -619,7 +620,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base(sim, num_articulations, device, articulation_type): """Test initialization for fixed base. @@ -676,7 +677,7 @@ def test_initialization_fixed_base(sim, num_articulations, device, articulation_ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -734,7 +735,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) def test_initialization_hand_with_tendons(sim, num_articulations, device, articulation_type): """Test initialization for fixed base articulated hand with tendons. @@ -783,7 +784,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device, articu @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_initialization_floating_base_made_fixed_base( @@ -837,7 +838,7 @@ def test_initialization_floating_base_made_fixed_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_initialization_fixed_base_made_floating_base( @@ -883,7 +884,7 @@ def test_initialization_fixed_base_made_floating_base( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -914,7 +915,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_out_of_range_default_joint_vel(sim, device, articulation_type): """Test that the default joint velocity from configuration is out of range. @@ -939,7 +940,7 @@ def test_out_of_range_default_joint_vel(sim, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1015,7 +1016,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1049,7 +1050,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_buffer(sim, num_articulations, device, articulation_type): """Test if external force buffer correctly updates in the force value is zero case. @@ -1134,7 +1135,7 @@ def test_external_force_buffer(sim, num_articulations, device, articulation_type @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation. @@ -1192,7 +1193,7 @@ def test_external_force_on_single_body(sim, num_articulations, device, articulat @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_single_body_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the base of the articulation at a given position. @@ -1287,7 +1288,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation. @@ -1347,7 +1348,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device, artic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device, articulation_type): """Test application of external force on the legs of the articulation at a given position. @@ -1441,7 +1442,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_loading_gains_from_usd(sim, num_articulations, device, articulation_type): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1503,7 +1504,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device, articulation_typ @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1538,7 +1539,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulation_type): """Test that gains are loaded from the configuration dictionary correctly. @@ -1571,7 +1572,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1643,7 +1644,7 @@ def test_setting_velocity_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1699,7 +1700,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) @@ -1756,7 +1757,7 @@ def test_setting_effort_limit_implicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) @pytest.mark.parametrize("articulation_type", ["single_joint_explicit"]) @@ -1822,7 +1823,7 @@ def test_setting_effort_limit_explicit( @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_reset(sim, num_articulations, device, articulation_type): """Test that reset method works properly.""" @@ -1866,7 +1867,7 @@ def test_reset(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -1906,7 +1907,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("articulation_type", ["single_joint_implicit"]) def test_body_root_state(sim, num_articulations, device, with_offset, articulation_type): @@ -2031,7 +2032,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset, articulati @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -2120,7 +2121,7 @@ def test_write_root_state( torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2139,7 +2140,7 @@ def test_setting_articulation_root_prim_path(sim, device, articulation_type): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["humanoid"]) def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_type): """Test that the articulation root prim path can be set explicitly.""" @@ -2158,7 +2159,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device, articulation_t @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled, articulation_type): @@ -2264,7 +2265,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["shadow_hand"]) @pytest.mark.skip(reason="Spatial tendons are not supported in Newton yet.") def test_spatial_tendons(sim, num_articulations, device, articulation_type): @@ -2318,7 +2319,7 @@ def test_spatial_tendons(sim, num_articulations, device, articulation_type): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane, articulation_type): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -2388,7 +2389,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_body_q_consistent_after_root_write(num_articulations, device, articulation_type): """Test that body_q is fresh when collide() runs after a root pose write. @@ -2470,7 +2471,7 @@ def _patched_simulate(cls): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) via view-level APIs.""" @@ -2526,7 +2527,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2550,7 +2551,7 @@ def test_randomize_rigid_body_com(sim, num_articulations, device, add_ground_pla @pytest.mark.parametrize("num_articulations", [2]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, add_ground_plane, articulation_type): @@ -2583,7 +2584,7 @@ def test_randomize_rigid_body_collider_offsets(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci @pytest.mark.xfail( @@ -2635,7 +2636,7 @@ def test_get_gravity_compensation_forces_not_implemented_on_newton(sim, num_arti @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2661,7 +2662,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2697,7 +2698,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2730,7 +2731,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2756,7 +2757,7 @@ def test_get_mass_matrix_shape_floating_base(sim, num_articulations, device, add assert M.dtype == torch.float32 -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2844,7 +2845,7 @@ def test_heterogeneous_scene_per_view_shapes(sim, device, add_ground_plane, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2923,7 +2924,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2974,7 +2975,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3025,7 +3026,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3058,7 +3059,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -3135,7 +3136,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_newton/test/assets/test_rigid_object.py b/source/isaaclab_newton/test/assets/test_rigid_object.py index 8811de5b9db7..deed30bd5c53 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -121,7 +122,7 @@ def generate_cubes_scene( @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -156,7 +157,7 @@ def test_initialization(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="Newton does not support kinematic rigid bodies") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -194,7 +195,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -212,7 +213,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -229,7 +230,7 @@ def test_initialization_with_articulation_root(num_cubes, device): @pytest.mark.isaacsim_ci -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -298,7 +299,7 @@ def test_external_force_buffer(device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -373,7 +374,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -462,7 +463,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -530,7 +531,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -573,7 +574,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object via view-level APIs.""" with _newton_sim_context(device, gravity_enabled=True, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -628,7 +629,7 @@ def _set_newton_material_properties(cube_object, friction_val, restitution_val, @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo contact at height=0 does not settle the same as PhysX — cube falls on z-axis") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -668,7 +669,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -681,7 +682,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo uses Coulomb friction (single mu), no static/dynamic distinction") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -761,7 +762,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.skip(reason="MuJoCo restitution model differs from PhysX — inelastic collisions still bounce") @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -839,7 +840,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _newton_sim_context(device, gravity_enabled=False, add_ground_plane=True, auto_add_lighting=True) as sim: @@ -877,7 +878,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -953,7 +954,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_cubes, device): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -1066,7 +1067,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -1137,7 +1138,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.isaacsim_ci @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): @@ -1297,7 +1298,7 @@ def test_warmup_attach_stage_not_called_for_cpu(): ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_link_pose_w_fresh_after_root_pose_write(device, writer): diff --git a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py index 18928bedb5d8..9d5355adc31b 100644 --- a/source/isaaclab_newton/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_newton/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -119,7 +120,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -151,7 +152,7 @@ def test_initialization(num_envs, num_cubes, device): @pytest.mark.skip(reason="Newton doesn't support kinematic rigid bodies yet") @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -187,7 +188,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -202,7 +203,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -258,7 +259,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -320,7 +321,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -403,7 +404,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_object_state(num_envs, num_cubes, device): """Test setting the state of the object. @@ -475,7 +476,7 @@ def test_set_object_state(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _newton_sim_context(device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -510,7 +511,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object collection via view-level APIs.""" with _newton_sim_context(device, auto_add_lighting=True) as sim: @@ -549,7 +550,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -622,7 +623,7 @@ def test_gravity_vec_w_tracks_model_gravity(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset): """Test the object_com_state_w and object_link_state_w properties.""" @@ -717,7 +718,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_object_state(num_envs, num_cubes, device, with_offset, state_location): @@ -795,7 +796,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_object_state_functions_data_consistency(num_envs, num_cubes, device, with_offset, state_location): @@ -932,7 +933,7 @@ def test_write_object_state_functions_data_consistency(num_envs, num_cubes, devi torch.testing.assert_close(body_com_vel_w[..., 3:], link_vel_w[..., 3:]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("writer", ["link_index", "link_mask", "com_index", "com_mask"]) @pytest.mark.isaacsim_ci def test_body_pose_write_marks_fk_reset_mask(device, writer): diff --git a/source/isaaclab_newton/test/sensors/test_contact_sensor.py b/source/isaaclab_newton/test/sensors/test_contact_sensor.py index 2d92e3274a7e..66452b62d553 100644 --- a/source/isaaclab_newton/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_newton/test/sensors/test_contact_sensor.py @@ -20,6 +20,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import math @@ -74,7 +76,7 @@ class ContactSensorTestSceneCfg(InteractiveSceneCfg): # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -193,7 +195,7 @@ def test_contact_lifecycle(device: str, use_mujoco_contacts: bool, shape_type: S assert no_contact_detected[env_idx], f"Env {env_idx}: Contact should stop after lift." -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) @pytest.mark.parametrize("shape_type", STABLE_SHAPES, ids=[shape_type_to_str(s) for s in STABLE_SHAPES]) def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: bool, shape_type: ShapeType): @@ -298,7 +300,7 @@ def test_horizontal_collision_detects_contact(device: str, use_mujoco_contacts: # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): """Test that resting object contact force equals weight and points upward. @@ -401,7 +403,7 @@ def test_resting_object_contact_force(device: str, use_mujoco_contacts: bool): assert not errs, "\n".join(errs) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("use_mujoco_contacts", COLLISION_PIPELINES) def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contacts: bool): """Test that dropping from higher produces larger peak impact force. @@ -483,7 +485,7 @@ def test_higher_drop_produces_larger_impact_force(device: str, use_mujoco_contac # =================================================================== -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -618,7 +620,7 @@ def test_filter_enables_force_matrix(device: str, use_mujoco_contacts: bool): } -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize( "use_mujoco_contacts", [ @@ -815,7 +817,7 @@ def _make_two_box_scene_cfg(num_envs: int) -> ContactSensorTestSceneCfg: return scene_cfg -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_sensor_metadata(device: str): """Verify sensor_names and filter_object_names match the underlying sensing and counterpart configuration across body-mode, body-mode-with-filter, and shape-mode. diff --git a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py index d114a1da2a80..9ff6c7a8610a 100644 --- a/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py +++ b/source/isaaclab_newton/test/sim/test_views_xform_prim_newton.py @@ -13,6 +13,8 @@ import sys from pathlib import Path +from isaaclab.test.utils import test_devices + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "isaaclab" / "test" / "sim")) @@ -102,7 +104,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_body_path(device): """FrameView rejects prim paths that resolve to a Newton physics body.""" ctx = _sim_context(device, num_envs=2) @@ -116,7 +118,7 @@ def test_reject_body_path(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_reject_shape_path(device): """FrameView rejects prim paths that resolve to a Newton collision shape.""" ctx = _sim_context(device, num_envs=2) @@ -183,7 +185,7 @@ def test_view_can_resolve_from_body_labels_after_reset(device): # ================================================================== -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_returns_initial_pose(device): """A world-rooted frame returns its configured position.""" ctx = _sim_context(device, num_envs=2) @@ -201,7 +203,7 @@ def test_world_attached_returns_initial_pose(device): ctx.__exit__(None, None, None) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_world_attached_set_world_roundtrip(device): """A world-attached prim can be repositioned via set_world_poses.""" ctx = _sim_context(device, num_envs=2) diff --git a/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_ovphysx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ovphysx/test/assets/test_articulation.py b/source/isaaclab_ovphysx/test/assets/test_articulation.py index 2dd13061cf2e..b14b416cf5bc 100644 --- a/source/isaaclab_ovphysx/test/assets/test_articulation.py +++ b/source/isaaclab_ovphysx/test/assets/test_articulation.py @@ -56,6 +56,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -82,7 +84,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _OMNI_PHYSX_SCHEMAS_GAP_REASON = ( @@ -329,7 +331,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -391,7 +393,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -454,7 +456,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -524,7 +526,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -595,7 +597,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -653,7 +655,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.xfail(reason=_OMNI_PHYSX_SCHEMAS_GAP_REASON, strict=False) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): @@ -720,7 +722,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -778,7 +780,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -808,7 +810,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -832,7 +834,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -907,7 +909,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -940,7 +942,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -1024,7 +1026,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1081,7 +1083,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1175,7 +1177,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1234,7 +1236,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1327,7 +1329,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1388,7 +1390,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1422,7 +1424,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1454,7 +1456,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1521,7 +1523,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1574,7 +1576,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1628,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1687,7 +1689,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1730,7 +1732,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1769,7 +1771,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1895,7 +1897,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1979,7 +1981,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, device): """Test the data.body_incoming_joint_wrench_b buffer is populated correctly and statically correct for single joint. @@ -2079,7 +2081,7 @@ def test_body_incoming_joint_wrench_b_single_joint(sim, num_articulations, devic ) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -2097,7 +2099,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -2115,7 +2117,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2220,7 +2222,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2272,7 +2274,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2365,7 +2367,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py index 87d4f2e9e882..9e875d08fb69 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object.py @@ -32,6 +32,8 @@ import warp as wp from flaky import flaky +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -55,7 +57,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _logger = logging.getLogger(__name__) @@ -176,7 +178,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -208,7 +210,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -244,7 +246,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -260,7 +262,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -275,7 +277,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -342,7 +344,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -418,7 +420,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -522,7 +524,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -586,7 +588,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" with _ovphysx_sim_context(device=device, gravity_enabled=True, auto_add_lighting=True) as sim: @@ -628,7 +630,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" raise NotImplementedError(_MATERIAL_GAP_REASON) @@ -636,7 +638,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" raise NotImplementedError(_MATERIAL_GAP_REASON) @@ -644,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" raise NotImplementedError(_MATERIAL_GAP_REASON) @@ -652,7 +654,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -666,7 +668,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -679,7 +681,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" with _ovphysx_sim_context( @@ -722,7 +724,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -760,7 +762,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @flaky(max_runs=3, min_passes=1) def test_body_root_state_properties(num_cubes, device, with_offset): @@ -876,7 +878,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) def test_write_root_state(num_cubes, device, with_offset, state_location): @@ -947,7 +949,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) def test_write_state_functions_data_consistency(num_cubes, device, with_offset, state_location): diff --git a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py index 4dae9628b6b3..0839efa6802f 100644 --- a/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_ovphysx/test/assets/test_rigid_object_collection.py @@ -28,6 +28,8 @@ import torch import warp as wp +from isaaclab.test.utils import test_devices + # The OVPhysX runtime wheel is optional. Skip gracefully when it is not installed; # CI jobs that need OVPhysX coverage install it explicitly. pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") @@ -52,7 +54,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated _LOCKED_DEVICE: list[str | None] = [None] @@ -172,7 +174,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -200,7 +202,7 @@ def test_initialization(num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(device): """Test environment and object index conversion to physics view indices.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -239,7 +241,7 @@ def test_id_conversion(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -274,7 +276,7 @@ def test_initialization_with_kinematic_enabled(num_envs, num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -288,7 +290,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -343,7 +345,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(num_envs, num_cubes, device): """Test application of external force on the base of the object.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -404,7 +406,7 @@ def test_external_force_on_single_body(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -486,7 +488,7 @@ def test_external_force_on_single_body_at_position(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -556,7 +558,7 @@ def test_set_object_state(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -655,7 +657,7 @@ def test_object_state_properties(num_envs, num_cubes, device, with_offset, gravi @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -733,7 +735,7 @@ def test_write_object_state(num_envs, num_cubes, device, with_offset, state_loca @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" with _ovphysx_sim_context(device=device, auto_add_lighting=True) as sim: @@ -768,7 +770,7 @@ def test_reset_object_collection(num_envs, num_cubes, device): @pytest.mark.xfail(reason=_MATERIAL_GAP_REASON, strict=False) @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" raise NotImplementedError(_MATERIAL_GAP_REASON) @@ -776,7 +778,7 @@ def test_set_material_properties(num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -810,7 +812,7 @@ def test_gravity_vec_w(num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py index eae32ddb82c4..72d9c546c71f 100644 --- a/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py +++ b/source/isaaclab_ovphysx/test/sensors/test_contact_sensor.py @@ -63,7 +63,7 @@ wp.init() -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated # --------------------------------------------------------------------------- # Device-lock autouse fixture diff --git a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py index e606ce7738b9..1b80ace67583 100644 --- a/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py +++ b/source/isaaclab_ovphysx/test/sim/test_views_xform_prim_ovphysx.py @@ -24,7 +24,7 @@ OVPHYSX_SIM_CFG = SimulationCfg(physics=OvPhysxCfg()) -pytestmark = pytest.mark.device_split +pytestmark = pytest.mark.device_isolated @pytest.mark.parametrize("device", ["cpu", "cuda:0"]) diff --git a/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip b/source/isaaclab_physx/changelog.d/jichuanh-multi-gpu-ci.skip new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_physx/test/assets/test_articulation.py b/source/isaaclab_physx/test/assets/test_articulation.py index af36a365cb66..b13e10d539bf 100644 --- a/source/isaaclab_physx/test/assets/test_articulation.py +++ b/source/isaaclab_physx/test/assets/test_articulation.py @@ -9,6 +9,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices HEADLESS = True @@ -310,7 +311,7 @@ def sim(request): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_non_root(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on a rigid body. @@ -366,7 +367,7 @@ def test_initialization_floating_base_non_root(sim, num_articulations, device, a @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base with articulation root on provided prim path. @@ -423,7 +424,7 @@ def test_initialization_floating_base(sim, num_articulations, device, add_ground @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_fixed_base(sim, num_articulations, device): """Test initialization for fixed base. @@ -487,7 +488,7 @@ def test_initialization_fixed_base(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_single_joint(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base articulation with a single joint. @@ -552,7 +553,7 @@ def test_initialization_fixed_base_single_joint(sim, num_articulations, device, @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_hand_with_tendons(sim, num_articulations, device): """Test initialization for fixed base articulated hand with tendons. @@ -605,7 +606,7 @@ def test_initialization_hand_with_tendons(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_floating_base_made_fixed_base(sim, num_articulations, device, add_ground_plane): """Test initialization for a floating-base articulation made fixed-base using schema properties. @@ -665,7 +666,7 @@ def test_initialization_floating_base_made_fixed_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_initialization_fixed_base_made_floating_base(sim, num_articulations, device, add_ground_plane): """Test initialization for fixed base made floating-base using schema properties. @@ -717,7 +718,7 @@ def test_initialization_fixed_base_made_floating_base(sim, num_articulations, de @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_ground_plane): """Test that the default joint position from configuration is out of range. @@ -747,7 +748,7 @@ def test_out_of_range_default_joint_pos(sim, num_articulations, device, add_grou sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_out_of_range_default_joint_vel(sim, device): """Test that the default joint velocity from configuration is out of range. @@ -771,7 +772,7 @@ def test_out_of_range_default_joint_vel(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): """Test write_joint_limits_to_sim API and when default pos falls outside of the new limits. @@ -846,7 +847,7 @@ def test_joint_pos_limits(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_joint_effort_limits(sim, num_articulations, device, add_ground_plane): """Validate joint effort limits via joint_effort_out_of_limit().""" @@ -879,7 +880,7 @@ def __init__(self, art): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, num_articulations, device): """Test if external force buffer correctly updates in the force value is zero case. @@ -963,7 +964,7 @@ def test_external_force_buffer(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_articulations, device): """Test application of external force on the base of the articulation. @@ -1020,7 +1021,7 @@ def test_external_force_on_single_body(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_articulations, device): """Test application of external force on the base of the articulation at a given position. @@ -1114,7 +1115,7 @@ def test_external_force_on_single_body_at_position(sim, num_articulations, devic @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies(sim, num_articulations, device): """Test application of external force on the legs of the articulation. @@ -1173,7 +1174,7 @@ def test_external_force_on_multiple_bodies(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, device): """Test application of external force on the legs of the articulation at a given position. @@ -1266,7 +1267,7 @@ def test_external_force_on_multiple_bodies_at_position(sim, num_articulations, d @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_loading_gains_from_usd(sim, num_articulations, device): """Test that gains are loaded from USD file if actuator model has them as None. @@ -1327,7 +1328,7 @@ def test_loading_gains_from_usd(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane): """Test that gains are loaded from the configuration correctly. @@ -1361,7 +1362,7 @@ def test_setting_gains_from_cfg(sim, num_articulations, device, add_ground_plane @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_gains_from_cfg_dict(sim, num_articulations, device): """Test that gains are loaded from the configuration dictionary correctly. @@ -1393,7 +1394,7 @@ def test_setting_gains_from_cfg_dict(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) @pytest.mark.parametrize("add_ground_plane", [False]) @@ -1460,7 +1461,7 @@ def test_setting_velocity_limit_implicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("vel_limit_sim", [1e5, None]) @pytest.mark.parametrize("vel_limit", [1e2, None]) def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_limit_sim, vel_limit): @@ -1513,7 +1514,7 @@ def test_setting_velocity_limit_explicit(sim, num_articulations, device, vel_lim @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [1e2, 80.0, None]) def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1565,7 +1566,7 @@ def test_setting_effort_limit_implicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("effort_limit_sim", [1e5, None]) @pytest.mark.parametrize("effort_limit", [80.0, 1e2, None]) def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_limit_sim, effort_limit): @@ -1626,7 +1627,7 @@ def test_setting_effort_limit_explicit(sim, num_articulations, device, effort_li @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset(sim, num_articulations, device): """Test that reset method works properly.""" articulation_cfg = generate_articulation_cfg(articulation_type="humanoid") @@ -1669,7 +1670,7 @@ def test_reset(sim, num_articulations, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("add_ground_plane", [True]) def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" @@ -1708,7 +1709,7 @@ def test_apply_joint_command(sim, num_articulations, device, add_ground_plane): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) def test_body_root_state(sim, num_articulations, device, with_offset): """Test for reading the `body_state_w` property. @@ -1831,7 +1832,7 @@ def test_body_root_state(sim, num_articulations, device, with_offset): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -1912,7 +1913,7 @@ def test_write_root_state(sim, num_articulations, device, with_offset, state_loc torch.testing.assert_close(rand_state[..., 7:], articulation.data.root_link_vel_w.torch) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1930,7 +1931,7 @@ def test_setting_articulation_root_prim_path(sim, device): assert articulation._is_initialized -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_setting_invalid_articulation_root_prim_path(sim, device): """Test that the articulation root prim path can be set explicitly.""" sim._app_control_on_stop_handle = None @@ -1948,7 +1949,7 @@ def test_setting_invalid_articulation_root_prim_path(sim, device): @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_write_joint_state_data_consistency(sim, num_articulations, device, gravity_enabled): """Test the setters for root_state using both the link frame and center of mass as reference frame. @@ -2053,7 +2054,7 @@ def test_write_joint_state_data_consistency(sim, num_articulations, device, grav @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_spatial_tendons(sim, num_articulations, device): """Test spatial tendons apis. This test verifies that: @@ -2105,7 +2106,7 @@ def test_spatial_tendons(sim, num_articulations, device): @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground_plane): """Test applying of joint position target functions correctly for a robotic arm.""" articulation_cfg = generate_articulation_cfg(articulation_type="panda") @@ -2198,7 +2199,7 @@ def test_write_joint_frictions_to_sim(sim, num_articulations, device, add_ground @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("num_articulations", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("articulation_type", ["panda"]) def test_set_material_properties(sim, num_articulations, device, add_ground_plane, articulation_type): """Test getting and setting material properties (friction/restitution) of articulation shapes.""" @@ -2245,7 +2246,7 @@ def test_set_material_properties(sim, num_articulations, device, add_ground_plan @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articulation_type): @@ -2262,7 +2263,7 @@ def test_get_jacobians_shape_fixed_base(sim, num_articulations, device, articula @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations, device, articulation_type): @@ -2289,7 +2290,7 @@ def test_get_mass_matrix_shape_and_nonsingular_fixed_base(sim, num_articulations @pytest.mark.parametrize("num_articulations", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("add_ground_plane", [True]) @pytest.mark.parametrize("articulation_type", ["anymal"]) @pytest.mark.isaacsim_ci @@ -2314,7 +2315,7 @@ def test_get_jacobians_shape_floating_base(sim, num_articulations, device, add_g @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2366,7 +2367,7 @@ def test_get_jacobians_link_origin_contract(sim, num_articulations, device, arti @pytest.mark.parametrize("num_articulations", [4]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2402,7 +2403,7 @@ def test_get_mass_matrix_symmetry_pd(sim, num_articulations, device, articulatio @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2453,7 +2454,7 @@ def test_jacobian_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda", "anymal"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2487,7 +2488,7 @@ def test_mass_matrix_refreshes_after_manual_joint_write( @pytest.mark.parametrize("num_articulations", [1]) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.isaacsim_ci def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulations, device, articulation_type): @@ -2576,7 +2577,7 @@ def test_get_gravity_compensation_forces_static_equilibrium(sim, num_articulatio ) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2634,7 +2635,7 @@ def test_franka_ik_tracking_accuracy(sim, device, articulation_type, gravity_ena assert rot_mean < 5e-2, f"IK rot_mean {rot_mean:.5f} > 0.05 rad — bridge regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [False]) @pytest.mark.isaacsim_ci @@ -2798,7 +2799,7 @@ def _run_osc_stay_still_under_gravity( return _summarize_history(pos_history), _summarize_history(rot_history) -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci @@ -2832,7 +2833,7 @@ def test_franka_osc_gravity_compensation_holds_under_gravity(sim, device, articu assert rot_mean < 5e-2, f"OSC + gravity_compensation rot_mean {rot_mean:.5f} > 0.05 rad — regression?" -@pytest.mark.parametrize("device", ["cuda:0"]) +@pytest.mark.parametrize("device", test_devices("01X")) @pytest.mark.parametrize("articulation_type", ["panda"]) @pytest.mark.parametrize("gravity_enabled", [True]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object.py b/source/isaaclab_physx/test/assets/test_rigid_object.py index b7f422c4f2f0..f55761b4bdad 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -98,7 +99,7 @@ def generate_cubes_scene( @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization(num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" @@ -132,7 +133,7 @@ def test_initialization(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_kinematic_enabled(num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" @@ -170,7 +171,7 @@ def test_initialization_with_kinematic_enabled(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_no_rigid_body(num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" @@ -188,7 +189,7 @@ def test_initialization_with_no_rigid_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_initialization_with_articulation_root(num_cubes, device): """Test that initialization fails when an articulation root is found at the provided prim path.""" @@ -205,7 +206,7 @@ def test_initialization_with_articulation_root(num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_buffer(device): """Test if external force buffer correctly updates in the force value is zero case. @@ -274,7 +275,7 @@ def test_external_force_buffer(device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_external_force_on_single_body(num_cubes, device): """Test application of external force on the base of the object. @@ -350,7 +351,7 @@ def test_external_force_on_single_body(num_cubes, device): @pytest.mark.parametrize("num_cubes", [2, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -455,7 +456,7 @@ def test_external_force_on_single_body_at_position(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_rigid_object_state(num_cubes, device): """Test setting the state of the rigid object. @@ -521,7 +522,7 @@ def test_set_rigid_object_state(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_reset_rigid_object(num_cubes, device): """Test resetting the state of the rigid object.""" @@ -564,7 +565,7 @@ def test_reset_rigid_object(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_material_properties(num_cubes, device): """Test getting and setting material properties of rigid object.""" @@ -605,7 +606,7 @@ def test_rigid_body_set_material_properties(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_set_material_properties_via_view(num_cubes, device): """Test setting material properties via the PhysX view-level API.""" @@ -645,7 +646,7 @@ def test_set_material_properties_via_view(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_no_friction(num_cubes, device): """Test that a rigid object with no friction will maintain it's velocity when sliding across a plane.""" @@ -694,7 +695,7 @@ def test_rigid_body_no_friction(num_cubes, device): cube_object.update(sim.cfg.dt) # Non-deterministic when on GPU, so we use different tolerances - if device == "cuda:0": + if device.startswith("cuda"): tolerance = 1e-2 else: tolerance = 1e-5 @@ -705,7 +706,7 @@ def test_rigid_body_no_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_static_friction(num_cubes, device): """Test that static friction applied to rigid object works as expected. @@ -791,7 +792,7 @@ def test_rigid_body_with_static_friction(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_with_restitution(num_cubes, device): """Test that restitution when applied to rigid object works as expected. @@ -874,7 +875,7 @@ def test_rigid_body_with_restitution(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.isaacsim_ci def test_rigid_body_set_mass(num_cubes, device): """Test getting and setting mass of rigid object.""" @@ -918,7 +919,7 @@ def test_rigid_body_set_mass(num_cubes, device): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) @pytest.mark.isaacsim_ci def test_gravity_vec_w(num_cubes, device, gravity_enabled): @@ -958,7 +959,7 @@ def test_gravity_vec_w(num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.isaacsim_ci @flaky(max_runs=3, min_passes=1) @@ -1069,7 +1070,7 @@ def test_body_root_state_properties(num_cubes, device, with_offset): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.isaacsim_ci @@ -1139,7 +1140,7 @@ def test_write_root_state(num_cubes, device, with_offset, state_location): @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py index f42f6dfcb085..4ec56828af82 100644 --- a/source/isaaclab_physx/test/assets/test_rigid_object_collection.py +++ b/source/isaaclab_physx/test/assets/test_rigid_object_collection.py @@ -10,6 +10,7 @@ """Launch Isaac Sim Simulator first.""" from isaaclab.app import AppLauncher +from isaaclab.test.utils import test_devices # launch omniverse app simulation_app = AppLauncher(headless=True).app @@ -110,7 +111,7 @@ def sim(request): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization(sim, num_envs, num_cubes, device): """Test initialization for prim with rigid body API at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -137,7 +138,7 @@ def test_initialization(sim, num_envs, num_cubes, device): object_collection.update(sim.cfg.dt) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_id_conversion(sim, device): """Test environment and object index conversion to physics view indices.""" object_collection, _ = generate_cubes_scene(num_envs=2, num_cubes=3, device=device) @@ -175,7 +176,7 @@ def test_id_conversion(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 3]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device): """Test that initialization for prim with kinematic flag enabled.""" object_collection, origins = generate_cubes_scene( @@ -209,7 +210,7 @@ def test_initialization_with_kinematic_enabled(sim, num_envs, num_cubes, device) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_initialization_with_no_rigid_body(sim, num_cubes, device): """Test that initialization fails when no rigid body is found at the provided prim path.""" object_collection, _ = generate_cubes_scene(num_cubes=num_cubes, has_api=False, device=device) @@ -222,7 +223,7 @@ def test_initialization_with_no_rigid_body(sim, num_cubes, device): sim.reset() -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_buffer(sim, device): """Test if external force buffer correctly updates in the force value is zero case.""" num_envs = 2 @@ -276,7 +277,7 @@ def test_external_force_buffer(sim, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object.""" object_collection, origins = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -336,7 +337,7 @@ def test_external_force_on_single_body(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 2]) @pytest.mark.parametrize("num_cubes", [1, 4]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, device): """Test application of external force on the base of the object at a specific position. @@ -415,7 +416,7 @@ def test_external_force_on_single_body_at_position(sim, num_envs, num_cubes, dev @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [False]) def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): """Test setting the state of the object. @@ -480,7 +481,7 @@ def test_set_object_state(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 4]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("gravity_enabled", [False]) def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, gravity_enabled): @@ -576,7 +577,7 @@ def test_object_state_properties(sim, num_envs, num_cubes, device, with_offset, @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True, False]) @pytest.mark.parametrize("state_location", ["com", "link"]) @pytest.mark.parametrize("gravity_enabled", [False]) @@ -656,7 +657,7 @@ def test_write_object_state(sim, num_envs, num_cubes, device, with_offset, state @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_reset_object_collection(sim, num_envs, num_cubes, device): """Test resetting the state of the rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -689,7 +690,7 @@ def test_reset_object_collection(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) def test_set_material_properties(sim, num_envs, num_cubes, device): """Test getting and setting material properties of rigid object.""" object_collection, _ = generate_cubes_scene(num_envs=num_envs, num_cubes=num_cubes, device=device) @@ -724,7 +725,7 @@ def test_set_material_properties(sim, num_envs, num_cubes, device): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("gravity_enabled", [True, False]) def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): """Test that gravity vector direction is set correctly for the rigid object.""" @@ -757,7 +758,7 @@ def test_gravity_vec_w(sim, num_envs, num_cubes, device, gravity_enabled): @pytest.mark.parametrize("num_envs", [1, 3]) @pytest.mark.parametrize("num_cubes", [1, 2]) -@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.parametrize("with_offset", [True]) @pytest.mark.parametrize("state_location", ["com", "link", "root"]) @pytest.mark.parametrize("gravity_enabled", [False]) diff --git a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py index 3cfe70095fd3..9a1c5e9e34ea 100644 --- a/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py +++ b/source/isaaclab_physx/test/sim/test_views_xform_prim_fabric.py @@ -10,7 +10,6 @@ Camera prim type for Fabric SelectPrims compatibility). """ -import os import sys from pathlib import Path @@ -30,6 +29,7 @@ from pxr import Gf, UsdGeom # noqa: E402 import isaaclab.sim as sim_utils # noqa: E402 +from isaaclab.test.utils import test_devices # noqa: E402 pytestmark = pytest.mark.isaacsim_ci PARENT_POS = (0.0, 0.0, 1.0) @@ -126,7 +126,7 @@ def factory(num_envs: int, device: str) -> ViewBundle: # ------------------------------------------------------------------ -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) @pytest.mark.xfail( reason=( "Issue #5: FabricFrameView.set_world_poses writes to Fabric worldMatrix only. " @@ -155,7 +155,7 @@ def _fill_position(out: wp.array(dtype=wp.float32, ndim=2), x: float, y: float, out[i, 2] = wp.float32(z) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): """Verify that set_world_poses in Fabric mode does NOT sync back to USD. @@ -199,7 +199,7 @@ def test_fabric_set_world_does_not_write_back_to_usd(device, view_factory): ) -@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +@pytest.mark.parametrize("device", test_devices()) def test_fabric_rebuild_after_topology_change(device, view_factory, monkeypatch): """Forcing the topology-changed branch on a write triggers :meth:`_rebuild_fabric_arrays` and leaves the view in a state where @@ -252,11 +252,7 @@ def force_topology_changed(): # ------------------------------------------------------------------ -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): """set_world_poses -> get_world_poses roundtrip works on cuda:1. @@ -276,11 +272,7 @@ def test_fabric_cuda1_world_pose_roundtrip(device, view_factory): assert torch.allclose(pos_torch, expected, atol=1e-7), f"Roundtrip failed on {device}: {pos_torch}" -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_no_usd_writeback(device, view_factory): """set_world_poses on cuda:1 does not write back to USD. @@ -308,11 +300,7 @@ def test_fabric_cuda1_no_usd_writeback(device, view_factory): ) -@pytest.mark.skipif( - not os.environ.get("ISAACLAB_TEST_MULTI_GPU"), - reason="Multi-GPU tests disabled (set ISAACLAB_TEST_MULTI_GPU=1 to enable)", -) -@pytest.mark.parametrize("device", ["cuda:1"]) +@pytest.mark.parametrize("device", test_devices("00X")) def test_fabric_cuda1_scales_roundtrip(device, view_factory): """set_scales -> get_scales roundtrip works on cuda:1. diff --git a/tools/_device_split.py b/tools/_device_split.py deleted file mode 100644 index e8bad4069133..000000000000 --- a/tools/_device_split.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Helpers for detecting and driving the ``device_split`` pytest marker. - -Test files that declare ``pytestmark = pytest.mark.device_split`` at module -scope must be re-invoked once per device (CPU and GPU) in separate processes -to work around process-global device locks such as ``ovphysx<=0.3.7`` gap G5. -The :func:`is_device_split_file` predicate lets the per-file CI runner in -``tools/conftest.py`` detect this without importing the test module. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -_DEVICE_SPLIT_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_split\b", re.MULTILINE) -"""Match a module-level ``pytestmark`` assignment that mentions ``device_split``. - -Recognises both single-mark and single-line list forms: - -* ``pytestmark = pytest.mark.device_split`` -* ``pytestmark = [pytest.mark.device_split, pytest.mark.foo]`` - -Multi-line list forms are not supported (currently no test file uses one); if -a future test needs that, expand the parsing rule. -""" - -# Per-pass pytest ``-k`` selectors used by ``tools/conftest.py`` when a file -# declares the ``device_split`` marker. Each entry is ``(suffix, k_expr)``: -# - ``suffix`` is appended to the JUnit report filename to keep both passes' XML. -# - ``k_expr`` is the ``-k`` keyword expression. ``"cpu or not cuda"`` keeps -# non-parametrized tests in the CPU pass; ``"cuda"`` catches GPU-parametrized -# tests only. -DEVICE_SPLIT_PASSES: list[tuple[str, str]] = [ - ("-cpu", "cpu or not cuda"), - ("-cuda", "cuda"), -] - - -def is_device_split_file(path: Path | str, source: str | None = None) -> bool: - """Return whether the test file at ``path`` declares the ``device_split`` marker. - - Matches :data:`_DEVICE_SPLIT_MARK_RE` against ``source`` when supplied. - Otherwise, reads the file source from ``path``. A missing or unreadable - file returns ``False`` so the caller falls back to the default single-pass - invocation. - - Args: - path: Filesystem path to a candidate test file. - source: Optional preloaded source text to inspect. - - Returns: - ``True`` when the file's module-level ``pytestmark`` mentions - ``device_split``; ``False`` otherwise. - """ - if source is None: - try: - source = Path(path).read_text(encoding="utf-8", errors="replace") - except OSError: - return False - return bool(_DEVICE_SPLIT_MARK_RE.search(source)) diff --git a/tools/conftest.py b/tools/conftest.py index a45f8cd9183c..dedc9f8b3087 100644 --- a/tools/conftest.py +++ b/tools/conftest.py @@ -3,1047 +3,55 @@ # # SPDX-License-Identifier: BSD-3-Clause -import contextlib -import os -import select -import signal -import subprocess -import sys -import time -from dataclasses import dataclass - -import pytest -from junitparser import Error, JUnitXml, TestCase, TestSuite -from prettytable import PrettyTable - -# Local imports -import test_settings as test_settings # isort: skip -from _device_split import DEVICE_SPLIT_PASSES, is_device_split_file # isort: skip - - -def pytest_ignore_collect(collection_path, config): - # Skip collection and run each test script individually - return True - - -COLD_CACHE_BUFFER = 700 -"""Extra seconds added to the first camera-enabled test's hard timeout. +"""CI entry point for the per-file test runner. -The first test that uses ``enable_cameras=True`` may compile shaders during its -run (~600 s). This buffer prevents that from being misreported as a test -timeout. Only the first such test gets the extension — after it runs, the -on-disk cache is populated. +This conftest does not run tests in-process. It hands the whole run to the +:mod:`test_runner` framework, which executes each test file in its own pytest +subprocess (so a crash or device lock in one file cannot affect the next). All +behavior is in the framework; this file is just the two pytest hooks that wire +it in. """ -STARTUP_DEADLINE = 120 -"""Seconds to wait for AppLauncher init or pytest collection before declaring a -startup hang. - -AppLauncher prints ``[ISAACLAB] AppLauncher initialization complete`` to -``sys.__stderr__`` (never suppressed) when Kit finishes initializing, and pytest -prints ``collected N items`` to stdout after collection. If neither appears -within this deadline the process is treated as hung. Kit startup can exceed -60 s on cold CI workers, so this catches real startup hangs without killing -legitimate slow launches. -""" - -STARTUP_HANG_RETRIES = 2 -"""Number of times to retry a test that hangs during startup before giving up.""" - -TIMEOUT_RETRIES = 0 -"""Number of times to retry a test that reaches its hard timeout before giving up.""" - -PROCESS_FAILURE_RETRIES_BY_FILE = { - "test_visualizer_integration_physx.py": 4, - "test_visualizer_integration_newton.py": 4, - "test_visualizer_tiled_integration_physx.py": 4, - "test_visualizer_tiled_integration_newton.py": 4, -} -"""Extra fresh-process attempts for visualizer tests that can enter stale render states.""" - -SHUTDOWN_GRACE_PERIOD = 30 -"""Seconds to wait for clean exit after the JUnit XML report file appears. +import os -When a test completes and writes its JUnit report, the subprocess may hang -during ``SimulationApp.close()`` or Kit shutdown. Rather than wasting the -full hard timeout, we give the process a short grace period to exit, then -kill it. The test results are taken from the report file (pass/fail), not -from the kill. -""" +import pytest +from test_runner.planning import RunnerConfig +from test_runner.session import Session -def capture_test_output_with_timeout(cmd, timeout, env, startup_deadline=0, report_file=""): - """Run a command with timeout and capture all output while streaming in real-time. +def pytest_ignore_collect(collection_path, config): + """PYTEST HOOK — skip pytest's own collection; the runner drives files itself. Args: - cmd: Command to execute. - timeout: Maximum wall-clock seconds before the process is killed. - env: Environment variables for the subprocess. - startup_deadline: If > 0, the process is killed early when neither - ``AppLauncher initialization complete`` (stderr) nor ``collected`` - (stdout) appears within this many seconds. - report_file: Path to the JUnit XML report file. When set, the process - is given only :data:`SHUTDOWN_GRACE_PERIOD` seconds to exit after - the file appears on disk. + collection_path: candidate path pytest is about to collect (unused). + config: the session ``pytest.Config`` (unused). Returns: - Tuple of ``(returncode, stdout_bytes, stderr_bytes, kill_reason, - wall_time, pre_kill_diag)``. *kill_reason* is ``""`` for normal exits, - ``"timeout"`` for hard timeouts, ``"startup_hang"`` when the process - did not reach pytest collection in time, or ``"shutdown_hang"`` when - the test completed but the process hung during shutdown. - """ - stdout_data = b"" - stderr_data = b"" - process = None - - try: - # Each test gets its own session so orphaned Kit/Isaac Sim child - # processes cannot send SIGHUP to the next test's process group. - process = subprocess.Popen( - cmd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0, - universal_newlines=False, - start_new_session=True, - ) - pgid = os.getpgid(process.pid) - - stdout_fd = process.stdout.fileno() - stderr_fd = process.stderr.fileno() - - try: - import fcntl - - for fd in [stdout_fd, stderr_fd]: - flags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) - except ImportError: - pass - - start_time = time.time() - startup_done = startup_deadline <= 0 - shutdown_deadline = 0.0 - - while process.poll() is None: - elapsed = time.time() - start_time - - if not startup_done: - if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: - startup_done = True - - if report_file and not shutdown_deadline and os.path.exists(report_file): - shutdown_deadline = time.time() + SHUTDOWN_GRACE_PERIOD - - kill_reason = None - if not startup_done and elapsed > startup_deadline: - kill_reason = "startup_hang" - elif shutdown_deadline and time.time() > shutdown_deadline: - kill_reason = "shutdown_hang" - elif elapsed > timeout: - kill_reason = "timeout" - - if kill_reason: - pre_kill_diag = _capture_system_diagnostics() - - # Kill the entire process group (test + any Kit children). - try: - os.killpg(pgid, signal.SIGKILL) - except (ProcessLookupError, PermissionError, OSError): - process.kill() - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=5) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except subprocess.TimeoutExpired: - pass - wall_time = time.time() - start_time - return -1, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag - - try: - ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) - - for fd in ready_fds: - with contextlib.suppress(OSError): - if fd == stdout_fd: - chunk = process.stdout.read(1024) - if chunk: - stdout_data += chunk - sys.stdout.buffer.write(chunk) - sys.stdout.buffer.flush() - elif fd == stderr_fd: - chunk = process.stderr.read(1024) - if chunk: - stderr_data += chunk - sys.stderr.buffer.write(chunk) - sys.stderr.buffer.flush() - except OSError: - time.sleep(0.1) - continue - - # Drain any output the process wrote before or just after exiting. - try: - remaining_stdout, remaining_stderr = process.communicate(timeout=10) - stdout_data += remaining_stdout - stderr_data += remaining_stderr - except Exception: - pass - - # Kill any orphaned child processes (Kit, Isaac Sim) left by the test. - try: - os.killpg(pgid, signal.SIGKILL) - time.sleep(1) - except (ProcessLookupError, PermissionError, OSError): - pass - - wall_time = time.time() - start_time - return process.returncode, stdout_data, stderr_data, "", wall_time, "" - - except Exception as e: - if process is not None and process.poll() is None: - process.kill() - with contextlib.suppress(Exception): - rem_out, rem_err = process.communicate(timeout=5) - stdout_data += rem_out - stderr_data += rem_err - stdout_data += f"\n[capture error: {e}]\n".encode() - return -1, stdout_data, stderr_data, "", 0.0, "" - - -_SIGNAL_DESCRIPTIONS = { - 1: "SIGHUP — session leader exit or orphaned process cleanup", - 6: "SIGABRT", - 9: "SIGKILL — likely OOM killed", - 11: "SIGSEGV — segmentation fault", - 15: "SIGTERM", -} - - -def _signal_description(sig): - """Return a human-readable description for a process killed by a signal.""" - base = f"Process killed by signal {sig}" - desc = _SIGNAL_DESCRIPTIONS.get(sig) - return f"{base} ({desc})" if desc else base - - -def _create_error_report(prefix, file_name, message, details): - """Create a JUnit XML error report for a test that failed to produce its own. - - Returns a :class:`JUnitXml` object ready to be written to disk. - """ - suite_name = os.path.splitext(file_name)[0] - suite = TestSuite(name=f"{prefix}_{suite_name}") - case = TestCase(name="test_execution", classname=suite_name) - error = Error(message=message) - error.text = details - case.result = error - suite.add_testcase(case) - report = JUnitXml() - report.add_testsuite(suite) - return report - - -def _get_diagnostics(pre_kill_diag=""): - """Return system diagnostics, truncated to 10 000 chars.""" - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - return diag - - -def _capture_system_diagnostics(): - """Capture system diagnostics (GPU, memory, processes) for crash investigation. - - All errors are caught and reported inline so this never raises. + ``True`` always, so nothing is collected in this process; the per-file + subprocesses launched by :class:`~test_runner.session.Session` do the + real collection. """ - sections = [] - - try: - r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) - if r.stdout: - sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- nvidia-smi --- FAILED: {e}") - - try: - with open("/proc/meminfo") as f: - lines = f.readlines() - keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") - relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] - if relevant: - sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) - except Exception as e: - sections.append(f"--- /proc/meminfo --- FAILED: {e}") - - cgroup_lines = [] - for path in ( - "/sys/fs/cgroup/memory.current", - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory.events", - "/sys/fs/cgroup/memory/memory.usage_in_bytes", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - "/sys/fs/cgroup/memory/memory.oom_control", - ): - try: - with open(path) as f: - cgroup_lines.append(f"{path}: {f.read().strip()}") - except FileNotFoundError: - pass - except Exception as e: - cgroup_lines.append(f"{path}: FAILED ({e})") - if cgroup_lines: - sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) - - try: - r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) - if r.stdout: - sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") - except Exception as e: - sections.append(f"--- process tree --- FAILED: {e}") - - try: - r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) - if r.stdout: - lines = r.stdout.strip().split("\n") - sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) - except Exception: - pass - - return "\n\n".join(sections) - - -def _read_test_report(report_file, file_name): - """Read a pytest JUnit report and return its summary fields.""" - report = JUnitXml.fromfile(report_file) - for suite in report: - if suite.name == "pytest": - suite.name = os.path.splitext(file_name)[0] - report.write(report_file) - - errors = int(report.errors) if report.errors is not None else 0 - failures = int(report.failures) if report.failures is not None else 0 - skipped = int(report.skipped) if report.skipped is not None else 0 - tests = int(report.tests) if report.tests is not None else 0 - time_elapsed = float(report.time) if report.time is not None else 0.0 - return report, errors, failures, skipped, tests, time_elapsed - - -def _retry_failed_test_in_fresh_process( - *, - test_file, - file_name, - cmd, - timeout, - env, - startup_deadline, - report_file, - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, -): - """Retry selected failed test files in a fresh subprocess.""" - has_test_failures = errors > 0 or failures > 0 - process_failure_attempts = 0 - max_process_failure_retries = PROCESS_FAILURE_RETRIES_BY_FILE.get(file_name, 0) - - while has_test_failures and process_failure_attempts < max_process_failure_retries: - process_failure_attempts += 1 - print( - f"⚠️ {test_file}: failed in subprocess" - f" (attempt {process_failure_attempts}/{max_process_failure_retries + 1}), retrying in fresh process..." - ) - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, timeout, env, startup_deadline=startup_deadline, report_file=report_file - ) - if not os.path.exists(report_file): - break - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, file_name) - has_test_failures = errors > 0 or failures > 0 - except Exception as e: - print(f"Error reading retry test report {report_file}: {e}") - has_test_failures = True - errors = 1 - failures = 0 - skipped = 0 - tests = 0 - time_elapsed = 0.0 - break - - return ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) - - -@dataclass -class _PassContext: - """Inputs shared across all pytest invocations for a single test file. - - Attributes: - test_file: Absolute path to the test file being driven. - file_name: Basename of ``test_file`` (used for JUnit naming). - workspace_root: Repository root; passed to pytest's ``--config-file``. - isaacsim_ci: Whether ``ISAACSIM_CI_SHORT`` is active; toggles the - ``-m isaacsim_ci`` selector. - timeout: Per-pass hard timeout in seconds. - startup_deadline: Per-pass startup-hang deadline in seconds. - env: Environment passed to the pytest subprocess. - """ - - test_file: str - file_name: str - workspace_root: str - isaacsim_ci: bool - timeout: int - startup_deadline: int - env: dict - - -_RESULT_PRIORITY = { - "STARTUP_HANG": 5, - "CRASHED": 4, - "TIMEOUT": 3, - "FAILED": 2, - "passed (shutdown hanged)": 1, - "passed": 0, -} - - -def _merge_pass_status(prev: dict | None, new: dict) -> dict: - """Merge per-pass status dicts into a single per-file entry. + return True - Counters (``errors``, ``failures``, ``skipped``, ``tests``, - ``time_elapsed``, ``wall_time``) are summed. ``result`` becomes the more - severe of the two via :data:`_RESULT_PRIORITY`. - """ - if prev is None: - return new - return { - "errors": prev["errors"] + new["errors"], - "failures": prev["failures"] + new["failures"], - "skipped": prev["skipped"] + new["skipped"], - "tests": prev["tests"] + new["tests"], - "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], - "wall_time": prev["wall_time"] + new["wall_time"], - "result": prev["result"] - if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) - else new["result"], - } +def pytest_sessionstart(session): + """PYTEST HOOK — build the runner from the environment and run the lifecycle. -def _run_one_pass( - ctx: _PassContext, - k_expr: str | None, - suffix: str, -) -> tuple[JUnitXml | None, dict, bool]: - """Drive one pytest subprocess for ``ctx.test_file`` and return its results. + Called by pytest at session start. It exits pytest with the runner's code so + normal pytest does not run (and overwrite the aggregate report) afterward. Args: - ctx: Static per-file context (paths, timeouts, env). - k_expr: Optional ``-k`` selector. ``None`` means no selector (default - single-pass invocation). - suffix: Suffix appended to the JUnit report filename, e.g. ``"-cpu"`` - or ``""`` for the unsplit default. + session: the ``pytest.Session`` (only its startup timing is used). Returns: - A 3-tuple ``(xml_report, status_dict, was_failure)``: - * ``xml_report``: parsed JUnit XML, or ``None`` if the pass produced - no report (e.g. startup hang). - * ``status_dict``: per-pass counters compatible with the entries - currently appended to ``test_status``. - * ``was_failure``: whether the pass should add ``ctx.test_file`` to - the ``failed_tests`` list. + None — terminates the process via :func:`pytest.exit` with 0 when every + file passed, else 1. """ - pass_file_label = f"{ctx.file_name}{suffix}" - report_file = f"tests/test-reports-{pass_file_label}.xml" - - cmd = [ - sys.executable, - "-m", - "pytest", - "-s", - "--no-header", - f"--config-file={ctx.workspace_root}/pyproject.toml", - f"--junitxml={report_file}", - "--tb=short", - ] - if ctx.isaacsim_ci: - cmd += ["-m", "isaacsim_ci"] - if k_expr is not None: - cmd += ["-k", k_expr] - cmd.append(str(ctx.test_file)) - - # -- Run with retry on startup hang or hard timeout ----------------- - returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", "" - wall_time, pre_kill_diag = 0.0, "" - startup_hang_attempts = 0 - timeout_attempts = 0 - while True: - with contextlib.suppress(FileNotFoundError): - os.remove(report_file) - - returncode, stdout_data, stderr_data, kill_reason, wall_time, pre_kill_diag = capture_test_output_with_timeout( - cmd, ctx.timeout, ctx.env, startup_deadline=ctx.startup_deadline, report_file=report_file - ) - - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES: - startup_hang_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: startup hang detected after {ctx.startup_deadline}s" - f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..." - ) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - - if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES: - timeout_attempts += 1 - print( - f"⚠️ {ctx.test_file}{suffix}: timeout detected after {ctx.timeout}s" - f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..." - ) - if stdout_data: - print("=== STDOUT (last 5000 chars) ===") - print(stdout_data.decode("utf-8", errors="replace")[-5000:]) - if stderr_data: - print("=== STDERR (last 5000 chars) ===") - print(stderr_data.decode("utf-8", errors="replace")[-5000:]) - diag = pre_kill_diag or _capture_system_diagnostics() - if len(diag) > 10000: - diag = diag[:10000] + "\n... (truncated)" - print(diag) - continue - break - - # -- Resolve result from kill_reason and report file ---------------- - has_report = os.path.exists(report_file) - - if kill_reason == "startup_hang": - diag = _get_diagnostics(pre_kill_diag) - print(f"⚠️ {ctx.test_file}{suffix}: startup hang after {STARTUP_HANG_RETRIES + 1} attempt(s)") - print(diag) - - msg = f"Startup hang after {ctx.startup_deadline}s (retried {STARTUP_HANG_RETRIES} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("startup_hang", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "STARTUP_HANG", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - if kill_reason == "timeout" and not has_report: - diag = _get_diagnostics(pre_kill_diag) - print(f"Test {ctx.test_file}{suffix} timed out after {ctx.timeout} seconds...") - print(diag) - - msg = f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))" - details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 5000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-5000:] + "\n" - if stderr_data: - details += "=== STDERR (last 5000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-5000:] + "\n" - - error_report = _create_error_report("timeout", pass_file_label, msg, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "TIMEOUT", - "time_elapsed": ctx.timeout, - "wall_time": wall_time, - }, - True, - ) - - if not has_report: - reason = ( - _signal_description(-returncode) - if returncode < 0 - else f"Process exited with code {returncode} but produced no report" - ) - diag = _get_diagnostics() - print(f"⚠️ {ctx.test_file}{suffix}: {reason}") - print(diag) - - details = f"{reason}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" - if stdout_data: - details += "=== STDOUT (last 2000 chars) ===\n" - details += stdout_data.decode("utf-8", errors="replace")[-2000:] + "\n" - if stderr_data: - details += "=== STDERR (last 2000 chars) ===\n" - details += stderr_data.decode("utf-8", errors="replace")[-2000:] + "\n" - - error_report = _create_error_report("crash", pass_file_label, reason, details) - error_report.write(report_file) - return ( - error_report, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 1, - "result": "CRASHED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - # -- Report file exists: parse actual test results ----------------- - if kill_reason in ("shutdown_hang", "timeout"): - print(f"⚠️ {ctx.test_file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test had completed)") - - try: - report, errors, failures, skipped, tests, time_elapsed = _read_test_report(report_file, pass_file_label) - except Exception as e: - print(f"Error reading test report {report_file}: {e}") - return ( - None, - { - "errors": 1, - "failures": 0, - "skipped": 0, - "tests": 0, - "result": "FAILED", - "time_elapsed": 0.0, - "wall_time": wall_time, - }, - True, - ) - - ( - report, - errors, - failures, - skipped, - tests, - time_elapsed, - returncode, - stdout_data, - stderr_data, - kill_reason, - wall_time, - pre_kill_diag, - has_test_failures, - ) = _retry_failed_test_in_fresh_process( - test_file=ctx.test_file, - file_name=ctx.file_name, - cmd=cmd, - timeout=ctx.timeout, - env=ctx.env, - startup_deadline=ctx.startup_deadline, - report_file=report_file, - report=report, - errors=errors, - failures=failures, - skipped=skipped, - tests=tests, - time_elapsed=time_elapsed, - returncode=returncode, - stdout_data=stdout_data, - stderr_data=stderr_data, - kill_reason=kill_reason, - wall_time=wall_time, - pre_kill_diag=pre_kill_diag, - ) - - shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_test_failures - was_failure = has_test_failures or (returncode != 0 and not shutdown_hanged) - - if shutdown_hanged: - result = "passed (shutdown hanged)" - elif has_test_failures: - result = "FAILED" - else: - result = "passed" - - return ( - report, - { - "errors": errors, - "failures": failures, - "skipped": skipped, - "tests": tests, - "result": result, - "time_elapsed": time_elapsed, - "wall_time": wall_time, - }, - was_failure, - ) - - -def run_individual_tests(test_files, workspace_root, isaacsim_ci): - """Run each test file separately, ensuring one finishes before starting the next.""" - failed_tests = [] - test_status = {} - xml_reports = [] - cold_cache_applied = False - - for test_file in test_files: - print(f"\n\n🚀 Running {test_file} independently...\n") - file_name = os.path.basename(test_file) - env = os.environ.copy() - env["PYTHONFAULTHANDLER"] = "1" - - timeout = test_settings.PER_TEST_TIMEOUTS.get(file_name, test_settings.DEFAULT_TIMEOUT) - - # Read the test file once for cold-cache and device-split detection. - try: - with open(test_file) as fh: - test_content = fh.read() - except OSError: - test_content = "" - - # The first camera-enabled test in a fresh container compiles shaders - # (~600 s). Give it extra time so that doesn't look like a test timeout. - is_cold_cache_test = not cold_cache_applied and "enable_cameras=True" in test_content - if is_cold_cache_test: - timeout += COLD_CACHE_BUFFER - cold_cache_applied = True - print(f"⏱️ Adding {COLD_CACHE_BUFFER}s cold-cache buffer (timeout now {timeout}s)") - - extra = COLD_CACHE_BUFFER if is_cold_cache_test else 0 - startup_deadline = min(timeout, STARTUP_DEADLINE + extra) - - ctx = _PassContext( - test_file=test_file, - file_name=file_name, - workspace_root=workspace_root, - isaacsim_ci=isaacsim_ci, - timeout=timeout, - startup_deadline=startup_deadline, - env=env, - ) - - if is_device_split_file(test_file, source=test_content): - print(f"⚙️ device_split detected — invoking {file_name} once per device (CPU then GPU)") - passes = DEVICE_SPLIT_PASSES - else: - passes = [("", None)] - - merged_status: dict | None = None - for suffix, k_expr in passes: - report, status, was_failure = _run_one_pass(ctx, k_expr=k_expr, suffix=suffix) - if report is not None: - xml_reports.append(report) - if was_failure and test_file not in failed_tests: - failed_tests.append(test_file) - merged_status = _merge_pass_status(merged_status, status) - - assert merged_status is not None # the pass list is never empty - test_status[test_file] = merged_status - - print("~~~~~~~~~~~~ Finished running all tests") - - return failed_tests, test_status, xml_reports - - -def _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, -): - """Collect test files from source directories, applying all active filters.""" - test_files = [] - for source_dir in source_dirs: - if not os.path.exists(source_dir): - print(f"Error: source directory not found at {source_dir}") - pytest.exit("Source directory not found", returncode=1) - - for root, _, files in os.walk(source_dir): - # source/isaaclab/test/install_ci/ has its own pytest config and conftest. - # It is run via .github/actions/install-ci-run, never via this collector, - # so skip the whole subtree to keep install_ci tests out of build.yaml jobs. - if "install_ci" in root.replace("\\", "/").split("/"): - continue - - for file in files: - if not (file.startswith("test_") and file.endswith(".py")): - continue - - # Mode-exclusive filters (each bypasses TESTS_TO_SKIP) - if quarantined_only: - if file not in test_settings.QUARANTINED_TESTS: - continue - elif curobo_only: - if file not in test_settings.CUROBO_TESTS: - continue - else: - # An explicit include_files entry overrides TESTS_TO_SKIP, allowing - # dedicated jobs (e.g. test-environments-training) to run tests that - # are otherwise excluded from general CI runs. - if file in test_settings.TESTS_TO_SKIP and file not in include_files: - print(f"Skipping {file} as it's in the skip list") - continue - - full_path = os.path.join(root, file) - - if filter_pattern and filter_pattern not in full_path: - print(f"Skipping {full_path} (does not match include pattern: {filter_pattern})") - continue - if exclude_pattern and any(p.strip() in full_path for p in exclude_pattern.split(",")): - print(f"Skipping {full_path} (matches exclude pattern: {exclude_pattern})") - continue - if include_files and file not in include_files: - print(f"Skipping {full_path} (not in include files list)") - continue - - test_files.append(full_path) - - # Apply file-level sharding: sort deterministically, then select every Nth file. - # Skip when include_files is set — in that case the test's own conftest handles - # sharding at the test-item level (e.g. parametrized test cases). - shard_index = os.environ.get("TEST_SHARD_INDEX", "") - shard_count = os.environ.get("TEST_SHARD_COUNT", "") - if shard_index and shard_count and not include_files: - shard_index = int(shard_index) - shard_count = int(shard_count) - test_files.sort() - test_files = [f for i, f in enumerate(test_files) if i % shard_count == shard_index] - print(f"Shard {shard_index}/{shard_count}: selected {len(test_files)} test files") - - return test_files - - -def _write_empty_report(): - """Write an empty JUnit XML report so downstream CI steps find a valid file.""" - os.makedirs("tests", exist_ok=True) - result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") - report = JUnitXml() - report.write(f"tests/{result_file}") - print(f"Wrote empty report to tests/{result_file}") - - -def pytest_sessionstart(session): - """Intercept pytest startup to execute tests in the correct order.""" - # Get the workspace root directory (one level up from tools) workspace_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - source_dirs = [ - os.path.join(workspace_root, "scripts"), - os.path.join(workspace_root, "source"), - ] - - # Get filter pattern from environment variable or command line - filter_pattern = os.environ.get("TEST_FILTER_PATTERN", "") - exclude_pattern = os.environ.get("TEST_EXCLUDE_PATTERN", "") - include_files_str = os.environ.get("TEST_INCLUDE_FILES", "") - quarantined_only = os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true" - curobo_only = os.environ.get("TEST_CUROBO_ONLY", "false") == "true" - - isaacsim_ci = os.environ.get("ISAACSIM_CI_SHORT", "false") == "true" - - # Parse include files list (comma-separated paths) - include_files = set() - if include_files_str: - for f in include_files_str.split(","): - f = f.strip() - if f: - include_files.add(os.path.basename(f)) - - # Also try to get from pytest config - if hasattr(session.config, "option") and hasattr(session.config.option, "filter_pattern"): - filter_pattern = filter_pattern or getattr(session.config.option, "filter_pattern", "") - if hasattr(session.config, "option") and hasattr(session.config.option, "exclude_pattern"): - exclude_pattern = exclude_pattern or getattr(session.config.option, "exclude_pattern", "") - - print("=" * 50) - print("CONFTEST.PY DEBUG INFO") - print("=" * 50) - print(f"Filter pattern: '{filter_pattern}'") - print(f"Exclude pattern: '{exclude_pattern}'") - print(f"Include files: {include_files if include_files else 'none'}") - print(f"Quarantined-only mode: {quarantined_only}") - print(f"Curobo-only mode: {curobo_only}") - print(f"TEST_FILTER_PATTERN env var: '{os.environ.get('TEST_FILTER_PATTERN', 'NOT_SET')}'") - print(f"TEST_EXCLUDE_PATTERN env var: '{os.environ.get('TEST_EXCLUDE_PATTERN', 'NOT_SET')}'") - print(f"TEST_INCLUDE_FILES env var: '{os.environ.get('TEST_INCLUDE_FILES', 'NOT_SET')}'") - print(f"TEST_QUARANTINED_ONLY env var: '{os.environ.get('TEST_QUARANTINED_ONLY', 'NOT_SET')}'") - print(f"TEST_CUROBO_ONLY env var: '{os.environ.get('TEST_CUROBO_ONLY', 'NOT_SET')}'") - print("=" * 50) - - # Get all test files in the source directories - test_files = _collect_test_files( - source_dirs, - filter_pattern, - exclude_pattern, - include_files, - quarantined_only, - curobo_only, - ) - - if isaacsim_ci: - new_test_files = [] - for test_file in test_files: - with open(test_file) as f: - if "@pytest.mark.isaacsim_ci" in f.read(): - new_test_files.append(test_file) - test_files = new_test_files - - if not test_files: - if quarantined_only: - print("No quarantined tests configured — nothing to run.") - _write_empty_report() - pytest.exit("No quarantined tests configured", returncode=0) - if filter_pattern: - print(f"No test files found matching filter pattern '{filter_pattern}' — nothing to run.") - _write_empty_report() - pytest.exit("No test files found for filter", returncode=0) - print("No test files found in source directory") - pytest.exit("No test files found", returncode=1) - - print(f"Found {len(test_files)} test files after filtering:") - for test_file in test_files: - print(f" - {test_file}") - - # Run all tests individually - failed_tests, test_status, xml_reports = run_individual_tests(test_files, workspace_root, isaacsim_ci) - - print("failed tests:", failed_tests) - - # Collect reports - print("~~~~~~~~~ Collecting final report...") - - # Merge in-memory report objects collected during the test run. Reading the - # on-disk files again risks losing elements if the junitparser - # read/write round-trip does not preserve them faithfully. - full_report = JUnitXml() - for xml_report in xml_reports: - print(xml_report) - full_report += xml_report - print("~~~~~~~~~~~~ Writing final report...") - # write content to full report - result_file = os.environ.get("TEST_RESULT_FILE", "full_report.xml") - full_report_path = f"tests/{result_file}" - print(f"Using result file: {result_file}") - full_report.write(full_report_path) - print("~~~~~~~~~~~~ Report written to", full_report_path) - - # print test status in a nice table - # Calculate the number and percentage of passing tests - num_tests = len(test_status) - num_passing = len([p for p in test_files if test_status[p]["result"].startswith("passed")]) - num_failing = len([p for p in test_files if test_status[p]["result"] == "FAILED"]) - num_timeout = len([p for p in test_files if test_status[p]["result"] == "TIMEOUT"]) - num_crashed = len([p for p in test_files if test_status[p]["result"] == "CRASHED"]) - num_startup_hang = len([p for p in test_files if test_status[p]["result"] == "STARTUP_HANG"]) - - if num_tests == 0: - passing_percentage = 100 - else: - passing_percentage = num_passing / num_tests * 100 - - # Print summaries of test results - summary_str = "\n\n" - summary_str += "===================\n" - summary_str += "Test Result Summary\n" - summary_str += "===================\n" - - summary_str += f"Total: {num_tests}\n" - summary_str += f"Passing: {num_passing}\n" - summary_str += f"Failing: {num_failing}\n" - summary_str += f"Crashed: {num_crashed}\n" - summary_str += f"Startup Hang: {num_startup_hang}\n" - summary_str += f"Timeout: {num_timeout}\n" - summary_str += f"Passing Percentage: {passing_percentage:.2f}%\n" - - total_wall = sum(test_status[test_path]["wall_time"] for test_path in test_files) - total_test = sum(test_status[test_path]["time_elapsed"] for test_path in test_files) - - summary_str += f"Total Wall Time: {total_wall // 3600:.0f}h{total_wall // 60 % 60:.0f}m{total_wall % 60:.2f}s\n" - summary_str += f"Total Test Time: {total_test // 3600:.0f}h{total_test // 60 % 60:.0f}m{total_test % 60:.2f}s" - - summary_str += "\n\n=======================\n" - summary_str += "Per Test Result Summary\n" - summary_str += "=======================\n" - - per_test_result_table = PrettyTable(field_names=["Test Path", "Result", "Test (s)", "Wall (s)", "# Tests"]) - per_test_result_table.align["Test Path"] = "l" - per_test_result_table.align["Test (s)"] = "r" - per_test_result_table.align["Wall (s)"] = "r" - for test_path in test_files: - num_tests_passed = ( - test_status[test_path]["tests"] - - test_status[test_path]["failures"] - - test_status[test_path]["errors"] - - test_status[test_path]["skipped"] - ) - per_test_result_table.add_row( - [ - test_path, - test_status[test_path]["result"], - f"{test_status[test_path]['time_elapsed']:0.2f}", - f"{test_status[test_path]['wall_time']:0.2f}", - f"{num_tests_passed}/{test_status[test_path]['tests']}", - ] - ) - - summary_str += per_test_result_table.get_string() - - # Print summary to console and log file - print(summary_str) - - # Exit pytest after custom execution to prevent normal pytest from overwriting our report - pytest.exit( - "Custom test execution completed", - returncode=0 if (num_failing == 0 and num_timeout == 0 and num_crashed == 0 and num_startup_hang == 0) else 1, + config = RunnerConfig.from_env(workspace_root) + print( + f"test_runner config: mask={config.runtime_mask} queue={'on' if config.queue_path else 'off'}" + f" isaacsim_ci={config.isaacsim_ci} filter={config.filter_pattern!r}" + f" exclude={config.exclude_pattern!r} include={sorted(config.include_files) or 'none'}" ) + pytest.exit("Custom test execution completed", returncode=Session(config).run()) diff --git a/tools/test_device_split.py b/tools/test_device_split.py deleted file mode 100644 index fb34ce40c4f1..000000000000 --- a/tools/test_device_split.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - -"""Unit tests for ``tools/_device_split.py``.""" - -from __future__ import annotations - -import textwrap -from pathlib import Path - -from _device_split import is_device_split_file - - -def _write(tmp_path: Path, content: str) -> Path: - p = tmp_path / "test_foo.py" - p.write_text(textwrap.dedent(content)) - return p - - -def test_single_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.device_split - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_list_form_single_line(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = [pytest.mark.device_split, pytest.mark.foo] - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is True - - -def test_preloaded_source(tmp_path): - source = textwrap.dedent( - """ - import pytest - - pytestmark = pytest.mark.device_split - """ - ) - assert is_device_split_file(tmp_path / "does_not_exist.py", source=source) is True - - -def test_no_mark(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_word_in_comment_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - # This file mentions device_split in a comment but is not marked. - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_unrelated_pytestmark_does_not_match(tmp_path): - f = _write( - tmp_path, - """ - import pytest - - pytestmark = pytest.mark.skipif(False, reason="x") - - def test_x(): - pass - """, - ) - assert is_device_split_file(f) is False - - -def test_missing_file(tmp_path): - # A path that does not exist must not raise; treat as not-marked. - assert is_device_split_file(tmp_path / "does_not_exist.py") is False diff --git a/tools/test_runner/__init__.py b/tools/test_runner/__init__.py new file mode 100644 index 000000000000..3ce7c128f7b9 --- /dev/null +++ b/tools/test_runner/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Per-file CI test runner framework. + +``tools/conftest.py`` runs each test file as its own pytest subprocess rather +than collecting everything into one process. This package is that runner, split +into the stages of a test-run lifecycle: + +* :mod:`test_runner.planning` — what to run: the :class:`~test_runner.planning.RunnerConfig` + knobs, the :class:`~test_runner.planning.Unit` work item, and the + :class:`~test_runner.planning.Planner` that turns a file + device mask into units. +* :mod:`test_runner.execution` — how to run one unit: subprocess capture, + timeout/hang handling, report parsing, retries. +* :mod:`test_runner.selector` — an in-process pytest plugin the executor injects + into a unit's subprocess to keep only that unit's device variants. +* :mod:`test_runner.session` — the lifecycle: collect files, plan units, run + them, aggregate and report. + +The whole runner is driven by one :class:`~test_runner.planning.RunnerConfig`, so +its behavior (timeouts, retries, cold-cache budget, device mask, filters) is +configuration rather than scattered constants. +""" diff --git a/tools/test_runner/execution.py b/tools/test_runner/execution.py new file mode 100644 index 000000000000..41f07c1f95f5 --- /dev/null +++ b/tools/test_runner/execution.py @@ -0,0 +1,355 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""How to run one unit: build its command, drive the subprocess, classify the result. + +:class:`UnitRunner` turns a :class:`~test_runner.planning.Unit` into a pytest +subprocess via :class:`~test_runner.process.ProcessCapture`, then resolves the +outcome (startup hang / timeout / crash / pass / fail) into a status dict, with +fresh-process retries for flaky files. The per-file dynamic inputs that vary +within a run live in :class:`ExecContext`; the static knobs come from the +:class:`~test_runner.planning.RunnerConfig`. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +from dataclasses import dataclass + +from test_runner.planning import RunnerConfig, Unit +from test_runner.process import JUnitReport, ProcessCapture + +_RESULT_PRIORITY = { + "STARTUP_HANG": 5, + "CRASHED": 4, + "TIMEOUT": 3, + "FAILED": 2, + "passed (shutdown hanged)": 1, + "passed": 0, +} + + +@dataclass +class ExecContext: + """Per-file inputs that vary within a run (static knobs are in RunnerConfig). + + Attributes: + file_name: Basename of the file, used in JUnit naming and messages. + timeout: Per-unit hard timeout [s] (already cold-cache-adjusted). + startup_deadline: Per-unit startup-hang deadline [s]. + env: Base environment; :class:`UnitRunner` copies it and adds the unit's mask. + """ + + file_name: str + timeout: int + startup_deadline: int + env: dict + + +def merge_statuses(prev: dict | None, new: dict) -> dict: + """Combine per-unit status dicts into a per-file entry. + + Counters sum; ``result`` becomes the more severe of the two by + :data:`_RESULT_PRIORITY`. Used by the session to fold a split file's units. + """ + if prev is None: + return new + return { + "errors": prev["errors"] + new["errors"], + "failures": prev["failures"] + new["failures"], + "skipped": prev["skipped"] + new["skipped"], + "tests": prev["tests"] + new["tests"], + "time_elapsed": prev["time_elapsed"] + new["time_elapsed"], + "wall_time": prev["wall_time"] + new["wall_time"], + "result": prev["result"] + if _RESULT_PRIORITY.get(prev["result"], 0) >= _RESULT_PRIORITY.get(new["result"], 0) + else new["result"], + } + + +class UnitRunner: + """Run one unit as a pytest subprocess and classify its outcome.""" + + def __init__(self, config: RunnerConfig): + self._config = config + self._capture = ProcessCapture(config.shutdown_grace_period) + + @staticmethod + def _status(result, *, errors=0, failures=0, skipped=0, tests=1, time_elapsed=0.0, wall_time=0.0) -> dict: + """A per-unit status dict (the shape the reporter and merge_statuses expect).""" + return { + "errors": errors, + "failures": failures, + "skipped": skipped, + "tests": tests, + "result": result, + "time_elapsed": time_elapsed, + "wall_time": wall_time, + } + + def _error_result( + self, + kind, + label, + suffix, + unit, + ctx, + result, + wall_time, + cap, + pre, + *, + msg, + stdout, + stderr, + report_file, + time_elapsed=0.0, + ): + """Print diagnostics, write a synthetic error report, and return its failure status.""" + diag = cap.diagnostics(pre) + print(f"⚠️ {unit.file}{suffix}: {msg}") + print(diag) + details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n" + if stdout: + details += "=== STDOUT (last 5000 chars) ===\n" + stdout.decode("utf-8", errors="replace")[-5000:] + "\n" + if stderr: + details += "=== STDERR (last 5000 chars) ===\n" + stderr.decode("utf-8", errors="replace")[-5000:] + "\n" + report = JUnitReport.error(kind, label, msg, details) + report.write(report_file) + return report, self._status(result, errors=1, tests=1, time_elapsed=time_elapsed, wall_time=wall_time), True + + def _build_cmd(self, unit: Unit, ctx: ExecContext): + """Build ``(cmd, env, report_file, label, report_suffix)`` for a unit. + + Device-variant selection is the unit's mask, set as ``ISAACLAB_TEST_DEVICES`` + for ``test_devices()`` — never ``-k``. A split unit (either half) or a + cpu-less shard also injects the selector plugin to drop out-of-scope + variants (including literal device-param ones the mask cannot narrow). + """ + config = self._config + # Split units of one file share a slug, so suffix the report with the mask; + # a lone unit keeps the suffix-free name the mgpu aggregator unslugs to a path. + report_suffix = f"-{unit.mask}" if unit.split else "" + pass_file_label = f"{ctx.file_name}{report_suffix}" + # Slug the full path (not the basename) so concurrent shards running + # same-basename files don't collide on the shared workspace mount. + report_slug = str(unit.file).replace("/", "__").replace("\\", "__") + report_file = os.path.join(config.report_dir, f"test-reports-{report_slug}{report_suffix}.xml") + + env = dict(ctx.env) + env["ISAACLAB_TEST_DEVICES"] = unit.mask + select_by_device = unit.split or unit.mask[:1] == "0" + if select_by_device: + env["PYTHONPATH"] = config.tools_dir + os.pathsep + env.get("PYTHONPATH", "") + + cmd = [ + sys.executable, + "-m", + "pytest", + "-s", + "-v", # per-test names in the log: if a file hangs, the last name pinpoints the culprit + "--no-header", + f"--config-file={config.config_file}", + f"--junitxml={report_file}", + "--tb=short", + ] + if select_by_device: + cmd += ["-p", "test_runner.selector"] + if config.isaacsim_ci: + cmd += ["-m", "isaacsim_ci"] + cmd.append(str(unit.file)) + return cmd, env, report_file, pass_file_label, report_suffix + + def _retry_failed_in_fresh_process( + self, + *, + unit, + ctx, + cmd, + env, + report_file, + report, + errors, + failures, + skipped, + tests, + time_elapsed, + returncode, + stdout, + stderr, + kill_reason, + wall_time, + pre, + ): + """Re-run a failed file in fresh processes (for files with stale-state flakiness).""" + has_failures = errors > 0 or failures > 0 + attempts = 0 + max_retries = self._config.process_failure_retries.get(ctx.file_name, 0) + while has_failures and attempts < max_retries: + attempts += 1 + print( + f"⚠️ {unit.file}: failed in subprocess" + f" (attempt {attempts}/{max_retries + 1}), retrying in fresh process..." + ) + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + returncode, stdout, stderr, kill_reason, wall_time, pre = self._capture.run( + cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) + if not os.path.exists(report_file): + break + try: + report, errors, failures, skipped, tests, time_elapsed = JUnitReport.read(report_file, ctx.file_name) + has_failures = errors > 0 or failures > 0 + except Exception as e: + print(f"Error reading retry test report {report_file}: {e}") + report, errors, failures, skipped, tests, time_elapsed = report, 1, 0, 0, 0, 0.0 + has_failures = True + break + return report, errors, failures, skipped, tests, time_elapsed, returncode, kill_reason, wall_time, has_failures + + def run(self, unit: Unit, ctx: ExecContext) -> tuple: + """Run ``unit`` and return ``(xml_report_or_None, status_dict, was_failure)``.""" + cap = self._capture + cmd, env, report_file, label, suffix = self._build_cmd(unit, ctx) + + # -- launch, retrying startup hangs / hard timeouts -- + returncode, stdout, stderr, kill_reason, wall_time, pre = -1, b"", b"", "", 0.0, "" + startup_attempts = timeout_attempts = 0 + while True: + with contextlib.suppress(FileNotFoundError): + os.remove(report_file) + returncode, stdout, stderr, kill_reason, wall_time, pre = cap.run( + cmd, ctx.timeout, env, startup_deadline=ctx.startup_deadline, report_file=report_file + ) + has_report = os.path.exists(report_file) + if kill_reason == "startup_hang" and startup_attempts < self._config.startup_hang_retries: + startup_attempts += 1 + print( + f"⚠️ {unit.file}{suffix}: startup hang after {ctx.startup_deadline}s" + f" (attempt {startup_attempts}/{self._config.startup_hang_retries + 1}), retrying..." + ) + if stderr: + print("=== STDERR (last 5000 chars) ===") + print(stderr.decode("utf-8", errors="replace")[-5000:]) + print(cap.diagnostics(pre)) + continue + if kill_reason == "timeout" and not has_report and timeout_attempts < self._config.timeout_retries: + timeout_attempts += 1 + print( + f"⚠️ {unit.file}{suffix}: timeout after {ctx.timeout}s" + f" (attempt {timeout_attempts}/{self._config.timeout_retries + 1}), retrying..." + ) + if stdout: + print("=== STDOUT (last 5000 chars) ===") + print(stdout.decode("utf-8", errors="replace")[-5000:]) + print(cap.diagnostics(pre)) + continue + break + + # -- classify the outcome -- + has_report = os.path.exists(report_file) + if kill_reason == "startup_hang": + return self._error_result( + "startup_hang", + label, + suffix, + unit, + ctx, + "STARTUP_HANG", + wall_time, + cap, + pre, + msg=f"Startup hang after {ctx.startup_deadline}s (retried {self._config.startup_hang_retries} time(s))", + stdout=stdout, + stderr=stderr, + report_file=report_file, + ) + if kill_reason == "timeout" and not has_report: + return self._error_result( + "timeout", + label, + suffix, + unit, + ctx, + "TIMEOUT", + wall_time, + cap, + pre, + msg=f"Timeout after {ctx.timeout} seconds (retried {timeout_attempts} time(s))", + stdout=stdout, + stderr=stderr, + report_file=report_file, + time_elapsed=ctx.timeout, + ) + if not has_report: + reason = ( + cap.signal_description(-returncode) + if returncode < 0 + else f"Process exited with code {returncode} but produced no report" + ) + return self._error_result( + "crash", + label, + suffix, + unit, + ctx, + "CRASHED", + wall_time, + cap, + "", + msg=reason, + stdout=stdout, + stderr=stderr, + report_file=report_file, + ) + + if kill_reason in ("shutdown_hang", "timeout"): + print(f"⚠️ {unit.file}{suffix}: shutdown hanged (killed after {wall_time:.0f}s, test completed)") + try: + report, errors, failures, skipped, tests, time_elapsed = JUnitReport.read(report_file, label) + except Exception as e: + print(f"Error reading test report {report_file}: {e}") + return None, self._status("FAILED", errors=1, tests=0, wall_time=wall_time), True + + report, errors, failures, skipped, tests, time_elapsed, returncode, kill_reason, wall_time, has_failures = ( + self._retry_failed_in_fresh_process( + unit=unit, + ctx=ctx, + cmd=cmd, + env=env, + report_file=report_file, + report=report, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + returncode=returncode, + stdout=stdout, + stderr=stderr, + kill_reason=kill_reason, + wall_time=wall_time, + pre=pre, + ) + ) + shutdown_hanged = kill_reason in ("shutdown_hang", "timeout") and not has_failures + was_failure = has_failures or (returncode != 0 and not shutdown_hanged) + result = "passed (shutdown hanged)" if shutdown_hanged else ("FAILED" if has_failures else "passed") + return ( + report, + self._status( + result, + errors=errors, + failures=failures, + skipped=skipped, + tests=tests, + time_elapsed=time_elapsed, + wall_time=wall_time, + ), + was_failure, + ) diff --git a/tools/test_runner/planning.py b/tools/test_runner/planning.py new file mode 100644 index 000000000000..831c14a21cfb --- /dev/null +++ b/tools/test_runner/planning.py @@ -0,0 +1,212 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""What to run: the runner configuration, the work unit, and the planner. + +:class:`RunnerConfig` centralizes every knob the runner used to read ad hoc from +the environment or hardcode as a module constant. :class:`Unit` is the work item +(a file plus the device mask to run it under). :class:`Planner` turns a file into +its units, splitting a ``device_isolated`` file into one single-device unit per +device when the run spans more than one. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field + +# Per-file fresh-process retry counts for tests that can enter stale render states. +# A default so the runner has no hardcoded file list; override via RunnerConfig. +_DEFAULT_PROCESS_FAILURE_RETRIES = { + "test_visualizer_integration_physx.py": 4, + "test_visualizer_integration_newton.py": 4, + "test_visualizer_tiled_integration_physx.py": 4, + "test_visualizer_tiled_integration_newton.py": 4, +} + + +@dataclass +class RunnerConfig: + """All runner behavior as data, so nothing is hardcoded mid-pipeline. + + Build it once with :meth:`from_env` at the pytest entry point and pass it to + every stage. Fields group into: run scope (mask, queue, isaacsim_ci), + collection (filters/sharding), per-unit execution (timeouts, deadlines, + retries, cold-cache), and reporting (paths). + + Attributes: + workspace_root: Repository root; pytest ``--config-file`` and the + ``tools/`` plugin dir derive from it. + runtime_mask: Device mask for the run (e.g. ``"110"`` cpu+cuda:0, + ``"0001"`` a shard). Concrete — never the open-ended ``"X"`` form. + queue_path: Shared work-queue root for mgpu work-stealing; empty when the + runner iterates its own file list. + sim_device: Kit boot device; names the per-shard work-queue subdir and + labels rows in the report. + isaacsim_ci: Whether to pass ``-m isaacsim_ci`` to each subprocess. + filter_pattern: Substring a test path must contain to run; empty = all. + exclude_pattern: Comma-separated substrings; a path matching any is skipped. + include_files: Basenames to restrict the run to; empty = no restriction. + quarantined_only: Run only the quarantined set. + curobo_only: Run only the cuRobo set. + shard_index: This file-shard's index, or ``None`` for no file sharding. + shard_count: Total file shards, or ``None``. + default_timeout: Per-unit hard timeout [s] when a file has no override. + per_file_timeouts: Basename -> hard timeout [s] overrides. + cold_cache_buffer: Extra timeout [s] for the first camera-enabled test + (shader compile on a cold cache). + cold_cache_marker: Source substring that marks a camera-enabled test. + startup_deadline: Seconds to reach AppLauncher init / pytest collection + before declaring a startup hang. + startup_hang_retries: Startup-hang retries before giving up. + timeout_retries: Hard-timeout retries before giving up. + shutdown_grace_period: Seconds to wait for clean exit after the report + file appears before killing a shutdown-hung process. + process_failure_retries: Basename -> extra fresh-process retries on test + failure (for tests with stale-state flakiness). + report_dir: Directory for per-unit and aggregate JUnit XML. + result_file: Aggregate report filename within :attr:`report_dir`. + """ + + workspace_root: str + # -- run scope -- + runtime_mask: str = "110" + queue_path: str = "" + sim_device: str = "cuda:0" + isaacsim_ci: bool = False + # -- collection -- + filter_pattern: str = "" + exclude_pattern: str = "" + include_files: frozenset[str] = frozenset() + quarantined_only: bool = False + curobo_only: bool = False + shard_index: int | None = None + shard_count: int | None = None + # -- per-unit execution -- + default_timeout: int = 600 + per_file_timeouts: dict = field(default_factory=dict) + cold_cache_buffer: int = 700 + cold_cache_marker: str = "enable_cameras=True" + startup_deadline: int = 120 + startup_hang_retries: int = 2 + timeout_retries: int = 0 + shutdown_grace_period: int = 30 + process_failure_retries: dict = field(default_factory=lambda: dict(_DEFAULT_PROCESS_FAILURE_RETRIES)) + # -- reporting -- + report_dir: str = "tests" + result_file: str = "full_report.xml" + + @property + def config_file(self) -> str: + """Path passed to pytest's ``--config-file`` for each subprocess.""" + return os.path.join(self.workspace_root, "pyproject.toml") + + @property + def tools_dir(self) -> str: + """Directory added to a subprocess PYTHONPATH so it can import the selector.""" + return os.path.join(self.workspace_root, "tools") + + @classmethod + def from_env(cls, workspace_root: str) -> RunnerConfig: + """Build a config from the environment and ``test_settings``. + + This is the single place environment variables are read; the rest of the + runner takes the resulting config. + + Args: + workspace_root: Repository root. + + Returns: + A fully-populated config for this run. + """ + import test_settings # tools/ module; imported lazily so unit tests need no env + + shard_index = os.environ.get("TEST_SHARD_INDEX", "") + shard_count = os.environ.get("TEST_SHARD_COUNT", "") + raw_includes = os.environ.get("TEST_INCLUDE_FILES", "").split(",") + include = {os.path.basename(f.strip()) for f in raw_includes if f.strip()} + return cls( + workspace_root=workspace_root, + # An unset mask means single-GPU CI: cpu + cuda:0, matching test_devices()'s default. + runtime_mask=os.environ.get("ISAACLAB_TEST_DEVICES") or "110", + queue_path=os.environ.get("ISAACLAB_TEST_QUEUE", ""), + # Kit boot device; names the per-shard work-queue subdir and labels the report. + sim_device=os.environ.get("ISAACLAB_SIM_DEVICE") or "cuda:0", + isaacsim_ci=os.environ.get("ISAACSIM_CI_SHORT", "false") == "true", + filter_pattern=os.environ.get("TEST_FILTER_PATTERN", ""), + exclude_pattern=os.environ.get("TEST_EXCLUDE_PATTERN", ""), + include_files=frozenset(include), + quarantined_only=os.environ.get("TEST_QUARANTINED_ONLY", "false") == "true", + curobo_only=os.environ.get("TEST_CUROBO_ONLY", "false") == "true", + shard_index=int(shard_index) if shard_index else None, + shard_count=int(shard_count) if shard_count else None, + default_timeout=test_settings.DEFAULT_TIMEOUT, + per_file_timeouts=dict(test_settings.PER_TEST_TIMEOUTS), + result_file=os.environ.get("TEST_RESULT_FILE", "full_report.xml"), + ) + + +@dataclass(frozen=True) +class Unit: + """One pytest subprocess to run. + + Attributes: + file: Test file to run. + mask: Device mask set as ``ISAACLAB_TEST_DEVICES`` for the subprocess; + ``test_devices()`` reads it to select the device variants. + split: Whether this is one of several units from splitting a + ``device_isolated`` file. The executor uses it only to give split + units distinct report filenames; a lone unit keeps the suffix-free + name the mgpu aggregator expects. + """ + + file: str + mask: str + split: bool = False + + +class Planner: + """Turns a test file into the units to run, given the run's device mask. + + A ``device_isolated`` file (a backend with a process-global device lock, e.g. + ovphysx) splits into one single-device unit per device, but only when the run + spans more than one device — an mgpu shard's single-device mask yields one + unit either way. + """ + + _ISOLATED_MARK_RE = re.compile(r"^\s*pytestmark\b.*\bdevice_isolated\b", re.MULTILINE) + """Match a module-level ``pytestmark`` mentioning ``device_isolated`` (single or list form).""" + + def __init__(self, runtime_mask: str): + if "X" in runtime_mask: + raise ValueError(f"runtime mask {runtime_mask!r} must be concrete; 'X' is a scope-only construct") + self._mask = runtime_mask + + @staticmethod + def _single_bit_mask(index: int, width: int) -> str: + """Return a width-``width`` mask with only ``index`` set, e.g. ``(1, 3) -> "010"``.""" + return "".join("1" if pos == index else "0" for pos in range(width)) + + def is_isolated(self, source: str) -> bool: + """Whether a test file's source declares the ``device_isolated`` marker.""" + return bool(self._ISOLATED_MARK_RE.search(source)) + + def plan(self, test_file: str, source: str) -> list[Unit]: + """Plan the units for one file. + + Args: + test_file: Path to the test file. + source: The file's source text (the caller already reads it for the + cold-cache check), inspected for the isolation marker. + + Returns: + One unit at the run's mask, or — for an isolated file on a + multi-device run — one single-device unit per active device. + """ + set_bits = [pos for pos, char in enumerate(self._mask) if char == "1"] + if self.is_isolated(source) and len(set_bits) > 1: + return [Unit(test_file, self._single_bit_mask(pos, len(self._mask)), split=True) for pos in set_bits] + return [Unit(test_file, self._mask)] diff --git a/tools/test_runner/process.py b/tools/test_runner/process.py new file mode 100644 index 000000000000..d1524494157a --- /dev/null +++ b/tools/test_runner/process.py @@ -0,0 +1,264 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Low-level subprocess execution and JUnit report I/O for one pytest run. + +:class:`ProcessCapture` launches a command, streams its output live, and kills it +on a startup hang, hard timeout, or shutdown hang, returning enough to classify +the outcome. :class:`JUnitReport` reads a produced report or synthesizes an error +report when the process produced none. Both are device- and unit-agnostic; the +unit-level orchestration that uses them is :class:`~test_runner.execution.UnitRunner`. +""" + +from __future__ import annotations + +import contextlib +import os +import select +import signal +import subprocess +import sys +import time + +from junitparser import Error, JUnitXml, TestCase, TestSuite + +_SIGNAL_DESCRIPTIONS = { + 1: "SIGHUP — session leader exit or orphaned process cleanup", + 6: "SIGABRT", + 9: "SIGKILL — likely OOM killed", + 11: "SIGSEGV — segmentation fault", + 15: "SIGTERM", +} +_DIAG_LIMIT = 10000 # chars; truncate diagnostics so a crash dump can't flood the log + + +class ProcessCapture: + """Run a pytest subprocess with hang/timeout detection and live output streaming.""" + + def __init__(self, shutdown_grace_period: int): + # Seconds to allow for clean exit after the report file appears, before + # killing a process that hung in SimulationApp.close() / Kit shutdown. + self._grace = shutdown_grace_period + + @staticmethod + def signal_description(sig: int) -> str: + """Human-readable reason a process was killed by signal ``sig``.""" + base = f"Process killed by signal {sig}" + desc = _SIGNAL_DESCRIPTIONS.get(sig) + return f"{base} ({desc})" if desc else base + + @staticmethod + def _system_diagnostics() -> str: + """GPU/memory/process snapshot for crash investigation. Never raises.""" + sections = [] + try: + r = subprocess.run(["nvidia-smi"], capture_output=True, text=True, timeout=10) + if r.stdout: + sections.append(f"--- nvidia-smi ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- nvidia-smi --- FAILED: {e}") + try: + with open("/proc/meminfo") as f: + lines = f.readlines() + keys = ("MemTotal", "MemFree", "MemAvailable", "Committed_AS", "SwapTotal", "SwapFree") + relevant = [line.strip() for line in lines if any(line.startswith(k) for k in keys)] + if relevant: + sections.append("--- /proc/meminfo ---\n" + "\n".join(relevant)) + except Exception as e: + sections.append(f"--- /proc/meminfo --- FAILED: {e}") + cgroup_lines = [] + for path in ( + "/sys/fs/cgroup/memory.current", + "/sys/fs/cgroup/memory.max", + "/sys/fs/cgroup/memory.events", + "/sys/fs/cgroup/memory/memory.usage_in_bytes", + "/sys/fs/cgroup/memory/memory.limit_in_bytes", + "/sys/fs/cgroup/memory/memory.oom_control", + ): + try: + with open(path) as f: + cgroup_lines.append(f"{path}: {f.read().strip()}") + except FileNotFoundError: + pass + except Exception as e: + cgroup_lines.append(f"{path}: FAILED ({e})") + if cgroup_lines: + sections.append("--- cgroup memory ---\n" + "\n".join(cgroup_lines)) + try: + r = subprocess.run(["ps", "auxf"], capture_output=True, text=True, timeout=5) + if r.stdout: + sections.append(f"--- process tree (ps auxf) ---\n{r.stdout.strip()}") + except Exception as e: + sections.append(f"--- process tree --- FAILED: {e}") + try: + r = subprocess.run(["dmesg", "-T"], capture_output=True, text=True, timeout=5) + if r.stdout: + lines = r.stdout.strip().split("\n") + sections.append("--- dmesg (last 30 lines) ---\n" + "\n".join(lines[-30:])) + except Exception: + pass + return "\n\n".join(sections) + + def diagnostics(self, pre_kill: str = "") -> str: + """Return diagnostics (a pre-kill snapshot if given, else a fresh one), truncated.""" + diag = pre_kill or self._system_diagnostics() + return diag[:_DIAG_LIMIT] + "\n... (truncated)" if len(diag) > _DIAG_LIMIT else diag + + def run(self, cmd, timeout, env, startup_deadline=0, report_file=""): + """Run ``cmd``, streaming output, and kill it on a hang or timeout. + + Args: + cmd: argv to execute. + timeout: hard wall-clock limit [s]. + env: subprocess environment. + startup_deadline: if > 0, kill when neither AppLauncher-init nor + pytest-collection markers appear within this many seconds. + report_file: when set, the process gets only the configured grace + period to exit after this file appears on disk. + + Returns: + ``(returncode, stdout_bytes, stderr_bytes, kill_reason, wall_time, + pre_kill_diag)``; ``kill_reason`` is ``""`` for a clean exit, else + ``"startup_hang"`` / ``"timeout"`` / ``"shutdown_hang"``. + """ + stdout_data = b"" + stderr_data = b"" + process = None + try: + # Own session so orphaned Kit children can't SIGHUP the next test's group. + process = subprocess.Popen( + cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + universal_newlines=False, + start_new_session=True, + ) + pgid = os.getpgid(process.pid) + stdout_fd = process.stdout.fileno() + stderr_fd = process.stderr.fileno() + try: + import fcntl + + for fd in [stdout_fd, stderr_fd]: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + except ImportError: + pass + + start_time = time.time() + startup_done = startup_deadline <= 0 + shutdown_deadline = 0.0 + while process.poll() is None: + elapsed = time.time() - start_time + if not startup_done: + if b"AppLauncher initialization complete" in stderr_data or b"collected " in stdout_data: + startup_done = True + if report_file and not shutdown_deadline and os.path.exists(report_file): + shutdown_deadline = time.time() + self._grace + + kill_reason = None + if not startup_done and elapsed > startup_deadline: + kill_reason = "startup_hang" + elif shutdown_deadline and time.time() > shutdown_deadline: + kill_reason = "shutdown_hang" + elif elapsed > timeout: + kill_reason = "timeout" + + if kill_reason: + pre_kill_diag = self._system_diagnostics() + try: # kill the whole group (test + any Kit children) + os.killpg(pgid, signal.SIGKILL) + except (ProcessLookupError, PermissionError, OSError): + process.kill() + try: + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + except subprocess.TimeoutExpired: + pass + return -1, stdout_data, stderr_data, kill_reason, time.time() - start_time, pre_kill_diag + + try: + ready_fds, _, _ = select.select([stdout_fd, stderr_fd], [], [], 0.1) + for fd in ready_fds: + with contextlib.suppress(OSError): + if fd == stdout_fd: + chunk = process.stdout.read(1024) + if chunk: + stdout_data += chunk + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif fd == stderr_fd: + chunk = process.stderr.read(1024) + if chunk: + stderr_data += chunk + sys.stderr.buffer.write(chunk) + sys.stderr.buffer.flush() + except OSError: + time.sleep(0.1) + continue + + try: # drain output written just before/after exit + rem_out, rem_err = process.communicate(timeout=10) + stdout_data += rem_out + stderr_data += rem_err + except Exception: + pass + try: # reap any orphaned Kit/Isaac Sim children + os.killpg(pgid, signal.SIGKILL) + time.sleep(1) + except (ProcessLookupError, PermissionError, OSError): + pass + return process.returncode, stdout_data, stderr_data, "", time.time() - start_time, "" + except Exception as e: + if process is not None and process.poll() is None: + process.kill() + with contextlib.suppress(Exception): + rem_out, rem_err = process.communicate(timeout=5) + stdout_data += rem_out + stderr_data += rem_err + stdout_data += f"\n[capture error: {e}]\n".encode() + return -1, stdout_data, stderr_data, "", 0.0, "" + + +class JUnitReport: + """Read a unit's JUnit report, or synthesize one when the process produced none.""" + + @staticmethod + def error(prefix: str, file_name: str, message: str, details: str) -> JUnitXml: + """Build a one-case JUnit report standing in for a run that wrote no report.""" + suite_name = os.path.splitext(file_name)[0] + suite = TestSuite(name=f"{prefix}_{suite_name}") + case = TestCase(name="test_execution", classname=suite_name) + result = Error(message=message) + result.text = details + case.result = result + suite.add_testcase(case) + report = JUnitXml() + report.add_testsuite(suite) + return report + + @staticmethod + def read(report_file: str, file_name: str): + """Parse a JUnit report; rename its synthetic ``pytest`` suite to the file. + + Returns: + ``(report, errors, failures, skipped, tests, time_elapsed)``. + """ + report = JUnitXml.fromfile(report_file) + for suite in report: + if suite.name == "pytest": + suite.name = os.path.splitext(file_name)[0] + report.write(report_file) + return ( + report, + int(report.errors) if report.errors is not None else 0, + int(report.failures) if report.failures is not None else 0, + int(report.skipped) if report.skipped is not None else 0, + int(report.tests) if report.tests is not None else 0, + float(report.time) if report.time is not None else 0.0, + ) diff --git a/tools/test_runner/selector.py b/tools/test_runner/selector.py new file mode 100644 index 000000000000..53710d095c54 --- /dev/null +++ b/tools/test_runner/selector.py @@ -0,0 +1,121 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""In-process pytest plugin: keep only the tests a unit's device mask allows. + +The executor injects this into a unit's subprocess with ``-p test_runner.selector`` +for any unit that must run on a single device — both halves of a ``device_isolated`` +split and every mgpu shard. It runs inside that subprocess, at collection time, so +a process locked to one device by a backend (e.g. ovphysx) never also tries another. + +It is registered as a class instance (see :func:`pytest_configure`): the hooks are +methods of :class:`DeviceSelect`, so the device mask is read once into instance +state instead of from the environment on every call. +""" + +from __future__ import annotations + +import os + +import pytest + +_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES" + + +class DeviceSelect: + """Decide, per collected test, whether it belongs to this unit's device mask. + + Device-parametrized tests are kept iff their device is active in the mask + (this catches literal ``["cuda:0", "cpu"]`` variants the mask cannot narrow at + parametrize time); agnostic tests (no ``device`` param) are kept only when cpu + is in the mask, i.e. the default unit — never on a cuda split unit or a shard. + """ + + def __init__(self, mask: str): + self._mask = mask + + @staticmethod + def _position(device) -> int | None: + """Mask position for a device: cpu -> 0, cuda:k -> k + 1, else None (agnostic/unknown).""" + if device == "cpu": + return 0 + if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit(): + return int(device[len("cuda:") :]) + 1 + return None + + def _active(self, position: int | None) -> bool: + """Whether a mask position is included; a trailing ``X`` includes all remaining.""" + if position is None: + return False + body, fill = (self._mask[:-1], True) if self._mask.endswith("X") else (self._mask, False) + return body[position] == "1" if 0 <= position < len(body) else fill + + def keeps(self, device: str | None) -> bool: + """Whether a test on ``device`` (or ``None`` if agnostic) runs in this unit.""" + if device is None: + return self._active(0) # agnostic: only in a cpu-inclusive unit + return self._active(self._position(device)) + + # -- pytest hooks (methods of the registered instance) -- + + def pytest_collection_modifyitems(self, config, items): + """PYTEST HOOK — filter the collected tests in place to this unit's devices. + + Called by pytest after collection, before any test runs. + + Args: + config: the session ``pytest.Config``; ``config.hook.pytest_deselected`` + reports the dropped tests so they count as deselected, not missing. + items: the live, ordered ``list[pytest.Item]`` of collected tests. This + hook mutates it in place (``items[:] = keep``) — that is how pytest + learns what to run; reassigning a new list would have no effect. + + Returns: + None. Its effect is the in-place edit of ``items`` plus the + ``pytest_deselected`` report. + """ + keep, drop = [], [] + for item in items: + callspec = getattr(item, "callspec", None) + device = callspec.params.get("device") if callspec is not None else None + (keep if self.keeps(device) else drop).append(item) + if drop: + items[:] = keep + config.hook.pytest_deselected(items=drop) + + def pytest_sessionfinish(self, session, exitstatus): + """PYTEST HOOK — treat a unit that selected nothing as a clean pass. + + Called by pytest once the whole session ends. + + Args: + session: the ``pytest.Session``; reassign ``session.exitstatus`` to + change the process exit code. + exitstatus: the proposed exit code (a ``pytest.ExitCode``). + + Returns: + None. When this unit deselected every test (NO_TESTS_COLLECTED), it + rewrites the exit to OK so the runner does not read a non-zero exit as + a failure — "nothing in scope for this file" is success here. + """ + if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED: + session.exitstatus = pytest.ExitCode.OK + + +def pytest_configure(config): + """PYTEST HOOK — the plugin's entry point: register the device selector. + + Loading this module with ``-p test_runner.selector`` makes pytest call this + module-level hook during config init. It builds a :class:`DeviceSelect` from + the unit's mask (the ``ISAACLAB_TEST_DEVICES`` the executor set) and registers + the instance, so the instance's hook methods become active for the session. + + Args: + config: the session ``pytest.Config`` (provides the plugin manager). + + Returns: + None. + """ + config.pluginmanager.register(DeviceSelect(os.environ.get(_RUNTIME_ENV, ""))) diff --git a/tools/test_runner/session.py b/tools/test_runner/session.py new file mode 100644 index 000000000000..79a333887dbf --- /dev/null +++ b/tools/test_runner/session.py @@ -0,0 +1,350 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""The test-run lifecycle: collect files, plan units, run them, report. + +:class:`Session` is the orchestrator that ``conftest.pytest_sessionstart`` +drives. It composes the stages: :class:`Collector` (which files), :class:`WorkQueue` +(how files are handed out across mgpu shards), the planner and unit runner from +the sibling modules (how each file runs), and :class:`Reporter` (merge + summary). +""" + +from __future__ import annotations + +import contextlib +import os +import re + +import pytest +import test_settings # tools/ module: QUARANTINED_TESTS / CUROBO_TESTS / TESTS_TO_SKIP +from junitparser import JUnitXml +from prettytable import PrettyTable + +from test_runner.execution import ExecContext, UnitRunner, merge_statuses +from test_runner.planning import Planner, RunnerConfig + + +class WorkQueue: + """Hand test files out across mgpu shards via an atomic-rename work queue. + + The queue is a directory of one file per pending test. A shard claims a test + by renaming it ``queue/`` -> ``inflight//`` (POSIX rename is + atomic, so racing shards are kernel-serialized), then ``-> done//`` + when it finishes. Anything left in ``inflight`` at job-end is a crashed claim. + When inactive (no queue configured) the runner just iterates its own list. + """ + + def __init__(self, config: RunnerConfig): + self._dir = config.queue_path + self._shard = config.sim_device.replace(":", "-") + + @property + def active(self) -> bool: + """Whether a shared queue is configured (mgpu) versus plain iteration.""" + return bool(self._dir) + + @staticmethod + def _slug(test_path: str) -> str: + """Encode a path as a flat queue filename (``/`` -> ``__``).""" + return test_path.replace("/", "__") + + def _claim(self) -> str | None: + """Atomically claim one pending test, or return None when the queue is empty.""" + pending_dir = os.path.join(self._dir, "queue") + inflight_dir = os.path.join(self._dir, "inflight", self._shard) + os.makedirs(inflight_dir, exist_ok=True) + try: + entries = sorted(os.listdir(pending_dir)) # uncached: another shard may have just taken one + except FileNotFoundError: + return None + for entry in entries: + try: + os.rename(os.path.join(pending_dir, entry), os.path.join(inflight_dir, entry)) + except FileNotFoundError: + continue # lost the race for this entry; try the next + return entry.replace("__", "/") + return None + + def mark_done(self, test_path: str) -> None: + """Move a finished claim from ``inflight//`` to ``done//``.""" + entry = self._slug(test_path) + done_dir = os.path.join(self._dir, "done", self._shard) + os.makedirs(done_dir, exist_ok=True) + # Suppress: already moved, or the runner died before marking — the reconciler catches that. + with contextlib.suppress(FileNotFoundError): + os.rename(os.path.join(self._dir, "inflight", self._shard, entry), os.path.join(done_dir, entry)) + + def iter(self, files: list[str]): + """Yield files to run: claimed from the queue when active, else ``files`` as-is.""" + if not self.active: + yield from files + return + while (claimed := self._claim()) is not None: + yield claimed + + +class Collector: + """Discover the test files to run, applying filters and file-level sharding.""" + + def __init__(self, config: RunnerConfig): + self._config = config + + def _source_dirs(self) -> list[str]: + """The roots walked for ``test_*.py`` files.""" + return [os.path.join(self._config.workspace_root, d) for d in ("scripts", "source")] + + def _selected(self, file_name: str, full_path: str) -> bool: + """Whether one ``test_*.py`` passes the active mode/skip/pattern filters.""" + config = self._config + if config.quarantined_only: + return file_name in test_settings.QUARANTINED_TESTS + if config.curobo_only: + return file_name in test_settings.CUROBO_TESTS + # An explicit include overrides the skip list (dedicated jobs run otherwise-skipped tests). + if file_name in test_settings.TESTS_TO_SKIP and file_name not in config.include_files: + return False + if config.filter_pattern and config.filter_pattern not in full_path: + return False + if config.exclude_pattern and any(p.strip() in full_path for p in config.exclude_pattern.split(",")): + return False + if config.include_files and file_name not in config.include_files: + return False + return True + + def _sharded(self, files: list[str]) -> list[str]: + """Select every Nth file for this file-shard; a no-op unless both shard env are set.""" + config = self._config + if config.shard_index is None or config.shard_count is None or config.include_files: + return files + files = sorted(files) + selected = [f for i, f in enumerate(files) if i % config.shard_count == config.shard_index] + print(f"Shard {config.shard_index}/{config.shard_count}: selected {len(selected)} test files") + return selected + + def _isaacsim_ci_only(self, files: list[str]) -> list[str]: + """Keep only files containing an ``@pytest.mark.isaacsim_ci`` test.""" + kept = [] + for test_file in files: + with open(test_file) as f: + if "@pytest.mark.isaacsim_ci" in f.read(): + kept.append(test_file) + return kept + + def discover(self) -> list[str]: + """Walk the source roots and return the filtered, sharded list of test files.""" + files = [] + for source_dir in self._source_dirs(): + if not os.path.exists(source_dir): + print(f"Error: source directory not found at {source_dir}") + pytest.exit("Source directory not found", returncode=1) + for root, _, names in os.walk(source_dir): + # install_ci has its own config/conftest and runs via its own action. + if "install_ci" in root.replace("\\", "/").split("/"): + continue + for name in names: + if name.startswith("test_") and name.endswith(".py"): + full_path = os.path.join(root, name) + if self._selected(name, full_path): + files.append(full_path) + files = self._sharded(files) + if self._config.isaacsim_ci: + files = self._isaacsim_ci_only(files) + return files + + +class Reporter: + """Merge per-unit JUnit reports and render the run summary.""" + + def __init__(self, config: RunnerConfig): + self._config = config + + @property + def _report_path(self) -> str: + return os.path.join(self._config.report_dir, self._config.result_file) + + def write_empty(self) -> None: + """Write an empty aggregate report so downstream CI steps find a valid file.""" + os.makedirs(self._config.report_dir, exist_ok=True) + JUnitXml().write(self._report_path) + print(f"Wrote empty report to {self._report_path}") + + @staticmethod + def _counts(files: list[str], status: dict) -> dict: + """Tally per-result-class file counts plus total wall/test time.""" + + def n(result): + return len([f for f in files if status[f]["result"] == result]) + + return { + "total": len(status), + "passing": len([f for f in files if status[f]["result"].startswith("passed")]), + "failing": n("FAILED"), + "timeout": n("TIMEOUT"), + "crashed": n("CRASHED"), + "startup_hang": n("STARTUP_HANG"), + "wall": sum(status[f]["wall_time"] for f in files), + "test": sum(status[f]["time_elapsed"] for f in files), + } + + def _per_file_table(self, files: list[str], status: dict) -> str: + """One row per file: path, GPU, result, times, pass/total.""" + table = PrettyTable(field_names=["Test Path", "GPU", "Result", "Test (s)", "Wall (s)", "# Tests"]) + table.align["Test Path"] = "l" + table.align["Test (s)"] = "r" + table.align["Wall (s)"] = "r" + for f in files: + s = status[f] + passed = s["tests"] - s["failures"] - s["errors"] - s["skipped"] + table.add_row( + [ + f, + self._config.sim_device, + s["result"], + f"{s['time_elapsed']:0.2f}", + f"{s['wall_time']:0.2f}", + f"{passed}/{s['tests']}", + ] + ) + return table.get_string() + + def _per_test_time_table(self, full_report: JUnitXml) -> str: + """Per-test run times, slowest first; device read from the test id, else the boot device.""" + table = PrettyTable(field_names=["Test", "Device", "Time (s)"]) + table.align["Test"] = "l" + table.align["Time (s)"] = "r" + rows = [] + for suite in full_report: + for case in suite: + name = f"{case.classname}::{case.name}" if case.classname else case.name + device = self._config.sim_device + bracket = re.search(r"\[(.*)\]", name) + if bracket and (dev := re.search(r"cuda:\d+|\bcpu\b", bracket.group(1))): + device = dev.group(0) + rows.append((name, device, float(case.time) if case.time is not None else 0.0)) + for name, device, elapsed in sorted(rows, key=lambda r: r[2], reverse=True): + table.add_row([name, device, f"{elapsed:0.3f}"]) + return table.get_string() + + def _summary(self, files: list[str], status: dict, counts: dict, full_report: JUnitXml) -> str: + """Assemble the printed summary: result tally, per-file table, per-test times.""" + pct = 100 if counts["total"] == 0 else counts["passing"] / counts["total"] * 100 + wall, test = counts["wall"], counts["test"] + lines = [ + "\n\n===================\nTest Result Summary\n===================", + f"Total: {counts['total']}", + f"Passing: {counts['passing']}", + f"Failing: {counts['failing']}", + f"Crashed: {counts['crashed']}", + f"Startup Hang: {counts['startup_hang']}", + f"Timeout: {counts['timeout']}", + f"Passing Percentage: {pct:.2f}%", + f"Total Wall Time: {wall // 3600:.0f}h{wall // 60 % 60:.0f}m{wall % 60:.2f}s", + f"Total Test Time: {test // 3600:.0f}h{test // 60 % 60:.0f}m{test % 60:.2f}s", + "\n=======================\nPer File Result Summary\n=======================", + self._per_file_table(files, status), + "\n=================\nPer Test Run Time\n=================", + self._per_test_time_table(full_report), + ] + return "\n".join(lines) + + def finalize(self, files: list[str], status: dict, xml_reports: list) -> int: + """Merge reports, write the aggregate, print the summary, return the exit code. + + Returns 0 when every file passed, else 1. + """ + print("~~~~~~~~~ Collecting final report...") + # Merge the in-memory report objects (re-reading the files risks dropping + # elements through the junitparser round-trip). + full_report = JUnitXml() + for report in xml_reports: + full_report += report + os.makedirs(self._config.report_dir, exist_ok=True) + full_report.write(self._report_path) + print("~~~~~~~~~~~~ Report written to", self._report_path) + + counts = self._counts(files, status) + print(self._summary(files, status, counts, full_report)) + clean = counts["failing"] == counts["timeout"] == counts["crashed"] == counts["startup_hang"] == 0 + return 0 if clean else 1 + + +class Session: + """The per-file test-run lifecycle, driven by ``conftest.pytest_sessionstart``.""" + + def __init__(self, config: RunnerConfig): + self._config = config + self._collector = Collector(config) + self._queue = WorkQueue(config) + self._planner = Planner(config.runtime_mask) + self._runner = UnitRunner(config) + self._reporter = Reporter(config) + self._cold_cache_applied = False + + def _exec_context(self, file_name: str, source: str) -> ExecContext: + """Per-file execution context, applying the one-time cold-cache timeout bump.""" + config = self._config + timeout = config.per_file_timeouts.get(file_name, config.default_timeout) + # The first camera-enabled test compiles shaders on a cold cache (~600 s); + # give it (and only it) extra time so that isn't misread as a hang. + is_cold = not self._cold_cache_applied and config.cold_cache_marker in source + if is_cold: + timeout += config.cold_cache_buffer + self._cold_cache_applied = True + print(f"⏱️ Adding {config.cold_cache_buffer}s cold-cache buffer (timeout now {timeout}s)") + startup_deadline = min(timeout, config.startup_deadline + (config.cold_cache_buffer if is_cold else 0)) + env = os.environ.copy() + env["PYTHONFAULTHANDLER"] = "1" + return ExecContext(file_name=file_name, timeout=timeout, startup_deadline=startup_deadline, env=env) + + def _run_file(self, test_file: str, status: dict, failed: list, reports: list) -> None: + """Plan ``test_file`` into units, run each, and fold their statuses into ``status``.""" + print(f"\n\n🚀 Running {test_file} independently...\n") + try: + with open(test_file) as fh: + source = fh.read() + except OSError: + source = "" + ctx = self._exec_context(os.path.basename(test_file), source) + units = self._planner.plan(test_file, source) + if len(units) > 1: + print(f"⚙️ device_isolated — invoking {ctx.file_name} once per device ({len(units)} units)") + merged = None + for unit in units: + report, unit_status, was_failure = self._runner.run(unit, ctx) + if report is not None: + reports.append(report) + if was_failure and test_file not in failed: + failed.append(test_file) + merged = merge_statuses(merged, unit_status) + status[test_file] = merged + + def run(self) -> int: + """Run the lifecycle and return the process exit code. + + Returns: + 0 when nothing failed (including a clean "nothing to run"), else 1. + """ + files = self._collector.discover() + if not files: + # A configured-but-empty scope is success; a bare empty run is an error. + if self._config.quarantined_only or self._config.filter_pattern: + print("No tests in scope — nothing to run.") + self._reporter.write_empty() + return 0 + print("No test files found in source directory") + return 1 + + print(f"Found {len(files)} test files after filtering") + status, failed, reports = {}, [], [] + for test_file in self._queue.iter(files): + self._run_file(test_file, status, failed, reports) + if self._queue.active: + self._queue.mark_done(test_file) + print("~~~~~~~~~~~~ Finished running all tests") + print("failed tests:", failed) + + # In work-queue mode this container ran only the files it claimed; report on those. + reported_files = list(status) if self._queue.active else files + return self._reporter.finalize(reported_files, status, reports) diff --git a/tools/test_runner/tests/__init__.py b/tools/test_runner/tests/__init__.py new file mode 100644 index 000000000000..440ba074bf57 --- /dev/null +++ b/tools/test_runner/tests/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for the test_runner framework (run locally; see package docstring).""" diff --git a/tools/test_runner/tests/test_execution.py b/tools/test_runner/tests/test_execution.py new file mode 100644 index 000000000000..3feaa2f30b0f --- /dev/null +++ b/tools/test_runner/tests/test_execution.py @@ -0,0 +1,94 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.execution: the unit command contract + status merge.""" + +from __future__ import annotations + +import os + +from test_runner.execution import ExecContext, UnitRunner, merge_statuses +from test_runner.planning import RunnerConfig, Unit + + +def _runner(isaacsim_ci=False): + return UnitRunner(RunnerConfig(workspace_root="/ws", isaacsim_ci=isaacsim_ci)) + + +def _ctx(): + return ExecContext(file_name="test_x.py", timeout=60, startup_deadline=30, env={"BASE": "1"}) + + +def _unit(mask, split=False): + return Unit("source/pkg/test_x.py", mask, split=split) + + +def test_mix_ok_unit_sets_mask_no_selector_no_k(): + cmd, env, report, label, suffix = _runner()._build_cmd(_unit("110"), _ctx()) + assert env["ISAACLAB_TEST_DEVICES"] == "110" + assert env["BASE"] == "1" # base env preserved + assert "test_runner.selector" not in cmd + assert "-k" not in cmd + assert report == "tests/test-reports-source__pkg__test_x.py.xml" + assert label == "test_x.py" + assert suffix == "" + assert cmd[-1] == "source/pkg/test_x.py" + + +def test_cpu_split_unit_injects_selector_and_is_suffixed(): + cmd, _, report, _, suffix = _runner()._build_cmd(_unit("100", split=True), _ctx()) + assert "test_runner.selector" in cmd # cpu half still drops out-of-scope cuda variants + assert suffix == "-100" + assert report.endswith("test_x.py-100.xml") + + +def test_cuda_split_unit_injects_selector_and_tools_pythonpath(): + cmd, env, _, _, _ = _runner()._build_cmd(_unit("010", split=True), _ctx()) + assert cmd.count("-p") == 1 and "test_runner.selector" in cmd + assert env["ISAACLAB_TEST_DEVICES"] == "010" + assert env["PYTHONPATH"].split(os.pathsep)[0] == os.path.join("/ws", "tools") + + +def test_shard_unit_injects_selector_without_a_suffix(): + cmd, env, report, _, suffix = _runner()._build_cmd(_unit("0001"), _ctx()) + assert "test_runner.selector" in cmd # cpu-less shard + assert suffix == "" + assert report == "tests/test-reports-source__pkg__test_x.py.xml" + + +def test_isaacsim_ci_adds_marker_selector(): + cmd, *_ = _runner(isaacsim_ci=True)._build_cmd(_unit("110"), _ctx()) + assert any(cmd[i] == "-m" and cmd[i + 1] == "isaacsim_ci" for i in range(len(cmd) - 1)) + + +def test_no_k_selector_for_any_mask(): + for mask, split in [("110", False), ("100", True), ("010", True), ("0001", False)]: + cmd, *_ = _runner()._build_cmd(_unit(mask, split=split), _ctx()) + assert "-k" not in cmd + + +def test_merge_statuses_sums_counters_and_takes_worst_result(): + a = { + "errors": 0, + "failures": 0, + "skipped": 1, + "tests": 3, + "time_elapsed": 1.0, + "wall_time": 2.0, + "result": "passed", + } + b = { + "errors": 1, + "failures": 0, + "skipped": 0, + "tests": 2, + "time_elapsed": 0.5, + "wall_time": 1.0, + "result": "FAILED", + } + merged = merge_statuses(a, b) + assert merged["tests"] == 5 and merged["skipped"] == 1 and merged["errors"] == 1 + assert merged["result"] == "FAILED" # more severe wins + assert merge_statuses(None, b) == b # first unit seeds the entry diff --git a/tools/test_runner/tests/test_planning.py b/tools/test_runner/tests/test_planning.py new file mode 100644 index 000000000000..359aeb6c65ff --- /dev/null +++ b/tools/test_runner/tests/test_planning.py @@ -0,0 +1,82 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.planning (Planner + RunnerConfig).""" + +from __future__ import annotations + +import pytest + +from test_runner.planning import Planner, RunnerConfig, Unit + +_ISOLATED = "import pytest\npytestmark = pytest.mark.device_isolated\n" + + +# ---- Planner.plan ---- + + +def test_mix_ok_file_is_a_single_unit(): + assert Planner("110").plan("a.py", "def test_x(): pass\n") == [Unit("a.py", "110")] + + +def test_isolated_file_splits_on_a_multi_device_run(): + assert Planner("110").plan("a.py", _ISOLATED) == [Unit("a.py", "100", split=True), Unit("a.py", "010", split=True)] + + +def test_isolated_file_does_not_split_on_a_single_device_run(): + assert Planner("0001").plan("a.py", _ISOLATED) == [Unit("a.py", "0001")] + + +def test_split_uses_each_set_bit_position(): + assert Planner("1010").plan("a.py", _ISOLATED) == [ + Unit("a.py", "1000", split=True), + Unit("a.py", "0010", split=True), + ] + + +def test_runtime_mask_with_trailing_x_is_rejected(): + with pytest.raises(ValueError): + Planner("11X") + + +# ---- Planner.is_isolated ---- + + +def test_is_isolated_detects_single_and_list_forms(): + p = Planner("110") + assert p.is_isolated("pytestmark = pytest.mark.device_isolated") is True + assert p.is_isolated("pytestmark = [pytest.mark.device_isolated, pytest.mark.slow]") is True + + +def test_is_isolated_ignores_comments_and_other_marks(): + p = Planner("110") + assert p.is_isolated("# device_isolated explains the lock") is False + assert p.is_isolated("pytestmark = pytest.mark.slow") is False + + +# ---- RunnerConfig ---- + + +def test_config_defaults_and_derived_paths(): + c = RunnerConfig(workspace_root="/ws") + assert c.config_file == "/ws/pyproject.toml" + assert c.tools_dir == "/ws/tools" + assert c.runtime_mask == "110" # single-GPU default + assert c.process_failure_retries["test_visualizer_integration_physx.py"] == 4 + + +def test_from_env_reads_mask_flags_and_includes(monkeypatch): + monkeypatch.setenv("ISAACLAB_TEST_DEVICES", "0001") + monkeypatch.setenv("ISAACSIM_CI_SHORT", "true") + monkeypatch.setenv("TEST_INCLUDE_FILES", "a/b/test_x.py, test_y.py") + c = RunnerConfig.from_env("/ws") + assert c.runtime_mask == "0001" + assert c.isaacsim_ci is True + assert c.include_files == frozenset({"test_x.py", "test_y.py"}) + + +def test_from_env_defaults_mask_when_unset(monkeypatch): + monkeypatch.delenv("ISAACLAB_TEST_DEVICES", raising=False) + assert RunnerConfig.from_env("/ws").runtime_mask == "110" diff --git a/tools/test_runner/tests/test_selector.py b/tools/test_runner/tests/test_selector.py new file mode 100644 index 000000000000..e3011b4e0126 --- /dev/null +++ b/tools/test_runner/tests/test_selector.py @@ -0,0 +1,112 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.selector.DeviceSelect. + +The class form is directly testable: construct DeviceSelect(mask) and call keeps() +or the hooks with fakes — no env var, no monkeypatch. +""" + +from __future__ import annotations + +import pytest + +from test_runner.selector import DeviceSelect + + +class _Callspec: + def __init__(self, params): + self.params = params + + +class _Item: + def __init__(self, name, params=None): + self.name = name + if params is not None: + self.callspec = _Callspec(params) + + +class _Hook: + def __init__(self): + self.deselected = [] + + def pytest_deselected(self, items): + self.deselected.extend(items) + + +class _Config: + def __init__(self): + self.hook = _Hook() + + +# ---- pure mask logic ---- + + +def test_position_maps_cpu_and_cuda(): + assert DeviceSelect._position("cpu") == 0 + assert DeviceSelect._position("cuda:0") == 1 + assert DeviceSelect._position("cuda:2") == 3 + assert DeviceSelect._position(None) is None + + +def test_active_handles_trailing_x(): + assert DeviceSelect("00X")._active(3) is True # cuda:2 via the trailing fill + assert DeviceSelect("110")._active(2) is False + + +def test_keeps_cpu_unit(): + s = DeviceSelect("100") + assert s.keeps("cpu") is True + assert s.keeps("cuda:0") is False # literal cuda variant dropped from the cpu unit + assert s.keeps(None) is True # agnostic kept (cpu in mask) + + +def test_keeps_cuda_unit(): + s = DeviceSelect("010") + assert s.keeps("cuda:0") is True + assert s.keeps("cpu") is False # literal cpu variant dropped from the cuda unit + assert s.keeps(None) is False # agnostic dropped (no cpu) + + +def test_keeps_shard(): + s = DeviceSelect("0001") # cuda:2 shard + assert s.keeps("cuda:2") is True + assert s.keeps("cuda:0") is False + assert s.keeps("cpu") is False + assert s.keeps(None) is False + + +# ---- the collection hook ---- + + +def test_modifyitems_drops_out_of_scope_and_agnostic_in_cuda_unit(): + cpu = _Item("t[cpu]", {"device": "cpu"}) + cuda = _Item("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _Item("t_logic") + items = [cpu, cuda, agnostic] + config = _Config() + DeviceSelect("010").pytest_collection_modifyitems(config, items) + assert items == [cuda] + assert set(config.hook.deselected) == {cpu, agnostic} + + +def test_modifyitems_keeps_everything_in_scope_for_default_mask(): + cpu = _Item("t[cpu]", {"device": "cpu"}) + cuda = _Item("t[cuda:0]", {"device": "cuda:0"}) + agnostic = _Item("t_logic") + items = [cpu, cuda, agnostic] + config = _Config() + DeviceSelect("110").pytest_collection_modifyitems(config, items) + assert items == [cpu, cuda, agnostic] + assert config.hook.deselected == [] + + +def test_sessionfinish_maps_no_tests_collected_to_ok(): + class _Session: + exitstatus = pytest.ExitCode.NO_TESTS_COLLECTED + + session = _Session() + DeviceSelect("010").pytest_sessionfinish(session, pytest.ExitCode.NO_TESTS_COLLECTED) + assert session.exitstatus == pytest.ExitCode.OK diff --git a/tools/test_runner/tests/test_session.py b/tools/test_runner/tests/test_session.py new file mode 100644 index 000000000000..9133b3239f1d --- /dev/null +++ b/tools/test_runner/tests/test_session.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit tests for test_runner.session: WorkQueue, Collector filters, Reporter, cold-cache.""" + +from __future__ import annotations + +from test_runner.planning import RunnerConfig +from test_runner.session import Collector, Reporter, Session, WorkQueue + + +def _status(result, *, tests=1, failures=0, errors=0, skipped=0, wall=1.0, elapsed=0.5): + return { + "result": result, + "tests": tests, + "failures": failures, + "errors": errors, + "skipped": skipped, + "wall_time": wall, + "time_elapsed": elapsed, + } + + +# ---- WorkQueue ---- + + +def test_workqueue_inactive_iterates_the_given_files(): + wq = WorkQueue(RunnerConfig(workspace_root="/ws")) # no queue_path + assert wq.active is False + assert list(wq.iter(["a.py", "b.py"])) == ["a.py", "b.py"] + + +def test_workqueue_claim_then_mark_done_roundtrip(tmp_path): + config = RunnerConfig(workspace_root="/ws", queue_path=str(tmp_path), sim_device="cuda:1") + (tmp_path / "queue").mkdir() + (tmp_path / "queue" / "src__pkg__test_x.py").write_text("") + wq = WorkQueue(config) + assert wq.active is True + claimed = list(wq.iter([])) + assert claimed == ["src/pkg/test_x.py"] # decoded path + assert (tmp_path / "inflight" / "cuda-1" / "src__pkg__test_x.py").exists() + wq.mark_done("src/pkg/test_x.py") + assert (tmp_path / "done" / "cuda-1" / "src__pkg__test_x.py").exists() + assert not (tmp_path / "inflight" / "cuda-1" / "src__pkg__test_x.py").exists() + + +# ---- Collector filters ---- + + +def test_collector_selects_by_filter_and_exclude_pattern(): + c = Collector(RunnerConfig(workspace_root="/ws", filter_pattern="physx")) + assert c._selected("test_zzz_synthetic.py", "/a/physx/test_zzz_synthetic.py") is True + assert c._selected("test_zzz_synthetic.py", "/a/newton/test_zzz_synthetic.py") is False + + c = Collector(RunnerConfig(workspace_root="/ws", exclude_pattern="newton")) + assert c._selected("test_zzz_synthetic.py", "/a/newton/test_zzz_synthetic.py") is False + + +def test_collector_respects_include_files(): + c = Collector(RunnerConfig(workspace_root="/ws", include_files=frozenset({"test_keep.py"}))) + assert c._selected("test_keep.py", "/a/test_keep.py") is True + assert c._selected("test_drop.py", "/a/test_drop.py") is False + + +# ---- Reporter counts ---- + + +def test_reporter_counts_tally_results_and_time(): + files = ["a.py", "b.py", "c.py"] + status = {"a.py": _status("passed"), "b.py": _status("FAILED"), "c.py": _status("passed (shutdown hanged)")} + counts = Reporter._counts(files, status) + assert counts["total"] == 3 + assert counts["passing"] == 2 # both "passed*" results + assert counts["failing"] == 1 + assert counts["wall"] == 3.0 + + +# ---- Session cold-cache timeout policy ---- + + +def test_cold_cache_buffer_applies_once_to_the_first_camera_file(): + config = RunnerConfig(workspace_root="/ws", default_timeout=600, cold_cache_buffer=700, startup_deadline=120) + session = Session(config) + first = session._exec_context("test_cam.py", "enable_cameras=True") + assert first.timeout == 1300 # 600 + 700 + assert first.startup_deadline == 820 # min(1300, 120 + 700) + second = session._exec_context("test_cam2.py", "enable_cameras=True") + assert second.timeout == 600 # buffer already spent + assert second.startup_deadline == 120 + + +def test_per_file_timeout_override(): + config = RunnerConfig(workspace_root="/ws", default_timeout=600, per_file_timeouts={"test_slow.py": 1200}) + ctx = Session(config)._exec_context("test_slow.py", "") + assert ctx.timeout == 1200