[CI] Cross-platform — Part 4: Multi-GPU pytest workflow with cuda:1 coverage

.github/actions/multi-gpu/aggregate_test_summary.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""Aggregate per-shard JUnit XML reports from the multi-GPU pytest lane.
+
+Reads ``RUNTIME_DIR`` (the host launcher's work-queue root, holding
+``queue/done/<shard>/<slug>``) and the ``tests/test-reports-*.xml`` JUnit files,
+then prints a per-file + per-shard table to stdout and, when
+``GITHUB_STEP_SUMMARY`` is set, writes the same content as markdown so it
+renders at the top of the run page.
+"""
+
+import glob
+import os
+import pathlib
+import xml.etree.ElementTree as ET
+
+runtime_dir = os.environ.get("RUNTIME_DIR", "")
+done_root = pathlib.Path(runtime_dir) / "queue" / "done" if runtime_dir else None
+
+# Map slug -> shard from done/<shard>/<slug>; slug uses "__" for "/".
+slug_to_shard = {}
+if done_root and done_root.is_dir():
+    for shard_dir in done_root.iterdir():
+        if not shard_dir.is_dir():
+            continue
+        shard = shard_dir.name  # e.g. "cuda-1"
+        for entry in shard_dir.iterdir():
+            slug_to_shard[entry.name] = shard
+
+
+def unslug(name: str) -> str:
+    # report file pattern: test-reports-<slug>.xml
+    if name.startswith("test-reports-"):
+        name = name[len("test-reports-") :]
+    if name.endswith(".xml"):
+        name = name[:-4]
+    return name  # this is the slug; the file path is slug.replace("__", "/")
+
+
+# Per-file rows from JUnit XMLs.
+rows = []
+for path in sorted(glob.glob("tests/test-reports-*.xml")):
+    slug = unslug(os.path.basename(path))
+    file_path = slug.replace("__", "/")
+    shard = slug_to_shard.get(slug, "?")
+    try:
+        tree = ET.parse(path)
+        root = tree.getroot()
+    except ET.ParseError:
+        rows.append((file_path, shard, "PARSE-ERROR", 0, 0, 0, 0, 0.0))
+        continue
+    tests = failures = errors = skipped = 0
+    total_time = 0.0
+    for ts in root.iter("testsuite"):
+        tests += int(ts.get("tests", 0))
+        failures += int(ts.get("failures", 0))
+        errors += int(ts.get("errors", 0))
+        skipped += int(ts.get("skipped", 0))
+        total_time += float(ts.get("time", 0) or 0)
+    passed = tests - failures - errors - skipped
+    status = "PASS"
+    if failures + errors > 0:
+        status = "FAIL"
+    elif tests == 0:
+        status = "EMPTY"
+    rows.append((file_path, shard, status, passed, tests, skipped, failures + errors, total_time))
+
+# Per-shard aggregate.
+shards = {}
+for fp, sh, st, p, t, sk, f, tm in rows:
+    if sh not in shards:
+        shards[sh] = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0}
+    s = shards[sh]
+    s["files"] += 1
+    s["passed"] += p
+    s["tests"] += t
+    s["skipped"] += sk
+    s["failed"] += f
+    s["time"] += tm
+
+
+# ---------- Output: plain text to stdout ----------
+def fmt_row(cols, widths):
+    return " | ".join(c.ljust(w) for c, w in zip(cols, widths))
+
+
+out = []
+out.append("PER-FILE RESULTS")
+out.append("=" * 110)
+hdr = ["File", "Shard", "Status", "Pass/Total", "Skip", "Fail", "Time(s)"]
+widths = [60, 7, 8, 12, 5, 5, 8]
+out.append(fmt_row(hdr, widths))
+out.append("-" * 110)
+for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])):
+    out.append(fmt_row([fp[-60:], sh, st, f"{p}/{t}", str(sk), str(f), f"{tm:.1f}"], widths))
+out.append("")
+out.append("PER-SHARD AGGREGATE")
+out.append("=" * 80)
+shard_hdr = ["Shard", "Files", "Pass/Total", "Skipped", "Failed", "Test time (s)"]
+shard_widths = [10, 7, 12, 9, 8, 14]
+out.append(fmt_row(shard_hdr, shard_widths))
+out.append("-" * 80)
+grand = {"files": 0, "passed": 0, "tests": 0, "skipped": 0, "failed": 0, "time": 0.0}
+for sh in sorted(shards):
+    s = shards[sh]
+    for k in grand:
+        grand[k] += s[k]
+    out.append(
+        fmt_row(
+            [
+                sh,
+                str(s["files"]),
+                f"{s['passed']}/{s['tests']}",
+                str(s["skipped"]),
+                str(s["failed"]),
+                f"{s['time']:.1f}",
+            ],
+            shard_widths,
+        )
+    )
+out.append("-" * 80)
+out.append(
+    fmt_row(
+        [
+            "TOTAL",
+            str(grand["files"]),
+            f"{grand['passed']}/{grand['tests']}",
+            str(grand["skipped"]),
+            str(grand["failed"]),
+            f"{grand['time']:.1f}",
+        ],
+        shard_widths,
+    )
+)
+print("\n".join(out))
+
+# ---------- Output: markdown to $GITHUB_STEP_SUMMARY ----------
+summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+if summary_path:
+    md = [
+        "## Multi-GPU CI: Aggregated Test Summary",
+        "",
+        "### Per-shard totals",
+        "",
+        "| Shard | Files | Pass/Total | Skipped | Failed | Test time (s) |",
+        "|---|---:|---:|---:|---:|---:|",
+    ]
+    for sh in sorted(shards):
+        s = shards[sh]
+        md.append(
+            f"| `{sh}` | {s['files']} | {s['passed']}/{s['tests']} | {s['skipped']} | {s['failed']} | {s['time']:.1f} |"
+        )
+    md.append(
+        f"| **TOTAL** | **{grand['files']}** | **{grand['passed']}/{grand['tests']}** | "
+        f"**{grand['skipped']}** | **{grand['failed']}** | **{grand['time']:.1f}** |"
+    )
+    md.append("")
+    md.append("### Per-file results")
+    md.append("")
+    md.append("| File | Shard | Status | Pass/Total | Skipped | Failed | Test time (s) |")
+    md.append("|---|---|---|---:|---:|---:|---:|")
+    for fp, sh, st, p, t, sk, f, tm in sorted(rows, key=lambda r: (r[1], r[0])):
+        status_md = {
+            "PASS": ":white_check_mark: PASS",
+            "FAIL": ":x: FAIL",
+            "EMPTY": ":white_circle: EMPTY",
+            "PARSE-ERROR": ":warning: PARSE-ERROR",
+        }.get(st, st)
+        md.append(f"| `{fp}` | `{sh}` | {status_md} | {p}/{t} | {sk} | {f} | {tm:.1f} |")
+    md.append("")
+    with open(summary_path, "a") as fh:
+        fh.write("\n".join(md) + "\n")

.github/actions/multi-gpu/mgpu_shard_select.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md).
+# All rights reserved.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+"""pytest plugin: select which tests a non-default GPU shard runs.
+
+Loaded only by the multi-GPU lane: ``tools/conftest.py`` injects it via
+``-p mgpu_shard_select`` into each per-file pytest subprocess (and prepends this
+directory to ``PYTHONPATH`` so it is importable). It lives next to the lane
+scripts rather than as a repo-root ``conftest.py`` so it only affects the lane.
+
+Signal: ``ISAACLAB_TEST_DEVICES`` -- the runtime device mask. This is the SAME
+env var ``isaaclab.test.utils.devices.test_devices()`` reads to decide a test's
+device parametrization, so this plugin's keep/drop decision agrees with what was
+parametrized. Kit-backed tests derive their AppLauncher device from the same mask.
+The plugin reads the mask string directly -- no isaaclab import -- matching the
+lib-free convention of every other conftest in the repo. The only shared knowledge
+is the mask grammar below, mirrored from ``devices.py`` (position 0 = cpu,
+position k = cuda:(k-1), trailing ``X`` = include all remaining); keep the two in sync.
+
+Selection rule, on a shard whose mask excludes cpu + cuda:0:
+
+    keep a test iff it is parametrized over ``device`` AND that variant's device
+    is active in the mask; deselect everything else.
+
+That single rule is both criteria at once: candidate = device-parametrized (a
+test with no ``device`` param is device-agnostic, covered by single-GPU CI on
+cuda:0); scope = only this shard's device variant runs (cpu / cuda:0 / other-index
+variants are dropped). Out-of-scope items are deselected (not skipped) so the
+shard report shows only what ran.
+"""
+
+import os
+
+import pytest
+
+_RUNTIME_ENV = "ISAACLAB_TEST_DEVICES"
+
+
+def _position(device):
+    """Mask position for a device value: cpu -> 0, cuda:k -> k + 1 (else None)."""
+    if device == "cpu":
+        return 0
+    if isinstance(device, str) and device.startswith("cuda:") and device[len("cuda:") :].isdigit():
+        return int(device[len("cuda:") :]) + 1
+    return None  # not device-parametrized, or an unrecognized device value
+
+
+def _active(position, mask):
+    """Whether a mask position is included. Trailing ``X`` includes all remaining."""
+    if position is None:
+        return False
+    body, fill = (mask[:-1], True) if mask.endswith("X") else (mask, False)
+    return body[position] == "1" if 0 <= position < len(body) else fill
+
+
+def _shard_mask():
+    """Return the runtime mask iff this is a multi-GPU shard run, else None.
+
+    A shard targets non-default GPUs only, so cpu (position 0) and cuda:0
+    (position 1) are excluded. Anything else -- unset, or single-GPU CI's default
+    ``"110"`` -- is not a shard and the plugin is a no-op.
+    """
+    mask = os.environ.get(_RUNTIME_ENV, "")
+    return mask if mask and not _active(0, mask) and not _active(1, mask) else None
+
+
+def pytest_collection_modifyitems(config, items):
+    mask = _shard_mask()
+    if mask is None:
+        return
+    keep, drop = [], []
+    for item in items:
+        callspec = getattr(item, "callspec", None)
+        device = callspec.params.get("device") if callspec is not None else None
+        if _active(_position(device), mask):
+            keep.append(item)  # device-parametrized AND in this shard's scope
+        else:
+            drop.append(item)  # non-device, or cpu / cuda:0 / other-index variant
+    if drop:
+        items[:] = keep
+        config.hook.pytest_deselected(items=drop)
+
+
+def pytest_sessionfinish(session, exitstatus):
+    # A file whose tests are all out of scope deselects to zero, so pytest exits
+    # NO_TESTS_COLLECTED (5). The lane orchestrator treats any non-zero per-file
+    # exit as a failure (tools/conftest.py), so report "nothing in scope for this
+    # file" as success rather than a false failure.
+    if _shard_mask() is not None and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED:
+        session.exitstatus = pytest.ExitCode.OK