diff --git a/.github/actions/windows-instance-state/action.yml b/.github/actions/windows-instance-state/action.yml new file mode 100644 index 000000000000..ead6e994436d --- /dev/null +++ b/.github/actions/windows-instance-state/action.yml @@ -0,0 +1,100 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Windows instance state report + minimum cleanup' +description: >- + Print disk + relevant directory sizes on the Windows runner before and after + a job, and (in 'post' phase) wipe non-cache user state so the runner is left + net-zero except for content-addressed wheel caches (uv, pip). + See the per-dir comments below for what's cleaned and why. + +inputs: + phase: + description: '"pre" (report only) or "post" (report → cleanup → report).' + required: true + +runs: + using: composite + steps: + - name: Report and optionally clean + shell: powershell + run: | + $ErrorActionPreference = "Continue" + + # Directories we observe on every run. uv/pip are content-addressed + # wheel caches (safe across runs, big speedup). The rest is user state + # that can chain bad behavior between runs and is cleaned in 'post'. + $observed = [ordered]@{ + "uv cache" = "$env:LOCALAPPDATA\uv\cache" # KEEP — content-addressed + "pip cache" = "$env:LOCALAPPDATA\pip\Cache" # KEEP — content-addressed + "Kit shader cache" = "$env:LOCALAPPDATA\NVIDIA\Omniverse" # KEEP — invalidated by Kit on version mismatch + "Kit user state" = "$env:APPDATA\NVIDIA Corporation\Omniverse Kit" # CLEAN — settings + last-used renderer chain + "Kit docs" = "$env:USERPROFILE\Documents\Kit" # CLEAN — recent files / persistent_state + "user site-pkgs" = "$env:APPDATA\Python\Python312\site-packages" # observe — escaped pip --user installs + } + + function Show-State($label) { + Write-Host "=== Windows instance state: $label ===" + Get-PSDrive C | + Select-Object @{n='Drive';e={$_.Name}}, + @{n='Used GB';e={[math]::Round($_.Used/1GB,1)}}, + @{n='Free GB';e={[math]::Round($_.Free/1GB,1)}} | + Format-Table + # GPU presence check via nvidia-smi. Surfaces "is there a GPU at all" + # before any Kit boot, so a Vulkan failure can be classified as + # "no GPU on the runner" vs "GPU present but Vulkan/driver issue". + $nvsmi = & nvidia-smi --query-gpu=name,driver_version,memory.total --format=csv,noheader 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "nvidia-smi: $nvsmi" + } else { + Write-Host "nvidia-smi: not available or no GPU detected (exit=$LASTEXITCODE)" + } + foreach ($k in $observed.Keys) { + $p = $observed[$k] + if (Test-Path $p) { + $s = (Get-ChildItem -Recurse -File -ErrorAction SilentlyContinue $p | + Measure-Object -Sum Length -ErrorAction SilentlyContinue).Sum + if ($null -eq $s) { $s = 0 } + "{0,-20} {1,10:N1} MB ({2})" -f $k, ($s/1MB), $p + } else { + "{0,-20} {1,10} ({2})" -f $k, "(absent)", $p + } + } + } + + if ("${{ inputs.phase }}" -eq "pre") { + Show-State "BEFORE" + exit 0 + } + + # phase == post: report → minimum cleanup → report + Show-State "AFTER (pre-cleanup)" + + $toRemove = @( + "$env:APPDATA\NVIDIA Corporation\Omniverse Kit", + "$env:USERPROFILE\Documents\Kit" + ) + foreach ($p in $toRemove) { + if (Test-Path $p) { + Remove-Item -LiteralPath $p -Recurse -Force -ErrorAction SilentlyContinue + } + } + # Crash-leftover scratch dirs in %TEMP%. + Get-ChildItem -Path $env:TEMP -Filter "Kit*" -ErrorAction SilentlyContinue | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + Get-ChildItem -Path $env:TEMP -Filter "hub-*" -ErrorAction SilentlyContinue | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + Get-ChildItem -Path $env:TEMP -Filter "omniverse-*" -ErrorAction SilentlyContinue | + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue + # build.sh fallback can leave 'build' / 'wheel' in user site-packages. + $userSite = "$env:APPDATA\Python\Python312\site-packages" + foreach ($pkg in @("build", "wheel")) { + $p = Join-Path $userSite $pkg + if (Test-Path $p) { + Remove-Item -LiteralPath $p -Recurse -Force -ErrorAction SilentlyContinue + } + } + + Show-State "AFTER (post-cleanup)" diff --git a/.github/actions/windows-sim-paths/action.yml b/.github/actions/windows-sim-paths/action.yml new file mode 100644 index 000000000000..2a5bc92b45d3 --- /dev/null +++ b/.github/actions/windows-sim-paths/action.yml @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +name: 'Resolve Isaac Sim paths and DLL search dirs on Windows' +description: >- + Discover the active Isaac Sim install root (via pip show isaacsim-kernel), + export ISAAC_PATH / CARB_APP_PATH / EXP_PATH / RESOURCE_NAME to subsequent + steps in this job, and prepend Isaac Sim's bin + kit/plugins directories to + PATH so the Vulkan loader can find NVIDIA's ICD DLLs and Kit can find its + plugin DLLs. + Mirrors PR #4018's known-working Windows env setup so Kit's RTX path can + initialise on a self-hosted Windows runner where DLL search defaults are not + pointed at the Sim install. + +inputs: + venv-path: + description: 'Path to the uv/python venv whose site-packages contains isaacsim (relative to workspace).' + required: false + default: 'env_isaaclab_uv' + +runs: + using: composite + steps: + - name: Resolve Isaac Sim paths + shell: powershell + run: | + $ErrorActionPreference = "Stop" + # Re-activate the caller's venv inside this fresh PowerShell session + # so `python -m pip show isaacsim-kernel` finds the right interpreter. + $activate = Join-Path "${{ inputs.venv-path }}" "Scripts\Activate.ps1" + if (-not (Test-Path $activate)) { throw "venv activate not found at $activate" } + & $activate + # Discover Sim location from pip metadata (avoids `import isaacsim`, + # which would bootstrap the kernel and is the exact thing we are about + # to launch deliberately). + # Use `uv pip show` rather than `python -m pip show` since uv venvs are + # created without pip installed inside the venv by default. + # Capture stdout only; uv writes its banner ("Using Python ...") to + # stderr and merging with 2>&1 trips $ErrorActionPreference=Stop via + # PowerShell's NativeCommandError handling on stderr lines. + $pipShow = (uv pip show isaacsim-kernel) | Out-String + if ($LASTEXITCODE -ne 0) { $pipShow = "" } + $loc = $pipShow -split "`n" | Where-Object { $_ -match "^Location:" } | Select-Object -First 1 + if (-not $loc) { + $pipShow = (uv pip show isaacsim) | Out-String + if ($LASTEXITCODE -ne 0) { $pipShow = "" } + $loc = $pipShow -split "`n" | Where-Object { $_ -match "^Location:" } | Select-Object -First 1 + } + if (-not $loc) { throw "Could not resolve isaacsim install path from pip" } + $sitePackages = ($loc -split "Location: ", 2)[1].Trim() + + # The Sim root is either ${sitePackages}\isaacsim or a versioned + # ${sitePackages}\isaacsim-* dir. Pick whichever holds kit/ or apps/. + $candidates = @() + $candidates += Join-Path $sitePackages "isaacsim" + $candidates += Join-Path $sitePackages "isaacsim_kernel" + Get-ChildItem -Path $sitePackages -Directory -Filter "isaacsim-*" -ErrorAction SilentlyContinue | + ForEach-Object { $candidates += $_.FullName } + + $isaacRoot = $null + foreach ($c in $candidates) { + if (Test-Path $c) { + if ((Test-Path (Join-Path $c "kit")) -or (Test-Path (Join-Path $c "apps"))) { + $isaacRoot = $c + break + } + } + } + if (-not $isaacRoot) { + Write-Host "Searched candidates for Sim root:" + $candidates | ForEach-Object { Write-Host " - $_ (exists: $(Test-Path $_))" } + throw "Could not find Isaac Sim install (no kit/ or apps/ under any candidate)" + } + + $carb = Join-Path $isaacRoot "kit" + # EXP_PATH should point at IsaacLab's workspace apps dir if present + # (IsaacLab ships its own .kit files there); fall back to Sim's apps. + $workspaceApps = Join-Path $PWD "apps" + $expPath = if (Test-Path $workspaceApps) { $workspaceApps } else { Join-Path $isaacRoot "apps" } + + # Export to subsequent steps via $GITHUB_ENV. + "ISAAC_PATH=$isaacRoot" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + "CARB_APP_PATH=$carb" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + "EXP_PATH=$expPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + "RESOURCE_NAME=IsaacSim" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + + # Prepend Sim DLL search dirs to PATH so the Vulkan loader can find + # NVIDIA's ICD .json + .dll and Kit can resolve plugin DLLs. + $extra = @() + foreach ($d in @((Join-Path $carb "plugins"), (Join-Path $isaacRoot "bin"))) { + if (Test-Path $d) { $extra += $d } + } + if ($extra.Count -gt 0) { + $newPath = ($extra -join ";") + ";" + $env:PATH + "PATH=$newPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + + Write-Host "Resolved:" + Write-Host " ISAAC_PATH = $isaacRoot" + Write-Host " CARB_APP_PATH = $carb" + Write-Host " EXP_PATH = $expPath" + Write-Host " PATH prepend = $($extra -join ';')" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 39f9ffab1b2c..2dbe9cab04e9 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -79,7 +79,10 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - run_docker_tests: ${{ steps.detect.outputs.run_docker_tests }} + # TEMP (revert before final review): force run_docker_tests=false while + # iterating Windows CI on PR #5700. Saves runner time + cost during the + # back-and-forth. + run_docker_tests: 'false' steps: - id: detect env: diff --git a/.github/workflows/install-ci.yml b/.github/workflows/install-ci.yml index 226222d425bd..a1e5e4ea5c39 100644 --- a/.github/workflows/install-ci.yml +++ b/.github/workflows/install-ci.yml @@ -36,7 +36,10 @@ jobs: name: Detect Changes runs-on: ubuntu-latest outputs: - run_install_tests: ${{ steps.detect.outputs.run_install_tests }} + # TEMP (revert before final review): force run_install_tests=false while + # iterating Windows CI on PR #5700. Saves runner time + cost during the + # back-and-forth. + run_install_tests: 'false' steps: - id: detect env: diff --git a/.github/workflows/test-multi-gpu.yaml b/.github/workflows/test-multi-gpu.yaml index edc51369fc9e..fa72a48da910 100644 --- a/.github/workflows/test-multi-gpu.yaml +++ b/.github/workflows/test-multi-gpu.yaml @@ -30,6 +30,10 @@ concurrency: jobs: test-multi-gpu: name: Multi-GPU (${{ matrix.physics }}, ${{ matrix.renderer }}) + # TEMP (revert before final review): skipped while iterating Windows CI on + # PR #5700 (this PR touches app_launcher.py, which would otherwise trigger + # the multi-GPU self-hosted runners). Saves runner time + cost. + if: false # Use dedicated multi-GPU runner to avoid blocking standard CI resources # Configure this label on a runner with 2+ GPUs (e.g., g5.12xlarge with 4x A10G) runs-on: [self-hosted, linux, x64, gpu, multi-gpu] diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 39be0686637c..7261c54a47fb 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -39,6 +39,14 @@ jobs: run: | set -euo pipefail + # TEMP (revert before final review): force run_build=false while + # iterating Windows CI on PR #5700. The detect step still runs so the + # required check stays green; only the heavy build steps are skipped. + echo "run_build=false" >> "$GITHUB_OUTPUT" + echo "## Wheel build gating" >> "$GITHUB_STEP_SUMMARY" + echo "Skipped: TEMP disabled while iterating Windows CI (PR #5700)." >> "$GITHUB_STEP_SUMMARY" + exit 0 + # Keep this workflow unconditionally triggered on PRs so required # branch-protection checks are always reported. The build steps below # run only when inputs that can affect the wheel have changed. diff --git a/.github/workflows/windows-ci.yaml b/.github/workflows/windows-ci.yaml new file mode 100644 index 000000000000..5ef5ff5f74bf --- /dev/null +++ b/.github/workflows/windows-ci.yaml @@ -0,0 +1,412 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# Windows CI on self-hosted GPU runners. Single consolidated job so the +# autoscaler can't tear the runner between sub-jobs. Kit-launching steps +# use Start-Process + WaitForExit as an OS-level watchdog (Python thread +# watchdogs are GIL-vulnerable). pytest uses --timeout-method=thread +# (SIGALRM is Unix-only). + +name: Windows CI + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - develop + - 'release/**' + push: + branches: + - main + - develop + - 'release/**' + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + checks: write + +# EULA + headless env. Without these Kit bootstrap blocks on stdin +# ("Unable to bootstrap inner kit kernel: EOF when reading a line") or +# the global watchdog kills the headless process. Mirrors PR #4018. +env: + OMNI_KIT_ACCEPT_EULA: "yes" + ACCEPT_EULA: "Y" + ISAACSIM_ACCEPT_EULA: "YES" + PRIVACY_CONSENT: "Y" + HEADLESS: "1" + ISAAC_SIM_HEADLESS: "1" + ISAAC_SIM_LOW_MEMORY: "1" + WINDOWS_PLATFORM: "true" + OMNI_KIT_NO_WINDOW: "1" + OMNI_KIT_DISABLE_WATCHDOG: "1" + OMNI_KIT_TELEMETRY: "0" + CARB_LOGGING_SEVERITY: "error" + PYTHONUNBUFFERED: "1" + PYTHONIOENCODING: "utf-8" + +jobs: + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + run_windows_ci: ${{ steps.detect.outputs.run_windows_ci }} + steps: + - id: detect + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + EVENT_NAME: ${{ github.event_name }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + patterns=( + $'^source/\tLibrary source code' + $'^tools/\tBuild tooling' + $'^apps/\tStandalone apps' + $'(^|/)pyproject\\.toml$\tPython project metadata' + $'^\\.github/workflows/windows-ci\\.yaml$\tThis workflow file' + $'^VERSION$\tVersion file' + ) + any_match() { + local files="$1" entry regex + for entry in "${patterns[@]}"; do + IFS=$'\t' read -r regex _ <<< "$entry" + if grep -qE "$regex" <<< "$files"; then + return 0 + fi + done + return 1 + } + if [ "$EVENT_NAME" != "pull_request" ]; then + echo "run_windows_ci=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + changed_files="$(gh api --paginate "repos/$REPO/pulls/$PR_NUMBER/files" --jq '.[].filename' || true)" + if [ -z "$changed_files" ] || any_match "$changed_files"; then + echo "run_windows_ci=true" >> "$GITHUB_OUTPUT" + else + echo "run_windows_ci=false" >> "$GITHUB_OUTPUT" + fi + + windows-ci: + name: windows-ci + needs: [changes] + if: needs.changes.outputs.run_windows_ci == 'true' + runs-on: [self-hosted, gpu-windows] + timeout-minutes: 90 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 1 + lfs: false + + - name: Report instance state (BEFORE) + uses: ./.github/actions/windows-instance-state + with: { phase: pre } + + - name: Install uv + shell: powershell + run: | + $ErrorActionPreference = "Stop" + if (-not (Get-Command uv -ErrorAction SilentlyContinue)) { + irm https://astral.sh/uv/install.ps1 | iex + } + Add-Content -Path $env:GITHUB_PATH -Value "$HOME\.local\bin" + + # Diagnostic only: proves whether a setup failure is DNS/reachability vs a + # missing package, decoupled from pip. pypi.nvidia.com serves the pinned + # public Isaac Sim wheels and pypi.org their dependencies. nvcr.io is the + # NGC registry the Linux CI pulls the develop container from; it is probed + # here to qualify these runners for a future develop-content lane (fetching + # develop bits from NGC needs only the repo-level NGC_API_KEY secret, no + # per-instance setup). + - name: Probe index reachability (diagnostic) + if: always() + continue-on-error: true + shell: powershell + run: | + Write-Host "runner=$env:COMPUTERNAME RUNNER_NAME=$env:RUNNER_NAME" + Write-Host "proxies: HTTP_PROXY=$env:HTTP_PROXY HTTPS_PROXY=$env:HTTPS_PROXY NO_PROXY=$env:NO_PROXY http_proxy=$env:http_proxy https_proxy=$env:https_proxy" + Write-Host "DNS servers:" + Get-DnsClientServerAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " $($_.InterfaceAlias): $($_.ServerAddresses -join ', ')" } + foreach ($h in @("pypi.nvidia.com", "pypi.org", "nvcr.io")) { + Write-Host "=== $h ===" + try { + $ips = [System.Net.Dns]::GetHostAddresses($h) + Write-Host "DNS OK -> $(( $ips | ForEach-Object { $_.IPAddressToString }) -join ', ')" + } catch { + Write-Host "DNS FAIL -> $($_.Exception.Message)" + } + try { + $r = Test-NetConnection -ComputerName $h -Port 443 -WarningAction SilentlyContinue + Write-Host "TCP 443 reachable -> $($r.TcpTestSucceeded)" + } catch { + Write-Host "TCP 443 test error -> $($_.Exception.Message)" + } + } + + # Shared setup. Hard fail aborts the job (no continue-on-error) since + # downstream steps all depend on this venv. + - name: Setup venv + install Isaac Sim (public pin) + isaaclab + test deps + id: setup + shell: powershell + timeout-minutes: 25 + run: | + $ErrorActionPreference = "Stop" + # --seed because the wheel-builder step runs `python -m pip install build wheel`. + uv venv --python 3.12 --seed env_isaaclab_uv + & "env_isaaclab_uv\Scripts\Activate.ps1" + uv pip install pytest pytest-timeout h5py + + # Install Isaac Sim from pypi.nvidia.com at the version pinned in + # [tool.isaaclab.versions], plus editable isaaclab and the RL frameworks + # the training smoke needs — all through the install CLI, so this job + # exercises the same install path the docs give Windows users. The + # develop-container build the Linux CI tests is not reachable from these + # runners (internal Artifactory is corp-DNS only); the public pin is + # IsaacLab's declared isaacsim dependency, so native Windows validates + # exactly what `isaaclab -i isaacsim` gives users. A develop-content + # lane via NGC (see the reachability probe above) can layer on later. + .\isaaclab.bat -i 'isaacsim,rl[rsl_rl,rl_games]' + python -c "import importlib.metadata as m; print('isaacsim build:', m.version('isaacsim'))" + python -c "import isaaclab, isaaclab_assets, isaaclab_tasks, isaaclab_newton, isaaclab_physx, isaaclab_ppisp; print('editable imports ok')" + New-Item -ItemType Directory -Force -Path "reports" | Out-Null + + - name: Resolve Isaac Sim paths (ISAAC_PATH / CARB_APP_PATH / EXP_PATH / DLL search) + id: sim-paths + uses: ./.github/actions/windows-sim-paths + with: { venv-path: 'env_isaaclab_uv' } + + # ===== Test branches (each independent, continue-on-error). ===== + + - name: Deps smoke (torch + scipy) + id: test-deps + if: always() && steps.setup.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 5 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + python -m pytest ` + source/isaaclab/test/deps ` + --ignore=tools/conftest.py ` + -m windows_ci ` + --continue-on-collection-errors ` + --timeout=60 ` + --timeout-method=thread ` + -v ` + --junitxml=reports/deps-smoke.xml + + - name: Path-IO tests (utils) + id: test-pathio + if: always() && steps.setup.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 10 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + # Explicit files only; neighbor tests import AppLauncher/argparse at + # module load and crash on Windows without Sim initialised. + python -m pytest ` + source/isaaclab/test/utils/test_configclass.py ` + source/isaaclab/test/utils/test_dict.py ` + source/isaaclab/test/utils/test_episode_data.py ` + source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py ` + --continue-on-collection-errors ` + --timeout=120 ` + --timeout-method=thread ` + -v ` + --junitxml=reports/path-io.xml + + - name: Kit headless boot smoke + id: test-kit-launch + if: always() && steps.sim-paths.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 8 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + $script = @' + import sys + from isaaclab.app import AppLauncher + + app_launcher = AppLauncher(headless=True) + sim = app_launcher.app + assert sim is not None, "AppLauncher did not return a SimulationApp" + sim.close() + sys.exit(0) + '@ + $script | Out-File -FilePath kit_launch_smoke.py -Encoding utf8 + $proc = Start-Process -PassThru -NoNewWindow -FilePath python -ArgumentList "kit_launch_smoke.py" + if (-not $proc.WaitForExit(300000)) { + Write-Host "::error::kit-launch hard timeout (5 min) - Kit hung; killing python tree" + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $proc.WaitForExit() + exit 124 + } + exit $proc.ExitCode + + # Runs both subcases: state (rsl_rl) and perception (rl_games, + # --enable_cameras). The perception subcase needs the runner GPU in + # WDDM mode so Kit's RTX/Vulkan path initialises; the data-center (TCC) + # driver does not expose Vulkan. + - name: Cartpole training smoke (state + perception) + id: test-training-smoke + if: always() && steps.sim-paths.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 25 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + python -m pytest ` + source/isaaclab_tasks/test/test_cartpole_training_smoke.py ` + --continue-on-collection-errors ` + --timeout=600 ` + --timeout-method=thread ` + -v ` + --junitxml=reports/training-smoke.xml + + # Cartpole-camera perception smoke — exercises Kit's RTX/Vulkan path + # directly (lighter than the perception training subcase, so a Vulkan + # init failure surfaces here first). Needs the runner GPU in WDDM mode + # so Kit can enumerate a Vulkan device; the data-center (TCC) driver + # does not expose Vulkan. + - name: Cartpole-camera perception smoke (RTX / Vulkan path) + id: test-perception + if: always() && steps.sim-paths.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 8 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + $script = @' + import os + import sys + import traceback + + from isaaclab.app import AppLauncher + + app_launcher = AppLauncher(headless=True, enable_cameras=True) + sim = app_launcher.app + assert sim is not None, "AppLauncher did not return a SimulationApp" + try: + import gymnasium as gym + import isaaclab_tasks # noqa: F401 (gym env registration) + env = gym.make("Isaac-Cartpole-Camera-Direct", num_envs=1) + obs, info = env.reset() + assert obs is not None, "env.reset returned None observation" + for step_i in range(3): + action = env.action_space.sample() + obs, reward, terminated, truncated, info = env.step(action) + assert obs is not None, f"env.step {step_i} returned None observation" + env.close() + except BaseException: + # Print the real failure and skip Kit teardown: a wedged shutdown + # would otherwise hide the traceback behind the 3-min watchdog kill. + traceback.print_exc() + sys.stdout.flush() + sys.stderr.flush() + os._exit(1) + sim.close() + sys.exit(0) + '@ + $script | Out-File -FilePath perception_smoke.py -Encoding utf8 + $proc = Start-Process -PassThru -NoNewWindow -FilePath python -ArgumentList "perception_smoke.py" + if (-not $proc.WaitForExit(180000)) { + Write-Host "::error::perception hard timeout (3 min) - Kit/Vulkan hung; killing python tree" + Stop-Process -Id $proc.Id -Force -ErrorAction SilentlyContinue + $proc.WaitForExit() + exit 124 + } + exit $proc.ExitCode + + # Last test step — destructively uninstalls editable isaaclab and + # reinstalls from the built wheel. Placed last so the test branches + # above run against the editable install. + - name: Wheel build + reinstall + smoke import + id: test-wheel-build + if: always() && steps.setup.outcome == 'success' + continue-on-error: true + shell: powershell + timeout-minutes: 20 + run: | + $ErrorActionPreference = "Stop" + & "env_isaaclab_uv\Scripts\Activate.ps1" + $gitBash = "C:\Program Files\Git\bin\bash.exe" + if (-not (Test-Path $gitBash)) { throw "Git Bash not found at $gitBash" } + # git-bash on Windows ships `python` only, not `python3`. + $env:PYTHON = "python" + & $gitBash tools/wheel_builder/build.sh + if ($LASTEXITCODE -ne 0) { throw "wheel_builder/build.sh failed with exit $LASTEXITCODE" } + $wheel = Get-ChildItem -Path "tools/wheel_builder/build/dist" -Filter "isaaclab-*.whl" | Select-Object -First 1 + if (-not $wheel) { throw "no wheel found in tools/wheel_builder/build/dist" } + uv pip uninstall isaaclab + # Bare wheel (no extras): Isaac Sim is already installed at the same + # public pin, so this only validates that the freshly built isaaclab + # wheel installs. + uv pip install "$($wheel.FullName)" + python -c "import isaaclab; print('wheel install ok:', isaaclab.__file__)" + + # ===== Reporting + cleanup. ===== + + - name: Upload all test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: windows-ci-reports + path: | + reports/ + kit_launch_smoke.py + perception_smoke.py + retention-days: 7 + if-no-files-found: ignore + + # Every active test step gates the job. + - name: Aggregate test results + if: always() + shell: powershell + run: | + $results = [ordered]@{ + "setup" = "${{ steps.setup.outcome }}" + "sim-paths" = "${{ steps.sim-paths.outcome }}" + "deps" = "${{ steps.test-deps.outcome }}" + "path-io" = "${{ steps.test-pathio.outcome }}" + "kit-launch" = "${{ steps.test-kit-launch.outcome }}" + "training-smoke" = "${{ steps.test-training-smoke.outcome }}" + "perception" = "${{ steps.test-perception.outcome }}" + "wheel-build" = "${{ steps.test-wheel-build.outcome }}" + } + Write-Host "=== windows-ci step outcomes ===" + foreach ($k in $results.Keys) { + "{0,-16} {1}" -f $k, $results[$k] + } + $blocking = @("setup", "sim-paths", "deps", "path-io", "kit-launch", "training-smoke", "perception", "wheel-build") + $failed = @() + foreach ($k in $blocking) { + if ($results[$k] -eq "failure") { $failed += $k } + } + if ($failed.Count -gt 0) { + Write-Host "::error::Failing job - these steps failed: $($failed -join ', ')" + exit 1 + } + Write-Host "All gating steps passed." + + - name: Report instance state + cleanup (AFTER) + if: always() + uses: ./.github/actions/windows-instance-state + with: { phase: post } diff --git a/pyproject.toml b/pyproject.toml index 92cca8dfdf02..5a26ec3228a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "torchaudio>=2.11", "onnx>=1.18.0", "prettytable>=3.3.0", + "psutil", "protobuf>=4.25.8,!=5.26.0", "hidapi>=0.14.0", "gymnasium>=1.2.0", diff --git a/source/isaaclab/changelog.d/jichuanh-windows-ci-cli-install.rst b/source/isaaclab/changelog.d/jichuanh-windows-ci-cli-install.rst new file mode 100644 index 000000000000..1bf63fceef18 --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-windows-ci-cli-install.rst @@ -0,0 +1,10 @@ +Fixed +^^^^^ + +* Fixed ``isaaclab -i isaacsim`` failing in uv environments because uv rejects + Isaac Sim's transitive pre-release pins (e.g. ``tinyobjloader==2.0.0rc13``); + the install now passes ``--prerelease=allow`` alongside the existing index + strategy flag. +* Fixed comma-separated RL framework selectors (e.g. ``-i "rl[rsl_rl,rl_games]"``) + installing nothing; each listed framework extra is now installed, mirroring the + ``ov`` selector handling. diff --git a/source/isaaclab/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..3ae87e252fbd --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-windows-ci.skip @@ -0,0 +1 @@ +Skip changelog: CI-infrastructure only (no user-facing API change). Adds .github/workflows/windows-ci.yaml carrying the Windows CI pipeline against [self-hosted, gpu-windows] runners. Tier 1 (smoke, install probe with wheel build + reinstall, Kit launch) plus Tier 2 (path-IO marker-driven discovery, cartpole-camera perception smoke). All jobs use continue-on-error: true and pytest --timeout to fail fast on hangs. Inline scripts assert explicitly and exit nonzero on any failure (fixes the previous pattern where Vulkan failures hung the job instead of erroring). diff --git a/source/isaaclab/changelog.d/jichuanh-windows-spark-ci-min.skip b/source/isaaclab/changelog.d/jichuanh-windows-spark-ci-min.skip new file mode 100644 index 000000000000..bfa2b75a780a --- /dev/null +++ b/source/isaaclab/changelog.d/jichuanh-windows-spark-ci-min.skip @@ -0,0 +1 @@ +Skip changelog: CI/test-infrastructure foundation (no user-facing API change). Registers the windows / windows_ci / arm / arm_ci pytest markers in pyproject.toml, teaches AppLauncher to recognize them in argv so they do not leak into Isaac Sim's argparse, and moves the AssetConverterBase USD scratch dir from hardcoded /tmp/IsaacLab to tempfile.gettempdir() for cross-platform compatibility. Workflow files (arm-ci.yaml, windows-ci.yaml) ship in follow-up PRs. diff --git a/source/isaaclab/isaaclab/cli/commands/install.py b/source/isaaclab/isaaclab/cli/commands/install.py index ec7fe246d215..53619e9befd0 100644 --- a/source/isaaclab/isaaclab/cli/commands/install.py +++ b/source/isaaclab/isaaclab/cli/commands/install.py @@ -646,8 +646,10 @@ def _install_isaacsim() -> None: extra_flags = [] if using_uv: # uv needs unsafe-best-match to resolve packages across multiple indexes - # (isaacsim is on pypi.nvidia.com, its deps are on pypi.org). - extra_flags = ["--index-strategy", "unsafe-best-match"] + # (isaacsim is on pypi.nvidia.com, its deps are on pypi.org), and + # prerelease=allow because uv rejects transitive pre-release pins + # (isaacsim depends on e.g. tinyobjloader==2.0.0rc13); pip allows both. + extra_flags = ["--index-strategy", "unsafe-best-match", "--prerelease=allow"] run_command( pip_cmd @@ -856,8 +858,12 @@ def _install_extra_feature(feature_name: str, selector: str = "") -> None: elif feature_name == "rl": extra = selector if selector else "all" # rl[all] installs every RL framework extra; other selectors map by name - # (rsl_rl -> rsl-rl, skrl, sb3, rl-games). - frameworks = {"sb3", "skrl", "rl-games", "rsl-rl"} if extra == "all" else {extra.replace("_", "-")} + # (rsl_rl -> rsl-rl, skrl, sb3, rl-games) and may be comma-separated + # (e.g. rl[rsl_rl,rl_games]), mirroring the ov selector handling. + if extra == "all": + frameworks = {"sb3", "skrl", "rl-games", "rsl-rl"} + else: + frameworks = {item.strip().replace("_", "-") for item in extra.split(",") if item.strip()} print_info(f"Installing RL framework extras: {extra}...") for framework in sorted(frameworks): _install_root_extra(framework) diff --git a/source/isaaclab/test/cli/test_install_command_parsing.py b/source/isaaclab/test/cli/test_install_command_parsing.py index 8a6abe523130..8f97327d1c5f 100644 --- a/source/isaaclab/test/cli/test_install_command_parsing.py +++ b/source/isaaclab/test/cli/test_install_command_parsing.py @@ -411,3 +411,31 @@ def test_isaaclab_is_first_when_mimic_added(self): mocks = self._run("mimic") installed = mocks["_install_isaaclab_submodules"].call_args[0][0] assert installed[0] == "isaaclab" + + +# --------------------------------------------------------------------------- +# _install_extra_feature selector handling +# --------------------------------------------------------------------------- + + +class TestInstallExtraFeatureSelectors: + """Tests for _install_extra_feature() selector normalization.""" + + def _run_rl(self, selector: str) -> set[str]: + """Invoke the rl feature with _install_root_extra mocked; return extras installed.""" + from isaaclab.cli.commands.install import _install_extra_feature + + with patch(f"{_INSTALL_MODULE}._install_root_extra") as mock_extra: + _install_extra_feature("rl", selector) + return {c.args[0] for c in mock_extra.call_args_list} + + def test_rl_single_selector_maps_underscores(self): + assert self._run_rl("rsl_rl") == {"rsl-rl"} + + def test_rl_comma_selector_installs_each_framework(self): + # split_install_items() deliberately keeps "rl[rsl_rl,rl_games]" as one + # token, so the selector itself must be comma-split here. + assert self._run_rl("rsl_rl,rl_games") == {"rsl-rl", "rl-games"} + + def test_rl_all_installs_every_framework(self): + assert self._run_rl("") == {"sb3", "skrl", "rl-games", "rsl-rl"} diff --git a/source/isaaclab/test/deps/test_scipy.py b/source/isaaclab/test/deps/test_scipy.py index 91d47a5c9e62..b07b0c0d5435 100644 --- a/source/isaaclab/test/deps/test_scipy.py +++ b/source/isaaclab/test/deps/test_scipy.py @@ -13,7 +13,7 @@ import numpy as np import scipy.interpolate as interpolate -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci, pytest.mark.arm_ci] @pytest.mark.isaacsim_ci diff --git a/source/isaaclab/test/deps/test_torch.py b/source/isaaclab/test/deps/test_torch.py index 2d4881346f72..a32e8e6c4836 100644 --- a/source/isaaclab/test/deps/test_torch.py +++ b/source/isaaclab/test/deps/test_torch.py @@ -7,7 +7,7 @@ import torch import torch.utils.benchmark as benchmark -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci, pytest.mark.arm_ci] @pytest.mark.isaacsim_ci diff --git a/source/isaaclab/test/utils/test_configclass.py b/source/isaaclab/test/utils/test_configclass.py index 1d3327e48944..c5ff75a7e106 100644 --- a/source/isaaclab/test/utils/test_configclass.py +++ b/source/isaaclab/test/utils/test_configclass.py @@ -16,6 +16,8 @@ import torch from isaaclab.utils.configclass import _field_module_dir, configclass + +pytestmark = pytest.mark.windows_ci from isaaclab.utils.dict import class_to_dict, dict_to_md5_hash, update_class_from_dict from isaaclab.utils.io import dump_yaml, load_yaml from isaaclab.utils.string import ResolvableString diff --git a/source/isaaclab/test/utils/test_dict.py b/source/isaaclab/test/utils/test_dict.py index 7546736497f6..5646dac32851 100644 --- a/source/isaaclab/test/utils/test_dict.py +++ b/source/isaaclab/test/utils/test_dict.py @@ -10,7 +10,7 @@ import isaaclab.utils.dict as dict_utils import isaaclab.utils.string as string_utils -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def _test_function(x): diff --git a/source/isaaclab/test/utils/test_episode_data.py b/source/isaaclab/test/utils/test_episode_data.py index c767521ceb86..c4a543d8918e 100644 --- a/source/isaaclab/test/utils/test_episode_data.py +++ b/source/isaaclab/test/utils/test_episode_data.py @@ -7,7 +7,7 @@ from isaaclab.utils.datasets import EpisodeData -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] @pytest.mark.parametrize("device", ["cuda:0", "cpu"]) diff --git a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py index f28b9277d814..40c28f6d8424 100644 --- a/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py +++ b/source/isaaclab/test/utils/test_hdf5_dataset_file_handler.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause import os -import shutil import tempfile import uuid @@ -12,7 +11,7 @@ from isaaclab.utils.datasets import EpisodeData, HDF5DatasetFileHandler -pytestmark = pytest.mark.unit +pytestmark = [pytest.mark.unit, pytest.mark.windows_ci] def create_test_episode(device): @@ -38,10 +37,12 @@ def create_test_episode(device): @pytest.fixture def temp_dir(): """Create a temporary directory for test datasets.""" - temp_dir = tempfile.mkdtemp() - yield temp_dir - # cleanup after tests - shutil.rmtree(temp_dir) + # ignore_cleanup_errors absorbs a Windows-specific PermissionError: + # libhdf5 keeps an internal file handle briefly after .close(), and + # rmtree races with that handle release. On Linux/macOS this flag is + # a no-op since no cleanup error is raised. + with tempfile.TemporaryDirectory(ignore_cleanup_errors=True) as d: + yield d def test_create_dataset_file(temp_dir): diff --git a/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip b/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip new file mode 100644 index 000000000000..8dad5a2f809c --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/jichuanh-windows-ci.skip @@ -0,0 +1 @@ +Skip changelog: CI/test-only (no user-facing API change). Adds source/isaaclab_tasks/test/test_cartpole_training_smoke.py — a minimal cartpole training smoke (state rsl_rl + perception rl_games, two PPO iters each) tagged with the arm_ci and windows_ci markers so cross-platform CI shapes can invoke it via marker-driven discovery. diff --git a/source/isaaclab_tasks/test/test_cartpole_training_smoke.py b/source/isaaclab_tasks/test/test_cartpole_training_smoke.py new file mode 100644 index 000000000000..ad558fd6f12b --- /dev/null +++ b/source/isaaclab_tasks/test/test_cartpole_training_smoke.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Minimal end-to-end training smoke for cartpole. + +Two cases — state-only and perception (RGB tiled camera) — each spawn the +unified ``isaaclab train`` entrypoint for two PPO iterations on a small env +count. They validate the full pipeline (``./isaaclab.sh`` wrapper, gym +registration, env build, RL wrapper, optimizer step, checkpoint write) +without the cost of a real training run, so the orchestrator can include +them in every CI shape (Linux, ARM/Spark). + +The state case uses rsl_rl and the perception case rl_games, so the two +smokes cover two different RL-library dispatch paths. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest + +# Cross-platform: ARM (Linux/aarch64) and Windows CI both opt in. +pytestmark = [pytest.mark.arm_ci, pytest.mark.windows_ci] + +_REPO_ROOT = Path(__file__).resolve().parents[3] +# isaaclab.bat on Windows, isaaclab.sh on Linux/macOS — same CLI surface. +_LAUNCHER = str(_REPO_ROOT / ("isaaclab.bat" if os.name == "nt" else "isaaclab.sh")) + + +def _run_train( + rl_library: str, task_name: str, num_envs: str = "16", extra_args: list[str] | None = None, timeout: int = 600 +) -> None: + """Spawn a trainer for two iterations and assert it exits cleanly.""" + cmd = [ + _LAUNCHER, + "train", + "--rl_library", + rl_library, + "--task", + task_name, + "--headless", + "--num_envs", + num_envs, + "--max_iterations", + "2", + "--seed", + "42", + ] + if extra_args: + cmd.extend(extra_args) + + result = subprocess.run( + cmd, + cwd=_REPO_ROOT, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + assert result.returncode == 0, ( + f"Training command failed for {task_name}: {' '.join(cmd)}\n" + f"--- stdout (tail) ---\n{result.stdout[-4000:]}\n" + f"--- stderr (tail) ---\n{result.stderr[-4000:]}\n" + ) + + +def test_train_cartpole_state(): + """State-observation cartpole trains for two rsl_rl PPO iterations without errors.""" + _run_train("rsl_rl", "Isaac-Cartpole-Direct") + + +def test_train_cartpole_perception(): + """RGB-camera cartpole trains for two rl_games PPO iterations without errors.""" + # 32 envs: rl_games asserts batch_size (horizon_length * num_envs = 64 * 32) + # is divisible by the agent cfg's minibatch_size (2048). + _run_train( + "rl_games", + "Isaac-Cartpole-Camera-Direct", + num_envs="32", + extra_args=["--enable_cameras"], + ) diff --git a/tools/wheel_builder/build.sh b/tools/wheel_builder/build.sh index 1c9f22685b53..44791a1ffba2 100755 --- a/tools/wheel_builder/build.sh +++ b/tools/wheel_builder/build.sh @@ -1,6 +1,10 @@ #!/bin/bash set -e +# Python interpreter override. Linux installs typically expose `python3`; +# Windows git-bash only has `python`. Callers can set PYTHON=python to override. +PYTHON="${PYTHON:-python3}" + SELF_DIR="$(dirname "$(realpath "$0")")" cd "$SELF_DIR/../.." @@ -86,20 +90,20 @@ cp "$SELF_DIR/res/__init__.py" "$BUILD_DIR/src/isaaclab/" cp "$SELF_DIR/res/__main__.py" "$BUILD_DIR/src/isaaclab/" # 3. Generate pyproject.toml with dependencies from the root pyproject.toml -python3 "$SELF_DIR/gen_pyproject.py" "$SELF_DIR/../../pyproject.toml" "$BUILD_DIR/pyproject.toml" "$WHEEL_VERSION" +"$PYTHON" "$SELF_DIR/gen_pyproject.py" "$SELF_DIR/../../pyproject.toml" "$BUILD_DIR/pyproject.toml" "$WHEEL_VERSION" # 4. Build the wheel cd "$BUILD_DIR" # Prefer --user to avoid polluting system Python; fall back to --break-system-packages # for environments where --user is unsupported (e.g. Docker, ephemeral CI runners). -python3 -m pip install --user build wheel 2>/dev/null || python3 -m pip install --break-system-packages build wheel -python3 -m build --wheel --outdir "$DIST_DIR/" +"$PYTHON" -m pip install --user build wheel 2>/dev/null || "$PYTHON" -m pip install --break-system-packages build wheel +"$PYTHON" -m build --wheel --outdir "$DIST_DIR/" # 5. Retag the wheel to match official platform tags # cd "$DIST_DIR" # GENERIC_WHL=$(ls isaaclab-*.whl) # echo "Retagging $GENERIC_WHL -> $PYTHON_TAG-$ABI_TAG-$PLATFORM_TAG" -# python3 -m wheel tags --python-tag "$PYTHON_TAG" --abi-tag "$ABI_TAG" --platform-tag "$PLATFORM_TAG" "$GENERIC_WHL" +# "$PYTHON" -m wheel tags --python-tag "$PYTHON_TAG" --abi-tag "$ABI_TAG" --platform-tag "$PLATFORM_TAG" "$GENERIC_WHL" # # Remove the generic wheel (wheel tags creates a new file) # TAGGED_WHL=$(ls isaaclab-*"$PLATFORM_TAG"*.whl 2>/dev/null) # if [ "$GENERIC_WHL" != "$TAGGED_WHL" ] && [ -n "$TAGGED_WHL" ]; then