diff --git a/.github/versions.json b/.github/versions.json index 28b14f23b..dfb667712 100644 --- a/.github/versions.json +++ b/.github/versions.json @@ -35,6 +35,7 @@ "region": "us-east-2", "cpu_ami": "ami-0e14a711dad782a70", "gpu_ami": "ami-0f5c0b65fde08ae43", + "fgpu_ami": "ami-0b5dca767eafff9d8", "subnets": [ { "az": "us-east-2a", "id": "subnet-0b75bb13b6048a2f1" }, { "az": "us-east-2b", "id": "subnet-03f2320d6e6e0005b" }, diff --git a/.github/workflows/cu128.yml b/.github/workflows/cu128.yml index 381559bb6..a6d19e198 100644 --- a/.github/workflows/cu128.yml +++ b/.github/workflows/cu128.yml @@ -166,7 +166,9 @@ jobs: source .venv/bin/activate NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" env: NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }} @@ -249,8 +251,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS diff --git a/.github/workflows/cu130.yml b/.github/workflows/cu130.yml index 9de66e293..8241defbf 100644 --- a/.github/workflows/cu130.yml +++ b/.github/workflows/cu130.yml @@ -166,7 +166,9 @@ jobs: source .venv/bin/activate NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list "${NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR}" env: NEEDS_VERSIONS_OUTPUTS_CUDA_ARCH_PR: ${{ needs.versions.outputs.cuda-arch-pr }} @@ -249,8 +251,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS diff --git a/.github/workflows/load-versions.yml b/.github/workflows/load-versions.yml index 293c3c8eb..fccdaa5dd 100644 --- a/.github/workflows/load-versions.yml +++ b/.github/workflows/load-versions.yml @@ -30,6 +30,8 @@ on: value: ${{ jobs.load.outputs.cuda-128-patch }} cuda-130-patch: value: ${{ jobs.load.outputs.cuda-130-patch }} + cuda-default-patch: + value: ${{ jobs.load.outputs.cuda-default-patch }} cuda-conda-override: value: ${{ jobs.load.outputs.cuda-conda-override }} cuda-arch-pr: @@ -50,10 +52,14 @@ on: value: ${{ jobs.load.outputs.aws-cpu-ami }} aws-gpu-ami: value: ${{ jobs.load.outputs.aws-gpu-ami }} + aws-fgpu-ami: + value: ${{ jobs.load.outputs.aws-fgpu-ami }} aws-cpu-az-config: value: ${{ jobs.load.outputs.aws-cpu-az-config }} aws-gpu-az-config: value: ${{ jobs.load.outputs.aws-gpu-az-config }} + aws-fgpu-az-config: + value: ${{ jobs.load.outputs.aws-fgpu-az-config }} aws-security-group: value: ${{ jobs.load.outputs.aws-security-group }} aws-cloudfront-distribution-domain: @@ -79,6 +85,7 @@ jobs: cuda-default: ${{ steps.parse.outputs.cuda-default }} cuda-128-patch: ${{ steps.parse.outputs.cuda-128-patch }} cuda-130-patch: ${{ steps.parse.outputs.cuda-130-patch }} + cuda-default-patch: ${{ steps.parse.outputs.cuda-default-patch }} cuda-conda-override: ${{ steps.parse.outputs.cuda-conda-override }} cuda-arch-pr: ${{ steps.parse.outputs.cuda-arch-pr }} cuda-arch-publish: ${{ steps.parse.outputs.cuda-arch-publish }} @@ -89,8 +96,10 @@ jobs: aws-region: ${{ steps.parse.outputs.aws-region }} aws-cpu-ami: ${{ steps.parse.outputs.aws-cpu-ami }} aws-gpu-ami: ${{ steps.parse.outputs.aws-gpu-ami }} + aws-fgpu-ami: ${{ steps.parse.outputs.aws-fgpu-ami }} aws-cpu-az-config: ${{ steps.parse.outputs.aws-cpu-az-config }} aws-gpu-az-config: ${{ steps.parse.outputs.aws-gpu-az-config }} + aws-fgpu-az-config: ${{ steps.parse.outputs.aws-fgpu-az-config }} aws-security-group: ${{ steps.parse.outputs.aws-security-group }} aws-cloudfront-distribution-domain: ${{ steps.parse.outputs.aws-cloudfront-distribution-domain }} publish-matrix: ${{ steps.parse.outputs.publish-matrix }} @@ -118,6 +127,7 @@ jobs: echo "cuda-default=$(jq -r '.cuda.default' $CFG)" echo "cuda-128-patch=$(jq -r '.cuda.versions["12.8"].patch' $CFG)" echo "cuda-130-patch=$(jq -r '.cuda.versions["13.0"].patch' $CFG)" + echo "cuda-default-patch=$(jq -r '.cuda.versions[.cuda.default].patch' $CFG)" echo "cuda-conda-override=$(jq -r '.cuda.conda_override' $CFG)" echo "cuda-arch-pr=$(jq -r '.cuda.arch_list_pr' $CFG)" echo "cuda-arch-publish=$(jq -r '.cuda.arch_list_publish' $CFG)" @@ -128,8 +138,10 @@ jobs: echo "aws-region=$(jq -r '.aws.region' $CFG)" echo "aws-cpu-ami=$(jq -r '.aws.cpu_ami' $CFG)" echo "aws-gpu-ami=$(jq -r '.aws.gpu_ami' $CFG)" + echo "aws-fgpu-ami=$(jq -r '.aws.fgpu_ami' $CFG)" echo "aws-cpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.cpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" echo "aws-gpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.gpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" + echo "aws-fgpu-az-config=$(jq -c '.aws as $a | [$a.subnets[] | {imageId: $a.fgpu_ami, subnetId: .id, securityGroupId: $a.security_group}]' $CFG)" echo "aws-security-group=$(jq -r '.aws.security_group' $CFG)" echo "aws-cloudfront-distribution-domain=$(jq -r '.aws.cloudfront_distribution_domain // ""' $CFG)" echo "publish-matrix=$(jq -c '.publish_matrix' $CFG)" diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index 21382af90..7e6b4fc89 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -507,3 +507,147 @@ jobs: github-token: ${{ secrets.EC2_RUNNER_TOKEN }} label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }} + + ############################################################################## + # START NIGHTLY TESTS GPU RUNNER + # + # The PR workflows build with NANOVDB_USE_SYNC_CUDA_MALLOC because they test on + # fractional-GPU (vGPU) runners, which cannot use cudaMallocAsync. This whole + # GPU instance is therefore the only place the default asynchronous NanoVDB + # allocation path gets exercised, so it runs once a night against the published + # nightly wheel instead of on every pull request. + ############################################################################## + start-tests-gpu-runner: + name: Start EC2 GPU runner for nightly tests + needs: [nightly-build, versions] + if: ${{ !cancelled() && needs.nightly-build.result == 'success' }} + runs-on: ubuntu-latest + permissions: + id-token: write # Required for AWS OIDC + outputs: + label: ${{ steps.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 + with: + role-to-assume: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + - name: Start EC2 GPU runner + id: start-tests-gpu-runner + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + + ############################################################################## + # RUN NIGHTLY UNIT TESTS + ############################################################################## + nightly-unit-tests: + name: fVDB Nightly Unit Tests + needs: [start-tests-gpu-runner, versions] + runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} + container: + image: nvidia/cuda:${{ needs.versions.outputs.cuda-default-patch }}-cudnn-devel-${{ needs.versions.outputs.docker-os-rocky }} + env: + PYTHONPATH: "" + options: --rm + defaults: + run: + shell: bash -el {0} + steps: + - name: Install prerequisites + run: | + dnf install -y git wget procps-ng findutils gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET} + echo "source /opt/rh/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}/enable" > /etc/profile.d/gcc-toolset-${NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET}.sh + env: + NEEDS_VERSIONS_OUTPUTS_GCC_TOOLSET: ${{ needs.versions.outputs.gcc-toolset }} + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: main + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 + with: + version: "${{ needs.versions.outputs.uv-version }}" + + - name: Set up Python + run: | + uv python install ${NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT} + uv venv + env: + NEEDS_VERSIONS_OUTPUTS_PYTHON_DEFAULT: ${{ needs.versions.outputs.python-default }} + + - name: Install system dependencies + run: dnf install -y zlib-devel libpng-devel + + - name: Install pip dependencies + run: | + CUDA_TAG="cu$(echo "${NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT}" | tr -d '.')" + echo "torch==${NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION}" > "${RUNNER_TEMP}/torch-constraints.txt" + uv pip install --no-cache -c "${RUNNER_TEMP}/torch-constraints.txt" -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/${CUDA_TAG} --index-strategy unsafe-best-match + uv pip install --no-cache setuptools + env: + NEEDS_VERSIONS_OUTPUTS_CUDA_DEFAULT: ${{ needs.versions.outputs.cuda-default }} + NEEDS_VERSIONS_OUTPUTS_TORCH_FULL_VERSION: ${{ needs.versions.outputs.torch-full-version }} + + # Test the default Python/CUDA cell of the publish matrix; the other cells + # differ only in interpreter and CUDA minor version. + - name: Download package + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: fvdb-nightly-${{ needs.versions.outputs.python-default }}-torch${{ needs.versions.outputs.torch-version }}-cu${{ needs.versions.outputs.cuda-default }} + path: ./dist + + - name: Install package + run: | + source .venv/bin/activate + pip install ./dist/*.whl + + - name: Run unit tests + run: | + source .venv/bin/activate + NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) + pip install "$NANOVDB_EDITOR_SPEC" + cd tests; + pytest -v unit + + - name: Cleanup + if: always() + run: | + echo "Cleaning up /__w/_temp directory" + rm -rf /__w/_temp/* + echo "Cleanup completed" + + ############################################################################## + # STOP NIGHTLY TESTS GPU RUNNER + ############################################################################## + stop-tests-gpu-runner: + name: Stop GPU EC2 runner for nightly tests + needs: + - start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job + - nightly-unit-tests # required to wait when the main job is done + - versions + runs-on: ubuntu-latest + permissions: + id-token: write # Required for AWS OIDC + # required to stop the runner even if the error happened in the previous jobs, + # but only if the start-tests-gpu-runner job was not skipped + if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@254c19bd240aabef8777f48595e9d2d7b972184b # v6.2.1 + with: + role-to-assume: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + - name: Stop EC2 runner + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ needs.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9a00cb7d..6bed5be1a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -132,7 +132,9 @@ jobs: micromamba activate fvdb_build NANOVDB_EDITOR_SPEC=$(python .github/scripts/get_viewer_dependency.py) pip install "$NANOVDB_EDITOR_SPEC" - ./build.sh wheel verbose gtests benchmarks --cuda-arch-list '8.9+PTX' + # sync_cuda_malloc: the test runners are fractional-GPU (vGPU) instances + # where cudaMallocAsync is unavailable, so NanoVDB must use cudaMalloc. + ./build.sh wheel verbose gtests benchmarks sync_cuda_malloc --cuda-arch-list '8.9+PTX' - name: Upload wheel uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -204,8 +206,8 @@ jobs: with: mode: start github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + ec2-instance-type: g6f.2xlarge # 8 CPU-core, 1/4 L4 GPU + availability-zones-config: ${{ needs.versions.outputs.aws-fgpu-az-config }} ############################################################################## # RUN FVDB GTESTS diff --git a/README.md b/README.md index 3cc68acd8..692a33910 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,9 @@ The build script supports the following build modifiers: - `debug`: Build in debug mode with full debug symbols and no optimizations. - `strip_symbols`: Strip symbols from the build (will be ignored if debug is enabled). - `verbose`: Enable verbose build output for pip and CMake. +- `sync_cuda_malloc`: Compile with `-DNANOVDB_USE_SYNC_CUDA_MALLOC` so NanoVDB allocates with + `cudaMalloc`/`cudaFree` instead of the async variants. Required on vGPU slices, where async + allocation is unavailable because it depends on GPU unified memory. ### Running Tests diff --git a/build.sh b/build.sh index 34415f0ac..90baa5621 100755 --- a/build.sh +++ b/build.sh @@ -26,6 +26,8 @@ usage() { echo " debug Build in debug mode with full debug symbols and no optimizations." echo " lineinfo Enable CUDA lineinfo (sets FVDB_LINEINFO=ON)." echo " strip_symbols Strip symbols from the build (will be ignored if debug is enabled)." + echo " sync_cuda_malloc Compile with -DNANOVDB_USE_SYNC_CUDA_MALLOC so NanoVDB uses" + echo " synchronous cudaMalloc/cudaFree (required on vGPU slices)." echo " verbose Enable verbose build output for pip and CMake." echo " trace Enable CMake trace output for debugging configuration." echo "" @@ -316,6 +318,10 @@ while (( "$#" )); do echo "Enabling strip symbols build" CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_STRIP_SYMBOLS=ON" is_config_arg_handled=true + elif [[ "$1" == "sync_cuda_malloc" ]]; then + echo "Enabling synchronous NanoVDB CUDA allocation (-DNANOVDB_USE_SYNC_CUDA_MALLOC)" + CONFIG_SETTINGS+=" --config-settings=cmake.define.FVDB_USE_SYNC_CUDA_MALLOC=ON" + is_config_arg_handled=true fi fi @@ -422,22 +428,32 @@ elif [ "$BUILD_TYPE" == "ctest" ]; then # --- Find and Run Tests --- echo "Searching for test build directory..." - # Find CMakeCache.txt to locate the build root - CMAKE_CACHE=$(find build -name CMakeCache.txt -type f -print -quit 2>/dev/null) + # Find CMakeCache.txt to locate the build root. Every vendored dependency that + # CPM/FetchContent configures leaves a CMakeCache.txt of its own under _deps, so + # that subtree is pruned; otherwise find can return a dependency's cache first, + # since it walks directories in filesystem order rather than a defined one. + FOUND_CMAKE_CACHE=false + BUILD_DIR="" + while IFS= read -r cmake_cache; do + FOUND_CMAKE_CACHE=true + # Construct the test directory path (where CTestTestfile.cmake is generated) + # This discovers all tests from both src/tests/ and src/dispatch/ + candidate_dir="$(dirname "$cmake_cache")/src" + if [ -f "$candidate_dir/CTestTestfile.cmake" ]; then + BUILD_DIR="$candidate_dir" + break + fi + done < <(find build -type d -name _deps -prune -o -name CMakeCache.txt -type f -print 2>/dev/null) - if [ -z "$CMAKE_CACHE" ]; then + if [ "$FOUND_CMAKE_CACHE" = false ]; then echo "Error: Could not find CMakeCache.txt in build directory" echo "Please build the project first with tests enabled:" echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON" exit 1 fi - # Construct the test directory path (where CTestTestfile.cmake is generated) - # This discovers all tests from both src/tests/ and src/dispatch/ - BUILD_DIR="$(dirname "$CMAKE_CACHE")/src" - - if [ ! -f "$BUILD_DIR/CTestTestfile.cmake" ]; then - echo "Error: No CTestTestfile.cmake found in $BUILD_DIR" + if [ -z "$BUILD_DIR" ]; then + echo "Error: No CTestTestfile.cmake found under build/*/src" echo "Please enable tests by building with:" echo "pip install . -C cmake.define.FVDB_BUILD_TESTS=ON" exit 1 diff --git a/src/cmake/get_nanovdb.cmake b/src/cmake/get_nanovdb.cmake index d8ce343af..731a34f21 100644 --- a/src/cmake/get_nanovdb.cmake +++ b/src/cmake/get_nanovdb.cmake @@ -4,7 +4,7 @@ CPMAddPackage( NAME nanovdb GITHUB_REPOSITORY AcademySoftwareFoundation/openvdb - GIT_TAG f9754140ba6031813b37d8e1b239ed0253ebd96d + GIT_TAG b7fc4fc7af73e84071b5f625482cfad4c50eb247 SOURCE_SUBDIR nanovdb/nanovdb DOWNLOAD_ONLY YES ) @@ -15,3 +15,19 @@ if(nanovdb_ADDED) add_library(nanovdb INTERFACE) target_include_directories(nanovdb INTERFACE ${nanovdb_SOURCE_DIR}/nanovdb) endif() + +# nanovdb::util::cuda::mallocAsync resolves to either cudaMallocAsync or plain +# cudaMalloc depending on whether NANOVDB_USE_SYNC_CUDA_MALLOC is defined. The +# choice is made in a header, so the macro has to be visible to every C++/CUDA +# translation unit that includes NanoVDB or an inline function ends up with two +# different bodies across the build. Hence a directory-wide compile definition +# rather than a per-target one. +# +# Async allocation depends on GPU unified memory, which is unavailable on vGPU +# slices (e.g. fractional-GPU cloud instances), so builds targeting those need +# the synchronous path. +option(FVDB_USE_SYNC_CUDA_MALLOC "Define NANOVDB_USE_SYNC_CUDA_MALLOC to force synchronous NanoVDB CUDA allocation" OFF) +if(FVDB_USE_SYNC_CUDA_MALLOC) + message(STATUS "FVDB: compiling with NANOVDB_USE_SYNC_CUDA_MALLOC") + add_compile_definitions(NANOVDB_USE_SYNC_CUDA_MALLOC) +endif()