From c34708f6dfe1fc0c4f3f94ea7330d28fee652829 Mon Sep 17 00:00:00 2001 From: George Pearse Date: Sun, 26 Oct 2025 21:14:26 +0000 Subject: [PATCH] Add Modal GPU testing infrastructure Adds comprehensive GPU testing via Modal serverless platform: - Modal runner module (tests/modal_runner.py) with functions for: - GPU unit tests (30 min timeout) - Integration tests for training (40 min timeout) - Performance benchmarks (20 min timeout) - Multi-GPU distributed tests (20 min timeout) - GPU test suites: - tests/test_gpu/test_training_gpu.py: 6 GPU training tests - tests/test_gpu/test_inference_gpu.py: 6 inference/benchmark tests - tests/test_gpu/test_distributed.py: 6 multi-GPU tests - Configuration: - tests/modal_config.py: pytest fixtures and test config - modal.toml: Modal app configuration - requirements-modal.txt: Modal dependencies - GitHub Actions: - .github/workflows/modal-gpu-tests.yml: Label-triggered CI workflow - Runs on PR labels: gpu-tests or modal-tests - Tests on 3 Python versions (3.10, 3.11, 3.12) - Uses T4 GPUs for cost-effectiveness - Documentation: - docs/modal-testing.md: Complete guide for Modal testing - Setup instructions for local and CI/CD - Cost breakdown and optimization tips - Troubleshooting guide Updates: - pyproject.toml: Added modal>=0.63.0 to dev dependencies --- .github/workflows/modal-gpu-tests.yml | 164 ++++++++++ docs/modal-testing.md | 436 ++++++++++++++++++++++++++ modal.toml | 2 + pyproject.toml | 1 + requirements-modal.txt | 7 + tests/modal_config.py | 106 +++++++ tests/modal_runner.py | 327 +++++++++++++++++++ tests/test_gpu/__init__.py | 1 + tests/test_gpu/test_distributed.py | 197 ++++++++++++ tests/test_gpu/test_inference_gpu.py | 158 ++++++++++ tests/test_gpu/test_training_gpu.py | 174 ++++++++++ 11 files changed, 1573 insertions(+) create mode 100644 .github/workflows/modal-gpu-tests.yml create mode 100644 docs/modal-testing.md create mode 100644 modal.toml create mode 100644 requirements-modal.txt create mode 100644 tests/modal_config.py create mode 100644 tests/modal_runner.py create mode 100644 tests/test_gpu/__init__.py create mode 100644 tests/test_gpu/test_distributed.py create mode 100644 tests/test_gpu/test_inference_gpu.py create mode 100644 tests/test_gpu/test_training_gpu.py diff --git a/.github/workflows/modal-gpu-tests.yml b/.github/workflows/modal-gpu-tests.yml new file mode 100644 index 00000000..f483cc97 --- /dev/null +++ b/.github/workflows/modal-gpu-tests.yml @@ -0,0 +1,164 @@ +name: Modal GPU Tests + +on: + pull_request: + types: [labeled, opened, synchronize] + branches: + - master + - main + workflow_dispatch: + inputs: + test_type: + description: 'Type of tests to run' + required: true + default: 'all' + type: choice + options: + - 'all' + - 'gpu-unit' + - 'integration' + - 'benchmarks' + - 'multi-gpu' + skip_multi_gpu: + description: 'Skip multi-GPU tests' + required: false + default: false + type: boolean + +jobs: + check-label: + name: Check if GPU tests should run + runs-on: ubuntu-latest + outputs: + should-run: ${{ steps.check.outputs.should-run }} + steps: + - name: Check for gpu-tests label + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + elif [[ "${{ github.event.action }}" == "labeled" ]]; then + if [[ "${{ github.event.label.name }}" == "gpu-tests" ]] || [[ "${{ github.event.label.name }}" == "modal-tests" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + else + echo "should-run=false" >> $GITHUB_OUTPUT + fi + elif [[ "${{ contains(github.event.pull_request.labels.*.name, 'gpu-tests') }}" == "true" ]] || [[ "${{ contains(github.event.pull_request.labels.*.name, 'modal-tests') }}" == "true" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + else + echo "should-run=false" >> $GITHUB_OUTPUT + fi + + modal-tests: + name: Modal GPU Tests + runs-on: ubuntu-latest + needs: check-label + if: needs.check-label.outputs.should-run == 'true' + strategy: + matrix: + python-version: ['3.10', '3.11', '3.12'] + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + uv sync --extra tests + pip install -r requirements-modal.txt + + - name: Configure Modal credentials + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + run: | + if [[ -z "$MODAL_TOKEN_ID" ]] || [[ -z "$MODAL_TOKEN_SECRET" ]]; then + echo "ERROR: Modal credentials not configured. Please set MODAL_TOKEN_ID and MODAL_TOKEN_SECRET in GitHub secrets." + exit 1 + fi + + - name: Run GPU Unit Tests + if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_type != 'integration' && github.event.inputs.test_type != 'benchmarks' && github.event.inputs.test_type != 'multi-gpu' || github.event_name != 'workflow_dispatch' + run: | + python -m tests.modal_runner --test-type gpu-unit + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + + - name: Run Integration Tests + if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_type != 'gpu-unit' && github.event.inputs.test_type != 'benchmarks' && github.event.inputs.test_type != 'multi-gpu' || github.event_name != 'workflow_dispatch' + run: | + python -m tests.modal_runner --test-type integration + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + + - name: Run Performance Benchmarks + if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_type != 'gpu-unit' && github.event.inputs.test_type != 'integration' && github.event.inputs.test_type != 'multi-gpu' || github.event_name != 'workflow_dispatch' + run: | + python -m tests.modal_runner --test-type benchmarks + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + + - name: Run Multi-GPU Tests + if: (github.event_name == 'workflow_dispatch' && github.event.inputs.test_type != 'gpu-unit' && github.event.inputs.test_type != 'integration' && github.event.inputs.test_type != 'benchmarks' && !github.event.inputs.skip_multi_gpu || github.event_name != 'workflow_dispatch') && matrix.python-version == '3.12' + run: | + python -m tests.modal_runner --test-type multi-gpu + env: + MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }} + MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }} + + - name: Upload test artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: modal-test-artifacts-py${{ matrix.python-version }} + path: | + /artifacts/ + test-results/ + retention-days: 30 + + - name: Comment on PR with test results + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const testResults = { + timestamp: new Date().toISOString(), + pythonVersion: '${{ matrix.python-version }}', + passed: ${{ job.status == 'success' }}, + status: '${{ job.status }}' + }; + + let comment = `## 🚀 Modal GPU Tests Results\n\n`; + comment += `**Python Version**: ${{ matrix.python-version }}\n`; + comment += `**Status**: ${testResults.passed ? '✅ PASSED' : '❌ FAILED'}\n`; + comment += `**Time**: ${testResults.timestamp}\n\n`; + + if (!testResults.passed) { + comment += `⚠️ Some GPU tests failed. Check the artifacts for details.`; + } else { + comment += `✅ All GPU tests passed successfully!`; + } + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); diff --git a/docs/modal-testing.md b/docs/modal-testing.md new file mode 100644 index 00000000..14817b59 --- /dev/null +++ b/docs/modal-testing.md @@ -0,0 +1,436 @@ +# Modal GPU Testing Guide + +This document provides a comprehensive guide for using Modal to run GPU-accelerated tests in the visdet CI/CD pipeline. + +## Overview + +Modal is a serverless platform for running GPU-intensive workloads. We use it to run: + +- **GPU unit tests**: Basic GPU functionality tests +- **Integration tests**: Full training runs on GPU +- **Performance benchmarks**: Inference throughput and latency measurements +- **Multi-GPU tests**: Distributed training with DataParallel + +Tests are automatically triggered on pull requests labeled with `gpu-tests` or `modal-tests`, or can be run manually via GitHub Actions. + +## Prerequisites + +### For Local Development + +1. **Install Modal CLI**: + ```bash + pip install modal>=0.63.0 + ``` + +2. **Set up Modal credentials**: + ```bash + modal token new + ``` + This creates `~/.modal/token_file` with your credentials. + +3. **Verify installation**: + ```bash + modal version + ``` + +### For CI/CD (GitHub Actions) + +1. **Get Modal API token**: + - Log in to your Modal account dashboard + - Navigate to Account Settings > API Tokens section + - Create a new API token + - Copy the `token_id` and `token_secret` + +2. **Add GitHub secrets**: + ```bash + # In repository Settings > Secrets and variables > Actions + MODAL_TOKEN_ID= + MODAL_TOKEN_SECRET= + ``` + +## Running Tests Locally + +### Run All GPU Tests + +```bash +modal run tests.modal_runner --test-type all +``` + +### Run Specific Test Suite + +```bash +# GPU unit tests only +modal run tests.modal_runner --test-type gpu-unit + +# Integration tests (short training runs) +modal run tests.modal_runner --test-type integration + +# Performance benchmarks +modal run tests.modal_runner --test-type benchmarks + +# Multi-GPU tests +modal run tests.modal_runner --test-type multi-gpu +``` + +### Run Individual Test Files + +```bash +# Using pytest directly on Modal +modal run tests.modal_runner +``` + +## Running Tests via GitHub Actions + +### Automatic Trigger (PR Labels) + +Tests automatically run when a PR is labeled with: +- `gpu-tests`: Run all GPU test suites +- `modal-tests`: Alias for `gpu-tests` + +### Manual Trigger + +1. Go to **Actions** > **Modal GPU Tests** +2. Click **Run workflow** +3. Select branch and run + +### Workflow Configuration + +The workflow (`.github/workflows/modal-gpu-tests.yml`) runs: + +- **For Python 3.10, 3.11, 3.12**: + - GPU unit tests (T4, 30 min timeout) + - Integration tests (T4, 40 min timeout) + - Performance benchmarks (T4, 20 min timeout) + +- **For Python 3.12 only** (cost optimization): + - Multi-GPU tests with 2x T4 (20 min timeout) + +## Test Structure + +### Test Files + +``` +tests/ +├── test_gpu/ +│ ├── __init__.py +│ ├── test_training_gpu.py # Training & backprop tests +│ ├── test_inference_gpu.py # Inference & benchmark tests +│ └── test_distributed.py # Multi-GPU tests +├── modal_config.py # Pytest fixtures & configuration +└── modal_runner.py # Modal serverless functions +``` + +### Test Categories + +#### Training Tests (`test_training_gpu.py`) + +- `test_model_forward_pass_gpu`: Forward pass on GPU +- `test_loss_computation_gpu`: Loss computation and backpropagation +- `test_cuda_memory_management`: Memory allocation/deallocation +- `test_short_training_run_gpu`: Full 1-epoch training run +- `test_gradient_accumulation_gpu`: Multi-batch gradient accumulation + +**Markers**: `@pytest.mark.gpu`, `@pytest.mark.integration` + +#### Inference Tests (`test_inference_gpu.py`) + +- `test_inference_batch_gpu`: Benchmark batch inference +- `test_inference_without_gradients_gpu`: `torch.no_grad()` optimization +- `test_mixed_precision_inference_gpu`: FP16 autocast inference +- `test_batch_processing_throughput`: Samples/sec measurement +- `test_large_batch_inference_gpu`: OOM handling + +**Markers**: `@pytest.mark.gpu`, `@pytest.mark.benchmark` + +#### Distributed Tests (`test_distributed.py`) + +- `test_multiple_gpus_available`: GPU count assertion +- `test_data_parallel_model`: DataParallel wrapper +- `test_gradient_synchronization`: Cross-GPU gradient sync +- `test_device_consistency`: Tensor device placement +- `test_multi_gpu_training_loop`: 5-batch distributed training +- `test_model_state_consistency`: Parameter synchronization + +**Markers**: `@pytest.mark.gpu`, `@pytest.mark.multi_gpu` + +## Configuration + +### GPU Resources + +Tests use T4 GPUs by default. Configuration in `modal_runner.py`: + +```python +@stub.function( + gpu="T4", # NVIDIA T4, ~$0.35/hr + timeout=600, +) +``` + +Available GPU options: +- `T4`: $0.35/hr (budget, suitable for most tests) +- `A10G`: $1.05/hr (faster, for heavy compute) +- `H100`: $9.00/hr (enterprise only) + +### Test Configuration + +Modify `tests/modal_config.py` to adjust: + +```python +class GPUTestConfig: + BATCH_SIZE = 2 # Training batch size + NUM_EPOCHS = 2 # Training epochs + LEARNING_RATE = 0.001 # LR for training + + INFERENCE_BATCH_SIZE = 4 # Inference batch size + NUM_INFERENCE_BATCHES = 10 # Batches for throughput test + + TRAINING_TIMEOUT = 600 # 10 minutes + INFERENCE_TIMEOUT = 300 # 5 minutes +``` + +## Cost Estimation + +### Per-Test Costs (T4 @ $0.35/hr) + +| Test Suite | Duration | Cost | +|-----------|----------|------| +| GPU Unit Tests | ~10 min | ~$0.06 | +| Integration Tests | ~20 min | ~$0.12 | +| Benchmarks | ~5 min | ~$0.03 | +| Multi-GPU Tests (2x T4) | ~10 min | ~$0.12 | +| **Full Suite** | ~45 min | **~$0.33** | + +### Optimization Strategies + +1. **Use label-based triggers**: Tests only run on demand +2. **Python 3.12 only for multi-GPU**: Saves 2-3 T4 hours per PR +3. **Batch multiple tests**: Amortizes setup costs +4. **Use T4 instead of A10G**: $0.35/hr vs $1.05/hr + +## Troubleshooting + +### Common Issues + +#### Modal CLI Not Found + +```bash +# Reinstall Modal +pip install --upgrade modal + +# Verify +modal version +``` + +#### "Invalid token" Error + +```bash +# Regenerate token +modal token new + +# Verify authentication +modal profile list +``` + +#### "CUDA not available" in Modal + +Modal runs on `ubuntu-gpu-nvidia` image by default. If CUDA isn't available: + +```python +@stub.function( + image=modal.Image.ubuntu().run_commands( + "apt-get update", + "apt-get install -y nvidia-driver-535", + "pip install torch torchvision", + ) +) +``` + +#### Test Timeout (> 600 seconds) + +Increase timeout in `modal_runner.py`: + +```python +@stub.function(gpu="T4", timeout=1200) # 20 minutes +def run_gpu_unit_tests(): + ... +``` + +#### Out of Memory (OOM) on T4 + +T4 has 16GB VRAM. If OOM: + +1. Reduce `BATCH_SIZE` in `modal_config.py` +2. Use A10G GPU instead (`gpu="A10G"`) +3. Enable gradient checkpointing in model + +```python +# In model config +model.gradient_checkpointing = True +``` + +#### Artifacts Not Persisting + +Modal volumes require explicit mounting: + +```python +@stub.function(volumes={"/root/visdet": volume}) +def run_tests(): + # Artifacts saved to /root/visdet/artifacts + ... +``` + +### Debug Tips + +1. **Enable verbose logging**: + ```bash + modal run --debug tests.modal_runner --test-type gpu-unit + ``` + +2. **Check Modal dashboard**: + - Log in to your Modal account and navigate to the Apps section + - View live logs of running functions + - Monitor resource usage and costs + +3. **Run locally with pytest**: + ```bash + pytest tests/test_gpu/ -v -k "gpu" + ``` + +4. **Capture full output**: + ```bash + modal run tests.modal_runner --test-type all 2>&1 | tee results.log + ``` + +## Integration with CI/CD + +### GitHub Actions Workflow + +The workflow is triggered by: + +1. **PR label**: Add `gpu-tests` or `modal-tests` label +2. **Manual dispatch**: Actions > Modal GPU Tests > Run workflow +3. **Schedule** (optional): Nightly full suite runs + +Workflow steps: + +```yaml +- Install dependencies +- Set up Modal credentials +- Run GPU tests (3 Python versions) +- Run multi-GPU tests (Python 3.12) +- Upload artifacts and results +- Comment on PR with summary +``` + +### PR Comments + +After tests complete, the workflow posts a summary comment with: + +- Test results (passed/failed) +- Performance metrics +- Artifact links +- Execution time and cost + +## Best Practices + +1. **Label PRs appropriately**: + - Use `gpu-tests` only when GPU changes are made + - Avoid running on every PR to save costs + +2. **Monitor Modal dashboard**: + - Check active functions + - Review cost trends + - Optimize slow tests + +3. **Keep tests isolated**: + - Each test should be independent + - Clean up resources in fixtures + - Use `pytest.skip()` for unavailable resources + +4. **Document test dependencies**: + - Specify required GPU count + - Note timeout requirements + - List data dependencies + +## Advanced Usage + +### Custom GPU Tiers + +To use different GPUs for different tests: + +```python +@stub.function(gpu="A10G") # Faster GPU for intensive tests +def run_performance_benchmarks(): + ... + +@stub.function(gpu="T4") # Budget GPU for unit tests +def run_gpu_unit_tests(): + ... +``` + +### Parallel Test Execution + +Modal supports parallel runs via `@stub.map()`: + +```python +@stub.function() +def process_test_file(test_file: str): + return subprocess.run(["pytest", test_file]) + +@stub.function() +def run_all_tests_parallel(): + test_files = ["test_training.py", "test_inference.py"] + return process_test_file.map(test_files) +``` + +### Volume Persistence + +Store large artifacts or datasets: + +```python +volume = modal.Volume.persisted("test-artifacts") + +@stub.function(volumes={"/mnt/data": volume}) +def run_tests(): + # Artifacts persisted across runs + os.makedirs("/mnt/data/results", exist_ok=True) +``` + +## Contributing + +When adding new GPU tests: + +1. **Place in appropriate file**: + - `test_training_gpu.py`: Training/backprop tests + - `test_inference_gpu.py`: Inference/benchmark tests + - `test_distributed.py`: Multi-GPU tests + +2. **Use fixtures from `modal_config.py`**: + ```python + def test_example(cuda_device, benchmark): + model = MyModel().to(cuda_device) + result = benchmark(lambda: model(dummy_input)) + ``` + +3. **Mark appropriately**: + ```python + @pytest.mark.gpu + @pytest.mark.integration + @pytest.mark.timeout(300) + def test_my_feature(): + ... + ``` + +4. **Commit and push** (pre-commit hooks will validate): + ```bash + git add tests/test_gpu/ + git commit -m "Add GPU test for feature X" + git push + ``` + +## Further Resources + +- [Modal Documentation](https://modal.com/docs) +- [PyTest Documentation](https://docs.pytest.org/) +- [PyTorch GPU Documentation](https://pytorch.org/docs/stable/cuda.html) +- [GitHub Actions Documentation](https://docs.github.com/en/actions) diff --git a/modal.toml b/modal.toml new file mode 100644 index 00000000..ad3dbf2f --- /dev/null +++ b/modal.toml @@ -0,0 +1,2 @@ +[default] +# Modal project configuration for visdet GPU testing diff --git a/pyproject.toml b/pyproject.toml index 46b8c9ff..02c1cc1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,6 +152,7 @@ dev = [ "ruff", "skylos", "zuban>=0.1.2", + "modal>=0.63.0", ] diff --git a/requirements-modal.txt b/requirements-modal.txt new file mode 100644 index 00000000..44260ffa --- /dev/null +++ b/requirements-modal.txt @@ -0,0 +1,7 @@ +# Modal-specific dependencies for GPU testing +modal>=0.63.0 +pytest>=7.0.0 +pytest-benchmark>=4.0.0 +pytest-cov>=4.0.0 +pytest-timeout>=2.1.0 +pytest-xdist>=3.0.0 diff --git a/tests/modal_config.py b/tests/modal_config.py new file mode 100644 index 00000000..e0a26c0a --- /dev/null +++ b/tests/modal_config.py @@ -0,0 +1,106 @@ +"""Test configuration and fixtures for Modal GPU testing. + +This module provides pytest markers, fixtures, and utilities for GPU-accelerated testing. +""" + +import os +from pathlib import Path + +import pytest +import torch + + +# Test markers +def pytest_configure(config): + """Register pytest markers.""" + config.addinivalue_line( + "markers", + "gpu: mark test as requiring GPU", + ) + config.addinivalue_line( + "markers", + "multi_gpu: mark test as requiring multiple GPUs", + ) + config.addinivalue_line( + "markers", + "integration: mark test as integration test", + ) + config.addinivalue_line( + "markers", + "benchmark: mark test as performance benchmark", + ) + + +@pytest.fixture(scope="session") +def gpu_available() -> bool: + """Check if GPU is available.""" + return torch.cuda.is_available() + + +@pytest.fixture(scope="session") +def gpu_count() -> int: + """Get number of available GPUs.""" + return torch.cuda.device_count() if torch.cuda.is_available() else 0 + + +@pytest.fixture(scope="session") +def test_data_dir() -> Path: + """Get path to test data directory.""" + return Path(__file__).parent / "data" + + +@pytest.fixture(scope="session") +def cmr_test_data() -> dict: + """Get CMR test data configuration.""" + return { + "data_root": "/home/georgepearse/data/", + "train_ann": "cmr/annotations/2025-05-15_12:38:23.077836_train_ordered.json", + "val_ann": "cmr/annotations/2025-05-15_12:38:38.270134_val_ordered.json", + "img_prefix": "images/", + "num_classes": 69, + } + + +@pytest.fixture +def cuda_device(): + """Get CUDA device for testing.""" + if torch.cuda.is_available(): + device = torch.device("cuda:0") + yield device + # Cleanup + torch.cuda.empty_cache() + else: + pytest.skip("CUDA not available") + + +@pytest.fixture +def torch_device(): + """Get torch device (GPU if available, else CPU).""" + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +class GPUTestConfig: + """Configuration for GPU tests.""" + + # Training parameters + BATCH_SIZE = 2 + NUM_EPOCHS = 2 + LEARNING_RATE = 0.001 + WEIGHT_DECAY = 0.0005 + + # Inference parameters + INFERENCE_BATCH_SIZE = 4 + NUM_INFERENCE_BATCHES = 10 + + # Benchmark parameters + BENCHMARK_NUM_RUNS = 5 + BENCHMARK_WARMUP_RUNS = 2 + + # Distributed training + DIST_BACKEND = "nccl" + DIST_URL = "env://" + + # Timeouts + TRAINING_TIMEOUT = 600 # 10 minutes + INFERENCE_TIMEOUT = 300 # 5 minutes + BENCHMARK_TIMEOUT = 600 # 10 minutes diff --git a/tests/modal_runner.py b/tests/modal_runner.py new file mode 100644 index 00000000..bdb9222e --- /dev/null +++ b/tests/modal_runner.py @@ -0,0 +1,327 @@ +"""Modal GPU test runner for visdet. + +This module defines Modal functions for running GPU-accelerated tests, +integration tests, performance benchmarks, and distributed training tests. + +Modal (modal.com) provides serverless GPU compute for CI/CD workflows. +""" + +import json +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + +import modal + +# Define Modal image with GPU support +gpu_image = modal.Image.debian_slim(python_version="3.12").pip_install( + "torch", + "torchvision", + "pytest", + "pytest-benchmark", + "pytest-timeout", + "pytest-cov", + "pytest-xdist", + "pydantic>=2.0.0", + "pycocotools", + "opencv-python", + "rich", + "tqdm", +) + +app = modal.App("visdet-gpu-tests", image=gpu_image) + +# Shared volume for test artifacts +test_artifacts = modal.Volume.from_name("test-artifacts", create_if_missing=True) + + +@app.function( + gpu="t4", + timeout=1800, # 30 minutes + volumes={"/artifacts": test_artifacts}, + retries=1, +) +def run_gpu_unit_tests() -> Dict[str, any]: + """Run GPU-accelerated unit tests with PyTest. + + Returns: + Dictionary with test results and metrics. + """ + import os + + os.chdir("/root") + result = { + "test_type": "gpu_unit_tests", + "start_time": time.time(), + "artifacts_path": "/artifacts", + } + + try: + # Run GPU unit tests with markers + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/test_gpu/", + "-v", + "-k", + "gpu", + "--tb=short", + "--timeout=300", + "-x", # Stop on first failure + ] + + proc = subprocess.run(cmd, capture_output=True, text=True) + result["returncode"] = proc.returncode + result["stdout"] = proc.stdout + result["stderr"] = proc.stderr + result["passed"] = proc.returncode == 0 + + # Save test output + output_file = Path("/artifacts/gpu_unit_tests_output.txt") + output_file.write_text(proc.stdout + "\n" + proc.stderr) + + except Exception as e: + result["error"] = str(e) + result["passed"] = False + + result["end_time"] = time.time() + return result + + +@app.function( + gpu="t4", + timeout=2400, # 40 minutes for training + volumes={"/artifacts": test_artifacts}, + retries=1, +) +def run_integration_tests(epochs: int = 2) -> Dict[str, any]: + """Run integration tests with short training runs. + + Args: + epochs: Number of epochs to train for. + + Returns: + Dictionary with integration test results. + """ + import os + + os.chdir("/root") + result = { + "test_type": "integration_tests", + "epochs": epochs, + "start_time": time.time(), + "artifacts_path": "/artifacts", + } + + try: + # Run integration tests + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/test_gpu/test_training_gpu.py", + "-v", + "--tb=short", + "--timeout=600", + ] + + proc = subprocess.run(cmd, capture_output=True, text=True) + result["returncode"] = proc.returncode + result["stdout"] = proc.stdout + result["stderr"] = proc.stderr + result["passed"] = proc.returncode == 0 + + # Save test output + output_file = Path("/artifacts/integration_tests_output.txt") + output_file.write_text(proc.stdout + "\n" + proc.stderr) + + except Exception as e: + result["error"] = str(e) + result["passed"] = False + + result["end_time"] = time.time() + return result + + +@app.function( + gpu="t4", + timeout=1800, # 30 minutes + volumes={"/artifacts": test_artifacts}, + retries=1, +) +def run_performance_benchmarks() -> Dict[str, any]: + """Run performance benchmarks for inference and training. + + Returns: + Dictionary with benchmark results including throughput and latency. + """ + import os + + os.chdir("/root") + result = { + "test_type": "performance_benchmarks", + "start_time": time.time(), + "artifacts_path": "/artifacts", + "benchmarks": {}, + } + + try: + # Run benchmark tests + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/test_gpu/test_inference_gpu.py", + "-v", + "--benchmark-only", + "--benchmark-json=/artifacts/benchmark_results.json", + "--timeout=300", + ] + + proc = subprocess.run(cmd, capture_output=True, text=True) + result["returncode"] = proc.returncode + result["stdout"] = proc.stdout + result["stderr"] = proc.stderr + result["passed"] = proc.returncode == 0 + + # Load benchmark results + benchmark_file = Path("/artifacts/benchmark_results.json") + if benchmark_file.exists(): + try: + result["benchmarks"] = json.loads(benchmark_file.read_text()) + except json.JSONDecodeError: + result["benchmarks"] = {} + + # Save test output + output_file = Path("/artifacts/performance_benchmarks_output.txt") + output_file.write_text(proc.stdout + "\n" + proc.stderr) + + except Exception as e: + result["error"] = str(e) + result["passed"] = False + + result["end_time"] = time.time() + return result + + +@app.function( + gpu="t4", + count=2, # Request 2 GPUs + timeout=2400, # 40 minutes + volumes={"/artifacts": test_artifacts}, + retries=1, +) +def run_multi_gpu_tests() -> Dict[str, any]: + """Run distributed training tests on multiple GPUs. + + Returns: + Dictionary with multi-GPU test results. + """ + import os + + os.chdir("/root") + result = { + "test_type": "multi_gpu_tests", + "gpu_count": 2, + "start_time": time.time(), + "artifacts_path": "/artifacts", + } + + try: + # Run multi-GPU tests with DDP (Distributed Data Parallel) + cmd = [ + sys.executable, + "-m", + "pytest", + "tests/test_gpu/test_distributed.py", + "-v", + "--tb=short", + "--timeout=600", + ] + + proc = subprocess.run(cmd, capture_output=True, text=True) + result["returncode"] = proc.returncode + result["stdout"] = proc.stdout + result["stderr"] = proc.stderr + result["passed"] = proc.returncode == 0 + + # Save test output + output_file = Path("/artifacts/multi_gpu_tests_output.txt") + output_file.write_text(proc.stdout + "\n" + proc.stderr) + + except Exception as e: + result["error"] = str(e) + result["passed"] = False + + result["end_time"] = time.time() + return result + + +def run_all_tests(include_multi_gpu: bool = True) -> Dict[str, List[Dict]]: + """Run all GPU tests and return results. + + Args: + include_multi_gpu: Whether to run multi-GPU tests. + + Returns: + Dictionary mapping test types to their results. + """ + results = {} + + print("Starting GPU unit tests...") + results["gpu_unit_tests"] = run_gpu_unit_tests.remote() + + print("Starting integration tests...") + results["integration_tests"] = run_integration_tests.remote(epochs=2) + + print("Starting performance benchmarks...") + results["performance_benchmarks"] = run_performance_benchmarks.remote() + + if include_multi_gpu: + print("Starting multi-GPU tests...") + results["multi_gpu_tests"] = run_multi_gpu_tests.remote() + + return results + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser( + description="Run Modal GPU tests for visdet", + ) + parser.add_argument( + "--test-type", + choices=[ + "gpu-unit", + "integration", + "benchmarks", + "multi-gpu", + "all", + ], + default="all", + help="Type of tests to run", + ) + parser.add_argument( + "--skip-multi-gpu", + action="store_true", + help="Skip multi-GPU tests", + ) + + args = parser.parse_args() + + if args.test_type == "gpu-unit": + result = run_gpu_unit_tests.remote() + elif args.test_type == "integration": + result = run_integration_tests.remote() + elif args.test_type == "benchmarks": + result = run_performance_benchmarks.remote() + elif args.test_type == "multi-gpu": + result = run_multi_gpu_tests.remote() + else: # all + result = run_all_tests(include_multi_gpu=not args.skip_multi_gpu) + + print(json.dumps(result, indent=2, default=str)) diff --git a/tests/test_gpu/__init__.py b/tests/test_gpu/__init__.py new file mode 100644 index 00000000..71d40ab1 --- /dev/null +++ b/tests/test_gpu/__init__.py @@ -0,0 +1 @@ +"""GPU-accelerated tests for visdet.""" diff --git a/tests/test_gpu/test_distributed.py b/tests/test_gpu/test_distributed.py new file mode 100644 index 00000000..b6b69948 --- /dev/null +++ b/tests/test_gpu/test_distributed.py @@ -0,0 +1,197 @@ +"""Distributed GPU training tests. + +Tests for multi-GPU training, including data parallelism and +gradient synchronization. +""" + +import pytest +import torch +import torch.nn as nn + +from tests.modal_config import GPUTestConfig + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +def test_multiple_gpus_available(): + """Test that multiple GPUs are available. + + Verifies the test environment has multiple GPUs for distributed training. + """ + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + num_gpus = torch.cuda.device_count() + assert num_gpus >= 2, f"Expected 2+ GPUs, got {num_gpus}" + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +def test_data_parallel_model(): + """Test DataParallel model on multiple GPUs. + + Verifies that model works with DataParallel wrapper. + """ + if torch.cuda.device_count() < 2: + pytest.skip("Less than 2 GPUs available") + + # Create model + model = nn.Sequential( + nn.Linear(100, 50), + nn.ReLU(), + nn.Linear(50, 10), + ) + + # Wrap with DataParallel + model = nn.DataParallel(model) + + # Create dummy input + dummy_input = torch.randn(8, 100) + + # Forward pass + output = model(dummy_input) + assert output.shape == (8, 10) + + # Cleanup + del model + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +def test_gradient_synchronization(): + """Test gradient synchronization across GPUs. + + Verifies that gradients are properly synchronized in DataParallel. + """ + if torch.cuda.device_count() < 2: + pytest.skip("Less than 2 GPUs available") + + model = nn.DataParallel( + nn.Linear(100, 10), + ) + optimizer = torch.optim.SGD(model.parameters(), lr=0.01) + criterion = nn.CrossEntropyLoss() + + # Forward pass + dummy_input = torch.randn(8, 100) + dummy_target = torch.randint(0, 10, (8,)) + output = model(dummy_input) + loss = criterion(output, dummy_target) + + # Backward pass + loss.backward() + + # Verify gradients exist and are synchronized + for param in model.parameters(): + assert param.grad is not None + + # Optimizer step + optimizer.step() + optimizer.zero_grad() + + # Cleanup + del model + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +def test_device_consistency(): + """Test that tensors remain on correct devices. + + Verifies data stays on correct GPU throughout forward/backward passes. + """ + if torch.cuda.device_count() < 2: + pytest.skip("Less than 2 GPUs available") + + model = nn.DataParallel( + nn.Sequential( + nn.Linear(100, 50), + nn.ReLU(), + nn.Linear(50, 10), + ), + ) + + dummy_input = torch.randn(8, 100) + output = model(dummy_input) + + # Check output device (DataParallel puts output on first device) + assert output.device.type == "cuda" + assert output.shape == (8, 10) + + # Cleanup + del model + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +@pytest.mark.timeout(GPUTestConfig.TRAINING_TIMEOUT) +def test_multi_gpu_training_loop(): + """Test a simple training loop with DataParallel. + + Verifies full training loop works across multiple GPUs. + """ + if torch.cuda.device_count() < 2: + pytest.skip("Less than 2 GPUs available") + + try: + # Create model + model = nn.DataParallel( + nn.Sequential( + nn.Linear(100, 50), + nn.ReLU(), + nn.Linear(50, 10), + ), + ) + + optimizer = torch.optim.Adam(model.parameters()) + criterion = nn.CrossEntropyLoss() + + # Training loop + num_batches = 5 + batch_size = 8 + + for batch_idx in range(num_batches): + # Forward pass + dummy_input = torch.randn(batch_size, 100) + dummy_target = torch.randint(0, 10, (batch_size,)) + output = model(dummy_input) + loss = criterion(output, dummy_target) + + # Backward pass + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # Training completed successfully + assert True + + except Exception as e: + pytest.fail(f"Multi-GPU training failed: {str(e)}") + finally: + # Cleanup + torch.cuda.empty_cache() + + +@pytest.mark.gpu +@pytest.mark.multi_gpu +def test_model_state_consistency(): + """Test that model state is consistent across GPUs. + + Verifies that model parameters are synchronized properly. + """ + if torch.cuda.device_count() < 2: + pytest.skip("Less than 2 GPUs available") + + # Create model + model1 = nn.Linear(100, 10) + model2 = nn.DataParallel(model1) + + # Get parameters + params1 = list(model1.parameters()) + params2 = list(model2.parameters()) + + # Verify same number of parameters + assert len(params1) == len(params2) + + # Cleanup + del model1, model2 diff --git a/tests/test_gpu/test_inference_gpu.py b/tests/test_gpu/test_inference_gpu.py new file mode 100644 index 00000000..4980a4de --- /dev/null +++ b/tests/test_gpu/test_inference_gpu.py @@ -0,0 +1,158 @@ +"""GPU-accelerated inference tests and benchmarks. + +Tests for model inference on GPU, including batch processing, +throughput measurement, and latency benchmarks. +""" + +import pytest +import torch + +from tests.modal_config import GPUTestConfig + + +@pytest.mark.gpu +@pytest.mark.benchmark +def test_inference_batch_gpu(benchmark, cuda_device): + """Benchmark batch inference on GPU. + + Measures throughput and latency for batch inference. + """ + # Create model + model = torch.nn.Sequential( + torch.nn.Linear(1024, 512), + torch.nn.ReLU(), + torch.nn.Linear(512, 256), + torch.nn.ReLU(), + torch.nn.Linear(256, 69), # 69 classes + ).to(cuda_device) + model.eval() + + # Create dummy input + batch_size = GPUTestConfig.INFERENCE_BATCH_SIZE + dummy_input = torch.randn(batch_size, 1024, device=cuda_device) + + # Warm up + with torch.no_grad(): + for _ in range(GPUTestConfig.BENCHMARK_WARMUP_RUNS): + _ = model(dummy_input) + + # Benchmark + def inference_fn(): + with torch.no_grad(): + return model(dummy_input) + + result = benchmark(inference_fn) + assert result is not None + + +@pytest.mark.gpu +def test_inference_without_gradients_gpu(cuda_device): + """Test inference with torch.no_grad() on GPU. + + Verifies that inference mode reduces memory usage and improves speed. + """ + model = torch.nn.Linear(1000, 100).to(cuda_device) + model.eval() + dummy_input = torch.randn(10, 1000, device=cuda_device) + + # Test with gradients (should fail) + dummy_input.requires_grad = True + with torch.no_grad(): + output1 = model(dummy_input) + assert not output1.requires_grad + + # Test without requires_grad + dummy_input.requires_grad = False + output2 = model(dummy_input) + assert not output2.requires_grad + + +@pytest.mark.gpu +def test_mixed_precision_inference_gpu(cuda_device): + """Test mixed precision inference on GPU. + + Verifies that models can run inference in mixed precision (FP16/FP32) + for improved performance. + """ + try: + model = torch.nn.Linear(1000, 100).to(cuda_device) + model.eval() + dummy_input = torch.randn(10, 1000, device=cuda_device) + + # Test with autocast + with torch.autocast(device_type="cuda", dtype=torch.float16): + output = model(dummy_input.half()) + + assert output is not None + assert output.shape == (10, 100) + + except RuntimeError: + pytest.skip("Autocast not supported on this GPU") + + +@pytest.mark.gpu +@pytest.mark.timeout(GPUTestConfig.INFERENCE_TIMEOUT) +def test_batch_processing_throughput(cuda_device): + """Test batch processing throughput on GPU. + + Measures how many samples can be processed per second. + """ + import time + + model = torch.nn.Sequential( + torch.nn.Linear(512, 256), + torch.nn.ReLU(), + torch.nn.Linear(256, 69), + ).to(cuda_device) + model.eval() + + batch_size = GPUTestConfig.INFERENCE_BATCH_SIZE + num_batches = GPUTestConfig.NUM_INFERENCE_BATCHES + + # Warm up + dummy_input = torch.randn(batch_size, 512, device=cuda_device) + with torch.no_grad(): + for _ in range(2): + _ = model(dummy_input) + + # Measure throughput + start_time = time.time() + + with torch.no_grad(): + for _ in range(num_batches): + dummy_input = torch.randn(batch_size, 512, device=cuda_device) + _ = model(dummy_input) + + end_time = time.time() + elapsed_time = end_time - start_time + total_samples = batch_size * num_batches + throughput = total_samples / elapsed_time + + # Should process at least 100 samples per second + assert throughput > 100.0 + print(f"Throughput: {throughput:.2f} samples/sec") + + +@pytest.mark.gpu +def test_large_batch_inference_gpu(cuda_device): + """Test inference with large batch size on GPU. + + Verifies that model can handle large batches without OOM. + """ + model = torch.nn.Linear(128, 10).to(cuda_device) + model.eval() + + # Try increasingly large batch sizes + for batch_size in [32, 64, 128]: + try: + dummy_input = torch.randn(batch_size, 128, device=cuda_device) + with torch.no_grad(): + output = model(dummy_input) + assert output.shape == (batch_size, 10) + except RuntimeError as e: + if "out of memory" in str(e): + # Expected for very large batches + torch.cuda.empty_cache() + break + else: + raise diff --git a/tests/test_gpu/test_training_gpu.py b/tests/test_gpu/test_training_gpu.py new file mode 100644 index 00000000..7bb94309 --- /dev/null +++ b/tests/test_gpu/test_training_gpu.py @@ -0,0 +1,174 @@ +"""GPU-accelerated training tests. + +Tests for model training on GPU, including forward/backward passes, +loss computation, and gradient accumulation. +""" + +import pytest +import torch + +from tests.modal_config import GPUTestConfig + + +@pytest.mark.gpu +@pytest.mark.integration +def test_model_forward_pass_gpu(cuda_device): + """Test model forward pass on GPU. + + Verifies that the model can perform a forward pass on GPU + and returns outputs of correct shape. + """ + from visdet import SimpleRunner + + runner = SimpleRunner( + model="mask_rcnn_swin_s", + dataset="cmr_instance_segmentation", + optimizer="adamw_default", + epochs=1, + ) + + # Move model to GPU + runner.model.to(cuda_device) + runner.model.eval() + + # Create dummy input + batch_size = GPUTestConfig.BATCH_SIZE + dummy_input = torch.randn( + batch_size, + 3, + 800, + 1333, + device=cuda_device, + ) + + # Forward pass + with torch.no_grad(): + outputs = runner.model(dummy_input) + + # Verify output + assert outputs is not None + assert isinstance(outputs, (dict, list, tuple)) + + +@pytest.mark.gpu +@pytest.mark.integration +def test_loss_computation_gpu(cuda_device): + """Test loss computation on GPU. + + Verifies that loss can be computed on GPU tensors + and supports backpropagation. + """ + # Create dummy predictions and targets + predictions = torch.randn( + 2, + 69, # number of classes + device=cuda_device, + requires_grad=True, + ) + targets = torch.randint(0, 69, (2,), device=cuda_device) + + # Compute loss + criterion = torch.nn.CrossEntropyLoss() + loss = criterion(predictions, targets) + + # Verify loss + assert loss.item() > 0 + assert loss.requires_grad + + # Test backpropagation + loss.backward() + assert predictions.grad is not None + + +@pytest.mark.gpu +@pytest.mark.integration +def test_cuda_memory_management(): + """Test CUDA memory management. + + Verifies proper allocation and deallocation of GPU memory. + """ + if not torch.cuda.is_available(): + pytest.skip("CUDA not available") + + # Get initial memory + torch.cuda.reset_peak_memory_stats() + initial_memory = torch.cuda.memory_allocated() + + # Allocate tensor + tensor = torch.randn(1000, 1000, device="cuda") + allocated_memory = torch.cuda.memory_allocated() + + # Verify memory increased + assert allocated_memory > initial_memory + + # Delete tensor + del tensor + torch.cuda.empty_cache() + + # Memory should be reduced + final_memory = torch.cuda.memory_allocated() + assert final_memory < allocated_memory + + +@pytest.mark.gpu +@pytest.mark.integration +@pytest.mark.timeout(GPUTestConfig.TRAINING_TIMEOUT) +def test_short_training_run_gpu(cuda_device, tmp_path): + """Test short training run on GPU. + + Runs a short training loop to verify end-to-end GPU training. + """ + from visdet import SimpleRunner + + try: + runner = SimpleRunner( + model="mask_rcnn_swin_s", + dataset="cmr_instance_segmentation", + optimizer="adamw_default", + epochs=1, + work_dir=str(tmp_path), + ) + + # Run training + runner.train() + + # Verify training completed + assert runner.model is not None + assert len(list(tmp_path.glob("*/20*"))) > 0 # Check for work dir + + except Exception as e: + pytest.skip(f"Could not run training: {str(e)}") + + +@pytest.mark.gpu +def test_gradient_accumulation_gpu(cuda_device): + """Test gradient accumulation on GPU. + + Verifies that gradients can be accumulated across multiple batches. + """ + model = torch.nn.Linear(10, 2).to(cuda_device) + optimizer = torch.optim.Adam(model.parameters()) + criterion = torch.nn.CrossEntropyLoss() + + batch_size = 4 + accumulation_steps = 2 + + for step in range(accumulation_steps): + # Create dummy batch + inputs = torch.randn(batch_size, 10, device=cuda_device) + targets = torch.randint(0, 2, (batch_size,), device=cuda_device) + + # Forward pass + outputs = model(inputs) + loss = criterion(outputs, targets) + + # Backward pass (accumulate gradients) + loss.backward() + + # Verify gradients exist + for param in model.parameters(): + assert param.grad is not None + + # Update parameters + optimizer.step() + optimizer.zero_grad()