diff --git a/.github/workflows/cu130.yml b/.github/workflows/cu130.yml index 7ba89345f..af29e5cb5 100644 --- a/.github/workflows/cu130.yml +++ b/.github/workflows/cu130.yml @@ -33,29 +33,19 @@ jobs: start-build-runner: name: Start CPU-only EC2 runner for build needs: [check-changes, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC if: >- needs.check-changes.outputs.should_test == 'true' && (github.event.pull_request.draft == false || github.event_name != 'pull_request_target') - outputs: - label: ${{ steps.start-build-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 runner - id: start-build-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: m6a.8xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: m6a.8xlarge + az-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # BUILD FVDB @@ -205,25 +195,19 @@ jobs: - start-build-runner # required to get output from the start-build-runner job - fvdb-build # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs # but only if the start-build-runner job was not skipped if: ${{ always() && needs.start-build-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # START FVDB TESTS GPU RUNNER @@ -231,26 +215,16 @@ jobs: start-tests-gpu-runner: name: Start EC2 GPU runner for gtests needs: [fvdb-build, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC - outputs: - label: ${{ steps.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 GPU runner - id: start-tests-gpu-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: g6.xlarge # 4 CPU-core, L4 GPU + az-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # RUN FVDB GTESTS @@ -573,22 +547,16 @@ jobs: - fvdb-unit-test # required to wait when the main job is done - fvdb-docs-test # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs # but only if the start-tests-gpu-runner job was not skipped if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} diff --git a/.github/workflows/cu132.yml b/.github/workflows/cu132.yml index 59e55ce3f..7430d6b10 100644 --- a/.github/workflows/cu132.yml +++ b/.github/workflows/cu132.yml @@ -33,29 +33,19 @@ jobs: start-build-runner: name: Start CPU-only EC2 runner for build needs: [check-changes, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC if: >- needs.check-changes.outputs.should_test == 'true' && (github.event.pull_request.draft == false || github.event_name != 'pull_request_target') - outputs: - label: ${{ steps.start-build-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 runner - id: start-build-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: m6a.8xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: m6a.8xlarge + az-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # BUILD FVDB @@ -205,25 +195,19 @@ jobs: - start-build-runner # required to get output from the start-build-runner job - fvdb-build # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs # but only if the start-build-runner job was not skipped if: ${{ always() && needs.start-build-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # START FVDB TESTS GPU RUNNER @@ -231,26 +215,16 @@ jobs: start-tests-gpu-runner: name: Start EC2 GPU runner for gtests needs: [fvdb-build, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC - outputs: - label: ${{ steps.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 GPU runner - id: start-tests-gpu-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: g6.xlarge # 4 CPU-core, L4 GPU + az-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # RUN FVDB GTESTS @@ -573,22 +547,16 @@ jobs: - fvdb-unit-test # required to wait when the main job is done - fvdb-docs-test # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs # but only if the start-tests-gpu-runner job was not skipped if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} diff --git a/.github/workflows/nightly-publish.yml b/.github/workflows/nightly-publish.yml index 0e11e941a..f332d1e25 100644 --- a/.github/workflows/nightly-publish.yml +++ b/.github/workflows/nightly-publish.yml @@ -59,42 +59,29 @@ jobs: name: Start CPU-only EC2 runner for nightly build needs: [check-new-commits, versions] if: needs.check-new-commits.outputs.should_build == 'true' - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Stagger job starts to avoid API rate limits - run: | - DELAY=$((RANDOM % 16)) - echo "Delaying start by ${DELAY} seconds for Python ${MATRIX_PYTHON_VERSION}" - sleep $DELAY - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 runner - id: start-build-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: m6a.xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }} - label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - aws-resource-tags: > - [ - {"Key": "RunnerLabel", "Value": "nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, - {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, - {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, - {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, - {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} - ] + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: m6a.xlarge + az-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + runner-label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-resource-tags: > + [ + {"Key": "RunnerLabel", "Value": "nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, + {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, + {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, + {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, + {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} + ] + stagger-seconds: 15 + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # BUILD NIGHTLY WHEELS @@ -360,47 +347,16 @@ jobs: - start-build-runner - nightly-build - versions - runs-on: ubuntu-latest if: ${{ always() && needs.start-build-runner.result != 'skipped' }} permissions: id-token: write # Required for AWS OIDC strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Find EC2 instance ID by label - id: find-instance - run: | - LABEL="nightly-${MATRIX_PYTHON_VERSION}-pt${MATRIX_TORCH_VERSION}-cu${MATRIX_CUDA_VERSION}-${{ github.run_id }}" - echo "Looking for instance with RunnerLabel: $LABEL" - - INSTANCE_ID=$(aws ec2 describe-instances \ - --filters "Name=tag:RunnerLabel,Values=$LABEL" "Name=instance-state-name,Values=running,pending,stopping,stopped" \ - --query 'Reservations[0].Instances[0].InstanceId' \ - --output text) - - if [ "$INSTANCE_ID" == "None" ] || [ -z "$INSTANCE_ID" ]; then - echo "ERROR: No instance found with RunnerLabel=$LABEL" - echo "instance-id=" >> $GITHUB_OUTPUT - exit 1 - else - echo "Found instance: $INSTANCE_ID" - echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT - fi - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - MATRIX_TORCH_VERSION: ${{ matrix.torch-version }} - MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }} - - name: Stop EC2 runner - if: steps.find-instance.outputs.instance-id != '' - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: nightly-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e020a2a44..baf956175 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -119,44 +119,30 @@ jobs: start-build-runner: name: Start CPU-only EC2 runner for build - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC needs: versions strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Stagger job starts to avoid API rate limits - run: | - # Random delay between 0 and 15 seconds to stagger runner registrations - DELAY=$((RANDOM % 16)) - echo "Delaying start by ${DELAY} seconds for Python ${MATRIX_PYTHON_VERSION}" - sleep $DELAY - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 runner - id: start-build-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: m6a.8xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }} - label: ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - aws-resource-tags: > - [ - {"Key": "RunnerLabel", "Value": "ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, - {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, - {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, - {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, - {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} - ] + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: m6a.8xlarge + az-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + runner-label: ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-resource-tags: > + [ + {"Key": "RunnerLabel", "Value": "ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, + {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, + {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, + {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, + {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} + ] + stagger-seconds: 15 + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} fvdb-build: name: fVDB Build @@ -533,42 +519,29 @@ jobs: name: Start GPU EC2 runner for validation needs: [pr-flags, fvdb-build, versions] if: ${{ !cancelled() && needs.fvdb-build.result == 'success' && (needs.pr-flags.outputs.release_push == 'true' || needs.pr-flags.outputs.release == 'true' || inputs.run_validation == true) }} - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Stagger job starts to avoid API rate limits - run: | - DELAY=$((RANDOM % 16)) - echo "Delaying start by ${DELAY} seconds for Python ${MATRIX_PYTHON_VERSION}" - sleep $DELAY - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 GPU runner - id: start-validation-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} - label: val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - aws-resource-tags: > - [ - {"Key": "RunnerLabel", "Value": "val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, - {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, - {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, - {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, - {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} - ] + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: g6.xlarge + az-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + runner-label: val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-resource-tags: > + [ + {"Key": "RunnerLabel", "Value": "val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }}"}, + {"Key": "PythonVersion", "Value": "${{ matrix.python-version }}"}, + {"Key": "TorchVersion", "Value": "${{ matrix.torch-version }}"}, + {"Key": "CudaVersion", "Value": "${{ matrix.cuda-version }}"}, + {"Key": "GitHubRunId", "Value": "${{ github.run_id }}"} + ] + stagger-seconds: 15 + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} validate-smoke-test: name: Smoke test built wheel @@ -701,50 +674,19 @@ jobs: - validate-smoke-test - validate-unit-tests - versions - runs-on: ubuntu-latest if: ${{ always() && needs.start-validation-runner.result != 'skipped' }} permissions: id-token: write # Required for AWS OIDC strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Find EC2 instance ID by label - id: find-instance - run: | - LABEL="val-${MATRIX_PYTHON_VERSION}-pt${MATRIX_TORCH_VERSION}-cu${MATRIX_CUDA_VERSION}-${{ github.run_id }}" - echo "Looking for instance with RunnerLabel: $LABEL" - - INSTANCE_ID=$(aws ec2 describe-instances \ - --filters "Name=tag:RunnerLabel,Values=$LABEL" "Name=instance-state-name,Values=running,pending,stopping,stopped" \ - --query 'Reservations[0].Instances[0].InstanceId' \ - --output text) - - if [ "$INSTANCE_ID" == "None" ] || [ -z "$INSTANCE_ID" ]; then - echo "ERROR: No instance found with RunnerLabel=$LABEL" - echo "instance-id=" >> $GITHUB_OUTPUT - exit 1 - else - echo "Found instance: $INSTANCE_ID" - echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT - fi - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - MATRIX_TORCH_VERSION: ${{ matrix.torch-version }} - MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }} - - name: Stop EC2 runner - if: steps.find-instance.outputs.instance-id != '' - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: val-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} fvdb-build-stop-runner: name: Stop CPU-only EC2 runner for build @@ -752,48 +694,16 @@ jobs: - start-build-runner # required to get output from the start-build-runner job - fvdb-build # required to wait when the main job is done - versions - runs-on: ubuntu-latest if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs permissions: id-token: write # Required for AWS OIDC strategy: fail-fast: false matrix: ${{ fromJSON(needs.versions.outputs.publish-matrix) }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Find EC2 instance ID by label - id: find-instance - run: | - LABEL="ec2-${MATRIX_PYTHON_VERSION}-pt${MATRIX_TORCH_VERSION}-cu${MATRIX_CUDA_VERSION}-${{ github.run_id }}" - echo "Looking for instance with RunnerLabel: $LABEL" - - # Query by the custom RunnerLabel tag we set during instance creation - INSTANCE_ID=$(aws ec2 describe-instances \ - --filters "Name=tag:RunnerLabel,Values=$LABEL" "Name=instance-state-name,Values=running,pending,stopping,stopped" \ - --query 'Reservations[0].Instances[0].InstanceId' \ - --output text) - - if [ "$INSTANCE_ID" == "None" ] || [ -z "$INSTANCE_ID" ]; then - echo "ERROR: No instance found with RunnerLabel=$LABEL" - echo "instance-id=" >> $GITHUB_OUTPUT - exit 1 - else - echo "Found instance: $INSTANCE_ID" - echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT - fi - env: - MATRIX_PYTHON_VERSION: ${{ matrix.python-version }} - MATRIX_TORCH_VERSION: ${{ matrix.torch-version }} - MATRIX_CUDA_VERSION: ${{ matrix.cuda-version }} - - name: Stop EC2 runner - if: steps.find-instance.outputs.instance-id != '' - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} - ec2-instance-id: ${{ steps.find-instance.outputs.instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ec2-${{ matrix.python-version }}-pt${{ matrix.torch-version }}-cu${{ matrix.cuda-version }}-${{ github.run_id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} diff --git a/.github/workflows/start-ec2-runner.yml b/.github/workflows/start-ec2-runner.yml new file mode 100644 index 000000000..7d9e2c498 --- /dev/null +++ b/.github/workflows/start-ec2-runner.yml @@ -0,0 +1,248 @@ +# Copyright Contributors to the OpenVDB Project +# SPDX-License-Identifier: Apache-2.0 +# +# Start a self-hosted EC2 runner, retrying with backoff when the region has no +# capacity for the requested instance type. +# +# Every workflow that needs an EC2 runner calls this instead of inlining the +# machulav action, so the retry policy (attempt count, backoff, the AZ sweep, +# the pinned action SHA) lives in exactly one place and cannot drift between +# tests.yml, cu130.yml, cu132.yml, publish.yml and nightly-publish.yml. +# +# Security note: this workflow holds the admin-scoped runner token, so it +# deliberately has no `actions/checkout` and no local actions -- see +# .github/scripts/check_runner_token_policy.py, which enforces that and which +# also covers this file, because it is a *local* reusable workflow. Callers must +# forward that one secret explicitly by name; `secrets: inherit` is rejected by +# the same check. (The token's name is deliberately not spelled in this comment: +# the policy allows it to appear only in the three positions it documents, so a +# single grep for the name enumerates every place it can actually reach.) +name: Start EC2 Runner + +on: + workflow_call: + inputs: + instance-type: + description: EC2 instance type to launch, e.g. m6a.8xlarge or g6.xlarge. + type: string + required: true + az-config: + description: >- + JSON array of availability-zone configs, tried in order by the action + within a single attempt. Usually needs.versions.outputs.aws-*-az-config. + type: string + required: true + aws-role: + description: IAM role to assume via OIDC. + type: string + required: true + aws-region: + description: AWS region to launch in. + type: string + required: true + runner-label: + description: >- + Explicit runner label. Leave empty to let the action generate one -- + only the publish workflows need a predictable label, because they find + the instance again by tag at teardown. + type: string + required: false + default: '' + aws-resource-tags: + description: >- + JSON array of EC2 tags to apply to the instance. The publish workflows + set a RunnerLabel tag here so stop-ec2-runner.yml can find the instance + again without a job output. + type: string + required: false + default: '' + stagger-seconds: + description: >- + Upper bound (inclusive) on a random delay before the first attempt. + Matrix fan-outs set this so N simultaneous registrations do not hit the + GitHub API rate limit at the same instant. 0 disables the delay. + type: number + required: false + default: 0 + backoff-seconds: + description: >- + Seconds to wait after the first failed attempt. The second wait is + twice this, so the default gives 90s then 180s between three attempts. + type: number + required: false + default: 90 + secrets: + EC2_RUNNER_TOKEN: + description: Admin-scoped token used to register the runner with this repo. + required: true + outputs: + label: + description: Runner label to pass as `runs-on` in the job being provisioned. + value: ${{ jobs.start.outputs.label }} + ec2-instance-id: + description: Instance id, to be handed back to stop-ec2-runner.yml. + value: ${{ jobs.start.outputs.ec2-instance-id }} + +permissions: + id-token: write # Required for AWS OIDC + +jobs: + start: + name: Start EC2 runner + runs-on: ubuntu-latest + # Taken from the attempt that actually SUCCEEDED -- see the select step + # below for why "first non-empty" is not the same thing. + outputs: + label: ${{ steps.select.outputs.label }} + ec2-instance-id: ${{ steps.select.outputs.ec2-instance-id }} + steps: + - name: Stagger job start to avoid API rate limits + if: inputs.stagger-seconds > 0 + env: + STAGGER: ${{ inputs.stagger-seconds }} + run: | + DELAY=$((RANDOM % (STAGGER + 1))) + echo "Delaying start by ${DELAY}s to spread out runner registrations." + sleep "${DELAY}" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.aws-region }} + + - name: Start EC2 runner (attempt 1) + id: a1 + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + ec2-instance-type: ${{ inputs.instance-type }} + availability-zones-config: ${{ inputs.az-config }} + # Empty label => the action generates a unique one (its documented + # behaviour). aws-resource-tags is NOT so forgiving: the action does a + # bare JSON.parse() on it, which throws on '', so default to '[]'. + label: ${{ inputs.runner-label }} + aws-resource-tags: ${{ inputs.aws-resource-tags || '[]' }} + + # The action publishes its outputs as soon as the instance launches, i.e. + # BEFORE it waits for the runner to register. So a failed attempt can still + # have left an instance running -- terminate it rather than leak it. + - name: Clean up the instance left behind by attempt 1 + if: steps.a1.outcome == 'failure' && steps.a1.outputs.ec2-instance-id != '' + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ steps.a1.outputs.label }} + ec2-instance-id: ${{ steps.a1.outputs.ec2-instance-id }} + + - name: Back off before attempt 2 + if: steps.a1.outcome == 'failure' + env: + BACKOFF: ${{ inputs.backoff-seconds }} + run: | + echo "::warning::Runner start attempt 1 found no capacity; retrying in ${BACKOFF}s." + sleep "${BACKOFF}" + + - name: Start EC2 runner (attempt 2) + id: a2 + if: steps.a1.outcome == 'failure' + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + ec2-instance-type: ${{ inputs.instance-type }} + availability-zones-config: ${{ inputs.az-config }} + # Empty label => the action generates a unique one (its documented + # behaviour). aws-resource-tags is NOT so forgiving: the action does a + # bare JSON.parse() on it, which throws on '', so default to '[]'. + label: ${{ inputs.runner-label }} + aws-resource-tags: ${{ inputs.aws-resource-tags || '[]' }} + + # The action publishes its outputs as soon as the instance launches, i.e. + # BEFORE it waits for the runner to register. So a failed attempt can still + # have left an instance running -- terminate it rather than leak it. + - name: Clean up the instance left behind by attempt 2 + if: steps.a2.outcome == 'failure' && steps.a2.outputs.ec2-instance-id != '' + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ steps.a2.outputs.label }} + ec2-instance-id: ${{ steps.a2.outputs.ec2-instance-id }} + + - name: Back off before attempt 3 + if: steps.a1.outcome == 'failure' && steps.a2.outcome == 'failure' + env: + BACKOFF: ${{ inputs.backoff-seconds }} + run: | + echo "::warning::Runner start attempt 2 found no capacity; retrying in $((BACKOFF * 2))s." + sleep "$((BACKOFF * 2))" + + - name: Start EC2 runner (attempt 3) + id: a3 + if: steps.a1.outcome == 'failure' && steps.a2.outcome == 'failure' + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: start + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + ec2-instance-type: ${{ inputs.instance-type }} + availability-zones-config: ${{ inputs.az-config }} + # Empty label => the action generates a unique one (its documented + # behaviour). aws-resource-tags is NOT so forgiving: the action does a + # bare JSON.parse() on it, which throws on '', so default to '[]'. + label: ${{ inputs.runner-label }} + aws-resource-tags: ${{ inputs.aws-resource-tags || '[]' }} + + # The action publishes its outputs as soon as the instance launches, i.e. + # BEFORE it waits for the runner to register. So a failed attempt can still + # have left an instance running -- terminate it rather than leak it. + - name: Clean up the instance left behind by attempt 3 + if: steps.a3.outcome == 'failure' && steps.a3.outputs.ec2-instance-id != '' + continue-on-error: true + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ steps.a3.outputs.label }} + ec2-instance-id: ${{ steps.a3.outputs.ec2-instance-id }} + + # Pick the outputs of the attempt that succeeded. This must key on the + # step OUTCOME, not on which output is non-empty: an attempt whose + # instance launched but never registered a runner fails with its outputs + # already populated, so "first non-empty" would hand back a dead label -- + # the dependent job would then queue against a runner that never appears + # instead of failing fast. + - name: Select the successful attempt + id: select + env: + INSTANCE_TYPE: ${{ inputs.instance-type }} + A1_OUTCOME: ${{ steps.a1.outcome }} + A1_LABEL: ${{ steps.a1.outputs.label }} + A1_ID: ${{ steps.a1.outputs.ec2-instance-id }} + A2_OUTCOME: ${{ steps.a2.outcome }} + A2_LABEL: ${{ steps.a2.outputs.label }} + A2_ID: ${{ steps.a2.outputs.ec2-instance-id }} + A3_OUTCOME: ${{ steps.a3.outcome }} + A3_LABEL: ${{ steps.a3.outputs.label }} + A3_ID: ${{ steps.a3.outputs.ec2-instance-id }} + run: | + if [ "${A1_OUTCOME}" = "success" ]; then + LABEL="${A1_LABEL}"; ID="${A1_ID}"; WON=1 + elif [ "${A2_OUTCOME}" = "success" ]; then + LABEL="${A2_LABEL}"; ID="${A2_ID}"; WON=2 + elif [ "${A3_OUTCOME}" = "success" ]; then + LABEL="${A3_LABEL}"; ID="${A3_ID}"; WON=3 + else + echo "::error::Could not obtain a ${INSTANCE_TYPE} instance after 3 attempts across every configured availability zone; the region is likely out of this instance type." + exit 1 + fi + echo "Runner provisioned on attempt ${WON}: ${LABEL} (${ID})" + echo "label=${LABEL}" >> "$GITHUB_OUTPUT" + echo "ec2-instance-id=${ID}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/stop-ec2-runner.yml b/.github/workflows/stop-ec2-runner.yml new file mode 100644 index 000000000..7e8b3b6ab --- /dev/null +++ b/.github/workflows/stop-ec2-runner.yml @@ -0,0 +1,90 @@ +# Copyright Contributors to the OpenVDB Project +# SPDX-License-Identifier: Apache-2.0 +# +# De-register and terminate a self-hosted EC2 runner started by +# start-ec2-runner.yml. Callers keep their own `needs:` and +# `if: always() && needs..result != 'skipped'` guards; this workflow +# is only the teardown itself. +# +# Two call shapes, matching the two ways the repo provisions runners: +# * pass `ec2-instance-id` straight from the start job's output (tests.yml, +# cu130.yml, cu132.yml, nightly-publish.yml); +# * leave it empty and the instance is looked up by its `RunnerLabel` tag +# (publish.yml, whose start job fans out over a matrix, so there is no +# single job output to read). +# +# Security note: like its start counterpart, this workflow holds the +# admin-scoped runner token and therefore never checks out code or runs a +# local action -- see .github/scripts/check_runner_token_policy.py. +name: Stop EC2 Runner + +on: + workflow_call: + inputs: + label: + description: Runner label, as emitted or as given to start-ec2-runner.yml. + type: string + required: true + ec2-instance-id: + description: >- + Instance id emitted by start-ec2-runner.yml. Leave empty to resolve it + from the instance's RunnerLabel tag instead. + type: string + required: false + default: '' + aws-role: + description: IAM role to assume via OIDC. + type: string + required: true + aws-region: + description: AWS region the instance is running in. + type: string + required: true + secrets: + EC2_RUNNER_TOKEN: + description: Admin-scoped token used to de-register the runner. + required: true + +permissions: + id-token: write # Required for AWS OIDC + +jobs: + stop: + name: Stop EC2 runner + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + role-to-assume: ${{ inputs.aws-role }} + aws-region: ${{ inputs.aws-region }} + + - name: Find EC2 instance ID by label + id: find-instance + if: inputs.ec2-instance-id == '' + env: + LABEL: ${{ inputs.label }} + run: | + echo "Looking for instance with RunnerLabel: $LABEL" + INSTANCE_ID=$(aws ec2 describe-instances \ + --filters "Name=tag:RunnerLabel,Values=$LABEL" "Name=instance-state-name,Values=running,pending,stopping,stopped" \ + --query 'Reservations[0].Instances[0].InstanceId' \ + --output text) + + if [ "$INSTANCE_ID" == "None" ] || [ -z "$INSTANCE_ID" ]; then + echo "ERROR: No instance found with RunnerLabel=$LABEL" + echo "instance-id=" >> "$GITHUB_OUTPUT" + exit 1 + fi + echo "Found instance: $INSTANCE_ID" + echo "instance-id=$INSTANCE_ID" >> "$GITHUB_OUTPUT" + + - name: Stop EC2 runner + # The lookup step is skipped when the caller supplied an id, so its + # output is empty then; `||` picks whichever of the two we actually have. + uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 + with: + mode: stop + github-token: ${{ secrets.EC2_RUNNER_TOKEN }} + label: ${{ inputs.label }} + ec2-instance-id: ${{ inputs.ec2-instance-id || steps.find-instance.outputs.instance-id }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 360cf95e0..9fbf12404 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,29 +39,19 @@ jobs: start-build-runner: name: Start CPU-only EC2 runner for build needs: [check-changes, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC if: >- needs.check-changes.outputs.should_test == 'true' && (github.event.pull_request.draft == false || github.event_name != 'pull_request_target') - outputs: - label: ${{ steps.start-build-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 runner - id: start-build-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: m6a.8xlarge - availability-zones-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: m6a.8xlarge + az-config: ${{ needs.versions.outputs.aws-cpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} fvdb-build: name: fVDB Build (Conda) needs: [start-build-runner, versions] # required to start the main job when the runner is ready @@ -159,25 +149,19 @@ jobs: - start-build-runner # required to get output from the start-build-runner job - fvdb-build # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs, but only if the # start-build-runner job was not skipped if: ${{ always() && needs.start-build-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-build-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-build-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## @@ -186,26 +170,16 @@ jobs: start-tests-gpu-runner: name: Start EC2 GPU runner for tests needs: [fvdb-build, versions] - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC - outputs: - label: ${{ steps.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Start EC2 GPU runner - id: start-tests-gpu-runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - ec2-instance-type: g6.xlarge # 4 CPU-core, L4 GPU - availability-zones-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + uses: ./.github/workflows/start-ec2-runner.yml + with: + instance-type: g6.xlarge # 4 CPU-core, L4 GPU + az-config: ${{ needs.versions.outputs.aws-gpu-az-config }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} ############################################################################## # RUN FVDB GTESTS @@ -446,22 +420,16 @@ jobs: - fvdb-unit-tests # required to wait when the main job is done - fvdb-docs-test # required to wait when the main job is done - versions - runs-on: ubuntu-latest permissions: id-token: write # Required for AWS OIDC # required to stop the runner even if the error happened in the previous jobs # but only if the start-tests-gpu-runner job was not skipped if: ${{ always() && needs.start-tests-gpu-runner.result != 'skipped' }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - role-to-assume: ${{ needs.versions.outputs.aws-role }} - aws-region: ${{ needs.versions.outputs.aws-region }} - - name: Stop EC2 runner - uses: machulav/ec2-github-runner@343a1b2ae682e681c3cec9a235d882da17ff04ef # v2.6.1 - with: - mode: stop - github-token: ${{ secrets.EC2_RUNNER_TOKEN }} - label: ${{ needs.start-tests-gpu-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + uses: ./.github/workflows/stop-ec2-runner.yml + with: + label: ${{ needs.start-tests-gpu-runner.outputs.label }} + ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }} + aws-role: ${{ needs.versions.outputs.aws-role }} + aws-region: ${{ needs.versions.outputs.aws-region }} + secrets: + EC2_RUNNER_TOKEN: ${{ secrets.EC2_RUNNER_TOKEN }} diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index 49cb16077..61ae282ab 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -64,8 +64,10 @@ jobs: echo "Scanning PR workflow files:" ls -1 .github/workflows - # 3a. Repo-specific policy: the admin runner token may only be used as the - # github-token input to machulav/ec2-github-runner. The policy script + # 3a. Repo-specific policy: the admin runner token may only reach + # machulav/ec2-github-runner -- either as that action's github-token + # input, or forwarded by name to a local reusable workflow that this + # same scan covers (never via secrets: inherit). The policy script # and its tests run from the trusted base checkout (not the PR). # Rules 2-4 scan the overlaid PR workflow files; the Rule 1 leak check # runs against the PR head commit tree (read-only) so it also catches