diff --git a/.github/scripts/Publish-MaesterSmokeReport.ps1 b/.github/scripts/Publish-MaesterSmokeReport.ps1 new file mode 100644 index 000000000..f4f3a294a --- /dev/null +++ b/.github/scripts/Publish-MaesterSmokeReport.ps1 @@ -0,0 +1,140 @@ +#Requires -Version 7.2 + +[CmdletBinding()] +param( + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$')] + [string] $Repository, + + [Parameter(Mandatory)] + [ValidatePattern('^smoke-[0-9]+-[0-9]+$')] + [string] $Tag, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $ReleaseName, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $ReleaseBody, + + [Parameter(Mandatory)] + [ValidateNotNullOrEmpty()] + [string] $AssetPath, + + [Parameter(Mandatory)] + [ValidatePattern('^[A-Za-z0-9_.-]+\.html$')] + [string] $AssetName, + + [ValidateNotNullOrEmpty()] + [string] $ApiBaseUrl = 'https://api.github.com' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$token = $env:MAESTER_REPORTS_TOKEN +if ([string]::IsNullOrWhiteSpace($token)) { + throw 'MAESTER_REPORTS_TOKEN is required to publish the private report.' +} + +$resolvedAsset = Resolve-Path -LiteralPath $AssetPath -ErrorAction Stop +if (-not (Test-Path -LiteralPath $resolvedAsset.Path -PathType Leaf)) { + throw "The Maester HTML report '$AssetPath' does not exist." +} + +$apiUri = [uri]$ApiBaseUrl +if ($apiUri.Scheme -ne 'https' -and -not $apiUri.IsLoopback) { + throw 'ApiBaseUrl must use HTTPS unless it targets a loopback address.' +} +$apiBase = $ApiBaseUrl.TrimEnd('/') +$headers = @{ + Accept = 'application/vnd.github+json' + Authorization = "Bearer $token" + 'User-Agent' = 'maester-action-smoke-test' + 'X-GitHub-Api-Version' = '2022-11-28' +} + +function Get-HttpStatusCode { + param( + [Parameter(Mandatory)] + [System.Management.Automation.ErrorRecord] $ErrorRecord + ) + + try { + return [int]$ErrorRecord.Exception.Response.StatusCode + } catch { + return $null + } +} + +function Get-PrivateRelease { + $escapedTag = [uri]::EscapeDataString($Tag) + $releaseUri = "$apiBase/repos/$Repository/releases/tags/$escapedTag" + + try { + return Invoke-RestMethod -Method Get -Uri $releaseUri -Headers $headers + } catch { + if ((Get-HttpStatusCode -ErrorRecord $_) -eq 404) { + return $null + } + throw + } +} + +$release = Get-PrivateRelease +if ($null -eq $release) { + $createUri = "$apiBase/repos/$Repository/releases" + $createBody = @{ + tag_name = $Tag + target_commitish = 'main' + name = $ReleaseName + body = $ReleaseBody + draft = $false + prerelease = $true + generate_release_notes = $false + make_latest = 'false' + } | ConvertTo-Json + + try { + $release = Invoke-RestMethod ` + -Method Post ` + -Uri $createUri ` + -Headers $headers ` + -ContentType 'application/json' ` + -Body $createBody + } catch { + if ((Get-HttpStatusCode -ErrorRecord $_) -ne 422) { + throw + } + + # All three matrix jobs may try to create the shared release. If + # another operating system won that race, wait for the release to + # become readable instead of failing the remaining report uploads. + for ($attempt = 1; $attempt -le 10 -and $null -eq $release; $attempt++) { + Start-Sleep -Milliseconds 500 + $release = Get-PrivateRelease + } + if ($null -eq $release) { + throw "GitHub reported that release '$Tag' already exists, but it could not be retrieved." + } + } +} + +$existingAssets = @(@($release.assets) | Where-Object name -EQ $AssetName) +if ($existingAssets.Count -gt 0) { + throw "Release '$Tag' already contains an asset named '$AssetName'." +} + +$uploadBase = [string]$release.upload_url -replace '\{\?name,label\}$', '' +$escapedAssetName = [uri]::EscapeDataString($AssetName) +$uploadUri = "${uploadBase}?name=$escapedAssetName" + +[void](Invoke-RestMethod ` + -Method Post ` + -Uri $uploadUri ` + -Headers $headers ` + -ContentType 'text/html; charset=utf-8' ` + -InFile $resolvedAsset.Path) + +Write-Output "Uploaded private report '$AssetName' to $Repository release '$Tag'." diff --git a/.github/workflows/maester-action-smoke-test.md b/.github/workflows/maester-action-smoke-test.md new file mode 100644 index 000000000..f658e2711 --- /dev/null +++ b/.github/workflows/maester-action-smoke-test.md @@ -0,0 +1,156 @@ +# Maester action tenant smoke test + +`maester-action-smoke-test.yaml` exercises the published standard +`maester365/maester-action` on Linux, Windows, and macOS against a real +Microsoft 365 tenant. It runs one fast, platform-neutral Graph test (`MT.1068`) +and verifies that Maester produced a complete result for the configured tenant. +Each operating system's self-contained HTML report is then written directly to +the private `maester365/maester-smoke-reports` repository for core-maintainer +review. + +The check intentionally accepts either `Passed` or `Failed` for `MT.1068`. +The test tenant's policy state is not a release gate. Authentication errors, +Graph errors, skipped/not-run tests, missing results, or the wrong test/tenant/OS +fail the workflow. + +## Protected environment configuration + +In `maester365/maester`, create a GitHub Actions environment named exactly +`maester-smoke-test`. Configure its deployment protection rules before adding +credentials: + +1. Add `maester365/core-module`, the core Maester maintainer GitHub team, as + the only required reviewer. Do not add outside collaborators, bots, or + broad contributor teams. +2. Enable **Prevent self-review**. +3. Disable **Allow administrators to bypass configured protection rules**. +4. Restrict deployment branches and tags to the `main` branch. + +Create these **environment secrets** on `maester-smoke-test`: + +| Secret | Value | +| --- | --- | +| `MAESTER_SMOKE_TENANT_ID` | The Microsoft Entra **Directory (tenant) ID** (GUID) for the test tenant. For the `elapora.com` tenant, use its directory ID rather than the domain name. | +| `MAESTER_SMOKE_CLIENT_ID` | The **Application (client) ID** (GUID) of the dedicated Entra app registration in that tenant. | +| `MAESTER_REPORTS_APP_PRIVATE_KEY` | The PEM private key generated for the dedicated `do-not-delete-maester-reports` GitHub App. | + +Create this **environment variable** on `maester-smoke-test`: + +| Variable | Value | +| --- | --- | +| `MAESTER_REPORTS_APP_CLIENT_ID` | The GitHub App's Client ID. | + +Do not create repository-level copies of these secrets, and remove any that +already exist there. Do not create a client secret. The published action +supports workload identity federation and authenticates with GitHub's +short-lived OIDC token. The GitHub App private key is used only to mint a +short-lived installation token scoped to the private report repository. + +GitHub evaluates the environment's required-reviewer and branch rules before +starting each credentialed matrix job. The job cannot read the environment +secrets or request an OIDC token until a core maintainer approves the pending +deployment. + +## Private GitHub report storage + +GitHub Actions artifacts in the public `maester365/maester` repository inherit +that repository's read audience, so they must not contain tenant assessment +data. Store the HTML reports as private release assets instead: + +1. Set the `maester365` organization base repository permission to **None**. + Before changing it, explicitly preserve the intended readers of every + existing private repository. In particular, preserve the current + organization members' read access to the private `.github` repository. +2. Create a private repository named `maester365/maester-smoke-reports`. + Disable issues, projects, wikis, and private forks. Grant + `maester365/core-module` **Read** access and do not add other teams or direct + collaborators. Organization owners retain their GitHub-defined + administrative access. +3. Create an organization-owned GitHub App named + `do-not-delete-maester-reports`: + - Set its homepage to `https://github.com/maester365/maester`. + - Start its description with **DO NOT DELETE** and link to the same source + repository. + - Disable webhooks, user authorization, and device flow. + - Allow installation only on the `maester365` account. + - Grant only **Contents: Read and write**. GitHub adds + **Metadata: Read** automatically. + - Install it using **Only select repositories** and select only + `maester-smoke-reports`. + - Generate one private key, store it in + `MAESTER_REPORTS_APP_PRIVATE_KEY`, and delete the downloaded copy after + confirming the environment secret was created. +4. Add a scheduled cleanup workflow to the private repository that deletes + smoke-test releases and tags after 30 days. + +The public workflow requests an installation token for only +`maester-smoke-reports`, explicitly reduces it to `contents: write`, and lets +`actions/create-github-app-token` revoke it when the matrix job ends. Each run +creates one private prerelease tagged `smoke--` containing: + +- `maester-report-ubuntu.html` +- `maester-report-windows.html` +- `maester-report-macos.html` + +Core maintainers review a result from the private repository's **Releases** +page: open the source run's prerelease, download the required HTML asset, and +open the self-contained file locally. Release descriptions link to the source +workflow run and commit without disclosing tenant data in the public +repository. + +## Microsoft Entra configuration + +In the test tenant: + +1. Create a dedicated single-tenant app registration named + `DO NOT DELETE - Maester GitHub Action Smoke Test` and its service principal. + Set its **Notes** to: + + > Used by the Maester repository GitHub Actions smoke test: + > https://github.com/maester365/maester. Authenticates only through the + > protected maester-smoke-test GitHub Environment. + +2. Grant and admin-consent these Microsoft Graph **application** permissions: + - `Directory.Read.All` + - `Policy.Read.All` +3. Add a federated identity credential to the app registration with: + + | Setting | Value | + | --- | --- | + | Issuer | `https://token.actions.githubusercontent.com` | + | Subject | `repo:maester365/maester:environment:maester-smoke-test` | + | Audience | `api://AzureADTokenExchange` | + +No Azure subscription role, Exchange Online permission/role, Teams role, mail +permission, or client secret is required for this scoped smoke test. The app +needs only read access; `MT.1068` may pass or fail without affecting the smoke +result. + +## Trigger and disclosure strategy + +- A successful `publish-module-preview` run starts the cross-platform smoke + test against the newly published preview module, then waits for a core + maintainer to approve the protected-environment deployment. +- A weekly Monday 05:17 UTC run catches action, runner-image, dependency, and + tenant-authentication drift, subject to the same approval. +- A manual run from `main` can exercise either the `preview` or `latest` module + channel. Use this as the explicit pre-release check; it also requires approval. +- Pull requests and ordinary pushes are not workflow triggers. Manual runs from + another ref fail in the uncredentialed preflight job without requesting + approval or reading secrets. +- The federated credential trusts only jobs assigned to the + `maester-smoke-test` environment. The environment restricts deployments to + `main` and its only required reviewer is the core maintainer team. + +The workflow has no default token permissions. Only the protected, +credentialed job receives `contents: read` and `id-token: write`; the +uncredentialed preflight job cannot mint an OIDC token. No trigger or broader +permission is needed for the environment gate: GitHub applies it to manual, +scheduled, and `workflow_run` executions before the job starts. + +The action is pinned to the immutable commit for `maester-action` v1.2.0. +Public result summaries, public artifacts, and telemetry remain disabled. +After Maester produces a result, the workflow transfers only the HTML report +directly to the core-only private release repository; no plaintext report is +uploaded to the public workflow run. The matrix uses `fail-fast: false` so a +failure on one operating system does not hide the other platform results. diff --git a/.github/workflows/maester-action-smoke-test.yaml b/.github/workflows/maester-action-smoke-test.yaml new file mode 100644 index 000000000..70348320c --- /dev/null +++ b/.github/workflows/maester-action-smoke-test.yaml @@ -0,0 +1,202 @@ +name: Maester action tenant smoke test + +on: + workflow_dispatch: + inputs: + maester_version: + description: Maester module channel to exercise + required: true + default: preview + type: choice + options: + - preview + - latest + schedule: + # Exercise the published preview regularly, even when no module preview was + # published during the week. + - cron: "17 5 * * 1" + workflow_run: + # Test each successfully published module preview against a real tenant. + workflows: ["publish-module-preview"] + types: [completed] + branches: [main] + +permissions: {} + +concurrency: + group: maester-action-tenant-smoke-test + cancel-in-progress: false + +env: + SMOKE_MAESTER_VERSION: ${{ inputs.maester_version || 'preview' }} + +jobs: + preflight: + name: Validate trusted ref + if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Validate trusted ref + shell: bash + run: | + set -euo pipefail + + if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then + echo "::error title=Untrusted smoke-test ref::Run this workflow from the main branch." + exit 1 + fi + + smoke: + name: Run published action (${{ matrix.os }}) + needs: preflight + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + # Required reviewers on this environment must approve the deployment before + # the job starts, its secrets are exposed, or an OIDC token can be requested. + environment: + name: maester-smoke-test + permissions: + # azure/login inside the published Maester action exchanges GitHub's OIDC + # token for this environment's federated Entra service principal. + id-token: write + contents: read + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + report_name: ubuntu + - os: windows-latest + report_name: windows + - os: macos-latest + report_name: macos + + steps: + - name: Validate environment secrets + shell: pwsh + env: + MAESTER_SMOKE_TENANT_ID: ${{ secrets.MAESTER_SMOKE_TENANT_ID }} + MAESTER_SMOKE_CLIENT_ID: ${{ secrets.MAESTER_SMOKE_CLIENT_ID }} + run: | + $ErrorActionPreference = 'Stop' + + try { + [void][guid]::Parse($env:MAESTER_SMOKE_TENANT_ID) + } catch { + throw 'MAESTER_SMOKE_TENANT_ID must be an environment secret containing the Microsoft Entra Directory (tenant) ID.' + } + try { + [void][guid]::Parse($env:MAESTER_SMOKE_CLIENT_ID) + } catch { + throw 'MAESTER_SMOKE_CLIENT_ID must be an environment secret containing the app registration Application (client) ID.' + } + + - name: Check out private report publisher + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Create private report uploader token + id: reports-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ vars.MAESTER_REPORTS_APP_CLIENT_ID }} + private-key: ${{ secrets.MAESTER_REPORTS_APP_PRIVATE_KEY }} + owner: maester365 + repositories: maester-smoke-reports + permission-contents: write + + - name: Run Maester against the test tenant + id: maester + uses: maester365/maester-action@910da5adbc61adcd6464ee69e3ab15805c1ad213 # v1.2.0 + with: + tenant_id: ${{ secrets.MAESTER_SMOKE_TENANT_ID }} + client_id: ${{ secrets.MAESTER_SMOKE_CLIENT_ID }} + include_public_tests: true + include_private_tests: false + include_exchange: false + include_teams: false + include_longrunning_tests: false + include_preview_tests: false + include_tags: MT.1068 + maester_version: ${{ env.SMOKE_MAESTER_VERSION }} + pester_verbosity: None + step_summary: false + artifact_upload: false + artifact_upload_html: false + disable_telemetry: true + + - name: Verify Maester executed successfully + shell: pwsh + env: + EXPECTED_OS: ${{ runner.os }} + EXPECTED_TENANT_ID: ${{ secrets.MAESTER_SMOKE_TENANT_ID }} + RESULTS_JSON: ${{ steps.maester.outputs.results_json }} + TESTS_TOTAL: ${{ steps.maester.outputs.tests_total }} + run: | + $ErrorActionPreference = 'Stop' + + if ([string]::IsNullOrWhiteSpace($env:RESULTS_JSON) -or + -not (Test-Path -LiteralPath $env:RESULTS_JSON -PathType Leaf)) { + throw 'The published action did not produce a Maester results JSON file.' + } + + $results = Get-Content -LiteralPath $env:RESULTS_JSON -Raw | ConvertFrom-Json + $tests = @($results.Tests) + $executedTests = @($tests | Where-Object Result -ne 'NotRun') + $smokeTests = @($tests | Where-Object Id -eq 'MT.1068') + + if ([int]$env:TESTS_TOTAL -le 0 -or + [int]$env:TESTS_TOTAL -ne [int]$results.TotalCount -or + [int]$results.TotalCount -ne $tests.Count) { + throw "Maester result counts are inconsistent (action=$($env:TESTS_TOTAL), JSON=$($results.TotalCount), tests=$($tests.Count))." + } + if ($executedTests.Count -ne 1 -or $smokeTests.Count -ne 1 -or + $executedTests[0].Id -ne 'MT.1068') { + throw "Expected only MT.1068 to execute, but Maester executed $($executedTests.Count) test(s)." + } + if ($smokeTests[0].Result -notin @('Passed', 'Failed') -or + [int]$results.ErrorCount -ne 0) { + throw "MT.1068 did not complete normally (result=$($smokeTests[0].Result), errors=$($results.ErrorCount)). Check OIDC trust and Graph application permissions." + } + if ([string]::IsNullOrWhiteSpace($results.TenantName) -or + $results.TenantId -ne $env:EXPECTED_TENANT_ID) { + throw 'The Maester result did not identify the configured Microsoft 365 test tenant.' + } + if ($results.SystemInfo.OSPlatform -ne $env:EXPECTED_OS) { + throw "Maester reported OS '$($results.SystemInfo.OSPlatform)' on the '$($env:EXPECTED_OS)' runner." + } + if ($results.PowerShellInfo.Edition -ne 'Core' -or + [string]::IsNullOrWhiteSpace($results.CurrentVersion) -or + $results.EndOfJson -ne 'EndOfJson') { + throw 'The Maester result metadata is incomplete.' + } + + Write-Host "Published Maester action completed MT.1068 on $($env:EXPECTED_OS)." + + - name: Publish private HTML report + if: always() && steps.maester.outputs.results_json != '' + shell: pwsh + env: + MAESTER_REPORTS_TOKEN: ${{ steps.reports-token.outputs.token }} + REPORT_NAME: ${{ matrix.report_name }} + SOURCE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + $ErrorActionPreference = 'Stop' + + $releaseBody = @" + Private reports for [$env:GITHUB_REPOSITORY run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)]($env:SOURCE_RUN_URL). + + Source commit: ``$env:GITHUB_SHA`` + Maester version: ``$env:SMOKE_MAESTER_VERSION`` + "@ + + ./.github/scripts/Publish-MaesterSmokeReport.ps1 ` + -Repository 'maester365/maester-smoke-reports' ` + -Tag "smoke-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" ` + -ReleaseName "Maester smoke run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)" ` + -ReleaseBody $releaseBody ` + -AssetPath 'test-results/test-results.html' ` + -AssetName "maester-report-$env:REPORT_NAME.html"