From a6fa2ba839420632d0fdd0a1a4c2fee4aa5a6e16 Mon Sep 17 00:00:00 2001 From: Merill Fernando Date: Fri, 24 Jul 2026 20:45:20 +1000 Subject: [PATCH 1/2] Add quick and full demo tenant smoke tests --- .../demo-tenant-quick-approval-trigger.yaml | 27 ++ .../workflows/maester-action-smoke-test.md | 174 +++++---- .../workflows/maester-action-smoke-test.yaml | 341 ++++++++++++++---- 3 files changed, 388 insertions(+), 154 deletions(-) create mode 100644 .github/workflows/demo-tenant-quick-approval-trigger.yaml diff --git a/.github/workflows/demo-tenant-quick-approval-trigger.yaml b/.github/workflows/demo-tenant-quick-approval-trigger.yaml new file mode 100644 index 000000000..075317db8 --- /dev/null +++ b/.github/workflows/demo-tenant-quick-approval-trigger.yaml @@ -0,0 +1,27 @@ +name: Demo tenant quick approval trigger + +on: + pull_request_review: + types: [submitted] + +permissions: {} + +jobs: + approved: + name: Approved review received + if: >- + github.event.review.state == 'approved' && + github.event.pull_request.base.ref == 'main' && + !github.event.pull_request.draft + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Request the trusted quick smoke test + shell: bash + env: + PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + REVIEWER: ${{ github.event.review.user.login }} + run: >- + echo "Approved review from @${REVIEWER} requested the trusted + quick smoke test for pull request #${PULL_REQUEST_NUMBER}." diff --git a/.github/workflows/maester-action-smoke-test.md b/.github/workflows/maester-action-smoke-test.md index 46722dbda..5a41ff823 100644 --- a/.github/workflows/maester-action-smoke-test.md +++ b/.github/workflows/maester-action-smoke-test.md @@ -1,28 +1,30 @@ -# Maester action tenant smoke test +# Demo tenant Invoke-Maester smoke test `maester-action-smoke-test.yaml` exercises the published standard `maester365/maester-action` on Linux, Windows, and macOS against a real -Microsoft 365 tenant. It runs one fast, platform-neutral Graph test (`MT.1068`) -and verifies that Maester produced a complete result for the configured tenant. -Each operating system's self-contained HTML report is then written directly to -the private `maester365/maester-smoke-reports` repository for core-maintainer +Microsoft 365 demo tenant. The same workflow supports two run types: + +- **Quick** runs only the fast, platform-neutral Graph check `MT.1068`. +- **Full** runs all public Graph checks. Exchange, Teams, private, preview, and + long-running tests remain disabled. + +Each operating system's self-contained HTML report is written directly to the +private `maester365/maester-smoke-reports` repository for core-maintainer review. -The check intentionally accepts either `Passed` or `Failed` for `MT.1068`. -The test tenant's policy state is not a release gate. Authentication errors, -Graph errors, skipped/not-run tests, missing results, or the wrong test/tenant/OS -fail the workflow. Maester may emit more than one result instance for the -selected check; every executed result must still be `MT.1068` and must complete -normally. +The gate intentionally accepts either `Passed` or `Failed` test results because +the demo tenant's security posture is not a release gate. Authentication, +Microsoft Graph, action, result-integrity, tenant, or runner errors fail the +workflow. The full run also accepts intentionally skipped tests but requires at +least one public Graph check to complete. ## Protected environment configuration In `maester365/maester`, create a GitHub Actions environment named exactly -`maester-smoke-test`. This is the credentials environment used by automatic -trusted-branch runs. Configure it before adding credentials: +`maester-smoke-test`. Configure it before adding credentials: -1. Do not configure required reviewers. Runs from `main`, including every merge, - must start automatically. +1. Do not configure required reviewers. Approved-pull-request quick runs and + every full run after a merge to `main` must start automatically. 2. Disable **Allow administrators to bypass configured protection rules**. 3. Restrict deployment branches and tags to the `main` branch. @@ -30,8 +32,8 @@ Create these **environment secrets** on `maester-smoke-test`: | Secret | Value | | --- | --- | -| `MAESTER_SMOKE_TENANT_ID` | The Microsoft Entra **Directory (tenant) ID** (GUID) for the test tenant. For the `elapora.com` tenant, use its directory ID rather than the domain name. | -| `MAESTER_SMOKE_CLIENT_ID` | The **Application (client) ID** (GUID) of the dedicated Entra app registration in that tenant. | +| `MAESTER_SMOKE_TENANT_ID` | The Microsoft Entra **Directory (tenant) ID** (GUID) for the demo tenant. For the `elapora.com` tenant, use `0817c655-a853-4d8f-9723-3a333b5b9235`. | +| `MAESTER_SMOKE_CLIENT_ID` | The **Application (client) ID** (GUID) of the dedicated Entra app registration. | | `MAESTER_REPORTS_APP_PRIVATE_KEY` | The PEM private key generated for the dedicated `do-not-delete-maester-reports` GitHub App. | Create this **environment variable** on `maester-smoke-test`: @@ -41,27 +43,15 @@ Create this **environment variable** on `maester-smoke-test`: | `MAESTER_REPORTS_APP_CLIENT_ID` | The GitHub App's Client ID. | Do not create repository-level copies of these secrets, and remove any that -already exist there. Do not create a client secret. The published action -supports workload identity federation and authenticates with GitHub's -short-lived OIDC token. The GitHub App private key is used only to mint a -short-lived installation token scoped to the private report repository. - -Create a second environment named exactly `maester-smoke-test-pr`. This is an -approval-only gate and must not contain secrets or variables: - -1. Add `maester365/core-module`, the core Maester maintainer GitHub team, as - the only required reviewer. Do not add outside collaborators, bots, or broad - contributor teams. -2. Disable **Prevent self-review** so an authorized core maintainer can approve - a pull request smoke run they initiated. -3. Disable **Allow administrators to bypass configured protection rules**. -4. Restrict deployment branches and tags to the `main` branch. - -GitHub evaluates `maester-smoke-test-pr` before starting the approval job for a -pull request. Only after a core maintainer approves that job can the workflow -continue to the credentialed matrix. Runs caused by pushes to `main`, the -schedule, or a manual dispatch from `main` skip the approval environment and -start automatically. +already exist there. Do not create a client secret. The published action uses +workload identity federation and GitHub's short-lived OIDC token. The GitHub +App private key is used only to mint a short-lived installation token scoped to +the private report repository. + +The earlier `maester-smoke-test-pr` approval-only environment is no longer used +and can be deleted after this workflow is deployed. Pull request authorization +now comes from a current core-maintainer approval and is independently +revalidated by trusted default-branch code. ## Private GitHub report storage @@ -98,11 +88,11 @@ data. Store the HTML reports as private release assets instead: The public workflow requests an installation token for only `maester-smoke-reports`, explicitly reduces it to `contents: write`, and lets `actions/create-github-app-token` revoke it when the matrix job ends. Each run -creates one private prerelease tagged `smoke--` containing: +creates one private prerelease tagged `smoke--` containing +assets named: -- `maester-report-ubuntu.html` -- `maester-report-windows.html` -- `maester-report-macos.html` +- `maester-report-quick-.html`, or +- `maester-report-full-.html`. Core maintainers review a result from the private repository's **Releases** page: open the source run's prerelease, download the required HTML asset, and @@ -112,9 +102,9 @@ repository. ## Microsoft Entra configuration -In the test tenant: +In the demo tenant: -1. Create a dedicated single-tenant app registration named +1. Use a dedicated single-tenant app registration named `DO NOT DELETE - Maester GitHub Action Smoke Test` and its service principal. Set its **Notes** to: @@ -122,9 +112,35 @@ In the test tenant: > https://github.com/maester365/maester. Authenticates only through the > protected maester-smoke-test GitHub Environment. -2. Grant and admin-consent these Microsoft Graph **application** permissions: +2. Grant and admin-consent the following Microsoft Graph **application** + permissions. These are the default read-only scopes returned by + `Get-MtGraphScope`: + + - `AuditLog.Read.All` + - `DeviceManagementConfiguration.Read.All` + - `DeviceManagementManagedDevices.Read.All` + - `DeviceManagementRBAC.Read.All` + - `DeviceManagementServiceConfig.Read.All` - `Directory.Read.All` + - `DirectoryRecommendations.Read.All` + - `EntitlementManagement.Read.All` + - `IdentityRiskEvent.Read.All` + - `OnPremDirectorySynchronization.Read.All` + - `OrgSettings-AppsAndServices.Read.All` + - `OrgSettings-Forms.Read.All` - `Policy.Read.All` + - `Policy.Read.ConditionalAccess` + - `Reports.Read.All` + - `ReportSettings.Read.All` + - `RoleEligibilitySchedule.Read.Directory` + - `RoleManagement.Read.All` + - `RoleManagementAlert.Read.Directory` + - `SecurityIdentitiesSensors.Read.All` + - `SecurityIdentitiesHealth.Read.All` + - `SharePointTenantSettings.Read.All` + - `ThreatHunting.Read.All` + - `UserAuthenticationMethod.Read.All` + 3. Add a federated identity credential to the app registration with: | Setting | Value | @@ -133,41 +149,41 @@ In the test tenant: | Subject | `repo:maester365/maester:environment:maester-smoke-test` | | Audience | `api://AzureADTokenExchange` | -No Azure subscription role, Exchange Online permission/role, Teams role, mail -permission, or client secret is required for this scoped smoke test. The app -needs only read access; `MT.1068` may pass or fail without affecting the smoke -result. - -## Trigger and disclosure strategy - -- Every push to `main`, including a merged pull request, starts the - cross-platform smoke test automatically. -- A pull request targeting `main` starts a pending smoke run when it is opened, - updated, reopened, or marked ready for review. A core maintainer must approve - `maester-smoke-test-pr` before the workflow can access the tenant. -- A weekly Monday 05:17 UTC run catches action, runner-image, dependency, and - tenant-authentication drift automatically. -- A manual run from `main` can exercise either the `preview` or `latest` module - channel without approval. Manual runs from another ref fail in the - uncredentialed preflight job without reading secrets. -- Pull request runs use `pull_request_target`, so GitHub loads the workflow and - checkout from the trusted default branch. The workflow never checks out, - builds, or executes pull request code. The event associates the published - action smoke test with the pull request; it does not test the pull request's - unmerged implementation. -- The federated credential trusts only jobs assigned to the - `maester-smoke-test` environment. The environment restricts deployments to - `main`; the separate approval-only environment has no tenant or report - credentials. - -The workflow has no default token permissions. Only the protected, -credentialed job receives `contents: read` and `id-token: write`; the -uncredentialed preflight job cannot mint an OIDC token. No trigger or broader -permission is needed for the pull request gate. +No Azure subscription role, Exchange Online permission/role, Teams role, +write permission, mail permission, or client secret is required. + +## Trigger and merge-gate strategy + +- An approved review on a ready pull request targeting `main` starts the quick + cross-platform run immediately. +- The secretless `Demo tenant quick approval trigger` workflow only records the + review event. A separate `workflow_run` job loaded from the default branch + independently confirms that the pull request is open, is still on its + approved commit, and that the approving reviewer has `write`, `maintain`, or + `admin` repository permission before requesting tenant credentials. +- The trusted runner creates a check named exactly + **`Demo tenant quick smoke`** on the pull request's current test merge commit + and marks it successful only when all three operating systems complete. Add + that check to the `Protect main` ruleset's required status checks after this + workflow has been merged to `main`; enabling it earlier would prevent this + pull request from producing the new check. +- Every push to `main`, including a merged pull request, starts the full + cross-platform Graph run automatically. +- A weekly Monday 05:17 UTC run exercises the full published preview. +- A manual run from `main` can select quick or full and either the `preview` or + `latest` module channel. Manual runs from another ref fail before tenant + access. + +Neither workflow checks out or executes pull request code. The credentialed +workflow always runs trusted default-branch code, and the protected environment +only permits `main`. The listener has no token permissions or secrets. The +trusted context job can read actions and pull requests and create the explicit +merge-gate check, but cannot request an OIDC token. Only the protected matrix +receives `contents: read` and `id-token: write`. The action is pinned to the immutable commit for `maester-action` v1.2.0. Public result summaries, public artifacts, and telemetry remain disabled. After Maester produces a result, the workflow transfers only the HTML report -directly to the core-only private release repository; no plaintext report is -uploaded to the public workflow run. The matrix uses `fail-fast: false` so a -failure on one operating system does not hide the other platform results. +directly to the core-only private release repository. The matrix uses +`fail-fast: false` so a failure on one operating system does not hide the other +platform results. diff --git a/.github/workflows/maester-action-smoke-test.yaml b/.github/workflows/maester-action-smoke-test.yaml index f53f43162..74b133fc1 100644 --- a/.github/workflows/maester-action-smoke-test.yaml +++ b/.github/workflows/maester-action-smoke-test.yaml @@ -1,13 +1,21 @@ -name: Maester action tenant smoke test +name: Demo tenant Invoke-Maester smoke test on: push: branches: [main] - pull_request_target: - branches: [main] - types: [opened, reopened, synchronize, ready_for_review] + workflow_run: + workflows: ["Demo tenant quick approval trigger"] + types: [completed] workflow_dispatch: inputs: + run_type: + description: Smoke-test depth + required: true + default: quick + type: choice + options: + - quick + - full maester_version: description: Maester module channel to exercise required: true @@ -17,73 +25,191 @@ on: - preview - latest schedule: - # Exercise the published preview regularly, even when no module preview was - # published during the week. + # Exercise the full published preview regularly, even when no module preview + # was published during the week. - cron: "17 5 * * 1" permissions: {} -concurrency: - # A pull request waiting for approval must not block trusted main runs. - group: maester-action-tenant-smoke-test-${{ github.event.pull_request.number || 'trusted' }} - cancel-in-progress: ${{ github.event_name == 'pull_request_target' }} - -env: - SMOKE_MAESTER_VERSION: ${{ inputs.maester_version || 'preview' }} - jobs: - preflight: - name: Validate trusted execution context - if: github.event_name != 'pull_request_target' || !github.event.pull_request.draft + context: + name: Select and validate smoke-test context runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + actions: read + checks: write + contents: read + pull-requests: read + outputs: + check_run_id: ${{ steps.context.outputs.check_run_id }} + include_tags: ${{ steps.context.outputs.include_tags }} + maester_version: ${{ steps.context.outputs.maester_version }} + pull_request_number: ${{ steps.context.outputs.pull_request_number }} + run_type: ${{ steps.context.outputs.run_type }} steps: - - name: Validate trusted execution context - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - PULL_REQUEST_BASE: ${{ github.event.pull_request.base.ref || '' }} - run: | - set -euo pipefail - - if [[ "${GITHUB_REF}" != "refs/heads/main" ]]; then - echo "::error title=Untrusted smoke-test ref::Run this workflow from the main branch." - exit 1 - fi - if [[ "${EVENT_NAME}" == "pull_request_target" && "${PULL_REQUEST_BASE}" != "main" ]]; then - echo "::error title=Unexpected pull request base::Pull request smoke tests must target main." - exit 1 - fi - - approve-pr: - name: Approve tenant access for pull request - needs: preflight - if: github.event_name == 'pull_request_target' - runs-on: ubuntu-latest - timeout-minutes: 5 - environment: - name: maester-smoke-test-pr - deployment: false - permissions: {} - steps: - - name: Record approval - shell: bash - env: - PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} - run: echo "A core maintainer approved tenant access for pull request #${PULL_REQUEST_NUMBER}." + - name: Select and validate smoke-test context + id: context + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const setContext = ({ + checkRunId = '', + includeTags = '', + maesterVersion = 'preview', + pullRequestNumber = '', + runType = 'none', + } = {}) => { + core.setOutput('check_run_id', checkRunId); + core.setOutput('include_tags', includeTags); + core.setOutput('maester_version', maesterVersion); + core.setOutput('pull_request_number', pullRequestNumber); + core.setOutput('run_type', runType); + }; + + if (context.eventName !== 'workflow_run') { + if (process.env.GITHUB_REF !== 'refs/heads/main') { + core.setFailed('Manual and trusted-branch smoke tests must run from main.'); + setContext(); + return; + } + + const requestedType = context.eventName === 'workflow_dispatch' + ? context.payload.inputs?.run_type + : 'full'; + if (!['quick', 'full'].includes(requestedType)) { + core.setFailed(`Unsupported smoke-test type '${requestedType}'.`); + setContext(); + return; + } + + setContext({ + includeTags: requestedType === 'quick' ? 'MT.1068' : '', + maesterVersion: context.payload.inputs?.maester_version || 'preview', + runType: requestedType, + }); + return; + } + + const sourceRun = context.payload.workflow_run; + const ignore = (message) => { + core.notice(`No tenant access requested: ${message}`); + setContext(); + }; + + if (sourceRun.name !== 'Demo tenant quick approval trigger' || + sourceRun.event !== 'pull_request_review' || + sourceRun.conclusion !== 'success') { + ignore('the source run was not a successful approved-review listener.'); + return; + } + + const sourceJobs = await github.paginate( + github.rest.actions.listJobsForWorkflowRun, + { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: sourceRun.id, + per_page: 100, + }, + ); + const approvalJob = sourceJobs.find( + (job) => job.name === 'Approved review received', + ); + if (approvalJob?.conclusion !== 'success') { + ignore('the source event was not an approved review for main.'); + return; + } + + const sourcePullRequests = sourceRun.pull_requests || []; + if (sourcePullRequests.length !== 1) { + ignore(`the source run referenced ${sourcePullRequests.length} pull requests.`); + return; + } + + const pullNumber = sourcePullRequests[0].number; + const {data: pull} = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + }); + if (pull.state !== 'open' || pull.draft || pull.base.ref !== 'main') { + ignore('the pull request is not an open, ready-for-review change targeting main.'); + return; + } + if (!pull.merge_commit_sha || + sourceRun.head_sha !== pull.merge_commit_sha) { + ignore('the approved review no longer matches the current test merge commit.'); + return; + } + + const reviewer = sourceRun.actor?.login; + if (!reviewer) { + ignore('the source run did not identify a reviewer.'); + return; + } + + const {data: permission} = + await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: reviewer, + }); + if (!['admin', 'maintain', 'write'].includes(permission.permission)) { + ignore(`@${reviewer} does not have maintainer-level repository permission.`); + return; + } + + const reviews = await github.paginate(github.rest.pulls.listReviews, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullNumber, + per_page: 100, + }); + const currentApproval = reviews.find((review) => + review.user?.login?.toLowerCase() === reviewer.toLowerCase() && + review.state === 'APPROVED' && + review.commit_id === pull.head.sha + ); + if (!currentApproval) { + ignore(`@${reviewer} has no current approval for ${pull.head.sha}.`); + return; + } + + const detailsUrl = + `${process.env.GITHUB_SERVER_URL}/${context.repo.owner}/` + + `${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; + const {data: checkRun} = await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'Demo tenant quick smoke', + head_sha: pull.merge_commit_sha, + details_url: detailsUrl, + status: 'in_progress', + started_at: new Date().toISOString(), + output: { + title: 'Running the demo-tenant quick smoke test', + summary: + `A current approval from @${reviewer} authorized the ` + + `cross-platform MT.1068 run for pull request #${pullNumber}.`, + }, + }); + setContext({ + checkRunId: String(checkRun.id), + includeTags: 'MT.1068', + pullRequestNumber: String(pullNumber), + runType: 'quick', + }); smoke: - name: Run published action (${{ matrix.os }}) - needs: [preflight, approve-pr] + name: Run ${{ needs.context.outputs.run_type }} published action (${{ matrix.os }}) + needs: context if: >- - always() && - needs.preflight.result == 'success' && - (needs.approve-pr.result == 'success' || needs.approve-pr.result == 'skipped') + needs.context.outputs.run_type == 'quick' || + needs.context.outputs.run_type == 'full' runs-on: ${{ matrix.os }} - timeout-minutes: 30 - # This credentials environment is restricted to main but has no reviewer - # gate. Pull request runs pass the approval-only job above before reaching it. + timeout-minutes: 45 environment: name: maester-smoke-test permissions: @@ -101,6 +227,9 @@ jobs: report_name: windows - os: macos-latest report_name: macos + env: + SMOKE_MAESTER_VERSION: ${{ needs.context.outputs.maester_version }} + SMOKE_RUN_TYPE: ${{ needs.context.outputs.run_type }} steps: - name: Validate environment secrets @@ -125,8 +254,8 @@ jobs: - name: Check out private report publisher uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - # pull_request_target resolves github.sha to trusted default-branch - # code. Never check out the pull request head in this privileged job. + # All credentialed jobs run from the trusted default branch. Never + # check out or execute pull-request code in this workflow. ref: ${{ github.sha }} persist-credentials: false @@ -140,7 +269,7 @@ jobs: repositories: maester-smoke-reports permission-contents: write - - name: Run Maester against the test tenant + - name: Run Maester against the demo tenant id: maester uses: maester365/maester-action@910da5adbc61adcd6464ee69e3ab15805c1ad213 # v1.2.0 with: @@ -152,7 +281,7 @@ jobs: include_teams: false include_longrunning_tests: false include_preview_tests: false - include_tags: MT.1068 + include_tags: ${{ needs.context.outputs.include_tags }} maester_version: ${{ env.SMOKE_MAESTER_VERSION }} pester_verbosity: None step_summary: false @@ -166,6 +295,7 @@ jobs: EXPECTED_OS: ${{ runner.os }} EXPECTED_TENANT_ID: ${{ secrets.MAESTER_SMOKE_TENANT_ID }} RESULTS_JSON: ${{ steps.maester.outputs.results_json }} + RUN_TYPE: ${{ needs.context.outputs.run_type }} TESTS_TOTAL: ${{ steps.maester.outputs.tests_total }} run: | $ErrorActionPreference = 'Stop' @@ -178,10 +308,11 @@ jobs: $results = Get-Content -LiteralPath $env:RESULTS_JSON -Raw | ConvertFrom-Json $tests = @($results.Tests) $executedTests = @($tests | Where-Object Result -ne 'NotRun') - $smokeTests = @($tests | Where-Object Id -eq 'MT.1068') - $unexpectedTests = @($executedTests | Where-Object Id -ne 'MT.1068') - $incompleteSmokeTests = @( - $smokeTests | Where-Object { $_.Result -notin @('Passed', 'Failed') } + $completedTests = @( + $tests | Where-Object Result -In @('Passed', 'Failed') + ) + $unexpectedResults = @( + $tests | Where-Object Result -NotIn @('Passed', 'Failed', 'Skipped', 'NotRun') ) if ([int]$env:TESTS_TOTAL -le 0 -or @@ -189,15 +320,33 @@ jobs: [int]$results.TotalCount -ne $tests.Count) { throw "Maester result counts are inconsistent (action=$($env:TESTS_TOTAL), JSON=$($results.TotalCount), tests=$($tests.Count))." } - if ($smokeTests.Count -lt 1 -or $unexpectedTests.Count -gt 0) { - throw "Expected only MT.1068 to execute, but Maester produced $($executedTests.Count) executed result(s), including $($unexpectedTests.Count) unexpected result(s)." + if ($unexpectedResults.Count -gt 0 -or [int]$results.ErrorCount -ne 0) { + throw "Maester did not complete normally (unexpected results=$($unexpectedResults.Count), errors=$($results.ErrorCount)). Check OIDC trust and Graph application permissions." } - if ($incompleteSmokeTests.Count -gt 0 -or [int]$results.ErrorCount -ne 0) { - throw "MT.1068 did not complete normally (incomplete=$($incompleteSmokeTests.Count), errors=$($results.ErrorCount)). Check OIDC trust and Graph application permissions." + + if ($env:RUN_TYPE -eq 'quick') { + $smokeTests = @($tests | Where-Object Id -eq 'MT.1068') + $unexpectedTests = @($executedTests | Where-Object Id -ne 'MT.1068') + $incompleteSmokeTests = @( + $smokeTests | Where-Object Result -NotIn @('Passed', 'Failed') + ) + if ($smokeTests.Count -lt 1 -or $unexpectedTests.Count -gt 0) { + throw "Expected only MT.1068 to execute, but Maester produced $($executedTests.Count) executed result(s), including $($unexpectedTests.Count) unexpected result(s)." + } + if ($incompleteSmokeTests.Count -gt 0) { + throw "MT.1068 did not complete normally ($($incompleteSmokeTests.Count) incomplete result(s))." + } + } elseif ($env:RUN_TYPE -eq 'full') { + if ($completedTests.Count -lt 1) { + throw 'The full Graph smoke test did not complete any public Maester checks.' + } + } else { + throw "Unsupported smoke-test type '$($env:RUN_TYPE)'." } + if ([string]::IsNullOrWhiteSpace($results.TenantName) -or $results.TenantId -ne $env:EXPECTED_TENANT_ID) { - throw 'The Maester result did not identify the configured Microsoft 365 test tenant.' + throw 'The Maester result did not identify the configured Microsoft 365 demo tenant.' } if ($results.SystemInfo.OSPlatform -ne $env:EXPECTED_OS) { throw "Maester reported OS '$($results.SystemInfo.OSPlatform)' on the '$($env:EXPECTED_OS)' runner." @@ -208,7 +357,7 @@ jobs: throw 'The Maester result metadata is incomplete.' } - Write-Host "Published Maester action completed MT.1068 on $($env:EXPECTED_OS)." + Write-Host "Published Maester action completed the $($env:RUN_TYPE) Graph smoke test on $($env:EXPECTED_OS)." - name: Publish private HTML report if: always() && steps.maester.outputs.results_json != '' @@ -221,7 +370,7 @@ jobs: $ErrorActionPreference = 'Stop' $releaseBody = @" - Private reports for [$env:GITHUB_REPOSITORY run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)]($env:SOURCE_RUN_URL). + Private $env:SMOKE_RUN_TYPE reports for [$env:GITHUB_REPOSITORY run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)]($env:SOURCE_RUN_URL). Source commit: ``$env:GITHUB_SHA`` Maester version: ``$env:SMOKE_MAESTER_VERSION`` @@ -230,7 +379,49 @@ jobs: ./.github/scripts/Publish-MaesterSmokeReport.ps1 ` -Repository 'maester365/maester-smoke-reports' ` -Tag "smoke-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" ` - -ReleaseName "Maester smoke run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)" ` + -ReleaseName "Maester $env:SMOKE_RUN_TYPE smoke run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)" ` -ReleaseBody $releaseBody ` -AssetPath 'test-results/test-results.html' ` - -AssetName "maester-report-$env:REPORT_NAME.html" + -AssetName "maester-report-$env:SMOKE_RUN_TYPE-$env:REPORT_NAME.html" + + finalize-quick-check: + name: Finalize pull request quick check + needs: [context, smoke] + if: always() && needs.context.outputs.check_run_id != '' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + checks: write + + steps: + - name: Complete pull request quick check + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + CHECK_RUN_ID: ${{ needs.context.outputs.check_run_id }} + PULL_REQUEST_NUMBER: ${{ needs.context.outputs.pull_request_number }} + SMOKE_RESULT: ${{ needs.smoke.result }} + with: + script: | + const succeeded = process.env.SMOKE_RESULT === 'success'; + const detailsUrl = + `${process.env.GITHUB_SERVER_URL}/${context.repo.owner}/` + + `${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; + + await github.rest.checks.update({ + owner: context.repo.owner, + repo: context.repo.repo, + check_run_id: Number(process.env.CHECK_RUN_ID), + details_url: detailsUrl, + status: 'completed', + conclusion: succeeded ? 'success' : 'failure', + completed_at: new Date().toISOString(), + output: { + title: succeeded + ? 'Demo-tenant quick smoke test passed' + : 'Demo-tenant quick smoke test failed', + summary: + `The cross-platform MT.1068 run for pull request ` + + `#${process.env.PULL_REQUEST_NUMBER} ` + + `${succeeded ? 'completed successfully.' : 'did not complete successfully.'}`, + }, + }); From ef41d2c717f25f82892bec1ee38910d894564651 Mon Sep 17 00:00:00 2001 From: Merill Fernando Date: Fri, 24 Jul 2026 20:51:30 +1000 Subject: [PATCH 2/2] Link private reports from smoke summaries --- .../workflows/maester-action-smoke-test.md | 5 ++++- .../workflows/maester-action-smoke-test.yaml | 21 +++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maester-action-smoke-test.md b/.github/workflows/maester-action-smoke-test.md index 5a41ff823..024b4cdd9 100644 --- a/.github/workflows/maester-action-smoke-test.md +++ b/.github/workflows/maester-action-smoke-test.md @@ -98,7 +98,10 @@ Core maintainers review a result from the private repository's **Releases** page: open the source run's prerelease, download the required HTML asset, and open the self-contained file locally. Release descriptions link to the source workflow run and commit without disclosing tenant data in the public -repository. +repository. After each successful upload, the corresponding operating-system +job adds **Open the private HTML report** and **View the private release** links +to the public workflow run summary. The links disclose no report content and +resolve only for users who can read the private report repository. ## Microsoft Entra configuration diff --git a/.github/workflows/maester-action-smoke-test.yaml b/.github/workflows/maester-action-smoke-test.yaml index 74b133fc1..43200a01b 100644 --- a/.github/workflows/maester-action-smoke-test.yaml +++ b/.github/workflows/maester-action-smoke-test.yaml @@ -364,11 +364,14 @@ jobs: shell: pwsh env: MAESTER_REPORTS_TOKEN: ${{ steps.reports-token.outputs.token }} + REPORT_LABEL: ${{ runner.os }} REPORT_NAME: ${{ matrix.report_name }} SOURCE_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | $ErrorActionPreference = 'Stop' + $tag = "smoke-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" + $assetName = "maester-report-$env:SMOKE_RUN_TYPE-$env:REPORT_NAME.html" $releaseBody = @" Private $env:SMOKE_RUN_TYPE reports for [$env:GITHUB_REPOSITORY run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)]($env:SOURCE_RUN_URL). @@ -378,11 +381,25 @@ jobs: ./.github/scripts/Publish-MaesterSmokeReport.ps1 ` -Repository 'maester365/maester-smoke-reports' ` - -Tag "smoke-$env:GITHUB_RUN_ID-$env:GITHUB_RUN_ATTEMPT" ` + -Tag $tag ` -ReleaseName "Maester $env:SMOKE_RUN_TYPE smoke run $env:GITHUB_RUN_ID (attempt $env:GITHUB_RUN_ATTEMPT)" ` -ReleaseBody $releaseBody ` -AssetPath 'test-results/test-results.html' ` - -AssetName "maester-report-$env:SMOKE_RUN_TYPE-$env:REPORT_NAME.html" + -AssetName $assetName + + $reportUrl = "https://github.com/maester365/maester-smoke-reports/releases/download/$tag/$assetName" + $releaseUrl = "https://github.com/maester365/maester-smoke-reports/releases/tag/$tag" + $reportSummary = @" + ### $env:REPORT_LABEL $env:SMOKE_RUN_TYPE report + + [Open the private HTML report]($reportUrl) ยท [View the private release]($releaseUrl) + + Access is limited to maintainers who can read ``maester365/maester-smoke-reports``. + "@ + Add-Content ` + -LiteralPath $env:GITHUB_STEP_SUMMARY ` + -Value $reportSummary ` + -Encoding utf8 finalize-quick-check: name: Finalize pull request quick check