From 898e7c02e7c17ac2bdbdbe66464a1cfcef2773ea Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 19 Jun 2026 11:30:44 -0700 Subject: [PATCH 1/3] Add aspnetcore perf-build pipeline (moved from dotnet/aspnetcore) Stand up an Azure DevOps perf-build pipeline hosted in dotnet/performance that builds every commit on dotnet/aspnetcore main from source, packs per-RID Microsoft.AspNetCore.App.Runtime archives, and uploads them to the Build Cache Service for dotnet/crank bisection. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/aspnetcore-perf-build-jobs.yml | 286 ++++++++++++++++++ eng/pipelines/aspnetcore-perf-build.yml | 177 +++++++++++ eng/pipelines/register-build-jobs.yml | 6 + .../templates/register-build-job.yml | 12 +- .../templates/upload-build-artifacts-job.yml | 14 +- eng/pipelines/tools/pack-bcs-archives.ps1 | 187 ++++++++++++ eng/pipelines/upload-build-artifacts-jobs.yml | 12 + 7 files changed, 691 insertions(+), 3 deletions(-) create mode 100644 eng/pipelines/aspnetcore-perf-build-jobs.yml create mode 100644 eng/pipelines/aspnetcore-perf-build.yml create mode 100644 eng/pipelines/tools/pack-bcs-archives.ps1 diff --git a/eng/pipelines/aspnetcore-perf-build-jobs.yml b/eng/pipelines/aspnetcore-perf-build-jobs.yml new file mode 100644 index 00000000000..28f50935ffa --- /dev/null +++ b/eng/pipelines/aspnetcore-perf-build-jobs.yml @@ -0,0 +1,286 @@ +# aspnetcore-perf-build-jobs.yml +# +# Authored build jobs for the ASP.NET Core perf-build pipeline (consumed by +# eng/pipelines/aspnetcore-perf-build.yml). These jobs are written from scratch +# here in dotnet/performance instead of reusing dotnet/aspnetcore's +# `.azure/pipelines/jobs/default-build.yml`, because that template has hard +# `@self` references (e.g. /eng/common/templates-official/job/job.yml@self) and +# `@self` always resolves to the ROOT pipeline repo (performance), which does not +# carry aspnetcore's eng/common layout/contract. So we stand up the build +# ourselves: pick the dnceng internal pool, check out the aspnetcore mirror at the +# triggering commit alongside this repo (for the pack script), run aspnetcore's +# own eng/build with lean perf args (PUBLIC feeds only -- no internal runtime +# download), pack the per-RID runtime nupkgs into the BCS archive layout via the +# moved eng/pipelines/tools/pack-bcs-archives.ps1, and publish one pipeline +# artifact per config. +# +# Job names (Windows_build / Linux_x64_build / Linux_arm64_build) are load-bearing: +# they must match the `dependencyJobName` values in +# eng/pipelines/upload-build-artifacts-jobs.yml's aspnetcore_* branches. +# +# PUBLIC-FEED rationale: aspnetcore's own ci-public.yml builds every public PR with +# `_InternalRuntimeDownloadArgs` empty, proving public feeds suffice for a from- +# source pack build. We therefore drop enable-internal-runtimes / get-delegation-sas +# / the dotnetbuilds-internal-read connection entirely and pass no +# -RuntimeSourceFeed args. + +parameters: + aspnetcoreRepoAlias: aspnetcore + performanceRepoAlias: self + # Per-config enablement booleans (forward-compat with the MissingBuildsTrigger + # PerConfiguration indexer). In v1's eager per-commit CI-trigger mode all five + # are true and all configs build; the booleans let a future Function queue a + # subset per config. + aspnetcore_x64_linux: true + aspnetcore_arm64_linux: true + aspnetcore_x64_windows: true + aspnetcore_arm64_windows: true + aspnetcore_x86_windows: true + +jobs: + +# =========================================================================== +# Windows multi-arch build job (x64 + x86 + arm64 in one agent) +# +# Mirrors aspnetcore ci-public.yml's Windows_build pattern: build.cmd is invoked +# x64-first WITH native (-nativeToolsOnMachine), then x86 and arm64 with +# -noBuildNative because they cross-pack against the x64 native bits. Each +# invocation emits its own per-RID Microsoft.AspNetCore.App.Runtime.win-{arch}.*.nupkg +# into artifacts/packages/Release/Shipping. The x64 build always runs when any +# Windows arch is enabled (it produces the native base the others reuse); pack + +# publish for each arch are individually gated by that arch's boolean. +# =========================================================================== +- ${{ if or(eq(parameters.aspnetcore_x64_windows, true), eq(parameters.aspnetcore_arm64_windows, true), eq(parameters.aspnetcore_x86_windows, true)) }}: + - job: Windows_build + displayName: 'Build: Windows x64/x86/arm64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2026preview.scout.amd64 + variables: + - _AspNetCoreRoot: $(Agent.BuildDirectory)\s\aspnetcore + - _PackScript: $(Agent.BuildDirectory)\s\performance\eng\pipelines\tools\pack-bcs-archives.ps1 + - _ShippingDir: $(Agent.BuildDirectory)\s\aspnetcore\artifacts\packages\Release\Shipping + - _StagingRoot: $(Build.ArtifactStagingDirectory)\bcs + - MSBUILDUSESERVER: "1" + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + # x64 -- native base build (always runs when any Windows arch is requested). + - script: >- + eng\build.cmd + -ci + -prepareMachine + -nativeToolsOnMachine + -Configuration Release + -arch x64 + -pack + -all + -noBuildJava + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.x64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build x64 + env: + MSBUILDUSESERVER: "1" + + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - script: >- + eng\build.cmd + -ci + -prepareMachine + -Configuration Release + -arch x86 + -pack + -all + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + -ExcludeCIBinaryLog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build x86 + env: + MSBUILDUSESERVER: "1" + + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - script: >- + eng\build.cmd + -ci + -prepareMachine + -Configuration Release + -arch arm64 + -pack + -noBuildJava + -noBuildNative + /p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.arm64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build ARM64 + env: + MSBUILDUSESERVER: "1" + + - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - pwsh: >- + & "$(_PackScript)" -Rids win-x64 -Format zip -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Pack BCS archive (win-x64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_x64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_x64_Release_aspnetcore + artifactName: BuildArtifacts_windows_x64_Release_aspnetcore + + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - pwsh: >- + & "$(_PackScript)" -Rids win-x86 -Format zip -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Pack BCS archive (win-x86)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_x86_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_x86_Release_aspnetcore + artifactName: BuildArtifacts_windows_x86_Release_aspnetcore + + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - pwsh: >- + & "$(_PackScript)" -Rids win-arm64 -Format zip -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Pack BCS archive (win-arm64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_windows_arm64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)\BuildArtifacts_windows_arm64_Release_aspnetcore + artifactName: BuildArtifacts_windows_arm64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Windows build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)\artifacts\log + artifactName: Windows_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Linux x64 build job +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - job: Linux_x64_build + displayName: 'Build: Linux x64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + - _AspNetCoreRoot: $(Agent.BuildDirectory)/s/aspnetcore + - _PackScript: $(Agent.BuildDirectory)/s/performance/eng/pipelines/tools/pack-bcs-archives.ps1 + - _ShippingDir: $(Agent.BuildDirectory)/s/aspnetcore/artifacts/packages/Release/Shipping + - _StagingRoot: $(Build.ArtifactStagingDirectory)/bcs + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng/build.sh + --ci + --configuration Release + --arch x64 + --pack + --all + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.linux-x64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build linux-x64 + + - pwsh: >- + & "$(_PackScript)" -Rids linux-x64 -Format targz -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Pack BCS archive (linux-x64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_linux_x64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)/BuildArtifacts_linux_x64_Release_aspnetcore + artifactName: BuildArtifacts_linux_x64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Linux x64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)/artifacts/log + artifactName: Linux_x64_perf_build_Logs_Attempt_$(System.JobAttempt) + +# =========================================================================== +# Linux arm64 build job (cross-build from the x64 host) +# =========================================================================== +- ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - job: Linux_arm64_build + displayName: 'Build: Linux ARM64 (perf-build)' + timeoutInMinutes: 180 + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-ubuntu-2204 + variables: + - _AspNetCoreRoot: $(Agent.BuildDirectory)/s/aspnetcore + - _PackScript: $(Agent.BuildDirectory)/s/performance/eng/pipelines/tools/pack-bcs-archives.ps1 + - _ShippingDir: $(Agent.BuildDirectory)/s/aspnetcore/artifacts/packages/Release/Shipping + - _StagingRoot: $(Build.ArtifactStagingDirectory)/bcs + steps: + - checkout: ${{ parameters.aspnetcoreRepoAlias }} + path: s/aspnetcore + fetchDepth: 1 + fetchTags: false + clean: true + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + fetchDepth: 1 + fetchTags: false + clean: true + + - script: >- + eng/build.sh + --ci + --configuration Release + --arch arm64 + --pack + --all + --no-build-java + -p:OnlyPackPlatformSpecificPackages=true + $(_BuildArgs) + /bl:artifacts/log/Release/Build.linux-arm64.binlog + workingDirectory: $(_AspNetCoreRoot) + displayName: Build linux-arm64 + + - pwsh: >- + & "$(_PackScript)" -Rids linux-arm64 -Format targz -ShippingDir "$(_ShippingDir)" -StagingRoot "$(_StagingRoot)" + displayName: 'Pack BCS archive (linux-arm64)' + - task: PublishPipelineArtifact@1 + displayName: 'Publish BuildArtifacts_linux_arm64_Release_aspnetcore' + inputs: + targetPath: $(_StagingRoot)/BuildArtifacts_linux_arm64_Release_aspnetcore + artifactName: BuildArtifacts_linux_arm64_Release_aspnetcore + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Linux arm64 build logs' + condition: always() + continueOnError: true + inputs: + targetPath: $(_AspNetCoreRoot)/artifacts/log + artifactName: Linux_arm64_perf_build_Logs_Attempt_$(System.JobAttempt) diff --git a/eng/pipelines/aspnetcore-perf-build.yml b/eng/pipelines/aspnetcore-perf-build.yml new file mode 100644 index 00000000000..e5a02accc79 --- /dev/null +++ b/eng/pipelines/aspnetcore-perf-build.yml @@ -0,0 +1,177 @@ +# +# aspnetcore-perf-build.yml +# +# Azure DevOps pipeline (hosted in dotnet/performance) that builds every commit +# on dotnet/aspnetcore `main` from source, packs per-RID +# Microsoft.AspNetCore.App.Runtime archives, and uploads them to the Build Cache +# Service (BCS @ pvscmdupload) so dotnet/crank can resolve ASP.NET Core runtime +# binaries by commit SHA for performance-regression bisection. +# +# WHY THIS LIVES HERE (and not in dotnet/aspnetcore): moving it into +# dotnet/performance gives the perf team full merge control. The previous +# aspnetcore-hosted version reused aspnetcore's `default-build.yml`, which cannot +# be consumed cross-repo (its `@self` references resolve to the ROOT pipeline +# repo = performance). So the build jobs are re-authored from source in +# eng/pipelines/aspnetcore-perf-build-jobs.yml. +# +# HOW PER-COMMIT FIRING WORKS: the aspnetcore AzDO mirror +# (internal/dotnet-aspnetcore) is declared as a repository resource with a CI +# `trigger` on main. A push to aspnetcore main fires this pipeline +# (Build.Reason == ResourceTrigger). The build jobs check out that resource at +# the triggering commit. +# +# SHA INVERSION (load-bearing): because the pipeline is hosted in performance, +# `Build.SourceVersion` is the PERFORMANCE commit, not the aspnetcore one. The +# correct BCS {sha} is the triggering aspnetcore commit, exposed as +# `$(resources.repositories.aspnetcore.version)`. We pass it as `sha:` to the +# shared register/upload templates (which default `sha` to $(Build.SourceVersion) +# for the runtime perf-build that is hosted in dotnet/runtime). +# +# PUBLIC FEEDS ONLY: like aspnetcore's ci-public.yml, the from-source pack build +# uses public feeds (no internal runtime download, no dotnetbuilds-internal-read +# connection). Only the BCS upload uses the proven `.NET Performance` connection. +# +# NOT 1ES: per-commit perf artifacts go to a private BCS cache (unsigned, not +# redistributed), so we do not extend 1ES.Official -- matching dotnet/runtime's +# perf-build. +# +# ATOMICITY: continueOnError:false everywhere; any failure sinks the build to +# Failed and the MissingBuildsTrigger Function skips indexing that SHA. +# +# BCS output layout: +# builds/aspnetcore/buildArtifacts/{sha}/{configKey}/buildInfo.json +# builds/aspnetcore/buildArtifacts/{sha}/{configKey}/{archiveFile} +# + +parameters: +# Per-config enablement booleans (default true). Surfaced as pipeline parameters +# so they appear in the build's templateParameters for the dotnet-performance-infra +# MissingBuildsTrigger PerConfiguration indexer to count expected configs. In v1's +# eager per-commit mode all five are true; they are forward-compat for a future +# Function that queues a subset per config. +- name: aspnetcore_x64_linux + displayName: 'Build aspnetcore_x64_linux' + type: boolean + default: true +- name: aspnetcore_arm64_linux + displayName: 'Build aspnetcore_arm64_linux' + type: boolean + default: true +- name: aspnetcore_x64_windows + displayName: 'Build aspnetcore_x64_windows' + type: boolean + default: true +- name: aspnetcore_arm64_windows + displayName: 'Build aspnetcore_arm64_windows' + type: boolean + default: true +- name: aspnetcore_x86_windows + displayName: 'Build aspnetcore_x86_windows' + type: boolean + default: true + +# Self (performance) pushes must not fire this pipeline; only the aspnetcore +# resource trigger (below) does. +trigger: none +pr: none + +variables: +- name: _TeamName + value: AspNetCore +# Lean build args mirrored from aspnetcore ci-public.yml. Deliberately omit +# signing / installers / helix / publishing and any internal runtime download +# (public feeds only) -- none contribute to the per-RID runtime pack. +- name: _BuildArgs + value: /p:TeamName=$(_TeamName) + /p:OfficialBuildId=$(Build.BuildNumber) + /p:SkipTestBuild=true +- template: /eng/common/templates/variables/pool-providers.yml + +resources: + repositories: + # aspnetcore AzDO mirror. The CI trigger on main is what fires this pipeline + # per aspnetcore commit; the build jobs check it out at the triggering commit, + # and $(resources.repositories.aspnetcore.version) is the BCS {sha}. + - repository: aspnetcore + type: git + name: internal/dotnet-aspnetcore + ref: refs/heads/main + trigger: + batch: false + branches: + include: + - main + +stages: + +# ============================================================================ +# RegisterBuild -- writes a per-configKey buildInfo.json marker to BCS at +# $web/builds/aspnetcore/buildArtifacts/{sha}/{configKey}/buildInfo.json +# Runs in parallel with Build (dependsOn: []). Gated to internal + +# (ResourceTrigger | Manual): ResourceTrigger is the steady-state per-commit +# flow, Manual covers seed/runbook queues. +# ============================================================================ +- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: + - stage: RegisterBuild + displayName: 'Register Build' + dependsOn: [] + jobs: + - template: /eng/pipelines/register-build-jobs.yml + parameters: + repoName: aspnetcore + sha: $(resources.repositories.aspnetcore.version) + buildType: + - ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - aspnetcore_x64_linux + - ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - aspnetcore_arm64_linux + - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - aspnetcore_x64_windows + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - aspnetcore_arm64_windows + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - aspnetcore_x86_windows + +- stage: Build + displayName: Build + dependsOn: [] + jobs: + - template: /eng/pipelines/aspnetcore-perf-build-jobs.yml + parameters: + aspnetcoreRepoAlias: aspnetcore + performanceRepoAlias: self + aspnetcore_x64_linux: ${{ parameters.aspnetcore_x64_linux }} + aspnetcore_arm64_linux: ${{ parameters.aspnetcore_arm64_linux }} + aspnetcore_x64_windows: ${{ parameters.aspnetcore_x64_windows }} + aspnetcore_arm64_windows: ${{ parameters.aspnetcore_arm64_windows }} + aspnetcore_x86_windows: ${{ parameters.aspnetcore_x86_windows }} + +# ============================================================================ +# UploadArtifacts -- downloads each Build-stage pipeline artifact and uploads it +# to $web/builds/aspnetcore/buildArtifacts/{sha}/{configKey}/{file}. Depends on +# Build + RegisterBuild; condition succeeded() so any failure skips indexing. +# Same internal + (ResourceTrigger | Manual) gate as RegisterBuild. +# ============================================================================ +- ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: + - stage: UploadArtifacts + displayName: 'Upload Artifacts to BCS' + dependsOn: + - Build + - RegisterBuild + condition: succeeded() + jobs: + - template: /eng/pipelines/upload-build-artifacts-jobs.yml + parameters: + repoName: aspnetcore + sha: $(resources.repositories.aspnetcore.version) + buildType: + - ${{ if eq(parameters.aspnetcore_x64_linux, true) }}: + - aspnetcore_x64_linux + - ${{ if eq(parameters.aspnetcore_arm64_linux, true) }}: + - aspnetcore_arm64_linux + - ${{ if eq(parameters.aspnetcore_x64_windows, true) }}: + - aspnetcore_x64_windows + - ${{ if eq(parameters.aspnetcore_arm64_windows, true) }}: + - aspnetcore_arm64_windows + - ${{ if eq(parameters.aspnetcore_x86_windows, true) }}: + - aspnetcore_x86_windows diff --git a/eng/pipelines/register-build-jobs.yml b/eng/pipelines/register-build-jobs.yml index 7b3aa53a1bd..e998dd3d9b9 100644 --- a/eng/pipelines/register-build-jobs.yml +++ b/eng/pipelines/register-build-jobs.yml @@ -10,6 +10,11 @@ parameters: # pipeline; callers from other tenants (e.g. dotnet/aspnetcore) pass # repoName explicitly. repoName: 'runtime' + # sha is forwarded to each register-build-job so the buildInfo.json marker + # lands under builds/{repoName}/buildArtifacts/{sha}/... Defaults to the self + # repo commit; resource-built tenants (e.g. aspnetcore) pass the resource's + # commit, e.g. sha: $(resources.repositories.aspnetcore.version). + sha: '$(Build.SourceVersion)' jobs: - ${{ each type in parameters.buildType }}: @@ -18,3 +23,4 @@ jobs: buildType: ${{ type }} buildId: $(Build.BuildId) repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} diff --git a/eng/pipelines/templates/register-build-job.yml b/eng/pipelines/templates/register-build-job.yml index 7a275f64f2b..d397cbe22f0 100644 --- a/eng/pipelines/templates/register-build-job.yml +++ b/eng/pipelines/templates/register-build-job.yml @@ -7,6 +7,16 @@ parameters: # callers from other tenants (e.g. dotnet/aspnetcore) pass repoName # explicitly. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path + # (builds/{repoName}/buildArtifacts/{sha}/...). Defaults to the self repo's + # triggering commit ($(Build.SourceVersion)) for back-compat with the runtime + # perf-build pipeline (which is hosted in dotnet/runtime, so self == the repo + # being built). Callers where the built repo is a *resource* rather than self + # (e.g. the dotnet/aspnetcore perf-build hosted in dotnet/performance) must + # pass the resource's commit explicitly, e.g. + # sha: $(resources.repositories.aspnetcore.version), because Build.SourceVersion + # would otherwise resolve to the performance pipeline commit. + sha: '$(Build.SourceVersion)' jobs: - job: RegisterBuild_${{ parameters.buildType }} @@ -20,4 +30,4 @@ jobs: scriptType: 'pscore' scriptLocation: 'inlineScript' inlineScript: | - az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --data '{"buildId":"${{ parameters.buildId }}"}' --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/buildInfo.json" --overwrite true + az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --data '{"buildId":"${{ parameters.buildId }}"}' --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/buildInfo.json" --overwrite true diff --git a/eng/pipelines/templates/upload-build-artifacts-job.yml b/eng/pipelines/templates/upload-build-artifacts-job.yml index 5da08e5553d..fd81e2d74d3 100644 --- a/eng/pipelines/templates/upload-build-artifacts-job.yml +++ b/eng/pipelines/templates/upload-build-artifacts-job.yml @@ -8,6 +8,16 @@ parameters: # Callers building a different repo (e.g. dotnet/aspnetcore) pass repoName explicitly # so that the upload lands at a tenant-specific path the indexer/crank then read. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path: + # builds/{repoName}/buildArtifacts/{sha}/{buildType}/{file} + # Defaults to the self repo's triggering commit ($(Build.SourceVersion)) for + # back-compat with the runtime perf-build pipeline (hosted in dotnet/runtime, + # so self == the repo being built). Callers where the built repo is a + # *resource* rather than self (e.g. dotnet/aspnetcore perf-build hosted in + # dotnet/performance) must pass the resource's commit explicitly, e.g. + # sha: $(resources.repositories.aspnetcore.version), because Build.SourceVersion + # would otherwise resolve to the performance pipeline commit. + sha: '$(Build.SourceVersion)' jobs: - ${{ each artifact in parameters.artifacts }}: @@ -20,7 +30,7 @@ jobs: displayName: Download Build Artifact ${{ artifact.artifactName }} artifact: ${{ artifact.artifactName }} - ${{ each fileName in artifact.files }}: - - script: echo "pvscmdupload/\$web/builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/${{ fileName }}" + - script: echo "pvscmdupload/\$web/builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/${{ fileName }}" displayName: 'Artifact upload path' - task: AzureCLI@2 displayName: 'Upload ${{ fileName }} file' @@ -29,4 +39,4 @@ jobs: scriptType: 'pscore' scriptLocation: 'inlineScript' inlineScript: | - az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --file "$(Pipeline.Workspace)/${{ artifact.artifactName }}/${{ fileName }}" --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ variables['Build.SourceVersion'] }}/${{ parameters.buildType }}/${{ fileName }}" --overwrite true + az storage blob upload --auth-mode login --account-name pvscmdupload --container-name '$web' --file "$(Pipeline.Workspace)/${{ artifact.artifactName }}/${{ fileName }}" --name "builds/${{ parameters.repoName }}/buildArtifacts/${{ parameters.sha }}/${{ parameters.buildType }}/${{ fileName }}" --overwrite true diff --git a/eng/pipelines/tools/pack-bcs-archives.ps1 b/eng/pipelines/tools/pack-bcs-archives.ps1 new file mode 100644 index 00000000000..6a943bf96fc --- /dev/null +++ b/eng/pipelines/tools/pack-bcs-archives.ps1 @@ -0,0 +1,187 @@ +<# +.SYNOPSIS + Packs per-RID Microsoft.AspNetCore.App runtime packs into Build Cache + Service (BCS) archives for the perf-build pipeline. + +.DESCRIPTION + This is the single, canonical "find the runtime-pack nupkg and zip it into + the BCS archive layout" recipe used by every job in perf-build.yml (Windows + x64/x86/arm64, Linux x64, Linux arm64). The build jobs invoke it from their + `afterBuild` hook with the RID(s) they produced and the archive format that + matches their platform convention (.zip on Windows, .tar.gz on Linux). + + For each RID it: + 1. Locates artifacts/packages/Release/Shipping/Microsoft.AspNetCore.App.Runtime.{rid}.*.nupkg + (excluding the *.symbols.nupkg). + 2. Extracts it into the LOWERCASE archive root + microsoft.aspnetcore.app.runtime.{rid}/Release/ -- this directory name + is a load-bearing contract for the future dotnet/crank aspnetcore + overlay path, which mirrors BuildCacheClient's + FindDirectory(extractDir, "microsoft.aspnetcore.app.runtime.{rid}"). + 3. Validates the managed Microsoft.AspNetCore.*.dll assemblies landed under + runtimes/{rid}/lib and strips any stray *.pdb / *.dbg files. + 4. Compresses the lowercase root directory into the per-config archive. + 5. Asserts the archive's single root entry matches the contract. + +.PARAMETER Rids + One or more .NET RIDs to pack (e.g. win-x64, win-x86, win-arm64, linux-x64, + linux-arm64). os/arch and the BCS naming are derived from each RID. + +.PARAMETER Format + Archive format: 'zip' (Windows convention) or 'targz' (Linux convention). + +.PARAMETER ShippingDir + Directory containing the runtime-pack nupkgs. Defaults to + $(Build.SourcesDirectory)/artifacts/packages/Release/Shipping so the + pipeline can call this with no path arguments; override it for local runs. + +.PARAMETER StagingRoot + Directory under which the per-config archive staging folders are created. + Defaults to $(Build.ArtifactStagingDirectory)/bcs. + +.EXAMPLE + ./pack-bcs-archives.ps1 -Rids win-x64,win-x86,win-arm64 -Format zip + +.EXAMPLE + ./pack-bcs-archives.ps1 -Rids linux-x64 -Format targz +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string[]]$Rids, + + [Parameter(Mandatory = $true)] + [ValidateSet('zip', 'targz')] + [string]$Format, + + [string]$ShippingDir = (Join-Path $env:BUILD_SOURCESDIRECTORY 'artifacts/packages/Release/Shipping'), + + [string]$StagingRoot = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY 'bcs') +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest +Add-Type -AssemblyName System.IO.Compression.FileSystem + +function Get-RidParts { + param([string]$Rid) + + $dash = $Rid.IndexOf('-') + if ($dash -lt 1) { + throw "RID '$Rid' is not in the expected '-' form." + } + + $osToken = $Rid.Substring(0, $dash) + $arch = $Rid.Substring($dash + 1) + $os = switch ($osToken) { + 'win' { 'windows' } + 'linux' { 'linux' } + 'osx' { 'osx' } + default { throw "Unsupported RID os token '$osToken' in RID '$Rid'." } + } + + return [pscustomobject]@{ Os = $os; Arch = $arch } +} + +if (-not (Test-Path $ShippingDir)) { + throw "Shipping directory '$ShippingDir' does not exist." +} + +New-Item -ItemType Directory -Force -Path $StagingRoot | Out-Null + +foreach ($rid in $Rids) { + $parts = Get-RidParts -Rid $rid + $os = $parts.Os + $arch = $parts.Arch + $configKey = "aspnetcore_${arch}_${os}" + $artifactName = "BuildArtifacts_${os}_${arch}_Release_aspnetcore" + $ext = if ($Format -eq 'zip') { 'zip' } else { 'tar.gz' } + $archiveFile = "${artifactName}.${ext}" + + Write-Host '' + Write-Host "=== Packing $configKey (rid=$rid, format=$Format) ===" + + $pattern = "Microsoft.AspNetCore.App.Runtime.$rid.*.nupkg" + $nupkg = Get-ChildItem -Path $ShippingDir -Filter $pattern -File -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notmatch '\.symbols\.nupkg$' } | + Sort-Object Name | + Select-Object -First 1 + if (-not $nupkg) { + throw "Could not find runtime pack nupkg matching '$pattern' under '$ShippingDir'." + } + Write-Host "Found nupkg: $($nupkg.FullName)" + + # microsoft.aspnetcore.app.runtime.{rid}/Release/ <-- archive root (lowercase!) + $stageDir = Join-Path $StagingRoot $artifactName + $payloadDir = Join-Path $stageDir "microsoft.aspnetcore.app.runtime.$rid" + $releaseDir = Join-Path $payloadDir 'Release' + if (Test-Path $payloadDir) { + Remove-Item -LiteralPath $payloadDir -Recurse -Force + } + New-Item -ItemType Directory -Force -Path $releaseDir | Out-Null + + # Extract the nupkg (it's a zip) into Release/. Using the .NET API keeps the + # extraction identical on Windows and Linux agents (no unzip/Expand-Archive split). + [System.IO.Compression.ZipFile]::ExtractToDirectory($nupkg.FullName, $releaseDir) + + $runtimesDir = Join-Path $releaseDir "runtimes/$rid" + if (-not (Test-Path $runtimesDir)) { + throw "Extracted runtime pack is missing expected directory '$runtimesDir'." + } + + # Sanity-check that managed assemblies landed where we expect. + $libMatches = @(Get-ChildItem -Path (Join-Path $runtimesDir 'lib') -Recurse -Filter 'Microsoft.AspNetCore.*.dll' -ErrorAction SilentlyContinue) + if ($libMatches.Count -eq 0) { + throw "Extracted runtime pack for $rid is missing managed Microsoft.AspNetCore.*.dll under 'runtimes/$rid/lib'." + } + Write-Host "Validated managed lib dir contains $($libMatches.Count) Microsoft.AspNetCore.*.dll(s)." + + # Defensive: strip debug-symbol files that occasionally tag along. + Get-ChildItem -Path $runtimesDir -Recurse -Include '*.pdb', '*.dbg' -ErrorAction SilentlyContinue | + ForEach-Object { Remove-Item -LiteralPath $_.FullName -Force } + + $archivePath = Join-Path $stageDir $archiveFile + if (Test-Path $archivePath) { + Remove-Item -LiteralPath $archivePath -Force + } + + $expectedRoot = "microsoft.aspnetcore.app.runtime.$rid/" + + if ($Format -eq 'zip') { + # Zip the lowercase root directory so the archive contents start with + # `microsoft.aspnetcore.app.runtime.{rid}/...`. + Compress-Archive -Path $payloadDir -DestinationPath $archivePath -CompressionLevel Optimal -Force + + # Assert the archive root matches the load-bearing contract. + $zip = [System.IO.Compression.ZipFile]::OpenRead($archivePath) + try { + $rootEntries = @($zip.Entries | ForEach-Object { ($_.FullName -split '/')[0] } | Sort-Object -Unique) + if ($rootEntries.Count -ne 1 -or "$($rootEntries[0])/" -ne $expectedRoot) { + throw "Archive '$archivePath' has root entries [$($rootEntries -join ', ')] but expected exactly '$expectedRoot' (load-bearing contract for future crank PR)." + } + } + finally { + $zip.Dispose() + } + } + else { + # Tar the lowercase root directory (so the archive contents start with + # `microsoft.aspnetcore.app.runtime.{rid}/...`). + & tar -czf $archivePath -C $stageDir "microsoft.aspnetcore.app.runtime.$rid" + if ($LASTEXITCODE -ne 0) { + throw "tar of '$payloadDir' failed (exit $LASTEXITCODE)." + } + + # Assert the archive root matches the load-bearing contract. + $firstEntry = (& tar -tzf $archivePath | Select-Object -First 1) + if (-not $firstEntry.StartsWith($expectedRoot)) { + throw "Archive '$archivePath' first entry '$firstEntry' does not start with '$expectedRoot' (load-bearing contract for future crank PR)." + } + } + + # Drop the working payload directory so only the archive ends up in the published pipeline artifact. + Remove-Item -LiteralPath $payloadDir -Recurse -Force + + $size = (Get-Item $archivePath).Length + Write-Host "Produced archive: $archivePath ($size bytes)" +} diff --git a/eng/pipelines/upload-build-artifacts-jobs.yml b/eng/pipelines/upload-build-artifacts-jobs.yml index b917db9457a..1979a48a8aa 100644 --- a/eng/pipelines/upload-build-artifacts-jobs.yml +++ b/eng/pipelines/upload-build-artifacts-jobs.yml @@ -7,6 +7,13 @@ parameters: # back-compat with the existing dotnet/runtime perf-build pipeline; callers from # other tenants (e.g. dotnet/aspnetcore) pass repoName explicitly. repoName: 'runtime' + # sha controls the {sha} segment of the BCS blob path. Defaults to the self + # repo commit ($(Build.SourceVersion)) for back-compat with the runtime + # perf-build pipeline. Resource-built tenants (e.g. aspnetcore, whose source + # is a repository resource rather than self) pass the resource's commit, e.g. + # sha: $(resources.repositories.aspnetcore.version). Runtime config branches + # below omit sha and inherit the leaf template's $(Build.SourceVersion) default. + sha: '$(Build.SourceVersion)' jobs: - ${{ if containsValue(parameters.buildType, 'coreclr_arm64_linux') }}: @@ -174,6 +181,7 @@ jobs: buildType: 'aspnetcore_x64_linux' dependencyJobName: Linux_x64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_x64_Release_aspnetcore' files: [ 'BuildArtifacts_linux_x64_Release_aspnetcore.tar.gz' ] @@ -184,6 +192,7 @@ jobs: buildType: 'aspnetcore_arm64_linux' dependencyJobName: Linux_arm64_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_linux_arm64_Release_aspnetcore' files: [ 'BuildArtifacts_linux_arm64_Release_aspnetcore.tar.gz' ] @@ -194,6 +203,7 @@ jobs: buildType: 'aspnetcore_x64_windows' dependencyJobName: Windows_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x64_Release_aspnetcore' files: [ 'BuildArtifacts_windows_x64_Release_aspnetcore.zip' ] @@ -204,6 +214,7 @@ jobs: buildType: 'aspnetcore_arm64_windows' dependencyJobName: Windows_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_arm64_Release_aspnetcore' files: [ 'BuildArtifacts_windows_arm64_Release_aspnetcore.zip' ] @@ -214,6 +225,7 @@ jobs: buildType: 'aspnetcore_x86_windows' dependencyJobName: Windows_build repoName: ${{ parameters.repoName }} + sha: ${{ parameters.sha }} artifacts: - artifactName: 'BuildArtifacts_windows_x86_Release_aspnetcore' files: [ 'BuildArtifacts_windows_x86_Release_aspnetcore.zip' ] From 657cd3dc87d17ac2f358e40e9b300f239da3d156 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 19 Jun 2026 12:00:41 -0700 Subject: [PATCH 2/3] Tag perf-build with aspnetcore-sha for infra indexer correlation Add an early, unconditional RegisterBuild job that tags the run with aspnetcore-sha:<40-char sha> (the aspnetcore commit from the repo-resource version macro). The build record's sourceVersion is the performance commit, so this tag is the dotnet-performance-infra indexer's sole aspnetcore-sha signal. Additive; the BCS {sha} override and buildInfo.json are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/aspnetcore-perf-build.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/eng/pipelines/aspnetcore-perf-build.yml b/eng/pipelines/aspnetcore-perf-build.yml index e5a02accc79..75519150975 100644 --- a/eng/pipelines/aspnetcore-perf-build.yml +++ b/eng/pipelines/aspnetcore-perf-build.yml @@ -110,12 +110,37 @@ stages: # Runs in parallel with Build (dependsOn: []). Gated to internal + # (ResourceTrigger | Manual): ResourceTrigger is the steady-state per-commit # flow, Manual covers seed/runbook queues. +# +# The first job tags the build with `aspnetcore-sha:<40-char sha>`. Because the +# pipeline is hosted in performance, the build record's sourceVersion is the +# PERFORMANCE commit, so the dotnet-performance-infra indexer cannot correlate a +# build to its aspnetcore commit via sourceVersion. This tag is its SOLE +# aspnetcore-sha signal on the build record. It is additive and independent of +# the BCS {sha} path override (which stays $(resources.repositories.aspnetcore.version)). +# Added once, early, and unconditionally so it is present even if later stages fail. # ============================================================================ - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: - stage: RegisterBuild displayName: 'Register Build' dependsOn: [] jobs: + - job: TagBuild + displayName: 'Tag build with aspnetcore sha' + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals 1es-ubuntu-2204 + steps: + - checkout: none + # $(resources.repositories.aspnetcore.version) is the AzDO runtime macro for + # the triggering aspnetcore commit -- the same value passed as `sha:` to the + # register/upload templates below. AzDO expands the macro before pwsh runs, so + # $resolvedSha holds the literal 40-char commit sha. The echo lets us eyeball + # the resolved value in the first real run. + - pwsh: | + $resolvedSha = "$(resources.repositories.aspnetcore.version)" + Write-Host "Resolved aspnetcore commit sha: $resolvedSha" + Write-Host "##vso[build.addbuildtag]aspnetcore-sha:$resolvedSha" + displayName: 'Add aspnetcore-sha build tag' - template: /eng/pipelines/register-build-jobs.yml parameters: repoName: aspnetcore From ddfcf04476099456ade810c4a0576c41b7d09ddb Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 19 Jun 2026 12:11:36 -0700 Subject: [PATCH 3/3] Drop aspnetcore-sha build tag (Option B: indexer reads repo-resource version) The dotnet-performance-infra indexer derives the aspnetcore commit directly from the build's resources.repositories.aspnetcore.version via the runs API, so the dedicated TagBuild job is redundant. The BCS {sha} path override and buildInfo.json are unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/pipelines/aspnetcore-perf-build.yml | 29 +++++-------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/aspnetcore-perf-build.yml b/eng/pipelines/aspnetcore-perf-build.yml index 75519150975..88e67db79e4 100644 --- a/eng/pipelines/aspnetcore-perf-build.yml +++ b/eng/pipelines/aspnetcore-perf-build.yml @@ -111,36 +111,17 @@ stages: # (ResourceTrigger | Manual): ResourceTrigger is the steady-state per-commit # flow, Manual covers seed/runbook queues. # -# The first job tags the build with `aspnetcore-sha:<40-char sha>`. Because the -# pipeline is hosted in performance, the build record's sourceVersion is the -# PERFORMANCE commit, so the dotnet-performance-infra indexer cannot correlate a -# build to its aspnetcore commit via sourceVersion. This tag is its SOLE -# aspnetcore-sha signal on the build record. It is additive and independent of -# the BCS {sha} path override (which stays $(resources.repositories.aspnetcore.version)). -# Added once, early, and unconditionally so it is present even if later stages fail. +# The aspnetcore commit this build corresponds to is NOT tagged on the run: the +# dotnet-performance-infra indexer reads it directly from the build's +# `resources.repositories.aspnetcore.version` via the runs API (intrinsic to the +# build carrying the aspnetcore repo resource). The BCS {sha} path override below +# still uses that same value. # ============================================================================ - ${{ if and(ne(variables['System.TeamProject'], 'public'), or(eq(variables['Build.Reason'], 'ResourceTrigger'), eq(variables['Build.Reason'], 'Manual'))) }}: - stage: RegisterBuild displayName: 'Register Build' dependsOn: [] jobs: - - job: TagBuild - displayName: 'Tag build with aspnetcore sha' - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals 1es-ubuntu-2204 - steps: - - checkout: none - # $(resources.repositories.aspnetcore.version) is the AzDO runtime macro for - # the triggering aspnetcore commit -- the same value passed as `sha:` to the - # register/upload templates below. AzDO expands the macro before pwsh runs, so - # $resolvedSha holds the literal 40-char commit sha. The echo lets us eyeball - # the resolved value in the first real run. - - pwsh: | - $resolvedSha = "$(resources.repositories.aspnetcore.version)" - Write-Host "Resolved aspnetcore commit sha: $resolvedSha" - Write-Host "##vso[build.addbuildtag]aspnetcore-sha:$resolvedSha" - displayName: 'Add aspnetcore-sha build tag' - template: /eng/pipelines/register-build-jobs.yml parameters: repoName: aspnetcore