From 6310a64039b4025630f3f3d9032d78bea2a67a60 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 25 Mar 2021 13:37:13 -0700 Subject: [PATCH 01/50] Tool changes to support nightly test package release --- eng/scripts/Initialize-VcpkgRelease.ps1 | 27 ++++++++++++++++++++----- eng/scripts/Update-VcpkgPort.ps1 | 21 ++++++++++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/eng/scripts/Initialize-VcpkgRelease.ps1 b/eng/scripts/Initialize-VcpkgRelease.ps1 index f10cb582af..df83076b9f 100644 --- a/eng/scripts/Initialize-VcpkgRelease.ps1 +++ b/eng/scripts/Initialize-VcpkgRelease.ps1 @@ -12,6 +12,10 @@ Location of the relevant package-info.json file .PARAMETER GitHubRepo Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp) +.PARAMETER TestReleaseRef +If supplied update the portfile.cmake file's REF and SHA512 with values +associated with the given ref. + #> param ( @@ -25,7 +29,9 @@ param ( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] - [string] $GitHubRepo + [string] $GitHubRepo, + + [string] $TestReleaseRef ) # If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true @@ -36,8 +42,14 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) { exit } -$packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json -$tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" +$tarGzUri = if (-not $TestReleaseRef) { + Write-Verbose "Initializing Production Release" + $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json + "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" +} else { + Write-Verbose "Initializing Test Release" + "https://github.com/$GitHubRepo/archive/$TestReleaseRef.tar.gz" +} Write-Host "Downloading tarball to compute hash from $tarGzUri" $localTarGzPath = New-TemporaryFile @@ -53,7 +65,12 @@ $portfileLocation = "$SourceDirectory/port/portfile.cmake" # recommended in vcpkg documentation # Before: " SHA512 1" # After: " SHA512 f6cf1c16c52" -$newContent = Get-Content -Raw -Path $portfileLocation ` - | ForEach-Object { $_ -replace '(SHA512\s+)1', "`${1}$sha512" } +$portFileContent = Get-Content -Raw -Path $portfileLocation +$newContent = $portFileContent -replace '(SHA512\s+)1', "`${1}$sha512" + +if ($TestReleaseRef) { + Write-Verbose "Overriding REF with test release ref: $TestReleaseRef" + $newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $TestReleaseRef" +} $newContent | Set-Content $portfileLocation -NoNewLine diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index e0e5c32d5d..b0e660e67d 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -11,9 +11,9 @@ param ( [ValidateNotNullOrEmpty()] [string] $VcpkgPortName, - [Parameter(Mandatory = $true)] - [ValidateNotNullOrEmpty()] - [string] $GitCommitParameters + [string] $GitCommitParameters, + + [switch] $TestRelease ) ."$PSScriptRoot/../common/scripts/common.ps1" @@ -77,8 +77,14 @@ if ($LASTEXITCODE -ne 0) { exit 1 } -Write-Host "./vcpkg.exe x-add-version $VcpkgPortName" -./vcpkg.exe x-add-version $VcpkgPortName +$addVersionAdditionalParameters = if ($TestRelease) { + '--overwrite-version' +} else { + '' +} + +Write-Host "./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters" +./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters if ($LASTEXITCODE -ne 0) { Write-Error "Failed to run vcpkg x-add-version $VcpkgPortName" @@ -147,4 +153,9 @@ Additional parameters to supply to the `git commit` command. These are useful in the context of Azure DevOps where the git client does not have a configured user.name and user.email. +.PARAMETER TestRelease +In the case of a test release set this to ensure that the x-add-version step +includes `--overwrite-version` to ensure nightly packages are properly updated +in the vcpkg repo. + #> From 2f6d0b5aa504ecd82beb6182d56b80d5541767ed Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 29 Mar 2021 08:52:40 -0700 Subject: [PATCH 02/50] Initial wiring up and refactor vcpkg-publish.yml --- .../stages/archetype-cpp-release.yml | 90 +++++++++---------- .../templates/steps/vcpkg-publish.yml | 55 ++++++++++++ 2 files changed, 99 insertions(+), 46 deletions(-) create mode 100644 eng/pipelines/templates/steps/vcpkg-publish.yml diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index efb67f2915..d75dadb0e4 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -4,6 +4,45 @@ parameters: TestPipeline: false stages: + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - stage: Integration + dependsOn: Build + + jobs: + - deployment: PublishNightlyVcpkg + displayName: Publish to vcpkg nightly branch + dependsOn: Build + # TODO: Condition? + environment: github + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + + - pwsh: | + $formattedDate = Get-Date -Format 'yyyy-MM-dd' + $branchName = "nightly/$formattedDate" + Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" + displayName: Set branch name in fork repo + + # Placing artifacts in the same branch in sequence reduces the + # potential for race conditions. If this job does fail to push + # because another job has already pushed to the same branch it + # can be re-run. + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + PRBranchName: $(PrBranchName) + DisplayNameExtension: ${{ artifact.Name }} + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - ${{ each artifact in parameters.Artifacts }}: - stage: Release_${{ replace(artifact.Name, '-', '_') }} @@ -122,52 +161,11 @@ stages: Write-Host "##vso[task.setvariable variable=PrTitle]$title" displayName: Set PR title - - task: Powershell@2 - inputs: - filePath: eng/scripts/Initialize-VcpkgRelease.ps1 - arguments: >- - -GitHubRepo "$(Build.Repository.Name)" - -PackageSpecPath $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json - -SourceDirectory $(Pipeline.Workspace)/packages/${{artifact.Name}}/vcpkg - -Verbose - pwsh: true - displayName: Initialize vcpkg release - - # There are potential race conditions if this script runs - # in parallel against the same branch name. Release only - # one package at a time. - - pwsh: | - Write-Host "git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg" - git clone https://github.com/azure-sdk/vcpkg $(Pipeline.Workspace)/vcpkg - if ($LASTEXITCODE -ne 0) { - Write-Error "Unable to check out vcpkg fork repo" - exit $LASTEXITCODE - } - Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]$(Pipeline.Workspace)/vcpkg" - displayName: Clone vcpkg from upstream - - # Check out the PR branch if it's already in remote. - # Ignore failures. - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(PrBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout Previous PRBranch if it exist. - workingDirectory: $(VcpkgWorkspace) - - - task: Powershell@2 - inputs: - pwsh: true - targetType: filePath - filePath: eng/scripts/Update-VcpkgPort.ps1 - arguments: >- - -ReleaseArtifactSourceDirectory "$(Pipeline.Workspace)/packages/${{ artifact.Name }}" - -PortDestinationDirectory 'ports/${{ artifact.VcpkgPortName }}' - -VcpkgPortName '${{ artifact.VcpkgPortName }}' - -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' - workingDirectory: $(VcpkgWorkspace) - displayName: Update vcpkg port + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + PRBranchName: $(PrBranchName) # Set $(HasChanges) to $true so that # create-pull-request.yml completes the push and PR diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml new file mode 100644 index 0000000000..63490df0b0 --- /dev/null +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -0,0 +1,55 @@ +parameters: + GitHubRepo: $(Build.Repository.Name) + Workspace: $(Pipeline.Workspace) + ArtifactName: not-set + VcpkgPortName: not-set + PRBranchName: not-set + DisplayNameExtension: + +steps: + - task: Powershell@2 + inputs: + filePath: eng/scripts/Initialize-VcpkgRelease.ps1 + arguments: >- + -GitHubRepo "${{ parameters.GitHubRepo }}" + -PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json + -SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg + -Verbose + pwsh: true + displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} + + # There are potential race conditions if this script runs + # in parallel against the same branch name. Release only + # one package at a time. + - pwsh: | + Write-Host "git clone https://github.com/azure-sdk/vcpkg ${{ parameters.Workspace }}/vcpkg" + git clone https://github.com/azure-sdk/vcpkg ${{ parameters.Workspace }}/vcpkg + if ($LASTEXITCODE -ne 0) { + Write-Error "Unable to check out vcpkg fork repo" + exit $LASTEXITCODE + } + Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]${{ parameters.Workspace }}/vcpkg" + displayName: Clone vcpkg from upstream ${{ parameters.DisplayNameExtension }} + + # Check out the PR branch if it's already in remote. + # Ignore failures. + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout Previous PRBranch if it exist. ${{ parameters.DisplayNameExtension }} + workingDirectory: $(VcpkgWorkspace) + + - task: Powershell@2 + inputs: + pwsh: true + targetType: filePath + filePath: eng/scripts/Update-VcpkgPort.ps1 + arguments: >- + -ReleaseArtifactSourceDirectory "${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}" + -PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}' + -VcpkgPortName '${{ parameters.VcpkgPortName }}' + -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' + workingDirectory: $(VcpkgWorkspace) + displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file From 6e859033f0000288a23bbebc4cc8cc65d7b2b75e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:12:11 -0700 Subject: [PATCH 03/50] Use vcpkg-clone.yml --- .../stages/archetype-cpp-release.yml | 29 +++++++++++++++++-- eng/pipelines/templates/steps/vcpkg-clone.yml | 28 ++++++++++++++++++ .../templates/steps/vcpkg-publish.yml | 24 --------------- 3 files changed, 55 insertions(+), 26 deletions(-) create mode 100644 eng/pipelines/templates/steps/vcpkg-clone.yml diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index d75dadb0e4..a122c3a65e 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -1,6 +1,7 @@ parameters: Artifacts: [] ArtifactName: 'not-specified' + ServiceDirectory: not-specified TestPipeline: false stages: @@ -31,6 +32,12 @@ stages: Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" displayName: Set branch name in fork repo + # TODO: Can we do a sparse checkout? + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(PrBranchName) + # Placing artifacts in the same branch in sequence reduces the # potential for race conditions. If this job does fail to push # because another job has already pushed to the same branch it @@ -40,9 +47,23 @@ stages: parameters: ArtifactName: ${{ artifact.Name }} VcpkgPortName: ${{ artifact.VcpkgPortName }} - PRBranchName: $(PrBranchName) DisplayNameExtension: ${{ artifact.Name }} + - pwsh: | + Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" + git reset --soft HEAD~${{ length(parameters.Artifacts) }} + displayName: Reset commits to enable pushing a single commit + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(PrBranchName) + CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} + TargetRepoOwner: azure-sdk + TargetRepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - ${{ each artifact in parameters.Artifacts }}: - stage: Release_${{ replace(artifact.Name, '-', '_') }} @@ -161,11 +182,15 @@ stages: Write-Host "##vso[task.setvariable variable=PrTitle]$title" displayName: Set PR title + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(PrBranchName) + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml parameters: ArtifactName: ${{ artifact.Name }} VcpkgPortName: ${{ artifact.VcpkgPortName }} - PRBranchName: $(PrBranchName) # Set $(HasChanges) to $true so that # create-pull-request.yml completes the push and PR diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml new file mode 100644 index 0000000000..223b1f7d65 --- /dev/null +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -0,0 +1,28 @@ +parameters: + Workspace: $(Pipeline.Workspace) + RepoOwner: not-set + RepoName: vcpkg + PRBranchName: not-set + +steps: + # There are potential race conditions if this script runs + # in parallel against the same branch name. Release only + # one package at a time. + - pwsh: | + Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg" + git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg + if ($LASTEXITCODE -ne 0) { + Write-Error "Unable to check out vcpkg fork repo" + exit $LASTEXITCODE + } + displayName: Clone vcpkg from upstream + + # Check out the PR branch if it's already in remote. + # Ignore failures. + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout Previous PRBranch if it exists + workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml index 63490df0b0..6c433cfe47 100644 --- a/eng/pipelines/templates/steps/vcpkg-publish.yml +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -3,7 +3,6 @@ parameters: Workspace: $(Pipeline.Workspace) ArtifactName: not-set VcpkgPortName: not-set - PRBranchName: not-set DisplayNameExtension: steps: @@ -18,29 +17,6 @@ steps: pwsh: true displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} - # There are potential race conditions if this script runs - # in parallel against the same branch name. Release only - # one package at a time. - - pwsh: | - Write-Host "git clone https://github.com/azure-sdk/vcpkg ${{ parameters.Workspace }}/vcpkg" - git clone https://github.com/azure-sdk/vcpkg ${{ parameters.Workspace }}/vcpkg - if ($LASTEXITCODE -ne 0) { - Write-Error "Unable to check out vcpkg fork repo" - exit $LASTEXITCODE - } - Write-Host "##vso[task.setvariable variable=VcpkgWorkspace]${{ parameters.Workspace }}/vcpkg" - displayName: Clone vcpkg from upstream ${{ parameters.DisplayNameExtension }} - - # Check out the PR branch if it's already in remote. - # Ignore failures. - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout Previous PRBranch if it exist. ${{ parameters.DisplayNameExtension }} - workingDirectory: $(VcpkgWorkspace) - - task: Powershell@2 inputs: pwsh: true From 6ee7833df9eda1bee0a4c634a8f570f2a5de3f30 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:12:58 -0700 Subject: [PATCH 04/50] Disable publishing for verification of nightly builds --- .../stages/archetype-cpp-release.yml | 302 +++++++++--------- 1 file changed, 151 insertions(+), 151 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index a122c3a65e..96a0fc1481 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -64,157 +64,157 @@ stages: ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts - - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - - ${{ each artifact in parameters.Artifacts }}: - - stage: Release_${{ replace(artifact.Name, '-', '_') }} - displayName: 'Release ${{artifact.name}}' - dependsOn: Build - condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) - - jobs: - - deployment: TagRepository - displayName: "Create release tag" - condition: ne(variables['Skip.TagRepository'], 'true') - environment: github - - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: ${{ artifact.Name }} - ServiceDirectory: ${{ parameters.ServiceDirectory }} - TestPipeline: ${{ parameters.TestPipeline }} - - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}} - displayName: Output Visible Artifacts - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLog.ps1 - arguments: > - -ChangeLogLocation $(Pipeline.Workspace)/packages/${{artifact.Name}}/CHANGELOG.md - -VersionString (Get-Content $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json | ConvertFrom-Json).version - -ForRelease $true - pwsh: true - displayName: Verify CHANGELOG.md contents - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: "$(Pipeline.Workspace)/packages/${{artifact.Name}}" - PackageRepository: CPP - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-cpp - - - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - deployment: PublishDocs - displayName: Publish Docs to GitHub pages - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio - # Release tag must exist for link substitution to work - dependsOn: TagRepository - - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - pwsh: Write-Host 'publish docs' - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }} - displayName: Output Visible Artifacts - - template: ../../../common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' - BlobSASKey: '$(azure-sdk-docs-prod-sas)' - BlobName: '$(azure-sdk-docs-prod-blob-name)' - TargetLanguage: 'cpp' - ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}' - # we override the regular script path because we have cloned the build tools repo as a separate artifact. - ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1' - - - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: - - deployment: PublishVcpkg - displayName: Publish to vcpkg - condition: and(succeeded(), ne(variables['Skip.PublishVcpkg'], 'true')) - environment: github - # This step requires the tag step - dependsOn: TagRepository - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - - pwsh: | - $branchName = "azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId)" - if ('$(VcpkgForkBranchName)') { - Write-Host "Using queue time branch name" - $branchName = '$(VcpkgForkBranchName)' - } - Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" - displayName: Set fork branch name - - - pwsh: | - $packageSpec = Get-Content -Raw -Path "$(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json" | ConvertFrom-Json - $version = $packageSpec.version - Write-Host "##vso[task.setvariable variable=PackageVersion]$version" - displayName: Set PackageVersion variable - - - pwsh: | - $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)" - - if ('$(VcpkgPRTitle)') { - Write-Host "Using queue time PR title" - $title = '$(VcpkgPRTitle)' - } - Write-Host "##vso[task.setvariable variable=PrTitle]$title" - displayName: Set PR title - - - template: /eng/pipelines/templates/steps/vcpkg-clone.yml - parameters: - RepoOwner: azure-sdk - PRBranchName: $(PrBranchName) - - - template: /eng/pipelines/templates/steps/vcpkg-publish.yml - parameters: - ArtifactName: ${{ artifact.Name }} - VcpkgPortName: ${{ artifact.VcpkgPortName }} - - # Set $(HasChanges) to $true so that - # create-pull-request.yml completes the push and PR - # submission steps - - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" - displayName: Set $(HasChanges) to $true for create-pull-request.yml - - # SkipCheckingForChanges is true to skip the commit step - # (which is already done by Update-VcpkgPort.ps1) - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoOwner: Microsoft - RepoName: vcpkg - WorkingDirectory: $(Pipeline.Workspace)/vcpkg - PrBranchName: $(PrBranchName) - PRTitle: $(PrTitle) - PRBody: Update vcpkg ports for Azure SDK release. This release may contain multiple ports. - SkipCheckingForChanges: true - BaseBranchName: master - OpenAsDraft: ${{ parameters.TestPipeline }} - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - - - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: - - deployment: UpdatePackageVersion + # - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + # - ${{ each artifact in parameters.Artifacts }}: + # - stage: Release_${{ replace(artifact.Name, '-', '_') }} + # displayName: 'Release ${{artifact.name}}' + # dependsOn: Build + # condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) + + # jobs: + # - deployment: TagRepository + # displayName: "Create release tag" + # condition: ne(variables['Skip.TagRepository'], 'true') + # environment: github + + # pool: + # name: azsdk-pool-mms-win-2019-general + # vmImage: MMS2019 + + # strategy: + # runOnce: + # deploy: + # steps: + # - checkout: self + # - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + # parameters: + # PackageName: ${{ artifact.Name }} + # ServiceDirectory: ${{ parameters.ServiceDirectory }} + # TestPipeline: ${{ parameters.TestPipeline }} + # - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}} + # displayName: Output Visible Artifacts + # - task: Powershell@2 + # inputs: + # filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLog.ps1 + # arguments: > + # -ChangeLogLocation $(Pipeline.Workspace)/packages/${{artifact.Name}}/CHANGELOG.md + # -VersionString (Get-Content $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json | ConvertFrom-Json).version + # -ForRelease $true + # pwsh: true + # displayName: Verify CHANGELOG.md contents + # - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + # parameters: + # ArtifactLocation: "$(Pipeline.Workspace)/packages/${{artifact.Name}}" + # PackageRepository: CPP + # ReleaseSha: $(Build.SourceVersion) + # RepoId: Azure/azure-sdk-for-cpp + + # - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: + # - deployment: PublishDocs + # displayName: Publish Docs to GitHub pages + # condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + # environment: githubio + # # Release tag must exist for link substitution to work + # dependsOn: TagRepository + + # pool: + # name: azsdk-pool-mms-win-2019-general + # vmImage: MMS2019 + + # strategy: + # runOnce: + # deploy: + # steps: + # - checkout: self + # - pwsh: Write-Host 'publish docs' + # - pwsh: | + # Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }} + # displayName: Output Visible Artifacts + # - template: ../../../common/pipelines/templates/steps/publish-blobs.yml + # parameters: + # FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' + # BlobSASKey: '$(azure-sdk-docs-prod-sas)' + # BlobName: '$(azure-sdk-docs-prod-blob-name)' + # TargetLanguage: 'cpp' + # ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}' + # # we override the regular script path because we have cloned the build tools repo as a separate artifact. + # ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1' + + # - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: + # - deployment: PublishVcpkg + # displayName: Publish to vcpkg + # condition: and(succeeded(), ne(variables['Skip.PublishVcpkg'], 'true')) + # environment: github + # # This step requires the tag step + # dependsOn: TagRepository + # pool: + # name: azsdk-pool-mms-win-2019-general + # vmImage: MMS2019 + + # strategy: + # runOnce: + # deploy: + # steps: + # - checkout: self + + # - pwsh: | + # $branchName = "azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId)" + # if ('$(VcpkgForkBranchName)') { + # Write-Host "Using queue time branch name" + # $branchName = '$(VcpkgForkBranchName)' + # } + # Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" + # displayName: Set fork branch name + + # - pwsh: | + # $packageSpec = Get-Content -Raw -Path "$(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json" | ConvertFrom-Json + # $version = $packageSpec.version + # Write-Host "##vso[task.setvariable variable=PackageVersion]$version" + # displayName: Set PackageVersion variable + + # - pwsh: | + # $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)" + + # if ('$(VcpkgPRTitle)') { + # Write-Host "Using queue time PR title" + # $title = '$(VcpkgPRTitle)' + # } + # Write-Host "##vso[task.setvariable variable=PrTitle]$title" + # displayName: Set PR title + + # - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + # parameters: + # RepoOwner: azure-sdk + # PRBranchName: $(PrBranchName) + + # - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + # parameters: + # ArtifactName: ${{ artifact.Name }} + # VcpkgPortName: ${{ artifact.VcpkgPortName }} + + # # Set $(HasChanges) to $true so that + # # create-pull-request.yml completes the push and PR + # # submission steps + # - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" + # displayName: Set $(HasChanges) to $true for create-pull-request.yml + + # # SkipCheckingForChanges is true to skip the commit step + # # (which is already done by Update-VcpkgPort.ps1) + # - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + # parameters: + # RepoOwner: Microsoft + # RepoName: vcpkg + # WorkingDirectory: $(Pipeline.Workspace)/vcpkg + # PrBranchName: $(PrBranchName) + # PRTitle: $(PrTitle) + # PRBody: Update vcpkg ports for Azure SDK release. This release may contain multiple ports. + # SkipCheckingForChanges: true + # BaseBranchName: master + # OpenAsDraft: ${{ parameters.TestPipeline }} + # CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + + # - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: + # - deployment: UpdatePackageVersion displayName: "Update Package Version" condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) environment: github From 70df80ea1550421cc07194a80e1586becebf489d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:14:58 -0700 Subject: [PATCH 05/50] Complete the comment --- .../stages/archetype-cpp-release.yml | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 96a0fc1481..58db98fd59 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -215,26 +215,26 @@ stages: # - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: # - deployment: UpdatePackageVersion - displayName: "Update Package Version" - condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - environment: github - - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - - strategy: - runOnce: - deploy: - steps: - - checkout: self - - pwsh: | - eng/scripts/Update-PkgVersion.ps1 -ServiceDirectory '${{parameters.ServiceDirectory}}' -PackageName '${{artifact.name}}' - displayName: Increment package version - - template: ../../../common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-cpp - PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file + # displayName: "Update Package Version" + # condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + # environment: github + + # pool: + # name: azsdk-pool-mms-win-2019-general + # vmImage: MMS2019 + + # strategy: + # runOnce: + # deploy: + # steps: + # - checkout: self + # - pwsh: | + # eng/scripts/Update-PkgVersion.ps1 -ServiceDirectory '${{parameters.ServiceDirectory}}' -PackageName '${{artifact.name}}' + # displayName: Increment package version + # - template: ../../../common/pipelines/templates/steps/create-pull-request.yml + # parameters: + # RepoName: azure-sdk-for-cpp + # PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + # CommitMsg: "Increment package version after release of ${{ artifact.name }}" + # PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + # CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file From 71e5a2fe887d4596256435123c57c1a84c2ef82f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:16:11 -0700 Subject: [PATCH 06/50] Dependency is enforced at the stage level, not the deployment/job level --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 58db98fd59..b1454882e8 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -12,7 +12,6 @@ stages: jobs: - deployment: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch - dependsOn: Build # TODO: Condition? environment: github From ea249f76a32e416dda83495308ed8e347bdf7332 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:28:16 -0700 Subject: [PATCH 07/50] Remove environment, no approval needed --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index b1454882e8..dff5eeea66 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -13,7 +13,6 @@ stages: - deployment: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch # TODO: Condition? - environment: github pool: name: azsdk-pool-mms-win-2019-general From 346ebc104d332e9db29fd32575ce6f0d46470a0b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:40:16 -0700 Subject: [PATCH 08/50] Add ability to enable test release parameters --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 3 +++ eng/pipelines/templates/steps/vcpkg-publish.yml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index dff5eeea66..a87774559a 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -5,6 +5,7 @@ parameters: TestPipeline: false stages: + # TODO: Check for scheduled run - ${{if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration dependsOn: Build @@ -46,6 +47,8 @@ stages: ArtifactName: ${{ artifact.Name }} VcpkgPortName: ${{ artifact.VcpkgPortName }} DisplayNameExtension: ${{ artifact.Name }} + TestRelease: true + TestReleaseRef: $(Build.SourceVersion) - pwsh: | Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml index 6c433cfe47..dd674685c8 100644 --- a/eng/pipelines/templates/steps/vcpkg-publish.yml +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -1,9 +1,12 @@ parameters: GitHubRepo: $(Build.Repository.Name) + ArtifactPath: $() Workspace: $(Pipeline.Workspace) ArtifactName: not-set VcpkgPortName: not-set DisplayNameExtension: + TestReleaseRef: + TestRelease: false steps: - task: Powershell@2 @@ -13,6 +16,7 @@ steps: -GitHubRepo "${{ parameters.GitHubRepo }}" -PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json -SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg + -TestReleaseRef '${{ parameters.TestReleaseRef }}' -Verbose pwsh: true displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} @@ -27,5 +31,6 @@ steps: -PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}' -VcpkgPortName '${{ parameters.VcpkgPortName }}' -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' + -TestRelease:$${{ parameters.TestRelease }} workingDirectory: $(VcpkgWorkspace) displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file From 20ae6686bd0b52e1d608b8a97a5bcb898583500c Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:42:12 -0700 Subject: [PATCH 09/50] deploy -> job --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index a87774559a..37bfe1bd3f 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -21,7 +21,7 @@ stages: strategy: runOnce: - deploy: + job: steps: - checkout: self From 3cbf95593ff61de3cd57c29c8e501e351602791f Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:43:23 -0700 Subject: [PATCH 10/50] deployment -> job --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 37bfe1bd3f..55e211ff4f 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -11,7 +11,7 @@ stages: dependsOn: Build jobs: - - deployment: PublishNightlyVcpkg + - job: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch # TODO: Condition? From f95e5a08b3fd8323a859eefd88dee8e4f75ad532 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:44:43 -0700 Subject: [PATCH 11/50] Remove strategy --- .../stages/archetype-cpp-release.yml | 85 +++++++++---------- 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 55e211ff4f..f23b9b7397 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -19,50 +19,47 @@ stages: name: azsdk-pool-mms-win-2019-general vmImage: MMS2019 - strategy: - runOnce: - job: - steps: - - checkout: self - - - pwsh: | - $formattedDate = Get-Date -Format 'yyyy-MM-dd' - $branchName = "nightly/$formattedDate" - Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" - displayName: Set branch name in fork repo - - # TODO: Can we do a sparse checkout? - - template: /eng/pipelines/templates/steps/vcpkg-clone.yml - parameters: - RepoOwner: azure-sdk - PRBranchName: $(PrBranchName) - - # Placing artifacts in the same branch in sequence reduces the - # potential for race conditions. If this job does fail to push - # because another job has already pushed to the same branch it - # can be re-run. - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/pipelines/templates/steps/vcpkg-publish.yml - parameters: - ArtifactName: ${{ artifact.Name }} - VcpkgPortName: ${{ artifact.VcpkgPortName }} - DisplayNameExtension: ${{ artifact.Name }} - TestRelease: true - TestReleaseRef: $(Build.SourceVersion) - - - pwsh: | - Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" - git reset --soft HEAD~${{ length(parameters.Artifacts) }} - displayName: Reset commits to enable pushing a single commit - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(PrBranchName) - CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} - TargetRepoOwner: azure-sdk - TargetRepoName: vcpkg - WorkingDirectory: $(Pipeline.Workspace)/vcpkg - ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + steps: + - checkout: self + + - pwsh: | + $formattedDate = Get-Date -Format 'yyyy-MM-dd' + $branchName = "nightly/$formattedDate" + Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" + displayName: Set branch name in fork repo + + # TODO: Can we do a sparse checkout? + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(PrBranchName) + + # Placing artifacts in the same branch in sequence reduces the + # potential for race conditions. If this job does fail to push + # because another job has already pushed to the same branch it + # can be re-run. + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + DisplayNameExtension: ${{ artifact.Name }} + TestRelease: true + TestReleaseRef: $(Build.SourceVersion) + + - pwsh: | + Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" + git reset --soft HEAD~${{ length(parameters.Artifacts) }} + displayName: Reset commits to enable pushing a single commit + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(PrBranchName) + CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} + TargetRepoOwner: azure-sdk + TargetRepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts # - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: From 208c4a1b738f9e34b8b11556e0c21d59a8c9c9c9 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 22:59:44 -0700 Subject: [PATCH 12/50] Download pipeline artifacts --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index f23b9b7397..7cc0321f19 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -22,6 +22,10 @@ stages: steps: - checkout: self + - task: DownloadPipelineArtifact@2 + inputs: + artifact: packages + - pwsh: | $formattedDate = Get-Date -Format 'yyyy-MM-dd' $branchName = "nightly/$formattedDate" From 256eaace32081c48c5905534f46de1524a81a3ec Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 23:01:24 -0700 Subject: [PATCH 13/50] task -> download --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 7cc0321f19..52c91d9ba5 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -22,9 +22,8 @@ stages: steps: - checkout: self - - task: DownloadPipelineArtifact@2 - inputs: - artifact: packages + - download: current + artifact: packages - pwsh: | $formattedDate = Get-Date -Format 'yyyy-MM-dd' From e474192bace57181b7e77d5e41bc7b8fb154ba9b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 23:27:16 -0700 Subject: [PATCH 14/50] Set appropriate working directory --- eng/pipelines/templates/steps/vcpkg-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml index dd674685c8..63117b59b8 100644 --- a/eng/pipelines/templates/steps/vcpkg-publish.yml +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -1,6 +1,5 @@ parameters: GitHubRepo: $(Build.Repository.Name) - ArtifactPath: $() Workspace: $(Pipeline.Workspace) ArtifactName: not-set VcpkgPortName: not-set @@ -32,5 +31,5 @@ steps: -VcpkgPortName '${{ parameters.VcpkgPortName }}' -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' -TestRelease:$${{ parameters.TestRelease }} - workingDirectory: $(VcpkgWorkspace) + workingDirectory: ${{ parameters.Workspace }}/vcpkg displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file From 229c6f3b3aea1709d27fcdc80cc74af60c90e38d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 23:38:53 -0700 Subject: [PATCH 15/50] git status --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 52c91d9ba5..250ef89376 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -52,7 +52,10 @@ stages: - pwsh: | Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" - git reset --soft HEAD~${{ length(parameters.Artifacts) }} + git reset --soft HEAD^${{ length(parameters.Artifacts) }} + + Write-Host "git status" + git status displayName: Reset commits to enable pushing a single commit - template: /eng/common/pipelines/templates/steps/git-push-changes.yml From b70c239fc299d7a9d2f22c8a91f918db3b0fe76e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 30 Mar 2021 23:39:40 -0700 Subject: [PATCH 16/50] Set working directory --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 250ef89376..33e1fa8b1b 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -57,6 +57,7 @@ stages: Write-Host "git status" git status displayName: Reset commits to enable pushing a single commit + workingDirectory: $(Pipeline.Workspace)/vcpkg - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: From e88e5f6cd92062dad663fe7f25bcdd1e949b1131 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 31 Mar 2021 23:10:45 -0700 Subject: [PATCH 17/50] Add pipeline for nightly vcpkg PR update --- eng/pipelines/cpp-nightly.yml | 58 +++++++++++++++++++++++ eng/scripts/Update-VcpkgNightlyBranch.ps1 | 54 +++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 eng/pipelines/cpp-nightly.yml create mode 100644 eng/scripts/Update-VcpkgNightlyBranch.ps1 diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml new file mode 100644 index 0000000000..491e526a81 --- /dev/null +++ b/eng/pipelines/cpp-nightly.yml @@ -0,0 +1,58 @@ + +jobs: + - job: + displayName: Nightly vcpkg PR + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + variables: + # The branch in the azure-sdk/vcpkg repo that is updated nigthly with the + # latest set of ports + TargetPrBranch: azure-sdk-for-cpp-nightly-verification + + steps: + - checkout: self + + - pwsh: | + $formattedDate = Get-Date -Format 'yyyy-MM-dd' + $branchName = "nightly/$formattedDate" + Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" + displayName: Set branch name in fork repo + + - template: eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + Workspace: $(Pipeline.Workspace) + RepoOwner: azure-sdk + PRBranchName: $(TargetPrBranch) + + - template: /eng/common/pipelines/templates/steps/set-default-branch.yml + parameters: + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + + - task: Powershell@2 + inputs: + filePath: eng/scripts/Update-VcpkgNightlyBranch.ps1 + arguments: >- + -SourceBranch $(NightlyBranchName) + -TargetBranch $(TargetPrBranch) + -MainBranchName $(DefaultBranch) + pwsh: true + workingDirectory: $(Pipeline.Workspace)/vcpkg + displayName: Update vcpkg Nightly Branch + + - pwsh: "##vso[task.setvariable variable=HasChanges]$true" + displayName: Set $(HasChanges) to $true + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + BaseBranchName: $(DefaultBranch) + PRBranchName: $(TargetPrBranch) + RepoOwner: Microsoft + RepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + PRTitle: "[DO NOT MERGE] Nightly Azure SDK for C++ CI validation" + PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI. + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts + SkipCheckingForChanges: true + OpenAsDraft: true diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 new file mode 100644 index 0000000000..355e4357fb --- /dev/null +++ b/eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -0,0 +1,54 @@ +param( + [string] $SourceBranch, + [string] $TargetBranch, + [string] $MainBranchName +) + +$ErrorActionPreference = "Continue" + +if ((Get-Command git | Measure-Object).Count -eq 0) { + LogError "Could not locate git. Install git https://git-scm.com/downloads" + exit 1 +} + +Write-Host "git fetch --all" +git fetch --all + +# Fail quickly if the $SourceBranch of nightly changes cannot be found +Write-Host "git checkout $SourceBranch" +git checkout $SourceBranch + +if ($LASTEXITCODE -ne 0) { + Write-Error "Cannot find branch: $SourceBranch, nothing to merge" + exit 1 +} + + +Write-Host "git branch --show-current" +$currentBranch = git branch --show-current + +if ($currentBranch -ne $TargetBranch) { + Write-Host "git checkout $TargetBranch" + git checkout $TargetBranch + + if ($LASTEXITCODE -ne 0) { + Write-Host "git checkout -b $TargetBranch" + git checkout -b $TargetBranch + } +} + +Write-Host "git merge $MainBranchName --strategy-option=theirs" +git merge $MainBranchName --strategy-option=theirs + +if ($LASTEXITCODE -ne 0) { + Write-Error "Could not merge $MainBranchName into $TargetBranch" + exit 1 +} + +Write-Host "git merge $SourceBranch --strategy-option=theirs" +git merge $SourceBranch --strategy-option=theirs + +if ($LASTEXITCODE -ne 0) { + Write-Error "Could not merge $SourceBranch into $TargetBranch" + exit 1 +} From a16df9fe37d872e3b9d5d12a2763abfafe96df24 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Wed, 31 Mar 2021 23:15:59 -0700 Subject: [PATCH 18/50] Correct vcpkg-clone.yml path --- eng/pipelines/cpp-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 491e526a81..dcccc7dee7 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -20,7 +20,7 @@ jobs: Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" displayName: Set branch name in fork repo - - template: eng/pipelines/templates/steps/vcpkg-clone.yml + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: Workspace: $(Pipeline.Workspace) RepoOwner: azure-sdk From 78bf4e3810c12d264f462ab8dcdb1b22b17f4c97 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 08:54:33 -0700 Subject: [PATCH 19/50] Add identity parameters to git merge commands --- eng/pipelines/cpp-nightly.yml | 1 + eng/scripts/Update-VcpkgNightlyBranch.ps1 | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index dcccc7dee7..cf34fa84fe 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -37,6 +37,7 @@ jobs: -SourceBranch $(NightlyBranchName) -TargetBranch $(TargetPrBranch) -MainBranchName $(DefaultBranch) + -GitIdentityParameters -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" pwsh: true workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 index 355e4357fb..0b2df68677 100644 --- a/eng/scripts/Update-VcpkgNightlyBranch.ps1 +++ b/eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -1,7 +1,8 @@ param( [string] $SourceBranch, [string] $TargetBranch, - [string] $MainBranchName + [string] $MainBranchName, + [string] $GitIdentityParameters = '' ) $ErrorActionPreference = "Continue" @@ -37,16 +38,16 @@ if ($currentBranch -ne $TargetBranch) { } } -Write-Host "git merge $MainBranchName --strategy-option=theirs" -git merge $MainBranchName --strategy-option=theirs +Write-Host "git merge $GitIdentityParameters $MainBranchName --strategy-option=theirs" +git merge $GitIdentityParameters $MainBranchName --strategy-option=theirs if ($LASTEXITCODE -ne 0) { Write-Error "Could not merge $MainBranchName into $TargetBranch" exit 1 } -Write-Host "git merge $SourceBranch --strategy-option=theirs" -git merge $SourceBranch --strategy-option=theirs +Write-Host "git merge $GitIdentityParameters $SourceBranch --strategy-option=theirs" +git merge $GitIdentityParameters $SourceBranch --strategy-option=theirs if ($LASTEXITCODE -ne 0) { Write-Error "Could not merge $SourceBranch into $TargetBranch" From a7973e762502c6ea19706f60705bb578c7702530 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:00:16 -0700 Subject: [PATCH 20/50] Remove GitIdentityParameters, the merge command does not suppor tthem. Add instead in the pipeline --- eng/pipelines/cpp-nightly.yml | 4 +++- eng/scripts/Update-VcpkgNightlyBranch.ps1 | 11 +++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index cf34fa84fe..593c143caa 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -30,6 +30,9 @@ jobs: parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg + - pwsh: git config --global user.name "azure-sdk" user.email "azuresdk@microsoft.com" + displayName: Configure git identity + - task: Powershell@2 inputs: filePath: eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -37,7 +40,6 @@ jobs: -SourceBranch $(NightlyBranchName) -TargetBranch $(TargetPrBranch) -MainBranchName $(DefaultBranch) - -GitIdentityParameters -c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com" pwsh: true workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 index 0b2df68677..355e4357fb 100644 --- a/eng/scripts/Update-VcpkgNightlyBranch.ps1 +++ b/eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -1,8 +1,7 @@ param( [string] $SourceBranch, [string] $TargetBranch, - [string] $MainBranchName, - [string] $GitIdentityParameters = '' + [string] $MainBranchName ) $ErrorActionPreference = "Continue" @@ -38,16 +37,16 @@ if ($currentBranch -ne $TargetBranch) { } } -Write-Host "git merge $GitIdentityParameters $MainBranchName --strategy-option=theirs" -git merge $GitIdentityParameters $MainBranchName --strategy-option=theirs +Write-Host "git merge $MainBranchName --strategy-option=theirs" +git merge $MainBranchName --strategy-option=theirs if ($LASTEXITCODE -ne 0) { Write-Error "Could not merge $MainBranchName into $TargetBranch" exit 1 } -Write-Host "git merge $GitIdentityParameters $SourceBranch --strategy-option=theirs" -git merge $GitIdentityParameters $SourceBranch --strategy-option=theirs +Write-Host "git merge $SourceBranch --strategy-option=theirs" +git merge $SourceBranch --strategy-option=theirs if ($LASTEXITCODE -ne 0) { Write-Error "Could not merge $SourceBranch into $TargetBranch" From dd8a06780cfc367158465f619359284e871f4714 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:07:31 -0700 Subject: [PATCH 21/50] Split lines --- eng/pipelines/cpp-nightly.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 593c143caa..3d05b633d6 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -30,7 +30,9 @@ jobs: parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - - pwsh: git config --global user.name "azure-sdk" user.email "azuresdk@microsoft.com" + - pwsh: | + git config --global user.name "azure-sdk" + git config --global user.email "azuresdk@microsoft.com" displayName: Configure git identity - task: Powershell@2 From 89951ea7bc6b89a464cc3a362eca612f6d84fb30 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:12:28 -0700 Subject: [PATCH 22/50] Write-Host --- eng/pipelines/cpp-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 3d05b633d6..48c9e66f0d 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -46,7 +46,7 @@ jobs: workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch - - pwsh: "##vso[task.setvariable variable=HasChanges]$true" + - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" displayName: Set $(HasChanges) to $true - template: /eng/common/pipelines/templates/steps/create-pull-request.yml From 899b9ca1be74ab6c98cb2092a0a63740ddb89377 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:36:46 -0700 Subject: [PATCH 23/50] Template for nightly branch name, update comments, Check for scheduling or "PublishNightlyVcpkg" --- eng/pipelines/cpp-nightly.yml | 11 ++++++----- .../templates/steps/generate-nightly-branch-name.yml | 10 ++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 eng/pipelines/templates/steps/generate-nightly-branch-name.yml diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 48c9e66f0d..8071b6662b 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -14,11 +14,7 @@ jobs: steps: - checkout: self - - pwsh: | - $formattedDate = Get-Date -Format 'yyyy-MM-dd' - $branchName = "nightly/$formattedDate" - Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" - displayName: Set branch name in fork repo + - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: @@ -30,6 +26,8 @@ jobs: parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg + # Git merge does not support the -c parameter. For the scope of this job + # it's ok to configure the user.name and user.email at a global level - pwsh: | git config --global user.name "azure-sdk" git config --global user.email "azuresdk@microsoft.com" @@ -46,9 +44,12 @@ jobs: workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch + # Set $(HasChanges) to $true so create-pull-request.yml will push the + # branch. - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" displayName: Set $(HasChanges) to $true + # This step will skip creating the PR if the PR already exists - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: BaseBranchName: $(DefaultBranch) diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml new file mode 100644 index 0000000000..ee14bd4647 --- /dev/null +++ b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml @@ -0,0 +1,10 @@ +steps: + - pwsh: | + $branchName = if ($env:NIGHTLYBRANCHOVERRIDE) { + $env:NIGHTLYBRANCHOVERRIDE + } else { + $formattedDate = Get-Date -Format 'yyyy-MM-dd' + "nightly/$formattedDate" + } + Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" + displayName: Set branch name in fork repo \ No newline at end of file From b32309c07df66e20badff55ed9cf194bbdf1e766 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:36:59 -0700 Subject: [PATCH 24/50] include archetype-cpp-release.yml changes --- .../templates/stages/archetype-cpp-release.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 33e1fa8b1b..84149809c5 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -5,15 +5,13 @@ parameters: TestPipeline: false stages: - # TODO: Check for scheduled run - - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - ${{if and(eq(variables['System.TeamProject'], 'internal'), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) }}: - stage: Integration dependsOn: Build jobs: - job: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch - # TODO: Condition? pool: name: azsdk-pool-mms-win-2019-general @@ -21,21 +19,15 @@ stages: steps: - checkout: self - - download: current artifact: packages - - pwsh: | - $formattedDate = Get-Date -Format 'yyyy-MM-dd' - $branchName = "nightly/$formattedDate" - Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" - displayName: Set branch name in fork repo + - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml - # TODO: Can we do a sparse checkout? - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: RepoOwner: azure-sdk - PRBranchName: $(PrBranchName) + PRBranchName: $(NightlyBranchName) # Placing artifacts in the same branch in sequence reduces the # potential for race conditions. If this job does fail to push @@ -61,7 +53,7 @@ stages: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: - BaseRepoBranch: $(PrBranchName) + BaseRepoBranch: $(NightlyBranchName) CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} TargetRepoOwner: azure-sdk TargetRepoName: vcpkg From 70e7de2abc8f3f7340ead26282d35de8599c28a8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:37:15 -0700 Subject: [PATCH 25/50] Re-enable publishing stage --- .../stages/archetype-cpp-release.yml | 348 +++++++++--------- 1 file changed, 174 insertions(+), 174 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 84149809c5..344d206bd7 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -61,177 +61,177 @@ stages: ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts - # - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - # - ${{ each artifact in parameters.Artifacts }}: - # - stage: Release_${{ replace(artifact.Name, '-', '_') }} - # displayName: 'Release ${{artifact.name}}' - # dependsOn: Build - # condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) - - # jobs: - # - deployment: TagRepository - # displayName: "Create release tag" - # condition: ne(variables['Skip.TagRepository'], 'true') - # environment: github - - # pool: - # name: azsdk-pool-mms-win-2019-general - # vmImage: MMS2019 - - # strategy: - # runOnce: - # deploy: - # steps: - # - checkout: self - # - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - # parameters: - # PackageName: ${{ artifact.Name }} - # ServiceDirectory: ${{ parameters.ServiceDirectory }} - # TestPipeline: ${{ parameters.TestPipeline }} - # - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}} - # displayName: Output Visible Artifacts - # - task: Powershell@2 - # inputs: - # filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLog.ps1 - # arguments: > - # -ChangeLogLocation $(Pipeline.Workspace)/packages/${{artifact.Name}}/CHANGELOG.md - # -VersionString (Get-Content $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json | ConvertFrom-Json).version - # -ForRelease $true - # pwsh: true - # displayName: Verify CHANGELOG.md contents - # - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - # parameters: - # ArtifactLocation: "$(Pipeline.Workspace)/packages/${{artifact.Name}}" - # PackageRepository: CPP - # ReleaseSha: $(Build.SourceVersion) - # RepoId: Azure/azure-sdk-for-cpp - - # - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - # - deployment: PublishDocs - # displayName: Publish Docs to GitHub pages - # condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - # environment: githubio - # # Release tag must exist for link substitution to work - # dependsOn: TagRepository - - # pool: - # name: azsdk-pool-mms-win-2019-general - # vmImage: MMS2019 - - # strategy: - # runOnce: - # deploy: - # steps: - # - checkout: self - # - pwsh: Write-Host 'publish docs' - # - pwsh: | - # Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }} - # displayName: Output Visible Artifacts - # - template: ../../../common/pipelines/templates/steps/publish-blobs.yml - # parameters: - # FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' - # BlobSASKey: '$(azure-sdk-docs-prod-sas)' - # BlobName: '$(azure-sdk-docs-prod-blob-name)' - # TargetLanguage: 'cpp' - # ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}' - # # we override the regular script path because we have cloned the build tools repo as a separate artifact. - # ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1' - - # - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: - # - deployment: PublishVcpkg - # displayName: Publish to vcpkg - # condition: and(succeeded(), ne(variables['Skip.PublishVcpkg'], 'true')) - # environment: github - # # This step requires the tag step - # dependsOn: TagRepository - # pool: - # name: azsdk-pool-mms-win-2019-general - # vmImage: MMS2019 - - # strategy: - # runOnce: - # deploy: - # steps: - # - checkout: self - - # - pwsh: | - # $branchName = "azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId)" - # if ('$(VcpkgForkBranchName)') { - # Write-Host "Using queue time branch name" - # $branchName = '$(VcpkgForkBranchName)' - # } - # Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" - # displayName: Set fork branch name - - # - pwsh: | - # $packageSpec = Get-Content -Raw -Path "$(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json" | ConvertFrom-Json - # $version = $packageSpec.version - # Write-Host "##vso[task.setvariable variable=PackageVersion]$version" - # displayName: Set PackageVersion variable - - # - pwsh: | - # $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)" - - # if ('$(VcpkgPRTitle)') { - # Write-Host "Using queue time PR title" - # $title = '$(VcpkgPRTitle)' - # } - # Write-Host "##vso[task.setvariable variable=PrTitle]$title" - # displayName: Set PR title - - # - template: /eng/pipelines/templates/steps/vcpkg-clone.yml - # parameters: - # RepoOwner: azure-sdk - # PRBranchName: $(PrBranchName) - - # - template: /eng/pipelines/templates/steps/vcpkg-publish.yml - # parameters: - # ArtifactName: ${{ artifact.Name }} - # VcpkgPortName: ${{ artifact.VcpkgPortName }} - - # # Set $(HasChanges) to $true so that - # # create-pull-request.yml completes the push and PR - # # submission steps - # - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" - # displayName: Set $(HasChanges) to $true for create-pull-request.yml - - # # SkipCheckingForChanges is true to skip the commit step - # # (which is already done by Update-VcpkgPort.ps1) - # - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - # parameters: - # RepoOwner: Microsoft - # RepoName: vcpkg - # WorkingDirectory: $(Pipeline.Workspace)/vcpkg - # PrBranchName: $(PrBranchName) - # PRTitle: $(PrTitle) - # PRBody: Update vcpkg ports for Azure SDK release. This release may contain multiple ports. - # SkipCheckingForChanges: true - # BaseBranchName: master - # OpenAsDraft: ${{ parameters.TestPipeline }} - # CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - - # - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: - # - deployment: UpdatePackageVersion - # displayName: "Update Package Version" - # condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - # environment: github - - # pool: - # name: azsdk-pool-mms-win-2019-general - # vmImage: MMS2019 - - # strategy: - # runOnce: - # deploy: - # steps: - # - checkout: self - # - pwsh: | - # eng/scripts/Update-PkgVersion.ps1 -ServiceDirectory '${{parameters.ServiceDirectory}}' -PackageName '${{artifact.name}}' - # displayName: Increment package version - # - template: ../../../common/pipelines/templates/steps/create-pull-request.yml - # parameters: - # RepoName: azure-sdk-for-cpp - # PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - # CommitMsg: "Increment package version after release of ${{ artifact.name }}" - # PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - # CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file + - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: + - ${{ each artifact in parameters.Artifacts }}: + - stage: Release_${{ replace(artifact.Name, '-', '_') }} + displayName: 'Release ${{artifact.name}}' + dependsOn: Build + condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) + + jobs: + - deployment: TagRepository + displayName: "Create release tag" + condition: ne(variables['Skip.TagRepository'], 'true') + environment: github + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: ${{ artifact.Name }} + ServiceDirectory: ${{ parameters.ServiceDirectory }} + TestPipeline: ${{ parameters.TestPipeline }} + - pwsh: Get-ChildItem -Recurse $(Pipeline.Workspace)/packages/${{artifact.Name}} + displayName: Output Visible Artifacts + - task: Powershell@2 + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/scripts/Verify-ChangeLog.ps1 + arguments: > + -ChangeLogLocation $(Pipeline.Workspace)/packages/${{artifact.Name}}/CHANGELOG.md + -VersionString (Get-Content $(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json | ConvertFrom-Json).version + -ForRelease $true + pwsh: true + displayName: Verify CHANGELOG.md contents + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: "$(Pipeline.Workspace)/packages/${{artifact.Name}}" + PackageRepository: CPP + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-cpp + + - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: + - deployment: PublishDocs + displayName: Publish Docs to GitHub pages + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + environment: githubio + # Release tag must exist for link substitution to work + dependsOn: TagRepository + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + - pwsh: Write-Host 'publish docs' + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/docs/${{ artifact.Name }} + displayName: Output Visible Artifacts + - template: ../../../common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}' + BlobSASKey: '$(azure-sdk-docs-prod-sas)' + BlobName: '$(azure-sdk-docs-prod-blob-name)' + TargetLanguage: 'cpp' + ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}' + # we override the regular script path because we have cloned the build tools repo as a separate artifact. + ScriptPath: '$(Build.SourcesDirectory)/eng/common/scripts/copy-docs-to-blobstorage.ps1' + + - ${{ if ne(artifact.skipPublishVcpkg, 'true') }}: + - deployment: PublishVcpkg + displayName: Publish to vcpkg + condition: and(succeeded(), ne(variables['Skip.PublishVcpkg'], 'true')) + environment: github + # This step requires the tag step + dependsOn: TagRepository + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + + - pwsh: | + $branchName = "azure-sdk-for-cpp-${{ artifact.Name }}-$(Build.BuildId)" + if ('$(VcpkgForkBranchName)') { + Write-Host "Using queue time branch name" + $branchName = '$(VcpkgForkBranchName)' + } + Write-Host "##vso[task.setvariable variable=PrBranchName]$branchName" + displayName: Set fork branch name + + - pwsh: | + $packageSpec = Get-Content -Raw -Path "$(Pipeline.Workspace)/packages/${{artifact.Name}}/package-info.json" | ConvertFrom-Json + $version = $packageSpec.version + Write-Host "##vso[task.setvariable variable=PackageVersion]$version" + displayName: Set PackageVersion variable + + - pwsh: | + $title = "[${{ artifact.VcpkgPortName }}] publish version $(PackageVersion)" + + if ('$(VcpkgPRTitle)') { + Write-Host "Using queue time PR title" + $title = '$(VcpkgPRTitle)' + } + Write-Host "##vso[task.setvariable variable=PrTitle]$title" + displayName: Set PR title + + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(PrBranchName) + + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + + # Set $(HasChanges) to $true so that + # create-pull-request.yml completes the push and PR + # submission steps + - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" + displayName: Set $(HasChanges) to $true for create-pull-request.yml + + # SkipCheckingForChanges is true to skip the commit step + # (which is already done by Update-VcpkgPort.ps1) + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoOwner: Microsoft + RepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + PrBranchName: $(PrBranchName) + PRTitle: $(PrTitle) + PRBody: Update vcpkg ports for Azure SDK release. This release may contain multiple ports. + SkipCheckingForChanges: true + BaseBranchName: master + OpenAsDraft: ${{ parameters.TestPipeline }} + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + + - ${{if ne(artifact.skipUpdatePackageVersion, 'true')}}: + - deployment: UpdatePackageVersion + displayName: "Update Package Version" + condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) + environment: github + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + strategy: + runOnce: + deploy: + steps: + - checkout: self + - pwsh: | + eng/scripts/Update-PkgVersion.ps1 -ServiceDirectory '${{parameters.ServiceDirectory}}' -PackageName '${{artifact.name}}' + displayName: Increment package version + - template: ../../../common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-cpp + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ artifact.name }}" + PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file From d7151114dec7f69f30d17e3e76ed22abd2ac08b5 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:40:37 -0700 Subject: [PATCH 26/50] Close the and --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 344d206bd7..1aea1e1ad9 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -5,7 +5,7 @@ parameters: TestPipeline: false stages: - - ${{if and(eq(variables['System.TeamProject'], 'internal'), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) }}: + - ${{if and(eq(variables['System.TeamProject'], 'internal'), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) }}: - stage: Integration dependsOn: Build From aea9d4a8226d958616b1a639232753ba488841b0 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:56:57 -0700 Subject: [PATCH 27/50] Runtime condition for integration stage --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 1aea1e1ad9..6d730831ff 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -5,10 +5,10 @@ parameters: TestPipeline: false stages: - - ${{if and(eq(variables['System.TeamProject'], 'internal'), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) }}: + - ${{if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration dependsOn: Build - + condition: or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) jobs: - job: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch From e13eff517448d1a88d4d871a0fb47e0999b1a0f8 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 09:59:14 -0700 Subject: [PATCH 28/50] Move integration below package publishing stages --- .../stages/archetype-cpp-release.yml | 113 +++++++++--------- 1 file changed, 56 insertions(+), 57 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 6d730831ff..7a514065c7 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -5,62 +5,6 @@ parameters: TestPipeline: false stages: - - ${{if eq(variables['System.TeamProject'], 'internal') }}: - - stage: Integration - dependsOn: Build - condition: or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) - jobs: - - job: PublishNightlyVcpkg - displayName: Publish to vcpkg nightly branch - - pool: - name: azsdk-pool-mms-win-2019-general - vmImage: MMS2019 - - steps: - - checkout: self - - download: current - artifact: packages - - - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml - - - template: /eng/pipelines/templates/steps/vcpkg-clone.yml - parameters: - RepoOwner: azure-sdk - PRBranchName: $(NightlyBranchName) - - # Placing artifacts in the same branch in sequence reduces the - # potential for race conditions. If this job does fail to push - # because another job has already pushed to the same branch it - # can be re-run. - - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/pipelines/templates/steps/vcpkg-publish.yml - parameters: - ArtifactName: ${{ artifact.Name }} - VcpkgPortName: ${{ artifact.VcpkgPortName }} - DisplayNameExtension: ${{ artifact.Name }} - TestRelease: true - TestReleaseRef: $(Build.SourceVersion) - - - pwsh: | - Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" - git reset --soft HEAD^${{ length(parameters.Artifacts) }} - - Write-Host "git status" - git status - displayName: Reset commits to enable pushing a single commit - workingDirectory: $(Pipeline.Workspace)/vcpkg - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(NightlyBranchName) - CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} - TargetRepoOwner: azure-sdk - TargetRepoName: vcpkg - WorkingDirectory: $(Pipeline.Workspace)/vcpkg - ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts - - - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - ${{ each artifact in parameters.Artifacts }}: - stage: Release_${{ replace(artifact.Name, '-', '_') }} @@ -234,4 +178,59 @@ stages: PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) CommitMsg: "Increment package version after release of ${{ artifact.name }}" PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' \ No newline at end of file + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + + - ${{if eq(variables['System.TeamProject'], 'internal') }}: + - stage: Integration + dependsOn: Build + condition: or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) + jobs: + - job: PublishNightlyVcpkg + displayName: Publish to vcpkg nightly branch + + pool: + name: azsdk-pool-mms-win-2019-general + vmImage: MMS2019 + + steps: + - checkout: self + - download: current + artifact: packages + + - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml + + - template: /eng/pipelines/templates/steps/vcpkg-clone.yml + parameters: + RepoOwner: azure-sdk + PRBranchName: $(NightlyBranchName) + + # Placing artifacts in the same branch in sequence reduces the + # potential for race conditions. If this job does fail to push + # because another job has already pushed to the same branch it + # can be re-run. + - ${{ each artifact in parameters.Artifacts }}: + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + DisplayNameExtension: ${{ artifact.Name }} + TestRelease: true + TestReleaseRef: $(Build.SourceVersion) + + - pwsh: | + Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" + git reset --soft HEAD^${{ length(parameters.Artifacts) }} + + Write-Host "git status" + git status + displayName: Reset commits to enable pushing a single commit + workingDirectory: $(Pipeline.Workspace)/vcpkg + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(NightlyBranchName) + CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} + TargetRepoOwner: azure-sdk + TargetRepoName: vcpkg + WorkingDirectory: $(Pipeline.Workspace)/vcpkg + ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts From bc9228082029cff9873aa4078394497bdf2fa899 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 13:19:41 -0700 Subject: [PATCH 29/50] try -- to disambiguate --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 7a514065c7..07ce022875 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -218,8 +218,8 @@ stages: TestReleaseRef: $(Build.SourceVersion) - pwsh: | - Write-Host "git reset --soft HEAD^${{ length(parameters.Artifacts) }}" - git reset --soft HEAD^${{ length(parameters.Artifacts) }} + Write-Host "git reset --soft -- HEAD^${{ length(parameters.Artifacts) }}" + git reset --soft -- HEAD^${{ length(parameters.Artifacts) }} Write-Host "git status" git status From 20eca311967404e9713dfb65fb1afd1d10414951 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 13:35:59 -0700 Subject: [PATCH 30/50] ^ -> ~ --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 07ce022875..eac1baf9ae 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -218,8 +218,8 @@ stages: TestReleaseRef: $(Build.SourceVersion) - pwsh: | - Write-Host "git reset --soft -- HEAD^${{ length(parameters.Artifacts) }}" - git reset --soft -- HEAD^${{ length(parameters.Artifacts) }} + Write-Host "git reset --soft -- HEAD~${{ length(parameters.Artifacts) }}" + git reset --soft -- HEAD~${{ length(parameters.Artifacts) }} Write-Host "git status" git status From b22b38dd5ec86fd4a6a11644193d9f035247f1d3 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 13:50:27 -0700 Subject: [PATCH 31/50] Remove -- --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index eac1baf9ae..7a913dccac 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -218,8 +218,8 @@ stages: TestReleaseRef: $(Build.SourceVersion) - pwsh: | - Write-Host "git reset --soft -- HEAD~${{ length(parameters.Artifacts) }}" - git reset --soft -- HEAD~${{ length(parameters.Artifacts) }} + Write-Host "git reset --soft HEAD~${{ length(parameters.Artifacts) }}" + git reset --soft HEAD~${{ length(parameters.Artifacts) }} Write-Host "git status" git status From 9ded0b195893b3f88a8f5e50a07a792d05a6c9b7 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 14:48:27 -0700 Subject: [PATCH 32/50] Documentation and variable naming --- eng/pipelines/cpp-nightly.yml | 4 +- eng/scripts/Update-VcpkgNightlyBranch.ps1 | 51 ++++++++++++++++------- 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 8071b6662b..5469af7f0d 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -37,8 +37,8 @@ jobs: inputs: filePath: eng/scripts/Update-VcpkgNightlyBranch.ps1 arguments: >- - -SourceBranch $(NightlyBranchName) - -TargetBranch $(TargetPrBranch) + -SourceBranchName $(NightlyBranchName) + -TargetBranchName $(TargetPrBranch) -MainBranchName $(DefaultBranch) pwsh: true workingDirectory: $(Pipeline.Workspace)/vcpkg diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 index 355e4357fb..366dd5f9ed 100644 --- a/eng/scripts/Update-VcpkgNightlyBranch.ps1 +++ b/eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -1,6 +1,27 @@ +<# +.SYNOPSIS +Updates a Target branch with changes from a Main branch and a Source branch + +.DESCRIPTION +Updat the long-lived nightly branch with changes from Main and a given Source +branch. Changes are first merged from the Main branch and then from the Source +branch. In both cases conflicts are resolved in favor of the branch which is +being merged. + +.PARAMETER SourceBranchName +Name of the branch with new changes to merge into the Target Branch + +.PARAMETER TargetBranchName +Name of the long-lived branch receiving the changes + +.PARAMETER MainBranchName +Name of the "Main" branch to merge into the Target Branch + +#> + param( - [string] $SourceBranch, - [string] $TargetBranch, + [string] $SourceBranchName, + [string] $TargetBranchName, [string] $MainBranchName ) @@ -14,12 +35,12 @@ if ((Get-Command git | Measure-Object).Count -eq 0) { Write-Host "git fetch --all" git fetch --all -# Fail quickly if the $SourceBranch of nightly changes cannot be found -Write-Host "git checkout $SourceBranch" -git checkout $SourceBranch +# Fail quickly if the $SourceBranchName of nightly changes cannot be found +Write-Host "git checkout $SourceBranchName" +git checkout $SourceBranchName if ($LASTEXITCODE -ne 0) { - Write-Error "Cannot find branch: $SourceBranch, nothing to merge" + Write-Error "Cannot find branch: $SourceBranchName, nothing to merge" exit 1 } @@ -27,13 +48,13 @@ if ($LASTEXITCODE -ne 0) { Write-Host "git branch --show-current" $currentBranch = git branch --show-current -if ($currentBranch -ne $TargetBranch) { - Write-Host "git checkout $TargetBranch" - git checkout $TargetBranch +if ($currentBranch -ne $TargetBranchName) { + Write-Host "git checkout $TargetBranchName" + git checkout $TargetBranchName if ($LASTEXITCODE -ne 0) { - Write-Host "git checkout -b $TargetBranch" - git checkout -b $TargetBranch + Write-Host "git checkout -b $TargetBranchName" + git checkout -b $TargetBranchName } } @@ -41,14 +62,14 @@ Write-Host "git merge $MainBranchName --strategy-option=theirs" git merge $MainBranchName --strategy-option=theirs if ($LASTEXITCODE -ne 0) { - Write-Error "Could not merge $MainBranchName into $TargetBranch" + Write-Error "Could not merge $MainBranchName into $TargetBranchName" exit 1 } -Write-Host "git merge $SourceBranch --strategy-option=theirs" -git merge $SourceBranch --strategy-option=theirs +Write-Host "git merge $SourceBranchName --strategy-option=theirs" +git merge $SourceBranchName --strategy-option=theirs if ($LASTEXITCODE -ne 0) { - Write-Error "Could not merge $SourceBranch into $TargetBranch" + Write-Error "Could not merge $SourceBranchName into $TargetBranchName" exit 1 } From 3876e9e68b3de8a002b8c08dc9102e6b3a402489 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Thu, 1 Apr 2021 14:53:07 -0700 Subject: [PATCH 33/50] Move documentation comment --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 3 +++ eng/pipelines/templates/steps/vcpkg-clone.yml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 7a913dccac..7f34745cf3 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -123,6 +123,9 @@ stages: Write-Host "##vso[task.setvariable variable=PrTitle]$title" displayName: Set PR title + # There are potential race conditions if this script runs + # in parallel against the same branch name. Release only + # one package at a time. - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: RepoOwner: azure-sdk diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index 223b1f7d65..6136766900 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -5,9 +5,6 @@ parameters: PRBranchName: not-set steps: - # There are potential race conditions if this script runs - # in parallel against the same branch name. Release only - # one package at a time. - pwsh: | Write-Host "git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg" git clone https://github.com/${{ parameters.RepoOwner }}/${{ parameters.RepoName }} ${{ parameters.Workspace }}/vcpkg From c735b55b149fee059eb19180bd236a87b18c8e98 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 11:48:21 -0700 Subject: [PATCH 34/50] Update eng/pipelines/templates/steps/generate-nightly-branch-name.yml Co-authored-by: Wes Haggard --- .../templates/steps/generate-nightly-branch-name.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml index ee14bd4647..12103a2753 100644 --- a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml +++ b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml @@ -1,10 +1,9 @@ steps: - pwsh: | - $branchName = if ($env:NIGHTLYBRANCHOVERRIDE) { - $env:NIGHTLYBRANCHOVERRIDE - } else { + $branchName = $env:NIGHTLYBRANCHOVERRIDE + if (!$branchName) { $formattedDate = Get-Date -Format 'yyyy-MM-dd' - "nightly/$formattedDate" + $branchName"nightly/$formattedDate" } Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" - displayName: Set branch name in fork repo \ No newline at end of file + displayName: Set branch name in fork repo From 099d4a92d1ac8820847ea04f7def64ecd0cc41a5 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 15:05:49 -0700 Subject: [PATCH 35/50] Apply suggestions from code review Co-authored-by: Wes Haggard --- eng/scripts/Update-VcpkgNightlyBranch.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 index 366dd5f9ed..8e8a7213c4 100644 --- a/eng/scripts/Update-VcpkgNightlyBranch.ps1 +++ b/eng/scripts/Update-VcpkgNightlyBranch.ps1 @@ -27,7 +27,7 @@ param( $ErrorActionPreference = "Continue" -if ((Get-Command git | Measure-Object).Count -eq 0) { +if (!(Get-Command git-ErrorAction SilentlyContinue)) { LogError "Could not locate git. Install git https://git-scm.com/downloads" exit 1 } From 6333c716227c348f5f80ccd5dca4afa69c265114 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 15:07:15 -0700 Subject: [PATCH 36/50] Review feedback --- eng/pipelines/cpp-nightly.yml | 32 +++----- .../stages/archetype-cpp-release.yml | 21 +++--- eng/pipelines/templates/steps/vcpkg-clone.yml | 19 ++--- eng/scripts/Initialize-VcpkgRelease.ps1 | 11 ++- eng/scripts/Update-VcpkgNightlyBranch.ps1 | 75 ------------------- eng/scripts/Update-VcpkgPort.ps1 | 7 +- 6 files changed, 39 insertions(+), 126 deletions(-) delete mode 100644 eng/scripts/Update-VcpkgNightlyBranch.ps1 diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 5469af7f0d..9209e7681f 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -7,7 +7,7 @@ jobs: vmImage: MMS2019 variables: - # The branch in the azure-sdk/vcpkg repo that is updated nigthly with the + # The branch in the azure-sdk/vcpkg repo that is updated nightly with the # latest set of ports TargetPrBranch: azure-sdk-for-cpp-nightly-verification @@ -20,45 +20,31 @@ jobs: parameters: Workspace: $(Pipeline.Workspace) RepoOwner: azure-sdk - PRBranchName: $(TargetPrBranch) + SkipCheckout: true - template: /eng/common/pipelines/templates/steps/set-default-branch.yml parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - # Git merge does not support the -c parameter. For the scope of this job - # it's ok to configure the user.name and user.email at a global level - pwsh: | - git config --global user.name "azure-sdk" - git config --global user.email "azuresdk@microsoft.com" - displayName: Configure git identity - - - task: Powershell@2 - inputs: - filePath: eng/scripts/Update-VcpkgNightlyBranch.ps1 - arguments: >- - -SourceBranchName $(NightlyBranchName) - -TargetBranchName $(TargetPrBranch) - -MainBranchName $(DefaultBranch) - pwsh: true - workingDirectory: $(Pipeline.Workspace)/vcpkg + Write-Host "git merge $(NightlyBranchName) ---strategy-option=theirs --no-commit" + git merge $(NightlyBranchName) ---strategy-option=theirs --no-commit + workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch - # Set $(HasChanges) to $true so create-pull-request.yml will push the - # branch. - - pwsh: Write-Host "##vso[task.setvariable variable=HasChanges]$true" - displayName: Set $(HasChanges) to $true - # This step will skip creating the PR if the PR already exists - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: + # Force push because Update-VcpkgNightlyBranch.ps1 re-creates the + # nightly branch from the tip of the default branch + PushArgs: --force BaseBranchName: $(DefaultBranch) PRBranchName: $(TargetPrBranch) RepoOwner: Microsoft RepoName: vcpkg WorkingDirectory: $(Pipeline.Workspace)/vcpkg + CommitMsg: Update with changes from $(NightlyBranchName) PRTitle: "[DO NOT MERGE] Nightly Azure SDK for C++ CI validation" PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI. ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts - SkipCheckingForChanges: true OpenAsDraft: true diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 7f34745cf3..4fe0fa4be9 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -186,7 +186,7 @@ stages: - ${{if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration dependsOn: Build - condition: or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule')) + condition: and(succeeded(), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) jobs: - job: PublishNightlyVcpkg displayName: Publish to vcpkg nightly branch @@ -220,14 +220,17 @@ stages: TestRelease: true TestReleaseRef: $(Build.SourceVersion) - - pwsh: | - Write-Host "git reset --soft HEAD~${{ length(parameters.Artifacts) }}" - git reset --soft HEAD~${{ length(parameters.Artifacts) }} - - Write-Host "git status" - git status - displayName: Reset commits to enable pushing a single commit - workingDirectory: $(Pipeline.Workspace)/vcpkg + # Reset commit after vcpkg-publish.yml so git-push-changes.yml + # has changes to commit and push, and all changes go into one + # commit + - pwsh: | + Write-Host "git reset HEAD^" + git reset HEAD^ + + Write-Host "git status" + git status + displayName: Reset commit + workingDirectory: $(Pipeline.Workspace)/vcpkg - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index 6136766900..6ba6c3d636 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -3,6 +3,7 @@ parameters: RepoOwner: not-set RepoName: vcpkg PRBranchName: not-set + SkipCheckout: false steps: - pwsh: | @@ -14,12 +15,12 @@ steps: } displayName: Clone vcpkg from upstream - # Check out the PR branch if it's already in remote. - # Ignore failures. - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout Previous PRBranch if it exists - workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file + - ${{ if ne(parameters.SkipCheckout, 'true') }}: + # Check out the PR branch if it's already in remote. Ignore failures. + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout Previous PRBranch if it exists + workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file diff --git a/eng/scripts/Initialize-VcpkgRelease.ps1 b/eng/scripts/Initialize-VcpkgRelease.ps1 index df83076b9f..46c9edcdb4 100644 --- a/eng/scripts/Initialize-VcpkgRelease.ps1 +++ b/eng/scripts/Initialize-VcpkgRelease.ps1 @@ -42,13 +42,12 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) { exit } -$tarGzUri = if (-not $TestReleaseRef) { - Write-Verbose "Initializing Production Release" - $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json - "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" -} else { +$packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json +$tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" + +if ($TestReleaseRef) { Write-Verbose "Initializing Test Release" - "https://github.com/$GitHubRepo/archive/$TestReleaseRef.tar.gz" + $tarGzUri = "https://github.com/$GitHubRepo/archive/$TestReleaseRef.tar.gz" } Write-Host "Downloading tarball to compute hash from $tarGzUri" diff --git a/eng/scripts/Update-VcpkgNightlyBranch.ps1 b/eng/scripts/Update-VcpkgNightlyBranch.ps1 deleted file mode 100644 index 8e8a7213c4..0000000000 --- a/eng/scripts/Update-VcpkgNightlyBranch.ps1 +++ /dev/null @@ -1,75 +0,0 @@ -<# -.SYNOPSIS -Updates a Target branch with changes from a Main branch and a Source branch - -.DESCRIPTION -Updat the long-lived nightly branch with changes from Main and a given Source -branch. Changes are first merged from the Main branch and then from the Source -branch. In both cases conflicts are resolved in favor of the branch which is -being merged. - -.PARAMETER SourceBranchName -Name of the branch with new changes to merge into the Target Branch - -.PARAMETER TargetBranchName -Name of the long-lived branch receiving the changes - -.PARAMETER MainBranchName -Name of the "Main" branch to merge into the Target Branch - -#> - -param( - [string] $SourceBranchName, - [string] $TargetBranchName, - [string] $MainBranchName -) - -$ErrorActionPreference = "Continue" - -if (!(Get-Command git-ErrorAction SilentlyContinue)) { - LogError "Could not locate git. Install git https://git-scm.com/downloads" - exit 1 -} - -Write-Host "git fetch --all" -git fetch --all - -# Fail quickly if the $SourceBranchName of nightly changes cannot be found -Write-Host "git checkout $SourceBranchName" -git checkout $SourceBranchName - -if ($LASTEXITCODE -ne 0) { - Write-Error "Cannot find branch: $SourceBranchName, nothing to merge" - exit 1 -} - - -Write-Host "git branch --show-current" -$currentBranch = git branch --show-current - -if ($currentBranch -ne $TargetBranchName) { - Write-Host "git checkout $TargetBranchName" - git checkout $TargetBranchName - - if ($LASTEXITCODE -ne 0) { - Write-Host "git checkout -b $TargetBranchName" - git checkout -b $TargetBranchName - } -} - -Write-Host "git merge $MainBranchName --strategy-option=theirs" -git merge $MainBranchName --strategy-option=theirs - -if ($LASTEXITCODE -ne 0) { - Write-Error "Could not merge $MainBranchName into $TargetBranchName" - exit 1 -} - -Write-Host "git merge $SourceBranchName --strategy-option=theirs" -git merge $SourceBranchName --strategy-option=theirs - -if ($LASTEXITCODE -ne 0) { - Write-Error "Could not merge $SourceBranchName into $TargetBranchName" - exit 1 -} diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index b0e660e67d..c62058c053 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -77,10 +77,9 @@ if ($LASTEXITCODE -ne 0) { exit 1 } -$addVersionAdditionalParameters = if ($TestRelease) { - '--overwrite-version' -} else { - '' +$addVersionAdditionalParameters = '' +if ($TestRelease) { + $addVersionAdditionalParameters = '--overwrite-version' } Write-Host "./vcpkg.exe x-add-version $VcpkgPortName $addVersionAdditionalParameters" From ea2cbd3b05e76605785bd40ba7adc9e69d3224ec Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 15:10:23 -0700 Subject: [PATCH 37/50] Add ability to skip all release steps --- eng/pipelines/templates/stages/archetype-cpp-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 4fe0fa4be9..61b72a8bea 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -10,7 +10,7 @@ stages: - stage: Release_${{ replace(artifact.Name, '-', '_') }} displayName: 'Release ${{artifact.name}}' dependsOn: Build - condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp')) + condition: and(succeeded(), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-cpp'), ne(variables['Skip.AllRelease'], 'true')) jobs: - deployment: TagRepository From b5554a225a7e299ac9090c301b9b89d4938d602e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 15:24:51 -0700 Subject: [PATCH 38/50] Equal sign --- eng/pipelines/templates/steps/generate-nightly-branch-name.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml index 12103a2753..9a4f55fb35 100644 --- a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml +++ b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml @@ -3,7 +3,7 @@ steps: $branchName = $env:NIGHTLYBRANCHOVERRIDE if (!$branchName) { $formattedDate = Get-Date -Format 'yyyy-MM-dd' - $branchName"nightly/$formattedDate" + $branchName = "nightly/$formattedDate" } Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" displayName: Set branch name in fork repo From 061331b51852361df17b9d5d6660e92e75e71e72 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 15:45:49 -0700 Subject: [PATCH 39/50] The right number of dashes --- eng/pipelines/cpp-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 9209e7681f..fa8890fb1b 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -28,7 +28,7 @@ jobs: - pwsh: | Write-Host "git merge $(NightlyBranchName) ---strategy-option=theirs --no-commit" - git merge $(NightlyBranchName) ---strategy-option=theirs --no-commit + git merge $(NightlyBranchName) --strategy-option=theirs --no-commit workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch From 7538b60f207b28b9bc72f8796d783b2aea1760ca Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 16:03:26 -0700 Subject: [PATCH 40/50] Checkout before merge --- eng/pipelines/cpp-nightly.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index fa8890fb1b..1be2ec8c4a 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -27,7 +27,13 @@ jobs: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - pwsh: | - Write-Host "git merge $(NightlyBranchName) ---strategy-option=theirs --no-commit" + Write-Host "git checkout $(NightlyBranchName)" + git checkout $(NightlyBranchName) + if ($LASTEXITCODE -ne 0) { + Write-Error "Cannot check out $(NightlyBranchName) LASTEXITCODE=$LASTEXITCODE" + } + + Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit" git merge $(NightlyBranchName) --strategy-option=theirs --no-commit workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch From 26402e2919a4b17143b27c3f0fe1d78841f52143 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 16:07:17 -0700 Subject: [PATCH 41/50] Checkout the default branch --- eng/pipelines/cpp-nightly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 1be2ec8c4a..e39b49434d 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -33,6 +33,8 @@ jobs: Write-Error "Cannot check out $(NightlyBranchName) LASTEXITCODE=$LASTEXITCODE" } + git checkout $(DefaultBranch) + Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit" git merge $(NightlyBranchName) --strategy-option=theirs --no-commit workingDirectory: $(Pipeline.Workspace)/vcpkg From 45208ef46b6ebf16ce41094457cf9249d292336e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 16:13:28 -0700 Subject: [PATCH 42/50] --no-ff --- eng/pipelines/cpp-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index e39b49434d..285818f729 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -35,8 +35,8 @@ jobs: git checkout $(DefaultBranch) - Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit" - git merge $(NightlyBranchName) --strategy-option=theirs --no-commit + Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit --no-ff" + git merge $(NightlyBranchName) --strategy-option=theirs --no-commit --no-ff workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch From b1a117c686658aec111a5e08fa89cfa967771b4d Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Fri, 2 Apr 2021 16:34:50 -0700 Subject: [PATCH 43/50] git reset --- eng/pipelines/cpp-nightly.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index 285818f729..eb24a2fab8 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -27,6 +27,7 @@ jobs: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - pwsh: | + git config user.email "azuresdk@microsoft.com" Write-Host "git checkout $(NightlyBranchName)" git checkout $(NightlyBranchName) if ($LASTEXITCODE -ne 0) { @@ -35,8 +36,11 @@ jobs: git checkout $(DefaultBranch) - Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit --no-ff" - git merge $(NightlyBranchName) --strategy-option=theirs --no-commit --no-ff + Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit" + git merge $(NightlyBranchName) --strategy-option=theirs --no-commit + + Write-Host "git reset HEAD~1" + git reset HEAD~1 workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch From 53c74b7f535434d81b2ed2885caa139356bcc73e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Sun, 4 Apr 2021 21:26:54 -0700 Subject: [PATCH 44/50] Skip publishing for artifacts which are not publishign to vcpkg --- .../stages/archetype-cpp-release.yml | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 61b72a8bea..89ce253e8b 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -212,25 +212,29 @@ stages: # because another job has already pushed to the same branch it # can be re-run. - ${{ each artifact in parameters.Artifacts }}: - - template: /eng/pipelines/templates/steps/vcpkg-publish.yml - parameters: - ArtifactName: ${{ artifact.Name }} - VcpkgPortName: ${{ artifact.VcpkgPortName }} - DisplayNameExtension: ${{ artifact.Name }} - TestRelease: true - TestReleaseRef: $(Build.SourceVersion) - - # Reset commit after vcpkg-publish.yml so git-push-changes.yml - # has changes to commit and push, and all changes go into one - # commit - - pwsh: | - Write-Host "git reset HEAD^" - git reset HEAD^ - - Write-Host "git status" - git status - displayName: Reset commit - workingDirectory: $(Pipeline.Workspace)/vcpkg + # Only run vcpkg publish if the artifact has a "VcpkgPortName" + # property. Absence of VcpkgPortName implies that the artifact + # does not ship to vcpkg. + - ${{ if ne(artifact.VcpkgPortName, '') }}: + - template: /eng/pipelines/templates/steps/vcpkg-publish.yml + parameters: + ArtifactName: ${{ artifact.Name }} + VcpkgPortName: ${{ artifact.VcpkgPortName }} + DisplayNameExtension: ${{ artifact.Name }} + TestRelease: true + TestReleaseRef: $(Build.SourceVersion) + + # Reset commit after vcpkg-publish.yml so git-push-changes.yml + # has changes to commit and push, and all changes go into one + # commit + - pwsh: | + Write-Host "git reset HEAD^" + git reset HEAD^ + + Write-Host "git status" + git status + displayName: Reset commit + workingDirectory: $(Pipeline.Workspace)/vcpkg - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: From 5423d0aaa59db2a4b1d801fcf20b7839b04b208e Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Mon, 5 Apr 2021 10:45:19 -0700 Subject: [PATCH 45/50] Add git config --unset --- eng/pipelines/cpp-nightly.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index eb24a2fab8..df3bbb979f 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -26,8 +26,14 @@ jobs: parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg + # We have to set git identity because git merge will create a merge commit + # in some cases and event when specifying --no-commit. - pwsh: | + Write-Host 'git config user.name "azure-sdk"' + git config user.name "azure-sdk" + Write-Host 'git config user.email "azuresdk@microsoft.com"' git config user.email "azuresdk@microsoft.com" + Write-Host "git checkout $(NightlyBranchName)" git checkout $(NightlyBranchName) if ($LASTEXITCODE -ne 0) { @@ -41,6 +47,12 @@ jobs: Write-Host "git reset HEAD~1" git reset HEAD~1 + + # Remove git identity configuration + Write-Host 'git config --unset user.name' + git config --unset user.name + Write-Host 'git config --unset user.email' + git config --unset user.email workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch From 8b35b331faaea8a6dada558b92d4708ebe0ba568 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 6 Apr 2021 09:09:11 -0700 Subject: [PATCH 46/50] Apply suggestions from code review Co-authored-by: Wes Haggard --- eng/pipelines/cpp-nightly.yml | 2 +- eng/pipelines/templates/steps/generate-nightly-branch-name.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/cpp-nightly.yml index df3bbb979f..901bfb451d 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/cpp-nightly.yml @@ -9,7 +9,7 @@ jobs: variables: # The branch in the azure-sdk/vcpkg repo that is updated nightly with the # latest set of ports - TargetPrBranch: azure-sdk-for-cpp-nightly-verification + TargetPrBranch: release/azure-sdk-for-cpp-daily-verification steps: - checkout: self diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml index 9a4f55fb35..f0a7f4dd43 100644 --- a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml +++ b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml @@ -1,6 +1,6 @@ steps: - pwsh: | - $branchName = $env:NIGHTLYBRANCHOVERRIDE + $branchName = $env:DAILYBRANCHOVERRIDE if (!$branchName) { $formattedDate = Get-Date -Format 'yyyy-MM-dd' $branchName = "nightly/$formattedDate" From 4bea21ba2c1b9715c39943f7f6cd0537b5446bac Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 6 Apr 2021 09:10:22 -0700 Subject: [PATCH 47/50] Review feedback --- ...cpp-nightly.yml => daily-verification.yml} | 23 +-------- .../steps/generate-nightly-branch-name.yml | 2 +- eng/pipelines/templates/steps/vcpkg-clone.yml | 18 +++---- eng/scripts/Update-VcpkgPort.ps1 | 50 ++++++++++--------- 4 files changed, 38 insertions(+), 55 deletions(-) rename eng/pipelines/{cpp-nightly.yml => daily-verification.yml} (68%) diff --git a/eng/pipelines/cpp-nightly.yml b/eng/pipelines/daily-verification.yml similarity index 68% rename from eng/pipelines/cpp-nightly.yml rename to eng/pipelines/daily-verification.yml index df3bbb979f..a5760879d1 100644 --- a/eng/pipelines/cpp-nightly.yml +++ b/eng/pipelines/daily-verification.yml @@ -9,7 +9,7 @@ jobs: variables: # The branch in the azure-sdk/vcpkg repo that is updated nightly with the # latest set of ports - TargetPrBranch: azure-sdk-for-cpp-nightly-verification + TargetPrBranch: release/azure-sdk-for-cpp-daily-verification steps: - checkout: self @@ -20,20 +20,12 @@ jobs: parameters: Workspace: $(Pipeline.Workspace) RepoOwner: azure-sdk - SkipCheckout: true - template: /eng/common/pipelines/templates/steps/set-default-branch.yml parameters: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - # We have to set git identity because git merge will create a merge commit - # in some cases and event when specifying --no-commit. - pwsh: | - Write-Host 'git config user.name "azure-sdk"' - git config user.name "azure-sdk" - Write-Host 'git config user.email "azuresdk@microsoft.com"' - git config user.email "azuresdk@microsoft.com" - Write-Host "git checkout $(NightlyBranchName)" git checkout $(NightlyBranchName) if ($LASTEXITCODE -ne 0) { @@ -41,18 +33,7 @@ jobs: } git checkout $(DefaultBranch) - - Write-Host "git merge $(NightlyBranchName) --strategy-option=theirs --no-commit" - git merge $(NightlyBranchName) --strategy-option=theirs --no-commit - - Write-Host "git reset HEAD~1" - git reset HEAD~1 - - # Remove git identity configuration - Write-Host 'git config --unset user.name' - git config --unset user.name - Write-Host 'git config --unset user.email' - git config --unset user.email + Write-Host "##vso[task.setvariable variable=HasChanges]$true" workingDirectory: $(Pipeline.Workspace)/vcpkg displayName: Update vcpkg Nightly Branch diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml index 9a4f55fb35..2e0ef971ee 100644 --- a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml +++ b/eng/pipelines/templates/steps/generate-nightly-branch-name.yml @@ -3,7 +3,7 @@ steps: $branchName = $env:NIGHTLYBRANCHOVERRIDE if (!$branchName) { $formattedDate = Get-Date -Format 'yyyy-MM-dd' - $branchName = "nightly/$formattedDate" + $branchName = "release/daily-$formattedDate" } Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" displayName: Set branch name in fork repo diff --git a/eng/pipelines/templates/steps/vcpkg-clone.yml b/eng/pipelines/templates/steps/vcpkg-clone.yml index 6ba6c3d636..94640bb22d 100644 --- a/eng/pipelines/templates/steps/vcpkg-clone.yml +++ b/eng/pipelines/templates/steps/vcpkg-clone.yml @@ -3,7 +3,6 @@ parameters: RepoOwner: not-set RepoName: vcpkg PRBranchName: not-set - SkipCheckout: false steps: - pwsh: | @@ -15,12 +14,11 @@ steps: } displayName: Clone vcpkg from upstream - - ${{ if ne(parameters.SkipCheckout, 'true') }}: - # Check out the PR branch if it's already in remote. Ignore failures. - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout Previous PRBranch if it exists - workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file + # Check out the PR branch if it's already in remote. Ignore failures. + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/${{ parameters.PRBranchName }}" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout Previous PRBranch if it exists + workingDirectory: ${{ parameters.Workspace }}/vcpkg \ No newline at end of file diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index c62058c053..320eb1fc61 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -94,30 +94,34 @@ if ($LASTEXITCODE -ne 0) { Write-Host "git reset HEAD^" git reset HEAD^ -# Grab content needed for commit message and place in a temporary file -$packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version -$commitMessageFile = New-TemporaryFile -$chagelogEntry = Get-ChangeLogEntryAsString ` - -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md ` - -VersionString $PackageVersion - -"[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" ` - | Set-Content $commitMessageFile - -Write-Host "Commit Message:" -Write-host (Get-Content $commitMessageFile -Raw) - - -Write-Host "git add -A" -git add -A +# Only perform the final commit if this is not a test release +if (!$TestRelease) { + # Grab content needed for commit message and place in a temporary file + $packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version + $commitMessageFile = New-TemporaryFile + $chagelogEntry = Get-ChangeLogEntryAsString ` + -ChangeLogLocation $ReleaseArtifactSourceDirectory/CHANGELOG.md ` + -VersionString $PackageVersion + + "[$VcpkgPortName] Update to $PackageVersion`n$chagelogEntry" ` + | Set-Content $commitMessageFile + + Write-Host "Commit Message:" + Write-host (Get-Content $commitMessageFile -Raw) + + + Write-Host "git add -A" + git add -A + + # Final commit using commit message from the temporary file. Using the file + # enables the commit message to be formatted properly without having to write + # code to escape certain characters that might appear in the changelog file. + Write-Host "git $GitCommitParameters commit --file $commitMessageFile" + "git $GitCommitParameters commit --file $commitMessageFile" ` + | Invoke-Expression -Verbose ` + | Write-Host +} -# Final commit using commit message from the temporary file. Using the file -# enables the commit message to be formatted properly without having to write -# code to escape certain characters that might appear in the changelog file. -Write-Host "git $GitCommitParameters commit --file $commitMessageFile" -"git $GitCommitParameters commit --file $commitMessageFile" ` - | Invoke-Expression -Verbose ` - | Write-Host <# .SYNOPSIS From d4e0acfa9628c72cf5eb8e23a6b2e043c437ef9b Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 6 Apr 2021 09:30:53 -0700 Subject: [PATCH 48/50] Review feedback --- eng/pipelines/daily-verification.yml | 23 +++++++++---------- .../stages/archetype-cpp-release.yml | 14 +++++------ ...ame.yml => generate-daily-branch-name.yml} | 2 +- eng/scripts/Update-VcpkgPort.ps1 | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) rename eng/pipelines/templates/steps/{generate-nightly-branch-name.yml => generate-daily-branch-name.yml} (75%) diff --git a/eng/pipelines/daily-verification.yml b/eng/pipelines/daily-verification.yml index a5760879d1..96dcea1290 100644 --- a/eng/pipelines/daily-verification.yml +++ b/eng/pipelines/daily-verification.yml @@ -1,20 +1,20 @@ jobs: - job: - displayName: Nightly vcpkg PR + displayName: Daily vcpkg PR pool: name: azsdk-pool-mms-win-2019-general vmImage: MMS2019 variables: - # The branch in the azure-sdk/vcpkg repo that is updated nightly with the + # The branch in the azure-sdk/vcpkg repo that is updated daily with the # latest set of ports TargetPrBranch: release/azure-sdk-for-cpp-daily-verification steps: - checkout: self - - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml + - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: @@ -26,30 +26,29 @@ jobs: WorkingDirectory: $(Pipeline.Workspace)/vcpkg - pwsh: | - Write-Host "git checkout $(NightlyBranchName)" - git checkout $(NightlyBranchName) + Write-Host "git checkout $(DailyBranchName)" + git checkout $(DailyBranchName) if ($LASTEXITCODE -ne 0) { - Write-Error "Cannot check out $(NightlyBranchName) LASTEXITCODE=$LASTEXITCODE" + Write-Error "Cannot check out $(DailyBranchName) LASTEXITCODE=$LASTEXITCODE" } - git checkout $(DefaultBranch) Write-Host "##vso[task.setvariable variable=HasChanges]$true" workingDirectory: $(Pipeline.Workspace)/vcpkg - displayName: Update vcpkg Nightly Branch + displayName: Update vcpkg Daily Branch # This step will skip creating the PR if the PR already exists - template: /eng/common/pipelines/templates/steps/create-pull-request.yml parameters: - # Force push because Update-VcpkgNightlyBranch.ps1 re-creates the - # nightly branch from the tip of the default branch + # Force push so this recreates the $(TargetPrBranch) PushArgs: --force BaseBranchName: $(DefaultBranch) PRBranchName: $(TargetPrBranch) RepoOwner: Microsoft RepoName: vcpkg WorkingDirectory: $(Pipeline.Workspace)/vcpkg - CommitMsg: Update with changes from $(NightlyBranchName) - PRTitle: "[DO NOT MERGE] Nightly Azure SDK for C++ CI validation" + CommitMsg: Update with changes from $(DailyBranchName) + PRTitle: "[DO NOT MERGE] Daily Azure SDK for C++ CI validation" PRBody: This is a long-lived draft PR should not be merged. Its branch is be updated periodically to validate the Azure SDK for C++ against the vcpkg CI. ScriptDirectory: $(System.DefaultWorkingDirectory)/eng/common/scripts OpenAsDraft: true + SkipCheckingForChanges: true diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 89ce253e8b..636b29ca01 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -186,10 +186,10 @@ stages: - ${{if eq(variables['System.TeamProject'], 'internal') }}: - stage: Integration dependsOn: Build - condition: and(succeeded(), or(eq(variables['PublishNightlyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) + condition: and(succeeded(), or(eq(variables['PublishDailyVcpkg'], 'true'), eq(variables['Build.Reason'],'Schedule'))) jobs: - - job: PublishNightlyVcpkg - displayName: Publish to vcpkg nightly branch + - job: PublishDailyVcpkg + displayName: Publish to vcpkg daily branch pool: name: azsdk-pool-mms-win-2019-general @@ -200,12 +200,12 @@ stages: - download: current artifact: packages - - template: /eng/pipelines/templates/steps/generate-nightly-branch-name.yml + - template: /eng/pipelines/templates/steps/generate-daily-branch-name.yml - template: /eng/pipelines/templates/steps/vcpkg-clone.yml parameters: RepoOwner: azure-sdk - PRBranchName: $(NightlyBranchName) + PRBranchName: $(DailyBranchName) # Placing artifacts in the same branch in sequence reduces the # potential for race conditions. If this job does fail to push @@ -238,8 +238,8 @@ stages: - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: - BaseRepoBranch: $(NightlyBranchName) - CommitMsg: Nightly vcpkg ports for ${{ parameters.ServiceDirectory }} + BaseRepoBranch: $(DailyBranchName) + CommitMsg: Daily vcpkg ports for ${{ parameters.ServiceDirectory }} TargetRepoOwner: azure-sdk TargetRepoName: vcpkg WorkingDirectory: $(Pipeline.Workspace)/vcpkg diff --git a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml b/eng/pipelines/templates/steps/generate-daily-branch-name.yml similarity index 75% rename from eng/pipelines/templates/steps/generate-nightly-branch-name.yml rename to eng/pipelines/templates/steps/generate-daily-branch-name.yml index f3a5cbcd8e..18762e5e52 100644 --- a/eng/pipelines/templates/steps/generate-nightly-branch-name.yml +++ b/eng/pipelines/templates/steps/generate-daily-branch-name.yml @@ -5,5 +5,5 @@ steps: $formattedDate = Get-Date -Format 'yyyy-MM-dd' $branchName = "release/daily-$formattedDate" } - Write-Host "##vso[task.setvariable variable=NightlyBranchName]$branchName" + Write-Host "##vso[task.setvariable variable=DailyBranchName]$branchName" displayName: Set branch name in fork repo diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index 320eb1fc61..0a6f0174d4 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -158,7 +158,7 @@ user.name and user.email. .PARAMETER TestRelease In the case of a test release set this to ensure that the x-add-version step -includes `--overwrite-version` to ensure nightly packages are properly updated +includes `--overwrite-version` to ensure daily packages are properly updated in the vcpkg repo. #> From 586b66f865e5be943a59105084f2f9d7f9909df0 Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 6 Apr 2021 10:06:26 -0700 Subject: [PATCH 49/50] Remove extra reset --- .../templates/stages/archetype-cpp-release.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 636b29ca01..39cba44284 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -224,18 +224,6 @@ stages: TestRelease: true TestReleaseRef: $(Build.SourceVersion) - # Reset commit after vcpkg-publish.yml so git-push-changes.yml - # has changes to commit and push, and all changes go into one - # commit - - pwsh: | - Write-Host "git reset HEAD^" - git reset HEAD^ - - Write-Host "git status" - git status - displayName: Reset commit - workingDirectory: $(Pipeline.Workspace)/vcpkg - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: BaseRepoBranch: $(DailyBranchName) From dd229819649d18b324066320e329271fcface7ff Mon Sep 17 00:00:00 2001 From: Daniel Jurek Date: Tue, 6 Apr 2021 10:37:11 -0700 Subject: [PATCH 50/50] TestRelease -> DailyRelease --- .../templates/stages/archetype-cpp-release.yml | 4 ++-- .../templates/steps/vcpkg-publish.yml | 8 ++++---- eng/scripts/Initialize-VcpkgRelease.ps1 | 18 +++++++++--------- eng/scripts/Update-VcpkgPort.ps1 | 13 ++++++++----- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/eng/pipelines/templates/stages/archetype-cpp-release.yml b/eng/pipelines/templates/stages/archetype-cpp-release.yml index 39cba44284..fb83fd5fdb 100644 --- a/eng/pipelines/templates/stages/archetype-cpp-release.yml +++ b/eng/pipelines/templates/stages/archetype-cpp-release.yml @@ -221,8 +221,8 @@ stages: ArtifactName: ${{ artifact.Name }} VcpkgPortName: ${{ artifact.VcpkgPortName }} DisplayNameExtension: ${{ artifact.Name }} - TestRelease: true - TestReleaseRef: $(Build.SourceVersion) + DailyRelease: true + DailyReleaseRef: $(Build.SourceVersion) - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: diff --git a/eng/pipelines/templates/steps/vcpkg-publish.yml b/eng/pipelines/templates/steps/vcpkg-publish.yml index 63117b59b8..0493eebc83 100644 --- a/eng/pipelines/templates/steps/vcpkg-publish.yml +++ b/eng/pipelines/templates/steps/vcpkg-publish.yml @@ -4,8 +4,8 @@ parameters: ArtifactName: not-set VcpkgPortName: not-set DisplayNameExtension: - TestReleaseRef: - TestRelease: false + DailyReleaseRef: + DailyRelease: false steps: - task: Powershell@2 @@ -15,7 +15,7 @@ steps: -GitHubRepo "${{ parameters.GitHubRepo }}" -PackageSpecPath ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/package-info.json -SourceDirectory ${{ parameters.Workspace }}/packages/${{ parameters.ArtifactName }}/vcpkg - -TestReleaseRef '${{ parameters.TestReleaseRef }}' + -DailyReleaseRef '${{ parameters.DailyReleaseRef }}' -Verbose pwsh: true displayName: Initialize vcpkg release ${{ parameters.DisplayNameExtension }} @@ -30,6 +30,6 @@ steps: -PortDestinationDirectory 'ports/${{ parameters.VcpkgPortName }}' -VcpkgPortName '${{ parameters.VcpkgPortName }}' -GitCommitParameters '-c user.name="azure-sdk" -c user.email="azuresdk@microsoft.com"' - -TestRelease:$${{ parameters.TestRelease }} + -DailyRelease:$${{ parameters.DailyRelease }} workingDirectory: ${{ parameters.Workspace }}/vcpkg displayName: Update vcpkg port ${{ parameters.DisplayNameExtension }} \ No newline at end of file diff --git a/eng/scripts/Initialize-VcpkgRelease.ps1 b/eng/scripts/Initialize-VcpkgRelease.ps1 index 46c9edcdb4..c62e54afd9 100644 --- a/eng/scripts/Initialize-VcpkgRelease.ps1 +++ b/eng/scripts/Initialize-VcpkgRelease.ps1 @@ -12,12 +12,12 @@ Location of the relevant package-info.json file .PARAMETER GitHubRepo Name of the GitHub repo (of the form Azure/azure-sdk-for-cpp) -.PARAMETER TestReleaseRef +.PARAMETER DailyReleaseRef If supplied update the portfile.cmake file's REF and SHA512 with values associated with the given ref. #> - +[CmdletBinding()] param ( [Parameter(Mandatory = $true)] [ValidateNotNullOrEmpty()] @@ -31,7 +31,7 @@ param ( [ValidateNotNullOrEmpty()] [string] $GitHubRepo, - [string] $TestReleaseRef + [string] $DailyReleaseRef ) # If there's nothing in the "port" folder to upload set SkipVcpkgUpdate to true @@ -45,9 +45,9 @@ if (!(Get-ChildItem -Path "$SourceDirectory/port/CONTROL")) { $packageSpec = Get-Content -Raw -Path $PackageSpecPath | ConvertFrom-Json $tarGzUri = "https://github.com/$GitHubRepo/archive/$($packageSpec.packageName).tar.gz" -if ($TestReleaseRef) { - Write-Verbose "Initializing Test Release" - $tarGzUri = "https://github.com/$GitHubRepo/archive/$TestReleaseRef.tar.gz" +if ($DailyReleaseRef) { + Write-Verbose "Initializing Daily Release" + $tarGzUri = "https://github.com/$GitHubRepo/archive/$DailyReleaseRef.tar.gz" } Write-Host "Downloading tarball to compute hash from $tarGzUri" @@ -67,9 +67,9 @@ $portfileLocation = "$SourceDirectory/port/portfile.cmake" $portFileContent = Get-Content -Raw -Path $portfileLocation $newContent = $portFileContent -replace '(SHA512\s+)1', "`${1}$sha512" -if ($TestReleaseRef) { - Write-Verbose "Overriding REF with test release ref: $TestReleaseRef" - $newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $TestReleaseRef" +if ($DailyReleaseRef) { + Write-Verbose "Overriding REF with test release ref: $DailyReleaseRef" + $newContent = $newContent -replace '(?m)^(\s+)REF azure.*$', "`${1}REF $DailyReleaseRef" } $newContent | Set-Content $portfileLocation -NoNewLine diff --git a/eng/scripts/Update-VcpkgPort.ps1 b/eng/scripts/Update-VcpkgPort.ps1 index 0a6f0174d4..35708e7201 100644 --- a/eng/scripts/Update-VcpkgPort.ps1 +++ b/eng/scripts/Update-VcpkgPort.ps1 @@ -13,7 +13,7 @@ param ( [string] $GitCommitParameters, - [switch] $TestRelease + [switch] $DailyRelease ) ."$PSScriptRoot/../common/scripts/common.ps1" @@ -78,7 +78,7 @@ if ($LASTEXITCODE -ne 0) { } $addVersionAdditionalParameters = '' -if ($TestRelease) { +if ($DailyRelease) { $addVersionAdditionalParameters = '--overwrite-version' } @@ -95,7 +95,7 @@ Write-Host "git reset HEAD^" git reset HEAD^ # Only perform the final commit if this is not a test release -if (!$TestRelease) { +if (!$DailyRelease) { # Grab content needed for commit message and place in a temporary file $packageVersion = (Get-Content $ReleaseArtifactSourceDirectory/package-info.json -Raw | ConvertFrom-Json).version $commitMessageFile = New-TemporaryFile @@ -109,7 +109,6 @@ if (!$TestRelease) { Write-Host "Commit Message:" Write-host (Get-Content $commitMessageFile -Raw) - Write-Host "git add -A" git add -A @@ -120,6 +119,10 @@ if (!$TestRelease) { "git $GitCommitParameters commit --file $commitMessageFile" ` | Invoke-Expression -Verbose ` | Write-Host + + # Set $(HasChanges) to $true so that create-pull-request.yml completes the + # push and PR submission steps + Write-Host "##vso[task.setvariable variable=HasChanges]$true" } @@ -156,7 +159,7 @@ Additional parameters to supply to the `git commit` command. These are useful in the context of Azure DevOps where the git client does not have a configured user.name and user.email. -.PARAMETER TestRelease +.PARAMETER DailyRelease In the case of a test release set this to ensure that the x-add-version step includes `--overwrite-version` to ensure daily packages are properly updated in the vcpkg repo.