diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 16d61f12bc..5b831ce51c 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,103 +1,103 @@ - - https://github.com/dotnet/arcade - e4b369ad59dad41fa93b546b3d0cd81686cf45d4 + + https://github.com/dotnet/dotnet + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 - + https://github.com/dotnet/dotnet - 6fc1ab49c125b1cff73ea18706d46a94646ef60a + 025cac314d7303a35dcc77dd4ef2b209ab98c641 diff --git a/eng/Versions.props b/eng/Versions.props index d8d683eaa4..26cfe5917e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,64 +11,64 @@ $(MicrosoftBuildPackageVersion) - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 10.0.8 9.0.16 8.0.27 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 10.0.8 9.0.16 8.0.27 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 10.0.8 9.0.16 8.0.27 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 11.0.0-preview.1.26104.118 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 3.0.0 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 - 11.0.0-preview.5.26268.103 + 11.0.0-preview.6.26305.101 0.5.638 0.47.0 5.3.0 diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 18397a60eb..4b4f6b0923 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -24,6 +24,7 @@ Param( [switch][Alias('pb')]$productBuild, [switch]$fromVMR, [switch][Alias('bl')]$binaryLog, + [string][Alias('bln')]$binaryLogName = '', [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, @@ -46,6 +47,7 @@ function Print-Usage() { Write-Host " -platform Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild" Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" Write-Host " -binaryLog Output binary log (short: -bl)" + Write-Host " -binaryLogName Binary log file name or path; implies -binaryLog (short: -bln)" Write-Host " -help Print help and exit" Write-Host "" @@ -102,7 +104,19 @@ function Build { $toolsetBuildProj = InitializeToolset InitializeCustomToolset - $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' } + $bl = '' + if ($binaryLog) { + $binaryLogPath = if ([string]::IsNullOrEmpty($binaryLogName)) { + Join-Path $LogDir 'Build.binlog' + } elseif ([System.IO.Path]::IsPathRooted($binaryLogName)) { + $binaryLogName + } else { + Join-Path $LogDir $binaryLogName + } + + Create-Directory (Split-Path -Parent $binaryLogPath) + $bl = '/bl:' + $binaryLogPath + } $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' } $check = if ($buildCheck) { '/check' } else { '' } @@ -162,6 +176,10 @@ try { $nodeReuse = $false } + if (-not [string]::IsNullOrEmpty($binaryLogName)) { + $binaryLog = $true + } + if ($nativeToolsOnMachine) { $env:NativeToolsOnMachine = $true } diff --git a/eng/common/build.sh b/eng/common/build.sh index 74605d2eff..719ee4b587 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -13,6 +13,7 @@ usage() echo " --configuration Build configuration: 'Debug' or 'Release' (short: -c)" echo " --verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)" echo " --binaryLog Create MSBuild binary log (short: -bl)" + echo " --binaryLogName Binary log file name or path; implies --binaryLog (short: -bln)" echo " --help Print help and exit (short: -h)" echo "" @@ -83,6 +84,7 @@ warn_not_as_error='' node_reuse=true build_check=false binary_log=false +binary_log_name='' exclude_ci_binary_log=false projects='' @@ -114,6 +116,11 @@ while [[ $# -gt 0 ]]; do -binarylog|-bl) binary_log=true ;; + -binarylogname|-bln) + binary_log=true + binary_log_name=$2 + shift + ;; -excludecibinarylog|-nobl) exclude_ci_binary_log=true ;; @@ -232,7 +239,17 @@ function Build { local bl="" if [[ "$binary_log" == true ]]; then - bl="/bl:\"$log_dir/Build.binlog\"" + local binary_log_path="" + if [[ -z "$binary_log_name" ]]; then + binary_log_path="$log_dir/Build.binlog" + elif [[ "$binary_log_name" = /* ]]; then + binary_log_path="$binary_log_name" + else + binary_log_path="$log_dir/$binary_log_name" + fi + + mkdir -p "$(dirname "$binary_log_path")" + bl="/bl:\"$binary_log_path\"" fi local check="" diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml new file mode 100644 index 0000000000..a8162c5116 --- /dev/null +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -0,0 +1,217 @@ +parameters: +# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes. +- name: timeoutInMinutes + type: number + default: 360 + +# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty. +- name: organization + type: string + default: '' + +# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty. +- name: repository + type: string + default: '' + +# Optional dependency list for the generated job. +- name: dependsOn + type: object + default: [] + +# Optional condition for the generated job. +- name: condition + type: string + default: '' + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the +# default code path the version is taken from the consuming repo's .config/dotnet-tools.json. +- name: toolVersion + type: string + default: '' + +# Base URI for the Helix service (--helix-base-uri). +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# Polling interval in seconds (--polling-interval-seconds). +- name: pollingIntervalSeconds + type: number + default: 30 + +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into +# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is +# primarily intended for the Arcade repository itself, where the Helix job monitor tool is +# built in the same pipeline that runs this template. +# +# When this parameter is empty (the default), the consuming repository must declare the tool +# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template +# will check out the repo and run 'dotnet tool restore' to install the version pinned there. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + +jobs: +- job: HelixJobMonitor + displayName: Monitor Helix Jobs + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64.open + ${{ else }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals build.azurelinux.3.amd64 + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Helix Job Monitor artifact + inputs: + buildType: current + artifactName: ${{ parameters.toolNupkgArtifactName }} + itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' + targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg + + - bash: | + set -euo pipefail + + toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" + mkdir -p "$toolPath" + + packageId='${{ parameters.toolPackageId }}' + toolVersion='${{ parameters.toolVersion }}' + nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' + nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath" + + if [ ! -d "$nupkgDir" ]; then + echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2 + exit 1 + fi + + nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1) + if [ -z "$nupkg" ]; then + echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2 + exit 1 + fi + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + nupkgBase=$(basename "$nupkg" .nupkg) + derivedVersion="${nupkgBase#${packageId}.}" + if [ -z "$toolVersion" ]; then + toolVersion="$derivedVersion" + fi + + echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + + # Create a minimal NuGet.config that only references the local nupkg directory. + # This avoids conflicts with the repo's package source mapping which blocks --add-source. + toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" + printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig" + + pushd "$(Build.SourcesDirectory)" > /dev/null + ./eng/common/dotnet.sh tool install \ + --tool-path "$toolPath" "$packageId" \ + --version "$toolVersion" \ + --configfile "$toolNugetConfig" + + # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. + toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) + toolDll="${toolDll%.deps.json}.dll" + if [ ! -f "$toolDll" ]; then + echo "Could not find tool DLL in '$toolPath/.store'." >&2 + exit 1 + fi + + echo "Tool DLL: $toolDll" + echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll" + displayName: Install Helix Job Monitor + + - ${{ else }}: + - bash: ./eng/common/dotnet.sh tool restore + displayName: Restore Helix Job Monitor + + - bash: | + set -euo pipefail + + toolArgs=( + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. + --stage-name '$(System.StageName)' + ) + + organization='${{ parameters.organization }}' + repository='${{ parameters.repository }}' + + # Fall back to Azure DevOps-provided environment variables when the caller did not + # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically + # 'owner/repo' for GitHub-backed builds. + if [ -z "$organization" ] || [ -z "$repository" ]; then + buildRepoName="${BUILD_REPOSITORY_NAME:-}" + if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then + repoOwner="${buildRepoName%%/*}" + repoName="${buildRepoName#*/}" + if [ -z "$organization" ]; then organization="$repoOwner"; fi + if [ -z "$repository" ]; then repository="$repoName"; fi + fi + fi + + if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi + if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + + # Build.Reason and Build.SourceBranch are required to derive the Helix source filter + # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official', + # otherwise -> 'ci'). Without these, manually-queued / scheduled / CI builds would + # be looked up under the wrong source prefix and find zero jobs. + toolArgs+=( --build-reason "$(Build.Reason)" ) + toolArgs+=( --source-branch "$(Build.SourceBranch)" ) + + if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then + # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet. + export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet" + ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}" + else + # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it + # through the manifest from the repo root. + pushd "$BUILD_SOURCESDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" + fi + displayName: Monitor Helix Jobs + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index 66c7988f22..cb60f52978 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -75,6 +75,12 @@ jobs: variables: - name: AllowPtrToDetectTestRunRetryFiles value: true + # Component Governance detection and CodeQL are not run in the public project + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: skipComponentGovernanceDetection + value: true + - name: Codeql.SkipTaskAutoInjection + value: true - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index 5fc099a114..2731e48cce 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -33,8 +33,6 @@ steps: '$(publishing-dnceng-devdiv-code-r-build-re)' '$(dn-bot-all-orgs-artifact-feeds-rw)' '$(akams-client-id)' - '$(dnceng-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' '$(System.AccessToken)' ${{parameters.CustomSensitiveDataList}} continueOnError: true diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index 3ad83b8c30..fdca622357 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,15 +1,15 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250906.1 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250906.1 + sourceIndexUploadPackageVersion: 2.0.0-20260521.2 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20260521.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog steps: - task: UseDotNet@2 - displayName: "Source Index: Use .NET 9 SDK" + displayName: "Source Index: Use .NET 10 SDK" inputs: packageType: sdk - version: 9.0.x + version: 10.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index cef5d2d671..f06854ccc1 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -76,7 +76,7 @@ __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" __FreeBSDBase="13.5-RELEASE" -__FreeBSDPkg="1.21.3" +__FreeBSDPkg="2.7.5" __FreeBSDABI="13" __FreeBSDPackages="libunwind" __FreeBSDPackages+=" icu" @@ -577,7 +577,7 @@ elif [[ "$__CodeName" == "freebsd" ]]; then ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. - INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) IGNORE_OSVERSION=yes "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update # shellcheck disable=SC2086 INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages elif [[ "$__CodeName" == "openbsd" ]]; then diff --git a/eng/common/cross/install-debs.py b/eng/common/cross/install-debs.py index 100c4378da..20ca770a1e 100644 --- a/eng/common/cross/install-debs.py +++ b/eng/common/cross/install-debs.py @@ -328,7 +328,18 @@ def extract_deb_file(deb_file, tmp_dir, extract_dir, ar_tool): raise ValueError(f"Unsupported compression format: {file_extension}") with tarfile.open(tar_file_path, mode) as tar: - tar.extractall(path=extract_dir, filter='tar') + tar.extractall(path=extract_dir, filter=_rootfs_extraction_filter) + +def _rootfs_extraction_filter(member, dest_path): + """Tarfile extraction filter based on the 'data' filter that additionally + rewrites absolute-target symlinks/hardlinks into rootfs-relative paths. + """ + if (member.issym() or member.islnk()) and os.path.isabs(member.linkname): + link_dir = os.path.dirname(member.name) + new_linkname = os.path.relpath(member.linkname.lstrip('/'), + start=link_dir or '.') + member = member.replace(linkname=new_linkname, deep=False) + return tarfile.data_filter(member, dest_path) def finalize_setup(rootfsdir): lib_dir = os.path.join(rootfsdir, 'lib') diff --git a/eng/common/native/LocateNativeCompiler.targets b/eng/common/native/LocateNativeCompiler.targets new file mode 100644 index 0000000000..028b33d944 --- /dev/null +++ b/eng/common/native/LocateNativeCompiler.targets @@ -0,0 +1,27 @@ + + + + + clang + $(ROOTFS_DIR) + + + + + + + + $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) + <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) + lld + + + diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 4742177a76..aff839fa09 100644 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -24,16 +24,16 @@ case "$os" in apt update apt install -y build-essential gettext locales cmake llvm clang lld lldb liblldb-dev libunwind8-dev libicu-dev liblttng-ust-dev \ - libssl-dev libkrb5-dev pigz cpio ninja-build + libssl-dev libkrb5-dev pigz cpio ninja-build file localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos" ]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" - $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build + $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file elif [ "$ID" = "amzn" ]; then - dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio ninja-build file elif [ "$ID" = "alpine" ]; then - apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja + apk add build-base cmake bash curl clang llvm llvm-dev lld lldb-dev krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio ninja file else echo "Unsupported distro. distro: $ID" exit 1 diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 5e261f34db..85501406a5 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -21,11 +21,6 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} - # we don't run CG in public - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - script: echo "##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable - artifactPublishSteps: - ${{ if ne(parameters.artifacts.publish, '') }}: - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 73157c2634..fc72fe6304 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -162,12 +162,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 } - # Keep repo builds isolated from machine-installed SDK state and workload advertising. - # This avoids preview SDK builds picking up mismatched workloads on CI images. - $env:DOTNET_MULTILEVEL_LOOKUP = '0' - $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE = '1' - $env:DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE = '1' - # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { $dotnetExecutable = GetExecutableFileName 'dotnet' @@ -230,9 +224,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { Write-PipelinePrependPath -Path $dotnetRoot Write-PipelineSetVariable -Name 'DOTNET_NOLOGO' -Value '1' - Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' - Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' - Write-PipelineSetVariable -Name 'DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE' -Value '1' return $global:_DotNetInstallDir = $dotnetRoot } @@ -599,16 +590,16 @@ function GetDefaultMSBuildEngine() { ExitWithExitCode 1 } -function GetNuGetPackageCachePath() { +function InitializeNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build. - # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116 # use global cache in dev builds to avoid cost of downloading packages. # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' + $userProfile = if (IsWindowsPlatform) { $env:UserProfile } else { $env:HOME } + $env:NUGET_PACKAGES = [IO.Path]::Combine($userProfile, '.nuget', 'packages') + [IO.Path]::DirectorySeparatorChar } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' + $env:NUGET_PACKAGES = [IO.Path]::Combine($RepoRoot, '.packages') + [IO.Path]::DirectorySeparatorChar } } @@ -657,8 +648,6 @@ function InitializeToolset() { return $global:_InitializeToolset } - $nugetCache = GetNuGetPackageCachePath - $toolsetVersion = Read-ArcadeSdkVersion $toolsetToolsDir = Join-Path $ToolsetDir $toolsetVersion @@ -679,7 +668,7 @@ function InitializeToolset() { ExitWithExitCode 1 } - $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetCache") + $downloadArgs = @("package", "download", "Microsoft.DotNet.Arcade.Sdk@$toolsetVersion", "--verbosity", "minimal", "--prerelease", "--output", "$nugetPackageCachePath") $nugetConfig = $env:NUGET_CONFIG if (-not $nugetConfig) { # Search for any variation of nuget.config in the RepoRoot @@ -696,7 +685,7 @@ function InitializeToolset() { } DotNet @downloadArgs - $packageDir = Join-Path $nugetCache (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) + $packageDir = Join-Path $nugetPackageCachePath (Join-Path 'microsoft.dotnet.arcade.sdk' $toolsetVersion) $packageToolsetDir = Join-Path $packageDir 'toolset' if (!(Test-Path $packageToolsetDir)) { @@ -747,58 +736,6 @@ function Stop-Processes() { # Terminates the script if the build fails. # function MSBuild() { - if ($ci) { - $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 - $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' - Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' - - Enable-Nuget-EnhancedRetry - } - - MSBuild-Core @args -} - -# -# Executes a dotnet command with arguments passed to the function. -# Terminates the script if the command fails. -# -function DotNet() { - $dotnetRoot = InitializeDotNetCli -install:$restore - $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') - - $cmdArgs = "" - foreach ($arg in $args) { - if ($null -ne $arg -and $arg.Trim() -ne "") { - if ($arg.EndsWith('\')) { - $arg = $arg + "\" - } - $cmdArgs += " `"$arg`"" - } - } - - $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" - - $exitCode = Exec-Process $dotnetPath $cmdArgs - - if ($exitCode -ne 0) { - Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red - - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { - Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." - ExitWithExitCode 0 - } else { - ExitWithExitCode $exitCode - } - } -} - -# -# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function. -# The arguments are automatically quoted. -# Terminates the script if the build fails. -# -function MSBuild-Core() { if ($ci) { if (!$binaryLog -and !$excludeCIBinarylog) { Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' @@ -811,16 +748,10 @@ function MSBuild-Core() { } } - Enable-Nuget-EnhancedRetry - $buildTool = InitializeBuildTool $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci" - if ($ci -and $buildTool.Tool -eq 'dotnet') { - $cmdArgs += ' /p:MSBuildEnableWorkloadResolver=false' - } - # Add -mt flag for MSBuild multithreaded mode if enabled via environment variable if ($env:MSBUILD_MT_ENABLED -eq "1") { $cmdArgs += ' -mt' @@ -874,6 +805,40 @@ function MSBuild-Core() { } } +# +# Executes a dotnet command with arguments passed to the function. +# Terminates the script if the command fails. +# +function DotNet() { + $dotnetRoot = InitializeDotNetCli -install:$restore + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + + $cmdArgs = "" + foreach ($arg in $args) { + if ($null -ne $arg -and $arg.Trim() -ne "") { + if ($arg.EndsWith('\')) { + $arg = $arg + "\" + } + $cmdArgs += " `"$arg`"" + } + } + + $env:ARCADE_BUILD_TOOL_COMMAND = "`"$dotnetPath`" $cmdArgs" + + $exitCode = Exec-Process $dotnetPath $cmdArgs + + if ($exitCode -ne 0) { + Write-Host "dotnet command failed with exit code $exitCode. Check errors above." -ForegroundColor Red + + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$fromVMR) { + Write-PipelineSetResult -Result "Failed" -Message "dotnet command execution failed." + ExitWithExitCode 0 + } else { + ExitWithExitCode $exitCode + } + } +} + function GetMSBuildBinaryLogCommandLineArgument($arguments) { foreach ($argument in $arguments) { if ($argument -ne $null) { @@ -958,19 +923,5 @@ if (!$disableConfigureToolsetImport) { } } -# -# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. -# -function Enable-Nuget-EnhancedRetry() { - if ($ci) { - Write-Host "Setting NUGET enhanced retry environment variables" - $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true' - $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6 - $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000 - $env:NUGET_RETRY_HTTP_429 = 'true' - Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6' - Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000' - Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true' - } -} +# Initialize the nuget package cache vars +$nugetPackageCachePath = InitializeNuGetPackageCachePath diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 8edf9f5a69..48cab70ebf 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -116,12 +116,6 @@ function InitializeDotNetCli { export DOTNET_CLI_TELEMETRY_OPTOUT=1 fi - # Keep repo builds isolated from machine-installed SDK state and workload advertising. - # This avoids preview SDK builds picking up mismatched workloads on CI images. - export DOTNET_MULTILEVEL_LOOKUP=0 - export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 - export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=1 - # LTTNG is the logging infrastructure used by Core CLR. Need this variable set # so it doesn't output warnings to the console. export LTTNG_HOME="$HOME" @@ -167,9 +161,6 @@ function InitializeDotNetCli { Write-PipelinePrependPath -path "$dotnet_root" Write-PipelineSetVariable -name "DOTNET_NOLOGO" -value "1" - Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0" - Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1" - Write-PipelineSetVariable -name "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE" -value "1" # return value _InitializeDotNetCli="$dotnet_root" @@ -373,7 +364,7 @@ function InitializeBuildTool { _InitializeBuildToolCommand="msbuild" } -function GetNuGetPackageCachePath { +function InitializeNuGetPackageCachePath { if [[ -z ${NUGET_PACKAGES:-} ]]; then if [[ "$use_global_nuget_cache" == true ]]; then export NUGET_PACKAGES="$HOME/.nuget/packages/" @@ -383,7 +374,7 @@ function GetNuGetPackageCachePath { fi # return value - _GetNuGetPackageCachePath=$NUGET_PACKAGES + _InitializeNuGetPackageCachePath=$NUGET_PACKAGES } function InitializeNativeTools() { @@ -405,8 +396,6 @@ function InitializeToolset { return fi - GetNuGetPackageCachePath - ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk" local toolset_version=$_ReadGlobalVersion @@ -429,7 +418,7 @@ function InitializeToolset { ExitWithExitCode 2 fi - local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_GetNuGetPackageCachePath") + local download_args=("package" "download" "Microsoft.DotNet.Arcade.Sdk@$toolset_version" "--verbosity" "minimal" "--prerelease" "--output" "$_InitializeNuGetPackageCachePath") local nuget_config="${NUGET_CONFIG:-}" if [[ -z "$nuget_config" ]]; then # Search for any variation of nuget.config in the RepoRoot @@ -446,7 +435,7 @@ function InitializeToolset { fi DotNet "${download_args[@]}" - local package_dir="$_GetNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" + local package_dir="$_InitializeNuGetPackageCachePath/microsoft.dotnet.arcade.sdk/$toolset_version" if [[ ! -d "$package_dir/toolset" ]]; then Write-PipelineTelemetryError -category 'InitializeToolset' "Arcade SDK package does not contain a toolset folder: $package_dir" @@ -502,19 +491,6 @@ function DotNet { } function MSBuild { - local args=( "$@" ) - - if [[ "$ci" == true ]]; then - export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 - export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 - Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" - Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" - fi - - MSBuild-Core "${args[@]}" -} - -function MSBuild-Core { if [[ "$ci" == true ]]; then if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." @@ -567,12 +543,7 @@ function MSBuild-Core { warnnotaserror_switch="/warnnotaserror:$warn_not_as_error /p:AdditionalWarningsNotAsErrors=$warn_not_as_error" fi - local workload_resolver_switch="" - if [[ "$ci" == true && -n "${_InitializeBuildToolCommand:-}" ]]; then - workload_resolver_switch="/p:MSBuildEnableWorkloadResolver=false" - fi - - RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch $workload_resolver_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch $mt_switch $warnnotaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } function GetDarc { @@ -663,3 +634,6 @@ fi if [[ -n "${useInstalledDotNetCli:-}" ]]; then use_installed_dotnet_cli="$useInstalledDotNetCli" fi + +# Initialize the nuget package cache vars +InitializeNuGetPackageCachePath diff --git a/global.json b/global.json index 8c88e38017..6bd4bde746 100644 --- a/global.json +++ b/global.json @@ -8,7 +8,7 @@ "dotnet": "11.0.100-preview.5.26227.104" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26269.7", + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26305.101", "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.23607.2" } }