[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -96,14 +96,14 @@
       <Uri>https://dev.azure.com/dnceng/internal/_git/dotnet-runtime</Uri>
       <Sha>3a25a7f1cc446b60678ed25c9d829420d6321eba</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24352.2">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.24426.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
+      <Sha>e3bdd9a0f2a65fe037ba1adb2261eea48a840fa4</Sha>
     </Dependency>
     <!-- Intermediate is necessary for source build. -->
-    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24352.2">
+    <Dependency Name="Microsoft.SourceBuild.Intermediate.arcade" Version="9.0.0-beta.24426.3">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>4a7d983f833d6b86365ea1b2b4d6ee72fbdbf944</Sha>
+      <Sha>e3bdd9a0f2a65fe037ba1adb2261eea48a840fa4</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DotNet.XliffTasks" Version="1.0.0-beta.23475.1" CoherentParentDependency="Microsoft.DotNet.Arcade.Sdk">

eng/Versions.props

@@ -50,7 +50,7 @@
     <MicrosoftNETCoreBrowserDebugHostTransportPackageVersion>6.0.2-servicing.22064.6</MicrosoftNETCoreBrowserDebugHostTransportPackageVersion>
     <MicrosoftNETCorePlatformsPackageVersion>6.0.1</MicrosoftNETCorePlatformsPackageVersion>
     <MicrosoftSourceBuildIntermediatesourcebuildreferencepackagesPackageVersion>9.0.0-alpha.1.24304.1</MicrosoftSourceBuildIntermediatesourcebuildreferencepackagesPackageVersion>
-    <MicrosoftSourceBuildIntermediatearcadePackageVersion>9.0.0-beta.24352.2</MicrosoftSourceBuildIntermediatearcadePackageVersion>
+    <MicrosoftSourceBuildIntermediatearcadePackageVersion>9.0.0-beta.24426.3</MicrosoftSourceBuildIntermediatearcadePackageVersion>
     <MicrosoftDotNetXliffTasksPackageVersion>1.0.0-beta.23475.1</MicrosoftDotNetXliffTasksPackageVersion>
     <MicrosoftSourceBuildIntermediatexlifftasksPackageVersion>1.0.0-beta.23475.1</MicrosoftSourceBuildIntermediatexlifftasksPackageVersion>
     <MicrosoftNetCompilersToolsetPackageVersion>4.11.0-3.24303.3</MicrosoftNetCompilersToolsetPackageVersion>

eng/common/SetupNugetSources.ps1

@@ -157,7 +157,7 @@ if ($dotnet31Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password
 }
 
-$dotnetVersions = @('5','6','7','8')
+$dotnetVersions = @('5','6','7','8','9')
 
 foreach ($dotnetVersion in $dotnetVersions) {
     $feedPrefix = "dotnet" + $dotnetVersion;

eng/common/SetupNugetSources.sh

@@ -99,7 +99,7 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet3.1-internal-transport')
 fi
 
-DotNetVersions=('5' '6' '7' '8')
+DotNetVersions=('5' '6' '7' '8' '9')
 
 for DotNetVersion in ${DotNetVersions[@]} ; do
     FeedPrefix="dotnet${DotNetVersion}";

eng/common/core-templates/job/job.yml

@@ -33,11 +33,6 @@ parameters:
   artifactPublishSteps: []
   runAsPublic: false
 
-# Sbom related params
-  enableSbom: true
-  PackageVersion: 9.0.0
-  BuildDropPath: '$(Build.SourcesDirectory)/artifacts'
-
 # 1es specific parameters
   is1ESPipeline: ''
 

eng/common/core-templates/job/publish-build-assets.yml

@@ -113,38 +113,19 @@ jobs:
           Add-Content -Path $filePath -Value "$(DefaultChannels)"
           Add-Content -Path $filePath -Value $(IsStableBuild)
 
-    - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
-      parameters:
-        is1ESPipeline: ${{ parameters.is1ESPipeline }}
-        args:
-          displayName: Publish ReleaseConfigs Artifact
-          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
-          publishLocation: Container
-          artifactName: ReleaseConfigs
-
-    - task: powershell@2
-      displayName: Check if SymbolPublishingExclusionsFile.txt exists
-      inputs:
-        targetType: inline
-        script: |
           $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt"
-          if(Test-Path -Path $symbolExclusionfile)
+          if (Test-Path -Path $symbolExclusionfile)
           {
             Write-Host "SymbolExclusionFile exists"
-            Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true"
-          }
-          else{
-           Write-Host "Symbols Exclusion file does not exist"
-           Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false"
+            Copy-Item -Path $symbolExclusionfile -Destination "$(Build.StagingDirectory)/ReleaseConfigs"
           }
 
     - template: /eng/common/core-templates/steps/publish-build-artifacts.yml
       parameters:
         is1ESPipeline: ${{ parameters.is1ESPipeline }}
         args:
-          displayName: Publish SymbolPublishingExclusionsFile Artifact
-          condition: eq(variables['SymbolExclusionFile'], 'true') 
-          pathToPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+          displayName: Publish ReleaseConfigs Artifact
+          pathToPublish: '$(Build.StagingDirectory)/ReleaseConfigs'
           publishLocation: Container
           artifactName: ReleaseConfigs
 
@@ -162,9 +143,10 @@ jobs:
           scriptType: ps
           scriptLocation: scriptPath
           scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-          arguments: -BuildId $(BARBuildId) 
+          arguments: >
+            -BuildId $(BARBuildId)
             -PublishingInfraVersion 3
-            -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+            -AzdoToken '$(System.AccessToken)'
             -WaitPublishingFinish true
             -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
             -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/job/source-index-stage1.yml

@@ -34,10 +34,12 @@ jobs:
     pool:
       ${{ if eq(variables['System.TeamProject'], 'public') }}:
         name: $(DncEngPublicBuildPool)
-        image: windows.vs2022.amd64.open
+        image: 1es-windows-2022-open
+        os: windows
       ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         name: $(DncEngInternalBuildPool)
-        image: windows.vs2022.amd64
+        image: 1es-windows-2022
+        os: windows
 
   steps:
   - ${{ if eq(parameters.is1ESPipeline, '') }}:

eng/common/core-templates/post-build/post-build.yml

@@ -307,9 +307,10 @@ stages:
             scriptType: ps
             scriptLocation: scriptPath
             scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1
-            arguments: -BuildId $(BARBuildId) 
+            arguments: >
+              -BuildId $(BARBuildId)
               -PublishingInfraVersion ${{ parameters.publishingInfraVersion }}
-              -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)'
+              -AzdoToken '$(System.AccessToken)'
               -WaitPublishingFinish true
               -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}'
               -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}'

eng/common/core-templates/steps/get-federated-access-token.yml

@@ -3,17 +3,31 @@ parameters:
   type: string
 - name: outputVariableName
   type: string
+- name: is1ESPipeline
+  type: boolean
+- name: stepName
+  type: string
+  default: 'getFederatedAccessToken'
+- name: condition
+  type: string
+  default: ''
 # Resource to get a token for. Common values include:
 # - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps
 # - 'https://storage.azure.com/' for storage
 # Defaults to Azure DevOps
 - name: resource
   type: string
   default: '499b84ac-1321-427f-aa17-267ca6975798'
+- name: isStepOutputVariable
+  type: boolean
+  default: false
 
 steps:
 - task: AzureCLI@2
   displayName: 'Getting federated access token for feeds'
+  name: ${{ parameters.stepName }}
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
   inputs:
     azureSubscription: ${{ parameters.federatedServiceConnection }}
     scriptType: 'pscore'
@@ -25,4 +39,4 @@ steps:
         exit 1
       }
       Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value"
-      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$accessToken"
+      Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken"

eng/common/core-templates/steps/publish-logs.yml

@@ -32,7 +32,6 @@ steps:
       '$(MaestroAccessToken)'
       '$(dn-bot-all-orgs-artifact-feeds-rw)'
       '$(akams-client-id)'
-      '$(akams-client-secret)'
       '$(microsoft-symbol-server-pat)'
       '$(symweb-symbol-server-pat)'
       '$(dn-bot-all-orgs-build-rw-code-rw)'

eng/common/cross/build-rootfs.sh

@@ -72,7 +72,7 @@ __AlpinePackages+=" krb5-dev"
 __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
 
-__FreeBSDBase="13.2-RELEASE"
+__FreeBSDBase="13.3-RELEASE"
 __FreeBSDPkg="1.17.0"
 __FreeBSDABI="13"
 __FreeBSDPackages="libunwind"
@@ -605,34 +605,34 @@ elif [[ "$__CodeName" == "illumos" ]]; then
     fi
     echo "Building binutils. Please wait.."
     if [[ "$__hasWget" == 1 ]]; then
-        wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+        wget -O- https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
     else
-        curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf -
+        curl -SL https://ftp.gnu.org/gnu/binutils/binutils-2.42.tar.xz | tar -xJf -
     fi
     mkdir build-binutils && cd build-binutils
-    ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
+    ../binutils-2.42/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir"
     make -j "$JOBS" && make install && cd ..
     echo "Building gcc. Please wait.."
     if [[ "$__hasWget" == 1 ]]; then
-        wget -O- https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+        wget -O- https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
     else
-        curl -SL https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf -
+        curl -SL https://ftp.gnu.org/gnu/gcc/gcc-13.3.0/gcc-13.3.0.tar.xz | tar -xJf -
     fi
     CFLAGS="-fPIC"
     CXXFLAGS="-fPIC"
     CXXFLAGS_FOR_TARGET="-fPIC"
     CFLAGS_FOR_TARGET="-fPIC"
     export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET
     mkdir build-gcc && cd build-gcc
-    ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
+    ../gcc-13.3.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.11" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as       \
         --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \
         --disable-libquadmath-support --disable-shared --enable-tls
     make -j "$JOBS" && make install && cd ..
     BaseUrl=https://pkgsrc.smartos.org
     if [[ "$__UseMirror" == 1 ]]; then
         BaseUrl=https://pkgsrc.smartos.skylime.net
     fi
-    BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All"
+    BaseUrl="$BaseUrl/packages/SmartOS/2019Q4/${__illumosArch}/All"
     echo "Downloading manifest"
     if [[ "$__hasWget" == 1 ]]; then
         wget "$BaseUrl"

eng/common/darc-init.ps1

@@ -1,6 +1,6 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
+    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20',
     $verbosity = 'minimal',
     $toolpath = $null
 )

eng/common/darc-init.sh

@@ -2,7 +2,7 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20'
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do

eng/common/internal/Tools.csproj

@@ -4,6 +4,7 @@
   <PropertyGroup>
     <TargetFramework>net472</TargetFramework>
     <AutomaticallyUseReferenceAssemblyPackages>false</AutomaticallyUseReferenceAssemblyPackages>
+    <BuildWithNetFrameworkHostedCompiler>false</BuildWithNetFrameworkHostedCompiler>
   </PropertyGroup>
   <ItemGroup>
     <!-- Clear references, the SDK may add some depending on UsuingToolXxx settings, but we only want to restore the following -->

eng/common/native/init-compiler.sh

@@ -19,19 +19,19 @@ case "$compiler" in
         # clangx.y or clang-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
 
-        if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
-            minorVersion=0;
-        fi
+        # LLVM based on v18 released in early 2024, with two releases per year
+        maxVersion="$((18 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 6)))"
         compiler=clang
         ;;
 
     gcc*|-gcc*|--gcc*)
         # gccx.y or gcc-x.y
         version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
         majorVersion="${version%%.*}"
-        [ -z "${version##*.*}" ] && minorVersion="${version#*.}"
+
+        # GCC based on v14 released in early 2024, with one release per year
+        maxVersion="$((14 + ((($(date +%Y) - 2024) * 12 + $(date +%-m) - 3) / 12)))"
         compiler=gcc
         ;;
 esac
@@ -49,12 +49,10 @@ check_version_exists() {
     desired_version=-1
 
     # Set up the environment to be used for building with the desired compiler.
-    if command -v "$compiler-$1.$2" > /dev/null; then
-        desired_version="-$1.$2"
-    elif command -v "$compiler$1$2" > /dev/null; then
-        desired_version="$1$2"
-    elif command -v "$compiler-$1$2" > /dev/null; then
-        desired_version="-$1$2"
+    if command -v "$compiler-$1" > /dev/null; then
+        desired_version="-$1"
+    elif command -v "$compiler$1" > /dev/null; then
+        desired_version="$1"
     fi
 
     echo "$desired_version"
@@ -75,7 +73,7 @@ set_compiler_version_from_CC() {
     fi
 
     # gcc and clang often display 3 part versions. However, gcc can show only 1 part in some environments.
-    IFS=. read -r majorVersion minorVersion _ <<EOF
+    IFS=. read -r majorVersion _ <<EOF
 $version
 EOF
 }
@@ -84,43 +82,29 @@ if [ -z "$CLR_CC" ]; then
 
     # Set default versions
     if [ -z "$majorVersion" ]; then
-        # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
-        if [ "$compiler" = "clang" ]; then versions="18 17 16 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
-        elif [ "$compiler" = "gcc" ]; then versions="14 13 12 11 10 9 8 7 6 5 4.9"; fi
-
-        for version in $versions; do
-            _major="${version%%.*}"
-            [ -z "${version##*.*}" ] && _minor="${version#*.}"
-            desired_version="$(check_version_exists "$_major" "$_minor")"
-            if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
+        minVersion=8
+        maxVersion="$((maxVersion + 1))" # +1 for headspace
+        i="$maxVersion"
+        while [ "$i" -ge $minVersion ]; do
+            desired_version="$(check_version_exists "$i")"
+            if [ "$desired_version" != "-1" ]; then majorVersion="$i"; break; fi
+            i=$((i - 1))
         done
 
         if [ -z "$majorVersion" ]; then
             if ! command -v "$compiler" > /dev/null; then
-                echo "Error: No usable version of $compiler found."
+                echo "Error: No compatible version of $compiler was found within the range of $minVersion to $maxVersion. Please upgrade your toolchain or specify the compiler explicitly using CLR_CC and CLR_CXX environment variables."
                 exit 1
             fi
 
             CC="$(command -v "$compiler" 2> /dev/null)"
             CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
             set_compiler_version_from_CC
-        else
-            if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ] && { [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; }; then
-                # If a major version was provided explicitly, and it was too old, find a newer compiler instead
-                if ! command -v "$compiler" > /dev/null; then
-                    echo "Error: Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
-                    exit 1
-                fi
-
-                CC="$(command -v "$compiler" 2> /dev/null)"
-                CXX="$(command -v "$cxxCompiler" 2> /dev/null)"
-                set_compiler_version_from_CC
-            fi
         fi
     else
-        desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
+        desired_version="$(check_version_exists "$majorVersion")"
         if [ "$desired_version" = "-1" ]; then
-            echo "Error: Could not find specific version of $compiler: $majorVersion $minorVersion."
+            echo "Error: Could not find specific version of $compiler: $majorVersion."
             exit 1
         fi
     fi

eng/common/post-build/publish-using-darc.ps1

@@ -42,6 +42,7 @@ try {
     --azdev-pat "$AzdoToken" `
     --bar-uri "$MaestroApiEndPoint" `
     --ci `
+    --verbose `
 	@optionalParams
 
   if ($LastExitCode -ne 0) {