From b5c434065c7de84b106e49c6a14ba924fce0ecad Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 25 Oct 2022 11:27:10 -0700 Subject: [PATCH 1/3] First, an arcade update --- eng/Version.Details.xml | 8 +-- eng/common/cross/arm/tizen-fetch.sh | 6 +-- eng/common/cross/arm64/tizen-fetch.sh | 8 +-- eng/common/cross/armel/tizen-fetch.sh | 8 +-- eng/common/cross/armel/tizen/tizen-dotnet.ks | 50 ------------------- eng/common/cross/build-android-rootfs.sh | 4 +- eng/common/cross/x86/tizen-fetch.sh | 6 +-- eng/common/templates/job/execute-sdl.yml | 3 +- eng/common/templates/job/onelocbuild.yml | 24 ++++----- .../templates/job/publish-build-assets.yml | 13 ++++- eng/common/templates/job/source-build.yml | 15 +++++- .../templates/job/source-index-stage1.yml | 5 +- eng/common/templates/jobs/jobs.yml | 9 ---- .../templates/post-build/post-build.yml | 10 ++-- .../templates/variables/pool-providers.yml | 48 ++++++++++++++++++ eng/common/tools.sh | 22 +++++--- global.json | 4 +- 17 files changed, 133 insertions(+), 110 deletions(-) delete mode 100644 eng/common/cross/armel/tizen/tizen-dotnet.ks create mode 100644 eng/common/templates/variables/pool-providers.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 9e44856cd7618..0cf9362e41a48 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,18 +13,18 @@ - + https://github.com/dotnet/arcade - 32f13f8a8af8085ca09fbf93513ac848582c4a41 + c5dd6a1da2e6d9b3423ab809fcda8af2927a408b https://github.com/dotnet/roslyn c1d8c6f043bc80425c6828455eb57f8a404759c6 - + https://github.com/dotnet/arcade - 32f13f8a8af8085ca09fbf93513ac848582c4a41 + c5dd6a1da2e6d9b3423ab809fcda8af2927a408b diff --git a/eng/common/cross/arm/tizen-fetch.sh b/eng/common/cross/arm/tizen-fetch.sh index eabd06c4afede..0adb0f11ec329 100644 --- a/eng/common/cross/arm/tizen-fetch.sh +++ b/eng/common/cross/arm/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/tizen +TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -155,7 +155,7 @@ fetch_tizen_pkgs() } Inform "Initialize arm base" -fetch_tizen_pkgs_init standard base +fetch_tizen_pkgs_init standard Tizen-Base Inform "fetch common packages" fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard unified +fetch_tizen_pkgs_init standard Tizen-Unified Inform "fetch corefx packages" fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/arm64/tizen-fetch.sh b/eng/common/cross/arm64/tizen-fetch.sh index 16d1301f21e4c..785bd85484e25 100644 --- a/eng/common/cross/arm64/tizen-fetch.sh +++ b/eng/common/cross/arm64/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/tizen/ +TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -154,8 +154,8 @@ fetch_tizen_pkgs() done } -Inform "Initialize arm base" -fetch_tizen_pkgs_init standard base +Inform "Initialize arm64 base" +fetch_tizen_pkgs_init standard Tizen-Base Inform "fetch common packages" fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard unified +fetch_tizen_pkgs_init standard Tizen-Unified Inform "fetch corefx packages" fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/armel/tizen-fetch.sh b/eng/common/cross/armel/tizen-fetch.sh index 64f0187e5aaa2..3d8f177a34444 100755 --- a/eng/common/cross/armel/tizen-fetch.sh +++ b/eng/common/cross/armel/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/tizen +TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -154,8 +154,8 @@ fetch_tizen_pkgs() done } -Inform "Initialize arm base" -fetch_tizen_pkgs_init standard base +Inform "Initialize armel base" +fetch_tizen_pkgs_init standard Tizen-Base Inform "fetch common packages" fetch_tizen_pkgs armv7l gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs armv7l libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard unified +fetch_tizen_pkgs_init standard Tizen-Unified Inform "fetch corefx packages" fetch_tizen_pkgs armv7l gssdp gssdp-devel tizen-release diff --git a/eng/common/cross/armel/tizen/tizen-dotnet.ks b/eng/common/cross/armel/tizen/tizen-dotnet.ks deleted file mode 100644 index 506d455bd4ff8..0000000000000 --- a/eng/common/cross/armel/tizen/tizen-dotnet.ks +++ /dev/null @@ -1,50 +0,0 @@ -lang en_US.UTF-8 -keyboard us -timezone --utc Asia/Seoul - -part / --fstype="ext4" --size=3500 --ondisk=mmcblk0 --label rootfs --fsoptions=defaults,noatime - -rootpw tizen -desktop --autologinuser=root -user --name root --groups audio,video --password 'tizen' - -repo --name=standard --baseurl=http://download.tizen.org/releases/milestone/tizen/unified/latest/repos/standard/packages/ --ssl_verify=no -repo --name=base --baseurl=http://download.tizen.org/releases/milestone/tizen/base/latest/repos/standard/packages/ --ssl_verify=no - -%packages -tar -gzip - -sed -grep -gawk -perl - -binutils -findutils -util-linux -lttng-ust -userspace-rcu -procps-ng -tzdata -ca-certificates - - -### Core FX -libicu -libunwind -iputils -zlib -krb5 -libcurl -libopenssl - -%end - -%post - -### Update /tmp privilege -chmod 777 /tmp -#################################### - -%end diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 42516bbeebc3f..f163fb9dae966 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ +for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do if [[ "$path" != "Filename:" ]]; then echo "Working on: $path" - wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" + wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir" fi done diff --git a/eng/common/cross/x86/tizen-fetch.sh b/eng/common/cross/x86/tizen-fetch.sh index fa5f88b7d6c0d..cf8d1ce4a8e55 100644 --- a/eng/common/cross/x86/tizen-fetch.sh +++ b/eng/common/cross/x86/tizen-fetch.sh @@ -51,7 +51,7 @@ if [ ! -d $TMPDIR ]; then mkdir -p $TMPDIR fi -TIZEN_URL=http://download.tizen.org/snapshots/tizen +TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen BUILD_XML=build.xml REPOMD_XML=repomd.xml PRIMARY_XML=primary.xml @@ -155,7 +155,7 @@ fetch_tizen_pkgs() } Inform "Initialize i686 base" -fetch_tizen_pkgs_init standard base +fetch_tizen_pkgs_init standard Tizen-Base Inform "fetch common packages" fetch_tizen_pkgs i686 gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils Inform "fetch coreclr packages" @@ -164,7 +164,7 @@ Inform "fetch corefx packages" fetch_tizen_pkgs i686 libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel Inform "Initialize standard unified" -fetch_tizen_pkgs_init standard unified +fetch_tizen_pkgs_init standard Tizen-Unified Inform "fetch corefx packages" fetch_tizen_pkgs i686 gssdp gssdp-devel tizen-release diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 65f87b40c66ae..7aabaa18017bf 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -46,6 +46,7 @@ jobs: - template: /eng/common/templates/variables/sdl-variables.yml - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} + - template: /eng/common/templates/variables/pool-providers.yml pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: @@ -53,7 +54,7 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - checkout: self diff --git a/eng/common/templates/job/onelocbuild.yml b/eng/common/templates/job/onelocbuild.yml index 10e75fc76c476..c2cabcf9e0686 100644 --- a/eng/common/templates/job/onelocbuild.yml +++ b/eng/common/templates/job/onelocbuild.yml @@ -31,6 +31,17 @@ jobs: displayName: OneLocBuild${{ parameters.JobNameSuffix }} + variables: + - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat + - name: _GenerateLocProjectArguments + value: -SourcesDirectory ${{ parameters.SourcesDirectory }} + -LanguageSet "${{ parameters.LanguageSet }}" + -CreateNeutralXlfs + - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: + - name: _GenerateLocProjectArguments + value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson + - template: /eng/common/templates/variables/pool-providers.yml + ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} ${{ if eq(parameters.pool, '') }}: @@ -41,20 +52,9 @@ jobs: demands: Cmd # If it's not devdiv, it's dnceng ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - steps: - task: Powershell@2 inputs: diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 1cbb6a0c56003..1f1b78f2d4598 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -40,9 +40,8 @@ jobs: ${{ else }}: displayName: Publish to Build Asset Registry - pool: ${{ parameters.pool }} - variables: + - template: /eng/common/templates/variables/pool-providers.yml - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: Publish-Build-Assets - group: AzureDevOps-Artifact-Feeds-Pats @@ -51,6 +50,16 @@ jobs: - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - template: /eng/common/templates/post-build/common-variables.yml + pool: + # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + name: VSEngSS-MicroBuild2022-1ES + demands: Cmd + # If it's not devdiv, it's dnceng + ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2019.amd64 + steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: DownloadBuildArtifacts@0 diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 88f6f75a622dc..e40bf35203b9e 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -44,13 +44,24 @@ jobs: ${{ if eq(parameters.platform.pool, '') }}: # The default VM host AzDO pool. This should be capable of running Docker containers: almost all # source-build builds run in Docker, including the default managed platform. + # /eng/common/templates/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: + # Main environments + ${{ if and(eq(variables['System.TeamProject'], 'public'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: name: NetCore-Public demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), true)) }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + + # Servicing build environments + ${{ if and(eq(variables['System.TeamProject'], 'public'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: + name: NetCore-Svc-Public + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release')) }}: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index 21fd12276b658..09c506d11855d 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -22,16 +22,17 @@ jobs: value: ${{ parameters.binlogPath }} - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: source-dot-net stage1 variables + - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: pool: ${{ parameters.pool }} ${{ if eq(parameters.pool, '') }}: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: NetCore-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2019.amd64.open ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 64e5929f22161..289bb2396ce83 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -88,15 +88,6 @@ jobs: - ${{ job.job }} - ${{ if eq(parameters.enableSourceBuild, true) }}: - Source_Build_Complete - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: VSEngSS-MicroBuild2022-1ES - demands: Cmd - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals windows.vs2019.amd64 runAsPublic: ${{ parameters.runAsPublic }} publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 258ed2d110810..91251d08973c6 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -95,6 +95,7 @@ stages: displayName: Validate Build Assets variables: - template: common-variables.yml + - template: /eng/common/templates/variables/pool-providers.yml jobs: - job: displayName: NuGet Validation @@ -106,7 +107,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: @@ -143,7 +144,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -203,7 +204,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml @@ -251,6 +252,7 @@ stages: displayName: Publish using Darc variables: - template: common-variables.yml + - template: /eng/common/templates/variables/pool-providers.yml jobs: - job: displayName: Publish Using Darc @@ -262,7 +264,7 @@ stages: demands: Cmd # If it's not devdiv, it's dnceng ${{ else }}: - name: NetCore1ESPool-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2019.amd64 steps: - template: setup-maestro-vars.yml diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml new file mode 100644 index 0000000000000..a7b943c2fa4c1 --- /dev/null +++ b/eng/common/templates/variables/pool-providers.yml @@ -0,0 +1,48 @@ +# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, +# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. + +# Motivation: +# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS +# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing +# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. +# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services +# team needs to move resources around and create new and potentially differently-named pools. Using this template +# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. + +# How to use: +# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). +# If we find alternate naming conventions in broad usage it can be added to the condition below. +# +# First, import the template in an arcade-ified repo to pick up the variables, e.g.: +# +# variables: +# - template: eng/common/templates/variables/pool-providers.yml +# +# ... then anywhere specifying the pool provider use the runtime variables, +# $(DncEngInternalBuildPool) and $ (DncEngPublicBuildPool), e.g.: +# +# pool: +# name: $(DncEngInternalBuildPool) +# demands: ImageOverride -equals windows.vs2019.amd64 + +variables: +# Coalesce the target and source branches so we know when a PR targets a release branch +# If these variables are somehow missing, fall back to main (tends to have more capacity) +- name: BranchNameForPoolSelection + value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + +# Any new -Svc alternative pools should have variables added here to allow for splitting work + +# Main branch pools +- ${{ if ne(contains(variables['BranchNameForPoolSelection'], 'release'), true) }}: + - name: DncEngPublicBuildPool + value: NetCore-Public + - name: DncEngInternalBuildPool + value: NetCore1ESPool-Internal + +# Release branch pools +- ${{ if contains(variables['BranchNameForPoolSelection'], 'release') }}: + - name: DncEngPublicBuildPool + value: NetCore-Svc-Public + - name: DncEngInternalBuildPool + value: NetCore1ESPool-Svc-Internal \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh index c110d0ed410c5..6586eab458e55 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -417,12 +417,11 @@ function MSBuild { Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" - export NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY=true - export NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT=6 - export NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS=1000 - Write-PipelineSetVariable -name "NUGET_ENABLE_EXPERIMENTAL_HTTP_RETRY" -value "true" - Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_MAX_NETWORK_TRY_COUNT" -value "6" - Write-PipelineSetVariable -name "NUGET_EXPERIMENTAL_NETWORK_RETRY_DELAY_MILLISECONDS" -value "1000" + # https://github.com/dotnet/arcade/issues/11369 - disable new MSBuild server feature on linux + # This feature is new and can result in build failures from connection timeout errors. + export DOTNET_CLI_DO_NOT_USE_MSBUILD_SERVER=1 + Write-PipelineSetVariable -name "DOTNET_CLI_DO_NOT_USE_MSBUILD_SERVER" -value "1" + fi local toolset_dir="${_InitializeToolset%/*}" @@ -493,6 +492,17 @@ function MSBuild-Core { RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } +function GetDarc { + darc_path="$temp_dir/darc" + version="$1" + + if [[ -n "$version" ]]; then + version="--darcversion $version" + fi + + "$eng_root/common/darc-init.sh" --toolpath "$darc_path" $version +} + ResolvePath "${BASH_SOURCE[0]}" _script_dir=`dirname "$_ResolvePath"` diff --git a/global.json b/global.json index 3ca774c19c257..cdcd9c978161e 100644 --- a/global.json +++ b/global.json @@ -17,7 +17,7 @@ "xcopy-msbuild": "17.2.1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22513.2", - "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.22513.2" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22524.5", + "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.22524.5" } } From 803ab64a9e27099631531fb2eaecf8a2e4c9e37b Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Tue, 25 Oct 2022 16:58:11 -0700 Subject: [PATCH 2/3] Use the new pool-providers.yml to automatically pick up pool to use --- azure-pipelines-integration-lsp.yml | 4 +++- azure-pipelines-integration.yml | 4 +++- azure-pipelines-official.yml | 7 ++++--- azure-pipelines-richnav.yml | 7 ++++--- azure-pipelines.yml | 14 +++++++++++--- eng/pipelines/build-unix-job.yml | 5 ++++- eng/pipelines/build-windows-job.yml | 5 ++++- eng/pipelines/test-unix-job-single-machine.yml | 5 ++++- eng/pipelines/test-windows-job-single-machine.yml | 5 ++++- 9 files changed, 41 insertions(+), 15 deletions(-) diff --git a/azure-pipelines-integration-lsp.yml b/azure-pipelines-integration-lsp.yml index 48dc9c4c0b5b8..3154e0e74c01a 100644 --- a/azure-pipelines-integration-lsp.yml +++ b/azure-pipelines-integration-lsp.yml @@ -36,8 +36,10 @@ pr: jobs: - job: VS_Integration_LSP + variables: + - template: /eng/common/templates/variables/pool-providers.yml pool: - name: NetCore-Svc-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2022.amd64.open timeoutInMinutes: 135 diff --git a/azure-pipelines-integration.yml b/azure-pipelines-integration.yml index 8d428d73692b7..258332644df5f 100644 --- a/azure-pipelines-integration.yml +++ b/azure-pipelines-integration.yml @@ -32,8 +32,10 @@ pr: jobs: - job: VS_Integration + variables: + - template: /eng/common/templates/variables/pool-providers.yml pool: - name: NetCore-Svc-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2022.amd64.open strategy: maxParallel: 4 diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index 31665d7e5a3c4..2e812b1a0b587 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -72,13 +72,14 @@ variables: value: '[Auto Insertion]' - name: Insertion.TitleSuffix value: '' + - template: /eng/common/templates/variables/pool-providers.yml stages: - stage: build displayName: Build and Test pool: - name: NetCore1ESPool-Svc-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022.amd64 jobs: @@ -308,7 +309,7 @@ stages: dependsOn: - OfficialBuild pool: - name: NetCore1ESPool-Svc-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022.amd64 - stage: insert @@ -320,7 +321,7 @@ stages: - job: insert displayName: Insert to VS pool: - name: NetCore1ESPool-Svc-Internal + name: $(DncEngInternalBuildPool) demands: ImageOverride -equals windows.vs2022.amd64 steps: - download: current diff --git a/azure-pipelines-richnav.yml b/azure-pipelines-richnav.yml index bfb97c6f25174..7b8a28e161383 100644 --- a/azure-pipelines-richnav.yml +++ b/azure-pipelines-richnav.yml @@ -18,11 +18,12 @@ pr: none jobs: - job: RichCodeNav_Indexing - pool: - name: NetCore-Svc-Public - demands: ImageOverride -equals windows.vs2022.amd64.open variables: EnableRichCodeNavigation: true + - template: /eng/common/templates/variables/pool-providers.yml + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open timeoutInMinutes: 200 steps: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d1a522d501568..21fca1bc6e9af 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,6 +14,9 @@ pr: - features/* - demos/* +variables: +- template: /eng/common/templates/variables/pool-providers.yml + # Windows Build and Test Jobs jobs: - template: eng/pipelines/build-windows-job.yml @@ -21,6 +24,7 @@ jobs: jobName: Build_Windows_Debug testArtifactName: Transport_Artifacts_Windows_Debug configuration: Debug + poolName: $(DncEngPublicBuildPool) queueName: windows.vs2022.amd64.open - template: eng/pipelines/build-windows-job.yml @@ -28,6 +32,7 @@ jobs: jobName: Build_Windows_Release testArtifactName: Transport_Artifacts_Windows_Release configuration: Release + poolName: $(DncEngPublicBuildPool) queueName: windows.vs2022.amd64.open - template: eng/pipelines/test-windows-job.yml @@ -65,6 +70,7 @@ jobs: testArtifactName: Transport_Artifacts_Windows_Debug configuration: Debug testArguments: -testCoreClr + poolName: $(DncEngPublicBuildPool) - template: eng/pipelines/test-windows-job.yml parameters: @@ -117,6 +123,7 @@ jobs: jobName: Build_Unix_Debug testArtifactName: Transport_Artifacts_Unix_Debug configuration: Debug + poolName: $(DncEngPublicBuildPool) queueName: Build.Ubuntu.1804.Amd64.Open - template: eng/pipelines/test-unix-job.yml @@ -136,6 +143,7 @@ jobs: testArtifactName: Transport_Artifacts_Unix_Debug configuration: Debug testArguments: --testCoreClr + poolName: $(DncEngPublicBuildPool) queueName: Build.Ubuntu.1804.Amd64.Open - template: eng/pipelines/test-unix-job.yml @@ -153,7 +161,7 @@ jobs: - job: Correctness_Determinism pool: - name: NetCore-Svc-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2022.amd64.open timeoutInMinutes: 90 steps: @@ -169,7 +177,7 @@ jobs: - job: Correctness_Build pool: - name: NetCore-Svc-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2022.amd64.open timeoutInMinutes: 90 steps: @@ -201,7 +209,7 @@ jobs: - job: Correctness_Rebuild pool: - name: NetCore-Svc-Public + name: $(DncEngPublicBuildPool) demands: ImageOverride -equals windows.vs2022.amd64.open timeoutInMinutes: 90 steps: diff --git a/eng/pipelines/build-unix-job.yml b/eng/pipelines/build-unix-job.yml index 4d295295984d6..b198156e9bad0 100644 --- a/eng/pipelines/build-unix-job.yml +++ b/eng/pipelines/build-unix-job.yml @@ -9,6 +9,9 @@ parameters: - name: configuration type: string default: 'Debug' +- name: poolName + type: string + default: 'NetCore-Svc-Public' - name: queueName type: string default: '' @@ -20,7 +23,7 @@ jobs: - job: ${{ parameters.jobName }} pool: ${{ if ne(parameters.queueName, '') }}: - name: NetCore-Svc-Public + name: ${{ parameters.poolName }} demands: ImageOverride -equals ${{ parameters.queueName }} ${{ if ne(parameters.vmImageName, '') }}: diff --git a/eng/pipelines/build-windows-job.yml b/eng/pipelines/build-windows-job.yml index eaba27ebaf7b1..f1ca147aa08c3 100644 --- a/eng/pipelines/build-windows-job.yml +++ b/eng/pipelines/build-windows-job.yml @@ -9,6 +9,9 @@ parameters: - name: configuration type: string default: 'Debug' +- name: poolName + type: string + default: 'NetCore-Svc-Public' - name: queueName type: string default: '' @@ -23,7 +26,7 @@ jobs: - job: ${{ parameters.jobName }} pool: ${{ if ne(parameters.queueName, '') }}: - name: NetCore-Svc-Public + name: ${{ parameters.poolName }} demands: ImageOverride -equals ${{ parameters.queueName }} ${{ if ne(parameters.vmImageName, '') }}: diff --git a/eng/pipelines/test-unix-job-single-machine.yml b/eng/pipelines/test-unix-job-single-machine.yml index 5bbdae6a587a9..14a3cb6a8807e 100644 --- a/eng/pipelines/test-unix-job-single-machine.yml +++ b/eng/pipelines/test-unix-job-single-machine.yml @@ -18,6 +18,9 @@ parameters: - name: testArguments type: string default: '' +- name: poolName + type: string + default: 'NetCore-Svc-Public' - name: queueName type: string default: '' @@ -30,7 +33,7 @@ jobs: dependsOn: ${{ parameters.buildJobName }} pool: ${{ if ne(parameters.queueName, '') }}: - name: NetCore-Svc-Public + name: ${{ parameters.poolName }} demands: ImageOverride -equals ${{ parameters.queueName }} ${{ if ne(parameters.vmImageName, '') }}: diff --git a/eng/pipelines/test-windows-job-single-machine.yml b/eng/pipelines/test-windows-job-single-machine.yml index 7fc25575c8bcc..1a19d8a3aae56 100644 --- a/eng/pipelines/test-windows-job-single-machine.yml +++ b/eng/pipelines/test-windows-job-single-machine.yml @@ -18,6 +18,9 @@ parameters: - name: testArguments type: string default: '' +- name: poolName + type: string + default: 'NetCore-Svc-Public' - name: queueName type: string default: 'windows.vs2022.amd64.open' @@ -26,7 +29,7 @@ jobs: - job: ${{ parameters.jobName }} dependsOn: ${{ parameters.buildJobName }} pool: - name: NetCore-Svc-Public + name: ${{ parameters.poolName }} demands: ImageOverride -equals ${{ parameters.queueName }} timeoutInMinutes: 120 variables: From 93689f7c5c5041c42c3b37a5e3a4ab2704021cd6 Mon Sep 17 00:00:00 2001 From: Matt Galbraith Date: Wed, 26 Oct 2022 10:55:49 -0700 Subject: [PATCH 3/3] Testing: Is system.pullRequest.targetBranch case sensitive system.pullRequest.targetBranch didn't seem to get picked up here, these variables may be case sensitive. --- eng/common/templates/variables/pool-providers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/templates/variables/pool-providers.yml b/eng/common/templates/variables/pool-providers.yml index a7b943c2fa4c1..d1f3ff10ed1f3 100644 --- a/eng/common/templates/variables/pool-providers.yml +++ b/eng/common/templates/variables/pool-providers.yml @@ -29,7 +29,7 @@ variables: # Coalesce the target and source branches so we know when a PR targets a release branch # If these variables are somehow missing, fall back to main (tends to have more capacity) - name: BranchNameForPoolSelection - value: ${{ coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} + value: ${{ coalesce(variables['system.pullRequest.targetBranch'], variables['Build.SourceBranch'], 'refs/heads/main') }} # Any new -Svc alternative pools should have variables added here to allow for splitting work @@ -45,4 +45,4 @@ variables: - name: DncEngPublicBuildPool value: NetCore-Svc-Public - name: DncEngInternalBuildPool - value: NetCore1ESPool-Svc-Internal \ No newline at end of file + value: NetCore1ESPool-Svc-Internal