diff --git a/.vsts-pr.yaml b/.vsts-pr.yaml index 9e882363e1..ab86e2cdca 100644 --- a/.vsts-pr.yaml +++ b/.vsts-pr.yaml @@ -1,3 +1,7 @@ +variables: + - name: DOTNET_ROOT + value: $(Build.SourcesDirectory)\.dotnet + jobs: - job: Windows pool: @@ -44,7 +48,7 @@ jobs: - task: PublishTestResults@2 inputs: testRunner: 'xUnit' - testResultsFiles: '**/*.xml' + testResultsFiles: '**/*.xml' searchFolder: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' configuration: '$(_configuration)' publishRunAttachments: true diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 70d8d60b69..0d23aab716 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - bd2a2b09716ddb54cb1e40f087beaaeeef859118 + f6192d1e284a08ac05041d05fa6e60dec74b24f5 diff --git a/eng/Versions.props b/eng/Versions.props index 7ffbd0a030..eae99b8596 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,6 +11,10 @@ true true 3.7.0-4.20351.2 + + + 16.1.1 + - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - - - - - - - - - - https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - https://dotnetfeed.blob.core.windows.net/arcade-validation/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json - https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json - https://dotnetfeed.blob.core.windows.net/nuget-nugetclient/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-entityframework6/index.json - https://dotnetfeed.blob.core.windows.net/aspnet-blazor/index.json - - - - - - - - - - - - diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj deleted file mode 100644 index 311e2bbe0f..0000000000 --- a/eng/common/PublishToSymbolServers.proj +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - netcoreapp2.1 - - - - - - - - - - - - - - - - 3650 - true - true - true - false - - - - - - - - - - - - - - - - - diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index a5a1e711d7..dc22178125 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -83,7 +83,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." @@ -123,19 +123,21 @@ if ($creds -eq $null) { $doc.DocumentElement.AppendChild($creds) | Out-Null } -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password +$userName = "dn-bot" -$dotnet3Source = $sources.SelectSingleNode("add[@key='dotnet3']") -if ($dotnet3Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password -} +# Insert credential nodes for Maestro's private feeds +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +} + +$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") +if ($dotnet5Source -ne $null) { + AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 7d6fef27fe..f33e37711e 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -79,53 +79,52 @@ fi PackageSources=() -# Ensure dotnet3-internal and dotnet3-internal-transport are in the packageSources if the public dotnet3 feeds are present -grep -i "" $ConfigFile + grep -i "" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3-internal') + PackageSources+=('dotnet3.1-internal') - grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3-internal-transport to the packageSources." + echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3-internal-transport') + PackageSources+=('dotnet3.1-internal-transport') fi -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal') + PackageSources+=('dotnet5-internal') - grep -i "" $ConfigFile + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." + echo "Adding dotnet5-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceTemplate="${TB}" sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') + PackageSources+=('dotnet5-internal-transport') fi # I want things split line by line diff --git a/eng/common/SigningValidation.proj b/eng/common/SigningValidation.proj deleted file mode 100644 index 3d0ac80af3..0000000000 --- a/eng/common/SigningValidation.proj +++ /dev/null @@ -1,83 +0,0 @@ - - - - - - netcoreapp2.1 - - - - - - - - $(NuGetPackageRoot)Microsoft.DotNet.SignCheck\$(SignCheckVersion)\tools\Microsoft.DotNet.SignCheck.exe - - $(PackageBasePath) - signcheck.log - signcheck.errors.log - signcheck.exclusions.txt - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/eng/common/SourceLinkValidation.ps1 b/eng/common/SourceLinkValidation.ps1 deleted file mode 100644 index 0e4c349e00..0000000000 --- a/eng/common/SourceLinkValidation.ps1 +++ /dev/null @@ -1,195 +0,0 @@ -param( - [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored - [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages -) - -function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { - Write-PipelineTelemetryError -Category "Build" -Message "GHRepoName should be in the format /" - $global:LASTEXITCODE = 1 - return - } - - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-PipelineTelemetryError -Category "Build" -Message "GHCommit should be a 40 chars hexadecimal string" - $global:LASTEXITCODE = 1 - return - } - - $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree - - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) - - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 - } - } - } - catch { - Write-PipelineTelemetryError -Category "Build" -Message "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - $global:LASTEXITCODE = 1 - return - } - - if (Test-Path $ExtractPath) { - Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue - } - - # Process each NuGet package in parallel - $Jobs = @() - Get-ChildItem "$InputPath\*.symbols.nupkg" | - ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName - } - - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job - } -} - -try { - . $PSScriptRoot\pipeline-logging-functions.ps1 - - # Cache/HashMap (File -> Exist flag) used to consult whether a file exist - # in the repository at a specific commit point. This is populated by inserting - # all files present in the repo at a specific commit point. - $global:RepoFiles = @{} - - $ValidatePackage = { - param( - [string] $PackagePath # Full path to a Symbols.NuGet package - ) - - # Ensure input file exist - if (!(Test-Path $PackagePath)) { - throw "Input file does not exist: $PackagePath" - } - - # Extensions for which we'll look for SourceLink information - # For now we'll only care about Portable & Embedded PDBs - $RelevantExtensions = @(".dll", ".exe", ".pdb") - - Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " - - $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) - $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId - $FailedFiles = 0 - - Add-Type -AssemblyName System.IO.Compression.FileSystem - - [System.IO.Directory]::CreateDirectory($ExtractPath); - - $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) - - $zip.Entries | - Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | - ForEach-Object { - $FileName = $_.FullName - $Extension = [System.IO.Path]::GetExtension($_.Name) - $FakeName = -Join((New-Guid), $Extension) - $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName - - # We ignore resource DLLs - if ($FileName.EndsWith(".resources.dll")) { - return - } - - [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) - - $ValidateFile = { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $RealPath, - [ref] $FailedFiles - ) - - # Makes easier to reference `sourcelink cli` - Push-Location $using:SourceLinkToolPath - - $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String - - if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { - $NumFailedLinks = 0 - - # We only care about Http addresses - $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches - - if ($Matches.Count -ne 0) { - $Matches.Value | - ForEach-Object { - $Link = $_ - $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") - $FilePath = $Link.Replace($CommitUrl, "") - $Status = 200 - $Cache = $using:RepoFiles - - if ( !($Cache.ContainsKey($FilePath)) ) { - try { - $Uri = $Link -as [System.URI] - - # Only GitHub links are valid - if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { - $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode - } - else { - $Status = 0 - } - } - catch { - $Status = 0 - } - } - - if ($Status -ne 200) { - if ($NumFailedLinks -eq 0) { - if ($FailedFiles.Value -eq 0) { - Write-Host - } - - Write-Host "`tFile $RealPath has broken links:" - } - - Write-Host "`t`tFailed to retrieve $Link" - - $NumFailedLinks++ - } - } - } - - if ($NumFailedLinks -ne 0) { - $FailedFiles.value++ - $global:LASTEXITCODE = 1 - } - } - - Pop-Location - } - - &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) - } - - $zip.Dispose() - - if ($FailedFiles -eq 0) { - Write-Host "Passed." - } - } - - Measure-Command { ValidateSourceLinkLinks } -} -catch { - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'SourceLink' -Message $_ - ExitWithExitCode 1 -} - - diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 88814514d8..67ee6d28d3 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -20,12 +20,18 @@ Param( [switch] $publish, [switch] $clean, [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, [switch] $ci, [switch] $prepareMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) +# Unset 'Platform' environment variable to avoid unwanted collision in InstallDotNetCore.targets file +# some computer has this env var defined (e.g. Some HP) +if($env:Platform) { + $env:Platform="" +} function Print-Usage() { Write-Host "Common settings:" Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" @@ -53,6 +59,7 @@ function Print-Usage() { Write-Host "Advanced settings:" Write-Host " -projects Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)" Write-Host " -ci Set when running on CI server" + Write-Host " -excludeCIBinarylog Don't output binary log (short: -nobl)" Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." @@ -129,7 +136,9 @@ try { } if ($ci) { - $binaryLog = $true + if (-not $excludeCIBinarylog) { + $binaryLog = $true + } $nodeReuse = $false } diff --git a/eng/common/build.sh b/eng/common/build.sh index 36f9aa0462..6d7c5a1f69 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -32,6 +32,7 @@ usage() echo "Advanced settings:" echo " --projects Project or solution file(s) to build" echo " --ci Set when running on CI server" + echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" echo " --prepareMachine Prepare machine for CI run, clean up processes after build" echo " --nodeReuse Sets nodereuse msbuild parameter ('true' or 'false')" echo " --warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" @@ -68,6 +69,7 @@ clean=false warn_as_error=true node_reuse=true binary_log=false +exclude_ci_binary_log=false pipelines_log=false projects='' @@ -98,6 +100,9 @@ while [[ $# > 0 ]]; do -binarylog|-bl) binary_log=true ;; + -excludeCIBinarylog|-nobl) + exclude_ci_binary_log=true + ;; -pipelineslog|-pl) pipelines_log=true ;; @@ -156,8 +161,10 @@ done if [[ "$ci" == true ]]; then pipelines_log=true - binary_log=true node_reuse=false + if [[ "$exclude_ci_binary_log" == false ]]; then + binary_log=true + fi fi . "$scriptroot/tools.sh" diff --git a/eng/common/cross/android/arm/toolchain.cmake b/eng/common/cross/android/arm/toolchain.cmake deleted file mode 100644 index a7e1c73501..0000000000 --- a/eng/common/cross/android/arm/toolchain.cmake +++ /dev/null @@ -1,41 +0,0 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR arm) - -## Specify the toolchain -set(TOOLCHAIN "arm-linux-androideabi") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) -add_compile_options(-mfloat-abi=soft) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/) -include_directories(SYSTEM ${CROSS_NDK_TOOLCHAIN}/include/c++/4.9.x/arm-linux-androideabi/) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/android/arm64/toolchain.cmake b/eng/common/cross/android/arm64/toolchain.cmake deleted file mode 100644 index 29415899c1..0000000000 --- a/eng/common/cross/android/arm64/toolchain.cmake +++ /dev/null @@ -1,42 +0,0 @@ -set(CROSS_NDK_TOOLCHAIN $ENV{ROOTFS_DIR}/../) -set(CROSS_ROOTFS ${CROSS_NDK_TOOLCHAIN}/sysroot) -set(CLR_CMAKE_PLATFORM_ANDROID "Android") - -set(CMAKE_SYSTEM_NAME Linux) -set(CMAKE_SYSTEM_VERSION 1) -set(CMAKE_SYSTEM_PROCESSOR aarch64) - -## Specify the toolchain -set(TOOLCHAIN "aarch64-linux-android") -set(CMAKE_PREFIX_PATH ${CROSS_NDK_TOOLCHAIN}) -set(TOOLCHAIN_PREFIX ${TOOLCHAIN}-) - -find_program(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}clang++) -find_program(CMAKE_ASM_COMPILER ${TOOLCHAIN_PREFIX}clang) -find_program(CMAKE_AR ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_LD ${TOOLCHAIN_PREFIX}ar) -find_program(CMAKE_OBJCOPY ${TOOLCHAIN_PREFIX}objcopy) -find_program(CMAKE_OBJDUMP ${TOOLCHAIN_PREFIX}objdump) - -add_compile_options(--sysroot=${CROSS_ROOTFS}) -add_compile_options(-fPIE) - -## Needed for Android or bionic specific conditionals -add_compile_options(-D__ANDROID__) -add_compile_options(-D__BIONIC__) - -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -B ${CROSS_ROOTFS}/usr/lib/gcc/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -L${CROSS_ROOTFS}/lib/${TOOLCHAIN}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} --sysroot=${CROSS_ROOTFS}") -set(CROSS_LINK_FLAGS "${CROSS_LINK_FLAGS} -fPIE -pie") - -set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) -set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${CROSS_LINK_FLAGS}" CACHE STRING "" FORCE) - -set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}") -set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) -set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) -set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) diff --git a/eng/common/cross/arm64/tizen-build-rootfs.sh b/eng/common/cross/arm64/tizen-build-rootfs.sh new file mode 100644 index 0000000000..13bfddb5e2 --- /dev/null +++ b/eng/common/cross/arm64/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm64 ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen/ +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs aarch64 gcc glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel +Inform "fetch coreclr packages" +fetch_tizen_pkgs aarch64 lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 libcom_err libcom_err-devel zlib zlib-devel libopenssl libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs aarch64 gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm64/tizen/tizen.patch b/eng/common/cross/arm64/tizen/tizen.patch new file mode 100644 index 0000000000..af7c8be059 --- /dev/null +++ b/eng/common/cross/arm64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-littleaarch64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-aarch64.so.1 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-aarch64.so.1 ) ) diff --git a/eng/common/cross/armel/tizen-build-rootfs.sh b/eng/common/cross/armel/tizen-build-rootfs.sh index 87c48e78fb..25a0efa0bd 100755 --- a/eng/common/cross/armel/tizen-build-rootfs.sh +++ b/eng/common/cross/armel/tizen-build-rootfs.sh @@ -9,13 +9,6 @@ if [[ -z "$ROOTFS_DIR" ]]; then exit 1; fi -# Clean-up (TODO-Cleanup: We may already delete $ROOTFS_DIR at ./cross/build-rootfs.sh.) -# hk0110 -if [ -d "$ROOTFS_DIR" ]; then - umount $ROOTFS_DIR/* - rm -rf $ROOTFS_DIR -fi - TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp mkdir -p $TIZEN_TMP_DIR diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index adceda877a..e7f12edb56 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -e -__NDK_Version=r14 +__NDK_Version=r21 usage() { @@ -16,11 +16,11 @@ usage() echo. echo "By default, the NDK will be downloaded into the cross/android-rootfs/android-ndk-$__NDK_Version directory. If you already have an NDK installation," echo "you can set the NDK_DIR environment variable to have this script use that installation of the NDK." - echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.21-arm64. This file is to replace '/etc/os-release', which is not available for Android." + echo "By default, this script will generate a file, android_platform, in the root of the ROOTFS_DIR directory that contains the RID for the supported and tested Android build: android.28-arm64. This file is to replace '/etc/os-release', which is not available for Android." exit 1 } -__ApiLevel=21 # The minimum platform for arm64 is API level 21 +__ApiLevel=28 # The minimum platform for arm64 is API level 21 but the minimum version that support glob(3) is 28. See $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/glob.h __BuildArch=arm64 __AndroidArch=aarch64 __AndroidToolchain=aarch64-linux-android @@ -53,13 +53,20 @@ for i in "$@" done # Obtain the location of the bash script to figure out where the root of the repo is. -__CrossDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +__ScriptBaseDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -__Android_Cross_Dir="$__CrossDir/android-rootfs" -__NDK_Dir="$__Android_Cross_Dir/android-ndk-$__NDK_Version" -__libunwind_Dir="$__Android_Cross_Dir/libunwind" -__lldb_Dir="$__Android_Cross_Dir/lldb" -__ToolchainDir="$__Android_Cross_Dir/toolchain/$__BuildArch" +__CrossDir="$__ScriptBaseDir/../../../.tools/android-rootfs" + +if [[ ! -f "$__CrossDir" ]]; then + mkdir -p "$__CrossDir" +fi + +# Resolve absolute path to avoid `../` in build logs +__CrossDir="$( cd "$__CrossDir" && pwd )" + +__NDK_Dir="$__CrossDir/android-ndk-$__NDK_Version" +__lldb_Dir="$__CrossDir/lldb" +__ToolchainDir="$__CrossDir/android-ndk-$__NDK_Version" if [[ -n "$TOOLCHAIN_DIR" ]]; then __ToolchainDir=$TOOLCHAIN_DIR @@ -78,60 +85,47 @@ echo "Target Toolchain location: $__ToolchainDir" if [ ! -d $__NDK_Dir ]; then echo Downloading the NDK into $__NDK_Dir mkdir -p $__NDK_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__Android_Cross_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/android-ndk-$__NDK_Version-linux-x86_64.zip -O $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip + unzip -q $__CrossDir/android-ndk-$__NDK_Version-linux-x86_64.zip -d $__CrossDir fi if [ ! -d $__lldb_Dir ]; then mkdir -p $__lldb_Dir echo Downloading LLDB into $__lldb_Dir - wget -nv -nc --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip - unzip -q $__Android_Cross_Dir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir + wget -q --progress=bar:force:noscroll --show-progress https://dl.google.com/android/repository/lldb-2.3.3614996-linux-x86_64.zip -O $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip + unzip -q $__CrossDir/lldb-2.3.3614996-linux-x86_64.zip -d $__lldb_Dir fi -# Create the RootFS for both arm64 as well as aarch -rm -rf $__Android_Cross_Dir/toolchain - -echo Generating the $__BuildArch toolchain -$__NDK_Dir/build/tools/make_standalone_toolchain.py --arch $__BuildArch --api $__ApiLevel --install-dir $__ToolchainDir - -# Install the required packages into the toolchain -# TODO: Add logic to get latest pkg version instead of specific version number -rm -rf $__Android_Cross_Dir/deb/ -rm -rf $__Android_Cross_Dir/tmp - -mkdir -p $__Android_Cross_Dir/deb/ -mkdir -p $__Android_Cross_Dir/tmp/$arch/ -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libicu-dev_60.2_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb - -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob-dev_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-glob_0.4_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support-dev_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libandroid-support_22_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma-dev_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/liblzma_5.2.3_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind-dev_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb -wget -nv -nc http://termux.net/dists/stable/main/binary-$__AndroidArch/libunwind_1.2.20170304_$__AndroidArch.deb -O $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb - -echo Unpacking Termux packages -dpkg -x $__Android_Cross_Dir/deb/libicu_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libicu-dev_60.2_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob-dev_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-glob_0.4_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support-dev_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libandroid-support_22_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma-dev_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/liblzma_5.2.3_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind-dev_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ -dpkg -x $__Android_Cross_Dir/deb/libunwind_1.2.20170304_$__AndroidArch.deb $__Android_Cross_Dir/tmp/$__AndroidArch/ - -cp -R $__Android_Cross_Dir/tmp/$__AndroidArch/data/data/com.termux/files/usr/* $__ToolchainDir/sysroot/usr/ +echo "Download dependencies..." +__TmpDir=$__CrossDir/tmp/$__BuildArch/ +mkdir -p "$__TmpDir" -# Generate platform file for build.sh script to assign to __DistroRid -echo "Generating platform file..." +# combined dependencies for coreclr, installer and libraries +__AndroidPackages="libicu" +__AndroidPackages+=" libandroid-glob" +__AndroidPackages+=" liblzma" +__AndroidPackages+=" krb5" +__AndroidPackages+=" openssl" -echo "RID=android.21-arm64" > $__ToolchainDir/sysroot/android_platform -echo Now run: -echo CONFIG_DIR=\`realpath cross/android/$__BuildArch\` ROOTFS_DIR=\`realpath $__ToolchainDir/sysroot\` ./build.sh cross $__BuildArch skipgenerateversion skipnuget cmakeargs -DENABLE_LLDBPLUGIN=0 +for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ + grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do + if [[ "$path" != "Filename:" ]]; then + echo "Working on: $path" + wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" + fi +done + +cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" + +# Generate platform file for build.sh script to assign to __DistroRid +echo "Generating platform file..." +echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform + +echo "Now to build coreclr, libraries and installers; run:" +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory coreclr +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory libraries +echo ROOTFS_DIR=\$\(realpath $__ToolchainDir/sysroot\) ./build.sh --cross --arch $__BuildArch \ + --subsetCategory installer diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index a23f895ba1..ffdff38542 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -1,19 +1,26 @@ #!/usr/bin/env bash +set -e + usage() { - echo "Usage: $0 [BuildArch] [LinuxCodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" echo "BuildArch can be: arm(default), armel, arm64, x86" - echo "LinuxCodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." - echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine" + echo "CodeName - optional, Code name for Linux, can be: trusty, xenial(default), zesty, bionic, alpine. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." + echo " for FreeBSD can be: freebsd11 or freebsd12." + echo " for illumos can be: illumos." + echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FReeBSD" echo "--skipunmount - optional, will skip the unmount of rootfs folder." + echo "--use-mirror - optional, use mirror URL to fetch resources, when available." exit 1 } -__LinuxCodeName=xenial +__CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) __InitialDir=$PWD __BuildArch=arm +__AlpineArch=armv7 +__QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" __LLDB_Package="liblldb-3.9-dev" @@ -25,9 +32,10 @@ __UbuntuPackages="build-essential" __AlpinePackages="alpine-base" __AlpinePackages+=" build-base" __AlpinePackages+=" linux-headers" -__AlpinePackagesEdgeTesting=" lldb-dev" -__AlpinePackagesEdgeMain=" llvm9-libs" +__AlpinePackagesEdgeCommunity=" lldb-dev" +__AlpinePackagesEdgeMain=" llvm10-libs" __AlpinePackagesEdgeMain+=" python3" +__AlpinePackagesEdgeMain+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" @@ -53,6 +61,21 @@ __AlpinePackages+=" krb5-dev" __AlpinePackages+=" openssl-dev" __AlpinePackages+=" zlib-dev" +__FreeBSDBase="12.1-RELEASE" +__FreeBSDPkg="1.12.0" +__FreeBSDPackages="libunwind" +__FreeBSDPackages+=" icu" +__FreeBSDPackages+=" libinotify" +__FreeBSDPackages+=" lttng-ust" +__FreeBSDPackages+=" krb5" + +__IllumosPackages="icu-64.2nb2" +__IllumosPackages+=" mit-krb5-1.16.2nb4" +__IllumosPackages+=" openssl-1.1.1e" +__IllumosPackages+=" zlib-1.2.11" + +__UseMirror=0 + __UnprocessedBuildArgs= while :; do if [ $# -le 0 ]; then @@ -68,7 +91,7 @@ while :; do arm) __BuildArch=arm __UbuntuArch=armhf - __AlpineArch=armhf + __AlpineArch=armv7 __QEMUArch=arm ;; arm64) @@ -81,7 +104,7 @@ while :; do __BuildArch=armel __UbuntuArch=armel __UbuntuRepo="http://ftp.debian.org/debian/" - __LinuxCodeName=jessie + __CodeName=jessie ;; x86) __BuildArch=x86 @@ -110,53 +133,66 @@ while :; do unset __LLDB_Package ;; trusty) # Ubuntu 14.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=trusty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=trusty fi ;; xenial) # Ubuntu 16.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=xenial + if [ "$__CodeName" != "jessie" ]; then + __CodeName=xenial fi ;; zesty) # Ubuntu 17.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=zesty + if [ "$__CodeName" != "jessie" ]; then + __CodeName=zesty fi ;; bionic) # Ubuntu 18.04 - if [ "$__LinuxCodeName" != "jessie" ]; then - __LinuxCodeName=bionic + if [ "$__CodeName" != "jessie" ]; then + __CodeName=bionic fi ;; jessie) # Debian 8 - __LinuxCodeName=jessie + __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" ;; stretch) # Debian 9 - __LinuxCodeName=stretch + __CodeName=stretch __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; buster) # Debian 10 - __LinuxCodeName=buster + __CodeName=buster __UbuntuRepo="http://ftp.debian.org/debian/" __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ]; then - echo "Tizen is available only for armel." + if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for armel and arm64." usage; exit 1; fi - __LinuxCodeName= + __CodeName= __UbuntuRepo= __Tizen=tizen ;; alpine) - __LinuxCodeName=alpine + __CodeName=alpine __UbuntuRepo= ;; + freebsd11) + __FreeBSDBase="11.3-RELEASE" + ;& + freebsd12) + __CodeName=freebsd + __BuildArch=x64 + __SkipUnmount=1 + ;; + illumos) + __CodeName=illumos + __BuildArch=x64 + __SkipUnmount=1 + ;; --skipunmount) __SkipUnmount=1 ;; @@ -164,6 +200,9 @@ while :; do shift __RootfsDir=$1 ;; + --use-mirror) + __UseMirror=1 + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -187,12 +226,15 @@ fi if [ -d "$__RootfsDir" ]; then if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi rm -rf $__RootfsDir fi -if [[ "$__LinuxCodeName" == "alpine" ]]; then +mkdir -p $__RootfsDir +__RootfsDir="$( cd "$__RootfsDir" && pwd )" + +if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 __AlpineVersion=3.9 __ApkToolsDir=$(mktemp -d) @@ -213,30 +255,92 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then add $__AlpinePackagesEdgeMain $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \ + -X http://dl-cdn.alpinelinux.org/alpine/edge/community \ -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ - add $__AlpinePackagesEdgeTesting + add $__AlpinePackagesEdgeCommunity rm -r $__ApkToolsDir -elif [[ -n $__LinuxCodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__LinuxCodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__LinuxCodeName $__RootfsDir/etc/apt/sources.list +elif [[ "$__CodeName" == "freebsd" ]]; then + mkdir -p $__RootfsDir/usr/local/etc + wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + # For now, ask for 11 ABI even on 12. This can be revisited later. + echo "ABI = \"FreeBSD:11:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf + echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf + mkdir -p $__RootfsDir/tmp + # get and build package manager + wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - + cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # needed for install to succeed + mkdir -p $__RootfsDir/host/etc + ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make && make install + rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + # install packages we need. + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages +elif [[ "$__CodeName" == "illumos" ]]; then + mkdir "$__RootfsDir/tmp" + pushd "$__RootfsDir/tmp" + JOBS="$(getconf _NPROCESSORS_ONLN)" + echo "Downloading sysroot." + wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - + echo "Building binutils. Please wait.." + wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - + mkdir build-binutils && cd build-binutils + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + make -j "$JOBS" && make install && cd .. + echo "Building gcc. Please wait.." + wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - + CFLAGS="-fPIC" + CXXFLAGS="-fPIC" + CXXFLAGS_FOR_TARGET="-fPIC" + CFLAGS_FOR_TARGET="-fPIC" + export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET + mkdir build-gcc && cd build-gcc + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ + --disable-libquadmath-support --disable-shared --enable-tls + make -j "$JOBS" && make install && cd .. + BaseUrl=https://pkgsrc.joyent.com + if [[ "$__UseMirror" == 1 ]]; then + BaseUrl=http://pkgsrc.smartos.skylime.net + fi + BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + echo "Downloading dependencies." + read -ra array <<<"$__IllumosPackages" + for package in "${array[@]}"; do + echo "Installing $package..." + wget "$BaseUrl"/"$package".tgz + ar -x "$package".tgz + tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + done + echo "Cleaning up temporary files." + popd + rm -rf "$__RootfsDir"/{tmp,+*} + mkdir -p "$__RootfsDir"/usr/include/net + mkdir -p "$__RootfsDir"/usr/include/netpacket + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/bpf.h + wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h + wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h + wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h +elif [[ -n $__CodeName ]]; then + qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install chroot $__RootfsDir apt-get -y install $__UbuntuPackages chroot $__RootfsDir symlinks -cr /usr if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* + umount $__RootfsDir/* || true fi - if [[ "$__BuildArch" == "arm" && "$__LinuxCodeName" == "trusty" ]]; then + if [[ "$__BuildArch" == "arm" && "$__CodeName" == "trusty" ]]; then pushd $__RootfsDir patch -p1 < $__CrossDir/$__BuildArch/trusty.patch patch -p1 < $__CrossDir/$__BuildArch/trusty-lttng-2.4.patch popd fi -elif [ "$__Tizen" == "tizen" ]; then +elif [[ "$__Tizen" == "tizen" ]]; then ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh else echo "Unsupported target platform." diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 556da2e135..b9fe796f0d 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,7 +1,14 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) -set(CMAKE_SYSTEM_NAME Linux) +if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) + set(CMAKE_SYSTEM_NAME FreeBSD) +elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) + set(CMAKE_SYSTEM_NAME SunOS) + set(ILLUMOS 1) +else() + set(CMAKE_SYSTEM_NAME Linux) +endif() set(CMAKE_SYSTEM_VERSION 1) if(TARGET_ARCH_NAME STREQUAL "armel") @@ -24,9 +31,18 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") else() set(TOOLCHAIN "aarch64-linux-gnu") endif() + if("$ENV{__DistroRid}" MATCHES "tizen.*") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") +elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(triple "x86_64-unknown-freebsd11") +elseif (ILLUMOS) + set(CMAKE_SYSTEM_PROCESSOR "x86_64") + set(TOOLCHAIN "x86_64-illumos") else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!") endif() @@ -36,34 +52,120 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) endif() + if(TARGET_ARCH_NAME STREQUAL "arm64") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) + endif() endif() -set(CMAKE_SYSROOT "${CROSS_ROOTFS}") -set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") -set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +if("$ENV{__DistroRid}" MATCHES "android.*") + if(TARGET_ARCH_NAME STREQUAL "arm") + set(ANDROID_ABI armeabi-v7a) + elseif(TARGET_ARCH_NAME STREQUAL "arm64") + set(ANDROID_ABI arm64-v8a) + endif() + + # extract platform number required by the NDK's toolchain + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + + set(ANDROID_TOOLCHAIN clang) + set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository + set(CMAKE_SYSTEM_LIBRARY_PATH "${CROSS_ROOTFS}/usr/lib") + set(CMAKE_SYSTEM_INCLUDE_PATH "${CROSS_ROOTFS}/usr/include") + + # include official NDK toolchain script + include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) +elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") + # we cross-compile by instructing clang + set(CMAKE_C_COMPILER_TARGET ${triple}) + set(CMAKE_CXX_COMPILER_TARGET ${triple}) + set(CMAKE_ASM_COMPILER_TARGET ${triple}) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") +elseif(ILLUMOS) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + include_directories(SYSTEM ${CROSS_ROOTFS}/include) + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + find_program(EXEC_LOCATION_${exec} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") +else() + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") + set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr") +endif() # Specify link flags +function(add_toolchain_linker_flag Flag) + set(Config "${ARGV1}") + set(CONFIG_SUFFIX "") + if (NOT Config STREQUAL "") + set(CONFIG_SUFFIX "_${Config}") + endif() + set("CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_EXE_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) + set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}} ${Flag}" PARENT_SCOPE) +endfunction() + + if(TARGET_ARCH_NAME STREQUAL "armel") if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only - add_link_options("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") - add_link_options("-L${CROSS_ROOTFS}/lib") - add_link_options("-L${CROSS_ROOTFS}/usr/lib") - add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "arm64") + if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") + add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") - add_link_options(-m32) + add_toolchain_linker_flag(-m32) +elseif(ILLUMOS) + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") endif() # Specify compile options -if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") +if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -84,16 +186,19 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) - if(DEFINED TIZEN_TOOLCHAIN) - add_compile_options(-Wno-deprecated-declarations) # compile-time option - add_compile_options(-D__extern_always_inline=inline) # compile-time option - endif() endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-m32) add_compile_options(-Wno-error=unused-command-line-argument) endif() +if(DEFINED TIZEN_TOOLCHAIN) + if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") + add_compile_options(-Wno-deprecated-declarations) # compile-time option + add_compile_options(-D__extern_always_inline=inline) # compile-time option + endif() +endif() + # Set LLDB include and library paths for builds that need lldb. if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") if(TARGET_ARCH_NAME STREQUAL "x86") @@ -122,7 +227,6 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") endif() endif() - set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) diff --git a/eng/common/darc-init.ps1 b/eng/common/darc-init.ps1 index 2a30ccfd74..435e764134 100644 --- a/eng/common/darc-init.ps1 +++ b/eng/common/darc-init.ps1 @@ -7,7 +7,7 @@ param ( . $PSScriptRoot\tools.ps1 -function InstallDarcCli ($darcVersion) { +function InstallDarcCli ($darcVersion, $toolpath) { $darcCliPackageName = 'microsoft.dotnet.darc' $dotnetRoot = InitializeDotNetCli -install:$true @@ -32,13 +32,13 @@ function InstallDarcCli ($darcVersion) { Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g }else { - Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity -g --tool-path '$toolpath'" + Write-Host "'$dotnet' tool install $darcCliPackageName --version $darcVersion --add-source '$arcadeServicesSource' -v $verbosity --tool-path '$toolpath'" & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath" } } try { - InstallDarcCli $darcVersion + InstallDarcCli $darcVersion $toolpath } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 50bc5e475c..ead6a1d9a2 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -63,7 +63,7 @@ case $cpuname in amd64|x86_64) buildarch=x64 ;; - armv7l) + armv*l) buildarch=arm ;; i686) diff --git a/eng/common/init-tools-native.sh b/eng/common/init-tools-native.sh index dbd7cbfdf6..29fc5db8ae 100755 --- a/eng/common/init-tools-native.sh +++ b/eng/common/init-tools-native.sh @@ -34,6 +34,14 @@ while (($# > 0)); do force=true shift 1 ;; + --donotabortonfailure) + donotabortonfailure=true + shift 1 + ;; + --donotdisplaywarnings) + donotdisplaywarnings=true + shift 1 + ;; --downloadretries) download_retries=$2 shift 2 @@ -52,6 +60,8 @@ while (($# > 0)); do echo " - (default) %USERPROFILE%/.netcoreeng/native" echo "" echo " --clean Switch specifying not to install anything, but cleanup native asset folders" + echo " --donotabortonfailure Switch specifiying whether to abort native tools installation on failure" + echo " --donotdisplaywarnings Switch specifiying whether to display warnings during native tools installation on failure" echo " --force Clean and then install tools" echo " --help Print help and exit" echo "" @@ -92,6 +102,7 @@ if [[ -z $install_directory ]]; then fi install_bin="${native_base_dir}/bin" +installed_any=false ReadGlobalJsonNativeTools @@ -103,8 +114,8 @@ else for tool in "${!native_assets[@]}" do tool_version=${native_assets[$tool]} - installer_name="install-$tool.sh" - installer_command="$native_installer_dir/$installer_name" + installer_path="$native_installer_dir/install-$tool.sh" + installer_command="$installer_path" installer_command+=" --baseuri $base_uri" installer_command+=" --installpath $install_bin" installer_command+=" --version $tool_version" @@ -118,11 +129,29 @@ else installer_command+=" --clean" fi - $installer_command - - if [[ $? != 0 ]]; then - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" - exit 1 + if [[ -a $installer_path ]]; then + $installer_command + if [[ $? != 0 ]]; then + if [[ $donotabortonfailure = true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed" + exit 1 + fi + else + $installed_any = true + fi + else + if [[ $donotabortonfailure == true ]]; then + if [[ $donotdisplaywarnings != true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + fi + else + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed: no install script" + exit 1 + fi fi done fi @@ -135,8 +164,10 @@ if [[ -d $install_bin ]]; then echo "Native tools are available from $install_bin" echo "##vso[task.prependpath]$install_bin" else - Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" - exit 1 + if [[ $installed_any = true ]]; then + Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed" + exit 1 + fi fi exit 0 diff --git a/eng/common/internal-feed-operations.ps1 b/eng/common/internal-feed-operations.ps1 index db0baac9a4..b8f6529fdc 100644 --- a/eng/common/internal-feed-operations.ps1 +++ b/eng/common/internal-feed-operations.ps1 @@ -63,6 +63,7 @@ function SetupCredProvider { } if (($endpoints | Measure-Object).Count -gt 0) { + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 5941ea2833..9ed225e7e5 100755 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -62,6 +62,7 @@ function SetupCredProvider { endpoints+=']' if [ ${#endpoints} -gt 2 ]; then + # [SuppressMessage("Microsoft.Security", "CS002:SecretInNextLine", Justification="Endpoint code example with no real credentials.")] # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}' local endpointCredentials="{\"endpointCredentials\": "$endpoints"}" diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index 1a39a7ef3f..f46d5efe2e 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -4,6 +4,7 @@ net472 false + false diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index 41416862d9..d7d1a65109 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -145,9 +145,12 @@ function Get-File { New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null } + $TempPath = "$Path.tmp" if (Test-Path -IsValid -Path $Uri) { - Write-Verbose "'$Uri' is a file path, copying file to '$Path'" - Copy-Item -Path $Uri -Destination $Path + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path return $? } else { @@ -157,8 +160,10 @@ function Get-File { while($Attempt -Lt $DownloadRetries) { try { - Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $Path - Write-Verbose "Downloaded to '$Path'" + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" return $True } catch { @@ -359,16 +364,21 @@ function Expand-Zip { return $False } } - if (-Not (Test-Path $OutputDirectory)) { - New-Item -path $OutputDirectory -Force -itemType "Directory" | Out-Null + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null Add-Type -assembly "system.io.compression.filesystem" - [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$OutputDirectory") + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") if ($? -Eq $False) { Write-Error "Unable to extract '$ZipPath'" return $False } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory } catch { Write-Host $_ diff --git a/eng/common/native/find-native-compiler.sh b/eng/common/native/find-native-compiler.sh new file mode 100644 index 0000000000..aed19d07d5 --- /dev/null +++ b/eng/common/native/find-native-compiler.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# This file locates the native compiler with the given name and version and sets the environment variables to locate it. +# + +source="${BASH_SOURCE[0]}" + +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ $# -lt 0 ] +then + echo "Usage..." + echo "find-native-compiler.sh " + echo "Specify the name of compiler (clang or gcc)." + echo "Specify the major version of compiler." + echo "Specify the minor version of compiler." + exit 1 +fi + +. $scriptroot/../pipeline-logging-functions.sh + +compiler="$1" +cxxCompiler="$compiler++" +majorVersion="$2" +minorVersion="$3" + +if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi + +check_version_exists() { + desired_version=-1 + + # Set up the environment to be used for building with the desired compiler. + if command -v "$compiler-$1.$2" > /dev/null; then + desired_version="-$1.$2" + elif command -v "$compiler$1$2" > /dev/null; then + desired_version="$1$2" + elif command -v "$compiler-$1$2" > /dev/null; then + desired_version="-$1$2" + fi + + echo "$desired_version" +} + +if [ -z "$CLR_CC" ]; then + + # Set default versions + if [ -z "$majorVersion" ]; then + # note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero. + if [ "$compiler" = "clang" ]; then versions=( 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 ) + elif [ "$compiler" = "gcc" ]; then versions=( 9 8 7 6 5 4.9 ); fi + + for version in "${versions[@]}"; do + parts=(${version//./ }) + desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")" + if [ "$desired_version" != "-1" ]; then majorVersion="${parts[0]}"; break; fi + done + + if [ -z "$majorVersion" ]; then + if command -v "$compiler" > /dev/null; then + if [ "$(uname)" != "Darwin" ]; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH." + fi + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found." + exit 1 + fi + else + if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then + if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then + if command -v "$compiler" > /dev/null; then + Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH." + export CC="$(command -v "$compiler")" + export CXX="$(command -v "$cxxCompiler")" + else + Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH." + exit 1 + fi + fi + fi + fi + else + desired_version="$(check_version_exists "$majorVersion" "$minorVersion")" + if [ "$desired_version" = "-1" ]; then + Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion." + exit 1 + fi + fi + + if [ -z "$CC" ]; then + export CC="$(command -v "$compiler$desired_version")" + export CXX="$(command -v "$cxxCompiler$desired_version")" + if [ -z "$CXX" ]; then export CXX="$(command -v "$cxxCompiler")"; fi + fi +else + if [ ! -f "$CLR_CC" ]; then + Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist" + exit 1 + fi + export CC="$CLR_CC" + export CXX="$CLR_CXX" +fi + +if [ -z "$CC" ]; then + Write-PipelineTelemetryError -category "Build" "Unable to find $compiler." + exit 1 +fi + +export CCC_CC="$CC" +export CCC_CXX="$CXX" +export SCAN_BUILD_COMMAND="$(command -v "scan-build$desired_version")" diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index e5826b5323..272366da95 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,7 +6,8 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe - $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd + + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline %HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + + + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + + --corerun $(CoreRun) @@ -55,6 +63,11 @@ $(WorkItemCommand) $(CliArguments) + + + 2:30 + 0:15 + @@ -63,7 +76,7 @@ - 5 + 30 @@ -71,6 +84,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -86,7 +124,7 @@ $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) - 4:00 + $(WorkItemTimeout) @@ -99,4 +137,56 @@ 4:00 + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen\test.py crossgen --test-name System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.Xml.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Linq.Expressions.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.VisualBasic.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single Microsoft.CodeAnalysis.CSharp.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --single System.Private.CoreLib.dll --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + + + $(WorkItemDirectory)\ScenarioCorrelation + $(Python) %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\test.py crossgen2 --composite %HELIX_CORRELATION_PAYLOAD%\performance\src\scenarios\crossgen2\framework-r2r.dll.rsp --core-root %HELIX_CORRELATION_PAYLOAD%\Core_Root + 1:00 + + + \ No newline at end of file diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index e337669929..656c0bd902 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -3,7 +3,7 @@ Param( [string] $CoreRootDirectory, [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", - [string] $Framework="netcoreapp5.0", + [string] $Framework="net5.0", [string] $CompilationMode="Tiered", [string] $Repository=$env:BUILD_REPOSITORY_NAME, [string] $Branch=$env:BUILD_SOURCEBRANCH, @@ -12,8 +12,12 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, + [string] $MonoDotnet="", [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) @@ -31,7 +35,8 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" -if ($Framework.StartsWith("netcoreapp")) { +# TODO: Implement a better logic to determine if Framework is .NET Core or >= .NET 5. +if ($Framework.StartsWith("netcoreapp") -or ($Framework -eq "net5.0")) { $Queue = "Windows.10.Amd64.ClientRS5.Open" } @@ -49,10 +54,37 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoInterpreter) +{ + $ExtraBenchmarkDotNetArguments = "--category-exclusion-filter NoInterpreter" +} + +if($MonoDotnet -ne "") +{ + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" + +#This grabs the LKG version number of dotnet and passes it to our scripts +$VersionJSON = Get-Content global.json | ConvertFrom-Json +$DotNetVersion = $VersionJSON.tools.dotnet +$SetupArguments = "--dotnet-versions $DotNetVersion $SetupArguments" + + if ($RunFromPerformanceRepo) { $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments" @@ -62,6 +94,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -97,6 +136,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 94a04e0fe5..c87cbf0fc2 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -4,7 +4,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= baseline_core_root_directory= architecture=x64 -framework=netcoreapp5.0 +framework=net5.0 compilation_mode=tiered repository=$BUILD_REPOSITORY_NAME branch=$BUILD_SOURCEBRANCH @@ -12,13 +12,18 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" +llvm=false +monointerpreter=false +monoaot=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -65,6 +70,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -79,6 +85,22 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; + --monodotnet) + mono_dotnet=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +129,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -164,9 +187,24 @@ if [[ "$internal" == true ]]; then fi fi +if [[ "$mono_dotnet" != "" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + +if [[ "$monointerpreter" == "true" ]]; then + extra_benchmark_dotnet_arguments="--category-exclusion-filter NoInterpreter" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" + +# Get the tools section from the global.json. +# This grabs the LKG version number of dotnet and passes it to our scripts +dotnet_version=`cat global.json | python3 -c 'import json,sys;obj=json.load(sys.stdin);print(obj["tools"]["dotnet"])'` +setup_arguments="--dotnet-versions $dotnet_version $setup_arguments" + + if [[ "$run_from_perf_repo" = true ]]; then payload_directory= workitem_directory=$source_directory @@ -179,6 +217,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -214,3 +258,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/common/pipeline-logging-functions.ps1 b/eng/common/pipeline-logging-functions.ps1 index a3e1317ad4..8484451f3a 100644 --- a/eng/common/pipeline-logging-functions.ps1 +++ b/eng/common/pipeline-logging-functions.ps1 @@ -31,7 +31,9 @@ function Write-PipelineTelemetryError { $PSBoundParameters.Remove('Category') | Out-Null - $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + if($Force -Or ((Test-Path variable:ci) -And $ci)) { + $Message = "(NETCORE_ENGINEERING_TELEMETRY=$Category) $Message" + } $PSBoundParameters.Remove('Message') | Out-Null $PSBoundParameters.Add('Message', $Message) Write-PipelineTaskError @PSBoundParameters @@ -65,12 +67,12 @@ function Write-PipelineTaskError { } if(($Type -ne 'error') -and ($Type -ne 'warning')) { - Write-Host $Message - return + Write-Host $Message + return } $PSBoundParameters.Remove('Force') | Out-Null if(-not $PSBoundParameters.ContainsKey('Type')) { - $PSBoundParameters.Add('Type', 'error') + $PSBoundParameters.Add('Type', 'error') } Write-LogIssue @PSBoundParameters } diff --git a/eng/common/pipeline-logging-functions.sh b/eng/common/pipeline-logging-functions.sh index 33c3f0d807..6cd0a3400e 100755 --- a/eng/common/pipeline-logging-functions.sh +++ b/eng/common/pipeline-logging-functions.sh @@ -31,26 +31,21 @@ function Write-PipelineTelemetryError { return fi - message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" - function_args+=("$message") if [[ $force == true ]]; then function_args+=("-force") fi - - Write-PipelineTaskError $function_args + message="(NETCORE_ENGINEERING_TELEMETRY=$telemetry_category) $message" + function_args+=("$message") + Write-PipelineTaskError ${function_args[@]} } function Write-PipelineTaskError { - if [[ $force != true ]] && [[ "$ci" != true ]]; then - echo "$@" >&2 - return - fi - local message_type="error" local sourcepath='' local linenumber='' local columnnumber='' local error_code='' + local force=false while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')" @@ -75,6 +70,9 @@ function Write-PipelineTaskError { error_code=$2 shift ;; + -force|-f) + force=true + ;; *) break ;; @@ -83,6 +81,11 @@ function Write-PipelineTaskError { shift done + if [[ $force != true ]] && [[ "$ci" != true ]]; then + echo "$@" >&2 + return + fi + local message="##vso[task.logissue" message="$message type=$message_type" diff --git a/eng/common/post-build/check-channel-consistency.ps1 b/eng/common/post-build/check-channel-consistency.ps1 new file mode 100644 index 0000000000..63f3464c98 --- /dev/null +++ b/eng/common/post-build/check-channel-consistency.ps1 @@ -0,0 +1,40 @@ +param( + [Parameter(Mandatory=$true)][string] $PromoteToChannels, # List of channels that the build should be promoted to + [Parameter(Mandatory=$true)][array] $AvailableChannelIds # List of channel IDs available in the YAML implementation +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + + if ($PromoteToChannels -eq "") { + Write-PipelineTaskError -Type 'warning' -Message "This build won't publish assets as it's not configured to any Maestro channel. If that wasn't intended use Darc to configure a default channel using add-default-channel for this branch or to promote it to a channel using add-build-to-channel. See https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#assigning-an-individual-build-to-a-channel for more info." + ExitWithExitCode 0 + } + + # Check that every channel that Maestro told to promote the build to + # is available in YAML + $PromoteToChannelsIds = $PromoteToChannels -split "\D" | Where-Object { $_ } + + $hasErrors = $false + + foreach ($id in $PromoteToChannelsIds) { + if (($id -ne 0) -and ($id -notin $AvailableChannelIds)) { + Write-PipelineTaskError -Message "Channel $id is not present in the post-build YAML configuration! This is an error scenario. Please contact @dnceng." + $hasErrors = $true + } + } + + # The `Write-PipelineTaskError` doesn't error the script and we might report several errors + # in the previous lines. The check below makes sure that we return an error state from the + # script if we reported any validation error + if ($hasErrors) { + ExitWithExitCode 1 + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'CheckChannelConsistency' -Message "There was an error while trying to check consistency of Maestro default channels for the build and post-build YAML configuration." + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 new file mode 100644 index 0000000000..d22eb439c4 --- /dev/null +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -0,0 +1,71 @@ +param( + [Parameter(Mandatory=$true)][int] $BuildId, + [Parameter(Mandatory=$true)][string] $AzdoToken, + [Parameter(Mandatory=$true)][string] $MaestroToken, + [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com', + [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, + [Parameter(Mandatory=$true)][string] $EnableSourceLinkValidation, + [Parameter(Mandatory=$true)][string] $EnableSigningValidation, + [Parameter(Mandatory=$true)][string] $EnableNugetValidation, + [Parameter(Mandatory=$true)][string] $PublishInstallersAndChecksums, + [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, + [Parameter(Mandatory=$false)][string] $SigningValidationAdditionalParameters +) + +try { + . $PSScriptRoot\post-build-utils.ps1 + . $PSScriptRoot\..\darc-init.ps1 + + $optionalParams = [System.Collections.ArrayList]::new() + + if ("" -ne $ArtifactsPublishingAdditionalParameters) { + $optionalParams.Add("artifact-publishing-parameters") | Out-Null + $optionalParams.Add($ArtifactsPublishingAdditionalParameters) | Out-Null + } + + if ("false" -eq $WaitPublishingFinish) { + $optionalParams.Add("--no-wait") | Out-Null + } + + if ("true" -eq $PublishInstallersAndChecksums) { + $optionalParams.Add("--publish-installers-and-checksums") | Out-Null + } + + if ("true" -eq $EnableNugetValidation) { + $optionalParams.Add("--validate-nuget") | Out-Null + } + + if ("true" -eq $EnableSourceLinkValidation) { + $optionalParams.Add("--validate-sourcelinkchecksums") | Out-Null + } + + if ("true" -eq $EnableSigningValidation) { + $optionalParams.Add("--validate-signingchecksums") | Out-Null + + if ("" -ne $SigningValidationAdditionalParameters) { + $optionalParams.Add("--signing-validation-parameters") | Out-Null + $optionalParams.Add($SigningValidationAdditionalParameters) | Out-Null + } + } + + & darc add-build-to-channel ` + --id $buildId ` + --default-channels ` + --source-branch master ` + --azdev-pat $AzdoToken ` + --bar-uri $MaestroApiEndPoint ` + --password $MaestroToken ` + @optionalParams + + if ($LastExitCode -ne 0) { + Write-Host "Problems using Darc to promote build ${buildId} to default channels. Stopping execution..." + exit 1 + } + + Write-Host 'done.' +} +catch { + Write-Host $_ + Write-PipelineTelemetryError -Category 'PromoteBuild' -Message "There was an error while trying to publish build '$BuildId' to default channels." + ExitWithExitCode 1 +} diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index cc9d059d04..cdd1bba4f2 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -196,6 +196,8 @@ function ValidateSourceLinkLinks { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $ValidationFailures = 0 + # Process each NuGet package in parallel Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { @@ -209,17 +211,20 @@ function ValidateSourceLinkLinks { } foreach ($Job in @(Get-Job -State 'Completed')) { - Receive-Job -Id $Job.Id + $jobResult = Receive-Job -Id $Job.Id + if ($jobResult -ne '0') { + $ValidationFailures++ + } Remove-Job -Id $Job.Id } } - $ValidationFailures = 0 foreach ($Job in @(Get-Job)) { $jobResult = Wait-Job -Id $Job.Id | Receive-Job if ($jobResult -ne '0') { $ValidationFailures++ } + Remove-Job -Id $Job.Id } if ($ValidationFailures -gt 0) { Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation." diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index f7cfe986dd..a36fa6f26b 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -1,72 +1,30 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion # Version of dotnet symbol to use + [Parameter(Mandatory=$true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use + [Parameter(Mandatory=$false)][switch] $ContinueOnError, # If we should keep checking symbols after an error + [Parameter(Mandatory=$false)][switch] $Clean # Clean extracted symbols directory after checking symbols ) -function FirstMatchingSymbolDescriptionOrDefault { - param( - [string] $FullPath, # Full path to the module that has to be checked - [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols - [string] $SymbolsPath - ) - - $FileName = [System.IO.Path]::GetFileName($FullPath) - $Extension = [System.IO.Path]::GetExtension($FullPath) - - # Those below are potential symbol files that the `dotnet symbol` might - # return. Which one will be returned depend on the type of file we are - # checking and which type of file was uploaded. - - # The file itself is returned - $SymbolPath = $SymbolsPath + '\' + $FileName - - # PDB file for the module - $PdbPath = $SymbolPath.Replace($Extension, '.pdb') - - # PDB file for R2R module (created by crossgen) - $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') - - # DBG file for a .so library - $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') - - # DWARF file for a .dylib - $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') - - $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 6 - & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 - if (Test-Path $PdbPath) { - return 'PDB' - } - elseif (Test-Path $NGenPdb) { - return 'NGen PDB' - } - elseif (Test-Path $SODbg) { - return 'DBG for SO' - } - elseif (Test-Path $DylibDwarf) { - return 'Dwarf for Dylib' - } - elseif (Test-Path $SymbolPath) { - return 'Module' - } - else { - return $null - } -} - -function CountMissingSymbols { +$CountMissingSymbols = { param( [string] $PackagePath # Path to a NuGet package ) + . $using:PSScriptRoot\..\tools.ps1 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + # Ensure input file exist if (!(Test-Path $PackagePath)) { Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 + return -2 } # Extensions for which we'll look for symbols @@ -77,23 +35,88 @@ function CountMissingSymbols { $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) $PackageGuid = New-Guid - $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols' - [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + try { + [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath) + } + catch { + Write-Host "Something went wrong extracting $PackagePath" + Write-Host $_ + return [pscustomobject]@{ + result = -1 + packagePath = $PackagePath + } + } Get-ChildItem -Recurse $ExtractPath | Where-Object {$RelevantExtensions -contains $_.Extension} | ForEach-Object { - if ($_.FullName -Match '\\ref\\') { - Write-Host "`t Ignoring reference assembly file " $_.FullName + $FileName = $_.FullName + if ($FileName -Match '\\ref\\') { + Write-Host "`t Ignoring reference assembly file " $FileName return } - $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--microsoft-symbol-server' $SymbolsPath - $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName '--internal-server' $SymbolsPath + $FirstMatchingSymbolDescriptionOrDefault = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols + [string] $SymbolsPath + ) + + $FileName = [System.IO.Path]::GetFileName($FullPath) + $Extension = [System.IO.Path]::GetExtension($FullPath) + + # Those below are potential symbol files that the `dotnet symbol` might + # return. Which one will be returned depend on the type of file we are + # checking and which type of file was uploaded. - Write-Host -NoNewLine "`t Checking file " $_.FullName "... " + # The file itself is returned + $SymbolPath = $SymbolsPath + '\' + $FileName + + # PDB file for the module + $PdbPath = $SymbolPath.Replace($Extension, '.pdb') + + # PDB file for R2R module (created by crossgen) + $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb') + + # DBG file for a .so library + $SODbg = $SymbolPath.Replace($Extension, '.so.dbg') + + # DWARF file for a .dylib + $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf') + + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" + + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + + if (Test-Path $PdbPath) { + return 'PDB' + } + elseif (Test-Path $NGenPdb) { + return 'NGen PDB' + } + elseif (Test-Path $SODbg) { + return 'DBG for SO' + } + elseif (Test-Path $DylibDwarf) { + return 'Dwarf for Dylib' + } + elseif (Test-Path $SymbolPath) { + return 'Module' + } + else { + return $null + } + } + + $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--microsoft-symbol-server' $SymbolsPath + $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault $FileName '--internal-server' $SymbolsPath + + Write-Host -NoNewLine "`t Checking file " $FileName "... " if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) { Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)" @@ -115,9 +138,35 @@ function CountMissingSymbols { } } + if ($using:Clean) { + Remove-Item $ExtractPath -Recurse -Force + } + + if ($MissingSymbols -ne 0) + { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $MissingSymbols modules in the package $PackagePath" + } + Pop-Location - return $MissingSymbols + return [pscustomobject]@{ + result = $MissingSymbols + packagePath = $PackagePath + } +} + +function CheckJobResult( + $result, + $packagePath, + [ref]$DupedSymbols, + [ref]$TotalFailures) { + if ($result -eq '-1') { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files" + $DupedSymbols.Value++ + } + elseif ($jobResult.result -ne '0') { + $TotalFailures.Value++ + } } function CheckSymbolsAvailable { @@ -125,9 +174,13 @@ function CheckSymbolsAvailable { Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue } + $TotalFailures = 0 + $DupedSymbols = 0 + Get-ChildItem "$InputPath\*.nupkg" | ForEach-Object { $FileName = $_.Name + $FullName = $_.FullName # These packages from Arcade-Services include some native libraries that # our current symbol uploader can't handle. Below is a workaround until @@ -144,15 +197,45 @@ function CheckSymbolsAvailable { } Write-Host "Validating $FileName " - $Status = CountMissingSymbols "$InputPath\$FileName" - if ($Status -ne 0) { - Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName" - ExitWithExitCode $exitCode + Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList $FullName | Out-Null + + $NumJobs = @(Get-Job -State 'Running').Count + Write-Host $NumJobs + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count } + foreach ($Job in @(Get-Job -State 'Completed')) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + Remove-Job -Id $Job.Id + } Write-Host } + + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures) + } + + if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) { + if ($TotalFailures -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures packages" + } + + if ($DupedSymbols -gt 0) { + Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols packages had duplicated symbol files" + } + + ExitWithExitCode 1 + } + else { + Write-Host "All symbols validated!" + } } function InstallDotnetSymbol { @@ -174,11 +257,13 @@ function InstallDotnetSymbol { try { . $PSScriptRoot\post-build-utils.ps1 - - Add-Type -AssemblyName System.IO.Compression.FileSystem InstallDotnetSymbol + foreach ($Job in @(Get-Job)) { + Remove-Job -Id $Job.Id + } + CheckSymbolsAvailable } catch { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 3872af59b9..32ad5c7e91 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -57,6 +57,24 @@ try { ExitWithExitCode 1 } + if( $msbuildEngine -eq "vs") { + # Ensure desktop MSBuild is available for sdk tasks. + if( -not ($GlobalJson.tools.PSObject.Properties.Name -contains "vs" )) { + $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty + } + if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "16.5.0-alpha" -MemberType NoteProperty + } + if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { + $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true + } + if ($xcopyMSBuildToolsFolder -eq $null) { + throw 'Unable to get xcopy downloadable version of msbuild' + } + + $global:_MSBuildExe = "$($xcopyMSBuildToolsFolder)\MSBuild\Current\Bin\MSBuild.exe" + } + $taskProject = GetSdkTaskProject $task if (!(Test-Path $taskProject)) { Write-PipelineTelemetryError -Category 'Build' -Message "Unknown task: $task" -ForegroundColor Red diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 9db582f279..b681d797cd 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -24,7 +24,8 @@ Param( [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. [string] $GuardianLoggerLevel='Standard', # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams, # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [bool] $BreakOnFailure=$False # Optional: Fail the build if there were errors during the run ) try { @@ -33,6 +34,10 @@ try { $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 #Replace repo names to the format of org/repo @@ -102,6 +107,11 @@ try { ExitWithExitCode 1 } } + + if ($BreakOnFailure) { + Write-Host "Failing the build in case of breaking results..." + & $guardianCliLocation break + } } catch { Write-Host $_.ScriptStackTrace diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 3c9bf10678..7f28d9c59e 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -6,10 +6,6 @@ param( $ErrorActionPreference = 'Stop' Set-StrictMode -Version 2.0 -# `tools.ps1` checks $ci to perform some actions. Since the post-build -# scripts don't necessarily execute in the same agent that run the -# build.ps1/sh script this variable isn't automatically set. -$ci = $true $disableConfigureToolsetImport = $true function ExtractArtifacts { @@ -29,6 +25,10 @@ function ExtractArtifacts { } try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 $ExtractPackage = { @@ -63,7 +63,7 @@ try { } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } @@ -74,7 +74,7 @@ try { Measure-Command { ExtractArtifacts } } catch { - Write-Host $_.ScriptStackTrace + Write-Host $_ Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 285f1ccdb0..a68bf0b88e 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -12,6 +12,10 @@ Set-StrictMode -Version 2.0 $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 +# `tools.ps1` checks $ci to perform some actions. Since the SDL +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true . $PSScriptRoot\..\tools.ps1 # Don't display the console progress UI - it's a huge perf hit @@ -20,7 +24,7 @@ $ProgressPreference = 'SilentlyContinue' # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0-preview.1" +$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" $zipFile = "$WorkingDirectory/gdn.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem @@ -58,6 +62,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 } diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 256ffbfb93..968b39bef5 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/sdl/push-gdn.ps1 b/eng/common/sdl/push-gdn.ps1 index 79d3d355c7..d8fd2d82a6 100644 --- a/eng/common/sdl/push-gdn.ps1 +++ b/eng/common/sdl/push-gdn.ps1 @@ -12,6 +12,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We create the temp directory where we'll store the sdl-config repository @@ -62,4 +66,4 @@ catch { Write-Host $_.ScriptStackTrace Write-PipelineTelemetryError -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index 40a084f796..fe95ab35aa 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -16,6 +16,10 @@ $disableConfigureToolsetImport = $true $LASTEXITCODE = 0 try { + # `tools.ps1` checks $ci to perform some actions. Since the SDL + # scripts don't necessarily execute in the same agent that run the + # build.ps1/sh script this variable isn't automatically set. + $ci = $true . $PSScriptRoot\..\tools.ps1 # We store config files in the r directory of .gdn @@ -64,6 +68,6 @@ try { } catch { Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Sdl' -Message $_ + Write-PipelineTelemetryError -Force -Category 'Sdl' -Message $_ ExitWithExitCode 1 -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 640f2b04e2..c64c4f5686 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -6,6 +6,7 @@ parameters: # 'continueOnError', the parameter value is not correctly picked up. # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; + downloadArtifacts: true # optional: determines if the artifacts should be dowloaded dependsOn: '' # Optional: dependencies of the job artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts # Usage: @@ -31,8 +32,20 @@ jobs: steps: - checkout: self clean: true - - ${{ if ne(parameters.artifactNames, '') }}: - - ${{ each artifactName in parameters.artifactNames }}: + - ${{ if ne(parameters.downloadArtifacts, 'false')}}: + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: - task: DownloadBuildArtifacts@0 displayName: Download Build Artifacts inputs: @@ -41,20 +54,9 @@ jobs: project: $(AzDOProjectName) pipeline: $(AzDOPipelineId) buildId: $(AzDOBuildId) - artifactName: ${{ artifactName }} + downloadType: specific files + itemPattern: "**" downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - - ${{ if eq(parameters.artifactNames, '') }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - downloadType: specific files - itemPattern: "**" - downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts @@ -81,7 +83,7 @@ jobs: continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 + -GuardianPackageName Microsoft.Guardian.Cli.win10-x64.0.20.1 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index ecebd0f03e..e78ed9a1c6 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -24,6 +24,9 @@ parameters: enablePublishBuildAssets: false enablePublishTestResults: false enablePublishUsingPipelines: false + useBuildManifest: false + mergeTestResults: false + testRunTitle: '' name: '' preSteps: [] runAsPublic: false @@ -62,6 +65,9 @@ jobs: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' + - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: + - name: EnableRichCodeNavigation + value: 'true' - ${{ each variable in parameters.variables }}: # handle name-value variable syntax # example: @@ -121,6 +127,15 @@ jobs: - ${{ each step in parameters.steps }}: - ${{ step }} + - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: + - task: RichCodeNavIndexer@0 + displayName: RichCodeNav Upload + inputs: + languages: 'csharp' + environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'prod') }} + richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin + continueOnError: true + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - task: MicroBuildCleanup@1 @@ -194,6 +209,8 @@ jobs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit + mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true condition: always() @@ -214,3 +231,12 @@ jobs: ArtifactName: AssetManifests continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: PublishBuildArtifacts@1 + displayName: Publish Build Manifest + inputs: + PathToPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/manifest.props' + PublishLocation: Container + ArtifactName: BuildManifests + continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index b722975f9c..055304ad89 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -37,6 +37,12 @@ jobs: - name: _BuildConfig value: ${{ parameters.configuration }} - group: Publish-Build-Assets + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml index 380122901d..12db36ebe7 100644 --- a/eng/common/templates/post-build/channels/generic-internal-channel.yml +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -23,7 +23,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -35,6 +35,10 @@ stages: pool: vmImage: 'windows-2019' steps: + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + - task: DownloadBuildArtifacts@0 displayName: Download Build Assets continueOnError: true @@ -84,6 +88,7 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -95,7 +100,7 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml index ae8c6349d4..bf98d990e8 100644 --- a/eng/common/templates/post-build/channels/generic-public-channel.yml +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -10,6 +10,8 @@ parameters: transportFeed: '' shippingFeed: '' symbolsFeed: '' + # If the channel name is empty, no links will be generated + akaMSChannelName: '' stages: - stage: ${{ parameters.stageName }} @@ -23,7 +25,7 @@ stages: - job: publish_symbols displayName: Symbol Publishing dependsOn: setupMaestroVars - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) variables: - group: DotNet-Symbol-Server-Pats - name: AzDOProjectName @@ -83,6 +85,7 @@ stages: - job: publish_assets displayName: Publish Assets dependsOn: setupMaestroVars + timeoutInMinutes: 120 variables: - name: BARBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] @@ -94,7 +97,9 @@ stages: value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - name: AzDOBuildId value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - condition: or(contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )), eq(dependencies.setupMaestroVars.outputs['setReleaseVars.PromoteToMaestroChannelId'], ${{ parameters.channelId }})) + - name: ArtifactsCategory + value: ${{ coalesce(variables._DotNetArtifactsCategory, '.NETCore') }} + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'], format('[{0}]', ${{ parameters.channelId }} )) pool: vmImage: 'windows-2019' steps: @@ -132,7 +137,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet - /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:ArtifactsCategory=$(ArtifactsCategory) /p:IsStableBuild=$(IsStableBuild) /p:IsInternalBuild=$(IsInternalBuild) /p:RepositoryName=$(Build.Repository.Name) @@ -158,6 +163,9 @@ stages: /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:LatestLinkShortUrlPrefix=dotnet/'${{ parameters.akaMSChannelName }}' + /p:AkaMSClientId=$(akams-client-id) + /p:AkaMSClientSecret=$(akams-client-secret) ${{ parameters.artifactsPublishingAdditionalParameters }} - template: ../../steps/publish-logs.yml diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index 9505cf170f..c99fd75037 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -9,8 +9,8 @@ variables: - name: PublicDevRelease_31_Channel_Id value: 128 - # .NET Core 5 Dev - - name: NetCore_5_Dev_Channel_Id + # .NET 5 Dev + - name: Net_5_Dev_Channel_Id value: 131 # .NET Eng - Validation @@ -63,7 +63,7 @@ variables: - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion - value: "2019-01-16" + value: "2020-02-20" - name: SourceLinkCLIVersion value: 3.0.0 @@ -90,3 +90,10 @@ variables: value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json - name: InternalInstallersBlobFeedKey value: $(dotnetclimsrc-access-key) + + # Skip component governance and codesign validation for SDL. These jobs + # create no content. + - name: skipComponentGovernanceDetection + value: true + - name: runCodesignValidationInjection + value: false diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index eff0d9ac79..0e79a546b5 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,4 +1,13 @@ parameters: + # When set to true the publishing templates from the repo will be used + # otherwise Darc add-build-to-channel will be used to trigger the promotion pipeline + inline: true + + # Only used if inline==false. When set to true will stall the current build until + # the Promotion Pipeline build finishes. Otherwise, the current build continue + # execution concurrently with the promotion build. + waitPublishingFinish: true + enableSourceLinkValidation: false enableSigningValidation: true enableSymbolValidation: false @@ -9,12 +18,14 @@ parameters: continueOnError: false params: '' artifactNames: '' + downloadArtifacts: true # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation symbolPublishingAdditionalParameters: '' artifactsPublishingAdditionalParameters: '' signingValidationAdditionalParameters: '' + useBuildManifest: false # Which stages should finish execution before post-build stages start validateDependsOn: @@ -22,293 +33,505 @@ parameters: publishDependsOn: - Validate + # Channel ID's instantiated in this file. + # When adding a new channel implementation the call to `check-channel-consistency.ps1` + # needs to be updated with the new channel ID + NetEngLatestChannelId: 2 + NetEngValidationChannelId: 9 + NetDev5ChannelId: 131 + GeneralTestingChannelId: 529 + NETCoreToolingDevChannelId: 548 + NETCoreToolingReleaseChannelId: 549 + NETInternalToolingChannelId: 551 + NETCoreExperimentalChannelId: 562 + NetEngServicesIntChannelId: 678 + NetEngServicesProdChannelId: 679 + Net5Preview7ChannelId: 1065 + Net5Preview8ChannelId: 1155 + Net5RC1ChannelId: 1157 + NetCoreSDK313xxChannelId: 759 + NetCoreSDK313xxInternalChannelId: 760 + NetCoreSDK314xxChannelId: 921 + NetCoreSDK314xxInternalChannelId: 922 + VS166ChannelId: 1010 + VS167ChannelId: 1011 + VS168ChannelId: 1154 + VSMasterChannelId: 1012 + stages: -- stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate - variables: - - template: common-variables.yml - jobs: - - template: setup-maestro-vars.yml - - - job: - displayName: NuGet Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableNugetValidation }}, 'true') - pool: - vmImage: 'windows-2019' - variables: - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSigningValidation }}, 'true') +- ${{ if ne(parameters.inline, 'true') }}: + - stage: publish_using_darc + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Publish using Darc variables: - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@0 - displayName: 'Authenticate to AzDO Feeds' - - - task: PowerShell@2 - displayName: Enable cross-org publishing - inputs: - filePath: eng\common\enable-cross-org-publishing.ps1 - arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine dotnet - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - - - job: - displayName: SourceLink Validation - dependsOn: setupMaestroVars - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Publish Using Darc + dependsOn: setupMaestroVars + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Publish Using Darc + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) + -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' + -MaestroToken '$(MaestroApiAccessToken)' + -WaitPublishingFinish ${{ parameters.waitPublishingFinish }} + -EnableSourceLinkValidation ${{ parameters.enableSourceLinkValidation }} + -EnableSigningValidation ${{ parameters.enableSourceLinkValidation }} + -EnableNugetValidation ${{ parameters.enableSourceLinkValidation }} + -PublishInstallersAndChecksums ${{ parameters.publishInstallersAndChecksums }} + -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' + -SigningValidationAdditionalParameters '${{ parameters.signingValidationAdditionalParameters }}' + +- ${{ if eq(parameters.inline, 'true') }}: + - stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} + displayName: Validate Build Assets variables: - template: common-variables.yml - - name: AzDOProjectName - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] - - name: AzDOPipelineId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] - - name: AzDOBuildId - value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] - pool: - vmImage: 'windows-2019' - steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - - - template: /eng/common/templates/job/execute-sdl.yml - parameters: - enable: ${{ parameters.SDLValidationParameters.enable }} + jobs: + - template: setup-maestro-vars.yml + + - job: + displayName: Post-build Checks dependsOn: setupMaestroVars - additionalParameters: ${{ parameters.SDLValidationParameters.params }} - continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} - artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NetCore_Dev5_Publish' - channelName: '.NET Core 5 Dev' - channelId: 131 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview1_Publish' - channelName: '.NET 5 Preview 1' - channelId: 737 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net5_Preview2_Publish' - channelName: '.NET 5 Preview 2' - channelId: 738 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Latest_Publish' - channelName: '.NET Eng - Latest' - channelId: 2 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Validation_Publish' - channelName: '.NET Eng - Validation' - channelId: 9 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'General_Testing_Publish' - channelName: 'General Testing' - channelId: 529 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Dev_Publishing' - channelName: '.NET Core Tooling Dev' - channelId: 548 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Tooling_Release_Publishing' - channelName: '.NET Core Tooling Release' - channelId: 549 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-internal-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NET_Internal_Tooling_Publishing' - channelName: '.NET Internal Tooling' - channelId: 551 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'NETCore_Experimental_Publishing' - channelName: '.NET Core Experimental' - channelId: 562 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Int_Publish' - channelName: '.NET Eng Services - Int' - channelId: 678 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' - -- template: \eng\common\templates\post-build\channels\generic-public-channel.yml - parameters: - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - dependsOn: ${{ parameters.publishDependsOn }} - publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} - stageName: 'Net_Eng_Services_Prod_Publish' - channelName: '.NET Eng Services - Prod' - channelId: 679 - transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' - symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + variables: + - name: TargetChannels + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.TargetChannels'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: PowerShell@2 + displayName: Maestro Channels Consistency + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/check-channel-consistency.ps1 + arguments: -PromoteToChannels "$(TargetChannels)" + -AvailableChannelIds ${{parameters.NetEngLatestChannelId}},${{parameters.NetEngValidationChannelId}},${{parameters.NetDev5ChannelId}},${{parameters.GeneralTestingChannelId}},${{parameters.NETCoreToolingDevChannelId}},${{parameters.NETCoreToolingReleaseChannelId}},${{parameters.NETInternalToolingChannelId}},${{parameters.NETCoreExperimentalChannelId}},${{parameters.NetEngServicesIntChannelId}},${{parameters.NetEngServicesProdChannelId}},${{parameters.Net5Preview7ChannelId}},${{parameters.Net5Preview8ChannelId}},${{parameters.Net5RC1ChannelId}},${{parameters.NetCoreSDK313xxChannelId}},${{parameters.NetCoreSDK313xxInternalChannelId}},${{parameters.NetCoreSDK314xxChannelId}},${{parameters.NetCoreSDK314xxInternalChannelId}},${{parameters.VS166ChannelId}},${{parameters.VS167ChannelId}},${{parameters.VS168ChannelId}},${{parameters.VSMasterChannelId}} + + - job: + displayName: NuGet Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableNugetValidation }}, 'true') + pool: + vmImage: 'windows-2019' + variables: + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ + + - job: + displayName: Signing Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSigningValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - ${{ if eq(parameters.useBuildManifest, true) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download build manifest + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BuildManifests + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + + - job: + displayName: SourceLink Validation + dependsOn: setupMaestroVars + condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') + variables: + - template: common-variables.yml + - name: AzDOProjectName + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOProjectName'] ] + - name: AzDOPipelineId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOPipelineId'] ] + - name: AzDOBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.AzDOBuildId'] ] + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true + + - template: /eng/common/templates/job/execute-sdl.yml + parameters: + enable: ${{ parameters.SDLValidationParameters.enable }} + dependsOn: setupMaestroVars + additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} + downloadArtifacts: ${{ parameters.SDLValidationParameters.downloadArtifacts }} + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Dev5_Publish' + channelName: '.NET 5 Dev' + akaMSChannelName: 'net5/dev' + channelId: ${{ parameters.NetDev5ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview7_Publish' + channelName: '.NET 5 Preview 7' + akaMSChannelName: 'net5/preview7' + channelId: ${{ parameters.Net5Preview7ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_Preview8_Publish' + channelName: '.NET 5 Preview 8' + akaMSChannelName: 'net5/preview8' + channelId: ${{ parameters.Net5Preview8ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net5_RC1_Publish' + channelName: '.NET 5 RC 1' + akaMSChannelName: 'net5/rc1' + channelId: ${{ parameters.Net5RC1ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Latest_Publish' + channelName: '.NET Eng - Latest' + akaMSChannelName: 'eng/daily' + channelId: ${{ parameters.NetEngLatestChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Validation_Publish' + channelName: '.NET Eng - Validation' + akaMSChannelName: 'eng/validation' + channelId: ${{ parameters.NetEngValidationChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'General_Testing_Publish' + channelName: 'General Testing' + akaMSChannelName: 'generaltesting' + channelId: ${{ parameters.GeneralTestingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Dev_Publishing' + channelName: '.NET Core Tooling Dev' + channelId: ${{ parameters.NETCoreToolingDevChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Release_Publishing' + channelName: '.NET Core Tooling Release' + channelId: ${{ parameters.NETCoreToolingReleaseChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NET_Internal_Tooling_Publishing' + channelName: '.NET Internal Tooling' + channelId: ${{ parameters.NETInternalToolingChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet-tools-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Experimental_Publishing' + channelName: '.NET Core Experimental' + channelId: ${{ parameters.NETCoreExperimentalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-experimental-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Int_Publish' + channelName: '.NET Eng Services - Int' + channelId: ${{ parameters.NetEngServicesIntChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'Net_Eng_Services_Prod_Publish' + channelName: '.NET Eng Services - Prod' + channelId: ${{ parameters.NetEngServicesProdChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Publishing' + channelName: '.NET Core SDK 3.1.4xx' + channelId: ${{ parameters.NetCoreSDK314xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_314xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.4xx Internal' + channelId: ${{ parameters.NetCoreSDK314xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Publishing' + channelName: '.NET Core SDK 3.1.3xx' + channelId: ${{ parameters.NetCoreSDK313xxChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet3.1-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-internal-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_SDK_313xx_Internal_Publishing' + channelName: '.NET Core SDK 3.1.3xx Internal' + channelId: ${{ parameters.NetCoreSDK313xxInternalChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_6_Publishing' + channelName: 'VS 16.6' + channelId: ${{ parameters.VS166ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_7_Publishing' + channelName: 'VS 16.7' + channelId: ${{ parameters.VS167ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS16_8_Publishing' + channelName: 'VS 16.8' + channelId: ${{ parameters.VS168ChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' + + - template: \eng\common\templates\post-build\channels\generic-public-channel.yml + parameters: + artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'VS_Master_Publishing' + channelName: 'VS Master' + channelId: ${{ parameters.VSMasterChannelId }} + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 05e611edb6..b3d29d4498 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -5,7 +5,9 @@ jobs: - template: common-variables.yml - name: BuildId value: $[ coalesce(variables.BARBuildId, 0) ] - - name: PromoteToChannelId + - name: PromoteToMaestroChannels + value: $[ coalesce(variables.PromoteToChannelIds, 0) ] + - name: PromoteToMaestroChannel value: $[ coalesce(variables.PromoteToMaestroChannelId, 0) ] pool: vmImage: 'windows-2019' @@ -14,7 +16,7 @@ jobs: - task: DownloadBuildArtifacts@0 displayName: Download Release Configs - condition: eq(variables.PromoteToChannelId, 0) + condition: and(eq(variables.PromoteToMaestroChannels, 0), eq(variables.PromoteToMaestroChannel, 0)) inputs: buildType: current artifactName: ReleaseConfigs @@ -26,20 +28,16 @@ jobs: targetType: inline script: | try { - if ($Env:PromoteToChannelId -eq 0) { + if ($Env:PromoteToMaestroChannels -eq 0 -and $Env:PromoteToMaestroChannel -eq 0) { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt $BarId = $Content | Select -Index 0 - - $Channels = "" - $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } - + $Channels = $Content | Select -Index 1 $IsStableBuild = $Content | Select -Index 2 $AzureDevOpsProject = $Env:System_TeamProject $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId $AzureDevOpsBuildId = $Env:Build_BuildId - $PromoteToMaestroChannelId = 0 } else { $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" @@ -51,25 +49,23 @@ jobs: $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } $BarId = $Env:BARBuildId - $Channels = 'None' - - #TODO: Fix this once this issue is done: https://github.com/dotnet/arcade/issues/3834 - $IsStableBuild = 'False' + $Channels = $Env:PromoteToMaestroChannels -split "," + $Channels = $Channels -join "][" + $Channels = "[$Channels][$Env:PromoteToMaestroChannel]" + $IsStableBuild = $buildInfo.stable $AzureDevOpsProject = $buildInfo.azureDevOpsProject $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - $PromoteToMaestroChannelId = $Env:PromoteToMaestroChannelId } Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" - Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=TargetChannels;isOutput=true]$Channels" Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" Write-Host "##vso[task.setvariable variable=AzDOProjectName;isOutput=true]$AzureDevOpsProject" Write-Host "##vso[task.setvariable variable=AzDOPipelineId;isOutput=true]$AzureDevOpsBuildDefinitionId" Write-Host "##vso[task.setvariable variable=AzDOBuildId;isOutput=true]$AzureDevOpsBuildId" - Write-Host "##vso[task.setvariable variable=PromoteToMaestroChannelId;isOutput=true]$PromoteToMaestroChannelId" } catch { Write-Host $_ diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 30becf01ea..5eceb48725 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -10,7 +10,7 @@ parameters: HelixPostCommands: '' # optional -- commands to run after Helix work item execution WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in seconds for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects + WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload XUnitProjects: '' # optional -- semicolon delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects @@ -18,8 +18,8 @@ parameters: XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases.json + DotNetCliPackageType: '' # optional -- either 'sdk' or 'runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json + DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/master/release-notes/releases-index.json EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a45302b377..6f8bc41116 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -7,9 +7,11 @@ # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. [string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' } +# Set to true to opt out of outputting binary log while running in CI +[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false } + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci } +[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog } # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -55,10 +57,8 @@ set-strictmode -version 2.0 $ErrorActionPreference = 'Stop' [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 -function Create-Directory([string[]] $path) { - if (!(Test-Path $path)) { - New-Item -path $path -force -itemType 'Directory' | Out-Null - } +function Create-Directory ([string[]] $path) { + New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } function Unzip([string]$zipfile, [string]$outpath) { @@ -124,7 +124,9 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Find the first path on %PATH% that contains the dotnet.exe if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) { - $dotnetCmd = Get-Command 'dotnet.exe' -ErrorAction SilentlyContinue + $dotnetExecutable = GetExecutableFileName 'dotnet' + $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue + if ($dotnetCmd -ne $null) { $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent } @@ -283,12 +285,19 @@ function InstallDotNet([string] $dotnetRoot, # Throws on failure. # function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) { + if (-not (IsWindowsPlatform)) { + throw "Cannot initialize Visual Studio on non-Windows" + } + if (Test-Path variable:global:_MSBuildExe) { return $global:_MSBuildExe } + $vsMinVersionReqdStr = '16.5' + $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr) + if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs } - $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { '15.9' } + $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr } $vsMinVersion = [Version]::new($vsMinVersionStr) # Try msbuild command available in the environment. @@ -321,8 +330,18 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild' $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0] } else { - $vsMajorVersion = $vsMinVersion.Major - $xcopyMSBuildVersion = "$vsMajorVersion.$($vsMinVersion.Minor).0-alpha" + #if vs version provided in global.json is incompatible then use the default version for xcopy msbuild download + if($vsMinVersion -lt $vsMinVersionReqd){ + Write-Host "Using xcopy-msbuild version of $vsMinVersionReqdStr.0-alpha since VS version $vsMinVersionStr provided in global.json is not compatible" + $vsMajorVersion = $vsMinVersionReqd.Major + $vsMinorVersion = $vsMinVersionReqd.Minor + } + else{ + $vsMajorVersion = $vsMinVersion.Major + $vsMinorVersion = $vsMinVersion.Minor + } + + $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0-alpha" } $vsInstallDir = $null @@ -366,7 +385,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath + Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath Unzip $packagePath $packageDir } @@ -387,6 +406,10 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { # or $null if no instance meeting the requirements is found on the machine. # function LocateVisualStudio([object]$vsRequirements = $null){ + if (-not (IsWindowsPlatform)) { + throw "Cannot run vswhere on non-Windows platforms." + } + if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { @@ -452,7 +475,8 @@ function InitializeBuildTool() { Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot 'dotnet.exe'; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } + $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet') + $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = 'netcoreapp2.1' } } elseif ($msbuildEngine -eq "vs") { try { $msbuildPath = InitializeVisualStudioMSBuild -install:$restore @@ -488,10 +512,11 @@ function GetNuGetPackageCachePath() { if ($env:NUGET_PACKAGES -eq $null) { # Use local cache on CI to ensure deterministic build, # use global cache in dev builds to avoid cost of downloading packages. + # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968 if ($useGlobalNuGetCache) { - $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages' + $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\' } else { - $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages' + $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\' } } @@ -546,7 +571,7 @@ function InitializeToolset() { MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile - $path = Get-Content $toolsetLocationFile -TotalCount 1 + $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1 if (!(Test-Path $path)) { throw "Invalid toolset path: $path" } @@ -604,8 +629,8 @@ function MSBuild() { # function MSBuild-Core() { if ($ci) { - if (!$binaryLog) { - Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build.' + if (!$binaryLog -and !$excludeCIBinarylog) { + Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.' ExitWithExitCode 1 } @@ -632,10 +657,12 @@ function MSBuild-Core() { } } + $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs" + $exitCode = Exec-Process $buildTool.Path $cmdArgs if ($exitCode -ne 0) { - Write-PipelineTelemetryError Category 'Build' -Message 'Build failed.' + Write-PipelineTelemetryError -Category 'Build' -Message 'Build failed.' $buildLog = GetMSBuildBinaryLogCommandLineArgument $args if ($buildLog -ne $null) { @@ -663,6 +690,19 @@ function GetMSBuildBinaryLogCommandLineArgument($arguments) { return $null } +function GetExecutableFileName($baseName) { + if (IsWindowsPlatform) { + return "$baseName.exe" + } + else { + return $baseName + } +} + +function IsWindowsPlatform() { + return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT +} + . $PSScriptRoot\pipeline-logging-functions.ps1 $RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 664ac1055b..e94fce22ec 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -18,9 +18,17 @@ fi # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names. configuration=${configuration:-'Debug'} +# Set to true to opt out of outputting binary log while running in CI +exclude_ci_binary_log=${exclude_ci_binary_log:-false} + +if [[ "$ci" == true && "$exclude_ci_binary_log" == false ]]; then + binary_log_default=true +else + binary_log_default=false +fi + # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build. -# Binary log must be enabled on CI. -binary_log=${binary_log:-$ci} +binary_log=${binary_log:-$binary_log_default} # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes). prepare_machine=${prepare_machine:-false} @@ -77,7 +85,7 @@ function ResolvePath { function ReadGlobalVersion { local key=$1 - local line=`grep -m 1 "$key" "$global_json_file"` + local line=$(awk "/$key/ {print; exit}" "$global_json_file") local pattern="\"$key\" *: *\"(.*)\"" if [[ ! $line =~ $pattern ]]; then @@ -201,7 +209,14 @@ function InstallDotNet { local runtimeSourceFeedKey='' if [[ -n "${7:-}" ]]; then - decodedFeedKey=`echo $7 | base64 --decode` + # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' + # '-d'. To work around this, do a simple detection and switch the parameter + # accordingly. + decodeArg="--decode" + if base64 --help 2>&1 | grep -q "BusyBox"; then + decodeArg="-d" + fi + decodedFeedKey=`echo $7 | base64 $decodeArg` runtimeSourceFeedKey="--feed-credential $decodedFeedKey" fi @@ -397,8 +412,8 @@ function MSBuild { function MSBuild-Core { if [[ "$ci" == true ]]; then - if [[ "$binary_log" != true ]]; then - Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build." + if [[ "$binary_log" != true && "$exclude_ci_binary_log" != true ]]; then + Write-PipelineTelemetryError -category 'Build' "Binary log must be enabled in CI build, or explicitly opted-out from with the -noBinaryLog switch." ExitWithExitCode 1 fi @@ -415,11 +430,17 @@ function MSBuild-Core { warnaserror_switch="/warnaserror" fi - "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || { - local exit_code=$? - Write-PipelineTelemetryError -category 'Build' "Build failed (exit code '$exit_code')." - ExitWithExitCode $exit_code + function RunBuildTool { + export ARCADE_BUILD_TOOL_COMMAND="$_InitializeBuildTool $@" + + "$_InitializeBuildTool" "$@" || { + local exit_code=$? + Write-PipelineTaskError "Build failed (exit code '$exit_code')." + ExitWithExitCode $exit_code + } } + + RunBuildTool "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" } ResolvePath "${BASH_SOURCE[0]}" @@ -438,7 +459,7 @@ temp_dir="$artifacts_dir/tmp/$configuration" global_json_file="$repo_root/global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false -dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true +dotnetlocal_key=$(awk "/runtimes/ {print; exit}" "$global_json_file") || true if [[ -n "$dotnetlocal_key" ]]; then global_json_has_runtimes=true fi diff --git a/global.json b/global.json index 264f27e2ea..2f8fb88dd0 100644 --- a/global.json +++ b/global.json @@ -1,12 +1,17 @@ { "tools": { - "dotnet": "3.1.300-preview-015115", + "dotnet": "5.0.100-preview.6.20310.4", + "runtimes": { + "dotnet": [ + "3.1.0" + ] + }, "vs": { "version": "16.6" }, - "xcopy-msbuild": "16.3.0-alpha" + "xcopy-msbuild": "16.6.5-alpha1" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20224.11" + "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20374.1" } } diff --git a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj index b2f9bada28..d37b89d6b4 100644 --- a/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj +++ b/samples/Shared/UnitTestFramework/Roslyn.UnitTestFramework.csproj @@ -7,7 +7,7 @@ - + diff --git a/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.props b/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.props index e520741fa3..812274722c 100644 --- a/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.props +++ b/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.props @@ -1,5 +1,5 @@ - + $(NoWarn);NU5125;NU3005 @@ -8,38 +8,17 @@ - - true - false + MSTest - - true - false + NUnit - - - 16.5.0 - 2.0.0 - $(MSTestVersion) - $(MSTestVersion) - - - - - 3.12.0 - 3.15.1 - - - - - netcoreapp3.1;net472;net46;netcoreapp1.1;netcoreapp2.0 @@ -71,7 +50,7 @@ true $(MSBuildThisFileDirectory)..\..\RoslynSDK.ruleset - + diff --git a/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.targets b/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.targets index b74217fee4..c70b0af6f8 100644 --- a/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.targets +++ b/tests/Microsoft.CodeAnalysis.Testing/Directory.Build.targets @@ -1,11 +1,8 @@ - + - - - \ No newline at end of file diff --git a/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.props b/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.props deleted file mode 100644 index 8c119d5413..0000000000 --- a/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.props +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.targets b/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.targets deleted file mode 100644 index a8967a255b..0000000000 --- a/tests/Microsoft.CodeAnalysis.Testing/tools/MSTest/MSTest.targets +++ /dev/null @@ -1,99 +0,0 @@ - - - - - - - - - - - - - - <_TestEnvironment>%(TestToRun.EnvironmentDisplay) - <_TestAssembly>%(TestToRun.Identity) - <_TestRuntime>%(TestToRun.TestRuntime) - <_TestTimeout>%(TestToRun.TestTimeout) - <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments) - - <_TestRunnerTargetFramework>net451 - - - - <_MSTestConsoleExe>vstest.console.exe - <_MSTestConsoleExePath>$(NuGetPackageRoot)microsoft.testplatform/$(MicrosoftTestPlatformVersion)/tools/$(_TestRunnerTargetFramework)/Common7/IDE/Extensions/TestPlatform/$(_MSTestConsoleExe) - - - <_TestRunnerArgs>"$(_TestAssembly)" /Platform:%(TestToRun.Architecture) /Framework:%(TestToRun.TargetFramework) /Logger:trx;LogFileName="%(TestToRun.ResultsXmlPath)" $(_TestRunnerAdditionalArguments) - <_TestRunnerArgs Condition="'$(_TestRuntime)' == 'Mono'">$(TestRuntimeAdditionalArguments) "$(_MSTestConsoleExePath)" $(_TestRunnerArgs) - - - <_TestRunner Condition="'$(_TestRuntime)' == 'Mono'">$(MonoTool) - <_TestRunner Condition="'$(_TestRuntime)' != 'Mono'">$(_MSTestConsoleExePath) - - - - <_TestRunnerCommand>"$(_TestRunner)" $(_TestRunnerArgs) - - - <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestToRun.ResultsStdOutPath)" 2>&1 - - - - - <_OutputFiles Include="%(TestToRun.ResultsXmlPath)" /> - <_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" /> - - - - - - - - - - - - - - - - - - <_ResultsFileToDisplay>%(TestToRun.ResultsHtmlPath) - <_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath) - - - - - - - - - - diff --git a/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.props b/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.props deleted file mode 100644 index 736332286e..0000000000 --- a/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.props +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.targets b/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.targets deleted file mode 100644 index 7720be5d22..0000000000 --- a/tests/Microsoft.CodeAnalysis.Testing/tools/NUnit/NUnit.targets +++ /dev/null @@ -1,98 +0,0 @@ - - - - - - - - - - - - - - <_TestEnvironment>%(TestToRun.EnvironmentDisplay) - <_TestAssembly>%(TestToRun.Identity) - <_TestRuntime>%(TestToRun.TestRuntime) - <_TestTimeout>%(TestToRun.TestTimeout) - <_TestRunnerAdditionalArguments>%(TestToRun.TestRunnerAdditionalArguments) - - <_TestRunnerTargetFramework>net451 - - - - <_VSTestConsoleExe>vstest.console.exe - <_VSTestConsoleExePath>$(NuGetPackageRoot)microsoft.testplatform/$(MicrosoftTestPlatformVersion)/tools/$(_TestRunnerTargetFramework)/Common7/IDE/Extensions/TestPlatform/$(_VSTestConsoleExe) - - - <_TestRunnerArgs>"$(_TestAssembly)" /Platform:%(TestToRun.Architecture) /Framework:%(TestToRun.TargetFramework) /Logger:trx;LogFileName="%(TestToRun.ResultsXmlPath)" $(_TestRunnerAdditionalArguments) - <_TestRunnerArgs Condition="'$(_TestRuntime)' == 'Mono'">$(TestRuntimeAdditionalArguments) "$(_VSTestConsoleExePath)" $(_TestRunnerArgs) - - - <_TestRunner Condition="'$(_TestRuntime)' == 'Mono'">$(MonoTool) - <_TestRunner Condition="'$(_TestRuntime)' != 'Mono'">$(_VSTestConsoleExePath) - - - - <_TestRunnerCommand>"$(_TestRunner)" $(_TestRunnerArgs) - - - <_TestRunnerCommand Condition="'$(TestCaptureOutput)' != 'false'">$(_TestRunnerCommand) > "%(TestToRun.ResultsStdOutPath)" 2>&1 - - - - <_OutputFiles Include="%(TestToRun.ResultsXmlPath)" /> - <_OutputFiles Include="%(TestToRun.ResultsStdOutPath)" /> - - - - - - - - - - - - - - - - - - <_ResultsFileToDisplay>%(TestToRun.ResultsHtmlPath) - <_ResultsFileToDisplay Condition="!Exists('$(_ResultsFileToDisplay)')">%(TestToRun.ResultsStdOutPath) - - - - - - - - - - diff --git a/tests/VisualStudio.Roslyn.SDK/Directory.Build.targets b/tests/VisualStudio.Roslyn.SDK/Directory.Build.targets index 5ddf69387d..4884c0c1f0 100644 --- a/tests/VisualStudio.Roslyn.SDK/Directory.Build.targets +++ b/tests/VisualStudio.Roslyn.SDK/Directory.Build.targets @@ -1,6 +1,6 @@ - +