Making arcade self buildable

azure-pipelines.yml

@@ -28,27 +28,25 @@ jobs:
       variables:
       # Only enable publishing in non-public, non PR scenarios.
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
+        # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat
+        # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
         - group: DotNet-Blob-Feed
         - group: DotNet-Symbol-Server-Pats
+        - group: Publish-Build-Assets
         - name: _PublishBlobFeedUrl
           value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
-        - name: _SignArgs
+        - name: _InternalBuildArgs
           value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
-        - name: _PublishArgs
-          value: /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+            /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
             /p:DotNetPublishBlobFeedUrl=$(_PublishBlobFeedUrl)
             /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
             /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
             /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
-        - name: _OfficialBuildIdArgs
-          value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+            /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
       # else
       - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-        - name: _PublishArgs
-          value: ''
-        - name: _OfficialBuildIdArgs
-          value: ''
-        - name: _SignArgs
+        - name: _InternalBuildArgs
           value: ''
       strategy:
         matrix:
@@ -57,27 +55,31 @@ jobs:
             _PublishType: none
             _SignType: test
             _DotNetPublishToBlobFeed : false
+            _Script: eng\common\cibuild.cmd
+            _ValidateSdkArgs: ''
           Build_Release:
             _BuildConfig: Release
             # PRs or external builds are not signed.
             ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
               _PublishType: none
               _SignType: test
               _DotNetPublishToBlobFeed : false
+              _Script: eng\common\cibuild.cmd
             ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
               _PublishType: blob
               _SignType: real
               _DotNetPublishToBlobFeed : true
+              _Script: eng\validate-sdk.cmd
+              _ValidateSdkArgs: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) -barToken $(MaestroAccessToken)
       steps:
       - checkout: self
         clean: true
       # Use utility script to run script command dependent on agent OS.
-      - script: eng\common\cibuild.cmd
+      - script: $(_Script)
           -configuration $(_BuildConfig) 
           -prepareMachine
-          $(_PublishArgs)
-          $(_SignArgs)
-          $(_OfficialBuildIdArgs)
+          $(_InternalBuildArgs)
+          $(_ValidateSdkArgs)
         displayName: Windows Build / Publish
 
     - job: Linux

eng/Tools.props

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- All Rights Reserved. Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<Project>
+
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <!--When we are doing SDK validation we place the locally built packages in a custom path. This has to be propagated so the next
+        build can restore the local packages.-->
+    <RestoreSources Condition="'$(ArcadeSelfTest)' == 'true'">
+        $(RestoreSources);
+        $(ArtifactsDir)validatesdk\packages\$(Configuration)\NonShipping
+    </RestoreSources>
+  </PropertyGroup>
+</Project>

eng/common/CIBuild.cmd

@@ -1,3 +1,2 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
-exit /b %ErrorLevel%
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/common/build.ps1

@@ -4,6 +4,7 @@ Param(
   [string] $projects = "",
   [string][Alias('v')]$verbosity = "minimal",
   [string] $msbuildEngine = $null,
+  [string] $logFileName = "Build.binlog",
   [bool] $warnAsError = $true,
   [bool] $nodeReuse = $true,
   [switch] $execute,
@@ -57,6 +58,8 @@ function Print-Usage() {
     Write-Host "  -prepareMachine         Prepare machine for CI run"
     Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
     Write-Host ""
+    Write-Host "Optional settings:"
+    Write-Host "  -logFileName <value>    Binlog file name"
     Write-Host "Command line arguments not listed above are passed thru to msbuild."
     Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
 }
@@ -77,7 +80,7 @@ function InitializeCustomToolset {
 function Build {
   $toolsetBuildProj = InitializeToolset
   InitializeCustomToolset
-  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }
+  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir $logFileName) } else { "" }
 
   MSBuild $toolsetBuildProj `
     $bl `

eng/common/build.sh

@@ -27,6 +27,8 @@ usage()
   echo "  --nodeReuse <value>      Sets nodereuse msbuild parameter ('true' or 'false')"
   echo "  --warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   echo ""
+  echo "Optional settings:"
+  echo "  -logFileName <value>    Binlog file name"
   echo "Command line arguments starting with '/p:' are passed through to MSBuild."
 }
 
@@ -62,6 +64,7 @@ projects=''
 configuration='Debug'
 prepare_machine=false
 verbosity='minimal'
+logFileName='Build.binlog'
 properties=''
 
 while [[ $# > 0 ]]; do
@@ -165,7 +168,7 @@ function Build {
 
   local bl=""
   if [[ "$binary_log" == true ]]; then
-    bl="/bl:\"$log_dir/Build.binlog\""
+    bl="/bl:\"$log_dir/$logFileName\""
   fi
 
   MSBuild $_InitializeToolset \

eng/validate-sdk.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0validate-sdk.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/validate-sdk.ps1

@@ -0,0 +1,95 @@
+Param(
+  [string] $barToken,
+  [string] $gitHubPat,
+  [string] $configuration = "Debug"
+)
+
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\common\tools.ps1
+$LocalNugetConfigSourceName = "arcade-local"
+
+function Check-ExitCode ($exitCode)
+{
+  if ($exitCode -ne 0) {
+    Write-Host "Arcade self-build failed"
+    ExitWithExitCode $exitCode
+  }
+}
+
+function StopDotnetIfRunning
+{
+    $dotnet = Get-Process "dotnet" -ErrorAction SilentlyContinue
+    if ($dotnet) {
+        stop-process $dotnet
+    }
+}
+
+function AddSourceToNugetConfig([string]$nugetConfigPath, [string]$source) 
+{
+    Write-Host "Adding '$source' to '$nugetConfigPath'..."
+
+    $nugetConfig = New-Object XML
+    $nugetConfig.PreserveWhitespace = $true
+    $nugetConfig.Load($nugetConfigPath)
+    $packageSources = $nugetConfig.SelectSingleNode("//packageSources")
+    $keyAttribute = $nugetConfig.CreateAttribute("key")
+    $keyAttribute.Value = $LocalNugetConfigSourceName
+    $valueAttribute = $nugetConfig.CreateAttribute("value")
+    $valueAttribute.Value = $source
+    $newSource = $nugetConfig.CreateElement("add")
+    $newSource.Attributes.Append($keyAttribute)
+    $newSource.Attributes.Append($valueAttribute)
+    $packageSources.AppendChild($newSource)
+    $nugetConfig.Save($nugetConfigPath)
+}
+
+try {
+  Write-Host "STEP 1: Build and create local packages"
+
+  Push-Location $PSScriptRoot
+
+  $validateSdkDir = "$ArtifactsDir\validatesdk\\"
+  $packagesSource = "$validateSdkDir\packages\$configuration\NonShipping"  
+  $toolsProjPath = "$RepoRoot\src\Microsoft.DotNet.Arcade.Sdk\tools\Tools.proj"
+  $nugetConfigPath = "$RepoRoot\NuGet.config"
+
+  # When restoring, we check if local sources defined in Tools.proj actually exist so we need to create
+  # the validation SDK folder beforehand
+  if (!(Test-Path -Path $packagesSource)) {
+    Create-Directory $packagesSource
+  }
+
+  . .\common\build.ps1 -restore -build -pack -configuration $configuration -logFileName "Build_Local.binlog" /p:ArtifactsDir=$validateSdkDir
+
+  Write-Host "STEP 2: Build using the local packages"
+
+  AddSourceToNugetConfig $nugetConfigPath $packagesSource
+
+  Write-Host "Updating Dependencies using Darc..."
+
+  . .\common\darc-init.ps1
+
+  $DarcExe = "$env:USERPROFILE\.dotnet\tools"
+  $DarcExe = Resolve-Path $DarcExe
+
+  & $DarcExe\darc.exe update-dependencies --packages-folder $packagesSource --password $barToken --github-pat $gitHubPat --channel ".NET Tools - Latest"
+
+  Check-ExitCode $lastExitCode
+  StopDotnetIfRunning
+
+  Write-Host "Building with updated dependencies"
+
+  . .\common\build.ps1 -configuration $configuration @Args  /p:ArcadeSelfTest=true
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}
+finally {
+  Write-Host "Cleaning up workspace..."
+  StopDotnetIfRunning
+  Pop-Location
+  Write-Host "Finished building Arcade SDK with validation enabled!"
+}