Making arcade self buildable

azure-pipelines.yml

@@ -26,29 +26,31 @@ jobs:
         ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
           name: dotnet-internal-temp
       variables:
+      - name: _Script
+        value: eng\common\cibuild.cmd
+      - name: _ValidateSdkArgs
+        value: ''          
       # Only enable publishing in non-public, non PR scenarios.
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        # DotNet-Blob-Feed provides: dotnetfeed-storage-access-key-1
+        # DotNet-Symbol-Server-Pats provides: microsoft-symbol-server-pat, symweb-symbol-server-pat
+        # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
         - group: DotNet-Blob-Feed
         - group: DotNet-Symbol-Server-Pats
+        - group: Publish-Build-Assets
         - name: _PublishBlobFeedUrl
           value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
-        - name: _SignArgs
+        - name: _InternalBuildArgs
           value: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
-        - name: _PublishArgs
-          value: /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+            /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
             /p:DotNetPublishBlobFeedUrl=$(_PublishBlobFeedUrl)
             /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
             /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
             /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
-        - name: _OfficialBuildIdArgs
-          value: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
+            /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
       # else
       - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
-        - name: _PublishArgs
-          value: ''
-        - name: _OfficialBuildIdArgs
-          value: ''
-        - name: _SignArgs
+        - name: _InternalBuildArgs
           value: ''
       strategy:
         matrix:
@@ -68,16 +70,17 @@ jobs:
               _PublishType: blob
               _SignType: real
               _DotNetPublishToBlobFeed : true
+              _Script: eng\validate-sdk.cmd
+              _ValidateSdkArgs: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) -barToken $(MaestroAccessToken)
       steps:
       - checkout: self
         clean: true
       # Use utility script to run script command dependent on agent OS.
-      - script: eng\common\cibuild.cmd
+      - script: $(_Script)
           -configuration $(_BuildConfig) 
           -prepareMachine
-          $(_PublishArgs)
-          $(_SignArgs)
-          $(_OfficialBuildIdArgs)
+          $(_InternalBuildArgs)
+          $(_ValidateSdkArgs)
         displayName: Windows Build / Publish
 
     - job: Linux

eng/Tools.props

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- All Rights Reserved. Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. See the LICENSE file in the project root for more information. -->
+<Project>
+
+  <PropertyGroup>
+    <MSBuildAllProjects>$(MSBuildAllProjects);$(MSBuildThisFileFullPath)</MSBuildAllProjects>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <!--When we are doing SDK validation we place the locally built packages in a custom path. This has to be propagated so the next
+        build can restore the local packages.-->
+    <RestoreSources Condition="'$(AdditionalRestoreSources)' != ''">
+        $(RestoreSources);
+        $(AdditionalRestoreSources)
+    </RestoreSources>
+  </PropertyGroup>
+</Project>

eng/Version.Details.xml

@@ -1,23 +1,23 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Dependencies>
   <ProductDependencies>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.18607.6">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.Feed" Version="2.2.0-beta.18610.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e8b77277ce388a80223941927a0e4388d02b7233</Sha>
+      <Sha>6f6f4e5b974772c9949ed0f0baaf521a8c7991e4</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Maestro.Tasks" Version="1.0.0-beta.18607.6">
+    <Dependency Name="Microsoft.DotNet.Maestro.Tasks" Version="1.0.0-beta.18610.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e8b77277ce388a80223941927a0e4388d02b7233</Sha>
+      <Sha>6f6f4e5b974772c9949ed0f0baaf521a8c7991e4</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.18607.6">
+    <Dependency Name="Microsoft.DotNet.SignTool" Version="1.0.0-beta.18610.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e8b77277ce388a80223941927a0e4388d02b7233</Sha>
+      <Sha>6f6f4e5b974772c9949ed0f0baaf521a8c7991e4</Sha>
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18607.6">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="1.0.0-beta.18610.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>e8b77277ce388a80223941927a0e4388d02b7233</Sha>
+      <Sha>6f6f4e5b974772c9949ed0f0baaf521a8c7991e4</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/Versions.props

@@ -46,9 +46,9 @@
     <WindowsAzureStorageVersion>8.5.0</WindowsAzureStorageVersion>
     <XUnitVersion>2.4.1-pre.build.4059</XUnitVersion>
     <XUnitVSRunnerVersion>2.4.1-pre.build.4059</XUnitVSRunnerVersion>
-    <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.18607.6</MicrosoftDotNetBuildTasksFeedVersion>
-    <MicrosoftDotNetMaestroTasksVersion>1.0.0-beta.18607.6</MicrosoftDotNetMaestroTasksVersion>
-    <MicrosoftDotNetSignToolVersion>1.0.0-beta.18607.6</MicrosoftDotNetSignToolVersion>
+    <MicrosoftDotNetBuildTasksFeedVersion>2.2.0-beta.18610.4</MicrosoftDotNetBuildTasksFeedVersion>
+    <MicrosoftDotNetMaestroTasksVersion>1.0.0-beta.18610.4</MicrosoftDotNetMaestroTasksVersion>
+    <MicrosoftDotNetSignToolVersion>1.0.0-beta.18610.4</MicrosoftDotNetSignToolVersion>
     <MicrosoftAzureDocumentDBVersion>1.22.0</MicrosoftAzureDocumentDBVersion>
     <MicrosoftAzureCosmosDBTableVersion>1.1.2</MicrosoftAzureCosmosDBTableVersion>
     <MicrosoftAspNetCoreAllVersion>2.0.0</MicrosoftAspNetCoreAllVersion>

eng/common/CIBuild.cmd

@@ -1,3 +1,2 @@
 @echo off
-powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"
-exit /b %ErrorLevel%
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0Build.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/validate-sdk.cmd

@@ -0,0 +1,2 @@
+@echo off
+powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0validate-sdk.ps1""" -restore -build -test -sign -pack -publish -ci %*"

eng/validate-sdk.ps1

@@ -0,0 +1,105 @@
+Param(
+  [string] $barToken,
+  [string] $gitHubPat,
+  [string] $configuration = "Debug"
+)
+
+$ErrorActionPreference = "Stop"
+. $PSScriptRoot\common\tools.ps1
+$LocalNugetConfigSourceName = "arcade-local"
+
+function Check-ExitCode ($exitCode)
+{
+  if ($exitCode -ne 0) {
+    Write-Host "Arcade self-build failed"
+    ExitWithExitCode $exitCode
+  }
+}
+
+function StopDotnetIfRunning
+{
+    $dotnet = Get-Process "dotnet" -ErrorAction SilentlyContinue
+    if ($dotnet) {
+        stop-process $dotnet
+    }
+}
+
+function AddSourceToNugetConfig([string]$nugetConfigPath, [string]$source) 
+{
+    Write-Host "Adding '$source' to '$nugetConfigPath'..."
+
+    $nugetConfig = New-Object XML
+    $nugetConfig.PreserveWhitespace = $true
+    $nugetConfig.Load($nugetConfigPath)
+    $packageSources = $nugetConfig.SelectSingleNode("//packageSources")
+    $keyAttribute = $nugetConfig.CreateAttribute("key")
+    $keyAttribute.Value = $LocalNugetConfigSourceName
+    $valueAttribute = $nugetConfig.CreateAttribute("value")
+    $valueAttribute.Value = $source
+    $newSource = $nugetConfig.CreateElement("add")
+    $newSource.Attributes.Append($keyAttribute)
+    $newSource.Attributes.Append($valueAttribute)
+    $packageSources.AppendChild($newSource)
+    $nugetConfig.Save($nugetConfigPath)
+}
+
+function MoveArtifactsToValidateSdkFolder([string]$artifactsDir, [string]$validateSdkFolderName, [string]$repoRoot)
+{
+    Rename-Item -Path $artifactsDir -NewName $validateSdkFolderName -Force
+
+    if (!(Test-Path -Path $artifactsDir)) {
+        Create-Directory $artifactsDir
+    }
+
+    Move-Item -Path (Join-Path $repoRoot $validateSdkFolderName) -Destination $artifactsDir -Force
+}
+
+try {
+  Write-Host "STEP 1: Build and create local packages"
+
+  Push-Location $PSScriptRoot
+
+  $validateSdkFolderName = "validatesdk"
+  $validateSdkDir = Join-Path $ArtifactsDir $validateSdkFolderName
+  $packagesSource = Join-Path (Join-Path (Join-Path $validateSdkDir "packages") $configuration) "NonShipping"
+  $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
+
+  . .\common\build.ps1 -restore -build -pack -configuration $configuration
+
+  # This is a temporary solution. When https://github.com/dotnet/arcade/issues/1293 is closed
+  # we'll be able to pass a container name to build.ps1 which will put the outputs in the
+  # artifacts-<container-name> folder.
+  MoveArtifactsToValidateSdkFolder $ArtifactsDir $validateSdkFolderName $RepoRoot
+
+  Write-Host "STEP 2: Build using the local packages"
+
+  AddSourceToNugetConfig $nugetConfigPath $packagesSource
+
+  Write-Host "Updating Dependencies using Darc..."
+
+  . .\common\darc-init.ps1
+
+  $DarcExe = "$env:USERPROFILE\.dotnet\tools"
+  $DarcExe = Resolve-Path $DarcExe
+
+  & $DarcExe\darc.exe update-dependencies --packages-folder $packagesSource --password $barToken --github-pat $gitHubPat --channel ".NET Tools - Latest"
+
+  Check-ExitCode $lastExitCode
+  StopDotnetIfRunning
+
+  Write-Host "Building with updated dependencies"
+
+  . .\common\build.ps1 -configuration $configuration @Args  /p:AdditionalRestoreSources=$packagesSource
+}
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+}
+finally {
+  Write-Host "Cleaning up workspace..."
+  StopDotnetIfRunning
+  Pop-Location
+  Write-Host "Finished building Arcade SDK with validation enabled!"
+}

global.json

@@ -6,6 +6,6 @@
     "version": "2.1.400-preview-009088"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18607.6"
+    "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.18610.4"
   }
 }