Build the "in build" crossgen2 into a pack that we can use within the VMR

eng/DotNetBuild.props

@@ -107,6 +107,9 @@
 
       <!-- Needed until https://github.com/dotnet/runtime/issues/109329 is fixed. -->
       <InnerBuildArgs Condition="'$(NetCoreAppToolCurrentVersion)' != ''">$(InnerBuildArgs) /p:NetCoreAppToolCurrentVersion=$(NetCoreAppToolCurrentVersion)</InnerBuildArgs>
+
+      <!-- Pass ShortStack to the inner build so we can skip building host-machine components for downstream repos when we're the last repo. -->
+      <InnerBuildArgs Condition="'$(ShortStack)' == 'true'">$(InnerBuildArgs) /p:ShortStack=true</InnerBuildArgs>
     </PropertyGroup>
   </Target>
 

eng/Signing.props

@@ -1,6 +1,6 @@
 <Project>
   <PropertyGroup>
-    <EnableDefaultArtifacts Condition="'$(DotNetBuild)' != 'true'">false</EnableDefaultArtifacts>
+    <EnableDefaultArtifacts>false</EnableDefaultArtifacts>
   </PropertyGroup>
 
   <ItemGroup>
@@ -27,10 +27,10 @@
 
     <FileExtensionSignInfo Include=".msi" CertificateName="MicrosoftDotNet500" />
 
-    <!-- 
+    <!--
       Removal is temporarily needed as we integrate support for these extensions into SignTool.
       Should be cleaned up after https://github.com/dotnet/arcade/issues/14432,
-      https://github.com/dotnet/arcade/issues/14433, and 
+      https://github.com/dotnet/arcade/issues/14433, and
       https://github.com/dotnet/arcade/issues/14435 are completed.
      -->
     <FileExtensionSignInfo Remove=".deb;.rpm;.pkg" />
@@ -45,20 +45,35 @@
     <FileSignInfo Update="@(FileSignInfo->WithMetadataValue('CertificateName','Microsoft400'))" CertificateName="MicrosoftDotNet500" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(DotNetBuildOrchestrator)' == 'true'">
+    <VerticalVisibilityNuGetPackages Condition="'$(RuntimeFlavor)' != 'Mono' and '$(ShortStack)' != 'true' and '$(OutputRID)' != '$(NETCoreSdkRuntimeIdentifier)'"
+                                     Include="$(ArtifactsPackagesDir)**\Microsoft.NETCore.App.Crossgen2.$(NETCoreSdkRuntimeIdentifier).*.nupkg"
+                                     Visibility="Vertical" />
+  </ItemGroup>
+
   <!-- In build signing and publishing without a join point -->
   <ItemGroup Condition="'$(DotNetBuild)' == 'true'">
-    <Artifact Include="$(ArtifactsPackagesDir)**\*.tar.gz;
-                       $(ArtifactsPackagesDir)**\*.zip;
-                       $(ArtifactsPackagesDir)**\*.deb;
-                       $(ArtifactsPackagesDir)**\*.rpm;
-                       $(ArtifactsPackagesDir)**\*.pkg;
-                       $(ArtifactsPackagesDir)**\*.exe;
-                       $(ArtifactsPackagesDir)**\*.msi"
-              Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz"
-              IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))">
-      <!-- Exclude wixpack.zip files from checksum generation -->
-      <ChecksumPath Condition="$([System.String]::Copy('%(Filename)%(Extension)').EndsWith('.wixpack.zip')) != 'true'">%(FullPath).sha512</ChecksumPath>
-    </Artifact>
+    <NuGetPackages Include="$(ArtifactsPackagesDir)**\*.nupkg" Exclude="@(VerticalVisibilityNuGetPackages)" />
+
+    <Archives Include="$(ArtifactsPackagesDir)**\*.tar.gz;
+                       $(ArtifactsPackagesDir)**\*.zip"
+              Exclude="$(ArtifactsPackagesDir)**\Symbols.runtime.tar.gz;
+                       $(ArtifactsPackagesDir)**\*.wixpack.zip">
+      <ChecksumPath>%(FullPath).sha512</ChecksumPath>
+    </Archives>
+
+    <WixPacks Include="$(ArtifactsPackagesDir)**\*.wixpack.zip" />
+
+    <Installers Include="$(ArtifactsPackagesDir)**\*.deb;
+                         $(ArtifactsPackagesDir)**\*.rpm;
+                         $(ArtifactsPackagesDir)**\*.pkg;
+                         $(ArtifactsPackagesDir)**\*.exe;
+                         $(ArtifactsPackagesDir)**\*.msi">
+      <ChecksumPath>%(FullPath).sha512</ChecksumPath>
+    </Installers>
+
+    <Artifact Include="@(VerticalVisibilityNuGetPackages);@(NuGetPackages);@(Archives);@(WixPacks);@(Installers)"
+              IsShipping="$([System.String]::Copy('%(RecursiveDir)').StartsWith('Shipping'))" />
   </ItemGroup>
 
   <!-- Only the following artifacts should be signed.
@@ -71,4 +86,4 @@
               SkipPublish="true" />
   </ItemGroup>
 
-</Project>
+</Project>

eng/Subsets.props

@@ -548,6 +548,13 @@
       <ItemGroup>
         <ProjectToBuild Condition="'$(NativeAotSupported)' == 'true' and '$(RuntimeFlavor)' != 'Mono' and '$(BuildOnlyPgoInstrumentedAssets)' != 'true'" Include="$(InstallerProjectRoot)\pkg\projects\nativeaot-packages.proj" Category="packs" />
       </ItemGroup>
+      <ItemGroup>
+        <!--
+          When we're building in the VMR, we need to provide a crossgen2 that runs on the host machine for downstream repos to use to R2R their code.
+          In non-VMR builds, downstream repos can use the crossgen2 built for the target host SDK from another build leg, but in the VMR we need to provide one to use.
+        -->
+        <ProjectToBuild Condition="'$(RuntimeFlavor)' != 'Mono' and '$(DotNetBuildOrchestrator)' == 'true' and '$(ShortStack)' != 'true' and '$(OutputRID)' != '$(NETCoreSdkRuntimeIdentifier)'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Crossgen2.Host.sfxproj" Category="packs" />
+      </ItemGroup>
       <ItemGroup>
         <SharedFrameworkProjectToBuild Condition="'$(BuildMonoAOTCrossCompilerOnly)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\Microsoft.NETCore.App\Microsoft.NETCore.App.Runtime.sfxproj" />
         <SharedFrameworkProjectToBuild Condition="'$(BuildNativeAOTRuntimePack)' != 'true' and '$(RuntimeFlavor)' == '$(PrimaryRuntimeFlavor)' and '$(TargetsMobile)' != 'true' and '$(DotNetBuildSourceOnly)' != 'true'" Include="$(InstallerProjectRoot)pkg\sfx\bundle\Microsoft.NETCore.App.Bundle.bundleproj" />

src/coreclr/crossgen-corelib.proj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.Build.NoTargets">
 
   <ItemGroup>
-    <ProjectReference Include="$(CoreClrProjectRoot)/tools/aot/crossgen2/crossgen2_inbuild.csproj" OutputItemType="Crossgen2" />
+    <ProjectReference Include="$(CoreClrProjectRoot)/tools/aot/crossgen2/crossgen2_inbuild.csproj" OutputItemType="Crossgen2Files" />
     <ProjectReference Include="$(CoreClrProjectRoot)/tools/dotnet-pgo/dotnet-pgo.csproj" OutputItemType="DotNetPgo" Condition="'$(DotNetBuildSourceOnly)' != 'true'" />
     <ProjectReference Include="$([MSBuild]::NormalizePath('$(CoreClrProjectRoot)', 'System.Private.CoreLib', 'System.Private.CoreLib.csproj'))" OutputItemType="CoreLib" />
   </ItemGroup>
@@ -92,21 +92,13 @@
     <ItemGroup>
       <Crossgen2Inputs Include="@(CoreLib)" />
       <Crossgen2Inputs Include="$(MergedMibcPath)" />
-    </ItemGroup>
-
-    <PropertyGroup>
-      <Crossgen2OutputPath>%(Crossgen2.RootDir)/%(Crossgen2.Directory)</Crossgen2OutputPath>
-    </PropertyGroup>
-    <ItemGroup>
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.dll" />
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.so" />
-      <Crossgen2Files Include="$(Crossgen2OutputPath)/*.dylib" />
+      <Crossgen2Inputs Include="@(Crossgen2Files->Metadata('OutputPath'))" />
     </ItemGroup>
   </Target>
 
   <Target Name="InvokeCrossgen"
           DependsOnTargets="PrepareInvokeCrossgen;CreateMergedMibcFile"
-          Inputs="@(Crossgen2Inputs);@(Crossgen2Files)"
+          Inputs="@(Crossgen2Inputs)"
           Outputs="$(CoreLibOutputPath);$(CoreLibNiPdbPath);$(CoreLibPerfMapPath)"
           AfterTargets="Build">
 
@@ -117,7 +109,7 @@
       Text="Generating native image of System.Private.CoreLib for $(OSPlatformConfig). Logging to $(CrossGenCoreLibLog)" />
 
     <PropertyGroup>
-      <CrossGenDllCmd>$(DotNetCli) @(Crossgen2)</CrossGenDllCmd>
+      <CrossGenDllCmd>@(Crossgen2Files->Metadata('OutputPath')->WithMetadataValue('Filename','crossgen2')->WithMetadataValue('Extension','$(ExeSuffix)'))</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -o:$(CoreLibOutputPath)</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) -r:$([MSBuild]::NormalizePath('$(BinDir)', 'IL', '*.dll'))</CrossGenDllCmd>
       <CrossGenDllCmd>$(CrossGenDllCmd) --targetarch:$(TargetArchitecture)</CrossGenDllCmd>

src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj

@@ -1,9 +1,20 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Publish;PublishItemsOutputGroup">
   <PropertyGroup>
     <CrossHostArch Condition="'$(CrossBuild)' == 'true' or '$(TargetArchitecture)' != '$(BuildArchitecture)' or '$(HostOS)' != '$(TargetOS)' or '$(EnableNativeSanitizers)' != ''">$(BuildArchitecture)</CrossHostArch>
-    <OutputPath>$(RuntimeBinDir)/$(BuildArchitecture)/crossgen2/</OutputPath>
-    <TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
-    <UseAppHost>false</UseAppHost>
+    <_IsPublishing>true</_IsPublishing>
+    <RuntimeIdentifier>$(NETCoreSdkRuntimeIdentifier)</RuntimeIdentifier>
+    <PublishDir>$(RuntimeBinDir)$(BuildArchitecture)/crossgen2/</PublishDir>
+    <SelfContained>true</SelfContained>
+    <PublishTrimmed>true</PublishTrimmed>
+    <!--
+      Publish with AOT on Windows as Single file doesn't work well when an app has to include DiaSymReader.Native.
+      Publish single-file elsewhere so we don't need to adjust our "AOT in build" logic to handle targeting the SDK RID.
+    -->
+    <PublishAot Condition="$(NETCoreSdkPortableRuntimeIdentifier.StartsWith('win-'))">true</PublishAot>
+    <PublishSingleFile Condition="'$(PublishAot)' != 'true'">true</PublishSingleFile>
+    <PublishReadyToRun>true</PublishReadyToRun>
+    <PublishReadyToRunComposite>true</PublishReadyToRunComposite>
+    <TrimmerSingleWarn>false</TrimmerSingleWarn>
   </PropertyGroup>
   <Import Project="crossgen2.props" />
 </Project>

src/installer/pkg/projects/Directory.Build.props

@@ -2,6 +2,19 @@
   <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props, $(MSBuildThisFileDirectory)..))" />
 
   <PropertyGroup>
+
+    <PackageIndexFile>$(MSBuildThisFileDirectory)packageIndex.json</PackageIndexFile>
+
+    <!-- this repo doesn't currently use the index so don't force it to be in sync -->
+    <SkipIndexCheck>true</SkipIndexCheck>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageIndex Include="$(PackageIndexFile)" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <Platform>$(TargetArchitecture)</Platform>
     <PackagePlatform>AnyCPU</PackagePlatform>
 
     <SkipPackageFileCheck>true</SkipPackageFileCheck>

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.Host.sfxproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Sdk Name="Microsoft.DotNet.SharedFramework.Sdk" />
+
+  <PropertyGroup>
+    <CrossgenPublishProject>$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_inbuild.csproj</CrossgenPublishProject>
+    <RuntimeIdentifier>$(NETCoreSdkRuntimeIdentifier)</RuntimeIdentifier>
+    <IsShipping>false</IsShipping>
+  </PropertyGroup>
+
+  <Import Project="$(MSBuildThisFileDirectory)Microsoft.NETCore.App.Crossgen2.props" />
+</Project>

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.props

@@ -0,0 +1,59 @@
+<Project>
+  <PropertyGroup>
+    <!-- Crossgen is not used for Mono -->
+    <SkipBuild Condition="'$(RuntimeFlavor)' == 'Mono'">true</SkipBuild>
+    <PlatformPackageType>ToolPack</PlatformPackageType>
+    <SharedFrameworkName>$(SharedFrameworkName).Crossgen2</SharedFrameworkName>
+    <OverridePackageId>$(SharedFrameworkName).$(RuntimeIdentifier)</OverridePackageId>
+    <ArchiveName>dotnet-crossgen2</ArchiveName>
+    <SharedFrameworkHostFileNameOverride>crossgen2</SharedFrameworkHostFileNameOverride>
+    <GenerateInstallers>false</GenerateInstallers>
+    <PermitDllAndExeFilesLackingFileVersion>true</PermitDllAndExeFilesLackingFileVersion>
+    <!-- Publishing as single-file or NativeAOT means we can't examine the interior DLLs -->
+    <ShouldVerifyClosure>false</ShouldVerifyClosure>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference
+      Include="$(CrossgenPublishProject)"
+      OutputItemType="_RawCrossgenPublishFiles"
+      ReferenceOutputAssembly="false" />
+  </ItemGroup>
+
+  <Target Name="PublishCrossgen"
+          BeforeTargets="GetFilesToPackage">
+    <ItemGroup>
+      <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')"
+                             KeepMetadata="REMOVE_ALL" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(NativeAotSupported)' != 'true'">
+      <FilesToPackage Include="@(_CrossgenPublishFiles)"
+                      Exclude="*.pdb;*.h;*.lib"
+                      TargetPath="tools/" />
+    </ItemGroup>
+    <ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
+      <!-- Treat all native aot assets as native runtime assets -->
+      <FilesToPackage Include="@(_CrossgenPublishFiles->Distinct())"
+                          Condition="'%(Extension)' != '.pdb'"
+                          TargetPath="tools/" />
+    </ItemGroup>
+  </Target>
+
+  <PropertyGroup>
+    <TargetOSComponent>unix</TargetOSComponent>
+    <TargetOSComponent Condition="'$(TargetOS)' == 'windows'">win</TargetOSComponent>
+    <TargetSpec>$(TargetOSComponent)-$(TargetArchitecture)</TargetSpec>
+  </PropertyGroup>
+
+  <Target Name="AddCrossgen2SymbolFilesToPackage" BeforeTargets="GetFilesToPackage" DependsOnTargets="PublishCrossgen">
+    <ItemGroup>
+      <_Crossgen2SymbolFilesToPackage Include="@(_CrossgenPublishFiles)" Condition="'%(Extension)' == '.pdb'" />
+      <!-- Symbol files for JIT libraries are placed in a different location for Windows builds -->
+      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPdbDir)%(FileName).pdb')" Condition="'$(TargetOS)' == 'windows' and '%(FileName)' != 'crossgen2'" />
+      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPath)%(FileName)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows' and '%(FileName)' != 'crossgen2'" />
+      <_Crossgen2SymbolFilesToPackage Remove="@(_Crossgen2SymbolFilesToPackage)" Condition="!Exists('%(Identity)')" />
+
+      <_SymbolFilesToPackage Include="@(_Crossgen2SymbolFilesToPackage)" TargetPath="tools/" />
+    </ItemGroup>
+  </Target>
+</Project>

src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj

@@ -1,51 +1,15 @@
-<Project>
-  <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
-  <Import Project="Sdk.props" Sdk="Microsoft.DotNet.SharedFramework.Sdk" />
+<Project Sdk="Microsoft.NET.Sdk">
+  <Sdk Name="Microsoft.DotNet.SharedFramework.Sdk" />
 
   <PropertyGroup>
-    <!-- Crossgen is not used for Mono -->
-    <SkipBuild Condition="'$(RuntimeFlavor)' == 'Mono'">true</SkipBuild>
-    <PlatformPackageType>ToolPack</PlatformPackageType>
-    <SharedFrameworkName>$(SharedFrameworkName).Crossgen2</SharedFrameworkName>
-    <OverridePackageId>$(SharedFrameworkName).$(RuntimeIdentifier)</OverridePackageId>
-    <ArchiveName>dotnet-crossgen2</ArchiveName>
-    <SharedFrameworkHostFileNameOverride>crossgen2</SharedFrameworkHostFileNameOverride>
-    <GenerateInstallers>false</GenerateInstallers>
-    <HostJsonTargetPath>tools/</HostJsonTargetPath>
-    <PermitDllAndExeFilesLackingFileVersion>true</PermitDllAndExeFilesLackingFileVersion>
-    <!-- Publishing as single-file or NativeAOT means we can't examine the interior DLLs -->
-    <ShouldVerifyClosure>false</ShouldVerifyClosure>
+    <CrossgenPublishProject>$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj</CrossgenPublishProject>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Microsoft.DotNet.Build.Tasks.Archives" Version="$(MicrosoftDotNetBuildTasksArchivesVersion)" />
   </ItemGroup>
 
-  <ItemGroup>
-    <ProjectReference
-      Include="$(RepoRoot)src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj"
-      OutputItemType="_RawCrossgenPublishFiles"
-      ReferenceOutputAssembly="false" />
-  </ItemGroup>
-
-  <Target Name="PublishCrossgen"
-          BeforeTargets="GetFilesToPackage">
-    <ItemGroup>
-      <_CrossgenPublishFiles Include="@(_RawCrossgenPublishFiles->'%(OutputPath)')"
-                             KeepMetadata="REMOVE_ALL" />
-    </ItemGroup>
-    <ItemGroup Condition="'$(NativeAotSupported)' != 'true'">
-      <FilesToPackage Include="@(_CrossgenPublishFiles)"
-                      Exclude="*.pdb;*.h;*.lib"
-                      TargetPath="tools/" />
-    </ItemGroup>
-    <ItemGroup Condition="'$(NativeAotSupported)' == 'true'">
-      <!-- Treat all native aot assets as native runtime assets -->
-      <FilesToPackage Include="@(_CrossgenPublishFiles->Distinct())"
-                          Condition="'%(Extension)' != '.pdb'"
-                          TargetPath="tools/" />
-    </ItemGroup>
-  </Target>
+  <Import Project="$(MSBuildThisFileDirectory)Microsoft.NETCore.App.Crossgen2.props" />
 
   <Target Name="RunPublishedCrossgen" AfterTargets="PublishCrossgen"
           Condition="'$(TargetOS)' == '$(HostOS)' and '$(TargetArchitecture)' == '$(BuildArchitecture)' and '$(CrossBuild)' != 'true'">
@@ -54,41 +18,4 @@
       <Output TaskParameter="ExitCode" PropertyName="CrossgenExitCode" />
     </Exec>
   </Target>
-
-  <PropertyGroup>
-    <TargetOSComponent>unix</TargetOSComponent>
-    <TargetOSComponent Condition="'$(TargetOS)' == 'windows'">win</TargetOSComponent>
-    <TargetSpec>$(TargetOSComponent)-$(TargetArchitecture)</TargetSpec>
-  </PropertyGroup>
-
-  <Target Name="AddCrossgen2SymbolFilesToPackage" BeforeTargets="GetFilesToPackage" DependsOnTargets="PublishCrossgen">
-    <ItemGroup>
-      <_Crossgen2SymbolFilesToPackage Include="@(_CrossgenPublishFiles)" Condition="'%(Extension)' == '.pdb'" />
-      <!-- Symbol files for JIT libraries are placed in a different location for Windows builds -->
-      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPdbDir)%(FileName).pdb')" Condition="'$(TargetOS)' == 'windows' and '%(FileName)' != 'crossgen2'" />
-      <_Crossgen2SymbolFilesToPackage Include="@(NativeRuntimeAsset->'$(CoreCLRArtifactsPath)%(FileName)%(Extension)$(SymbolsSuffix)')" Condition="'$(TargetOS)' != 'windows' and '%(FileName)' != 'crossgen2'" />
-      <_Crossgen2SymbolFilesToPackage Remove="@(_Crossgen2SymbolFilesToPackage)" Condition="!Exists('%(Identity)')" />
-
-      <_SymbolFilesToPackage Include="@(_Crossgen2SymbolFilesToPackage)" TargetPath="tools/" />
-    </ItemGroup>
-  </Target>
-
-  <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
-  <Import Project="Sdk.targets" Sdk="Microsoft.DotNet.SharedFramework.Sdk" />
-
-  <Target Name="GetFilesToPublish">
-    <MSBuild Projects="$(MSBuildProjectFullPath)"
-          Targets="_GetAllSharedFrameworkFiles"
-          RemoveProperties="OutputPath;SymbolsOutputPath">
-      <Output TaskParameter="TargetOutputs" ItemName="_FilesToPackage" />
-    </MSBuild>
-    <ItemGroup>
-      <_PackagedFilesToPublish Include="@(_FilesToPackage)" Condition="'%(_FilesToPackage.PackOnly)' != 'true'" />
-    </ItemGroup>
-    <ItemGroup>
-      <FilesToPublish Include="@(_PackagedFilesToPublish)"
-                       TargetPath="" />
-    </ItemGroup>
-  </Target>
-
 </Project>