Use SingleFileHost (#11797)

* Use SingleFileHost This commit implements the following changes for single-file apps: * When publishing self-contained single-file apps: * Use `SingleFileHost` instead of `apphost` * Trim the native components of the runtime published for the app. * Fixes #11567. * Implements the [optional additional settings](https://github.com/dotnet/designs/blob/master/accepted/2020/single-file/design.md#optional-settings) for .net 5 `PublishSingleFile`. * dotnet/runtime#36590 * Address feedback from @dplaisted. * Update global.json (To include update from runtime) * Updating stage0 caused crossgen2 tests to fail because of missing dependencies in the corssgen2 package. Therefore two tests are disabled until dotnet/runtime#37196 is fixed.
dotnet · Jun 1, 2020 · 8288f55 · 8288f55
1 parent 9ebff78
commit 8288f55
Show file tree

Hide file tree

Showing 14 changed files with 214 additions and 78 deletions.
diff --git a/global.json b/global.json
@@ -1,6 +1,6 @@
 {
   "tools": {
-    "dotnet": "5.0.100-preview.6.20266.3",
+    "dotnet": "5.0.100-preview.6.20280.6",
     "runtimes": {
       "dotnet": [
         "$(MicrosoftNETCoreAppRuntimePackageVersion)"

diff --git a/src/Assets/TestProjects/HelloWorldWithSubDirs/HelloWorldWithSubDirs.csproj b/src/Assets/TestProjects/HelloWorldWithSubDirs/HelloWorldWithSubDirs.csproj
@@ -2,18 +2,18 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp3.0</TargetFramework>
+    <TargetFramework>net5.0</TargetFramework>
   </PropertyGroup>
 
   <ItemGroup>
     <Content Include="$(MSBuildThisFileDirectory)\SmallNameDir\**\*.*" >
       <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
     </Content>
-    <Content Include="$(MSBuildThisFileDirectory)\Signature.Newest.Stamp" >
+    <Content Include="$(MSBuildThisFileDirectory)\Signature.Newest.Stamp" Condition="'$(PlaceStamp)' == 'true'">
       <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
       <ExcludeFromSingleFile>$(ExcludeNewest)</ExcludeFromSingleFile>
     </Content>
-    <Content Include="$(MSBuildThisFileDirectory)\Signature.Always.Stamp" >
+    <Content Include="$(MSBuildThisFileDirectory)\Signature.Always.Stamp" Condition="'$(PlaceStamp)' == 'true'">
       <CopyToPublishDirectory>Always</CopyToPublishDirectory>
       <ExcludeFromSingleFile>$(ExcludeAlways)</ExcludeFromSingleFile>
     </Content>

diff --git a/src/Tasks/Common/MetadataKeys.cs b/src/Tasks/Common/MetadataKeys.cs
@@ -80,6 +80,9 @@ internal static class MetadataKeys
         //  Targeting packs
         public const string PackageConflictPreferredPackages = "PackageConflictPreferredPackages";
 
+        //  Runtime packs
+        public const string DropFromSingleFile = "DropFromSingleFile";
+
         // Content files
         public const string PPOutputPath = "PPOutputPath";
         public const string CodeLanguage = "CodeLanguage";

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/GenerateBundle.cs b/src/Tasks/Microsoft.NET.Build.Tasks/GenerateBundle.cs
@@ -19,6 +19,10 @@ public class GenerateBundle : TaskBase
         [Required]
         public bool IncludeSymbols { get; set; }
         [Required]
+        public bool IncludeNativeLibraries { get; set; }
+        [Required]
+        public bool IncludeAllContent { get; set; }
+        [Required]
         public string TargetFrameworkVersion { get; set; }
         [Required]
         public string RuntimeIdentifier { get; set; }
@@ -35,7 +39,9 @@ protected override void ExecuteCore()
             OSPlatform targetOS = RuntimeIdentifier.StartsWith("win") ? OSPlatform.Windows :
                                   RuntimeIdentifier.StartsWith("osx") ? OSPlatform.OSX : OSPlatform.Linux;
 
-            BundleOptions options = BundleOptions.BundleAllContent;
+            BundleOptions options = BundleOptions.None;
+            options |= IncludeNativeLibraries ? BundleOptions.BundleNativeBinaries : BundleOptions.None;
+            options |= IncludeAllContent ? BundleOptions.BundleAllContent : BundleOptions.None;
             options |= IncludeSymbols ? BundleOptions.BundleSymbolFiles : BundleOptions.None;
 
             var bundler = new Bundler(AppHostName, OutputDir, options, targetOS, new Version(TargetFrameworkVersion), ShowDiagnosticOutput);

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/GenerateDepsFile.cs b/src/Tasks/Microsoft.NET.Build.Tasks/GenerateDepsFile.cs
@@ -90,6 +90,8 @@ public class GenerateDepsFile : TaskBase
 
         public bool IsSelfContained { get; set; }
 
+        public bool IsSingleFile { get; set; }
+
         public bool IncludeRuntimeFileVersions { get; set; }
 
         [Required]
@@ -169,7 +171,10 @@ private void WriteDepsFile(string depsFilePath)
                 SingleProjectInfo.CreateProjectReferenceInfos(ReferencePaths, ReferenceSatellitePaths, isUserRuntimeAssembly);
 
             IEnumerable<RuntimePackAssetInfo> runtimePackAssets =
-                IsSelfContained ? RuntimePackAssets.Select(item => RuntimePackAssetInfo.FromItem(item)) : Enumerable.Empty<RuntimePackAssetInfo>();
+                IsSelfContained ? 
+                    RuntimePackAssets.Where(item => !IsSingleFile || !item.GetMetadata(MetadataKeys.DropFromSingleFile).Equals("true"))
+                                     .Select(item => RuntimePackAssetInfo.FromItem(item)) :
+                    Enumerable.Empty<RuntimePackAssetInfo>();
 
             DependencyContextBuilder builder;
             if (projectContext != null)

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/ResolveAppHosts.cs b/src/Tasks/Microsoft.NET.Build.Tasks/ResolveAppHosts.cs
@@ -31,6 +31,9 @@ public class ResolveAppHosts : TaskBase
         [Required]
         public string DotNetAppHostExecutableNameWithoutExtension { get; set; }
 
+        [Required]
+        public string DotNetSingleFileHostExecutableNameWithoutExtension { get; set; }
+
         /// <summary>
         /// The file name of comhost asset.
         /// </summary>
@@ -57,6 +60,9 @@ public class ResolveAppHosts : TaskBase
         [Output]
         public ITaskItem[] AppHost { get; set; }
 
+        [Output]
+        public ITaskItem[] SingleFileHost { get; set; }
+
         [Output]
         public ITaskItem[] ComHost { get; set; }
 
@@ -107,6 +113,20 @@ protected override void ExecuteCore()
                     AppHost = new ITaskItem[] { appHostItem };
                 }
 
+                var singlefileHostItem = GetHostItem(
+                    AppHostRuntimeIdentifier,
+                    knownAppHostPacksForTargetFramework,
+                    packagesToDownload,
+                    DotNetSingleFileHostExecutableNameWithoutExtension,
+                    "SingleFileHost",
+                    isExecutable: true,
+                    errorIfNotFound: true);
+
+                if (singlefileHostItem != null)
+                {
+                    SingleFileHost = new ITaskItem[] { singlefileHostItem };
+                }
+
                 var comHostItem = GetHostItem(
                     AppHostRuntimeIdentifier,
                     knownAppHostPacksForTargetFramework,

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/ResolveRuntimePackAssets.cs b/src/Tasks/Microsoft.NET.Build.Tasks/ResolveRuntimePackAssets.cs
@@ -143,6 +143,7 @@ private void AddRuntimePackAssetsFromManifest(List<ITaskItem> runtimePackAssets,
                 assetItem.SetMetadata("AssemblyVersion", fileElement.Attribute("AssemblyVersion")?.Value);
                 assetItem.SetMetadata("FileVersion", fileElement.Attribute("FileVersion")?.Value);
                 assetItem.SetMetadata("PublicKeyToken", fileElement.Attribute("PublicKeyToken")?.Value);
+                assetItem.SetMetadata("DropFromSingleFile", fileElement.Attribute("DropFromSingleFile")?.Value);
 
                 runtimePackAssets.Add(assetItem);
             }

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Publish.targets b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Publish.targets
@@ -547,6 +547,12 @@ Copyright (c) .NET Foundation. All rights reserved.
         <RelativePath>shims/%(_EmbeddedApphostPaths.ShimRuntimeIdentifier)/%(_EmbeddedApphostPaths.Filename)%(_EmbeddedApphostPaths.Extension)</RelativePath>
         <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
       </ResolvedFileToPublish>
+
+      <!-- Filter files for PublishSingleFiles scenario -->
+      <_FilesToDrop Include="@(ResolvedFileToPublish)"
+                    Condition="'$(PublishSingleFile)' == 'true' and
+                               '%(ResolvedFileToPublish.DropFromSingleFile)' == 'true'"/>
+      <ResolvedFileToPublish Remove="@(_FilesToDrop)"/>
     </ItemGroup>
 
   </Target>
@@ -917,10 +923,12 @@ Copyright (c) .NET Foundation. All rights reserved.
     <!-- Check to see whether we can re-use the .deps.json file from the build for publish, or whether we have to
          generate a different one. -->
     <PropertyGroup>
+      <_TrimRuntimeAssets Condition="'$(PublishSingleFile)' == 'true' and '$(SelfContained)' == 'true'">true</_TrimRuntimeAssets>
       <_UseBuildDependencyFile Condition="'@(_ExcludeFromPublishPackageReference)' == '' and
                                           '@(RuntimeStorePackages)' == '' and
                                           '$(PreserveStoreLayout)' != 'true' and
-                                          '$(PublishTrimmed)' != 'true'">true</_UseBuildDependencyFile>
+                                          '$(PublishTrimmed)' != 'true' and 
+                                          '$(_TrimRuntimeAssets)' != 'true'">true</_UseBuildDependencyFile>
     </PropertyGroup>
 
   </Target>
@@ -937,17 +945,17 @@ Copyright (c) .NET Foundation. All rights reserved.
         Condition="'$(PublishSingleFile)' == 'true'">
 
     <ItemGroup>
-      <_FilesToBundle Include="@(ResolvedFileToPublish)" Condition="'%(ResolvedFileToPublish.ExcludeFromSingleFile)' != 'true'"/>
+      <_FilesToBundle Include="@(ResolvedFileToPublish)" 
+                      Condition="'%(ResolvedFileToPublish.ExcludeFromSingleFile)' != 'true'"/>
+
+      <ResolvedFileToPublish Remove="@(_FilesToBundle)"/>
     </ItemGroup>
 
     <PropertyGroup>
       <PublishedSingleFileName>$(AssemblyName)$(_NativeExecutableExtension)</PublishedSingleFileName>
       <PublishedSingleFilePath>$(PublishDir)$(PublishedSingleFileName)</PublishedSingleFilePath>
     </PropertyGroup>
-
-    <ItemGroup>
-      <ResolvedFileToPublish Remove="@(_FilesToBundle)"/>
-    </ItemGroup>
+
   </Target>
 
   <UsingTask TaskName="GenerateBundle" AssemblyFile="$(MicrosoftNETBuildTasksAssembly)" />
@@ -960,11 +968,15 @@ Copyright (c) .NET Foundation. All rights reserved.
     <PropertyGroup>
       <TraceSingleFileBundler Condition="'$(TraceSingleFileBundler)' == ''">false</TraceSingleFileBundler>
       <IncludeSymbolsInSingleFile Condition="'$(IncludeSymbolsInSingleFile)' == ''">false</IncludeSymbolsInSingleFile>
+      <IncludeNativeLibrariesInSingleFile Condition="'$(IncludeNativeLibrariesInSingleFile)' == ''">false</IncludeNativeLibrariesInSingleFile>
+      <IncludeAllContentInSingleFile Condition="'$(IncludeAllContentInSingleFile)' == ''">false</IncludeAllContentInSingleFile>
     </PropertyGroup>    
 
     <GenerateBundle FilesToBundle="@(_FilesToBundle)"
                     AppHostName="$(PublishedSingleFileName)"
                     IncludeSymbols="$(IncludeSymbolsInSingleFile)"
+                    IncludeNativeLibraries="$(IncludeNativeLibrariesInSingleFile)"
+                    IncludeAllContent="$(IncludeAllContentInSingleFile)"                    
                     TargetFrameworkVersion="$(_TargetFrameworkVersionWithoutV)"
                     RuntimeIdentifier="$(RuntimeIdentifier)"
                     OutputDir="$(PublishDir)"
@@ -1002,7 +1014,9 @@ Copyright (c) .NET Foundation. All rights reserved.
            PublishDepsFilePath is empty (by default) for PublishSingleFile, since the deps.json file is embedde within the single-file bundle -->      
       <IntermediateDepsFilePath Condition=" '$(PublishDepsFilePath)' != ''">$(PublishDepsFilePath)</IntermediateDepsFilePath >
       <IntermediateDepsFilePath Condition=" '$(PublishDepsFilePath)' == ''">$(IntermediateOutputPath)$(ProjectDepsFileName)</IntermediateDepsFilePath >
-      <PublishDepsFilePath Condition=" '$(PublishDepsFilePath)' == '' And '$(PublishSingleFile)' != 'true'">$(PublishDir)$(ProjectDepsFileName)</PublishDepsFilePath>      
+      <PublishDepsFilePath Condition=" '$(PublishDepsFilePath)' == '' And '$(PublishSingleFile)' != 'true'">$(PublishDir)$(ProjectDepsFileName)</PublishDepsFilePath>
+      <_IsSingleFilePublish Condition="'$(PublishSingleFile)' == ''">false</_IsSingleFilePublish>
+      <_IsSingleFilePublish Condition="'$(PublishSingleFile)' != ''">$(PublishSingleFile)</_IsSingleFilePublish>
     </PropertyGroup>
     <ItemGroup>
       <ResolvedCompileFileDefinitions Remove="@(_PublishConflictPackageFiles)" Condition="'%(_PublishConflictPackageFiles.ConflictItemType)' == 'Reference'" />
@@ -1013,6 +1027,7 @@ Copyright (c) .NET Foundation. All rights reserved.
       <_ResolvedNuGetFilesForPublish Include="@(ResourceCopyLocalItems)" Condition="'%(ResourceCopyLocalItems.CopyToPublishDirectory)' != 'false'" />
       <_ResolvedNuGetFilesForPublish Include="@(RuntimeCopyLocalItems)" Condition="'%(RuntimeCopyLocalItems.CopyToPublishDirectory)' != 'false'" />
       <_ResolvedNuGetFilesForPublish Remove="@(_PublishConflictPackageFiles)" Condition="'%(_PublishConflictPackageFiles.ConflictItemType)' != 'Reference'" />
+
     </ItemGroup>
 
     <GenerateDepsFile ProjectPath="$(MSBuildProjectFullPath)"
@@ -1039,6 +1054,7 @@ Copyright (c) .NET Foundation. All rights reserved.
                       ResolvedRuntimeTargetsFiles="@(RuntimeTargetsCopyLocalItems)"
                       UserRuntimeAssemblies="@(UserRuntimeAssembly)"
                       IsSelfContained="$(SelfContained)"
+                      IsSingleFile="$(_IsSingleFilePublish)"
                       IncludeRuntimeFileVersions="$(IncludeFileVersionsInDependencyFile)"
                       RuntimeGraphPath="$(BundledRuntimeIdentifierGraphFile)"/>
 

diff --git a/.../Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.FrameworkReferenceResolution.targets b/.../Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.FrameworkReferenceResolution.targets
@@ -97,13 +97,15 @@ Copyright (c) .NET Foundation. All rights reserved.
                      RuntimeFrameworkVersion="$(RuntimeFrameworkVersion)"
                      PackAsToolShimRuntimeIdentifiers="@(_PackAsToolShimRuntimeIdentifiers)"
                      DotNetAppHostExecutableNameWithoutExtension="$(_DotNetAppHostExecutableNameWithoutExtension)"
+                     DotNetSingleFileHostExecutableNameWithoutExtension="$(_DotNetSingleFileHostExecutableNameWithoutExtension)"
                      DotNetComHostLibraryNameWithoutExtension="$(_DotNetComHostLibraryNameWithoutExtension)"
                      DotNetIjwHostLibraryNameWithoutExtension="$(_DotNetIjwHostLibraryNameWithoutExtension)"
                      RuntimeGraphPath="$(BundledRuntimeIdentifierGraphFile)"
                      KnownAppHostPacks="@(KnownAppHostPack)">
 
       <Output TaskParameter="PackagesToDownload" ItemName="_PackageToDownload" />
       <Output TaskParameter="AppHost" ItemName="AppHostPack" />
+      <Output TaskParameter="SingleFileHost" ItemName="SingleFileHostPack" />
       <Output TaskParameter="ComHost" ItemName="ComHostPack" />
       <Output TaskParameter="IjwHost" ItemName="IjwHostPack" />
       <Output TaskParameter="PackAsToolShimAppHostPacks" ItemName="PackAsToolShimAppHostPack" />
@@ -187,6 +189,14 @@ Copyright (c) .NET Foundation. All rights reserved.
 
     </GetPackageDirectory>
 
+    <GetPackageDirectory
+      Items="@(SingleFileHostPack)"
+      PackageFolders="@(AssetsFilePackageFolder)">
+
+      <Output TaskParameter="Output" ItemName="ResolvedSingleFileHostPack" />
+
+    </GetPackageDirectory>
+
     <GetPackageDirectory
       Items="@(Crossgen2Pack)"
       PackageFolders="@(AssetsFilePackageFolder)">
@@ -234,6 +244,16 @@ Copyright (c) .NET Foundation. All rights reserved.
       <AppHostSourcePath>@(ResolvedAppHostPack->'%(Path)')</AppHostSourcePath>
     </PropertyGroup>
 
+    <ItemGroup>
+      <ResolvedSingleFileHostPack Condition="'%(ResolvedSingleFileHostPack.Path)' == '' and '%(ResolvedSingleFileHostPack.PackageDirectory)' != ''">
+        <Path>%(ResolvedSingleFileHostPack.PackageDirectory)\%(ResolvedSingleFileHostPack.PathInPackage)</Path>
+      </ResolvedSingleFileHostPack>
+    </ItemGroup>
+
+    <PropertyGroup Condition="'@(ResolvedSingleFileHostPack)' != '' And '$(SingleFileHostSourcePath)' == ''">
+      <SingleFileHostSourcePath>@(ResolvedSingleFileHostPack->'%(Path)')</SingleFileHostSourcePath>
+    </PropertyGroup>
+
     <ItemGroup>
       <ResolvedComHostPack Condition="'%(ResolvedComHostPack.Path)' == '' and '%(ResolvedComHostPack.PackageDirectory)' != ''">
         <Path>%(ResolvedComHostPack.PackageDirectory)\%(ResolvedComHostPack.PathInPackage)</Path>

diff --git a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.targets b/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.targets
@@ -83,6 +83,7 @@ Copyright (c) .NET Foundation. All rights reserved.
     <_DotNetAppHostExecutableNameWithoutExtension>apphost</_DotNetAppHostExecutableNameWithoutExtension>
     <_DotNetAppHostExecutableName>$(_DotNetAppHostExecutableNameWithoutExtension)$(_NativeExecutableExtension)</_DotNetAppHostExecutableName>
 
+    <_DotNetSingleFileHostExecutableNameWithoutExtension>singlefilehost</_DotNetSingleFileHostExecutableNameWithoutExtension>
     <_DotNetComHostLibraryNameWithoutExtension>comhost</_DotNetComHostLibraryNameWithoutExtension>
     <_DotNetComHostLibraryName>$(_DotNetComHostLibraryNameWithoutExtension)$(_ComHostLibraryExtension)</_DotNetComHostLibraryName>
     <_DotNetIjwHostLibraryNameWithoutExtension>Ijwhost</_DotNetIjwHostLibraryNameWithoutExtension>
@@ -415,7 +416,7 @@ Copyright (c) .NET Foundation. All rights reserved.
   <Target Name="_CreateAppHost"
           Inputs="@(IntermediateAssembly);$(AppHostSourcePath)"
           Outputs="$(AppHostIntermediatePath)"
-          DependsOnTargets="_GetAppHostPaths;CoreCompile"
+          DependsOnTargets="_ChooseAppHost;CoreCompile"
           Condition="'$(ComputeNETCoreBuildOutputFiles)' == 'true' and
                      '$(AppHostSourcePath)' != '' and
                      Exists('@(IntermediateAssembly)') and
@@ -434,6 +435,25 @@ Copyright (c) .NET Foundation. All rights reserved.
                    />
   </Target>
 
+  <!--
+    ============================================================
+                                        _ChooseAppHost
+      Choose the correct app-host for the build scenario:
+      * SingleFileHost if an app being published as a self-contained single-file .net5+ app 
+      * AppHost otherwise
+    ============================================================
+     -->
+  <Target Name="_ChooseAppHost"
+          DependsOnTargets="_GetAppHostPaths"
+          Condition="'$(UseAppHost)' == 'true' and '$(_IsExecutable)' == 'true'">
+    <PropertyGroup>
+      <AppHostSourcePath Condition="'$(PublishSingleFile)' == 'true' and 
+                                  '$(SelfContained)' == 'true' and
+                                  '$(_TargetFrameworkVersionWithoutV)' >= '5.0' and
+                                  '$(SingleFileHostSourcePath)' != ''">$(SingleFileHostSourcePath)</AppHostSourcePath>
+    </PropertyGroup>
+  </Target>
+
   <!--
     ============================================================
                                         _GetAppHostPaths

diff --git a/src/Tests/Microsoft.NET.Publish.Tests/GivenThatWeWantToPublishANetCoreAppForTelemetry.cs b/src/Tests/Microsoft.NET.Publish.Tests/GivenThatWeWantToPublishANetCoreAppForTelemetry.cs
@@ -60,9 +60,9 @@ public void It_collects_Trimmer_SingleFile_ReadyToRun_publishing_properties(stri
                 "\"CompileListCount\":\"[1-9]\\d?\"");  // Do not hardcode number of assemblies being compiled here, due to ILTrimmer
         }
 
-        [CoreMSBuildOnlyTheory]
-        [InlineData("net5.0")]
-        public void It_collects_crossgen2_publishing_properties(string targetFramework)
+        [CoreMSBuildOnlyTheory(Skip = "https://github.com/dotnet/runtime/issues/37196")]
+        [InlineData("net5.0")] 
+        void It_collects_crossgen2_publishing_properties(string targetFramework)
         {
             // Crossgen2 only supported for Linux/Windows x64 scenarios for now
             if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) || RuntimeInformation.OSArchitecture != Architecture.X64)