Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Single-File: Update Bundler Invocation. #11586

Merged
merged 4 commits into from
May 15, 2020
Merged

Single-File: Update Bundler Invocation. #11586

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b78bd59
Single-File: Update Bundler Invocation.
swaroop-sridhar May 12, 2020
8f99f05
Address feedback by @wli3
swaroop-sridhar May 13, 2020
a2c068e
Address more feedback from @wli3
swaroop-sridhar May 14, 2020
85c42de
Reduce the number of cases for It_leaves_host_components_unbundled_wh…
swaroop-sridhar May 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions src/Tasks/Microsoft.NET.Build.Tasks/GenerateBundle.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
using Microsoft.NET.HostModel.Bundle;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;

namespace Microsoft.NET.Build.Tasks
{
Expand All @@ -17,14 +19,27 @@ public class GenerateBundle : TaskBase
[Required]
public bool IncludeSymbols { get; set; }
[Required]
public string TargetFrameworkVersion { get; set; }
[Required]
public string RuntimeIdentifier { get; set; }
[Required]
public string OutputDir { get; set; }
[Required]
public bool ShowDiagnosticOutput { get; set; }

[Output]
public ITaskItem[] ExcludedFiles { get; set; }

protected override void ExecuteCore()
{
BundleOptions options = BundleOptions.BundleAllContent | (IncludeSymbols ? BundleOptions.BundleSymbolFiles : BundleOptions.None);
var bundler = new Bundler(AppHostName, OutputDir, options, diagnosticOutput: ShowDiagnosticOutput);
OSPlatform targetOS = RuntimeIdentifier.StartsWith("win") ? OSPlatform.Windows :
RuntimeIdentifier.StartsWith("osx") ? OSPlatform.OSX : OSPlatform.Linux;

BundleOptions options = BundleOptions.BundleAllContent;
options |= IncludeSymbols ? BundleOptions.BundleSymbolFiles : BundleOptions.None;

var bundler = new Bundler(AppHostName, OutputDir, options, targetOS, new Version(TargetFrameworkVersion), ShowDiagnosticOutput);

var fileSpec = new List<FileSpec>(FilesToBundle.Length);

foreach (var item in FilesToBundle)
Expand All @@ -34,6 +49,14 @@ protected override void ExecuteCore()
}

bundler.GenerateBundle(fileSpec);

// Certain files are excluded from the bundle, based on BundleOptions.
// For example:
// Native files and contents files are excluded by default.
// hostfxr and hostpolicy are excluded until singlefilehost is available.
// Return the set of excluded files in ExcludedFiles, so that they can be placed in the publish directory.

ExcludedFiles = FilesToBundle.Zip(fileSpec, (item, spec) => (spec.Excluded) ? item : null).Where(x => x != null).ToArray();
}
}
}
22 changes: 18 additions & 4 deletions src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Publish.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -947,9 +947,7 @@ Copyright (c) .NET Foundation. All rights reserved.

<ItemGroup>
<ResolvedFileToPublish Remove="@(_FilesToBundle)"/>
<!-- ResolvedFileToPublish shouldn't include PublishedSingleFilePath, since the single-file bundle is written directly to the publish directory -->
</ItemGroup>

</Target>

<UsingTask TaskName="GenerateBundle" AssemblyFile="$(MicrosoftNETBuildTasksAssembly)" />
Expand All @@ -959,12 +957,28 @@ Copyright (c) .NET Foundation. All rights reserved.
Inputs="@(_FilesToBundle)"
Outputs="$(PublishedSingleFilePath)">

<PropertyGroup>
<TraceSingleFileBundler Condition="'$(TraceSingleFileBundler)' == ''">false</TraceSingleFileBundler>
<IncludeSymbolsInSingleFile Condition="'$(IncludeSymbolsInSingleFile)' == ''">false</IncludeSymbolsInSingleFile>
<IncludeNativeLibrariesInSingleFile Condition="'$(IncludeNativeLibrariesInSingleFile)' == ''">false</IncludeNativeLibrariesInSingleFile>
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not used?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, removed the initialization.

<IncludeAllContentInSingleFile Condition="'$(IncludeAllContentInSingleFile)' == ''">false</IncludeAllContentInSingleFile>
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IncludeAllContentInSingleFile not used?

</PropertyGroup>

<GenerateBundle FilesToBundle="@(_FilesToBundle)"
AppHostName="$(PublishedSingleFileName)"
IncludeSymbols="$(IncludeSymbolsInSingleFile)"
TargetFrameworkVersion="$(_TargetFrameworkVersionWithoutV)"
RuntimeIdentifier="$(RuntimeIdentifier)"
OutputDir="$(PublishDir)"
ShowDiagnosticOutput="false"/>

ShowDiagnosticOutput="$(TraceSingleFileBundler)">
<Output TaskParameter="ExcludedFiles" ItemName="_FilesExcludedFromBundle"/>
</GenerateBundle>

<ItemGroup>
<ResolvedFileToPublish Include="@(_FilesExcludedFromBundle)"/>
<!-- ResolvedFileToPublish shouldn't include PublishedSingleFilePath, since the single-file bundle is written directly to the publish directory -->
</ItemGroup>

</Target>

<!--
Expand Down
40 changes: 38 additions & 2 deletions src/Tests/Microsoft.NET.Publish.Tests/GivenThatWeWantToPublishASingleFileApp.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ private PublishCommand GetPublishCommand()
return new PublishCommand(Log, testAsset.TestRoot);
}

private DirectoryInfo GetPublishDirectory(PublishCommand publishCommand)
private DirectoryInfo GetPublishDirectory(PublishCommand publishCommand, string targetFramework = "netcoreapp3.0")
{
return publishCommand.GetOutputDirectory(targetFramework: "netcoreapp3.0",
return publishCommand.GetOutputDirectory(targetFramework: targetFramework,
runtimeIdentifier: RuntimeInformation.RuntimeIdentifier);
}

Expand Down Expand Up @@ -352,5 +352,41 @@ public void It_rewrites_the_apphost_for_non_single_file_publish()

appHostSize.Should().BeLessThan(singleFileSize);
}

[CoreMSBuildOnlyTheory]
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

are all these combinations necessary too?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, for It_runs_single_file_apps I think we should have all combinations.
This guards against incompatibility between SDK and hostmodel that creates single-file apps that don't run correctly. I'm planning to add this test to integration tests in the installer branch, as recommended by Daniel.

[InlineData("netcoreapp3.0", false)]
[InlineData("netcoreapp3.0", true)]
[InlineData("netcoreapp3.1", false)]
[InlineData("netcoreapp3.1", true)]
[InlineData("netcoreapp5.0", false)]
[InlineData("netcoreapp5.0", true)]
public void It_runs_single_file_apps(string targetFramework, bool selfContained)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you change the assertion(assert different files are present or not) to reflect:

In .net 5, single-file apps will leave native binaries unbundled in the publish directory by default.
Since Single-file bundles are processed in the framework, hostpolicy and hostfxr DLLs cannot themselves be in the bundle (until statically linked singlefilehost is supported).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wli3 I've added a new test It_leaves_host_components_unbundled_when_necessary for this.

I thought about this, but I originally didn't add the test here because it is a temporary idiosyncrasy of the host/hostmodel, and not a requirement for the SDK.
The host-tests do test for this:
https://github.com/dotnet/runtime/blob/master/src/installer/test/Microsoft.NET.HostModel.Tests/AppHost.Bundle.Tests/BundleExtractToSpecificPath.cs#L58
https://github.com/dotnet/runtime/blob/master/src/installer/test/Microsoft.NET.HostModel.Tests/Helpers/BundleHelper.cs#L63-L66

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this is covered in runtime we can skip that. These are very slow tests * 6.

Maybe just have one test to ensure

ExcludedFiles = FilesToBundle.Zip(fileSpec, (item, spec) => (spec.Excluded) ? item : null).Where(x => x != null).ToArray(); works. As in assert one file in one tfm that should be exclude not being there.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

And is all these TFM combination necessary? Again, these are really slow tests. There is no SDK logic have if-else about TFM. Only one test in SDK is enough. Other coverage should be in dotnet/Runtime

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK @wli3. I'll keep the one interesting case of .net5 self-contained in the It_rewrites_the_apphost_for_non_single_file_publish test.

I think it is useful to have all the cases in the It_runs_single_file_apps case.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@wli3 This is done in 85c42de

{
var testProject = new TestProject()
{
Name = "SingleFileTest",
TargetFrameworks = targetFramework,
IsSdkProject = true,
IsExe = true,
};
testProject.AdditionalProperties.Add("SelfContained", $"{selfContained}");

var testAsset = _testAssetsManager.CreateTestProject(testProject);
var publishCommand = new PublishCommand(Log, Path.Combine(testAsset.TestRoot, testProject.Name));

publishCommand.Execute(PublishSingleFile, RuntimeIdentifier)
.Should()
.Pass();

var publishDir = GetPublishDirectory(publishCommand, targetFramework).FullName;
var singleFilePath = Path.Combine(publishDir, $"{testProject.Name}{Constants.ExeSuffix}");

var command = new RunExeCommand(Log, singleFilePath);
command.Execute()
.Should()
.Pass()
.And
.HaveStdOutContaining("Hello World");
}
}
}
6 changes: 3 additions & 3 deletions src/Tests/Microsoft.NET.Publish.Tests/GivenThatWeWantToPublishIncrementally.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,10 @@ public void It_cleans_for_mvc_projects()
newCommand.WorkingDirectory = testDir.Path;
newCommand.Execute("new", "mvc", "-n", assetName).Should().Pass();

var expectedRegularFiles = new string[] { ".dll", ".deps.json", ".runtimeconfig.json", ".Views.dll", ".Views.pdb" }
var expectedRegularFiles = new string[] { ".dll", ".deps.json", ".runtimeconfig.json", ".Views.dll"}
.Select(ending => assetName + ending);
var expectedSingleFiles = new string[] { ".pdb", ".exe" }.Select(ending => assetName + ending)
.Concat(new string[] { "appsettings.json", "appsettings.Development.json", "web.config" });
var expectedSingleFiles = new string[] { ".Views.pdb", ".pdb", ".exe" }.Select(ending => assetName + ending)
.Concat(new string[] { "hostfxr.dll", "hostpolicy.dll", "appsettings.json", "appsettings.Development.json", "web.config" });

// Publish normally
new PublishCommand(Log, Path.Combine(testDir.Path, assetName))
Expand Down