Add parallelism and copy-on-write links to Robocopy task - Use a new …

…version of the CopyOnWrite library that no-ops Mac and Linux calls, and an action block, to greatly speed up artifact copies. Use parallelism settings similar to those in the Microsoft.Build.CopyOnWrite SDK and allow turning off CoW linking with an env var setting. Fix an overcopying bug where the same source:destination pair would be copied multiple times. Increment minr version of Microsoft.Build.Artifacts. Migrate CoW SDK Copy task to use CoW package for clone file conpat checks.

Directory.Packages.props

@@ -4,6 +4,7 @@
     <MicrosoftBuildPackageVersion>17.6.3</MicrosoftBuildPackageVersion>
   </PropertyGroup>
   <ItemGroup>
+    <PackageVersion Include="CopyOnWrite" Version="0.3.3" Condition=" '$(TargetFramework)' != 'net46' " />
     <PackageVersion Include="Microsoft.Build.Utilities.Core" Version="$(MicrosoftBuildPackageVersion)" />
     <PackageVersion Update="Microsoft.Build.Utilities.Core" Version="16.9.0" Condition="'$(TargetFramework)' == 'netcoreapp3.1' Or '$(TargetFramework)' == 'netstandard2.0'" />
     <PackageVersion Update="Microsoft.Build.Utilities.Core" Version="15.9.20" Condition="'$(TargetFramework)' == 'net46'" />
@@ -15,9 +16,10 @@
     <PackageVersion Include="MSBuild.ProjectCreation" Version="10.0.0" />
     <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
+    <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="6.0.0" Condition=" '$(TargetFramework)' != 'net46' "/>
+    <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="4.11.1" Condition=" '$(TargetFramework)' == 'net46' "/>
     <PackageVersion Include="xunit" Version="2.4.2" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.4.5" />
-    <PackageVersion Include="CopyOnWrite" Version="0.3.2" />
   </ItemGroup>
 
   <ItemGroup>

src/Artifacts.UnitTests/Microsoft.Build.Artifacts.UnitTests.csproj

@@ -8,6 +8,7 @@
     <PackageReference Include="MSBuild.ProjectCreation" />
     <PackageReference Include="Newtonsoft.Json" />
     <PackageReference Include="Shouldly" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" />
     <PackageReference Include="xunit" />
     <PackageReference Include="xunit.runner.visualstudio" />
   </ItemGroup>

src/Artifacts.UnitTests/RobocopyTests.cs

@@ -45,7 +45,7 @@ public void NonRecursiveWildcards()
                         [nameof(RobocopyMetadata.IsRecursive)] = "false",
                     },
                 },
-                Sleep = duration => { },
+                Sleep = _ => { },
             };
 
             copyArtifacts.Execute().ShouldBeTrue(buildEngine.GetConsoleLog());
@@ -58,7 +58,8 @@ public void NonRecursiveWildcards()
                         "bar.txt",
                         "foo.txt",
                     }.Select(i => Path.Combine(destination.FullName, i)),
-                    ignoreOrder: true);
+                    ignoreOrder: true,
+                    customMessage: buildEngine.GetConsoleLog());
         }
 
         [Fact]
@@ -95,7 +96,7 @@ public void RecursiveWildcards()
                         ["FileMatch"] = "*exe *dll *exe.config",
                     },
                 },
-                Sleep = duration => { },
+                Sleep = _ => { },
             };
 
             copyArtifacts.Execute().ShouldBeTrue(buildEngine.GetConsoleLog());
@@ -111,7 +112,8 @@ public void RecursiveWildcards()
                         "foo.exe.config",
                         Path.Combine("baz", "baz.dll"),
                     }.Select(i => Path.Combine(destination.FullName, i)),
-                    ignoreOrder: true);
+                    ignoreOrder: true,
+                    customMessage: buildEngine.GetConsoleLog());
         }
 
         [Fact]
@@ -148,17 +150,19 @@ public void SingleFileMatch()
                         ["FileMatch"] = "foo.pdb",
                     },
                 },
-                Sleep = duration => { },
+                Sleep = _ => { },
             };
 
             copyArtifacts.Execute().ShouldBeTrue(buildEngine.GetConsoleLog());
 
             destination.GetFiles("*", SearchOption.AllDirectories)
                 .Select(i => i.FullName)
-                .ShouldBe(new[]
-                {
-                    "foo.pdb",
-                }.Select(i => Path.Combine(destination.FullName, i)));
+                .ShouldBe(
+                    new[]
+                    {
+                        "foo.pdb",
+                    }.Select(i => Path.Combine(destination.FullName, i)),
+                    customMessage: buildEngine.GetConsoleLog());
         }
 
         [Fact]
@@ -196,18 +200,75 @@ public void SingleFileMatchRecursive()
                         ["FileMatch"] = "foo.pdb",
                     },
                 },
-                Sleep = duration => { },
+                Sleep = _ => { },
             };
 
             copyArtifacts.Execute().ShouldBeTrue(buildEngine.GetConsoleLog());
 
             destination.GetFiles("*", SearchOption.AllDirectories)
                 .Select(i => i.FullName)
-                .ShouldBe(new[]
+                .ShouldBe(
+                    new[]
+                    {
+                        "foo.pdb",
+                        Path.Combine("foo", "foo", "foo", "foo.pdb"),
+                    }.Select(i => Path.Combine(destination.FullName, i)),
+                    customMessage: buildEngine.GetConsoleLog());
+        }
+
+        [Fact]
+        public void DuplicatedItemsShouldResultInOneCopy()
+        {
+            DirectoryInfo source = CreateFiles(
+                "source",
+                @"foo.txt");
+
+            DirectoryInfo destination = new DirectoryInfo(Path.Combine(TestRootPath, "destination"));
+
+            BuildEngine buildEngine = BuildEngine.Create();
+
+            MockTaskItem singleFileItem = new MockTaskItem(Path.Combine(source.FullName, "foo.txt"))
+            {
+                ["DestinationFolder"] = destination.FullName,
+            };
+
+            MockTaskItem recursiveDirItem = new MockTaskItem(source.FullName)
+            {
+                ["DestinationFolder"] = destination.FullName,
+            };
+
+            Robocopy copyArtifacts = new Robocopy
+            {
+                BuildEngine = buildEngine,
+                Sources = new ITaskItem[]
                 {
-                    "foo.pdb",
-                    Path.Combine("foo", "foo", "foo", "foo.pdb"),
-                }.Select(i => Path.Combine(destination.FullName, i)));
+                    singleFileItem,
+                    singleFileItem,
+                    singleFileItem,
+                    singleFileItem,
+                    singleFileItem,
+                    singleFileItem,
+
+                    recursiveDirItem,
+                    recursiveDirItem,
+                    recursiveDirItem,
+                    recursiveDirItem,
+                    recursiveDirItem,
+                    recursiveDirItem,
+                },
+                Sleep = _ => { },
+            };
+
+            copyArtifacts.Execute().ShouldBeTrue(buildEngine.GetConsoleLog());
+
+            destination.GetFiles("*", SearchOption.AllDirectories)
+                .Select(i => i.FullName)
+                .ShouldBe(
+                    new[]
+                    {
+                        "foo.txt",
+                    }.Select(i => Path.Combine(destination.FullName, i)),
+                    customMessage: buildEngine.GetConsoleLog());
         }
     }
 }

src/Artifacts/FileSystem.cs

@@ -2,13 +2,20 @@
 //
 // Licensed under the MIT license.
 
+#if NETSTANDARD2_0_OR_GREATER
+using Microsoft.CopyOnWrite;
+#endif
 using System.Collections.Generic;
 using System.IO;
 
 namespace Microsoft.Build.Artifacts
 {
     internal sealed class FileSystem : IFileSystem
     {
+#if NETSTANDARD2_0_OR_GREATER
+        private static readonly ICopyOnWriteFilesystem CoW = CopyOnWriteFilesystemFactory.GetInstance();
+#endif
+
         private FileSystem()
         {
         }
@@ -59,5 +66,27 @@ public bool FileExists(FileInfo file)
         {
             return file.Exists;
         }
+
+        public bool TryCloneFile(FileInfo sourceFile, FileInfo destinationFile)
+        {
+#if NETSTANDARD2_0_OR_GREATER
+            string sourcePath = sourceFile.FullName;
+            string destPath = destinationFile.FullName;
+            if (CoW.CopyOnWriteLinkSupportedBetweenPaths(sourcePath, destPath))
+            {
+                if (destinationFile.Exists)
+                {
+                    // CoW doesn't overwrite destination files.
+                    destinationFile.Delete();
+                }
+
+                // PathIsFullyResolved: FileInfo.FullName is fully resolved.
+                CoW.CloneFile(destPath, destPath, CloneFlags.PathIsFullyResolved);
+                return true;
+            }
+#endif
+
+            return false;
+        }
     }
 }

src/Artifacts/IFileSystem.cs

@@ -26,5 +26,11 @@ internal interface IFileSystem
         bool FileExists(string path);
 
         bool FileExists(FileInfo file);
+
+        /// <summary>
+        /// Attempts to create a copy-on-write link (file clone) if supported.
+        /// </summary>
+        /// <returns>True if the clone was created, false if unsupported.</returns>
+        bool TryCloneFile(FileInfo sourceFile, FileInfo destinationFile);
     }
 }

src/Artifacts/Microsoft.Build.Artifacts.csproj

@@ -12,7 +12,9 @@
     <EmbedUntrackedSources>true</EmbedUntrackedSources>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="CopyOnWrite" Condition="'$(TargetFramework)' != 'net46'" />
     <PackageReference Include="Microsoft.Build.Utilities.Core" ExcludeAssets="Runtime" PrivateAssets="All" />
+    <PackageReference Include="System.Threading.Tasks.Dataflow" />
   </ItemGroup>
   <ItemGroup>
     <None Include="build\*" Pack="true" PackagePath="build\" />