Improve NuGet.targets restore perf

Walk project to project references first while creating a minimum number of items to track the full closure of projects. Removing duplicates along the way is important for perf reasons, if too many items are created restore will grind to a halt as they build.

Once all projects have been found create specs for the projects directly instead of creating them during the walk. This also helps to keep the number of items down.

Performance improvement on a project with 127 projects with a maximum number of project references between them.
Before: 900 seconds
After: 21 seconds

In addition to the perf changes this also adds in the full set of dotnet cli tool references when recursive is set. Getting this information is much easier now that the full set of projects is found up front.

Fixes NuGet/Home#4592
Fixes NuGet/Home#4711

src/NuGet.Core/NuGet.Build.Tasks/GetRestorePackageReferencesTask.cs

@@ -37,13 +37,22 @@ public override bool Execute()
             log.LogDebug($"(in) PackageReferences '{string.Join(";", PackageReferences.Select(p => p.ItemSpec))}'");
 
             var entries = new List<ITaskItem>();
+            var seenIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
 
             foreach (var msbuildItem in PackageReferences)
             {
+                var packageId = msbuildItem.ItemSpec;
+
+                if (string.IsNullOrEmpty(packageId) || !seenIds.Add(packageId))
+                {
+                    // Skip empty or already processed ids
+                    continue;
+                }
+
                 var properties = new Dictionary<string, string>();
                 properties.Add("ProjectUniqueName", ProjectUniqueName);
                 properties.Add("Type", "Dependency");
-                properties.Add("Id", msbuildItem.ItemSpec);
+                properties.Add("Id", packageId);
                 BuildTasksUtility.CopyPropertyIfExists(msbuildItem, properties, "Version", "VersionRange");
 
                 if (!string.IsNullOrEmpty(TargetFrameworks))

src/NuGet.Core/NuGet.Build.Tasks/GetRestoreProjectReferencesTask.cs

@@ -47,6 +47,10 @@ public override bool Execute()
 
             var entries = new List<ITaskItem>();
 
+            // Filter obvious duplicates without considering OS case sensitivity.
+            // This will be filtered further when creating the spec.
+            var seen = new HashSet<string>(StringComparer.Ordinal);
+
             var parentDirectory = Path.GetDirectoryName(ParentProjectPath);
 
             foreach (var project in ProjectReferences)
@@ -61,6 +65,12 @@ public override bool Execute()
                     // Get the absolute path
                     var referencePath = Path.GetFullPath(Path.Combine(parentDirectory, project.ItemSpec));
 
+                    if (!seen.Add(referencePath))
+                    {
+                        // Skip already processed projects
+                        continue;
+                    }
+
                     var properties = new Dictionary<string, string>();
                     properties.Add("ProjectUniqueName", ProjectUniqueName);
                     properties.Add("Type", "ProjectReference");