Implement Visual Studio partial restore. Add a solution based up to d…

…ate check that's particular to VS and the fact that it's a long running process

src/NuGet.Clients/NuGet.SolutionRestoreManager/ISolutionRestoreChecker.cs

@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+using NuGet.Commands;
+using NuGet.ProjectModel;
+
+namespace NuGet.SolutionRestoreManager
+{
+    /// <summary>
+    /// An up to date checker for a solution.
+    /// </summary>
+    public interface ISolutionRestoreChecker
+    {
+        /// <summary>
+        /// Given the current dependency graph spec, perform a fast up to date check and return the dirty projects.
+        /// The checker itself caches the DependencyGraphSpec it is provided & the last restore status, reported through <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/>.
+        /// Accounts for changes in the PackageSpec and marks all the parent projects as dirty as well.
+        /// Additionally also ensures that the expected output files have the same timestamps as the last time a succesful status was reported through <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/>.
+        /// </summary>
+        /// <param name="dependencyGraphSpec">The current dependency graph spec.</param>
+        /// <returns>Unique ids of the dirty projects</returns>
+        /// <remarks>Note that this call is stateful. This method may end up caching the dependency graph spec, so do not invoke multiple times. Ideally <see cref="PerformUpToDateCheck(DependencyGraphSpec)"/> call should be followed by a <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/> call.</remarks>
+        IEnumerable<string> PerformUpToDateCheck(DependencyGraphSpec dependencyGraphSpec);
+
+        /// <summary>
+        /// Report the status of all the projects restored. 
+        /// </summary>
+        /// <param name="restoreSummaries"></param>
+        /// <remarks>Note that this call is stateful. This method may end up caching the dependency graph spec, so do not invoke multiple times. Ideally <see cref="PerformUpToDateCheck(DependencyGraphSpec)"/> call should be followed by a <see cref="ReportStatus(IReadOnlyList{RestoreSummary})"/> call.</remarks>
+
+        void ReportStatus(IReadOnlyList<RestoreSummary> restoreSummaries);
+
+        /// <summary>
+        /// Clears any cached values. This is meant to mimic restores that overwrite the incremental restore optimizations.
+        /// </summary>
+        /// <returns></returns>
+        void CleanCache();
+    }
+}

src/NuGet.Clients/NuGet.SolutionRestoreManager/NuGet.SolutionRestoreManager.csproj

@@ -32,6 +32,7 @@
     <Compile Include="BrokeredServicesUtility.cs" />
     <Compile Include="GlobalSuppressions.cs" />
     <Compile Include="Guids.cs" />
+    <Compile Include="ISolutionRestoreChecker.cs" />
     <Compile Include="ISolutionRestoreJob.cs" />
     <Compile Include="NuGetSolutionService.cs" />
     <Compile Include="PkgCmdID.cs" />
@@ -49,6 +50,7 @@
     <Compile Include="SolutionRestoreJob.cs" />
     <Compile Include="SolutionRestoreJobContext.cs" />
     <Compile Include="SolutionRestoreWorker.cs" />
+    <Compile Include="SolutionUpToDateChecker.cs" />
     <Compile Include="VerbosityLevel.cs" />
     <Compile Include="VSNominationUtilities.cs" />
     <Compile Include="VsSolutionRestoreService.cs" />

src/NuGet.Clients/NuGet.SolutionRestoreManager/SolutionRestoreBuildHandler.cs

@@ -3,11 +3,9 @@
 
 using System;
 using System.ComponentModel.Composition;
-using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft;
-using Microsoft.VisualStudio;
 using Microsoft.VisualStudio.Shell;
 using Microsoft.VisualStudio.Shell.Interop;
 using Microsoft.VisualStudio.Threading;
@@ -40,6 +38,9 @@ public sealed class SolutionRestoreBuildHandler
         [Import]
         private Lazy<ISolutionRestoreWorker> SolutionRestoreWorker { get; set; }
 
+        [Import]
+        private Lazy<ISolutionRestoreChecker> SolutionRestoreChecker { get; set; }
+
         /// <summary>
         /// The <see cref="IVsSolutionBuildManager3"/> object controlling the update solution events.
         /// </summary>
@@ -62,15 +63,17 @@ private SolutionRestoreBuildHandler()
         public SolutionRestoreBuildHandler(
             ISettings settings,
             ISolutionRestoreWorker restoreWorker,
-            IVsSolutionBuildManager3 buildManager)
+            IVsSolutionBuildManager3 buildManager,
+            ISolutionRestoreChecker solutionRestoreChecker)
         {
             Assumes.Present(settings);
             Assumes.Present(restoreWorker);
             Assumes.Present(buildManager);
+            Assumes.Present(solutionRestoreChecker);
 
             Settings = new Lazy<ISettings>(() => settings);
             SolutionRestoreWorker = new Lazy<ISolutionRestoreWorker>(() => restoreWorker);
-
+            SolutionRestoreChecker = new Lazy<ISolutionRestoreChecker>(() => solutionRestoreChecker);
             _solutionBuildManager = buildManager;
 
             _isMEFInitialized = true;
@@ -141,8 +144,9 @@ public async Task<bool> RestoreAsync(uint buildAction, CancellationToken token)
             if ((buildAction & (uint)VSSOLNBUILDUPDATEFLAGS.SBF_OPERATION_CLEAN) != 0 &&
                 (buildAction & (uint)VSSOLNBUILDUPDATEFLAGS3.SBF_FLAGS_UPTODATE_CHECK) == 0)
             {
-                // Clear the project.json restore cache on clean to ensure that the next build restores again
+                // Clear the transitive restore cache on clean to ensure that the next build restores again
                 await SolutionRestoreWorker.Value.CleanCacheAsync();
+                SolutionRestoreChecker.Value.CleanCache();
             }
             else if ((buildAction & (uint)VSSOLNBUILDUPDATEFLAGS.SBF_OPERATION_BUILD) != 0 &&
                     (buildAction & (uint)VSSOLNBUILDUPDATEFLAGS3.SBF_FLAGS_UPTODATE_CHECK) == 0 &&

src/NuGet.Clients/NuGet.SolutionRestoreManager/SolutionRestoreJob.cs

@@ -26,6 +26,7 @@
 using NuGet.ProjectManagement.Projects;
 using NuGet.ProjectModel;
 using NuGet.Protocol.Core.Types;
+using NuGet.Shared;
 using NuGet.VisualStudio;
 using NuGet.VisualStudio.Telemetry;
 using Task = System.Threading.Tasks.Task;
@@ -46,6 +47,7 @@ internal sealed class SolutionRestoreJob : ISolutionRestoreJob
         private readonly ISourceRepositoryProvider _sourceRepositoryProvider;
         private readonly ISettings _settings;
         private readonly IRestoreEventsPublisher _restoreEventsPublisher;
+        private readonly ISolutionRestoreChecker _solutionUpToDateChecker;
 
         private RestoreOperationLogger _logger;
         private INuGetProjectContext _nuGetProjectContext;
@@ -54,6 +56,7 @@ internal sealed class SolutionRestoreJob : ISolutionRestoreJob
         private NuGetOperationStatus _status;
         private int _packageCount;
         private int _noOpProjectsCount;
+        private int _upToDateProjectCount;
 
         // relevant to packages.config restore only
         private int _missingPackagesCount;
@@ -65,13 +68,15 @@ public SolutionRestoreJob(
             IVsSolutionManager solutionManager,
             ISourceRepositoryProvider sourceRepositoryProvider,
             IRestoreEventsPublisher restoreEventsPublisher,
-            ISettings settings)
+            ISettings settings,
+            ISolutionRestoreChecker solutionRestoreChecker)
             : this(AsyncServiceProvider.GlobalProvider,
                   packageRestoreManager,
                   solutionManager,
                   sourceRepositoryProvider,
                   restoreEventsPublisher,
-                  settings
+                  settings,
+                  solutionRestoreChecker
                 )
         { }
 
@@ -81,14 +86,16 @@ public SolutionRestoreJob(
             IVsSolutionManager solutionManager,
             ISourceRepositoryProvider sourceRepositoryProvider,
             IRestoreEventsPublisher restoreEventsPublisher,
-            ISettings settings)
+            ISettings settings,
+            ISolutionRestoreChecker solutionRestoreChecker)
         {
             Assumes.Present(asyncServiceProvider);
             Assumes.Present(packageRestoreManager);
             Assumes.Present(solutionManager);
             Assumes.Present(sourceRepositoryProvider);
             Assumes.Present(restoreEventsPublisher);
             Assumes.Present(settings);
+            Assumes.Present(solutionRestoreChecker);
 
             _asyncServiceProvider = asyncServiceProvider;
             _packageRestoreManager = packageRestoreManager;
@@ -97,6 +104,7 @@ public SolutionRestoreJob(
             _restoreEventsPublisher = restoreEventsPublisher;
             _settings = settings;
             _packageRestoreConsent = new PackageRestoreConsent(_settings);
+            _solutionUpToDateChecker = solutionRestoreChecker;
         }
 
         /// <summary>
@@ -148,8 +156,6 @@ public async Task<bool> ExecuteAsync(
             return _status == NuGetOperationStatus.NoOp || _status == NuGetOperationStatus.Succeeded;
         }
 
-
-
         private async Task RestoreAsync(bool forceRestore, RestoreOperationSource restoreSource, CancellationToken token)
         {
             var startTime = DateTimeOffset.Now;
@@ -279,6 +285,7 @@ private void EmitRestoreTelemetryEvent(IEnumerable<NuGetProject> projects,
                 startTime,
                 _status,
                 _packageCount,
+                _upToDateProjectCount,
                 _noOpProjectsCount,
                 DateTimeOffset.Now,
                 duration,
@@ -328,6 +335,7 @@ private async Task RestorePackageSpecProjectsAsync(
                 }
 
                 DependencyGraphCacheContext cacheContext;
+                DependencyGraphSpec originalDgSpec;
                 DependencyGraphSpec dgSpec;
                 IReadOnlyList<IAssetsLogMessage> additionalMessages;
 
@@ -338,12 +346,40 @@ private async Task RestorePackageSpecProjectsAsync(
                     var pathContext = NuGetPathContext.Create(_settings);
 
                     // Get full dg spec
-                    (dgSpec, additionalMessages) = await DependencyGraphRestoreUtility.GetSolutionRestoreSpecAndAdditionalMessages(_solutionManager, cacheContext);
+                    (originalDgSpec, additionalMessages) = await DependencyGraphRestoreUtility.GetSolutionRestoreSpecAndAdditionalMessages(_solutionManager, cacheContext);
+                }
+
+                using (intervalTracker.Start(RestoreTelemetryEvent.SolutionUpToDateCheck))
+                {
+                    // Run solution based up to date check.
+                    var projectsNeedingRestore = _solutionUpToDateChecker.PerformUpToDateCheck(originalDgSpec).AsList();
+
+                    dgSpec = originalDgSpec;
+                    // Only use the optimization results if the restore is not force.
+                    // Still run the optimization check anyways to prep the cache for a potential future non-force optimization
+                    if (!forceRestore)
+                    {
+                        // Update the dg spec.
+                        dgSpec = originalDgSpec.WithoutRestores();
+                        foreach (var uniqueProjectId in projectsNeedingRestore)
+                        {
+                            dgSpec.AddRestore(uniqueProjectId);
+                        }
+                        // loop through all legacy PackageReference projects. We don't know how to replay their warnings & errors yet.
+                        foreach(var project in (await _solutionManager.GetNuGetProjectsAsync()).Where(e => e is LegacyPackageReferenceProject).Select(e => e as LegacyPackageReferenceProject))
+                        {
+                            dgSpec.AddRestore(project.MSBuildProjectPath);
+                        }
+
+                        // recorded the number of up to date projects
+                        _upToDateProjectCount = originalDgSpec.Restore.Count - projectsNeedingRestore.Count;
+                        _noOpProjectsCount = _upToDateProjectCount;
+                    }
                 }
 
                 using (intervalTracker.Start(RestoreTelemetryEvent.PackageReferenceRestoreDuration))
                 {
-                    // Avoid restoring solutions with zero potential PackageReference projects.
+                    // Avoid restoring if all the projects are up to date, or the solution does not have build integrated projects.
                     if (DependencyGraphRestoreUtility.IsRestoreRequired(dgSpec))
                     {
                         // NOTE: During restore for build integrated projects,
@@ -379,8 +415,9 @@ await _logger.RunWithProgressAsync(
 
                                 _packageCount += restoreSummaries.Select(summary => summary.InstallCount).Sum();
                                 var isRestoreFailed = restoreSummaries.Any(summary => summary.Success == false);
-                                _noOpProjectsCount = restoreSummaries.Where(summary => summary.NoOpRestore == true).Count();
-
+                                _noOpProjectsCount += restoreSummaries.Where(summary => summary.NoOpRestore == true).Count();
+                                _solutionUpToDateChecker.ReportStatus(restoreSummaries);
+
                                 if (isRestoreFailed)
                                 {
                                     _status = NuGetOperationStatus.Failed;