Merge pull request #339: CloneVerb: Fall back to partial clone

This requires microsoft/git#249 in order to avoid a bad partial clone bug.

You can now try this:

```
$ scalar clone https://github.com/microsoft/scalar
Clone parameters:
  Repo URL:     https://github.com/microsoft/scalar
  Branch:       Default
  Cache Server: Default
  Local Cache:  C:\.scalarCache
  Destination:  C:\_git\t\scalar
  FullClone:     False
Authenticating...Succeeded
Fetching objects from remote...Succeeded
Checking out 'master'...Succeeded

$ ls scalar/src/
AuthoringTests.md  Dependencies.props     Directory.Build.targets  License.md    Protocol.md  Scalar.ruleset  SECURITY.md
CONTRIBUTING.md    Directory.Build.props  global.json              nuget.config  Readme.md    Scalar.sln      Signing.targets

$ cd scalar/src/

$ git sparse-checkout disable
remote: Enumerating objects: 418, done.
remote: Counting objects: 100% (418/418), done.
remote: Compressing objects: 100% (410/410), done.
remote: Total 483 (delta 71), reused 9 (delta 8), pack-reused 65
Receiving objects: 100% (483/483), 556.94 KiB | 8.57 MiB/s, done.
Resolving deltas: 100% (77/77), done.
Updating files: 100% (491/491), done.

$ ls
AuthoringTests.md        global.json   Scalar.Common/             Scalar.ruleset              Scalar.UnitTests/
CONTRIBUTING.md          License.md    Scalar.FunctionalTests/    Scalar.Service/             Scalar.Upgrader/
Dependencies.props       nuget.config  Scalar.Installer.Mac/      Scalar.Service.UI/          Scripts/
Directory.Build.props    Protocol.md   Scalar.Installer.Windows/  Scalar.Signing/             SECURITY.md
Directory.Build.targets  Readme.md     Scalar.MSBuild/            Scalar.sln                  Signing.targets
docs/                    Scalar/       Scalar.Notifications.Mac/  Scalar.TestInfrastructure/
```

or use SSH:

```
$ scalar clone [email protected]:microsoft/scalar.git scalar-ssh
Clone parameters:
  Repo URL:     [email protected]:microsoft/scalar.git
  Branch:       Default
  Cache Server: Default
  Local Cache:  C:\.scalarCache
  Destination:  C:\_git\t\scalar-ssh
  FullClone:     False
Fetching objects from remote...Succeeded
Checking out 'master'...Succeeded

$ cd scalar-ssh/src/

$ ls
AuthoringTests.md  Dependencies.props     Directory.Build.targets  License.md    Protocol.md  Scalar.ruleset  SECURITY.md
CONTRIBUTING.md    Directory.Build.props  global.json              nuget.config  Readme.md    Scalar.sln      Signing.targets

$ git sparse-checkout disable
remote: Enumerating objects: 418, done.
remote: Counting objects: 100% (418/418), done.
remote: Compressing objects: 100% (410/410), done.
remote: Total 483 (delta 71), reused 9 (delta 8), pack-reused 65
Receiving objects: 100% (483/483), 556.94 KiB | 9.95 MiB/s, done.
Resolving deltas: 100% (77/77), done.
Updating files: 100% (491/491), done.

$ ls
AuthoringTests.md        global.json   Scalar.Common/             Scalar.ruleset              Scalar.UnitTests/
CONTRIBUTING.md          License.md    Scalar.FunctionalTests/    Scalar.Service/             Scalar.Upgrader/
Dependencies.props       nuget.config  Scalar.Installer.Mac/      Scalar.Service.UI/          Scripts/
Directory.Build.props    Protocol.md   Scalar.Installer.Windows/  Scalar.Signing/             SECURITY.md
Directory.Build.targets  Readme.md     Scalar.MSBuild/            Scalar.sln                  Signing.targets
docs/                    Scalar/       Scalar.Notifications.Mac/  Scalar.TestInfrastructure/
```

I test a scenario for HTTPS clones, but SSH clones don't work due to the protections against man-in-the-middle attacks. Not sure how to resolve that, but the test I had would work from my dev box that had GitHub.com in my `known_hosts` file.

Directory.Build.props

@@ -36,7 +36,7 @@
 
     <!-- Version information -->
     <ScalarVersion>0.2.173.2</ScalarVersion>
-    <GitPackageVersion>2.20200221.5</GitPackageVersion>
+    <GitPackageVersion>2.20200227.1</GitPackageVersion>
     <MinimumGitVersion>v2.25.0.vfs.1.1</MinimumGitVersion>
     <WatchmanPackageUrl>https://github.com/facebook/watchman/suites/307436006/artifacts/304557</WatchmanPackageUrl>
     <GcmCoreOSXPackageUrl>https://github.com/microsoft/Git-Credential-Manager-Core/releases/download/v2.0.79-beta/gcmcore-osx-2.0.79.64449.pkg</GcmCoreOSXPackageUrl>

Scalar.Common/Git/GitAuthentication.cs

@@ -160,7 +160,7 @@ public bool TryGetCredentials(ITracer tracer, out string credentialString, out s
             return true;
         }
 
-        public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string errorMessage)
+        public Result TryInitialize(ITracer tracer, Enlistment enlistment, out string errorMessage)
         {
             if (this.isInitialized)
             {
@@ -173,18 +173,18 @@ public bool TryInitialize(ITracer tracer, Enlistment enlistment, out string erro
             if (!this.TryAnonymousQuery(tracer, enlistment, out isAnonymous))
             {
                 errorMessage = $"Unable to determine if authentication is required";
-                return false;
+                return Result.UnableToDetermine;
             }
 
             if (!isAnonymous &&
                 !this.TryCallGitCredential(tracer, out errorMessage))
             {
-                return false;
+                return Result.Failed;
             }
 
             this.IsAnonymous = isAnonymous;
             this.isInitialized = true;
-            return true;
+            return Result.Success;
         }
 
         public bool TryInitializeAndRequireAuth(ITracer tracer, out string errorMessage)
@@ -323,5 +323,12 @@ private bool TryCallGitCredential(ITracer tracer, out string errorMessage)
 
             return true;
         }
+
+        public enum Result
+        {
+            Success = 0,
+            Failed = 1,
+            UnableToDetermine = 2,
+        }
     }
 }

Scalar.Common/Git/GitProcess.cs

@@ -84,6 +84,24 @@ public GitProcess(string gitBinPath, string workingDirectoryRoot)
 
         public bool LowerPriority { get; set; }
 
+        public static Result Clone(string gitBinPath, string url, string targetDir, bool sparse, bool filterBlobs)
+        {
+            string sparseArg = sparse ? "--sparse " : string.Empty;
+            string filterArg = filterBlobs ? "--filter=blob:none" : string.Empty;
+
+            string dir = Paths.ConvertPathToGitFormat(targetDir);
+
+            GitProcess git = new GitProcess(gitBinPath, workingDirectoryRoot: null);
+
+            return git.InvokeGitImpl($"clone {sparseArg} {filterArg} {url} {dir}",
+                                     workingDirectory: Directory.GetCurrentDirectory(),
+                                     dotGitDirectory: null,
+                                     fetchMissingObjects: false,
+                                     writeStdIn: null,
+                                     parseStdOutLine: null,
+                                     timeoutMs: -1);
+        }
+
         public static Result Init(Enlistment enlistment)
         {
             return new GitProcess(enlistment).InvokeGitOutsideEnlistment("init \"" + enlistment.WorkingDirectoryRoot + "\"");
@@ -424,6 +442,18 @@ public Result ForceCheckoutAllFiles()
             return this.InvokeGitInWorkingDirectoryRoot("checkout HEAD -- .", fetchMissingObjects: true);
         }
 
+        public Result ForegroundFetch(string remote)
+        {
+            // By using "--refmap", we override the configured refspec,
+            // ignoring the normal "+refs/heads/*:refs/remotes/<remote>/*".
+            // The user will see their remote refs update
+            // normally when they do a foreground fetch.
+            return this.InvokeGitInWorkingDirectoryRoot(
+                $"-c credential.interactive=never fetch {remote} --quiet",
+                fetchMissingObjects: true,
+                userInteractive: false);
+        }
+
         public Result BackgroundFetch(string remote)
         {
             // By using this refspec, we do not create local refs, but instead store them in the "hidden"
@@ -468,7 +498,7 @@ public Result SparseCheckoutSet(List<string> foldersToSet)
                 {
                     foreach (string path in foldersToSet)
                     {
-                        string normalizedPath = path.Replace(Path.DirectorySeparatorChar, ScalarConstants.GitPathSeparator).TrimEnd(ScalarConstants.GitPathSeparator);
+                        string normalizedPath = Paths.ConvertPathToGitFormat(path).Trim(ScalarConstants.GitPathSeparator);
                         writer.Write(normalizedPath + "\n");
                     }
                 });

Scalar.FunctionalTests/Tests/MultiEnlistmentTests/ScalarCloneFromGithub.cs

@@ -0,0 +1,75 @@
+using NUnit.Framework;
+using Scalar.FunctionalTests.FileSystemRunners;
+using Scalar.FunctionalTests.Tools;
+using Scalar.Tests.Should;
+using System;
+using System.IO;
+using System.Linq;
+
+namespace Scalar.FunctionalTests.Tests.MultiEnlistmentTests
+{
+    [Category(Categories.GitRepository)]
+    public class ScalarCloneFromGithub : TestsWithMultiEnlistment
+    {
+        private static readonly string MicrosoftScalarHttp = "https://github.com/microsoft/scalar";
+        private static readonly string MicrosoftScalarSsh = "[email protected]:microsoft/scalar.git";
+
+        private FileSystemRunner fileSystem;
+
+        public ScalarCloneFromGithub()
+        {
+            this.fileSystem = new SystemIORunner();
+        }
+
+        [TestCase]
+        public void PartialCloneHttps()
+        {
+            ScalarFunctionalTestEnlistment enlistment = this.CreateNewEnlistment(
+                                                                url: MicrosoftScalarHttp,
+                                                                branch: "master",
+                                                                fullClone: false);
+
+            VerifyPartialCloneBehavior(enlistment);
+        }
+
+        private void VerifyPartialCloneBehavior(ScalarFunctionalTestEnlistment enlistment)
+        {
+            this.fileSystem.DirectoryExists(enlistment.RepoRoot).ShouldBeTrue($"'{enlistment.RepoRoot}' does not exist");
+
+            string gitPack = Path.Combine(enlistment.RepoRoot, ".git", "objects", "pack");
+            this.fileSystem.DirectoryExists(gitPack).ShouldBeTrue($"'{gitPack}' does not exist");
+
+            void checkPacks(string dir, int count, string when)
+            {
+                string dirContents = this.fileSystem
+                    .EnumerateDirectory(dir);
+
+                dirContents
+                    .Split()
+                    .Where(file => string.Equals(Path.GetExtension(file), ".pack", StringComparison.OrdinalIgnoreCase))
+                    .Count()
+                    .ShouldEqual(count, $"'{dir}' after '{when}': '{dirContents}'");
+            }
+
+            // Two packs for clone: commits and trees, blobs at root
+            checkPacks(gitPack, 2, "clone");
+
+            string srcScalar = Path.Combine(enlistment.RepoRoot, "Scalar");
+            this.fileSystem.DirectoryExists(srcScalar).ShouldBeFalse($"'{srcScalar}' should not exist due to sparse-checkout");
+
+            ProcessResult sparseCheckoutResult = GitProcess.InvokeProcess(enlistment.RepoRoot, "sparse-checkout disable");
+            sparseCheckoutResult.ExitCode.ShouldEqual(0, "git sparse-checkout disable exit code");
+
+            this.fileSystem.DirectoryExists(srcScalar).ShouldBeTrue($"'{srcScalar}' should exist after sparse-checkout");
+
+            // Three packs for sparse-chekcout: commits and trees, blobs at root, blobs outside root
+            checkPacks(gitPack, 3, "sparse-checkout");
+
+            ProcessResult checkoutResult = GitProcess.InvokeProcess(enlistment.RepoRoot, "checkout HEAD~10");
+            checkoutResult.ExitCode.ShouldEqual(0, "git checkout exit code");
+
+            // Four packs for chekcout: commits and trees, blobs at root, blobs outside root, checkout diff
+            checkPacks(gitPack, 4, "checkout");
+        }
+    }
+}

Scalar.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs

@@ -31,13 +31,17 @@ protected virtual void OnTearDownEnlistmentsDeleted()
         protected ScalarFunctionalTestEnlistment CreateNewEnlistment(
             string localCacheRoot = null,
             string branch = null,
-            bool skipFetchCommitsAndTrees = false)
+            bool skipFetchCommitsAndTrees = false,
+            string url = null,
+            bool fullClone = true)
         {
             ScalarFunctionalTestEnlistment output = ScalarFunctionalTestEnlistment.Clone(
                 ScalarTestConfig.PathToScalar,
                 branch,
                 localCacheRoot,
-                skipFetchCommitsAndTrees);
+                skipFetchCommitsAndTrees,
+                fullClone: fullClone,
+                url: url);
             this.enlistmentsToDelete.Add(output);
             return output;
         }

Scalar.FunctionalTests/Tools/ScalarFunctionalTestEnlistment.cs

@@ -100,10 +100,11 @@ public static ScalarFunctionalTestEnlistment Clone(
             string commitish = null,
             string localCacheRoot = null,
             bool skipFetchCommitsAndTrees = false,
-            bool fullClone = true)
+            bool fullClone = true,
+            string url = null)
         {
             string enlistmentRoot = ScalarFunctionalTestEnlistment.GetUniqueEnlistmentRoot();
-            return Clone(pathToScalar, enlistmentRoot, commitish, localCacheRoot, skipFetchCommitsAndTrees, fullClone);
+            return Clone(pathToScalar, enlistmentRoot, commitish, localCacheRoot, skipFetchCommitsAndTrees, fullClone, url);
         }
 
         public static ScalarFunctionalTestEnlistment CloneEnlistmentWithSpacesInPath(string pathToScalar, string commitish = null)
@@ -268,14 +269,15 @@ private static ScalarFunctionalTestEnlistment Clone(
             string commitish,
             string localCacheRoot,
             bool skipFetchCommitsAndTrees = false,
-            bool fullClone = true)
+            bool fullClone = true,
+            string url = null)
         {
             enlistmentRoot = enlistmentRoot ?? GetUniqueEnlistmentRoot();
 
             ScalarFunctionalTestEnlistment enlistment = new ScalarFunctionalTestEnlistment(
                 pathToScalar,
                 enlistmentRoot,
-                ScalarTestConfig.RepoToClone,
+                url ?? ScalarTestConfig.RepoToClone,
                 commitish ?? Properties.Settings.Default.Commitish,
                 localCacheRoot ?? ScalarTestConfig.LocalCacheRoot,
                 fullClone);

Scalar/CommandLine/CacheServerVerb.cs

@@ -1,5 +1,6 @@
 using CommandLine;
 using Scalar.Common;
+using Scalar.Common.Git;
 using Scalar.Common.Http;
 using Scalar.Common.Tracing;
 using System;
@@ -43,7 +44,7 @@ protected override void Execute(ScalarEnlistment enlistment)
             using (ITracer tracer = new JsonTracer(ScalarConstants.ScalarEtwProviderName, "CacheVerb"))
             {
                 string authErrorMessage;
-                if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage))
+                if (this.TryAuthenticate(tracer, enlistment, out authErrorMessage) != GitAuthentication.Result.Success)
                 {
                     this.ReportErrorAndExit(tracer, "Authentication failed: " + authErrorMessage);
                 }

Scalar/CommandLine/CloneVerb.cs

@@ -218,8 +218,22 @@ private Result DoClone(string fullEnlistmentRootPathParameter, string normalized
             this.Output.WriteLine("  Destination:  " + this.enlistment.EnlistmentRoot);
             this.Output.WriteLine("  FullClone:     " + this.FullClone);
 
-            string authErrorMessage;
-            if (!this.TryAuthenticate(this.tracer, this.enlistment, out authErrorMessage))
+            string authErrorMessage = null;
+            GitAuthentication.Result authResult = GitAuthentication.Result.UnableToDetermine;
+
+            // Do not try authentication on SSH URLs.
+            if (this.enlistment.RepoUrl.StartsWith("https://"))
+            {
+                authResult = this.TryAuthenticate(this.tracer, this.enlistment, out authErrorMessage);
+            }
+
+            if (authResult == GitAuthentication.Result.UnableToDetermine)
+            {
+                // We can't tell, because we don't have the right endpoint!
+                return this.GitClone();
+            }
+
+            if (authResult == GitAuthentication.Result.Failed)
             {
                 this.ReportErrorAndExit(this.tracer, "Cannot clone because authentication failed: " + authErrorMessage);
             }
@@ -291,6 +305,99 @@ private Result DoClone(string fullEnlistmentRootPathParameter, string normalized
             return cloneResult;
         }
 
+        private Result GitClone()
+        {
+            string gitBinPath = ScalarPlatform.Instance.GitInstallation.GetInstalledGitBinPath();
+            if (string.IsNullOrWhiteSpace(gitBinPath))
+            {
+                return new Result(ScalarConstants.GitIsNotInstalledError);
+            }
+
+            GitProcess git = new GitProcess(this.enlistment);
+
+            // protocol.version=2 is broken right now.
+            git.SetInLocalConfig("protocol.version", "1");
+
+            git.SetInLocalConfig("remote.origin.url", this.RepositoryURL);
+            git.SetInLocalConfig("remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*");
+            git.SetInLocalConfig("remote.origin.promisor", "true");
+            git.SetInLocalConfig("remote.origin.partialCloneFilter", "blob:none");
+
+            string branch = this.Branch ?? "master";
+            git.SetInLocalConfig($"branch.{branch}.remote", "origin");
+            git.SetInLocalConfig($"branch.{branch}.merge", $"refs/heads/{branch}");
+
+            if (!this.FullClone)
+            {
+                git.SetInLocalConfig($"core.sparseCheckout", "true");
+                git.SetInLocalConfig($"core.sparseCheckoutCone", "true");
+
+                this.fileSystem.CreateDirectory(Path.Combine(this.enlistment.DotGitRoot, "info"));
+                this.fileSystem.WriteAllText(Path.Combine(this.enlistment.DotGitRoot, "info", "sparse-checkout"), "/*\n!/*/*");
+            }
+
+            this.context = new ScalarContext(this.tracer, this.fileSystem, this.enlistment);
+
+            // Set required and optional config.
+            // Explicitly pass useGvfsProtocol: true as the enlistment can not discover that setting from
+            // Git config yet. Other verbs will discover this automatically from the config we set now.
+            ConfigStep configStep = new ConfigStep(this.context, useGvfsProtocol: false);
+
+            if (!configStep.TrySetConfig(out string configError))
+            {
+                return new Result($"Failed to set initial config: {configError}");
+            }
+
+            GitProcess.Result fetchResult = null;
+            if (!this.ShowStatusWhileRunning(() =>
+            {
+                using (ITracer activity = this.tracer.StartActivity("git-fetch-partial", EventLevel.LogAlways))
+                {
+                    fetchResult = git.ForegroundFetch("origin");
+                    return fetchResult.ExitCodeIsSuccess;
+                }
+            },
+                "Fetching objects from remote"))
+            {
+                if (!fetchResult.Errors.Contains("filtering not recognized by server"))
+                {
+                    return new Result($"Failed to complete regular clone: {fetchResult?.Errors}");
+                }
+            }
+
+            if (fetchResult.ExitCodeIsFailure &&
+                !this.ShowStatusWhileRunning(() =>
+                {
+                    using (ITracer activity = this.tracer.StartActivity("git-fetch", EventLevel.LogAlways))
+                    {
+                        git.DeleteFromLocalConfig("remote.origin.promisor");
+                        git.DeleteFromLocalConfig("remote.origin.partialCloneFilter");
+                        fetchResult = git.ForegroundFetch("origin");
+                        return fetchResult.ExitCodeIsSuccess;
+                    }
+                },
+                "Fetching objects from remote"))
+            {
+                return new Result($"Failed to complete regular clone: {fetchResult?.Errors}");
+            }
+
+            GitProcess.Result checkoutResult = null;
+
+            if (!this.ShowStatusWhileRunning(() =>
+                    {
+                        using (ITracer activity = this.tracer.StartActivity("git-checkout", EventLevel.LogAlways))
+                        {
+                            checkoutResult = git.ForceCheckout(branch);
+                            return checkoutResult.ExitCodeIsSuccess;
+                        }
+                    },
+                $"Checking out '{branch}'"))
+            {
+                return new Result($"Failed to complete regular clone: {checkoutResult?.Errors}");
+            }
+            return new Result(true);
+        }
+
         private Result TryCreateEnlistment(
             string fullEnlistmentRootPathParameter,
             string normalizedEnlistementRootPath,
@@ -329,7 +436,9 @@ private Result TryCreateEnlistment(
                 return new Result($"Error when creating a new Scalar enlistment at '{normalizedEnlistementRootPath}'. {e.Message}");
             }
 
-            return new Result(true);
+            GitProcess.Result initResult = GitProcess.Init(enlistment);
+
+            return new Result(initResult.ExitCodeIsSuccess);
         }
 
         private Result CreateScalarDirctories(string resolvedLocalCacheRoot)

Scalar/CommandLine/RunVerb.cs

@@ -231,7 +231,7 @@ private void InitializeServerConnection(
             if (!this.SkipVersionCheck)
             {
                 string authErrorMessage;
-                if (!this.TryAuthenticate(tracer, enlistment, out authErrorMessage))
+                if (this.TryAuthenticate(tracer, enlistment, out authErrorMessage) != GitAuthentication.Result.Success)
                 {
                     this.ReportErrorAndExit(tracer, "Unable to fetch because authentication failed: " + authErrorMessage);
                 }