[xharness] Improve listing files in the repository. (#18183)

When cloning projects, we need to list all the files in the original project
directory and manually include some of them in the cloned project (because in
.NET some files from the project directory are included automatically, and if
we clone a project and put the cloned project in a different directory, those
files won't be picked up automatically by the build anymore).

The previous code to list the files in the project directory would run 'git
ls-files' for each project directory. This is rather slow, since it happens
quite a few times. Instead modify the logic to run 'git ls-files' once for the
entire tests/ directory, store the result, and then when we need to list files
in a particular project directory, just look in that stored list for the
applicable files.

This is much, much faster.

tests/xharness/Harness.cs

@@ -1,10 +1,13 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.XHarness.Common;
+using Microsoft.DotNet.XHarness.Common.Execution;
 using Microsoft.DotNet.XHarness.Common.Logging;
 using Microsoft.DotNet.XHarness.iOS.Shared;
 using Microsoft.DotNet.XHarness.iOS.Shared.Execution;
@@ -74,6 +77,41 @@ public static string RootDirectory {
 					root_directory = Path.GetFullPath (root_directory).TrimEnd ('/');
 			}
 		}
+
+		static SemaphoreSlim ls_files_semaphore = new SemaphoreSlim (1);
+		static List<string> files_in_git = new List<string> ();
+		public static async Task<IEnumerable<string>> ListFilesInGitAsync (ILog log, string test_dir, IProcessManager processManager)
+		{
+			var acquired = await ls_files_semaphore.WaitAsync (TimeSpan.FromMinutes (5));
+			try {
+				if (!acquired)
+					log.WriteLine ($"Unable to acquire lock to run 'git ls-files {test_dir}' in 5 minutes; will try to run anyway.");
+				if (files_in_git.Count == 0) {
+					using var process = new Process ();
+					process.StartInfo.FileName = "git";
+					process.StartInfo.Arguments = "ls-files";
+					process.StartInfo.WorkingDirectory = RootDirectory;
+					var stdout = new MemoryLog () { Timestamp = false };
+					var result = await processManager.RunAsync (process, log, stdout, stdout, timeout: TimeSpan.FromSeconds (60));
+					if (!result.Succeeded)
+						throw new Exception ($"Failed to list the files in the directory {test_dir} (TimedOut: {result.TimedOut} ExitCode: {result.ExitCode}):\n{stdout}");
+					files_in_git.AddRange (stdout.ToString ().Split ('\n'));
+				}
+				var relative_dir = Path.GetFullPath (test_dir);
+				if (relative_dir.StartsWith (RootDirectory)) {
+					relative_dir = relative_dir.Substring (RootDirectory.Length);
+					relative_dir = relative_dir.TrimStart ('/');
+				}
+				if (!relative_dir.EndsWith ("/", StringComparison.Ordinal))
+					relative_dir += "/";
+				return files_in_git
+					.Where (v => v.StartsWith (relative_dir, StringComparison.OrdinalIgnoreCase))
+					.Select (v => v.Substring (relative_dir.Length));
+			} finally {
+				if (acquired)
+					ls_files_semaphore.Release ();
+			}
+		}
 	}
 
 	public class Harness : IHarness {

tests/xharness/TestProject.cs

@@ -4,6 +4,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
@@ -83,29 +84,6 @@ public Task CreateCopyAsync (ILog log, IProcessManager processManager, ITestTask
 			return CreateCopyAsync (log, processManager, test, rootDirectory, pr);
 		}
 
-		static SemaphoreSlim ls_files_semaphore = new SemaphoreSlim (1);
-
-		async Task<string []> ListFilesAsync (ILog log, string test_dir, IProcessManager processManager)
-		{
-			var acquired = await ls_files_semaphore.WaitAsync (TimeSpan.FromMinutes (5));
-			try {
-				if (!acquired)
-					log.WriteLine ($"Unable to acquire lock to run 'git ls-files {test_dir}' in 5 minutes; will try to run anyway.");
-				using var process = new Process ();
-				process.StartInfo.FileName = "git";
-				process.StartInfo.Arguments = "ls-files";
-				process.StartInfo.WorkingDirectory = test_dir;
-				var stdout = new MemoryLog () { Timestamp = false };
-				var result = await processManager.RunAsync (process, stdout, stdout, stdout, timeout: TimeSpan.FromSeconds (60));
-				if (!result.Succeeded)
-					throw new Exception ($"Failed to list the files in the directory {test_dir} (TimedOut: {result.TimedOut} ExitCode: {result.ExitCode}):\n{stdout}");
-				return stdout.ToString ().Split ('\n');
-			} finally {
-				if (acquired)
-					ls_files_semaphore.Release ();
-			}
-		}
-
 		async Task CreateCopyAsync (ILog log, IProcessManager processManager, ITestTask test, string rootDirectory, Dictionary<string, TestProject> allProjectReferences)
 		{
 			var directory = Cache.CreateTemporaryDirectory (test.TestName ?? System.IO.Path.GetFileNameWithoutExtension (Path));
@@ -162,7 +140,7 @@ async Task CreateCopyAsync (ILog log, IProcessManager processManager, ITestTask
 					// because the cloned project is stored in a very different directory.
 					var test_dir = System.IO.Path.GetDirectoryName (original_path);
 
-					var files = await ListFilesAsync (log, test_dir, processManager);
+					var files = await HarnessConfiguration.ListFilesInGitAsync (log, test_dir, processManager);
 					foreach (var file in files) {
 						var ext = System.IO.Path.GetExtension (file);
 						var full_path = System.IO.Path.Combine (test_dir, file);