Clean up and remove debug-only code

JakeYallop · May 24, 2024 · 6c2e257 · 6c2e257
1 parent 06cc051
commit 6c2e257
Show file tree

Hide file tree

Showing 7 changed files with 36 additions and 45 deletions.
diff --git a/WaybackDownloader.sln b/WaybackDownloader.sln
@@ -18,6 +18,13 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WaybackDownloader.Tests", "WaybackDownloader.Tests\WaybackDownloader.Tests.csproj", "{21B1BAA3-1CC9-4652-B5ED-B28FF11EA0C9}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{946C27DD-CB42-4E3F-A1D8-42537CEDFBF5}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{C0C489F1-82C6-4E56-AAE8-55C1AD60E141}"
+	ProjectSection(SolutionItems) = preProject
+		.github\workflows\build-and-test.yml = .github\workflows\build-and-test.yml
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -36,6 +43,9 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{C0C489F1-82C6-4E56-AAE8-55C1AD60E141} = {946C27DD-CB42-4E3F-A1D8-42537CEDFBF5}
+	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {5EAF9CEB-99CF-40E2-AF44-00F1A5A2AF3E}
 	EndGlobalSection

diff --git a/WaybackDownloader/DefaultCommand.Settings.cs b/WaybackDownloader/DefaultCommand.Settings.cs
@@ -85,10 +85,8 @@ public sealed class Settings : CommandSettings
         [DefaultValue(false)]
         public bool Verbose { get; init; }
 
-#if DEBUG
         [CommandOption("--useMockHandler")]
         public bool UseMockHandler { get; set; }
-#endif
 
         public override ValidationResult Validate()
         {

diff --git a/WaybackDownloader/DefaultCommand.cs b/WaybackDownloader/DefaultCommand.cs
@@ -24,7 +24,7 @@ public override async Task<int> ExecuteAsync(CommandContext context, Settings se
         Task? pageWokerRunnerTask = null;
 
         using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
-        cts.Token.Register(() => AnsiConsole.WriteLine("Shutting down."));
+        cts.Token.Register(() => AnsiConsole.WriteLine("Shutting down..."));
 
         try
         {
@@ -43,11 +43,7 @@ public override async Task<int> ExecuteAsync(CommandContext context, Settings se
 #pragma warning restore CA2000 // Dispose objects before losing scope
 
             services
-#if DEBUG
                 .AddCoreCommandServices(rateLimiter, settings.Verbose, mockData: settings.UseMockHandler)
-#else
-                .AddCoreCommandServices(rateLimiter, settings.Verbose)
-#endif
                 .AddSingleton(AnsiConsole.Create(new()))
                 .AddSingleton<Ui>()
                 .AddSingleton(new PageFilters([.. settings.PageFilters]));
@@ -120,12 +116,6 @@ public override async Task<int> ExecuteAsync(CommandContext context, Settings se
             }
             else
             {
-                //TODO: Move this to the dipose method of the pageWorkerRunnerTask, assuming it will not cause a deadlock
-                //if (pageWokerRunnerTask is not null)
-                //{
-                //    await pageWokerRunnerTask.WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None).ConfigureAwait(false);
-                //}
-                AnsiConsole.WriteLine("Background tasks finnished.");
                 var disposeException = await DisposeProviderAsync(serviceProvider).ConfigureAwait(false);
                 if (disposeException is not null)
                 {
@@ -211,12 +201,10 @@ private static void PrintSettings(DefaultCommand.Settings settings)
             AnsiConsole.WriteLine("Verbose logging enabled.");
         }
 
-#if DEBUG
         if (settings.UseMockHandler)
         {
             AnsiConsole.Write($"Use mock handler: {settings.UseMockHandler} ");
             AnsiConsole.MarkupLine("[yellow]No real API requests will be made. Garbage data will be saved to disk.[/]");
         }
-#endif
     }
 }
diff --git a/WaybackDownloader/MockDataHttpMessageHandler.cs b/WaybackDownloader/MockDataHttpMessageHandler.cs
@@ -15,20 +15,21 @@ var x when x.StartsWith("/cdx/search/cdx", StringComparison.OrdinalIgnoreCase) =
         return response;
     }
 
+    private static int _year = 1001;
     private static async Task<HttpResponseMessage> CreateCdxResponseAsync(CancellationToken cancellationToken)
     {
         var a = new HttpResponseMessage(HttpStatusCode.OK);
-        a.Content = new StringContent("""
-            org,archive)/about/"%20id= 20120731030657 http://www.archive.org/about/%22%20id= text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 338
-            org,archive)/about/"%20id= 20120731030659 http://archive.org/about/%22%20id= text/html 404 6GBF3VJD2SDHBHKQNNV7OAWW6YW33AKO 2943
-            org,archive)/about/%09identifier_marc.xml: 20120731030709 http://www.archive.org/about/%09IDENTIFIER_marc.xml: text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 350
-            org,archive)/about/%09identifier_marc.xml: 20120731030710 http://archive.org/about/%09IDENTIFIER_marc.xml: text/html 404 D6CHVFU5PP4OYNILSUWTASOOQLF6XSZC 2954
-            org,archive)/about/%09identifier_meta.xml 20120731030705 http://www.archive.org/about/%09IDENTIFIER_meta.xml text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 349
-            org,archive)/about/%09identifier_meta.xml 20120731030706 http://archive.org/about/%09IDENTIFIER_meta.xml text/html 404 YZNHYLIJFOUQZZYIYLJRAKMT42OXT2NS 2955
-            org,archive)/about/%0d%0aexclude.php 20060726224227 http://www.archive.org/about/%0d%0aexclude.php text/html 404 RECXT54QH34IINX54NU4VRFSGJD4MFMZ 2942
-            org,archive)/about/%0d%0aexclude.php 20060906001931 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2313
-            org,archive)/about/%0d%0aexclude.php 20060913011111 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2313
-            org,archive)/about/%0d%0aexclude.php 20060919234953 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2316
+        a.Content = new StringContent($"""
+            org,archive)/about/"%20id= {Interlocked.Increment(ref _year)}0731030657 http://www.archive.org/about/%22%20id= text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 338
+            org,archive)/about/"%20id= {Interlocked.Increment(ref _year)}0731030659 http://archive.org/about/%22%20id= text/html 404 6GBF3VJD2SDHBHKQNNV7OAWW6YW33AKO 2943
+            org,archive)/about/%09identifier_marc.xml: {Interlocked.Increment(ref _year)}0731030709 http://www.archive.org/about/%09IDENTIFIER_marc.xml: text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 350
+            org,archive)/about/%09identifier_marc.xml: {Interlocked.Increment(ref _year)}0731030710 http://archive.org/about/%09IDENTIFIER_marc.xml: text/html 404 D6CHVFU5PP4OYNILSUWTASOOQLF6XSZC 2954
+            org,archive)/about/%09identifier_meta.xml {Interlocked.Increment(ref _year)}0731030705 http://www.archive.org/about/%09IDENTIFIER_meta.xml text/html 302 OXCFCQFVBEUULMT2CUOBYU4T42VH7MYM 349
+            org,archive)/about/%09identifier_meta.xml {Interlocked.Increment(ref _year)}0731030706 http://archive.org/about/%09IDENTIFIER_meta.xml text/html 404 YZNHYLIJFOUQZZYIYLJRAKMT42OXT2NS 2955
+            org,archive)/about/%0d%0aexclude.php {Interlocked.Increment(ref _year)}0726224227 http://www.archive.org/about/%0d%0aexclude.php text/html 404 RECXT54QH34IINX54NU4VRFSGJD4MFMZ 2942
+            org,archive)/about/%0d%0aexclude.php {Interlocked.Increment(ref _year)}0906001931 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2313
+            org,archive)/about/%0d%0aexclude.php {Interlocked.Increment(ref _year)}0913011111 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2313
+            org,archive)/about/%0d%0aexclude.php {Interlocked.Increment(ref _year)}0919234953 http://www.archive.org/about/%0d%0aexclude.php text/html 404 CYSYSFG5DXSXMLLR25X5MNP2Z5FXLD22 2316
             """);
 
         await Task.Delay(15, cancellationToken).ConfigureAwait(false);

diff --git a/WaybackDownloader/ServiceCollectionExtensions.cs b/WaybackDownloader/ServiceCollectionExtensions.cs
@@ -11,11 +11,7 @@
 namespace WaybackDownloader;
 public static class ServiceCollectionExtensions
 {
-#if DEBUG
     public static IServiceCollection AddCoreCommandServices(this IServiceCollection services, RateLimiter pageWorkerHttpClientRateLimiter, bool verbose = false, bool mockData = false)
-#else
-    public static IServiceCollection AddCoreCommandServices(this IServiceCollection services, RateLimiter pageWorkerHttpClientRateLimiter, bool verbose = false)
-#endif
     {
         services.AddLogging(builder =>
         {
@@ -28,12 +24,11 @@ public static IServiceCollection AddCoreCommandServices(this IServiceCollection
         .ConfigureHttpClientDefaults(configure =>
         {
             configure.RemoveAllLoggers();
-#if DEBUG
+
             if (mockData)
             {
                 configure.ConfigurePrimaryHttpMessageHandler(s => new MockDataHttpMessageHandler());
             }
-#endif
             configure.ConfigureHttpClient(x =>
             {
                 x.DefaultRequestHeaders.UserAgent.Add(new("WaybackDownloader", DefaultCommand.Version));
@@ -86,15 +81,14 @@ public static IServiceCollection AddCoreCommandServices(this IServiceCollection
             configure
                 .AddRateLimiter(pageWorkerHttpClientRateLimiter)
                 .AddRetry(new HttpRetryStrategyOptions
-                {
-                    BackoffType = DelayBackoffType.Linear,
-                    MaxRetryAttempts = 3,
-                    Delay = TimeSpan.FromSeconds(10),
-                    ShouldRetryAfterHeader = true,
-                    UseJitter = true,
-                    ShouldHandle = static args => ValueTask.FromResult(HttpClientResiliencePredicates.IsTransient(args.Outcome) || args.Outcome.Result?.StatusCode is System.Net.HttpStatusCode.RequestTimeout),
-                })
-                .AddCircuitBreaker(new CircuitBreakerStrategyOptions<HttpResponseMessage>());
+                 {
+                     BackoffType = DelayBackoffType.Linear,
+                     MaxRetryAttempts = 3,
+                     Delay = TimeSpan.FromSeconds(10),
+                     ShouldRetryAfterHeader = true,
+                     UseJitter = true,
+                     ShouldHandle = static args => ValueTask.FromResult(HttpClientResiliencePredicates.IsTransient(args.Outcome) || args.Outcome.Result?.StatusCode is System.Net.HttpStatusCode.RequestTimeout),
+                 });
         })
         .Services
         .AddSingleton(Channel.CreateBounded<CdxRecord>(new BoundedChannelOptions(200)
@@ -104,9 +98,6 @@ public static IServiceCollection AddCoreCommandServices(this IServiceCollection
             AllowSynchronousContinuations = false
         }))
         .AddSingleton<DownloaderService>()
-        //Do not re-register WaybackCdxClient, otherwise previous HttpClient configuration is lost.
-        //.AddSingleton<WaybackCdxClient>()
-        //.AddSingleton<PageWorker>()
         .AddSingleton<PageWorkerRunner>()
         .AddSingleton<PagesStore>()
         .AddSingleton<IConsoleMessageColorProvider, SpectreConsoleMessageColorProvider>();

diff --git a/WaybackDownloader/Services/PageWorker.cs b/WaybackDownloader/Services/PageWorker.cs
@@ -2,6 +2,7 @@
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
 using Polly;
+using Spectre.Console;
 
 namespace WaybackDownloader.Services;
 

diff --git a/WaybackDownloader/Services/PageWorkerRunner.cs b/WaybackDownloader/Services/PageWorkerRunner.cs
@@ -1,6 +1,7 @@
 using System.Threading.RateLimiting;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using Spectre.Console;
 
 namespace WaybackDownloader.Services;
 
@@ -88,7 +89,8 @@ private async Task EvaluateLimitAsync(string outputDir, int requestedDownloadLim
 
     private async ValueTask DisposeAllAsync()
     {
-        await WaitForCompletionAsync().WaitAsync(TimeSpan.FromSeconds(5), CancellationToken.None).ConfigureAwait(false);
+        AnsiConsole.WriteLine("Waiting up to 10 seconds for workers to finish.");
+        await WaitForCompletionAsync().WaitAsync(TimeSpan.FromSeconds(10), CancellationToken.None).ConfigureAwait(false);
         return;
     }
 }