Simplify task management

CromwellOnAzure.sln

@@ -32,6 +32,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonUtilities", "src\ga4g
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CommonUtilities.Tests", "src\ga4gh-tes\src\CommonUtilities.Tests\CommonUtilities.Tests.csproj", "{E9B266DA-528E-44D5-B934-7C352E0E3BDE}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tes.Repository", "src\ga4gh-tes\src\Tes.Repository\Tes.Repository.csproj", "{515A4905-0522-4C72-BC18-41BE6A3BE880}"
+EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "build-push-acr", "src\ga4gh-tes\src\build-push-acr\build-push-acr.csproj", "{42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}"
 EndProject
 Global
@@ -80,6 +82,10 @@ Global
 		{E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E9B266DA-528E-44D5-B934-7C352E0E3BDE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{515A4905-0522-4C72-BC18-41BE6A3BE880}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{515A4905-0522-4C72-BC18-41BE6A3BE880}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{515A4905-0522-4C72-BC18-41BE6A3BE880}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{515A4905-0522-4C72-BC18-41BE6A3BE880}.Release|Any CPU.Build.0 = Release|Any CPU
 		{42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{42BF91C4-4A6A-4BA5-9F67-1B4E09D499E1}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs

@@ -142,7 +142,7 @@ private TriggerHostedService SetCromwellOnAzureEnvironment(string accountAuthori
             }
 
             var logger = new Mock<ILogger<TriggerHostedService>>().Object;
-            var postgreSqlOptions = new Mock<IOptions<PostgreSqlOptions>>().Object;
+            var postgreSqlOptions = new Mock<IOptions<Tes.Repository.Models.PostgreSqlOptions>>().Object;
             var cromwellApiClient = new Mock<ICromwellApiClient>().Object;
             var repository = new Mock<IRepository<TesTask>>().Object;
             var storageUtility = new Mock<IAzureStorageUtility>();

src/TriggerService.Tests/IntegrationTests.cs

@@ -30,10 +30,7 @@ public class IntegrationTests
         [TestMethod]
         public async Task RunScaleTestWithMutect2WaitTilDoneAsync()
         {
-            const string triggerFile = "https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json";
-            const string workflowFriendlyName = $"mutect2";
-
-            await RunIntegrationTestAsync(new List<(string triggerFileBlobUrl, string workflowFriendlyName)> { (triggerFile, workflowFriendlyName) });
+            await RunIntegrationTestAsync([("https://raw.githubusercontent.com/microsoft/CromwellOnAzure/main/src/TriggerService.Tests/test-wdls/mutect2/mutect2.trigger.json", "mutect2")]);
         }
 
         /// <summary>
@@ -44,7 +41,8 @@ public async Task RunScaleTestWithMutect2WaitTilDoneAsync()
         [TestMethod]
         public async Task RunAllCommonWorkflowsWaitTilDoneAsync()
         {
-            var workflowTriggerFiles = new List<(string triggerFileBlobUrl, string workflowFriendlyName)> {
+            await RunIntegrationTestAsync(
+            [
                 ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.b37.trigger.json", "preprocessing-b37"),
                 ("https://raw.githubusercontent.com/microsoft/gatk4-data-processing-azure/main-azure/processing-for-variant-discovery-gatk4.hg38.trigger.json", "preprocessing-hg38"),
                 ("https://raw.githubusercontent.com/microsoft/gatk4-genome-processing-pipeline-azure/main-azure/WholeGenomeGermlineSingleSample.trigger.json", "germline"),
@@ -53,9 +51,8 @@ public async Task RunAllCommonWorkflowsWaitTilDoneAsync()
                 ("https://raw.githubusercontent.com/microsoft/gatk4-cnn-variant-filter-azure/main-azure/cram2filtered.trigger.json", "cram-to-filtered"),
                 ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/interleaved-fastq-to-paired-fastq.trigger.json", "fastq-to-paired"),
                 ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/paired-fastq-to-unmapped-bam.trigger.json", "paired-fastq-to-unmapped-bam"),
-                ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/cram-to-bam.trigger.json", "cram-to-bam") };
-
-            await RunIntegrationTestAsync(workflowTriggerFiles);
+                ("https://raw.githubusercontent.com/microsoft/seq-format-conversion-azure/main-azure/cram-to-bam.trigger.json", "cram-to-bam")
+            ]);
         }
 
         /// <summary>
@@ -89,7 +86,7 @@ public async Task RunGlobTestWdlAsync()
             var n = DateTime.UtcNow;
             var date = $"{n.Year}-{n.Month}-{n.Day}-{n.Hour}-{n.Minute}";
             var triggerFileBlobName = $"new/globtest-{date}.json";
-            string triggerJson = System.Text.Json.JsonSerializer.Serialize(workflowTrigger).Replace(@"\r\n\", @"\n");
+            var triggerJson = System.Text.Json.JsonSerializer.Serialize(workflowTrigger).Replace(@"\r\n\", @"\n");
             container = blobServiceClient.GetBlobContainerClient("workflows");
             await container.GetBlobClient(triggerFileBlobName).UploadAsync(BinaryData.FromString(triggerJson), true);
         }
@@ -164,7 +161,7 @@ public async Task DeleteOldBatchPoolsAsync()
             var cutoffTime = DateTime.UtcNow.Subtract(maxAge);
             var pools = await batchClient.PoolOperations.ListPools().ToListAsync();
 
-            int count = 0;
+            var count = 0;
 
             foreach (var pool in pools)
             {
@@ -201,7 +198,7 @@ public void CountCompletedWorkflowsTest()
             Assert.IsTrue(CountWorkflowsByState(originalBlobNames, currentBlobNames, WorkflowState.Succeeded) == 1);
         }
 
-        private async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles)
+        private static async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles)
         {
             // This is set in the Azure Devops pipeline, which writes the file to the .csproj directory
             // The current working directory is this: /mnt/vss/_work/r1/a/CoaArtifacts/AllSource/TriggerService.Tests/bin/Debug/net7.0/
@@ -217,20 +214,20 @@ private async Task RunIntegrationTestAsync(List<(string triggerFileBlobUrl, stri
 
             Console.WriteLine($"Found path: {path}");
             var lines = await File.ReadAllLinesAsync(path);
-            string storageAccountName = lines[0].Trim();
-            string workflowsContainerSasToken = lines[1].Trim('"');
+            var storageAccountName = lines[0].Trim();
+            var workflowsContainerSasToken = lines[1].Trim('"');
 
-            int countOfEachWorkflowToRun = 1;
+            var countOfEachWorkflowToRun = 1;
 
             if (lines.Length > 2)
             {
-                int.TryParse(lines[2].Trim('"'), out countOfEachWorkflowToRun);
+                _ = int.TryParse(lines[2].Trim('"'), out countOfEachWorkflowToRun);
             }
 
             await StartWorkflowsAsync(countOfEachWorkflowToRun, triggerFiles, storageAccountName, waitTilDone: true, workflowsContainerSasToken);
         }
 
-        private async Task StartWorkflowsAsync(
+        private static async Task StartWorkflowsAsync(
             int countOfEachWorkflowToRun,
             List<(string triggerFileBlobUrl, string workflowFriendlyName)> triggerFiles,
             string storageAccountName,
@@ -265,9 +262,9 @@ private async Task StartWorkflowsAsync(
             using var httpClient = new HttpClient();
             var blobNames = new List<string>();
 
-            foreach (var triggerFile in triggerFiles)
+            foreach (var (triggerFileBlobUrl, workflowFriendlyName) in triggerFiles)
             {
-                var triggerFileJson = await (await httpClient.GetAsync(triggerFile.triggerFileBlobUrl)).Content.ReadAsStringAsync();
+                var triggerFileJson = await (await httpClient.GetAsync(triggerFileBlobUrl)).Content.ReadAsStringAsync();
 
                 // 2.  Start the workflows by uploading new trigger files
                 var date = $"{startTime.Year}-{startTime.Month}-{startTime.Day}-{startTime.Hour}-{startTime.Minute}";
@@ -276,7 +273,7 @@ private async Task StartWorkflowsAsync(
                 for (var i = 1; i <= countOfEachWorkflowToRun; i++)
                 {
                     // example: new/mutect2-001-of-100-2023-4-7-3-9.json
-                    var blobName = $"new/{triggerFile.workflowFriendlyName}-{i:D4}-of-{countOfEachWorkflowToRun:D4}-{date}.json";
+                    var blobName = $"new/{workflowFriendlyName}-{i:D4}-of-{countOfEachWorkflowToRun:D4}-{date}.json";
                     blobNames.Add(blobName);
                     await workflowsContainer.GetBlobClient(blobName).UploadAsync(BinaryData.FromString(triggerFileJson), true);
                 }
@@ -289,7 +286,7 @@ private async Task StartWorkflowsAsync(
             }
         }
 
-        private async Task<List<string>> ListContainerBlobNamesAsync(BlobContainerClient container)
+        private static async Task<List<string>> ListContainerBlobNamesAsync(BlobContainerClient container)
         {
             var enumerator = container.GetBlobsAsync().GetAsyncEnumerator();
             var existingBlobNames = new List<string>();
@@ -304,12 +301,12 @@ private async Task<List<string>> ListContainerBlobNamesAsync(BlobContainerClient
             return existingBlobNames;
         }
 
-        private int CountWorkflowsByState(List<string> originalBlobNames, List<string> currentBlobNames, WorkflowState state)
+        private static int CountWorkflowsByState(List<string> originalBlobNames, List<string> currentBlobNames, WorkflowState state)
         {
-            return GetWorkflowsByState(originalBlobNames, currentBlobNames, state).Count();
+            return GetWorkflowsByState(originalBlobNames, currentBlobNames, state).Count;
         }
 
-        private List<string> GetWorkflowsByState(List<string> originalBlobNames, List<string> currentBlobNames, WorkflowState state)
+        private static List<string> GetWorkflowsByState(List<string> originalBlobNames, List<string> currentBlobNames, WorkflowState state)
         {
             var stateString = state.ToString().ToLowerInvariant();
 
@@ -341,10 +338,10 @@ public void GetWorkflowsByStateTest()
             Assert.IsTrue(succeeded.Single() == currentBlobNames.Skip(1).First());
         }
 
-        private async Task WaitTilAllWorkflowsInTerminalStateAsync(BlobContainerClient container, List<string> originalBlobNames, DateTime startTime)
+        private static async Task WaitTilAllWorkflowsInTerminalStateAsync(BlobContainerClient container, List<string> originalBlobNames, DateTime startTime)
         {
-            int succeededCount = 0;
-            int failedCount = 0;
+            var succeededCount = 0;
+            var failedCount = 0;
 
             var sw = Stopwatch.StartNew();
 

src/TriggerService.Tests/ProcessAbortRequestTests.cs

@@ -95,7 +95,7 @@ public async Task FailedAbortRequestFileGetsMovedToFailedSubdirectory()
                 ApplicationInsightsAccountName = "fakeappinsights"
             });
 
-            var postgreSqlOptions = new Mock<IOptions<PostgreSqlOptions>>().Object;
+            var postgreSqlOptions = new Mock<IOptions<Tes.Repository.Models.PostgreSqlOptions>>().Object;
             var storageUtility = new Mock<IAzureStorageUtility>();
 
             storageUtility

src/TriggerService.Tests/ProcessNewWorkflowTests.cs

@@ -16,6 +16,7 @@
 using Newtonsoft.Json;
 using Tes.Models;
 using Tes.Repository;
+using Tes.Repository.Models;
 
 namespace TriggerService.Tests
 {

src/TriggerService.Tests/TriggerEngineTests.cs

@@ -71,7 +71,7 @@ public async Task TriggerEngineRunsAndOnlyLogsAvailabilityOncePerSystemUponAvail
                 MainRunIntervalMilliseconds = 25
             });
 
-            var postgreSqlOptions = new Mock<IOptions<PostgreSqlOptions>>().Object;
+            var postgreSqlOptions = new Mock<IOptions<Tes.Repository.Models.PostgreSqlOptions>>().Object;
             var cromwellApiClient = new Mock<ICromwellApiClient>().Object;
             var tesTaskRepository = new Mock<IRepository<TesTask>>().Object;
             var azureStorage = new Mock<IAzureStorage>();

src/TriggerService.Tests/UpdateWorkflowStatusTests.cs

@@ -423,7 +423,7 @@ static string ShardString(int shard) =>
                 DefaultStorageAccountName = "fakestorage",
                 ApplicationInsightsAccountName = "fakeappinsights"
             });
-            var postgreSqlOptions = new Mock<IOptions<PostgreSqlOptions>>().Object;
+            var postgreSqlOptions = new Mock<IOptions<Tes.Repository.Models.PostgreSqlOptions>>().Object;
             var storageUtility = new Mock<IAzureStorageUtility>();
 
             storageUtility

src/TriggerService/TriggerService.cs

@@ -84,7 +84,7 @@ await Host.CreateDefaultBuilder()
                 {
                     serviceCollection.Configure<CromwellApiClientOptions>(hostBuilderContext.Configuration.GetSection(CromwellApiClientOptions.CromwellApiClientOptionsSectionName));
                     serviceCollection.Configure<TriggerServiceOptions>(hostBuilderContext.Configuration.GetSection(TriggerServiceOptions.TriggerServiceOptionsSectionName));
-                    serviceCollection.Configure<PostgreSqlOptions>(hostBuilderContext.Configuration.GetSection(PostgreSqlOptions.GetConfigurationSectionName("Tes")));
+                    serviceCollection.Configure<Tes.Repository.Models.PostgreSqlOptions>(hostBuilderContext.Configuration.GetSection(Tes.Repository.Models.PostgreSqlOptions.GetConfigurationSectionName("Tes")));
                     serviceCollection.AddSingleton(azureCloudConfig);
                     serviceCollection.AddSingleton<ICromwellApiClient, CromwellApiClient.CromwellApiClient>();
                     serviceCollection.AddSingleton<IRepository<TesTask>, TesTaskPostgreSqlRepository>();

src/TriggerService/TriggerService.csproj

@@ -39,7 +39,7 @@
     <ProjectReference Include="..\Common\Common.csproj" />
     <ProjectReference Include="..\CromwellApiClient\CromwellApiClient.csproj" />
     <ProjectReference Include="..\ga4gh-tes\src\CommonUtilities\CommonUtilities.csproj" />
-    <ProjectReference Include="..\ga4gh-tes\src\Tes\Tes.csproj" />
+    <ProjectReference Include="..\ga4gh-tes\src\Tes.Repository\Tes.Repository.csproj" />
   </ItemGroup>
 
 </Project>

src/deploy-cromwell-on-azure.Tests/deploy-cromwell-on-azure.Tests.csproj

@@ -3,6 +3,8 @@
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
     <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+    <ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>None</ResolveAssemblyWarnOrErrorOnTargetArchitectureMismatch>
   </PropertyGroup>
 
   <ItemGroup>

src/deploy-cromwell-on-azure/scripts/helm/values-template.yaml

@@ -17,7 +17,6 @@ config:
     contentMD5: "False"
   batchScheduling:
     usePreemptibleVmsOnly: RUNTIME_PARAMETER
-    useLegacyAutopools: "False"
     poolRotationForcedDays: "7"
     taskMaxWallClockTimeDays: "7"
     prefix: RUNTIME_PARAMETER