From d89ff24495df701ad707cc88bbf7fb88d0078f89 Mon Sep 17 00:00:00 2001 From: davidreneuw Date: Mon, 9 Dec 2024 14:39:10 -0500 Subject: [PATCH 1/4] Fix variable extraction to always run --- .../Services/RepositoryService.cs | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs index cf5fa12d7..86017368a 100644 --- a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs +++ b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs @@ -530,17 +530,19 @@ public async Task ExecuteResourceRun(TerraformTemplate te { await terraformService.CopyTemplateAsync(template.Name, terraformWorkspace); await terraformService.ExtractVariables(template.Name, terraformWorkspace); - switch (template.Name) - { - case TerraformTemplate.NewProjectTemplate: - await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); - break; - case TerraformTemplate.VariableUpdate: - await terraformService.ExtractAllVariables(terraformWorkspace); - break; - } - await CommitTerraformTemplate(template, requestingUsername); - } + } + + switch (template.Name) + { + case TerraformTemplate.NewProjectTemplate: + await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); + break; + case TerraformTemplate.VariableUpdate: + await terraformService.ExtractAllVariables(terraformWorkspace); + break; + } + + await CommitTerraformTemplate(template, requestingUsername); return new RepositoryUpdateEvent() { From 107a3f6010ac9e109dc0b8818e58ebfa4a579fdd Mon Sep 17 00:00:00 2001 From: davidreneuw Date: Mon, 9 Dec 2024 16:34:34 -0500 Subject: [PATCH 2/4] Change it to extract variables on any status, not just create requested --- .../Services/RepositoryService.cs | 91 +++++++++++-------- ResourceProvisioner/template.settings.json | 1 - 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs index 86017368a..95799d501 100644 --- a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs +++ b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs @@ -46,7 +46,7 @@ public async Task HandleResourcing(CreateResourceRunCo { DirectoryUtils.tempDirectory = Guid.NewGuid().ToString().Substring(0, 8); CreateTemporaryDirectory(); - + var user = command.RequestingUserEmail ?? throw new NullReferenceException("Requesting user's email is null"); logger.LogInformation("Checking out workspace branch for {WorkspaceAcronym}", command.Workspace.Acronym); @@ -69,7 +69,7 @@ public async Task HandleResourcing(CreateResourceRunCo logger.LogInformation("Completing pull request for {WorkspaceAcronym}", command.Workspace.Acronym); await AutoApproveInfrastructurePullRequest(pullRequestValueObject.PullRequestId, command.Workspace.Acronym!); - + var pullRequestMessage = new PullRequestUpdateMessage { @@ -126,7 +126,7 @@ public async Task> GetModuleVersions() .ToList(); return versions; } - + private void CreateTemporaryDirectory() { CleanUpEnvironment(); @@ -147,26 +147,25 @@ public void FetchModuleRepository() var repositoryPath = DirectoryUtils.GetModuleRepositoryPath(resourceProvisionerConfiguration); DirectoryUtils.VerifyDirectoryDoesNotExist(repositoryPath); - logger.LogInformation("Cloning repository {RepositoryUrl} to {LocalPath}", repositoryUrl, repositoryPath); - Repository.Clone(repositoryUrl, repositoryPath); + logger.LogInformation("Cloning repository {RepositoryUrl} to {LocalPath}", repositoryUrl, repositoryPath); + Repository.Clone(repositoryUrl, repositoryPath); - if (resourceProvisionerConfiguration.ModuleRepository.Branch != ModuleRepositoryConfiguration.DefaultBranch) - { - using var repo = new Repository(repositoryPath); - var branch = - repo.Branches[$"refs/remotes/origin/{resourceProvisionerConfiguration.ModuleRepository.Branch}"]; - if (branch == null) + if (resourceProvisionerConfiguration.ModuleRepository.Branch != ModuleRepositoryConfiguration.DefaultBranch) { - logger.LogInformation("Branch {Branch} does not exist, checking out default branch", - resourceProvisionerConfiguration.ModuleRepository.Branch); - branch = repo.Branches[ModuleRepositoryConfiguration.DefaultBranch]; - } + using var repo = new Repository(repositoryPath); + var branch = + repo.Branches[$"refs/remotes/origin/{resourceProvisionerConfiguration.ModuleRepository.Branch}"]; + if (branch == null) + { + logger.LogInformation("Branch {Branch} does not exist, checking out default branch", + resourceProvisionerConfiguration.ModuleRepository.Branch); + branch = repo.Branches[ModuleRepositoryConfiguration.DefaultBranch]; + } Commands.Checkout(repo, branch); } logger.LogInformation("Repository {RepositoryUrl} cloned to {LocalPath}", repositoryUrl, repositoryPath); - } catch (Exception e) { @@ -204,10 +203,12 @@ public async Task FetchInfrastructureRepository() } }; - logger.LogInformation("Cloning repository {RepositoryUrl} to {LocalPath}", repositoryUrl, Path.GetFullPath(repositoryPath)); + logger.LogInformation("Cloning repository {RepositoryUrl} to {LocalPath}", repositoryUrl, + Path.GetFullPath(repositoryPath)); Repository.Clone(repositoryUrl, repositoryPath, cloneOptions); - logger.LogInformation("Repository {RepositoryUrl} cloned to {LocalPath}", repositoryUrl, Path.GetFullPath(repositoryPath)); + logger.LogInformation("Repository {RepositoryUrl} cloned to {LocalPath}", repositoryUrl, + Path.GetFullPath(repositoryPath)); } public async Task CheckoutInfrastructureBranch(string workspaceName) @@ -472,10 +473,10 @@ public async Task> ExecuteResourceRuns(List x.Name != TerraformTemplate.NewProjectTemplate).ToList(); - + foreach (var module in modules) { var result = await ExecuteResourceRun(module, terraformWorkspace, requestingUsername, resourcegroup); @@ -507,12 +508,6 @@ public async Task ExecuteResourceRun(TerraformTemplate te { try { - var templateStatusToIgnore = new List - { - TerraformStatus.Deleted, - TerraformStatus.DeleteInProgress - }; - if (template.Status == TerraformStatus.DeleteRequested) { if (template.Name == TerraformTemplate.NewProjectTemplate) @@ -523,27 +518,41 @@ public async Task ExecuteResourceRun(TerraformTemplate te { await terraformService.DeleteTemplateAsync(template.Name, terraformWorkspace); } - + await CommitTerraformTemplate(template, requestingUsername); } else if (template.Status == TerraformStatus.CreateRequested) { await terraformService.CopyTemplateAsync(template.Name, terraformWorkspace); await terraformService.ExtractVariables(template.Name, terraformWorkspace); + switch (template.Name) + { + case TerraformTemplate.NewProjectTemplate: + await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); + break; + case TerraformTemplate.VariableUpdate: + await terraformService.ExtractAllVariables(terraformWorkspace); + break; + } + + await CommitTerraformTemplate(template, requestingUsername); } - - switch (template.Name) + else { - case TerraformTemplate.NewProjectTemplate: - await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); - break; - case TerraformTemplate.VariableUpdate: - await terraformService.ExtractAllVariables(terraformWorkspace); - break; + await terraformService.ExtractVariables(template.Name, terraformWorkspace); + switch (template.Name) + { + case TerraformTemplate.NewProjectTemplate: + await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); + break; + case TerraformTemplate.VariableUpdate: + await terraformService.ExtractAllVariables(terraformWorkspace); + break; + } + + await CommitTerraformTemplate(template, requestingUsername); } - await CommitTerraformTemplate(template, requestingUsername); - return new RepositoryUpdateEvent() { Message = @@ -594,16 +603,18 @@ private async Task GetExistingPullRequestId(string workspaceAcronym) throw new NullReferenceException( $"Could not get existing pull request id for workspace {workspaceAcronym}"); } - + private void CleanUpEnvironment() { try { - logger.LogInformation("Deleting temporary directory {Directory} for resource run", DirectoryUtils.tempDirectory); + logger.LogInformation("Deleting temporary directory {Directory} for resource run", + DirectoryUtils.tempDirectory); var tempPath = DirectoryUtils.GetTempDirectoryPath(resourceProvisionerConfiguration); var dir = new DirectoryInfo(tempPath); DirectoryUtils.NormalizeAndDelete(dir); - } catch (Exception e) + } + catch (Exception e) { logger.LogError(e, "Error while cleaning up environment"); } diff --git a/ResourceProvisioner/template.settings.json b/ResourceProvisioner/template.settings.json index 6cec27ca2..824008eb3 100644 --- a/ResourceProvisioner/template.settings.json +++ b/ResourceProvisioner/template.settings.json @@ -39,7 +39,6 @@ "azure_databricks_enterprise_oid": "@Microsoft.KeyVault(VaultName=fsdh-key-$Environment;SecretName=datahub-databricks-sp)", "environment_classification": "U", "environment_name": "$Environment", - "budget_amount": "400", "aad_admin_group_oid": "@Microsoft.KeyVault(VaultName=fsdh-key-$Environment;SecretName=aad-admin-group-oid)", "storage_size_limit_tb": "5", "az_location": "canadacentral", From 186e1eaa5494c3de64b323b3f594e90b8146102a Mon Sep 17 00:00:00 2001 From: davidreneuw Date: Mon, 9 Dec 2024 16:48:12 -0500 Subject: [PATCH 3/4] Extract redundant code --- .../Services/RepositoryService.cs | 43 ++++++++----------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs index 95799d501..e4e82196e 100644 --- a/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs +++ b/ResourceProvisioner/src/ResourceProvisioner.Infrastructure/Services/RepositoryService.cs @@ -518,41 +518,19 @@ public async Task ExecuteResourceRun(TerraformTemplate te { await terraformService.DeleteTemplateAsync(template.Name, terraformWorkspace); } - - await CommitTerraformTemplate(template, requestingUsername); } else if (template.Status == TerraformStatus.CreateRequested) { await terraformService.CopyTemplateAsync(template.Name, terraformWorkspace); - await terraformService.ExtractVariables(template.Name, terraformWorkspace); - switch (template.Name) - { - case TerraformTemplate.NewProjectTemplate: - await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); - break; - case TerraformTemplate.VariableUpdate: - await terraformService.ExtractAllVariables(terraformWorkspace); - break; - } - - await CommitTerraformTemplate(template, requestingUsername); + await ExtractVariables(template, terraformWorkspace); } else { - await terraformService.ExtractVariables(template.Name, terraformWorkspace); - switch (template.Name) - { - case TerraformTemplate.NewProjectTemplate: - await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); - break; - case TerraformTemplate.VariableUpdate: - await terraformService.ExtractAllVariables(terraformWorkspace); - break; - } - - await CommitTerraformTemplate(template, requestingUsername); + await ExtractVariables(template, terraformWorkspace); } + await CommitTerraformTemplate(template, requestingUsername); + return new RepositoryUpdateEvent() { Message = @@ -584,6 +562,19 @@ public async Task ExecuteResourceRun(TerraformTemplate te } } + private async Task ExtractVariables(TerraformTemplate template, TerraformWorkspace terraformWorkspace) + { + await terraformService.ExtractVariables(template.Name, terraformWorkspace); + switch (template.Name) + { + case TerraformTemplate.NewProjectTemplate: + await terraformService.ExtractBackendConfig(terraformWorkspace.Acronym!); + break; + case TerraformTemplate.VariableUpdate: + await terraformService.ExtractAllVariables(terraformWorkspace); + break; + } + } private async Task GetExistingPullRequestId(string workspaceAcronym) { From 5fb4ee3547c07c2c2c2ce67d0a98eae8e0db8a3d Mon Sep 17 00:00:00 2001 From: davidreneuw Date: Mon, 9 Dec 2024 17:10:27 -0500 Subject: [PATCH 4/4] Revert config change --- ResourceProvisioner/template.settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/ResourceProvisioner/template.settings.json b/ResourceProvisioner/template.settings.json index 824008eb3..6cec27ca2 100644 --- a/ResourceProvisioner/template.settings.json +++ b/ResourceProvisioner/template.settings.json @@ -39,6 +39,7 @@ "azure_databricks_enterprise_oid": "@Microsoft.KeyVault(VaultName=fsdh-key-$Environment;SecretName=datahub-databricks-sp)", "environment_classification": "U", "environment_name": "$Environment", + "budget_amount": "400", "aad_admin_group_oid": "@Microsoft.KeyVault(VaultName=fsdh-key-$Environment;SecretName=aad-admin-group-oid)", "storage_size_limit_tb": "5", "az_location": "canadacentral",