Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Health check updates: Storage and webapp checks; monitoring page #1502

Merged
merged 6 commits into from
Dec 10, 2024
Merged

Health check updates: Storage and webapp checks; monitoring page #1502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
bfae7e3
Web app health check WIP
akhavich Dec 3, 2024
45ed45f
Merge remote-tracking branch 'origin/develop' into ak/webapp_healthcheck
akhavich Dec 4, 2024
83cdac3
Updated static class and cleaned up duplicate code
akhavich Dec 5, 2024
169a5e9
Updated storage and web app health checks
akhavich Dec 5, 2024
02fb6bc
Infrastructure health page code improvements
akhavich Dec 10, 2024
3a19573
Merge remote-tracking branch 'origin/develop' into ak/webapp_healthcheck
akhavich Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions Portal/src/Datahub.Core/Utils/TerraformVariableExtraction.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

using System.Text.Json;
using Datahub.Core.Model.Projects;
using Datahub.Shared;
using Datahub.Shared.Entities;

namespace Datahub.Core.Utils;
Expand Down Expand Up @@ -235,6 +236,29 @@ public static IList<string> ExtractEnvironmentVariableKeys(Project_Resources2 pr
"storage_account");
}

private static Project_Resources2? GetProjectResource(Datahub_Project? workspace, string terraformTemplateName)
{
var fullTemplateName = TerraformTemplate.GetTerraformServiceType(terraformTemplateName);
return workspace?.Resources?.FirstOrDefault(r => r.ResourceType == fullTemplateName);
}

/// <summary>
/// Checks if a specified resource has been requested for the given workspace.
/// </summary>
/// <param name="workspace">The Datahub workspace.</param>
/// <param name="terraformTemplateName">The template name for the resource. See <see cref="TerraformTemplate"/> for template names.</param>
/// <returns>True if the resource has been requested and not deleted</returns>
public static bool IsResourceRequested(Datahub_Project? workspace, string terraformTemplateName)
{
var resource = GetProjectResource(workspace, terraformTemplateName);
if (resource is null)
{
return false;
}

return TerraformStatus.CreatedOrInProcessOf(resource.Status);
}

/// <summary>
/// Extracts a string variable from the given JSON content based on the variable name.
/// </summary>
Expand Down
113 changes: 73 additions & 40 deletions Portal/src/Datahub.Infrastructure/Services/Helpers/HealthCheckHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using Azure.Storage.Queues;
using Datahub.Application.Configuration;
using Datahub.Application.Services;
using Datahub.Application.Services.WebApp;
using Datahub.Core.Model.Context;
using Datahub.Core.Model.Health;
using Datahub.Core.Utils;
Expand All @@ -15,6 +16,7 @@
using Datahub.Infrastructure.Services.Storage;
using Datahub.Shared.Clients;
using Datahub.Shared.Configuration;
using Datahub.Shared.Entities;
using MassTransit;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
Expand Down Expand Up @@ -51,6 +53,7 @@ public static class InfrastructureHealthCheckConstants

public class HealthCheckHelper(IDbContextFactory<DatahubProjectDBContext> dbContextFactory,
IProjectStorageConfigurationService projectStorageConfigurationService,
IWorkspaceWebAppManagementService workspaceWebAppManagementService,
IConfiguration configuration,
IHttpClientFactory httpClientFactory,
ILoggerFactory loggerFactory,
Expand Down Expand Up @@ -245,21 +248,63 @@ public async Task<IntermediateHealthCheckResult> CheckAzureStorageAccount(Infras
// Get the projects that match the request.Name
try
{
string accountName = projectStorageConfigurationService.GetProjectStorageAccountName(request.Name);
string accountKey = await projectStorageConfigurationService.GetProjectStorageAccountKey(request.Name);
await using var ctx = await dbContextFactory.CreateDbContextAsync();

var projectStorageManager = new AzureCloudStorageManager(accountName, accountKey);
var project = await ctx.Projects
.AsNoTracking()
.Include(p => p.Resources)
.FirstOrDefaultAsync(p => p.Project_Acronym_CD == request.Name);

if (projectStorageManager is null)
if (project == null)
{
status = InfrastructureHealthStatus.Degraded;
errors.Add("Unable to find the data container.");
status = InfrastructureHealthStatus.Unhealthy;
errors.Add("Unable to retrieve project.");
}
else
{
var isRequested = TerraformVariableExtraction.IsResourceRequested(project, TerraformTemplate.AzureStorageBlob);

if (!isRequested)
{
status = InfrastructureHealthStatus.Undefined;
}
else
{
string accountName = projectStorageConfigurationService.GetProjectStorageAccountName(request.Name);
string accountKey = await projectStorageConfigurationService.GetProjectStorageAccountKey(request.Name);

var projectStorageManager = new AzureCloudStorageManager(accountName, accountKey);

if (projectStorageManager is null)
{
status = InfrastructureHealthStatus.Unhealthy;
errors.Add("Unable to find the data container.");
}
else
{
var containers = await projectStorageManager.GetContainersAsync();
if (containers is null || containers.Count < 1)
{
errors.Add("Storage account appears to have no containers.");
status = InfrastructureHealthStatus.Degraded;
}
else
{
var metadata = await projectStorageManager.GetStorageMetadataAsync(containers[0]);
if (metadata is null)
{
errors.Add("Unable to get container metadata. There may be something wrong with the container.");
status = InfrastructureHealthStatus.Degraded;
}
}
}
}
}
}
catch (Exception ex)
{
status = InfrastructureHealthStatus.Unhealthy;
errors.Add("Unable to retrieve project. " + ex.GetType().ToString());
errors.Add("Error while verifying project storage. " + ex.GetType().ToString());
errors.Add($"Details: {ex.Message}");
}

Expand Down Expand Up @@ -573,54 +618,42 @@ public async Task<IntermediateHealthCheckResult> CheckWebApp(InfrastructureHealt
if (project == null)
{
errors.Add("Unable to retrieve project.");
status = InfrastructureHealthStatus.Create;
status = InfrastructureHealthStatus.Unhealthy;
}
else
{
// We check if the project has a web app resource. If not, we return a create status.
if (project.WebAppEnabled == null || project.WebAppEnabled == false)
var isRequested = TerraformVariableExtraction.IsResourceRequested(project, TerraformTemplate.AzureAppService);
var appServiceConfig = TerraformVariableExtraction.ExtractAppServiceConfiguration(project);

if (!isRequested)
{
status = InfrastructureHealthStatus.Create;
status = InfrastructureHealthStatus.Undefined;
}
else if (appServiceConfig is null)
{
errors.Add("Unable to retrieve App Service configuration from project resource.");
status = InfrastructureHealthStatus.Unhealthy;
}
else
{
string url = project.WebApp_URL;

// Validate if the URL is valid
if (!Uri.TryCreate(url, UriKind.Absolute, out var result))
var isProvisioned = !(string.IsNullOrEmpty(appServiceConfig.HostName) && string.IsNullOrEmpty(appServiceConfig.Id));
if (!isProvisioned)
{
status = InfrastructureHealthStatus.Unhealthy;
errors.Add("Invalid Web App URL.");
if (!string.IsNullOrEmpty(url) && !url.ToLower().StartsWith("http"))
{
url = "https://" + url; // add https if not present
}
errors.Add("App has not been provisioned - it may still be processing.");
status = InfrastructureHealthStatus.NeedHealthCheckRun;
}

try
else
{
// We attempt to connect to the URL. If we cannot, we return an unhealthy status.
using var httpClient = httpClientFactory.CreateClient();
var response = await httpClient.GetAsync(url);

if (!response.IsSuccessStatusCode)
var appIsRunning = await workspaceWebAppManagementService.GetState(appServiceConfig.Id);
if (!appIsRunning)
{
status = InfrastructureHealthStatus.Unhealthy;
errors.Add($"Web App returned an unhealthy status code: {response.StatusCode}. {response.ReasonPhrase}");
}
else
{
var content = await response.Content.ReadAsStringAsync();
errors.Add("App is provisioned but not running.");
status = InfrastructureHealthStatus.Degraded;
}
}
catch (Exception ex)
{
status = InfrastructureHealthStatus.Unhealthy;
errors.Add($"Error while checking Web App health: {ex.Message}");
}
}
}

return new(status, errors);
}

Expand Down
5 changes: 4 additions & 1 deletion Portal/src/Datahub.Infrastructure/Services/Helpers/LocalMessageReaderService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using Datahub.Application.Configuration;
using Datahub.Application.Services;
using Datahub.Application.Services.WebApp;
using Datahub.Core.Model.Context;
using Datahub.Core.Model.Health;
using Datahub.Infrastructure.Queues.Messages;
Expand Down Expand Up @@ -73,6 +74,7 @@ private async void OnCreated(object sender, FileSystemEventArgs e)
using (var scope = _serviceProvider.CreateScope())
{
var projectStorageConfigurationService = scope.ServiceProvider.GetRequiredService<ProjectStorageConfigurationService>();
var webAppManagementService = scope.ServiceProvider.GetRequiredService<IWorkspaceWebAppManagementService>();
var projectDBContext = scope.ServiceProvider.GetRequiredService<DatahubProjectDBContext>();
var portalConfiguration = scope.ServiceProvider.GetRequiredService<DatahubPortalConfiguration>();
var configuration = scope.ServiceProvider.GetRequiredService<IConfiguration>();
Expand All @@ -86,7 +88,8 @@ private async void OnCreated(object sender, FileSystemEventArgs e)
var dbContextFactory = scope.ServiceProvider.GetRequiredService<IDbContextFactory<DatahubProjectDBContext>>();
var sendEndpointProvider = scope.ServiceProvider.GetRequiredService<ISendEndpointProvider>();

var healthCheckHelper = new HealthCheckHelper(dbContextFactory, projectStorageConfigurationService, configuration,
// TODO: refactor this to make it more concise, and/or autowire it
var healthCheckHelper = new HealthCheckHelper(dbContextFactory, projectStorageConfigurationService, webAppManagementService, configuration,
httpClientFactory, _loggerFactory, sendEndpointProvider, resourceMessagingService, portalConfiguration);

// Deserialize the file contents into an InfrastructureHealthCheckMessage object
Expand Down
Loading
Loading