From 837ef26fb0cced40214b25f0f1f22a8a0d610fb2 Mon Sep 17 00:00:00 2001
From: Patrick Mueller <patrick.mueller@elastic.co>
Date: Tue, 6 Jun 2023 08:42:40 -0400
Subject: [PATCH] [TaskManager] log health on interval with background_tasks
 only role (#158890)

resolves https://github.com/elastic/kibana/issues/158870

## Summary

For Kibana servers that only have node role `background_tasks`, log the
task manager health report to the Kibana logs on an interval, currently
every hour.

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
---
 .../server/lib/log_health_metrics.ts          | 24 +++++++++++++++++++
 x-pack/plugins/task_manager/server/plugin.ts  | 17 ++++++++++++-
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugins/task_manager/server/lib/log_health_metrics.ts b/x-pack/plugins/task_manager/server/lib/log_health_metrics.ts
index bddafb802b41b..f639184624c66 100644
--- a/x-pack/plugins/task_manager/server/lib/log_health_metrics.ts
+++ b/x-pack/plugins/task_manager/server/lib/log_health_metrics.ts
@@ -6,6 +6,7 @@
  */
 
 import { isEmpty } from 'lodash';
+import { Observable } from 'rxjs';
 import { Logger, DocLinksServiceSetup } from '@kbn/core/server';
 import { HealthStatus } from '../monitoring';
 import { TaskManagerConfig } from '../config';
@@ -23,6 +24,29 @@ let lastLogLevel: LogLevel | null = null;
 export function resetLastLogLevel() {
   lastLogLevel = null;
 }
+
+export function setupIntervalLogging(
+  monitoredHealth$: Observable<MonitoredHealth>,
+  logger: Logger,
+  minutes: number
+) {
+  let monitoredHealth: MonitoredHealth | undefined;
+  monitoredHealth$.subscribe((m) => {
+    monitoredHealth = m;
+  });
+
+  setInterval(onInterval, 1000 * 60 * minutes);
+
+  function onInterval() {
+    const meta = { tags: ['task-manager-background-node-health'] };
+    if (!monitoredHealth) {
+      return logger.warn('unable to log health metrics, not initialized yet', meta);
+    }
+
+    logger.info(`background node health: ${JSON.stringify(monitoredHealth)}`, meta);
+  }
+}
+
 export function logHealthMetrics(
   monitoredHealth: MonitoredHealth,
   logger: Logger,
diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts
index fe6b6b4120ff0..64da6d6f84f95 100644
--- a/x-pack/plugins/task_manager/server/plugin.ts
+++ b/x-pack/plugins/task_manager/server/plugin.ts
@@ -34,6 +34,7 @@ import { EphemeralTask, ConcreteTaskInstance } from './task';
 import { registerTaskManagerUsageCollector } from './usage';
 import { TASK_MANAGER_INDEX } from './constants';
 import { AdHocTaskCounter } from './lib/adhoc_task_counter';
+import { setupIntervalLogging } from './lib/log_health_metrics';
 
 export interface TaskManagerSetupContract {
   /**
@@ -66,6 +67,8 @@ export type TaskManagerStartContract = Pick<
     getRegisteredTypes: () => string[];
   };
 
+const LogHealthForBackgroundTasksOnlyMinutes = 60;
+
 export class TaskManagerPlugin
   implements Plugin<TaskManagerSetupContract, TaskManagerStartContract>
 {
@@ -82,6 +85,7 @@ export class TaskManagerPlugin
   private shouldRunBackgroundTasks: boolean;
   private readonly kibanaVersion: PluginInitializerContext['env']['packageInfo']['version'];
   private adHocTaskCounter: AdHocTaskCounter;
+  private nodeRoles: PluginInitializerContext['node']['roles'];
 
   constructor(private readonly initContext: PluginInitializerContext) {
     this.initContext = initContext;
@@ -89,10 +93,16 @@ export class TaskManagerPlugin
     this.config = initContext.config.get<TaskManagerConfig>();
     this.definitions = new TaskTypeDictionary(this.logger);
     this.kibanaVersion = initContext.env.packageInfo.version;
-    this.shouldRunBackgroundTasks = initContext.node.roles.backgroundTasks;
+    this.nodeRoles = initContext.node.roles;
+    this.shouldRunBackgroundTasks = this.nodeRoles.backgroundTasks;
     this.adHocTaskCounter = new AdHocTaskCounter();
   }
 
+  isNodeBackgroundTasksOnly() {
+    const { backgroundTasks, migrator, ui } = this.nodeRoles;
+    return backgroundTasks && !migrator && !ui;
+  }
+
   public setup(
     core: CoreSetup,
     plugins: { usageCollection?: UsageCollectionSetup }
@@ -184,6 +194,11 @@ export class TaskManagerPlugin
       this.logger.warn(`Disabling authentication for background task utilization API`);
     }
 
+    // for nodes with background_tasks mode only, log health metrics every hour
+    if (this.isNodeBackgroundTasksOnly()) {
+      setupIntervalLogging(monitoredHealth$, this.logger, LogHealthForBackgroundTasksOnlyMinutes);
+    }
+
     return {
       index: TASK_MANAGER_INDEX,
       addMiddleware: (middleware: Middleware) => {