[Security Solution] Implement rule monitoring dashboard

x-pack/plugins/security_solution/common/detection_engine/rule_monitoring/api/urls.ts

@@ -28,6 +28,13 @@ export const GET_SPACE_HEALTH_URL = `${INTERNAL_URL}/health/_space` as const;
  */
 export const GET_RULE_HEALTH_URL = `${INTERNAL_URL}/health/_rule` as const;
 
+/**
+ * Similar to the "setup" command of beats, this endpoint installs resources
+ * (dashboards, data views, etc) related to rule monitoring and Detection Engine health,
+ * and can do any other setup work.
+ */
+export const SETUP_HEALTH_URL = `${INTERNAL_URL}/health/_setup` as const;
+
 // -------------------------------------------------------------------------------------------------
 // Rule execution logs API
 

x-pack/plugins/security_solution/public/app/home/index.tsx

@@ -25,8 +25,9 @@ import { TourContextProvider } from '../../common/components/guided_onboarding_t
 
 import { useUrlState } from '../../common/hooks/use_url_state';
 import { useUpdateBrowserTitle } from '../../common/hooks/use_update_browser_title';
-import { useUpgradeSecurityPackages } from '../../detection_engine/rule_management/logic/use_upgrade_security_packages';
 import { useUpdateExecutionContext } from '../../common/hooks/use_update_execution_context';
+import { useUpgradeSecurityPackages } from '../../detection_engine/rule_management/logic/use_upgrade_security_packages';
+import { useSetupDetectionEngineHealthApi } from '../../detection_engine/rule_monitoring';
 
 interface HomePageProps {
   children: React.ReactNode;
@@ -41,12 +42,14 @@ const HomePageComponent: React.FC<HomePageProps> = ({ children, setHeaderActionM
   useUpdateExecutionContext();
 
   const { browserFields } = useSourcererDataView(getScopeFromPath(pathname));
+
   // side effect: this will attempt to upgrade the endpoint package if it is not up to date
   // this will run when a user navigates to the Security Solution app and when they navigate between
   // tabs in the app. This is useful for keeping the endpoint package as up to date as possible until
   // a background task solution can be built on the server side. Once a background task solution is available we
   // can remove this.
   useUpgradeSecurityPackages();
+  useSetupDetectionEngineHealthApi();
 
   return (
     <SecuritySolutionAppWrapper id="security-solution-app" className="kbnAppWrapper">

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/__mocks__/api_client.ts

@@ -21,6 +21,8 @@ import type {
 } from '../api_client_interface';
 
 export const api: jest.Mocked<IRuleMonitoringApiClient> = {
+  setupDetectionEngineHealthApi: jest.fn<Promise<void>, []>().mockResolvedValue(),
+
   fetchRuleExecutionEvents: jest
     .fn<Promise<GetRuleExecutionEventsResponse>, [FetchRuleExecutionEventsArgs]>()
     .mockResolvedValue({

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.test.ts

@@ -27,6 +27,23 @@ describe('Rule Monitoring API Client', () => {
 
   const signal = new AbortController().signal;
 
+  describe('setupDetectionEngineHealthApi', () => {
+    const responseMock = {};
+
+    beforeEach(() => {
+      fetchMock.mockClear();
+      fetchMock.mockResolvedValue(responseMock);
+    });
+
+    it('calls API with correct parameters', async () => {
+      await api.setupDetectionEngineHealthApi();
+
+      expect(fetchMock).toHaveBeenCalledWith('/internal/detection_engine/health/_setup', {
+        method: 'POST',
+      });
+    });
+  });
+
   describe('fetchRuleExecutionEvents', () => {
     const responseMock: GetRuleExecutionEventsResponse = {
       events: [],

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client.ts

@@ -16,6 +16,7 @@ import type {
 import {
   getRuleExecutionEventsUrl,
   getRuleExecutionResultsUrl,
+  SETUP_HEALTH_URL,
 } from '../../../../common/detection_engine/rule_monitoring';
 
 import type {
@@ -25,6 +26,12 @@ import type {
 } from './api_client_interface';
 
 export const api: IRuleMonitoringApiClient = {
+  setupDetectionEngineHealthApi: async (): Promise<void> => {
+    await http().fetch(SETUP_HEALTH_URL, {
+      method: 'POST',
+    });
+  },
+
   fetchRuleExecutionEvents: (
     args: FetchRuleExecutionEventsArgs
   ): Promise<GetRuleExecutionEventsResponse> => {

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/api/api_client_interface.ts

@@ -16,6 +16,12 @@ import type {
 } from '../../../../common/detection_engine/rule_monitoring';
 
 export interface IRuleMonitoringApiClient {
+  /**
+   * Installs resources (dashboards, data views, etc) related to rule monitoring
+   * and Detection Engine health, and can do any other setup work.
+   */
+  setupDetectionEngineHealthApi(): Promise<void>;
+
   /**
    * Fetches plain rule execution events (status changes, metrics, generic events) from Event Log.
    * @throws An error if response is not OK.
@@ -33,7 +39,14 @@ export interface IRuleMonitoringApiClient {
   ): Promise<GetRuleExecutionResultsResponse>;
 }
 
-export interface FetchRuleExecutionEventsArgs {
+export interface RuleMonitoringApiCallArgs {
+  /**
+   * Optional signal for cancelling the request.
+   */
+  signal?: AbortSignal;
+}
+
+export interface FetchRuleExecutionEventsArgs extends RuleMonitoringApiCallArgs {
   /**
    * Saved Object ID of the rule (`rule.id`, not static `rule.rule_id`).
    */
@@ -63,14 +76,9 @@ export interface FetchRuleExecutionEventsArgs {
    * Number of results to fetch per page.
    */
   perPage?: number;
-
-  /**
-   * Optional signal for cancelling the request.
-   */
-  signal?: AbortSignal;
 }
 
-export interface FetchRuleExecutionResultsArgs {
+export interface FetchRuleExecutionResultsArgs extends RuleMonitoringApiCallArgs {
   /**
    * Saved Object ID of the rule (`rule.id`, not static `rule.rule_id`).
    */
@@ -116,9 +124,4 @@ export interface FetchRuleExecutionResultsArgs {
    * Number of results to fetch per page.
    */
   perPage?: number;
-
-  /**
-   * Optional signal for cancelling the request.
-   */
-  signal?: AbortSignal;
 }

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/index.ts

@@ -12,4 +12,5 @@ export * from './components/basic/indicators/execution_status_indicator';
 export * from './components/execution_events_table/execution_events_table';
 export * from './components/execution_results_table/use_execution_results';
 
+export * from './logic/detection_engine_health/use_setup_detection_engine_health_api';
 export * from './logic/execution_settings/use_execution_settings';

x-pack/plugins/security_solution/public/detection_engine/rule_monitoring/logic/detection_engine_health/use_setup_detection_engine_health_api.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { UseMutationOptions } from '@tanstack/react-query';
+import { useMutation } from '@tanstack/react-query';
+import { useEffect } from 'react';
+
+import { SETUP_HEALTH_URL } from '../../../../../common/detection_engine/rule_monitoring';
+import { api } from '../../api';
+
+export const SETUP_DETECTION_ENGINE_HEALTH_API_MUTATION_KEY = ['POST', SETUP_HEALTH_URL];
+
+export const useSetupDetectionEngineHealthApi = (options?: UseMutationOptions<void, Error>) => {
+  const { mutate: setupDetectionEngineHealthApi } = useMutation(
+    () => api.setupDetectionEngineHealthApi(),
+    {
+      ...options,
+      mutationKey: SETUP_DETECTION_ENGINE_HEALTH_API_MUTATION_KEY,
+    }
+  );
+
+  useEffect(() => {
+    setupDetectionEngineHealthApi();
+  }, [setupDetectionEngineHealthApi]);
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/detection_engine_health/setup/setup_health_route.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { transformError } from '@kbn/securitysolution-es-utils';
+import { buildSiemResponse } from '../../../../routes/utils';
+import type { SecuritySolutionPluginRouter } from '../../../../../../types';
+
+import { SETUP_HEALTH_URL } from '../../../../../../../common/detection_engine/rule_monitoring';
+
+/**
+ * Similar to the "setup" command of beats, this endpoint installs resources
+ * (dashboards, data views, etc) related to rule monitoring and Detection Engine health,
+ * and can do any other setup work.
+ */
+export const setupHealthRoute = (router: SecuritySolutionPluginRouter) => {
+  router.post(
+    {
+      path: SETUP_HEALTH_URL,
+      validate: {},
+      options: {
+        tags: ['access:securitySolution'],
+      },
+    },
+    async (context, request, response) => {
+      const siemResponse = buildSiemResponse(response);
+
+      try {
+        const ctx = await context.resolve(['securitySolution']);
+        const healthClient = ctx.securitySolution.getDetectionEngineHealthClient();
+
+        await healthClient.installAssetsForMonitoringHealth();
+
+        return response.ok({ body: {} });
+      } catch (err) {
+        const error = transformError(err);
+        return siemResponse.error({
+          body: error.message,
+          statusCode: error.statusCode,
+        });
+      }
+    }
+  );
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/api/register_routes.ts

@@ -9,6 +9,7 @@ import type { SecuritySolutionPluginRouter } from '../../../../types';
 import { getClusterHealthRoute } from './detection_engine_health/get_cluster_health/get_cluster_health_route';
 import { getRuleHealthRoute } from './detection_engine_health/get_rule_health/get_rule_health_route';
 import { getSpaceHealthRoute } from './detection_engine_health/get_space_health/get_space_health_route';
+import { setupHealthRoute } from './detection_engine_health/setup/setup_health_route';
 import { getRuleExecutionEventsRoute } from './rule_execution_logs/get_rule_execution_events/get_rule_execution_events_route';
 import { getRuleExecutionResultsRoute } from './rule_execution_logs/get_rule_execution_results/get_rule_execution_results_route';
 
@@ -17,6 +18,7 @@ export const registerRuleMonitoringRoutes = (router: SecuritySolutionPluginRoute
   getClusterHealthRoute(router);
   getSpaceHealthRoute(router);
   getRuleHealthRoute(router);
+  setupHealthRoute(router);
 
   // Rule execution logs API
   getRuleExecutionEventsRoute(router);

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/__mocks__/index.ts

@@ -16,6 +16,8 @@ import type { IDetectionEngineHealthClient } from '../detection_engine_health_cl
 type CalculateRuleHealth = IDetectionEngineHealthClient['calculateRuleHealth'];
 type CalculateSpaceHealth = IDetectionEngineHealthClient['calculateSpaceHealth'];
 type CalculateClusterHealth = IDetectionEngineHealthClient['calculateClusterHealth'];
+type InstallAssetsForMonitoringHealth =
+  IDetectionEngineHealthClient['installAssetsForMonitoringHealth'];
 
 export const detectionEngineHealthClientMock = {
   create: (): jest.Mocked<IDetectionEngineHealthClient> => ({
@@ -30,5 +32,12 @@ export const detectionEngineHealthClientMock = {
     calculateClusterHealth: jest
       .fn<ReturnType<CalculateClusterHealth>, Parameters<CalculateClusterHealth>>()
       .mockResolvedValue(clusterHealthSnapshotMock.getEmptyClusterHealthSnapshot()),
+
+    installAssetsForMonitoringHealth: jest
+      .fn<
+        ReturnType<InstallAssetsForMonitoringHealth>,
+        Parameters<InstallAssetsForMonitoringHealth>
+      >()
+      .mockResolvedValue(),
   }),
 };

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/assets/data_view_kibana_event_log.json

@@ -0,0 +1,14 @@
+{
+  "type": "index-pattern",
+  "id": "kibana-event-log-data-view",
+  "managed": true,
+  "coreMigrationVersion": "8.8.0",
+  "typeMigrationVersion": "8.0.0",
+  "attributes": {
+    "name": ".kibana-event-log-*",
+    "title": ".kibana-event-log-*",
+    "timeFieldName": "@timestamp",
+    "allowNoIndex": true
+  },
+  "references": []
+}

x-pack/plugins/security_solution/server/lib/detection_engine/rule_monitoring/logic/detection_engine_health/assets/install_assets_for_rule_monitoring.ts

@@ -0,0 +1,106 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { ISavedObjectsImporter, Logger } from '@kbn/core/server';
+import { SavedObjectsUtils } from '@kbn/core/server';
+import { cloneDeep } from 'lodash';
+import pRetry from 'p-retry';
+import { Readable } from 'stream';
+
+import sourceRuleMonitoringDashboard from './dashboard_rule_monitoring.json';
+import sourceKibanaEventLogDataView from './data_view_kibana_event_log.json';
+import sourceManagedTag from './tag_managed.json';
+import sourceSecuritySolutionTag from './tag_security_solution.json';
+
+const MAX_RETRIES = 2;
+
+/**
+ * Installs managed assets for monitoring rules and health of Detection Engine.
+ */
+export const installAssetsForRuleMonitoring = async (
+  savedObjectsImporter: ISavedObjectsImporter,
+  logger: Logger,
+  currentSpaceId: string
+): Promise<void> => {
+  const operation = async (attemptCount: number) => {
+    logger.debug(`Installing assets for rule monitoring (attempt ${attemptCount})...`);
+
+    const assets = getAssetsForRuleMonitoring(currentSpaceId);
+
+    // The assets are marked as "managed: true" at the saved object level, which in the future
+    // should be reflected in the UI for the user. Ticket to track:
+    // https://github.com/elastic/kibana/issues/140364
+    const importResult = await savedObjectsImporter.import({
+      readStream: Readable.from(assets),
+      managed: true,
+      overwrite: true,
+      createNewCopies: false,
+      refresh: false,
+      namespace: spaceIdToNamespace(currentSpaceId),
+    });
+
+    importResult.warnings.forEach((w) => {
+      logger.warn(w.message);
+    });
+
+    if (!importResult.success) {
+      const errors = (importResult.errors ?? []).map(
+        (e) => `Couldn't import "${e.type}:${e.id}": ${JSON.stringify(e.error)}`
+      );
+
+      errors.forEach((e) => {
+        logger.error(e);
+      });
+
+      // This will retry the operation
+      throw new Error(errors.length > 0 ? errors[0] : `Unknown error (attempt ${attemptCount})`);
+    }
+
+    logger.debug('Assets for rule monitoring installed');
+  };
+
+  await pRetry(operation, { retries: MAX_RETRIES });
+};
+
+const getAssetsForRuleMonitoring = (currentSpaceId: string) => {
+  const withSpaceId = appendSpaceId(currentSpaceId);
+
+  const assetRuleMonitoringDashboard = cloneDeep(sourceRuleMonitoringDashboard);
+  const assetKibanaEventLogDataView = cloneDeep(sourceKibanaEventLogDataView);
+  const assetManagedTag = cloneDeep(sourceManagedTag);
+  const assetSecuritySolutionTag = cloneDeep(sourceSecuritySolutionTag);
+
+  // Update ids of the assets to include the current space id
+  assetRuleMonitoringDashboard.id = withSpaceId('security-detection-rule-monitoring');
+  assetManagedTag.id = withSpaceId('fleet-managed');
+  assetSecuritySolutionTag.id = withSpaceId('security-solution');
+
+  // Update saved object references of the dashboard accordingly
+  assetRuleMonitoringDashboard.references = assetRuleMonitoringDashboard.references.map(
+    (reference) => {
+      if (reference.id === 'fleet-managed-<spaceId>') {
+        return { ...reference, id: assetManagedTag.id };
+      }
+      if (reference.id === 'security-solution-<spaceId>') {
+        return { ...reference, id: assetSecuritySolutionTag.id };
+      }
+
+      return reference;
+    }
+  );
+
+  return [
+    assetManagedTag,
+    assetSecuritySolutionTag,
+    assetKibanaEventLogDataView,
+    assetRuleMonitoringDashboard,
+  ];
+};
+
+const appendSpaceId = (spaceId: string) => (str: string) => `${str}-${spaceId}`;
+
+const spaceIdToNamespace = SavedObjectsUtils.namespaceStringToId;