[ML] Adding space aware jobs (#77916)

* [ML] Adding space aware jobs * adding mlClient * switching to type includes * adding additional job checks * fixing conflict * adding dfa checks * refactoring jobs in spaces checks * filtering calendars * adding initial job object status and repair endpoints * enabling repair endpoint * fixing listed jobs in status * adding datafeed repair * updating shared services * adding results job id check * fixing conflicts * don't remove SO on delete * fixing non-ml plugins * filtering job audit messages * fixing types * fixing tests * adding job ids wildcard support * removing empty migration test * fixing tests and disabling spaces test user * adding saved objects all permission * fixing calendars * updating job 404 * updating job wildcard search * renaming services * fixing conflicts * fixing log tests * disabling apm test * skipping more apm tests * optimzing repair * fixing types * updating apm test archive to include ML saved objects * enabling disabled test * removing comment * adding space assigning endpoints * adding saved object default permissions * removing commented code * loading all jobs for all spaces for status check * adding spaces list endpoint * adding job spaces to management page * adding trained model fltering * fixing trained model id check and job wildcard check * fixing types * fixing bug when adding new job to calendar * changes based on review * updating schema * changes based on review * fixing types * rolling back http service injection * fixing http service injection * adding errrors to repair endpoint response * updating api doc * improving types * disabling id check on ad get endpoints * fixing tests * fixing group requests * adding comments * using filter in saved object search * fixing fake request issue * removing console log * making job saved object hidden * removing acccidentally included file * renaming saved object client * updating apidoc * unhiding ml saved objects * moving route guard * improving error when SOC is null * fixing types after merge with master * fixing tests Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Kibana Machine <[email protected]>
elastic · Nov 3, 2020 · a0fded5 · a0fded5
1 parent 2db4244
commit a0fded5
Show file tree

Hide file tree

Showing 133 changed files with 6,110 additions and 1,399 deletions.
diff --git a/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts b/x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts
@@ -73,8 +73,14 @@ export function registerTransactionDurationAnomalyAlertType({
       }
       const alertParams = params;
       const request = {} as KibanaRequest;
-      const { mlAnomalySearch } = ml.mlSystemProvider(request);
-      const anomalyDetectors = ml.anomalyDetectorsProvider(request);
+      const { mlAnomalySearch } = ml.mlSystemProvider(
+        request,
+        services.savedObjectsClient
+      );
+      const anomalyDetectors = ml.anomalyDetectorsProvider(
+        request,
+        services.savedObjectsClient
+      );
 
       const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment);
 
@@ -94,6 +100,7 @@ export function registerTransactionDurationAnomalyAlertType({
         return {};
       }
 
+      const jobIds = mlJobs.map((job) => job.job_id);
       const anomalySearchParams = {
         terminateAfter: 1,
         body: {
@@ -102,7 +109,7 @@ export function registerTransactionDurationAnomalyAlertType({
             bool: {
               filter: [
                 { term: { result_type: 'record' } },
-                { terms: { job_id: mlJobs.map((job) => job.job_id) } },
+                { terms: { job_id: jobIds } },
                 {
                   range: {
                     timestamp: {
@@ -163,7 +170,8 @@ export function registerTransactionDurationAnomalyAlertType({
       };
 
       const response = ((await mlAnomalySearch(
-        anomalySearchParams
+        anomalySearchParams,
+        jobIds
       )) as unknown) as {
         hits: { total: { value: number } };
         aggregations?: {

diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.ts
@@ -123,8 +123,8 @@ function getMlSetup(
   request: KibanaRequest
 ) {
   return {
-    mlSystem: ml.mlSystemProvider(request),
-    anomalyDetectors: ml.anomalyDetectorsProvider(request),
+    mlSystem: ml.mlSystemProvider(request, savedObjectsClient),
+    anomalyDetectors: ml.anomalyDetectorsProvider(request, savedObjectsClient),
     modules: ml.modulesProvider(request, savedObjectsClient),
   };
 }

diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts
@@ -104,7 +104,7 @@ export async function getServiceAnomalies({
     },
   };
 
-  const response = await ml.mlSystem.mlAnomalySearch(params);
+  const response = await ml.mlSystem.mlAnomalySearch(params, mlJobIds);
 
   return {
     mlJobIds,

diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts
@@ -81,7 +81,7 @@ export async function anomalySeriesFetcher({
     const response: ESSearchResponse<
       unknown,
       typeof params
-    > = (await ml.mlSystem.mlAnomalySearch(params)) as any;
+    > = (await ml.mlSystem.mlAnomalySearch(params, [jobId])) as any;
 
     return response;
   } catch (err) {

diff --git a/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts b/x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts
@@ -49,7 +49,7 @@ export async function getMlBucketSize({
   };
 
   try {
-    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params);
+    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params, [jobId]);
     return resp.hits.hits[0]?._source.bucket_span;
   } catch (err) {
     const isHttpError = 'statusCode' in err;

diff --git a/x-pack/plugins/infra/server/lib/infra_ml/common.ts b/x-pack/plugins/infra/server/lib/infra_ml/common.ts
@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

diff --git a/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts
@@ -177,7 +177,8 @@ async function fetchMetricsHostsAnomalies(
 
   const results = decodeOrThrow(metricsHostsAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

diff --git a/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts b/x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts
@@ -173,7 +173,8 @@ async function fetchMetricK8sAnomalies(
 
   const results = decodeOrThrow(metricsK8sAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

diff --git a/x-pack/plugins/infra/server/lib/log_analysis/common.ts b/x-pack/plugins/infra/server/lib/log_analysis/common.ts
@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts
@@ -223,7 +223,8 @@ async function fetchLogEntryAnomalies(
 
   const results = decodeOrThrow(logEntryAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets)
+      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets),
+      jobIds
     )
   );
 

diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_analysis.ts
@@ -226,7 +226,8 @@ async function fetchTopLogEntryCategories(
         endTime,
         categoryCount,
         datasets
-      )
+      ),
+      [logEntryCategoriesCountJobId]
     )
   );
 
@@ -284,7 +285,8 @@ export async function fetchLogEntryCategories(
 
   const logEntryCategoriesResponse = decodeOrThrow(logEntryCategoriesResponseRT)(
     await context.infra.mlSystem.mlAnomalySearch(
-      createLogEntryCategoriesQuery(logEntryCategoriesCountJobId, categoryIds)
+      createLogEntryCategoriesQuery(logEntryCategoriesCountJobId, categoryIds),
+      [logEntryCategoriesCountJobId]
     )
   );
 
@@ -333,7 +335,8 @@ async function fetchTopLogEntryCategoryHistograms(
             startTime,
             endTime,
             bucketCount
-          )
+          ),
+          [logEntryCategoriesCountJobId]
         )
         .then(decodeOrThrow(logEntryCategoryHistogramsResponseRT))
         .then((response) => ({

diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts
@@ -40,7 +40,8 @@ export async function getLatestLogEntriesCategoriesDatasetsStats(
         endTime,
         COMPOSITE_AGGREGATION_BATCH_SIZE,
         afterLatestBatchKey
-      )
+      ),
+      jobIds
     );
 
     const { after_key: afterKey, buckets: latestBatchBuckets = [] } =

diff --git a/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts b/x-pack/plugins/infra/server/lib/log_analysis/log_entry_rate_analysis.ts
@@ -43,7 +43,8 @@ export async function getLogEntryRateBuckets(
         COMPOSITE_AGGREGATION_BATCH_SIZE,
         afterLatestBatchKey,
         datasets
-      )
+      ),
+      [logRateJobId]
     );
 
     const { after_key: afterKey, buckets: latestBatchBuckets = [] } =

diff --git a/x-pack/plugins/infra/server/plugin.ts b/x-pack/plugins/infra/server/plugin.ts
@@ -149,8 +149,11 @@ export class InfraServerPlugin {
     core.http.registerRouteHandlerContext(
       'infra',
       (context, request): InfraRequestHandlerContext => {
-        const mlSystem = plugins.ml?.mlSystemProvider(request);
-        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(request);
+        const mlSystem = plugins.ml?.mlSystemProvider(request, context.core.savedObjects.client);
+        const mlAnomalyDetectors = plugins.ml?.anomalyDetectorsProvider(
+          request,
+          context.core.savedObjects.client
+        );
         const spaceId = plugins.spaces?.spacesService.getSpaceId(request) || 'default';
 
         return {

diff --git a/x-pack/plugins/ml/common/types/calendars.ts b/x-pack/plugins/ml/common/types/calendars.ts
@@ -11,6 +11,7 @@ export interface Calendar {
   description: string;
   events: any[];
   job_ids: string[];
+  total_job_count?: number;
 }
 
 export interface UpdateCalendar extends Calendar {

diff --git a/x-pack/plugins/ml/common/types/capabilities.ts b/x-pack/plugins/ml/common/types/capabilities.ts
@@ -78,7 +78,7 @@ export function getPluginPrivileges() {
   const adminMlCapabilitiesKeys = Object.keys(adminMlCapabilities);
   const allMlCapabilitiesKeys = [...adminMlCapabilitiesKeys, ...userMlCapabilitiesKeys];
   // TODO: include ML in base privileges for the `8.0` release: https://github.com/elastic/kibana/issues/71422
-  const savedObjects = ['index-pattern', 'dashboard', 'search', 'visualization'];
+  const savedObjects = ['index-pattern', 'dashboard', 'search', 'visualization', 'ml-job'];
   const privilege = {
     app: [PLUGIN_ID, 'kibana'],
     excludeFromBasePrivileges: true,
@@ -116,7 +116,7 @@ export function getPluginPrivileges() {
       catalogue: [],
       savedObject: {
         all: [],
-        read: [],
+        read: ['ml-job'],
       },
       api: apmUserMlCapabilitiesKeys.map((k) => `ml:${k}`),
       ui: apmUserMlCapabilitiesKeys,

diff --git a/x-pack/plugins/ml/common/types/saved_objects.ts b/x-pack/plugins/ml/common/types/saved_objects.ts
@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export type JobType = 'anomaly-detector' | 'data-frame-analytics';
diff --git a/...ck/plugins/ml/server/lib/license/index.ts → ...ation/components/job_spaces_list/index.ts b/...ck/plugins/ml/server/lib/license/index.ts → ...ation/components/job_spaces_list/index.ts
@@ -4,4 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export { MlServerLicense } from './ml_server_license';
+export { JobSpacesList } from './job_spaces_list';
diff --git a/x-pack/plugins/ml/public/application/components/job_spaces_list/job_spaces_list.tsx b/x-pack/plugins/ml/public/application/components/job_spaces_list/job_spaces_list.tsx
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC } from 'react';
+
+import { EuiFlexGroup, EuiFlexItem, EuiBadge } from '@elastic/eui';
+
+interface Props {
+  spaces: string[];
+}
+
+export const JobSpacesList: FC<Props> = ({ spaces }) => (
+  <EuiFlexGroup wrap responsive={false} gutterSize="xs">
+    {spaces.map((space) => (
+      <EuiFlexItem grow={false} key={space}>
+        <EuiBadge color={'hollow'}>{space}</EuiBadge>
+      </EuiFlexItem>
+    ))}
+  </EuiFlexGroup>
+);
diff --git a/...a_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx b/...a_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx
@@ -116,7 +116,8 @@ export const DataFrameAnalyticsList: FC<Props> = ({
     setAnalyticsStats,
     setErrorMessage,
     setIsInitialized,
-    blockRefresh
+    blockRefresh,
+    isManagementTable
   );
 
   const updateFilteredItems = (queryClauses: any) => {

diff --git a/...ation/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts b/...ation/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts
@@ -112,6 +112,7 @@ export interface DataFrameAnalyticsListRow {
   mode: string;
   state: DataFrameAnalyticsStats['state'];
   stats: DataFrameAnalyticsStats;
+  spaces?: string[];
 }
 
 // Used to pass on attribute names to table columns

diff --git a/...data_frame_analytics/pages/analytics_management/components/analytics_list/use_columns.tsx b/...data_frame_analytics/pages/analytics_management/components/analytics_list/use_columns.tsx
@@ -32,6 +32,7 @@ import {
 import { useActions } from './use_actions';
 import { useMlLink } from '../../../../../contexts/kibana';
 import { ML_PAGES } from '../../../../../../../common/constants/ml_url_generator';
+import { JobSpacesList } from '../../../../../components/job_spaces_list';
 
 enum TASK_STATE_COLOR {
   analyzing = 'primary',
@@ -278,7 +279,8 @@ export const useColumns = (
       name: i18n.translate('xpack.ml.jobsList.analyticsSpacesLabel', {
         defaultMessage: 'Spaces',
       }),
-      render: () => <EuiBadge color={'hollow'}>{'all'}</EuiBadge>,
+      render: (item: DataFrameAnalyticsListRow) =>
+        Array.isArray(item.spaces) ? <JobSpacesList spaces={item.spaces} /> : null,
       width: '75px',
     });
 

diff --git a/...ta_frame_analytics/pages/analytics_management/services/analytics_service/get_analytics.ts b/...ta_frame_analytics/pages/analytics_management/services/analytics_service/get_analytics.ts
@@ -106,7 +106,8 @@ export const getAnalyticsFactory = (
     React.SetStateAction<GetDataFrameAnalyticsStatsResponseError | undefined>
   >,
   setIsInitialized: React.Dispatch<React.SetStateAction<boolean>>,
-  blockRefresh: boolean
+  blockRefresh: boolean,
+  isManagementTable: boolean
 ): GetAnalytics => {
   let concurrentLoads = 0;
 
@@ -123,6 +124,12 @@ export const getAnalyticsFactory = (
         const analyticsConfigs = await ml.dataFrameAnalytics.getDataFrameAnalytics();
         const analyticsStats = await ml.dataFrameAnalytics.getDataFrameAnalyticsStats();
 
+        let spaces: { [id: string]: string[] } = {};
+        if (isManagementTable) {
+          const allSpaces = await ml.savedObjects.jobsSpaces();
+          spaces = allSpaces['data-frame-analytics'];
+        }
+
         const analyticsStatsResult = isGetDataFrameAnalyticsStatsResponseOk(analyticsStats)
           ? getAnalyticsJobsStats(analyticsStats)
           : undefined;
@@ -148,6 +155,7 @@ export const getAnalyticsFactory = (
               mode: DATA_FRAME_MODE.BATCH,
               state: stats.state,
               stats,
+              spaces: spaces[config.id] ?? [],
             });
             return reducedtableRows;
           },

diff --git a/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.js b/x-pack/plugins/ml/public/application/jobs/components/custom_url_editor/utils.js
@@ -296,9 +296,12 @@ export function getTestUrl(job, customUrl) {
 
   return new Promise((resolve, reject) => {
     ml.results
-      .anomalySearch({
-        body,
-      })
+      .anomalySearch(
+        {
+          body,
+        },
+        [job.job_id]
+      )
       .then((resp) => {
         if (resp.hits.total.value > 0) {
           const record = resp.hits.hits[0]._source;

diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js
@@ -40,7 +40,6 @@ export class JobDetailsUI extends Component {
   }
 
   render() {
-    console.log('this.props', this.props);
     const { job } = this.state;
     const {
       services: {

diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list/jobs_list.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list/jobs_list.js
@@ -14,9 +14,10 @@ import { toLocaleString } from '../../../../util/string_utils';
 import { ResultLinks, actionsMenuContent } from '../job_actions';
 import { JobDescription } from './job_description';
 import { JobIcon } from '../../../../components/job_message_icon';
+import { JobSpacesList } from '../../../../components/job_spaces_list';
 import { TIME_FORMAT } from '../../../../../../common/constants/time_format';
 
-import { EuiBadge, EuiBasicTable, EuiButtonIcon, EuiScreenReaderOnly } from '@elastic/eui';
+import { EuiBasicTable, EuiButtonIcon, EuiScreenReaderOnly } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
 import { AnomalyDetectionJobIdLink } from './job_id_link';
@@ -251,7 +252,7 @@ export class JobsList extends Component {
         name: i18n.translate('xpack.ml.jobsList.spacesLabel', {
           defaultMessage: 'Spaces',
         }),
-        render: () => <EuiBadge color={'hollow'}>{'all'}</EuiBadge>,
+        render: (item) => <JobSpacesList spaces={item.spaces} />,
       });
       // Remove actions if Ml not enabled in current space
       if (this.props.isMlEnabledInSpace === false) {
-Original file line number
+Diff line change
@@ Expand Up / @@ -62,7 +62,8 @@ export async function getLogEntryDatasets( @@
               endTime,
               COMPOSITE_AGGREGATION_BATCH_SIZE,
               afterLatestBatchKey
-            )
+            ),
+            jobIds
           )
         );
@@ Expand Down @@