[ML] Split up data visualizer model, remove Logger

x-pack/plugins/data_visualizer/server/models/data_visualizer/check_fields_exist.ts

@@ -0,0 +1,183 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+import { get } from 'lodash';
+import { IScopedClusterClient } from 'kibana/server';
+import { AggCardinality, Aggs, FieldData } from '../../types';
+import {
+  buildBaseFilterCriteria,
+  buildSamplerAggregation,
+  getSafeAggregationName,
+  getSamplerAggregationsResponsePath,
+} from '../../../common/utils/query_utils';
+import { getDatafeedAggregations } from '../../../common/utils/datafeed_utils';
+import { isPopulatedObject } from '../../../common/utils/object_utils';
+
+export const checkAggregatableFieldsExist = async (
+  client: IScopedClusterClient,
+  indexPatternTitle: string,
+  query: any,
+  aggregatableFields: string[],
+  samplerShardSize: number,
+  timeFieldName: string | undefined,
+  earliestMs?: number,
+  latestMs?: number,
+  datafeedConfig?: estypes.Datafeed,
+  runtimeMappings?: estypes.RuntimeFields
+) => {
+  const { asCurrentUser } = client;
+
+  const index = indexPatternTitle;
+  const size = 0;
+  const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query);
+  const datafeedAggregations = getDatafeedAggregations(datafeedConfig);
+
+  // Value count aggregation faster way of checking if field exists than using
+  // filter aggregation with exists query.
+  const aggs: Aggs = datafeedAggregations !== undefined ? { ...datafeedAggregations } : {};
+
+  // Combine runtime fields from the index pattern as well as the datafeed
+  const combinedRuntimeMappings: estypes.RuntimeFields = {
+    ...(isPopulatedObject(runtimeMappings) ? runtimeMappings : {}),
+    ...(isPopulatedObject(datafeedConfig) && isPopulatedObject(datafeedConfig.runtime_mappings)
+      ? datafeedConfig.runtime_mappings
+      : {}),
+  };
+
+  aggregatableFields.forEach((field, i) => {
+    const safeFieldName = getSafeAggregationName(field, i);
+    aggs[`${safeFieldName}_count`] = {
+      filter: { exists: { field } },
+    };
+
+    let cardinalityField: AggCardinality;
+    if (datafeedConfig?.script_fields?.hasOwnProperty(field)) {
+      cardinalityField = aggs[`${safeFieldName}_cardinality`] = {
+        cardinality: { script: datafeedConfig?.script_fields[field].script },
+      };
+    } else {
+      cardinalityField = {
+        cardinality: { field },
+      };
+    }
+    aggs[`${safeFieldName}_cardinality`] = cardinalityField;
+  });
+
+  const searchBody = {
+    query: {
+      bool: {
+        filter: filterCriteria,
+      },
+    },
+    ...(isPopulatedObject(aggs) ? { aggs: buildSamplerAggregation(aggs, samplerShardSize) } : {}),
+    ...(isPopulatedObject(combinedRuntimeMappings)
+      ? { runtime_mappings: combinedRuntimeMappings }
+      : {}),
+  };
+
+  const { body } = await asCurrentUser.search({
+    index,
+    track_total_hits: true,
+    size,
+    body: searchBody,
+  });
+
+  const aggregations = body.aggregations;
+  // @ts-expect-error incorrect search response type
+  const totalCount = body.hits.total.value;
+  const stats = {
+    totalCount,
+    aggregatableExistsFields: [] as FieldData[],
+    aggregatableNotExistsFields: [] as FieldData[],
+  };
+
+  const aggsPath = getSamplerAggregationsResponsePath(samplerShardSize);
+  const sampleCount =
+    samplerShardSize > 0 ? get(aggregations, ['sample', 'doc_count'], 0) : totalCount;
+  aggregatableFields.forEach((field, i) => {
+    const safeFieldName = getSafeAggregationName(field, i);
+    const count = get(aggregations, [...aggsPath, `${safeFieldName}_count`, 'doc_count'], 0);
+    if (count > 0) {
+      const cardinality = get(
+        aggregations,
+        [...aggsPath, `${safeFieldName}_cardinality`, 'value'],
+        0
+      );
+      stats.aggregatableExistsFields.push({
+        fieldName: field,
+        existsInDocs: true,
+        stats: {
+          sampleCount,
+          count,
+          cardinality,
+        },
+      });
+    } else {
+      if (
+        datafeedConfig?.script_fields?.hasOwnProperty(field) ||
+        datafeedConfig?.runtime_mappings?.hasOwnProperty(field)
+      ) {
+        const cardinality = get(
+          aggregations,
+          [...aggsPath, `${safeFieldName}_cardinality`, 'value'],
+          0
+        );
+        stats.aggregatableExistsFields.push({
+          fieldName: field,
+          existsInDocs: true,
+          stats: {
+            sampleCount,
+            count,
+            cardinality,
+          },
+        });
+      } else {
+        stats.aggregatableNotExistsFields.push({
+          fieldName: field,
+          existsInDocs: false,
+        });
+      }
+    }
+  });
+
+  return stats;
+};
+
+export const checkNonAggregatableFieldExists = async (
+  client: IScopedClusterClient,
+  indexPatternTitle: string,
+  query: any,
+  field: string,
+  timeFieldName: string | undefined,
+  earliestMs: number | undefined,
+  latestMs: number | undefined,
+  runtimeMappings?: estypes.RuntimeFields
+) => {
+  const { asCurrentUser } = client;
+  const index = indexPatternTitle;
+  const size = 0;
+  const filterCriteria = buildBaseFilterCriteria(timeFieldName, earliestMs, latestMs, query);
+
+  const searchBody = {
+    query: {
+      bool: {
+        filter: filterCriteria,
+      },
+    },
+    ...(isPopulatedObject(runtimeMappings) ? { runtime_mappings: runtimeMappings } : {}),
+  };
+  filterCriteria.push({ exists: { field } });
+
+  const { body } = await asCurrentUser.search({
+    index,
+    size,
+    body: searchBody,
+  });
+  // @ts-expect-error incorrect search response type
+  return body.hits.total.value > 0;
+};

x-pack/plugins/file_upload/server/types.ts → x-pack/plugins/data_visualizer/server/models/data_visualizer/constants.ts

@@ -5,8 +5,9 @@
  * 2.0.
  */
 
-import { SecurityPluginStart } from '../..//security/server';
+export const SAMPLER_TOP_TERMS_THRESHOLD = 100000;
+export const SAMPLER_TOP_TERMS_SHARD_SIZE = 5000;
+export const AGGREGATABLE_EXISTS_REQUEST_BATCH_SIZE = 200;
+export const FIELDS_REQUEST_BATCH_SIZE = 10;
 
-export interface StartDeps {
-  security?: SecurityPluginStart;
-}
+export const MAX_CHART_COLUMNS = 20;