[Security Solution][Detection Engine] move lists to data stream (#162508

) ## Summary - addresses elastic/security-team#7198 - moves list/items indices to data stream - adds `@timestamp` mapping to indices mappings - migrate to data stream if indices already exist(for customers < 8.11) or create data stream(for customers 8.11+ or serverless) - adds [DLM](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-streams-put-lifecycle.html) to index templates - replaces update/delete queries with update_by_query/delete_by_query which supported in data streams - fixes existing issues with update/patch APIs for lists/items - update/patch for lists didn't save `version` parameter in ES - update and patch APIs for lists/items were identical, i.e. for both routes was called the same `update` method w/o any changes <details> <summary>Technical detail on moving API to (update/delete)_by_query</summary> `update_by_query`, `delete_by_query` do not support refresh=wait_for, [only false/true values](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-update-by-query.html#_refreshing_shards_2). Which might break some of the use cases on UI(when list is removed, we refetch all lists. Deleted list will be returned for some time. [Default refresh time is 1s](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-refresh.html)). So, we retry refetching deleted/updated document before finishing request, to return reindexed document `update_by_query` does not support OCC [as update API](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/optimistic-concurrency-control.html). Which is supported in both [list](https://www.elastic.co/guide/en/security/current/lists-api-update-container.html)/[list item ](https://www.elastic.co/guide/en/security/current/lists-api-update-item.html)updates through _version parameter. _version is base64 encoded "_seq_no", "_primary_term" props used for OCC So, to keep it without breaking changes: implemented check for version conflict within update method </details> ### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Aug 23, 2023 · 505d826 · 505d826
1 parent 154ca40
commit 505d826
Show file tree

Hide file tree

Showing 104 changed files with 2,607 additions and 767 deletions.
diff --git a/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx b/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx
@@ -211,6 +211,7 @@ describe('AutocompleteFieldListsComponent', () => {
 
     await waitFor(() => {
       expect(mockOnChange).toHaveBeenCalledWith({
+        '@timestamp': DATE_NOW,
         _version: undefined,
         created_at: DATE_NOW,
         created_by: 'some user',

diff --git a/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts b/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts
@@ -42,6 +42,7 @@ export const NAME = 'some name';
 // TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715
 // import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock';
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts
@@ -8,8 +8,10 @@
 
 export * from './src/bad_request_error';
 export * from './src/create_boostrap_index';
+export * from './src/create_data_stream';
 export * from './src/decode_version';
 export * from './src/delete_all_index';
+export * from './src/delete_data_stream';
 export * from './src/delete_index_template';
 export * from './src/delete_policy';
 export * from './src/delete_template';
@@ -18,11 +20,15 @@ export * from './src/get_bootstrap_index_exists';
 export * from './src/get_index_aliases';
 export * from './src/get_index_count';
 export * from './src/get_index_exists';
+export * from './src/get_data_stream_exists';
 export * from './src/get_index_template_exists';
 export * from './src/get_policy_exists';
 export * from './src/get_template_exists';
+export * from './src/migrate_to_data_stream';
 export * from './src/read_index';
 export * from './src/read_privileges';
+export * from './src/put_mappings';
+export * from './src/remove_policy_from_index';
 export * from './src/set_index_template';
 export * from './src/set_policy';
 export * from './src/set_template';

diff --git a/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * creates data stream
+ * @param esClient
+ * @param name
+ */
+export const createDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return esClient.indices.createDataStream({
+    name,
+  });
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts
@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * deletes data stream
+ * @param esClient
+ * @param name
+ */
+export const deleteDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<boolean> => {
+  return (
+    await esClient.indices.deleteDataStream(
+      {
+        name,
+      },
+      { meta: true }
+    )
+  ).body.acknowledged;
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts
@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * checks if data stream exists
+ * @param esClient
+ * @param name
+ */
+export const getDataStreamExists = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<boolean> => {
+  try {
+    const body = await esClient.indices.getDataStream({ name, expand_wildcards: 'all' });
+    return body.data_streams.length > 0;
+  } catch (err) {
+    if (err.body != null && err.body.status === 404) {
+      return false;
+    } else if (
+      // if index already created, _data_stream/${name} request will produce the following error
+      // data stream does not exist at this point, so we can return false
+      err?.body?.error?.reason?.includes(
+        `The provided expression [${name}] matches an alias, specify the corresponding concrete indices instead.`
+      )
+    ) {
+      return false;
+    } else {
+      throw err.body ? err.body : err;
+    }
+  }
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * migrate to data stream
+ * @param esClient
+ * @param name
+ */
+export const migrateToDataStream = async (
+  esClient: ElasticsearchClient,
+  name: string
+): Promise<unknown> => {
+  return esClient.indices.migrateToDataStream({
+    name,
+  });
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts
@@ -0,0 +1,26 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+import { MappingProperty } from '@elastic/elasticsearch/lib/api/types';
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+/**
+ * update mappings of index
+ * @param esClient
+ * @param index
+ * @param mappings
+ */
+export const putMappings = async (
+  esClient: ElasticsearchClient,
+  index: string,
+  mappings: Record<string, MappingProperty>
+): Promise<unknown> => {
+  return await esClient.indices.putMapping({
+    index,
+    properties: mappings,
+  });
+};
diff --git a/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts b/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts
@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { ElasticsearchClient } from '../elasticsearch_client';
+
+export const removePolicyFromIndex = async (
+  esClient: ElasticsearchClient,
+  index: string
+): Promise<unknown> => {
+  return (await esClient.ilm.removePolicy({ index }, { meta: true })).body;
+};
diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts
@@ -56,6 +56,7 @@ export * from './sort_field';
 export * from './sort_order';
 export * from './tags';
 export * from './tie_breaker_id';
+export * from './timestamp';
 export * from './total';
 export * from './type';
 export * from './underscore_version';

diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts
@@ -12,3 +12,6 @@ export const meta = t.object;
 export type Meta = t.TypeOf<typeof meta>;
 export const metaOrUndefined = t.union([meta, t.undefined]);
 export type MetaOrUndefined = t.TypeOf<typeof metaOrUndefined>;
+
+export const nullableMetaOrUndefined = t.union([metaOrUndefined, t.null]);
+export type NullableMetaOrUndefined = t.TypeOf<typeof nullableMetaOrUndefined>;
diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import * as t from 'io-ts';
+import { IsoDateString } from '@kbn/securitysolution-io-ts-types';
+
+export const timestamp = IsoDateString;
+export const timestampOrUndefined = t.union([IsoDateString, t.undefined]);
diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts
@@ -21,6 +21,7 @@ import {
 
 export const getListItemResponseMock = (): ListItemSchema => ({
   _version: undefined,
+  '@timestamp': DATE_NOW,
   created_at: DATE_NOW,
   created_by: USER,
   deserializer: undefined,

diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts
@@ -11,6 +11,7 @@ import * as t from 'io-ts';
 import { _versionOrUndefined } from '../../common/underscore_version';
 import { deserializerOrUndefined } from '../../common/deserializer';
 import { metaOrUndefined } from '../../common/meta';
+import { timestampOrUndefined } from '../../common/timestamp';
 import { serializerOrUndefined } from '../../common/serializer';
 import { created_at } from '../../common/created_at';
 import { created_by } from '../../common/created_by';
@@ -25,6 +26,7 @@ import { value } from '../../common/value';
 export const listItemSchema = t.exact(
   t.type({
     _version: _versionOrUndefined,
+    '@timestamp': timestampOrUndefined,
     created_at,
     created_by,
     deserializer: deserializerOrUndefined,

diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts
@@ -23,6 +23,7 @@ import {
 
 export const getListResponseMock = (): ListSchema => ({
   _version: undefined,
+  '@timestamp': DATE_NOW,
   created_at: DATE_NOW,
   created_by: USER,
   description: DESCRIPTION,

diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts
@@ -13,6 +13,7 @@ import { deserializerOrUndefined } from '../../common/deserializer';
 import { metaOrUndefined } from '../../common/meta';
 import { serializerOrUndefined } from '../../common/serializer';
 import { created_at } from '../../common/created_at';
+import { timestampOrUndefined } from '../../common/timestamp';
 import { created_by } from '../../common/created_by';
 import { description } from '../../common/description';
 import { id } from '../../common/id';
@@ -26,6 +27,7 @@ import { updated_by } from '../../common/updated_by';
 export const listSchema = t.exact(
   t.type({
     _version: _versionOrUndefined,
+    '@timestamp': timestampOrUndefined,
     created_at,
     created_by,
     description,

diff --git a/packages/kbn-securitysolution-list-api/src/api/index.ts b/packages/kbn-securitysolution-list-api/src/api/index.ts
@@ -37,7 +37,7 @@ import {
 
 import {
   ENDPOINT_LIST_URL,
-  EXCEPTION_FILTER,
+  INTERNAL_EXCEPTION_FILTER,
   EXCEPTION_LIST_ITEM_URL,
   EXCEPTION_LIST_URL,
 } from '@kbn/securitysolution-list-constants';
@@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({
   http,
   signal,
 }: GetExceptionFilterFromExceptionListIdsProps): Promise<ExceptionFilterResponse> =>
-  http.fetch(EXCEPTION_FILTER, {
+  http.fetch(INTERNAL_EXCEPTION_FILTER, {
     method: 'POST',
     body: JSON.stringify({
       exception_list_ids: exceptionListIds,
@@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({
   chunkSize,
   signal,
 }: GetExceptionFilterFromExceptionsProps): Promise<ExceptionFilterResponse> =>
-  http.fetch(EXCEPTION_FILTER, {
+  http.fetch(INTERNAL_EXCEPTION_FILTER, {
     method: 'POST',
     body: JSON.stringify({
       exceptions,

diff --git a/packages/kbn-securitysolution-list-api/src/list_api/index.ts b/packages/kbn-securitysolution-list-api/src/list_api/index.ts
@@ -37,7 +37,7 @@ import {
   LIST_ITEM_URL,
   LIST_PRIVILEGES_URL,
   LIST_URL,
-  FIND_LISTS_BY_SIZE,
+  INTERNAL_FIND_LISTS_BY_SIZE,
 } from '@kbn/securitysolution-list-constants';
 import { toError, toPromise } from '../fp_utils';
 
@@ -115,7 +115,7 @@ const findListsBySize = async ({
   per_page,
   signal,
 }: ApiParams & FindListSchemaEncoded): Promise<FoundListsBySizeSchema> => {
-  return http.fetch(`${FIND_LISTS_BY_SIZE}`, {
+  return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, {
     method: 'GET',
     query: {
       cursor,

diff --git a/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts
@@ -23,6 +23,7 @@ import {
 } from '../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

diff --git a/packages/kbn-securitysolution-list-constants/index.ts b/packages/kbn-securitysolution-list-constants/index.ts
@@ -20,8 +20,8 @@ export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`;
  * Internal value list routes
  */
 export const INTERNAL_LIST_URL = '/internal/lists';
-export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
-export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
+export const INTERNAL_FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const;
+export const INTERNAL_EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const;
 
 /**
  * Exception list routes

diff --git a/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts
@@ -23,6 +23,7 @@ import {
 } from '../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts
@@ -20,6 +20,7 @@ import {
 } from '../../constants.mock';
 
 export const getListItemResponseMock = (): ListItemSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,

diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts
@@ -22,6 +22,7 @@ import {
 } from '../../constants.mock';
 
 export const getListResponseMock = (): ListSchema => ({
+  '@timestamp': DATE_NOW,
   _version: undefined,
   created_at: DATE_NOW,
   created_by: USER,