From 505d8265c8efb8d92421019a59c31e137476ece6 Mon Sep 17 00:00:00 2001 From: Vitalii Dmyterko <92328789+vitaliidm@users.noreply.github.com> Date: Wed, 23 Aug 2023 19:42:57 +0100 Subject: [PATCH] [Security Solution][Detection Engine] move lists to data stream (#162508) ## Summary - addresses https://github.com/elastic/security-team/issues/7198 - moves list/items indices to data stream - adds `@timestamp` mapping to indices mappings - migrate to data stream if indices already exist(for customers < 8.11) or create data stream(for customers 8.11+ or serverless) - adds [DLM](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/data-streams-put-lifecycle.html) to index templates - replaces update/delete queries with update_by_query/delete_by_query which supported in data streams - fixes existing issues with update/patch APIs for lists/items - update/patch for lists didn't save `version` parameter in ES - update and patch APIs for lists/items were identical, i.e. for both routes was called the same `update` method w/o any changes
Technical detail on moving API to (update/delete)_by_query `update_by_query`, `delete_by_query` do not support refresh=wait_for, [only false/true values](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-update-by-query.html#_refreshing_shards_2). Which might break some of the use cases on UI(when list is removed, we refetch all lists. Deleted list will be returned for some time. [Default refresh time is 1s](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/docs-refresh.html)). So, we retry refetching deleted/updated document before finishing request, to return reindexed document `update_by_query` does not support OCC [as update API](https://www.elastic.co/guide/en/elasticsearch/reference/8.9/optimistic-concurrency-control.html). Which is supported in both [list](https://www.elastic.co/guide/en/security/current/lists-api-update-container.html)/[list item ](https://www.elastic.co/guide/en/security/current/lists-api-update-item.html)updates through _version parameter. _version is base64 encoded "_seq_no", "_primary_term" props used for OCC So, to keep it without breaking changes: implemented check for version conflict within update method
### Checklist Delete any items that are not applicable to this PR. - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --------- Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com> --- .../src/field_value_lists/index.test.tsx | 1 + .../src/list_schema/index.mock.ts | 1 + .../kbn-securitysolution-es-utils/index.ts | 6 + .../src/create_data_stream/index.ts | 23 ++ .../src/delete_data_stream/index.ts | 28 ++ .../src/get_data_stream_exists/index.ts | 38 ++ .../src/migrate_to_data_stream/index.ts | 23 ++ .../src/put_mappings/index.ts | 26 ++ .../src/remove_policy_from_index/index.ts | 16 + .../src/common/index.ts | 1 + .../src/common/meta/index.ts | 3 + .../src/common/timestamp/index.ts | 13 + .../response/list_item_schema/index.mock.ts | 1 + .../src/response/list_item_schema/index.ts | 2 + .../src/response/list_schema/index.mock.ts | 1 + .../src/response/list_schema/index.ts | 2 + .../src/api/index.ts | 6 +- .../src/list_api/index.ts | 4 +- .../mocks/response/list_schema.mock.ts | 1 + .../index.ts | 4 +- .../src/mocks/response/list_schema.mock.ts | 1 + .../schemas/response/list_item_schema.mock.ts | 1 + .../schemas/response/list_schema.mock.ts | 1 + .../server/routes/create_list_index_route.ts | 102 ----- .../server/routes/delete_list_index_route.ts | 112 ------ x-pack/plugins/lists/server/routes/index.ts | 40 +- .../create_exception_filter_route.ts} | 13 +- .../find_lists_by_size_route.ts | 13 +- .../routes/{ => list}/create_list_route.ts | 26 +- .../routes/{ => list}/delete_list_route.ts | 14 +- .../{ => list}/import_list_item_route.ts | 46 ++- .../routes/{ => list}/patch_list_route.ts | 22 +- .../routes/{ => list}/read_list_route.ts | 10 +- .../routes/{ => list}/update_list_route.ts | 20 +- .../list_index/create_list_index_route.ts | 84 ++++ .../list_index/delete_list_index_route.ts | 147 +++++++ .../export_list_item_route.ts | 10 +- .../{ => list_index}/find_list_route.ts | 9 +- .../{ => list_index}/read_list_index_route.ts | 28 +- .../{ => list_item}/create_list_item_route.ts | 10 +- .../{ => list_item}/delete_list_item_route.ts | 10 +- .../{ => list_item}/find_list_item_route.ts | 9 +- .../{ => list_item}/patch_list_item_route.ts | 22 +- .../{ => list_item}/read_list_item_route.ts | 10 +- .../{ => list_item}/update_list_item_route.ts | 20 +- .../read_list_privileges_route.mock.ts} | 0 .../read_list_privileges_route.ts} | 9 +- .../lists/server/routes/utils/index.ts | 1 + .../routes/utils/remove_templates_if_exist.ts | 26 ++ .../index_es_list_item_schema.mock.ts | 1 + .../index_es_list_item_schema.ts | 2 + .../index_es_list_schema.mock.ts | 1 + .../elastic_query/index_es_list_schema.ts | 2 + .../elastic_query/update_es_list_schema.ts | 2 + .../search_es_list_item_schema.mock.ts | 1 + .../search_es_list_item_schema.ts | 6 +- .../search_es_list_schema.mock.ts | 1 + .../elastic_response/search_es_list_schema.ts | 6 +- .../services/items/create_list_item.test.ts | 8 +- .../server/services/items/create_list_item.ts | 5 +- .../items/create_list_items_bulk.test.ts | 1 + .../services/items/create_list_items_bulk.ts | 1 + .../services/items/delete_list_item.test.ts | 14 +- .../server/services/items/delete_list_item.ts | 10 +- .../services/items/find_list_item.test.ts | 1 + .../services/items/get_list_item.test.ts | 1 + .../items/get_list_item_by_values.test.ts | 1 + .../items/get_list_item_template.test.ts | 5 +- .../services/items/get_list_item_template.ts | 10 +- .../services/items/list_item_mappings.json | 3 + .../items/search_list_item_by_values.test.ts | 1 + .../services/items/update_list_item.test.ts | 39 +- .../server/services/items/update_list_item.ts | 80 +++- .../items/write_lines_to_bulk_list_items.ts | 100 ++--- .../server/services/lists/create_list.test.ts | 10 +- .../server/services/lists/create_list.ts | 7 +- .../server/services/lists/delete_list.test.ts | 38 +- .../server/services/lists/delete_list.ts | 30 +- .../services/lists/get_list_template.test.ts | 5 +- .../services/lists/get_list_template.ts | 10 +- .../server/services/lists/list_client.mock.ts | 4 +- .../server/services/lists/list_client.test.ts | 4 +- .../server/services/lists/list_client.ts | 378 +++++++++++++----- .../server/services/lists/list_mappings.json | 3 + .../server/services/lists/update_list.test.ts | 39 +- .../server/services/lists/update_list.ts | 75 +++- .../services/utils/check_version_conflict.ts | 26 ++ .../services/utils/find_source_value.ts | 2 +- .../lists/server/services/utils/index.ts | 2 + .../utils/transform_elastic_to_list.ts | 3 + .../utils/transform_elastic_to_list_item.ts | 5 +- .../utils/wait_until_document_indexed.ts | 25 ++ .../threat_mapping/get_threat_list.ts | 2 +- .../tests/create_lists_index.ts | 16 +- .../tests/find_lists_by_size.ts | 6 +- .../tests/get_exception_filter.ts | 6 +- .../tests/import_list_items.ts | 34 +- .../security_and_spaces/tests/index.ts | 2 + .../tests/patch_list_items.ts | 281 +++++++++++++ .../security_and_spaces/tests/patch_lists.ts | 271 +++++++++++++ .../tests/read_list_privileges.ts | 2 +- .../tests/update_list_items.ts | 217 +++++++++- .../security_and_spaces/tests/update_lists.ts | 199 ++++++++- x-pack/test/lists_api_integration/utils.ts | 354 ++++++++++------ 104 files changed, 2607 insertions(+), 767 deletions(-) create mode 100644 packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts create mode 100644 packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts create mode 100644 packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts delete mode 100644 x-pack/plugins/lists/server/routes/create_list_index_route.ts delete mode 100644 x-pack/plugins/lists/server/routes/delete_list_index_route.ts rename x-pack/plugins/lists/server/routes/{get_exception_filter_route.ts => internal/create_exception_filter_route.ts} (88%) rename x-pack/plugins/lists/server/routes/{ => internal}/find_lists_by_size_route.ts (95%) rename x-pack/plugins/lists/server/routes/{ => list}/create_list_route.ts (74%) rename x-pack/plugins/lists/server/routes/{ => list}/delete_list_route.ts (96%) rename x-pack/plugins/lists/server/routes/{ => list}/import_list_item_route.ts (71%) rename x-pack/plugins/lists/server/routes/{ => list}/patch_list_route.ts (70%) rename x-pack/plugins/lists/server/routes/{ => list}/read_list_route.ts (87%) rename x-pack/plugins/lists/server/routes/{ => list}/update_list_route.ts (74%) create mode 100644 x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts create mode 100644 x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts rename x-pack/plugins/lists/server/routes/{ => list_index}/export_list_item_route.ts (89%) rename x-pack/plugins/lists/server/routes/{ => list_index}/find_list_route.ts (92%) rename x-pack/plugins/lists/server/routes/{ => list_index}/read_list_index_route.ts (63%) rename x-pack/plugins/lists/server/routes/{ => list_item}/create_list_item_route.ts (93%) rename x-pack/plugins/lists/server/routes/{ => list_item}/delete_list_item_route.ts (94%) rename x-pack/plugins/lists/server/routes/{ => list_item}/find_list_item_route.ts (95%) rename x-pack/plugins/lists/server/routes/{ => list_item}/patch_list_item_route.ts (75%) rename x-pack/plugins/lists/server/routes/{ => list_item}/read_list_item_route.ts (94%) rename x-pack/plugins/lists/server/routes/{ => list_item}/update_list_item_route.ts (77%) rename x-pack/plugins/lists/server/routes/{read_privileges_route.mock.ts => list_privileges/read_list_privileges_route.mock.ts} (100%) rename x-pack/plugins/lists/server/routes/{read_privileges_route.ts => list_privileges/read_list_privileges_route.ts} (89%) create mode 100644 x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts create mode 100644 x-pack/plugins/lists/server/services/utils/check_version_conflict.ts create mode 100644 x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts create mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts create mode 100644 x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts diff --git a/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx b/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx index 6cd92fe1a1b35..acd1a315d2c48 100644 --- a/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx +++ b/packages/kbn-securitysolution-autocomplete/src/field_value_lists/index.test.tsx @@ -211,6 +211,7 @@ describe('AutocompleteFieldListsComponent', () => { await waitFor(() => { expect(mockOnChange).toHaveBeenCalledWith({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: 'some user', diff --git a/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts b/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts index 9dca48e2d8b38..9875db2fab1f1 100644 --- a/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts +++ b/packages/kbn-securitysolution-autocomplete/src/list_schema/index.mock.ts @@ -42,6 +42,7 @@ export const NAME = 'some name'; // TODO: Once this mock is available within packages, use it instead, https://github.com/elastic/kibana/issues/100715 // import { getListResponseMock } from '../../../../../lists/common/schemas/response/list_schema.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/packages/kbn-securitysolution-es-utils/index.ts b/packages/kbn-securitysolution-es-utils/index.ts index 94aed92969627..049d5a7de492c 100644 --- a/packages/kbn-securitysolution-es-utils/index.ts +++ b/packages/kbn-securitysolution-es-utils/index.ts @@ -8,8 +8,10 @@ export * from './src/bad_request_error'; export * from './src/create_boostrap_index'; +export * from './src/create_data_stream'; export * from './src/decode_version'; export * from './src/delete_all_index'; +export * from './src/delete_data_stream'; export * from './src/delete_index_template'; export * from './src/delete_policy'; export * from './src/delete_template'; @@ -18,11 +20,15 @@ export * from './src/get_bootstrap_index_exists'; export * from './src/get_index_aliases'; export * from './src/get_index_count'; export * from './src/get_index_exists'; +export * from './src/get_data_stream_exists'; export * from './src/get_index_template_exists'; export * from './src/get_policy_exists'; export * from './src/get_template_exists'; +export * from './src/migrate_to_data_stream'; export * from './src/read_index'; export * from './src/read_privileges'; +export * from './src/put_mappings'; +export * from './src/remove_policy_from_index'; export * from './src/set_index_template'; export * from './src/set_policy'; export * from './src/set_template'; diff --git a/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts new file mode 100644 index 0000000000000..95427cc7aec56 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/create_data_stream/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * creates data stream + * @param esClient + * @param name + */ +export const createDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return esClient.indices.createDataStream({ + name, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts new file mode 100644 index 0000000000000..23009be11165e --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/delete_data_stream/index.ts @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * deletes data stream + * @param esClient + * @param name + */ +export const deleteDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return ( + await esClient.indices.deleteDataStream( + { + name, + }, + { meta: true } + ) + ).body.acknowledged; +}; diff --git a/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts new file mode 100644 index 0000000000000..8d5419405147a --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/get_data_stream_exists/index.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * checks if data stream exists + * @param esClient + * @param name + */ +export const getDataStreamExists = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + try { + const body = await esClient.indices.getDataStream({ name, expand_wildcards: 'all' }); + return body.data_streams.length > 0; + } catch (err) { + if (err.body != null && err.body.status === 404) { + return false; + } else if ( + // if index already created, _data_stream/${name} request will produce the following error + // data stream does not exist at this point, so we can return false + err?.body?.error?.reason?.includes( + `The provided expression [${name}] matches an alias, specify the corresponding concrete indices instead.` + ) + ) { + return false; + } else { + throw err.body ? err.body : err; + } + } +}; diff --git a/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts new file mode 100644 index 0000000000000..1ec831846e38f --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/migrate_to_data_stream/index.ts @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * migrate to data stream + * @param esClient + * @param name + */ +export const migrateToDataStream = async ( + esClient: ElasticsearchClient, + name: string +): Promise => { + return esClient.indices.migrateToDataStream({ + name, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts new file mode 100644 index 0000000000000..8a7c6c2b51a90 --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/put_mappings/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; +import type { ElasticsearchClient } from '../elasticsearch_client'; + +/** + * update mappings of index + * @param esClient + * @param index + * @param mappings + */ +export const putMappings = async ( + esClient: ElasticsearchClient, + index: string, + mappings: Record +): Promise => { + return await esClient.indices.putMapping({ + index, + properties: mappings, + }); +}; diff --git a/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts b/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts new file mode 100644 index 0000000000000..0b77c112a833f --- /dev/null +++ b/packages/kbn-securitysolution-es-utils/src/remove_policy_from_index/index.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import type { ElasticsearchClient } from '../elasticsearch_client'; + +export const removePolicyFromIndex = async ( + esClient: ElasticsearchClient, + index: string +): Promise => { + return (await esClient.ilm.removePolicy({ index }, { meta: true })).body; +}; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts index e50500f87f61a..9c7e1a4ad31b5 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/index.ts @@ -56,6 +56,7 @@ export * from './sort_field'; export * from './sort_order'; export * from './tags'; export * from './tie_breaker_id'; +export * from './timestamp'; export * from './total'; export * from './type'; export * from './underscore_version'; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts index 93c38facd8f4e..93376f3112530 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/meta/index.ts @@ -12,3 +12,6 @@ export const meta = t.object; export type Meta = t.TypeOf; export const metaOrUndefined = t.union([meta, t.undefined]); export type MetaOrUndefined = t.TypeOf; + +export const nullableMetaOrUndefined = t.union([metaOrUndefined, t.null]); +export type NullableMetaOrUndefined = t.TypeOf; diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts new file mode 100644 index 0000000000000..e4b89c7590530 --- /dev/null +++ b/packages/kbn-securitysolution-io-ts-list-types/src/common/timestamp/index.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import * as t from 'io-ts'; +import { IsoDateString } from '@kbn/securitysolution-io-ts-types'; + +export const timestamp = IsoDateString; +export const timestampOrUndefined = t.union([IsoDateString, t.undefined]); diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts index 33d3b49cf5492..3b85f03e2facb 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.mock.ts @@ -21,6 +21,7 @@ import { export const getListItemResponseMock = (): ListItemSchema => ({ _version: undefined, + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, deserializer: undefined, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts index ae75f3a7741bc..4a45903c82d98 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_item_schema/index.ts @@ -11,6 +11,7 @@ import * as t from 'io-ts'; import { _versionOrUndefined } from '../../common/underscore_version'; import { deserializerOrUndefined } from '../../common/deserializer'; import { metaOrUndefined } from '../../common/meta'; +import { timestampOrUndefined } from '../../common/timestamp'; import { serializerOrUndefined } from '../../common/serializer'; import { created_at } from '../../common/created_at'; import { created_by } from '../../common/created_by'; @@ -25,6 +26,7 @@ import { value } from '../../common/value'; export const listItemSchema = t.exact( t.type({ _version: _versionOrUndefined, + '@timestamp': timestampOrUndefined, created_at, created_by, deserializer: deserializerOrUndefined, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts index 3c78c0cdbc976..6ca277659c078 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.mock.ts @@ -23,6 +23,7 @@ import { export const getListResponseMock = (): ListSchema => ({ _version: undefined, + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts index 9db686e6de255..a9f076384fdf8 100644 --- a/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts +++ b/packages/kbn-securitysolution-io-ts-list-types/src/response/list_schema/index.ts @@ -13,6 +13,7 @@ import { deserializerOrUndefined } from '../../common/deserializer'; import { metaOrUndefined } from '../../common/meta'; import { serializerOrUndefined } from '../../common/serializer'; import { created_at } from '../../common/created_at'; +import { timestampOrUndefined } from '../../common/timestamp'; import { created_by } from '../../common/created_by'; import { description } from '../../common/description'; import { id } from '../../common/id'; @@ -26,6 +27,7 @@ import { updated_by } from '../../common/updated_by'; export const listSchema = t.exact( t.type({ _version: _versionOrUndefined, + '@timestamp': timestampOrUndefined, created_at, created_by, description, diff --git a/packages/kbn-securitysolution-list-api/src/api/index.ts b/packages/kbn-securitysolution-list-api/src/api/index.ts index 07d1b677a76bf..01a74756ce9be 100644 --- a/packages/kbn-securitysolution-list-api/src/api/index.ts +++ b/packages/kbn-securitysolution-list-api/src/api/index.ts @@ -37,7 +37,7 @@ import { import { ENDPOINT_LIST_URL, - EXCEPTION_FILTER, + INTERNAL_EXCEPTION_FILTER, EXCEPTION_LIST_ITEM_URL, EXCEPTION_LIST_URL, } from '@kbn/securitysolution-list-constants'; @@ -579,7 +579,7 @@ export const getExceptionFilterFromExceptionListIds = async ({ http, signal, }: GetExceptionFilterFromExceptionListIdsProps): Promise => - http.fetch(EXCEPTION_FILTER, { + http.fetch(INTERNAL_EXCEPTION_FILTER, { method: 'POST', body: JSON.stringify({ exception_list_ids: exceptionListIds, @@ -607,7 +607,7 @@ export const getExceptionFilterFromExceptions = async ({ chunkSize, signal, }: GetExceptionFilterFromExceptionsProps): Promise => - http.fetch(EXCEPTION_FILTER, { + http.fetch(INTERNAL_EXCEPTION_FILTER, { method: 'POST', body: JSON.stringify({ exceptions, diff --git a/packages/kbn-securitysolution-list-api/src/list_api/index.ts b/packages/kbn-securitysolution-list-api/src/list_api/index.ts index a7ca546ea2eef..81f4e9c94b3f0 100644 --- a/packages/kbn-securitysolution-list-api/src/list_api/index.ts +++ b/packages/kbn-securitysolution-list-api/src/list_api/index.ts @@ -37,7 +37,7 @@ import { LIST_ITEM_URL, LIST_PRIVILEGES_URL, LIST_URL, - FIND_LISTS_BY_SIZE, + INTERNAL_FIND_LISTS_BY_SIZE, } from '@kbn/securitysolution-list-constants'; import { toError, toPromise } from '../fp_utils'; @@ -115,7 +115,7 @@ const findListsBySize = async ({ per_page, signal, }: ApiParams & FindListSchemaEncoded): Promise => { - return http.fetch(`${FIND_LISTS_BY_SIZE}`, { + return http.fetch(`${INTERNAL_FIND_LISTS_BY_SIZE}`, { method: 'GET', query: { cursor, diff --git a/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts index 5421b4557c617..348c0506c9b26 100644 --- a/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts +++ b/packages/kbn-securitysolution-list-api/src/list_api/mocks/response/list_schema.mock.ts @@ -23,6 +23,7 @@ import { } from '../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/packages/kbn-securitysolution-list-constants/index.ts b/packages/kbn-securitysolution-list-constants/index.ts index 7bc44b534caf2..42249d5f305e9 100644 --- a/packages/kbn-securitysolution-list-constants/index.ts +++ b/packages/kbn-securitysolution-list-constants/index.ts @@ -20,8 +20,8 @@ export const LIST_PRIVILEGES_URL = `${LIST_URL}/privileges`; * Internal value list routes */ export const INTERNAL_LIST_URL = '/internal/lists'; -export const FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const; -export const EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const; +export const INTERNAL_FIND_LISTS_BY_SIZE = `${INTERNAL_LIST_URL}/_find_lists_by_size` as const; +export const INTERNAL_EXCEPTION_FILTER = `${INTERNAL_LIST_URL}/_create_filter` as const; /** * Exception list routes diff --git a/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts b/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts index 5421b4557c617..348c0506c9b26 100644 --- a/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts +++ b/packages/kbn-securitysolution-list-hooks/src/mocks/response/list_schema.mock.ts @@ -23,6 +23,7 @@ import { } from '../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts index 38bb32169beb5..cce497f87335c 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_item_schema.mock.ts @@ -20,6 +20,7 @@ import { } from '../../constants.mock'; export const getListItemResponseMock = (): ListItemSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts index adfe72d5e9125..abc52634e0232 100644 --- a/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts +++ b/x-pack/plugins/lists/common/schemas/response/list_schema.mock.ts @@ -22,6 +22,7 @@ import { } from '../../constants.mock'; export const getListResponseMock = (): ListSchema => ({ + '@timestamp': DATE_NOW, _version: undefined, created_at: DATE_NOW, created_by: USER, diff --git a/x-pack/plugins/lists/server/routes/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/create_list_index_route.ts deleted file mode 100644 index cdfd3b7646aae..0000000000000 --- a/x-pack/plugins/lists/server/routes/create_list_index_route.ts +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { validate } from '@kbn/securitysolution-io-ts-utils'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; - -import { createListIndexResponse } from '../../common/api'; -import type { ListsPluginRouter } from '../types'; - -import { buildSiemResponse } from './utils'; - -import { getListClient } from '.'; - -export const createListIndexRoute = (router: ListsPluginRouter): void => { - router.post( - { - options: { - tags: ['access:lists-all'], - }, - path: LIST_INDEX, - validate: false, - }, - async (context, _, response) => { - const siemResponse = buildSiemResponse(response); - - try { - const lists = await getListClient(context); - const listIndexExists = await lists.getListIndexExists(); - const listItemIndexExists = await lists.getListItemIndexExists(); - - const policyExists = await lists.getListPolicyExists(); - const policyListItemExists = await lists.getListItemPolicyExists(); - - if (!policyExists) { - await lists.setListPolicy(); - } - if (!policyListItemExists) { - await lists.setListItemPolicy(); - } - - const templateExists = await lists.getListTemplateExists(); - const templateListItemsExists = await lists.getListItemTemplateExists(); - const legacyTemplateExists = await lists.getLegacyListTemplateExists(); - const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); - - if (!templateExists) { - await lists.setListTemplate(); - } - - if (!templateListItemsExists) { - await lists.setListItemTemplate(); - } - - try { - // Check if the old legacy lists and items template exists and remove it - if (legacyTemplateExists) { - await lists.deleteLegacyListTemplate(); - } - if (legacyTemplateListItemsExists) { - await lists.deleteLegacyListItemTemplate(); - } - } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - } - - if (listIndexExists && listItemIndexExists) { - return siemResponse.error({ - body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" already exists`, - statusCode: 409, - }); - } else { - if (!listIndexExists) { - await lists.createListBootStrapIndex(); - } - if (!listItemIndexExists) { - await lists.createListItemBootStrapIndex(); - } - - const [validated, errors] = validate({ acknowledged: true }, createListIndexResponse); - if (errors != null) { - return siemResponse.error({ body: errors, statusCode: 500 }); - } else { - return response.ok({ body: validated ?? {} }); - } - } - } catch (err) { - const error = transformError(err); - return siemResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/delete_list_index_route.ts deleted file mode 100644 index 64607308adda4..0000000000000 --- a/x-pack/plugins/lists/server/routes/delete_list_index_route.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { validate } from '@kbn/securitysolution-io-ts-utils'; -import { transformError } from '@kbn/securitysolution-es-utils'; -import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; - -import type { ListsPluginRouter } from '../types'; -import { deleteListIndexResponse } from '../../common/api'; - -import { buildSiemResponse } from './utils'; - -import { getListClient } from '.'; - -/** - * Deletes all of the indexes, template, ilm policies, and aliases. You can check - * this by looking at each of these settings from ES after a deletion: - * - * GET /_template/.lists-default - * GET /.lists-default-000001/ - * GET /_ilm/policy/.lists-default - * GET /_alias/.lists-default - * - * GET /_template/.items-default - * GET /.items-default-000001/ - * GET /_ilm/policy/.items-default - * GET /_alias/.items-default - * - * And ensuring they're all gone - */ -export const deleteListIndexRoute = (router: ListsPluginRouter): void => { - router.delete( - { - options: { - tags: ['access:lists-all'], - }, - path: LIST_INDEX, - validate: false, - }, - async (context, _, response) => { - const siemResponse = buildSiemResponse(response); - - try { - const lists = await getListClient(context); - const listIndexExists = await lists.getListIndexExists(); - const listItemIndexExists = await lists.getListItemIndexExists(); - - if (!listIndexExists && !listItemIndexExists) { - return siemResponse.error({ - body: `index: "${lists.getListIndex()}" and "${lists.getListItemIndex()}" does not exist`, - statusCode: 404, - }); - } else { - if (listIndexExists) { - await lists.deleteListIndex(); - } - if (listItemIndexExists) { - await lists.deleteListItemIndex(); - } - - const listsPolicyExists = await lists.getListPolicyExists(); - const listItemPolicyExists = await lists.getListItemPolicyExists(); - - if (listsPolicyExists) { - await lists.deleteListPolicy(); - } - if (listItemPolicyExists) { - await lists.deleteListItemPolicy(); - } - - const listsTemplateExists = await lists.getListTemplateExists(); - const listItemTemplateExists = await lists.getListItemTemplateExists(); - - if (listsTemplateExists) { - await lists.deleteListTemplate(); - } - if (listItemTemplateExists) { - await lists.deleteListItemTemplate(); - } - - // check if legacy template exists - const legacyTemplateExists = await lists.getLegacyListTemplateExists(); - const legacyItemTemplateExists = await lists.getLegacyListItemTemplateExists(); - if (legacyTemplateExists) { - await lists.deleteLegacyListTemplate(); - } - - if (legacyItemTemplateExists) { - await lists.deleteLegacyListItemTemplate(); - } - - const [validated, errors] = validate({ acknowledged: true }, deleteListIndexResponse); - if (errors != null) { - return siemResponse.error({ body: errors, statusCode: 500 }); - } else { - return response.ok({ body: validated ?? {} }); - } - } - } catch (err) { - const error = transformError(err); - return siemResponse.error({ - body: error.message, - statusCode: error.statusCode, - }); - } - } - ); -}; diff --git a/x-pack/plugins/lists/server/routes/index.ts b/x-pack/plugins/lists/server/routes/index.ts index 6347d564981bf..b344977699d58 100644 --- a/x-pack/plugins/lists/server/routes/index.ts +++ b/x-pack/plugins/lists/server/routes/index.ts @@ -9,43 +9,43 @@ export * from './create_endpoint_list_item_route'; export * from './create_endpoint_list_route'; export * from './create_exception_list_item_route'; export * from './create_exception_list_route'; -export * from './create_list_index_route'; -export * from './create_list_item_route'; -export * from './create_list_route'; +export * from './list_index/create_list_index_route'; +export * from './list_item/create_list_item_route'; +export * from './list/create_list_route'; export * from './delete_endpoint_list_item_route'; export * from './delete_exception_list_route'; export * from './delete_exception_list_item_route'; -export * from './delete_list_index_route'; -export * from './delete_list_item_route'; -export * from './delete_list_route'; +export * from './list_index/delete_list_index_route'; +export * from './list_item/delete_list_item_route'; +export * from './list/delete_list_route'; export * from './duplicate_exception_list_route'; export * from './export_exception_list_route'; -export * from './export_list_item_route'; +export * from './list_index/export_list_item_route'; export * from './find_endpoint_list_item_route'; export * from './find_exception_list_item_route'; export * from './find_exception_list_route'; -export * from './find_list_item_route'; -export * from './find_list_route'; -export * from './find_lists_by_size_route'; -export * from './get_exception_filter_route'; +export * from './list_item/find_list_item_route'; +export * from './list_index/find_list_route'; +export * from './internal/find_lists_by_size_route'; +export * from './internal/create_exception_filter_route'; export * from './import_exceptions_route'; -export * from './import_list_item_route'; +export * from './list/import_list_item_route'; export * from './init_routes'; -export * from './patch_list_item_route'; -export * from './patch_list_route'; +export * from './list_item/patch_list_item_route'; +export * from './list/patch_list_route'; export * from './read_endpoint_list_item_route'; export * from './read_exception_list_item_route'; export * from './read_exception_list_route'; -export * from './read_list_index_route'; -export * from './read_list_item_route'; -export * from './read_list_route'; -export * from './read_privileges_route'; +export * from './list_index/read_list_index_route'; +export * from './list_item/read_list_item_route'; +export * from './list/read_list_route'; +export * from './list_privileges/read_list_privileges_route'; export * from './summary_exception_list_route'; export * from './update_endpoint_list_item_route'; export * from './update_exception_list_item_route'; export * from './update_exception_list_route'; -export * from './update_list_item_route'; -export * from './update_list_route'; +export * from './list_item/update_list_item_route'; +export * from './list/update_list_route'; export * from './utils'; // internal diff --git a/x-pack/plugins/lists/server/routes/get_exception_filter_route.ts b/x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts similarity index 88% rename from x-pack/plugins/lists/server/routes/get_exception_filter_route.ts rename to x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts index 1577040c9beb6..4fc6d2dfd402e 100644 --- a/x-pack/plugins/lists/server/routes/get_exception_filter_route.ts +++ b/x-pack/plugins/lists/server/routes/internal/create_exception_filter_route.ts @@ -11,13 +11,12 @@ import { ExceptionListItemSchema, FoundExceptionListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants'; +import { INTERNAL_EXCEPTION_FILTER } from '@kbn/securitysolution-list-constants'; -import { buildExceptionFilter } from '../services/exception_lists/build_exception_filter'; -import { ListsPluginRouter } from '../types'; -import { getExceptionFilterRequest } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; +import { buildExceptionFilter } from '../../services/exception_lists/build_exception_filter'; +import { ListsPluginRouter } from '../../types'; +import { getExceptionFilterRequest } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; export const getExceptionFilterRoute = (router: ListsPluginRouter): void => { router.post( @@ -25,7 +24,7 @@ export const getExceptionFilterRoute = (router: ListsPluginRouter): void => { options: { tags: ['access:securitySolution'], }, - path: `${EXCEPTION_FILTER}`, + path: INTERNAL_EXCEPTION_FILTER, validate: { body: buildRouteValidation(getExceptionFilterRequest), }, diff --git a/x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts b/x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts similarity index 95% rename from x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts rename to x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts index 1a6af1710ea61..6f65894d9d559 100644 --- a/x-pack/plugins/lists/server/routes/find_lists_by_size_route.ts +++ b/x-pack/plugins/lists/server/routes/internal/find_lists_by_size_route.ts @@ -8,17 +8,16 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { - FIND_LISTS_BY_SIZE, + INTERNAL_FIND_LISTS_BY_SIZE, MAXIMUM_SMALL_IP_RANGE_VALUE_LIST_DASH_SIZE, MAXIMUM_SMALL_VALUE_LIST_SIZE, } from '@kbn/securitysolution-list-constants'; import { chunk } from 'lodash'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; -import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; +import { findListsBySizeRequestQuery, findListsBySizeResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListsBySizeRoute = (router: ListsPluginRouter): void => { router.get( @@ -26,7 +25,7 @@ export const findListsBySizeRoute = (router: ListsPluginRouter): void => { options: { tags: ['access:lists-read'], }, - path: `${FIND_LISTS_BY_SIZE}`, + path: INTERNAL_FIND_LISTS_BY_SIZE, validate: { query: buildRouteValidation(findListsBySizeRequestQuery), }, diff --git a/x-pack/plugins/lists/server/routes/create_list_route.ts b/x-pack/plugins/lists/server/routes/list/create_list_route.ts similarity index 74% rename from x-pack/plugins/lists/server/routes/create_list_route.ts rename to x-pack/plugins/lists/server/routes/list/create_list_route.ts index 484d045f23aae..7a1eef80aa16c 100644 --- a/x-pack/plugins/lists/server/routes/create_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/create_list_route.ts @@ -9,12 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { CreateListRequestDecoded, createListRequest, createListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { + CreateListRequestDecoded, + createListRequest, + createListResponse, +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const createListRoute = (router: ListsPluginRouter): void => { router.post( @@ -35,13 +37,19 @@ export const createListRoute = (router: ListsPluginRouter): void => { const { name, description, deserializer, id, serializer, type, meta, version } = request.body; const lists = await getListClient(context); - const listExists = await lists.getListIndexExists(); - if (!listExists) { + const dataStreamExists = await lists.getListDataStreamExists(); + const indexExists = await lists.getListIndexExists(); + + if (!dataStreamExists && !indexExists) { return siemResponse.error({ - body: `To create a list, the index must exist first. Index "${lists.getListIndex()}" does not exist`, + body: `To create a list, the data stream must exist first. Data stream "${lists.getListName()}" does not exist`, statusCode: 400, }); } else { + // needs to be migrated to data stream + if (!dataStreamExists && indexExists) { + await lists.migrateListIndexToDataStream(); + } if (id != null) { const list = await lists.getList({ id }); if (list != null) { diff --git a/x-pack/plugins/lists/server/routes/delete_list_route.ts b/x-pack/plugins/lists/server/routes/list/delete_list_route.ts similarity index 96% rename from x-pack/plugins/lists/server/routes/delete_list_route.ts rename to x-pack/plugins/lists/server/routes/list/delete_list_route.ts index a8bb0fe01840d..ba14a2240d7e2 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/delete_list_route.ts @@ -18,14 +18,12 @@ import { import { getSavedObjectType } from '@kbn/securitysolution-list-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import type { ExceptionListClient } from '../services/exception_lists/exception_list_client'; -import { escapeQuotes } from '../services/utils/escape_query'; -import { deleteListRequestQuery, deleteListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getExceptionListClient, getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import type { ExceptionListClient } from '../../services/exception_lists/exception_list_client'; +import { escapeQuotes } from '../../services/utils/escape_query'; +import { deleteListRequestQuery, deleteListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getExceptionListClient, getListClient } from '..'; export const deleteListRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/import_list_item_route.ts b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts similarity index 71% rename from x-pack/plugins/lists/server/routes/import_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list/import_list_item_route.ts index 5155a32cb2441..60ecf15a07356 100644 --- a/x-pack/plugins/lists/server/routes/import_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list/import_list_item_route.ts @@ -10,14 +10,12 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { ConfigType } from '../config'; -import { importListItemRequestQuery, importListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; -import { createStreamFromBuffer } from './utils/create_stream_from_buffer'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { ConfigType } from '../../config'; +import { importListItemRequestQuery, importListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { createStreamFromBuffer } from '../utils/create_stream_from_buffer'; +import { getListClient } from '..'; export const importListItemRoute = (router: ListsPluginRouter, config: ConfigType): void => { router.post( @@ -45,13 +43,33 @@ export const importListItemRoute = (router: ListsPluginRouter, config: ConfigTyp const stream = createStreamFromBuffer(request.body); const { deserializer, list_id: listId, serializer, type } = request.query; const lists = await getListClient(context); - const listExists = await lists.getListIndexExists(); - if (!listExists) { - return siemResponse.error({ - body: `To import a list item, the index must exist first. Index "${lists.getListIndex()}" does not exist`, - statusCode: 400, - }); + + const listDataExists = await lists.getListDataStreamExists(); + if (!listDataExists) { + const listIndexExists = await lists.getListIndexExists(); + if (!listIndexExists) { + return siemResponse.error({ + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListName()}" does not exist`, + statusCode: 400, + }); + } + // otherwise migration is needed + await lists.migrateListIndexToDataStream(); + } + + const listItemDataExists = await lists.getListItemDataStreamExists(); + if (!listItemDataExists) { + const listItemIndexExists = await lists.getListItemIndexExists(); + if (!listItemIndexExists) { + return siemResponse.error({ + body: `To import a list item, the data steam must exist first. Data stream "${lists.getListItemName()}" does not exist`, + statusCode: 400, + }); + } + // otherwise migration is needed + await lists.migrateListItemIndexToDataStream(); } + if (listId != null) { const list = await lists.getList({ id: listId }); if (list == null) { diff --git a/x-pack/plugins/lists/server/routes/patch_list_route.ts b/x-pack/plugins/lists/server/routes/list/patch_list_route.ts similarity index 70% rename from x-pack/plugins/lists/server/routes/patch_list_route.ts rename to x-pack/plugins/lists/server/routes/list/patch_list_route.ts index 41f9e1c815e54..02edd41e4f074 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/patch_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { patchListRequest, patchListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { patchListRequest, patchListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const patchListRoute = (router: ListsPluginRouter): void => { router.patch( @@ -32,7 +30,17 @@ export const patchListRoute = (router: ListsPluginRouter): void => { try { const { name, description, id, meta, _version, version } = request.body; const lists = await getListClient(context); - const list = await lists.updateList({ _version, description, id, meta, name, version }); + + const dataStreamExists = await lists.getListDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListIndexExists(); + if (indexExists) { + await lists.migrateListIndexToDataStream(); + } + } + + const list = await lists.patchList({ _version, description, id, meta, name, version }); if (list == null) { return siemResponse.error({ body: `list id: "${id}" not found`, diff --git a/x-pack/plugins/lists/server/routes/read_list_route.ts b/x-pack/plugins/lists/server/routes/list/read_list_route.ts similarity index 87% rename from x-pack/plugins/lists/server/routes/read_list_route.ts rename to x-pack/plugins/lists/server/routes/list/read_list_route.ts index 583db5c065b62..d7a12a80ec7ef 100644 --- a/x-pack/plugins/lists/server/routes/read_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/read_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { readListRequestQuery, readListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { readListRequestQuery, readListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/update_list_route.ts b/x-pack/plugins/lists/server/routes/list/update_list_route.ts similarity index 74% rename from x-pack/plugins/lists/server/routes/update_list_route.ts rename to x-pack/plugins/lists/server/routes/list/update_list_route.ts index b2fe4f49950ad..a134341acd658 100644 --- a/x-pack/plugins/lists/server/routes/update_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list/update_list_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { updateListRequest, updateListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { updateListRequest, updateListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const updateListRoute = (router: ListsPluginRouter): void => { router.put( @@ -32,6 +30,16 @@ export const updateListRoute = (router: ListsPluginRouter): void => { try { const { name, description, id, meta, _version, version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListIndexExists(); + if (indexExists) { + await lists.migrateListIndexToDataStream(); + } + } + const list = await lists.updateList({ _version, description, id, meta, name, version }); if (list == null) { return siemResponse.error({ diff --git a/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts new file mode 100644 index 0000000000000..9116e5d338e5e --- /dev/null +++ b/x-pack/plugins/lists/server/routes/list_index/create_list_index_route.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { createListIndexResponse } from '../../../common/api'; +import type { ListsPluginRouter } from '../../types'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils'; +import { getListClient } from '..'; + +export const createListIndexRoute = (router: ListsPluginRouter): void => { + router.post( + { + options: { + tags: ['access:lists-all'], + }, + path: LIST_INDEX, + validate: false, + }, + async (context, _, response) => { + const siemResponse = buildSiemResponse(response); + + try { + const lists = await getListClient(context); + + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); + + const templateListExists = await lists.getListTemplateExists(); + const templateListItemsExists = await lists.getListItemTemplateExists(); + + if (!templateListExists || !listDataStreamExists) { + await lists.setListTemplate(); + } + + if (!templateListItemsExists || !listItemDataStreamExists) { + await lists.setListItemTemplate(); + } + + await removeLegacyTemplatesIfExist(lists); + + if (listDataStreamExists && listItemDataStreamExists) { + return siemResponse.error({ + body: `data stream: "${lists.getListName()}" and "${lists.getListItemName()}" already exists`, + statusCode: 409, + }); + } + + if (!listDataStreamExists) { + const listIndexExists = await lists.getListIndexExists(); + await (listIndexExists + ? lists.migrateListIndexToDataStream() + : lists.createListDataStream()); + } + + if (!listItemDataStreamExists) { + const listItemIndexExists = await lists.getListItemIndexExists(); + await (listItemIndexExists + ? lists.migrateListItemIndexToDataStream() + : lists.createListItemDataStream()); + } + + const [validated, errors] = validate({ acknowledged: true }, createListIndexResponse); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts new file mode 100644 index 0000000000000..2c8a2fb3212ce --- /dev/null +++ b/x-pack/plugins/lists/server/routes/list_index/delete_list_index_route.ts @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { validate } from '@kbn/securitysolution-io-ts-utils'; +import { transformError } from '@kbn/securitysolution-es-utils'; +import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { ListClient } from '../../services/lists/list_client'; +import type { ListsPluginRouter } from '../../types'; +import { deleteListIndexResponse } from '../../../common/api'; +import { buildSiemResponse, removeLegacyTemplatesIfExist } from '../utils'; +import { getListClient } from '..'; + +/** + * Deletes all of the indexes, template, ilm policies, and aliases. You can check + * this by looking at each of these settings from ES after a deletion: + * + * GET /_template/.lists-default + * GET /.lists-default-000001/ + * GET /_ilm/policy/.lists-default + * GET /_alias/.lists-default + * + * GET /_template/.items-default + * GET /.items-default-000001/ + * GET /_ilm/policy/.items-default + * GET /_alias/.items-default + * + * And ensuring they're all gone + */ +export const deleteListIndexRoute = (router: ListsPluginRouter): void => { + router.delete( + { + options: { + tags: ['access:lists-all'], + }, + path: LIST_INDEX, + validate: false, + }, + async (context, _, response) => { + const siemResponse = buildSiemResponse(response); + try { + const lists = await getListClient(context); + const listIndexExists = await lists.getListIndexExists(); + const listItemIndexExists = await lists.getListItemIndexExists(); + + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); + + // return early if no data stream or indices exist + if ( + !listDataStreamExists && + !listItemDataStreamExists && + !listIndexExists && + !listItemIndexExists + ) { + return siemResponse.error({ + body: `index and data stream: "${lists.getListName()}" and "${lists.getListItemName()}" does not exist`, + statusCode: 404, + }); + } + + // ensure data streams deleted if exist + await deleteDataStreams(lists, listDataStreamExists, listItemDataStreamExists); + + // ensure indices deleted if exist and were not migrated + await deleteIndices(lists, listIndexExists, listItemIndexExists); + + await deleteIndexTemplates(lists); + await removeLegacyTemplatesIfExist(lists); + + const [validated, errors] = validate({ acknowledged: true }, deleteListIndexResponse); + if (errors != null) { + return siemResponse.error({ body: errors, statusCode: 500 }); + } else { + return response.ok({ body: validated ?? {} }); + } + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; + +/** + * Delete list/item indices + */ +const deleteIndices = async ( + lists: ListClient, + listIndexExists: boolean, + listItemIndexExists: boolean +): Promise => { + if (listIndexExists) { + await lists.deleteListIndex(); + } + if (listItemIndexExists) { + await lists.deleteListItemIndex(); + } + + const listsPolicyExists = await lists.getListPolicyExists(); + const listItemPolicyExists = await lists.getListItemPolicyExists(); + + if (listsPolicyExists) { + await lists.deleteListPolicy(); + } + if (listItemPolicyExists) { + await lists.deleteListItemPolicy(); + } +}; + +/** + * Delete list/item data streams + */ +const deleteDataStreams = async ( + lists: ListClient, + listDataStreamExists: boolean, + listItemDataStreamExists: boolean +): Promise => { + if (listDataStreamExists) { + await lists.deleteListDataStream(); + } + if (listItemDataStreamExists) { + await lists.deleteListItemDataStream(); + } +}; + +/** + * Delete list/item index templates + */ +const deleteIndexTemplates = async (lists: ListClient): Promise => { + const listsTemplateExists = await lists.getListTemplateExists(); + const listItemTemplateExists = await lists.getListItemTemplateExists(); + + if (listsTemplateExists) { + await lists.deleteListTemplate(); + } + if (listItemTemplateExists) { + await lists.deleteListItemTemplate(); + } +}; diff --git a/x-pack/plugins/lists/server/routes/export_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts similarity index 89% rename from x-pack/plugins/lists/server/routes/export_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts index 5ab4dcaea6f8f..079a25ab24b02 100644 --- a/x-pack/plugins/lists/server/routes/export_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/export_list_item_route.ts @@ -10,12 +10,10 @@ import { Stream } from 'stream'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { exportListItemRequestQuery } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { exportListItemRequestQuery } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const exportListItemRoute = (router: ListsPluginRouter): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/find_list_route.ts b/x-pack/plugins/lists/server/routes/list_index/find_list_route.ts similarity index 92% rename from x-pack/plugins/lists/server/routes/find_list_route.ts rename to x-pack/plugins/lists/server/routes/list_index/find_list_route.ts index 411fc4e17c760..ab717a5f05ffb 100644 --- a/x-pack/plugins/lists/server/routes/find_list_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/find_list_route.ts @@ -9,11 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; -import { findListRequestQuery, findListResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; +import { findListRequestQuery, findListResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/read_list_index_route.ts b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts similarity index 63% rename from x-pack/plugins/lists/server/routes/read_list_index_route.ts rename to x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts index cd33c12371b39..3f1ffa8b95f5d 100644 --- a/x-pack/plugins/lists/server/routes/read_list_index_route.ts +++ b/x-pack/plugins/lists/server/routes/list_index/read_list_index_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_INDEX } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { readListIndexResponse } from '../../common/api'; - -import { buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { readListIndexResponse } from '../../../common/api'; +import { buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListIndexRoute = (router: ListsPluginRouter): void => { router.get( @@ -30,12 +28,12 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { try { const lists = await getListClient(context); - const listIndexExists = await lists.getListIndexExists(); - const listItemIndexExists = await lists.getListItemIndexExists(); + const listDataStreamExists = await lists.getListDataStreamExists(); + const listItemDataStreamExists = await lists.getListItemDataStreamExists(); - if (listIndexExists || listItemIndexExists) { + if (listDataStreamExists && listItemDataStreamExists) { const [validated, errors] = validate( - { list_index: listIndexExists, list_item_index: listItemIndexExists }, + { list_index: listDataStreamExists, list_item_index: listItemDataStreamExists }, readListIndexResponse ); if (errors != null) { @@ -43,19 +41,19 @@ export const readListIndexRoute = (router: ListsPluginRouter): void => { } else { return response.ok({ body: validated ?? {} }); } - } else if (!listIndexExists && listItemIndexExists) { + } else if (!listDataStreamExists && listItemDataStreamExists) { return siemResponse.error({ - body: `index ${lists.getListIndex()} does not exist`, + body: `data stream ${lists.getListName()} does not exist`, statusCode: 404, }); - } else if (!listItemIndexExists && listIndexExists) { + } else if (!listItemDataStreamExists && listDataStreamExists) { return siemResponse.error({ - body: `index ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListItemName()} does not exist`, statusCode: 404, }); } else { return siemResponse.error({ - body: `index ${lists.getListIndex()} and index ${lists.getListItemIndex()} does not exist`, + body: `data stream ${lists.getListName()} and data stream ${lists.getListItemName()} does not exist`, statusCode: 404, }); } diff --git a/x-pack/plugins/lists/server/routes/create_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts similarity index 93% rename from x-pack/plugins/lists/server/routes/create_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts index c5642139ae3ad..9a06a9dc2c4e0 100644 --- a/x-pack/plugins/lists/server/routes/create_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/create_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import { createListItemRequest, createListItemResponse } from '../../common/api'; -import type { ListsPluginRouter } from '../types'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import { createListItemRequest, createListItemResponse } from '../../../common/api'; +import type { ListsPluginRouter } from '../../types'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const createListItemRoute = (router: ListsPluginRouter): void => { router.post( diff --git a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/delete_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts index c580238455fff..a334d7012b461 100644 --- a/x-pack/plugins/lists/server/routes/delete_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/delete_list_item_route.ts @@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; +import type { ListsPluginRouter } from '../../types'; import { deleteListItemArrayResponse, deleteListItemRequestQuery, deleteListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const deleteListItemRoute = (router: ListsPluginRouter): void => { router.delete( diff --git a/x-pack/plugins/lists/server/routes/find_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts similarity index 95% rename from x-pack/plugins/lists/server/routes/find_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts index 077dd90b7323a..acd1bbd0f834f 100644 --- a/x-pack/plugins/lists/server/routes/find_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/find_list_item_route.ts @@ -9,15 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { decodeCursor } from '../services/utils'; +import type { ListsPluginRouter } from '../../types'; +import { decodeCursor } from '../../services/utils'; import { FindListItemRequestQueryDecoded, findListItemRequestQuery, findListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse, getListClient } from './utils'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse, getListClient } from '../utils'; export const findListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts similarity index 75% rename from x-pack/plugins/lists/server/routes/patch_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts index c4496197727e4..b71c949242546 100644 --- a/x-pack/plugins/lists/server/routes/patch_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/patch_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { patchListItemRequest, patchListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { patchListItemRequest, patchListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const patchListItemRoute = (router: ListsPluginRouter): void => { router.patch( @@ -32,7 +30,17 @@ export const patchListItemRoute = (router: ListsPluginRouter): void => { try { const { value, id, meta, _version } = request.body; const lists = await getListClient(context); - const listItem = await lists.updateListItem({ + + const dataStreamExists = await lists.getListItemDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListItemIndexExists(); + if (indexExists) { + await lists.migrateListItemIndexToDataStream(); + } + } + + const listItem = await lists.patchListItem({ _version, id, meta, diff --git a/x-pack/plugins/lists/server/routes/read_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts similarity index 94% rename from x-pack/plugins/lists/server/routes/read_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts index cb16d0d5f2df4..796be1ae2983d 100644 --- a/x-pack/plugins/lists/server/routes/read_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/read_list_item_route.ts @@ -9,16 +9,14 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; +import type { ListsPluginRouter } from '../../types'; import { readListItemArrayResponse, readListItemRequestQuery, readListItemResponse, -} from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +} from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const readListItemRoute = (router: ListsPluginRouter): void => { router.get( diff --git a/x-pack/plugins/lists/server/routes/update_list_item_route.ts b/x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts similarity index 77% rename from x-pack/plugins/lists/server/routes/update_list_item_route.ts rename to x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts index d05f68bf05262..f2ea42c538f61 100644 --- a/x-pack/plugins/lists/server/routes/update_list_item_route.ts +++ b/x-pack/plugins/lists/server/routes/list_item/update_list_item_route.ts @@ -9,12 +9,10 @@ import { validate } from '@kbn/securitysolution-io-ts-utils'; import { transformError } from '@kbn/securitysolution-es-utils'; import { LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; -import { updateListItemRequest, updateListItemResponse } from '../../common/api'; - -import { buildRouteValidation, buildSiemResponse } from './utils'; - -import { getListClient } from '.'; +import type { ListsPluginRouter } from '../../types'; +import { updateListItemRequest, updateListItemResponse } from '../../../common/api'; +import { buildRouteValidation, buildSiemResponse } from '../utils'; +import { getListClient } from '..'; export const updateListItemRoute = (router: ListsPluginRouter): void => { router.put( @@ -32,6 +30,16 @@ export const updateListItemRoute = (router: ListsPluginRouter): void => { try { const { value, id, meta, _version } = request.body; const lists = await getListClient(context); + + const dataStreamExists = await lists.getListItemDataStreamExists(); + // needs to be migrated to data stream if index exists + if (!dataStreamExists) { + const indexExists = await lists.getListItemIndexExists(); + if (indexExists) { + await lists.migrateListItemIndexToDataStream(); + } + } + const listItem = await lists.updateListItem({ _version, id, diff --git a/x-pack/plugins/lists/server/routes/read_privileges_route.mock.ts b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.mock.ts similarity index 100% rename from x-pack/plugins/lists/server/routes/read_privileges_route.mock.ts rename to x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.mock.ts diff --git a/x-pack/plugins/lists/server/routes/read_privileges_route.ts b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts similarity index 89% rename from x-pack/plugins/lists/server/routes/read_privileges_route.ts rename to x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts index 51ffaa9713b62..8a6c404220bbb 100644 --- a/x-pack/plugins/lists/server/routes/read_privileges_route.ts +++ b/x-pack/plugins/lists/server/routes/list_privileges/read_list_privileges_route.ts @@ -9,9 +9,8 @@ import { readPrivileges, transformError } from '@kbn/securitysolution-es-utils'; import { merge } from 'lodash/fp'; import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants'; -import type { ListsPluginRouter } from '../types'; - -import { buildSiemResponse, getListClient } from './utils'; +import type { ListsPluginRouter } from '../../types'; +import { buildSiemResponse, getListClient } from '../utils'; export const readPrivilegesRoute = (router: ListsPluginRouter): void => { router.get( @@ -27,8 +26,8 @@ export const readPrivilegesRoute = (router: ListsPluginRouter): void => { try { const esClient = (await context.core).elasticsearch.client.asCurrentUser; const lists = await getListClient(context); - const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListIndex()); - const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemIndex()); + const clusterPrivilegesLists = await readPrivileges(esClient, lists.getListName()); + const clusterPrivilegesListItems = await readPrivileges(esClient, lists.getListItemName()); const privileges = merge( { listItems: clusterPrivilegesListItems, diff --git a/x-pack/plugins/lists/server/routes/utils/index.ts b/x-pack/plugins/lists/server/routes/utils/index.ts index f035ae5dbfe9b..dd349bc5ec6e9 100644 --- a/x-pack/plugins/lists/server/routes/utils/index.ts +++ b/x-pack/plugins/lists/server/routes/utils/index.ts @@ -5,6 +5,7 @@ * 2.0. */ +export * from './remove_templates_if_exist'; export * from './get_error_message_exception_list_item'; export * from './get_error_message_exception_list'; export * from './get_list_client'; diff --git a/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts b/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts new file mode 100644 index 0000000000000..1394c628eba7b --- /dev/null +++ b/x-pack/plugins/lists/server/routes/utils/remove_templates_if_exist.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { ListClient } from '../../services/lists/list_client'; + +export const removeLegacyTemplatesIfExist = async (lists: ListClient): Promise => { + const legacyTemplateExists = await lists.getLegacyListTemplateExists(); + const legacyTemplateListItemsExists = await lists.getLegacyListItemTemplateExists(); + try { + // Check if the old legacy lists and items template exists and remove it + if (legacyTemplateExists) { + await lists.deleteLegacyListTemplate(); + } + if (legacyTemplateListItemsExists) { + await lists.deleteLegacyListItemTemplate(); + } + } catch (err) { + if (err.statusCode !== 404) { + throw err; + } + } +}; diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts index 9a41946c6677c..a2b9217c0ad7a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.mock.ts @@ -10,6 +10,7 @@ import { DATE_NOW, LIST_ID, META, TIE_BREAKER, USER, VALUE } from '../../../comm import { IndexEsListItemSchema } from './index_es_list_item_schema'; export const getIndexESListItemMock = (ip = VALUE): IndexEsListItemSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, deserializer: undefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts index b3130b95fe978..d4d94cd2a4bab 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_item_schema.ts @@ -14,6 +14,7 @@ import { metaOrUndefined, serializerOrUndefined, tie_breaker_id, + timestamp, updated_at, updated_by, } from '@kbn/securitysolution-io-ts-list-types'; @@ -23,6 +24,7 @@ import { esDataTypeUnion } from '../common/schemas'; export const indexEsListItemSchema = t.intersection([ t.exact( t.type({ + '@timestamp': timestamp, created_at, created_by, deserializer: deserializerOrUndefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts index 9e72c83223bab..b3cb37857cc9a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.mock.ts @@ -20,6 +20,7 @@ import { import { IndexEsListSchema } from './index_es_list_schema'; export const getIndexESListMock = (): IndexEsListSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts index 85e2eb95dd7e4..f6ca3bf1ecb2f 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/index_es_list_schema.ts @@ -16,6 +16,7 @@ import { name, serializerOrUndefined, tie_breaker_id, + timestamp, type, updated_at, updated_by, @@ -24,6 +25,7 @@ import { version } from '@kbn/securitysolution-io-ts-types'; export const indexEsListSchema = t.exact( t.type({ + '@timestamp': timestamp, created_at, created_by, description, diff --git a/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts index fe73d0fb9207f..15caf840d7dbd 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_query/update_es_list_schema.ts @@ -6,6 +6,7 @@ */ import * as t from 'io-ts'; +import { version } from '@kbn/securitysolution-io-ts-types'; import { descriptionOrUndefined, metaOrUndefined, @@ -21,6 +22,7 @@ export const updateEsListSchema = t.exact( name: nameOrUndefined, updated_at, updated_by, + version, }) ); diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts index 40427f0293488..b8369c8ed06a1 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.mock.ts @@ -22,6 +22,7 @@ import { getShardMock } from '../common/get_shard.mock'; import { SearchEsListItemSchema } from './search_es_list_item_schema'; export const getSearchEsListItemsAsAllUndefinedMock = (): SearchEsListItemSchema => ({ + '@timestamp': DATE_NOW, binary: undefined, boolean: undefined, byte: undefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts index 158783ce088b2..8b0fa71f2222f 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_item_schema.ts @@ -11,9 +11,10 @@ import { created_by, deserializerOrUndefined, list_id, - metaOrUndefined, + nullableMetaOrUndefined, serializerOrUndefined, tie_breaker_id, + timestampOrUndefined, updated_at, updated_by, } from '@kbn/securitysolution-io-ts-list-types'; @@ -46,6 +47,7 @@ import { export const searchEsListItemSchema = t.exact( t.type({ + '@timestamp': timestampOrUndefined, binary: binaryOrUndefined, boolean: booleanOrUndefined, byte: byteOrUndefined, @@ -70,7 +72,7 @@ export const searchEsListItemSchema = t.exact( list_id, long: longOrUndefined, long_range: longRangeOrUndefined, - meta: metaOrUndefined, + meta: nullableMetaOrUndefined, serializer: serializerOrUndefined, shape: shapeOrUndefined, short: shortOrUndefined, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts index 4e0cfef7c1352..1d4b282d5352a 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.mock.ts @@ -25,6 +25,7 @@ import { getShardMock } from '../common/get_shard.mock'; import { SearchEsListSchema } from './search_es_list_schema'; export const getSearchEsListMock = (): SearchEsListSchema => ({ + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, description: DESCRIPTION, diff --git a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts index 7e2ca2d6343cb..17d2e64fb7542 100644 --- a/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts +++ b/x-pack/plugins/lists/server/schemas/elastic_response/search_es_list_schema.ts @@ -12,10 +12,11 @@ import { description, deserializerOrUndefined, immutable, - metaOrUndefined, name, + nullableMetaOrUndefined, serializerOrUndefined, tie_breaker_id, + timestampOrUndefined, type, updated_at, updated_by, @@ -24,12 +25,13 @@ import { version } from '@kbn/securitysolution-io-ts-types'; export const searchEsListSchema = t.exact( t.type({ + '@timestamp': timestampOrUndefined, created_at, created_by, description, deserializer: deserializerOrUndefined, immutable, - meta: metaOrUndefined, + meta: nullableMetaOrUndefined, name, serializer: serializerOrUndefined, tie_breaker_id, diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts index ec8ba2876126c..700d85d935d3f 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.test.ts @@ -14,7 +14,7 @@ import { getIndexESListItemMock } from '../../schemas/elastic_query/index_es_lis import { CreateListItemOptions, createListItem } from './create_list_item'; import { getCreateListItemOptionsMock } from './create_list_item.mock'; -describe('crete_list_item', () => { +describe('create_list_item', () => { beforeEach(() => { jest.clearAllMocks(); }); @@ -26,7 +26,7 @@ describe('crete_list_item', () => { test('it returns a list item as expected with the id changed out for the elastic id', async () => { const options = getCreateListItemOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -46,14 +46,14 @@ describe('crete_list_item', () => { index: LIST_ITEM_INDEX, refresh: 'wait_for', }; - expect(options.esClient.index).toBeCalledWith(expected); + expect(options.esClient.create).toBeCalledWith(expected); }); test('It returns an auto-generated id if id is sent in undefined', async () => { const options = getCreateListItemOptionsMock(); options.id = undefined; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); diff --git a/x-pack/plugins/lists/server/services/items/create_list_item.ts b/x-pack/plugins/lists/server/services/items/create_list_item.ts index 9e80d8e4f4019..c51d3e3b944ea 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_item.ts @@ -52,6 +52,7 @@ export const createListItem = async ({ const createdAt = dateNow ?? new Date().toISOString(); const tieBreakerId = tieBreaker ?? uuidv4(); const baseBody = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, deserializer, @@ -68,9 +69,9 @@ export const createListItem = async ({ ...baseBody, ...elasticQuery, }; - const response = await esClient.index({ + const response = await esClient.create({ body, - id, + id: id ?? uuidv4(), index: listItemIndex, refresh: 'wait_for', }); diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts index ea2ff697c7d37..a2499a94d619e 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.test.ts @@ -56,6 +56,7 @@ describe('crete_list_item_bulk', () => { body: [ { create: { _index: LIST_ITEM_INDEX } }, { + '@timestamp': '2020-04-20T15:25:31.830Z', created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', deserializer: undefined, diff --git a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts index 0944e916c826a..54965751a058a 100644 --- a/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts +++ b/x-pack/plugins/lists/server/services/items/create_list_items_bulk.ts @@ -60,6 +60,7 @@ export const createListItemsBulk = async ({ }); if (elasticQuery != null) { const elasticBody: IndexEsListItemSchema = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, deserializer, diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts index d2c424804297d..a468eae0b8811 100644 --- a/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.test.ts @@ -40,16 +40,20 @@ describe('delete_list_item', () => { expect(deletedListItem).toEqual(listItem); }); - test('Delete calls "delete" if a list item is returned from "getListItem"', async () => { + test('Delete calls "deleteByQuery" if a list item is returned from "getListItem"', async () => { const listItem = getListItemResponseMock(); (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); const options = getDeleteListItemOptionsMock(); await deleteListItem(options); - const deleteQuery = { - id: LIST_ITEM_ID, + const deleteByQuery = { index: LIST_ITEM_INDEX, - refresh: 'wait_for', + query: { + ids: { + values: [LIST_ITEM_ID], + }, + }, + refresh: false, }; - expect(options.esClient.delete).toBeCalledWith(deleteQuery); + expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery); }); }); diff --git a/x-pack/plugins/lists/server/services/items/delete_list_item.ts b/x-pack/plugins/lists/server/services/items/delete_list_item.ts index 8fbdae3420acf..dffc12091d0fb 100644 --- a/x-pack/plugins/lists/server/services/items/delete_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/delete_list_item.ts @@ -25,10 +25,14 @@ export const deleteListItem = async ({ if (listItem == null) { return null; } else { - await esClient.delete({ - id, + await esClient.deleteByQuery({ index: listItemIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, }); } return listItem; diff --git a/x-pack/plugins/lists/server/services/items/find_list_item.test.ts b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts index 8b1457f0ce53d..93ed864c05f00 100644 --- a/x-pack/plugins/lists/server/services/items/find_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/find_list_item.test.ts @@ -31,6 +31,7 @@ describe('find_list_item', () => { { _id: 'some-list-item-id', _source: { + '@timestamp': '2020-04-20T15:25:31.830Z', _version: 'undefined', created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', diff --git a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts index 2c2b703246a5d..55dfcb8062176 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item.test.ts @@ -50,6 +50,7 @@ describe('get_list_item', () => { test('it returns null if all the values underneath the source type is undefined', async () => { const data = getSearchListItemMock(); data.hits.hits[0]._source = { + '@timestamp': DATE_NOW, binary: undefined, boolean: undefined, byte: undefined, diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts index 9f0b183b97b5c..d48a59fb48668 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_by_values.test.ts @@ -62,6 +62,7 @@ describe('get_list_item_by_values', () => { expect(listItem).toEqual([ { + '@timestamp': DATE_NOW, created_at: DATE_NOW, created_by: USER, id: LIST_ITEM_ID, diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts index 10d1a0be11efb..dd2547cdaca13 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.test.ts @@ -23,11 +23,12 @@ describe('get_list_item_template', () => { test('it returns a list template with the string filled in', async () => { const template = getListItemTemplate('some_index'); expect(template).toEqual({ - index_patterns: ['some_index-*'], + data_stream: {}, + index_patterns: ['some_index'], template: { + lifecycle: {}, mappings: { listMappings: {} }, settings: { - index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts index d42844e3e133a..69d20f465944d 100644 --- a/x-pack/plugins/lists/server/services/items/get_list_item_template.ts +++ b/x-pack/plugins/lists/server/services/items/get_list_item_template.ts @@ -9,16 +9,12 @@ import listsItemsMappings from './list_item_mappings.json'; export const getListItemTemplate = (index: string): Record => { const template = { - index_patterns: [`${index}-*`], + data_stream: {}, + index_patterns: [index], template: { + lifecycle: {}, mappings: listsItemsMappings, settings: { - index: { - lifecycle: { - name: index, - rollover_alias: index, - }, - }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/items/list_item_mappings.json b/x-pack/plugins/lists/server/services/items/list_item_mappings.json index 1381402c2f2f4..67a8d784d3a73 100644 --- a/x-pack/plugins/lists/server/services/items/list_item_mappings.json +++ b/x-pack/plugins/lists/server/services/items/list_item_mappings.json @@ -1,6 +1,9 @@ { "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "tie_breaker_id": { "type": "keyword" }, diff --git a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts index 7673347d488dd..e52654a7a696e 100644 --- a/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts +++ b/x-pack/plugins/lists/server/services/items/search_list_item_by_values.test.ts @@ -74,6 +74,7 @@ describe('search_list_item_by_values', () => { { items: [ { + '@timestamp': '2020-04-20T15:25:31.830Z', _version: undefined, created_at: '2020-04-20T15:25:31.830Z', created_by: 'some user', diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts index 14fe97638e710..d7bff1c11c5df 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.test.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.test.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import type { ListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; @@ -14,6 +13,12 @@ import { updateListItem } from './update_list_item'; import { getListItem } from './get_list_item'; import { getUpdateListItemOptionsMock } from './update_list_item.mock'; +jest.mock('../utils/check_version_conflict', () => ({ + checkVersionConflict: jest.fn(), +})); +jest.mock('../utils/wait_until_document_indexed', () => ({ + waitUntilDocumentIndexed: jest.fn(), +})); jest.mock('./get_list_item', () => ({ getListItem: jest.fn(), })); @@ -27,25 +32,22 @@ describe('update_list_item', () => { jest.clearAllMocks(); }); - test('it returns a list item as expected with the id changed out for the elastic id when there is a list item to update', async () => { + test('it returns a list item when updated', async () => { const listItem = getListItemResponseMock(); (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); const options = getUpdateListItemOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); - const updatedList = await updateListItem({ ...options, esClient }); - const expected: ListItemSchema = { ...getListItemResponseMock(), id: 'elastic-id-123' }; - expect(updatedList).toEqual(expected); + esClient.updateByQuery.mockResponse({ updated: 1 }); + const updatedListItem = await updateListItem({ ...options, esClient }); + const expected: ListItemSchema = getListItemResponseMock(); + expect(updatedListItem).toEqual(expected); }); test('it returns null when there is not a list item to update', async () => { (getListItem as unknown as jest.Mock).mockResolvedValueOnce(null); const options = getUpdateListItemOptionsMock(); - const updatedList = await updateListItem(options); - expect(updatedList).toEqual(null); + const updatedListItem = await updateListItem(options); + expect(updatedListItem).toEqual(null); }); test('it returns null when the serializer and type such as ip_range returns nothing', async () => { @@ -57,7 +59,18 @@ describe('update_list_item', () => { }; (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); const options = getUpdateListItemOptionsMock(); - const updatedList = await updateListItem(options); - expect(updatedList).toEqual(null); + const updatedListItem = await updateListItem(options); + expect(updatedListItem).toEqual(null); + }); + + test('throw error if no list item was updated', async () => { + const listItem = getListItemResponseMock(); + (getListItem as unknown as jest.Mock).mockResolvedValueOnce(listItem); + const options = getUpdateListItemOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResponse({ updated: 0 }); + await expect(updateListItem({ ...options, esClient })).rejects.toThrow( + 'No list item has been updated' + ); }); }); diff --git a/x-pack/plugins/lists/server/services/items/update_list_item.ts b/x-pack/plugins/lists/server/services/items/update_list_item.ts index 71f6f9eba4290..13132c525e9cc 100644 --- a/x-pack/plugins/lists/server/services/items/update_list_item.ts +++ b/x-pack/plugins/lists/server/services/items/update_list_item.ts @@ -12,10 +12,12 @@ import type { MetaOrUndefined, _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; -import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils'; -import { transformListItemToElasticQuery } from '../utils'; -import { UpdateEsListItemSchema } from '../../schemas/elastic_query'; +import { + checkVersionConflict, + transformListItemToElasticQuery, + waitUntilDocumentIndexed, +} from '../utils'; import { getListItem } from './get_list_item'; @@ -28,6 +30,7 @@ export interface UpdateListItemOptions { user: string; meta: MetaOrUndefined; dateNow?: string; + isPatch?: boolean; } export const updateListItem = async ({ @@ -39,6 +42,7 @@ export const updateListItem = async ({ user, meta, dateNow, + isPatch = false, }: UpdateListItemOptions): Promise => { const updatedAt = dateNow ?? new Date().toISOString(); const listItem = await getListItem({ esClient, id, listItemIndex }); @@ -53,30 +57,76 @@ export const updateListItem = async ({ if (elasticQuery == null) { return null; } else { - const doc: UpdateEsListItemSchema = { + checkVersionConflict(_version, listItem._version); + const keyValues = Object.entries(elasticQuery).map(([key, keyValue]) => ({ + key, + value: keyValue, + })); + + const params = { + // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !isPatch, + keyValues, meta, updated_at: updatedAt, updated_by: user, - ...elasticQuery, }; - const response = await esClient.update({ - ...decodeVersion(_version), - body: { - doc, - }, - id: listItem.id, + const response = await esClient.updateByQuery({ + conflicts: 'proceed', index: listItemIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, + script: { + lang: 'painless', + params, + source: ` + for (int i; i < params.keyValues.size(); i++) { + def entry = params.keyValues[i]; + ctx._source[entry.key] = entry.value; + } + if (params.assignEmpty == true || params.containsKey('meta')) { + ctx._source.meta = params.meta; + } + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; + // needed for list items that were created before migration to data streams + if (ctx._source.containsKey('@timestamp') == false) { + ctx._source['@timestamp'] = ctx._source.created_at; + } + `, + }, }); + + let updatedOCCVersion: string | undefined; + if (response.updated) { + const checkIfListUpdated = async (): Promise => { + const updatedListItem = await getListItem({ esClient, id, listItemIndex }); + if (updatedListItem?._version === listItem._version) { + throw Error('List item has not been re-indexed in time'); + } + updatedOCCVersion = updatedListItem?._version; + }; + + await waitUntilDocumentIndexed(checkIfListUpdated); + } else { + throw Error('No list item has been updated'); + } + return { - _version: encodeHitVersion(response), + '@timestamp': listItem['@timestamp'], + _version: updatedOCCVersion, created_at: listItem.created_at, created_by: listItem.created_by, deserializer: listItem.deserializer, - id: response._id, + id, list_id: listItem.list_id, - meta: meta ?? listItem.meta, + meta: isPatch ? meta ?? listItem.meta : meta, serializer: listItem.serializer, tie_breaker_id: listItem.tie_breaker_id, type: listItem.type, diff --git a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts index 9e9b15284cc67..9d18b78757943 100644 --- a/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts +++ b/x-pack/plugins/lists/server/services/items/write_lines_to_bulk_list_items.ts @@ -54,60 +54,68 @@ export const importListItemsToStream = ({ meta, version, }: ImportListItemsToStreamOptions): Promise => { - return new Promise((resolve) => { + return new Promise((resolve, reject) => { const readBuffer = new BufferLines({ bufferSize: config.importBufferSize, input: stream }); let fileName: string | undefined; let list: ListSchema | null = null; readBuffer.on('fileName', async (fileNameEmitted: string) => { - readBuffer.pause(); - fileName = decodeURIComponent(fileNameEmitted); - if (listId == null) { - list = await createListIfItDoesNotExist({ - description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', { - defaultMessage: 'File uploaded from file system of {fileName}', - values: { fileName }, - }), - deserializer, - esClient, - id: fileName, - immutable: false, - listIndex, - meta, - name: fileName, - serializer, - type, - user, - version, - }); + try { + readBuffer.pause(); + fileName = decodeURIComponent(fileNameEmitted); + if (listId == null) { + list = await createListIfItDoesNotExist({ + description: i18n.translate('xpack.lists.services.items.fileUploadFromFileSystem', { + defaultMessage: 'File uploaded from file system of {fileName}', + values: { fileName }, + }), + deserializer, + esClient, + id: fileName, + immutable: false, + listIndex, + meta, + name: fileName, + serializer, + type, + user, + version, + }); + } + readBuffer.resume(); + } catch (err) { + reject(err); } - readBuffer.resume(); }); readBuffer.on('lines', async (lines: string[]) => { - if (listId != null) { - await writeBufferToItems({ - buffer: lines, - deserializer, - esClient, - listId, - listItemIndex, - meta, - serializer, - type, - user, - }); - } else if (fileName != null) { - await writeBufferToItems({ - buffer: lines, - deserializer, - esClient, - listId: fileName, - listItemIndex, - meta, - serializer, - type, - user, - }); + try { + if (listId != null) { + await writeBufferToItems({ + buffer: lines, + deserializer, + esClient, + listId, + listItemIndex, + meta, + serializer, + type, + user, + }); + } else if (fileName != null) { + await writeBufferToItems({ + buffer: lines, + deserializer, + esClient, + listId: fileName, + listItemIndex, + meta, + serializer, + type, + user, + }); + } + } catch (err) { + reject(err); } }); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.test.ts b/x-pack/plugins/lists/server/services/lists/create_list.test.ts index 96744d9f55bae..86c0155c43ec4 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.test.ts @@ -15,7 +15,7 @@ import { getIndexESListMock } from '../../schemas/elastic_query/index_es_list_sc import { CreateListOptions, createList } from './create_list'; import { getCreateListOptionsMock } from './create_list.mock'; -describe('crete_list', () => { +describe('create_list', () => { beforeEach(() => { jest.clearAllMocks(); }); @@ -27,7 +27,7 @@ describe('crete_list', () => { test('it returns a list as expected with the id changed out for the elastic id', async () => { const options = getCreateListOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -43,7 +43,7 @@ describe('crete_list', () => { serializer: '(?)', }; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); @@ -67,14 +67,14 @@ describe('crete_list', () => { index: LIST_INDEX, refresh: 'wait_for', }; - expect(options.esClient.index).toBeCalledWith(expected); + expect(options.esClient.create).toBeCalledWith(expected); }); test('It returns an auto-generated id if id is sent in undefined', async () => { const options = getCreateListOptionsMock(); options.id = undefined; const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.index.mockResponse( + esClient.create.mockResponse( // @ts-expect-error not full response interface { _id: 'elastic-id-123' } ); diff --git a/x-pack/plugins/lists/server/services/lists/create_list.ts b/x-pack/plugins/lists/server/services/lists/create_list.ts index 3a31b7fdf6b9a..cd3497b6e891a 100644 --- a/x-pack/plugins/lists/server/services/lists/create_list.ts +++ b/x-pack/plugins/lists/server/services/lists/create_list.ts @@ -58,6 +58,7 @@ export const createList = async ({ }: CreateListOptions): Promise => { const createdAt = dateNow ?? new Date().toISOString(); const body: IndexEsListSchema = { + '@timestamp': createdAt, created_at: createdAt, created_by: user, description, @@ -72,12 +73,14 @@ export const createList = async ({ updated_by: user, version, }; - const response = await esClient.index({ + + const response = await esClient.create({ body, - id, + id: id ?? uuidv4(), index: listIndex, refresh: 'wait_for', }); + return { _version: encodeHitVersion(response), id: response._id, diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts index 9fb0c5c8d6469..dd2a8639e9278 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.test.ts @@ -12,6 +12,10 @@ import { getList } from './get_list'; import { deleteList } from './delete_list'; import { getDeleteListOptionsMock } from './delete_list.mock'; +jest.mock('../utils', () => ({ + waitUntilDocumentIndexed: jest.fn(), +})); + jest.mock('./get_list', () => ({ getList: jest.fn(), })); @@ -36,34 +40,45 @@ describe('delete_list', () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); const deletedList = await deleteList(options); expect(deletedList).toEqual(list); }); - test('Delete calls "deleteByQuery" and "delete" if a list is returned from getList', async () => { + test('Delete calls "deleteByQuery" for list items if a list is returned from getList', async () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); await deleteList(options); const deleteByQuery = { body: { query: { term: { list_id: LIST_ID } } }, + conflicts: 'proceed', index: LIST_ITEM_INDEX, refresh: false, }; - expect(options.esClient.deleteByQuery).toBeCalledWith(deleteByQuery); + expect(options.esClient.deleteByQuery).toHaveBeenNthCalledWith(1, deleteByQuery); }); - test('Delete calls "delete" second if a list is returned from getList', async () => { + test('Delete calls "deleteByQuery" for list if a list is returned from getList', async () => { const list = getListResponseMock(); (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 1 }); await deleteList(options); - const deleteQuery = { - id: LIST_ID, + const deleteByQuery = { + body: { + query: { + ids: { + values: [LIST_ID], + }, + }, + }, + conflicts: 'proceed', index: LIST_INDEX, - refresh: 'wait_for', + refresh: false, }; - expect(options.esClient.delete).toHaveBeenNthCalledWith(1, deleteQuery); + expect(options.esClient.deleteByQuery).toHaveBeenCalledWith(deleteByQuery); }); test('Delete does not call data client if the list returns null', async () => { @@ -72,4 +87,13 @@ describe('delete_list', () => { await deleteList(options); expect(options.esClient.delete).not.toHaveBeenCalled(); }); + + test('throw error if no list was deleted', async () => { + const list = getListResponseMock(); + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getDeleteListOptionsMock(); + options.esClient.deleteByQuery = jest.fn().mockResolvedValue({ deleted: 0 }); + + await expect(deleteList(options)).rejects.toThrow('No list has been deleted'); + }); }); diff --git a/x-pack/plugins/lists/server/services/lists/delete_list.ts b/x-pack/plugins/lists/server/services/lists/delete_list.ts index f82845e339ed7..c6b452675b7c3 100644 --- a/x-pack/plugins/lists/server/services/lists/delete_list.ts +++ b/x-pack/plugins/lists/server/services/lists/delete_list.ts @@ -8,6 +8,8 @@ import { ElasticsearchClient } from '@kbn/core/server'; import type { Id, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { waitUntilDocumentIndexed } from '../utils'; + import { getList } from './get_list'; export interface DeleteListOptions { @@ -35,15 +37,37 @@ export const deleteList = async ({ }, }, }, + conflicts: 'proceed', index: listItemIndex, refresh: false, }); - await esClient.delete({ - id, + const response = await esClient.deleteByQuery({ + body: { + query: { + ids: { + values: [id], + }, + }, + }, + conflicts: 'proceed', index: listIndex, - refresh: 'wait_for', + refresh: false, }); + + if (response.deleted) { + const checkIfListDeleted = async (): Promise => { + const deletedList = await getList({ esClient, id, listIndex }); + if (deletedList !== null) { + throw Error('List has not been re-indexed in time'); + } + }; + + await waitUntilDocumentIndexed(checkIfListDeleted); + } else { + throw Error('No list has been deleted'); + } + return list; } }; diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts index 57d67d6d383ca..d77eb8917c5ef 100644 --- a/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.test.ts @@ -24,11 +24,12 @@ describe('get_list_template', () => { test('it returns a list template with the string filled in', async () => { const template = getListTemplate('some_index'); expect(template).toEqual({ - index_patterns: ['some_index-*'], + data_stream: {}, + index_patterns: ['some_index'], template: { + lifecycle: {}, mappings: { dynamic: 'strict', properties: {} }, settings: { - index: { lifecycle: { name: 'some_index', rollover_alias: 'some_index' } }, mapping: { total_fields: { limit: 10000 } }, }, }, diff --git a/x-pack/plugins/lists/server/services/lists/get_list_template.ts b/x-pack/plugins/lists/server/services/lists/get_list_template.ts index 8b3cfa8d1df6e..81a66c0642af0 100644 --- a/x-pack/plugins/lists/server/services/lists/get_list_template.ts +++ b/x-pack/plugins/lists/server/services/lists/get_list_template.ts @@ -8,16 +8,12 @@ import listMappings from './list_mappings.json'; export const getListTemplate = (index: string): Record => ({ - index_patterns: [`${index}-*`], + data_stream: {}, + index_patterns: [index], template: { + lifecycle: {}, mappings: listMappings, settings: { - index: { - lifecycle: { - name: index, - rollover_alias: index, - }, - }, mapping: { total_fields: { limit: 10000, diff --git a/x-pack/plugins/lists/server/services/lists/list_client.mock.ts b/x-pack/plugins/lists/server/services/lists/list_client.mock.ts index a2f11e2ac1202..cd361526d2813 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.mock.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.mock.ts @@ -24,8 +24,8 @@ import { import { ListClient } from './list_client'; export class ListClientMock extends ListClient { - public getListIndex = jest.fn().mockReturnValue(LIST_INDEX); - public getListItemIndex = jest.fn().mockReturnValue(LIST_ITEM_INDEX); + public getListName = jest.fn().mockReturnValue(LIST_INDEX); + public getListItemName = jest.fn().mockReturnValue(LIST_ITEM_INDEX); public getList = jest.fn().mockResolvedValue(getListResponseMock()); public createList = jest.fn().mockResolvedValue(getListResponseMock()); public createListIfItDoesNotExist = jest.fn().mockResolvedValue(getListResponseMock()); diff --git a/x-pack/plugins/lists/server/services/lists/list_client.test.ts b/x-pack/plugins/lists/server/services/lists/list_client.test.ts index 7259f56b9a91b..e6d6c069ccb3a 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.test.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.test.ts @@ -14,12 +14,12 @@ describe('list_client', () => { describe('Mock client checks (not exhaustive tests against it)', () => { test('it returns the get list index as expected', () => { const mock = getListClientMock(); - expect(mock.getListIndex()).toEqual(LIST_INDEX); + expect(mock.getListName()).toEqual(LIST_INDEX); }); test('it returns the get list item index as expected', () => { const mock = getListClientMock(); - expect(mock.getListItemIndex()).toEqual(LIST_ITEM_INDEX); + expect(mock.getListItemName()).toEqual(LIST_ITEM_INDEX); }); test('it returns a mock list item', async () => { diff --git a/x-pack/plugins/lists/server/services/lists/list_client.ts b/x-pack/plugins/lists/server/services/lists/list_client.ts index b6dd681c8802c..d0b72313b7fd6 100644 --- a/x-pack/plugins/lists/server/services/lists/list_client.ts +++ b/x-pack/plugins/lists/server/services/lists/list_client.ts @@ -4,18 +4,24 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - +import { MappingProperty } from '@elastic/elasticsearch/lib/api/types'; import type { ElasticsearchClient } from '@kbn/core/server'; import { createBootstrapIndex, + createDataStream, deleteAllIndex, + deleteDataStream, deleteIndexTemplate, deletePolicy, deleteTemplate, getBootstrapIndexExists, + getDataStreamExists, getIndexTemplateExists, getPolicyExists, getTemplateExists, + migrateToDataStream, + putMappings, + removePolicyFromIndex, setIndexTemplate, setPolicy, } from '@kbn/securitysolution-es-utils'; @@ -47,8 +53,10 @@ import { updateListItem, } from '../items'; import listsItemsPolicy from '../items/list_item_policy.json'; +import listItemMappings from '../items/list_item_mappings.json'; import listPolicy from './list_policy.json'; +import listMappings from './list_mappings.json'; import type { ConstructorOptions, CreateListIfItDoesNotExistOptions, @@ -115,10 +123,10 @@ export class ListClient { } /** - * Returns the list index name - * @returns The list index name + * Returns the list data stream or index name + * @returns The list data stream/index name */ - public getListIndex = (): string => { + public getListName = (): string => { const { spaceId, config: { listIndex: listsIndexName }, @@ -127,10 +135,10 @@ export class ListClient { }; /** - * Returns the list item index name - * @returns The list item index name + * Returns the list item data stream or index name + * @returns The list item data stream/index name */ - public getListItemIndex = (): string => { + public getListItemName = (): string => { const { spaceId, config: { listItemIndex: listsItemsIndexName }, @@ -146,8 +154,8 @@ export class ListClient { */ public getList = async ({ id }: GetListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getList({ esClient, id, listIndex }); + const listName = this.getListName(); + return getList({ esClient, id, listIndex: listName }); }; /** @@ -178,14 +186,14 @@ export class ListClient { version, }: CreateListOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return createList({ description, deserializer, esClient, id, immutable, - listIndex, + listIndex: listName, meta, name, serializer, @@ -225,14 +233,14 @@ export class ListClient { version, }: CreateListIfItDoesNotExistOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return createListIfItDoesNotExist({ description, deserializer, esClient, id, immutable, - listIndex, + listIndex: listName, meta, name, serializer, @@ -248,8 +256,18 @@ export class ListClient { */ public getListIndexExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getBootstrapIndexExists(esClient, listIndex); + const listName = this.getListName(); + return getBootstrapIndexExists(esClient, listName); + }; + + /** + * True if the list data stream exists, otherwise false + * @returns True if the list data stream exists, otherwise false + */ + public getListDataStreamExists = async (): Promise => { + const { esClient } = this; + const listName = this.getListName(); + return getDataStreamExists(esClient, listName); }; /** @@ -258,28 +276,104 @@ export class ListClient { */ public getListItemIndexExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getBootstrapIndexExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getBootstrapIndexExists(esClient, listItemName); + }; + + /** + * True if the list item data stream exists, otherwise false + * @returns True if the list item data stream exists, otherwise false + */ + public getListItemDataStreamExists = async (): Promise => { + const { esClient } = this; + const listItemName = this.getListItemName(); + return getDataStreamExists(esClient, listItemName); }; /** * Creates the list boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch + * @deprecated after moving to data streams there should not be need to use it */ public createListBootStrapIndex = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return createBootstrapIndex(esClient, listIndex); + const listName = this.getListName(); + return createBootstrapIndex(esClient, listName); + }; + + /** + * Creates list data stream + * @returns The contents of the create data stream from Elasticsearch + */ + public createListDataStream = async (): Promise => { + const { esClient } = this; + const listName = this.getListName(); + return createDataStream(esClient, listName); + }; + + /** + * update list index mappings with @timestamp and migrates it to data stream + * @returns + */ + public migrateListIndexToDataStream = async (): Promise => { + const { esClient } = this; + const listName = this.getListName(); + // update list index template + await this.setListTemplate(); + // first need to update mapping of existing index to add @timestamp + await putMappings( + esClient, + listName, + listMappings.properties as Record + ); + await migrateToDataStream(esClient, listName); + await removePolicyFromIndex(esClient, listName); + if (await this.getListPolicyExists()) { + await this.deleteListPolicy(); + } + }; + + /** + * update list items index mappings with @timestamp and migrates it to data stream + * @returns + */ + public migrateListItemIndexToDataStream = async (): Promise => { + const { esClient } = this; + const listItemName = this.getListItemName(); + // update list items index template + await this.setListItemTemplate(); + // first need to update mapping of existing index to add @timestamp + await putMappings( + esClient, + listItemName, + listItemMappings.properties as Record + ); + await migrateToDataStream(esClient, listItemName); + await removePolicyFromIndex(esClient, listItemName); + if (await this.getListItemPolicyExists()) { + await this.deleteListItemPolicy(); + } }; /** * Creates the list item boot strap index for ILM policies. * @returns The contents of the bootstrap response from Elasticsearch + * @deprecated after moving to data streams there should not be need to use it */ public createListItemBootStrapIndex = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return createBootstrapIndex(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return createBootstrapIndex(esClient, listItemName); + }; + + /** + * Creates list item data stream + * @returns The contents of the create data stream from Elasticsearch + */ + public createListItemDataStream = async (): Promise => { + const { esClient } = this; + const listItemName = this.getListItemName(); + return createDataStream(esClient, listItemName); }; /** @@ -288,8 +382,8 @@ export class ListClient { */ public getListPolicyExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getPolicyExists(esClient, listIndex); + const listName = this.getListName(); + return getPolicyExists(esClient, listName); }; /** @@ -298,7 +392,7 @@ export class ListClient { */ public getListItemPolicyExists = async (): Promise => { const { esClient } = this; - const listsItemIndex = this.getListItemIndex(); + const listsItemIndex = this.getListItemName(); return getPolicyExists(esClient, listsItemIndex); }; @@ -308,8 +402,8 @@ export class ListClient { */ public getListTemplateExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getIndexTemplateExists(esClient, listIndex); + const listName = this.getListName(); + return getIndexTemplateExists(esClient, listName); }; /** @@ -318,8 +412,8 @@ export class ListClient { */ public getListItemTemplateExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getIndexTemplateExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getIndexTemplateExists(esClient, listItemName); }; /** @@ -328,8 +422,8 @@ export class ListClient { */ public getLegacyListTemplateExists = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return getTemplateExists(esClient, listIndex); + const listName = this.getListName(); + return getTemplateExists(esClient, listName); }; /** @@ -338,8 +432,8 @@ export class ListClient { */ public getLegacyListItemTemplateExists = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return getTemplateExists(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return getTemplateExists(esClient, listItemName); }; /** @@ -347,8 +441,8 @@ export class ListClient { * @returns The contents of the list template for ILM. */ public getListTemplate = (): Record => { - const listIndex = this.getListIndex(); - return getListTemplate(listIndex); + const listName = this.getListName(); + return getListTemplate(listName); }; /** @@ -356,8 +450,8 @@ export class ListClient { * @returns The contents of the list item template for ILM. */ public getListItemTemplate = (): Record => { - const listItemIndex = this.getListItemIndex(); - return getListItemTemplate(listItemIndex); + const listItemName = this.getListItemName(); + return getListItemTemplate(listItemName); }; /** @@ -367,8 +461,8 @@ export class ListClient { public setListTemplate = async (): Promise => { const { esClient } = this; const template = this.getListTemplate(); - const listIndex = this.getListIndex(); - return setIndexTemplate(esClient, listIndex, template); + const listName = this.getListName(); + return setIndexTemplate(esClient, listName, template); }; /** @@ -378,28 +472,30 @@ export class ListClient { public setListItemTemplate = async (): Promise => { const { esClient } = this; const template = this.getListItemTemplate(); - const listItemIndex = this.getListItemIndex(); - return setIndexTemplate(esClient, listItemIndex, template); + const listItemName = this.getListItemName(); + return setIndexTemplate(esClient, listItemName, template); }; /** * Sets the list policy * @returns The contents of the list policy set + * @deprecated after moving to data streams there should not be need to use it */ public setListPolicy = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return setPolicy(esClient, listIndex, listPolicy); + const listName = this.getListName(); + return setPolicy(esClient, listName, listPolicy); }; /** * Sets the list item policy * @returns The contents of the list policy set + * @deprecated after moving to data streams there should not be need to use it */ public setListItemPolicy = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return setPolicy(esClient, listItemIndex, listsItemsPolicy); + const listItemName = this.getListItemName(); + return setPolicy(esClient, listItemName, listsItemsPolicy); }; /** @@ -408,8 +504,8 @@ export class ListClient { */ public deleteListIndex = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteAllIndex(esClient, `${listIndex}-*`); + const listName = this.getListName(); + return deleteAllIndex(esClient, `${listName}-*`); }; /** @@ -418,8 +514,28 @@ export class ListClient { */ public deleteListItemIndex = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteAllIndex(esClient, `${listItemIndex}-*`); + const listItemName = this.getListItemName(); + return deleteAllIndex(esClient, `${listItemName}-*`); + }; + + /** + * Deletes the list data stream + * @returns True if the list index was deleted, otherwise false + */ + public deleteListDataStream = async (): Promise => { + const { esClient } = this; + const listName = this.getListName(); + return deleteDataStream(esClient, listName); + }; + + /** + * Deletes the list item data stream + * @returns True if the list index was deleted, otherwise false + */ + public deleteListItemDataStream = async (): Promise => { + const { esClient } = this; + const listItemName = this.getListItemName(); + return deleteDataStream(esClient, listItemName); }; /** @@ -428,8 +544,8 @@ export class ListClient { */ public deleteListPolicy = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deletePolicy(esClient, listIndex); + const listName = this.getListName(); + return deletePolicy(esClient, listName); }; /** @@ -438,8 +554,8 @@ export class ListClient { */ public deleteListItemPolicy = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deletePolicy(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deletePolicy(esClient, listItemName); }; /** @@ -448,8 +564,8 @@ export class ListClient { */ public deleteListTemplate = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteIndexTemplate(esClient, listIndex); + const listName = this.getListName(); + return deleteIndexTemplate(esClient, listName); }; /** @@ -458,8 +574,8 @@ export class ListClient { */ public deleteListItemTemplate = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteIndexTemplate(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deleteIndexTemplate(esClient, listItemName); }; /** @@ -468,8 +584,8 @@ export class ListClient { */ public deleteLegacyListTemplate = async (): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - return deleteTemplate(esClient, listIndex); + const listName = this.getListName(); + return deleteTemplate(esClient, listName); }; /** @@ -478,8 +594,8 @@ export class ListClient { */ public deleteLegacyListItemTemplate = async (): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteTemplate(esClient, listItemIndex); + const listItemName = this.getListItemName(); + return deleteTemplate(esClient, listItemName); }; /** @@ -488,8 +604,8 @@ export class ListClient { */ public deleteListItem = async ({ id }: DeleteListItemOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); - return deleteListItem({ esClient, id, listItemIndex }); + const listItemName = this.getListItemName(); + return deleteListItem({ esClient, id, listItemIndex: listItemName }); }; /** @@ -506,11 +622,11 @@ export class ListClient { type, }: DeleteListItemByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return deleteListItemByValue({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -524,13 +640,13 @@ export class ListClient { */ public deleteList = async ({ id }: DeleteListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return deleteList({ esClient, id, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, }); }; @@ -547,11 +663,11 @@ export class ListClient { stream, }: ExportListItemsToStreamOptions): void => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); exportListItemsToStream({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, stream, stringToAppend, }); @@ -580,15 +696,15 @@ export class ListClient { version, }: ImportListItemsToStreamOptions): Promise => { const { esClient, user, config } = this; - const listItemIndex = this.getListItemIndex(); - const listIndex = this.getListIndex(); + const listItemName = this.getListItemName(); + const listName = this.getListName(); return importListItemsToStream({ config, deserializer, esClient, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, meta, serializer, stream, @@ -612,11 +728,11 @@ export class ListClient { type, }: GetListItemByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItemByValue({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -646,13 +762,13 @@ export class ListClient { meta, }: CreateListItemOptions): Promise => { const { esClient, user } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return createListItem({ deserializer, esClient, id, listId, - listItemIndex, + listItemIndex: listItemName, meta, serializer, type, @@ -678,12 +794,43 @@ export class ListClient { meta, }: UpdateListItemOptions): Promise => { const { esClient, user } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return updateListItem({ _version, esClient, id, - listItemIndex, + isPatch: false, + listItemIndex: listItemName, + meta, + user, + value, + }); + }; + + /** + * Patches a list item's value given the id of the list item. + * See {@link https://www.elastic.co/guide/en/elasticsearch/reference/current/optimistic-concurrency-control.html} + * for more information around optimistic concurrency control. + * @param options + * @param options._version This is the version, useful for optimistic concurrency control. + * @param options.id id of the list to replace the list item with. + * @param options.value The value of the list item to replace. + * @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. + */ + public patchListItem = async ({ + _version, + id, + value, + meta, + }: UpdateListItemOptions): Promise => { + const { esClient, user } = this; + const listItemName = this.getListItemName(); + return updateListItem({ + _version, + esClient, + id, + isPatch: true, + listItemIndex: listItemName, meta, user, value, @@ -711,13 +858,48 @@ export class ListClient { version, }: UpdateListOptions): Promise => { const { esClient, user } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); + return updateList({ + _version, + description, + esClient, + id, + isPatch: false, + listIndex: listName, + meta, + name, + user, + version, + }); + }; + + /** + * Patches a list container's value given the id of the list. + * @param options + * @param options._version This is the version, useful for optimistic concurrency control. + * @param options.id id of the list to replace the list container data with. + * @param options.name The new name, or "undefined" if this should not be updated. + * @param options.description The new description, or "undefined" if this should not be updated. + * @param options.meta Additional meta data to associate with the list items as an object of "key/value" pairs. You can set this to "undefined" to not update meta values. + * @param options.version Updates the version of the list. + */ + public patchList = async ({ + _version, + id, + name, + description, + meta, + version, + }: UpdateListOptions): Promise => { + const { esClient, user } = this; + const listName = this.getListName(); return updateList({ _version, description, esClient, id, - listIndex, + isPatch: true, + listIndex: listName, meta, name, user, @@ -733,11 +915,11 @@ export class ListClient { */ public getListItem = async ({ id }: GetListItemOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItem({ esClient, id, - listItemIndex, + listItemIndex: listItemName, }); }; @@ -755,11 +937,11 @@ export class ListClient { value, }: GetListItemsByValueOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return getListItemByValues({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -779,11 +961,11 @@ export class ListClient { value, }: SearchListItemByValuesOptions): Promise => { const { esClient } = this; - const listItemIndex = this.getListItemIndex(); + const listItemName = this.getListItemName(); return searchListItemByValues({ esClient, listId, - listItemIndex, + listItemIndex: listItemName, type, value, }); @@ -813,12 +995,12 @@ export class ListClient { runtimeMappings, }: FindListOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); + const listName = this.getListName(); return findList({ currentIndexPosition, esClient, filter, - listIndex, + listIndex: listName, page, perPage, runtimeMappings, @@ -855,15 +1037,15 @@ export class ListClient { searchAfter, }: FindListItemOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return findListItem({ currentIndexPosition, esClient, filter, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, page, perPage, runtimeMappings, @@ -880,14 +1062,14 @@ export class ListClient { sortOrder, }: FindAllListItemsOptions): Promise => { const { esClient } = this; - const listIndex = this.getListIndex(); - const listItemIndex = this.getListItemIndex(); + const listName = this.getListName(); + const listItemName = this.getListItemName(); return findAllListItems({ esClient, filter, listId, - listIndex, - listItemIndex, + listIndex: listName, + listItemIndex: listItemName, sortField, sortOrder, }); diff --git a/x-pack/plugins/lists/server/services/lists/list_mappings.json b/x-pack/plugins/lists/server/services/lists/list_mappings.json index d00b00b6469a3..c097408ee9527 100644 --- a/x-pack/plugins/lists/server/services/lists/list_mappings.json +++ b/x-pack/plugins/lists/server/services/lists/list_mappings.json @@ -1,6 +1,9 @@ { "dynamic": "strict", "properties": { + "@timestamp": { + "type": "date" + }, "name": { "type": "keyword" }, diff --git a/x-pack/plugins/lists/server/services/lists/update_list.test.ts b/x-pack/plugins/lists/server/services/lists/update_list.test.ts index 8e68acd358861..4a088e01910de 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.test.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.test.ts @@ -14,6 +14,11 @@ import { updateList } from './update_list'; import { getList } from './get_list'; import { getUpdateListOptionsMock } from './update_list.mock'; +jest.mock('../utils', () => ({ + checkVersionConflict: jest.fn(), + waitUntilDocumentIndexed: jest.fn(), +})); + jest.mock('./get_list', () => ({ getList: jest.fn(), })); @@ -27,20 +32,6 @@ describe('update_list', () => { jest.clearAllMocks(); }); - test('it returns a list as expected with the id changed out for the elastic id when there is a list to update', async () => { - const list = getListResponseMock(); - (getList as unknown as jest.Mock).mockResolvedValueOnce(list); - const options = getUpdateListOptionsMock(); - const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); - const updatedList = await updateList({ ...options, esClient }); - const expected: ListSchema = { ...getListResponseMock(), id: 'elastic-id-123' }; - expect(updatedList).toEqual(expected); - }); - test('it returns a list with serializer and deserializer', async () => { const list: ListSchema = { ...getListResponseMock(), @@ -50,15 +41,12 @@ describe('update_list', () => { (getList as unknown as jest.Mock).mockResolvedValueOnce(list); const options = getUpdateListOptionsMock(); const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; - esClient.update.mockResponse( - // @ts-expect-error not full response interface - { _id: 'elastic-id-123' } - ); + esClient.updateByQuery.mockResolvedValue({ updated: 1 }); const updatedList = await updateList({ ...options, esClient }); const expected: ListSchema = { ...getListResponseMock(), deserializer: '{{value}}', - id: 'elastic-id-123', + id: list.id, serializer: '(?)', }; expect(updatedList).toEqual(expected); @@ -70,4 +58,17 @@ describe('update_list', () => { const updatedList = await updateList(options); expect(updatedList).toEqual(null); }); + + test('throw error if no list was updated', async () => { + const list: ListSchema = { + ...getListResponseMock(), + deserializer: '{{value}}', + serializer: '(?)', + }; + (getList as unknown as jest.Mock).mockResolvedValueOnce(list); + const options = getUpdateListOptionsMock(); + const esClient = elasticsearchClientMock.createScopedClusterClient().asCurrentUser; + esClient.updateByQuery.mockResolvedValue({ updated: 0 }); + await expect(updateList({ ...options, esClient })).rejects.toThrow('No list has been updated'); + }); }); diff --git a/x-pack/plugins/lists/server/services/lists/update_list.ts b/x-pack/plugins/lists/server/services/lists/update_list.ts index 300e4a20946c2..fcb976d7eff02 100644 --- a/x-pack/plugins/lists/server/services/lists/update_list.ts +++ b/x-pack/plugins/lists/server/services/lists/update_list.ts @@ -15,9 +15,9 @@ import type { _VersionOrUndefined, } from '@kbn/securitysolution-io-ts-list-types'; import { VersionOrUndefined } from '@kbn/securitysolution-io-ts-types'; -import { decodeVersion, encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { UpdateEsListSchema } from '../../schemas/elastic_query'; +import { checkVersionConflict, waitUntilDocumentIndexed } from '../utils'; import { getList } from '.'; @@ -32,6 +32,7 @@ export interface UpdateListOptions { meta: MetaOrUndefined; dateNow?: string; version: VersionOrUndefined; + isPatch?: boolean; } export const updateList = async ({ @@ -45,36 +46,90 @@ export const updateList = async ({ meta, dateNow, version, + isPatch = false, }: UpdateListOptions): Promise => { const updatedAt = dateNow ?? new Date().toISOString(); const list = await getList({ esClient, id, listIndex }); if (list == null) { return null; } else { + checkVersionConflict(_version, list._version); const calculatedVersion = version == null ? list.version + 1 : version; - const doc: UpdateEsListSchema = { + + const params: UpdateEsListSchema = { description, meta, name, updated_at: updatedAt, updated_by: user, + version: calculatedVersion, }; - const response = await esClient.update({ - ...decodeVersion(_version), - body: { doc }, - id, + + const response = await esClient.updateByQuery({ + conflicts: 'proceed', index: listIndex, - refresh: 'wait_for', + query: { + ids: { + values: [id], + }, + }, + refresh: false, + script: { + lang: 'painless', + params: { + ...params, + // when assigning undefined in painless, it will remove property and wil set it to null + // for patch we don't want to remove unspecified value in payload + assignEmpty: !isPatch, + }, + source: ` + if (params.assignEmpty == true || params.containsKey('description')) { + ctx._source.description = params.description; + } + if (params.assignEmpty == true || params.containsKey('meta')) { + ctx._source.meta = params.meta; + } + if (params.assignEmpty == true || params.containsKey('name')) { + ctx._source.name = params.name; + } + if (params.assignEmpty == true || params.containsKey('version')) { + ctx._source.version = params.version; + } + ctx._source.updated_at = params.updated_at; + ctx._source.updated_by = params.updated_by; + // needed for list that were created before migration to data streams + if (ctx._source.containsKey('@timestamp') == false) { + ctx._source['@timestamp'] = ctx._source.created_at; + } + `, + }, }); + + let updatedOCCVersion: string | undefined; + if (response.updated) { + const checkIfListUpdated = async (): Promise => { + const updatedList = await getList({ esClient, id, listIndex }); + if (updatedList?._version === list._version) { + throw Error('Document has not been re-indexed in time'); + } + updatedOCCVersion = updatedList?._version; + }; + + await waitUntilDocumentIndexed(checkIfListUpdated); + } else { + throw Error('No list has been updated'); + } + return { - _version: encodeHitVersion(response), + '@timestamp': list['@timestamp'], + _version: updatedOCCVersion, created_at: list.created_at, created_by: list.created_by, description: description ?? list.description, deserializer: list.deserializer, - id: response._id, + id, immutable: list.immutable, - meta, + meta: isPatch ? meta ?? list.meta : meta, name: name ?? list.name, serializer: list.serializer, tie_breaker_id: list.tie_breaker_id, diff --git a/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts b/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts new file mode 100644 index 0000000000000..66bebdae9ec05 --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/check_version_conflict.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import Boom from '@hapi/boom'; +import { decodeVersion } from '@kbn/securitysolution-es-utils'; + +/** + * checks if encoded OCC update _version matches actual version of list/item + * @param updateVersion - version in payload + * @param existingVersion - version in exiting list/item + */ +export const checkVersionConflict = ( + updateVersion: string | undefined, + existingVersion: string | undefined +): void => { + if (updateVersion && existingVersion && updateVersion !== existingVersion) { + throw Boom.conflict( + `Conflict: versions mismatch. Provided versions:${JSON.stringify( + decodeVersion(updateVersion) + )} does not match ${JSON.stringify(decodeVersion(existingVersion))}` + ); + } +}; diff --git a/x-pack/plugins/lists/server/services/utils/find_source_value.ts b/x-pack/plugins/lists/server/services/utils/find_source_value.ts index ed39b51dac6c6..5d03c8f6707b3 100644 --- a/x-pack/plugins/lists/server/services/utils/find_source_value.ts +++ b/x-pack/plugins/lists/server/services/utils/find_source_value.ts @@ -23,7 +23,7 @@ export const findSourceValue = ( const foundEntry = Object.entries(listItem).find( ([key, value]) => types.includes(key) && value != null ); - if (foundEntry != null) { + if (foundEntry != null && foundEntry[1] !== null) { const [foundType, value] = foundEntry; switch (foundType) { case 'shape': diff --git a/x-pack/plugins/lists/server/services/utils/index.ts b/x-pack/plugins/lists/server/services/utils/index.ts index 64e7c50d0e7b0..ecd2bbc5f469f 100644 --- a/x-pack/plugins/lists/server/services/utils/index.ts +++ b/x-pack/plugins/lists/server/services/utils/index.ts @@ -8,6 +8,7 @@ export * from './calculate_scroll_math'; export * from './encode_decode_cursor'; export * from './escape_query'; +export * from './check_version_conflict'; export * from './find_source_type'; export * from './find_source_value'; export * from './get_query_filter_from_type_value'; @@ -21,3 +22,4 @@ export * from './transform_elastic_named_search_to_list_item'; export * from './transform_elastic_to_list_item'; export * from './transform_elastic_to_list'; export * from './transform_list_item_to_elastic_query'; +export * from './wait_until_document_indexed'; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts index 3242742c1cfd6..e5e917a147ac9 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list.ts @@ -24,6 +24,9 @@ export const transformElasticToList = ({ _version: encodeHitVersion(hit), id: hit._id, ...hit._source, + // meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null + // return it as undefined to keep it consistent with payload + meta: hit._source?.meta ?? undefined, }; }); }; diff --git a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts index 3edbab94a0cfd..46ac86a5c8aae 100644 --- a/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts +++ b/x-pack/plugins/lists/server/services/utils/transform_elastic_to_list_item.ts @@ -56,13 +56,16 @@ export const transformElasticHitsToListItem = ({ throw new ErrorWithStatusCode(`Was expected ${type} to not be null/undefined`, 400); } else { return { + '@timestamp': _source?.['@timestamp'], _version: encodeHitVersion(hit), created_at, created_by, deserializer, id: _id, list_id, - meta, + // meta can be null if deleted (empty in PUT payload), since update_by_query set deleted values as null + // return it as undefined to keep it consistent with payload + meta: meta ?? undefined, serializer, tie_breaker_id, type, diff --git a/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts new file mode 100644 index 0000000000000..6bed7083b615e --- /dev/null +++ b/x-pack/plugins/lists/server/services/utils/wait_until_document_indexed.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import pRetry from 'p-retry'; + +// index.refresh_interval +// https://www.elastic.co/guide/en/elasticsearch/reference/8.9/index-modules.html#dynamic-index-settings +const DEFAULT_INDEX_REFRESH_TIME = 1000; + +/** + * retries until list/list item has been re-indexed + * After migration to data stream and using update_by_query, delete_by_query which do support only refresh=true/false, + * this utility needed response back when updates/delete applied + * @param fn execution function to retry + */ +export const waitUntilDocumentIndexed = async (fn: () => Promise): Promise => { + await new Promise((resolve) => setTimeout(resolve, DEFAULT_INDEX_REFRESH_TIME)); + await pRetry(fn, { + minTimeout: DEFAULT_INDEX_REFRESH_TIME, + retries: 5, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts index 73301b2c4dfb5..9009d20c379df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/indicator_match/threat_mapping/get_threat_list.ts @@ -64,7 +64,7 @@ export const getThreatList = async ({ runtime_mappings: runtimeMappings, sort: getSortForThreatList({ index, - listItemIndex: listClient.getListItemIndex(), + listItemIndex: listClient.getListItemName(), }), }, track_total_hits: false, diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts index 8ae35e0fb0426..ab549f27c4d2d 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/create_lists_index.ts @@ -28,14 +28,14 @@ export default ({ getService }: FtrProviderContext) => { await deleteListsIndex(supertest, log); }); - it('should create lists indices', async () => { + it('should create lists data streams', async () => { const { body: fetchedIndices } = await supertest .get(LIST_INDEX) .set('kbn-xsrf', 'true') .expect(404); expect(fetchedIndices).to.eql({ - message: 'index .lists-default and index .items-default does not exist', + message: 'data stream .lists-default and data stream .items-default does not exist', status_code: 404, }); @@ -46,14 +46,15 @@ export default ({ getService }: FtrProviderContext) => { expect(body).to.eql({ list_index: true, list_item_index: true }); }); - it('should update lists indices if old legacy templates exists', async () => { + it('should migrate lists indices to data streams and remove old legacy templates', async () => { // create legacy indices await createLegacyListsIndices(es); - const { body: listsIndex } = await supertest + await supertest .get(LIST_INDEX) .set('kbn-xsrf', 'true') - .expect(200); + // data stream does not exist + .expect(404); // confirm that legacy templates are in use const legacyListsTemplateExists = await getTemplateExists(es, '.lists-default'); @@ -65,10 +66,9 @@ export default ({ getService }: FtrProviderContext) => { expect(legacyItemsTemplateExists).to.equal(true); expect(nonLegacyListsTemplateExists).to.equal(false); expect(nonLegacyItemsTemplateExists).to.equal(false); - expect(listsIndex).to.eql({ list_index: true, list_item_index: true }); - // Expected 409 as index exists already, but now the templates should have been updated - await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(409); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); const { body } = await supertest.get(LIST_INDEX).set('kbn-xsrf', 'true').expect(200); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts index 49c1fc71ad27b..9452d95e7f972 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/find_lists_by_size.ts @@ -7,7 +7,7 @@ import expect from '@kbn/expect'; -import { LIST_URL, FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, INTERNAL_FIND_LISTS_BY_SIZE } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -35,7 +35,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return an empty found body correctly if no lists are loaded', async () => { const { body } = await supertest - .get(`${FIND_LISTS_BY_SIZE}`) + .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') .send() .expect(200); @@ -63,7 +63,7 @@ export default ({ getService }: FtrProviderContext): void => { // query the single list from _find_by_size const { body } = await supertest - .get(`${FIND_LISTS_BY_SIZE}`) + .get(`${INTERNAL_FIND_LISTS_BY_SIZE}`) .set('kbn-xsrf', 'true') .send() .expect(200); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts index abe119b066f88..a98f704d8a3d3 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/get_exception_filter.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import { - EXCEPTION_FILTER, + INTERNAL_EXCEPTION_FILTER, EXCEPTION_LIST_URL, EXCEPTION_LIST_ITEM_URL, } from '@kbn/securitysolution-list-constants'; @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext): void => { it('should return an exception filter if correctly passed exception items', async () => { const { body } = await supertest - .post(`${EXCEPTION_FILTER}`) + .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') .send(getExceptionFilterFromExceptionItemsSchemaMock()) .expect(200); @@ -119,7 +119,7 @@ export default ({ getService }: FtrProviderContext): void => { }) .expect(200); const { body } = await supertest - .post(`${EXCEPTION_FILTER}`) + .post(`${INTERNAL_EXCEPTION_FILTER}`) .set('kbn-xsrf', 'true') .send(getExceptionFilterFromExceptionIdsSchemaMock()) .expect(200); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts index 514ad3eb9f501..89ae216adc865 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/import_list_items.ts @@ -19,12 +19,14 @@ import { removeListServerGeneratedProperties, removeListItemServerGeneratedProperties, waitFor, + createListsIndices, } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext): void => { const supertest = getService('supertest'); const log = getService('log'); + const es = getService('es'); describe('import_list_items', () => { describe('importing list items without an index', () => { @@ -39,7 +41,7 @@ export default ({ getService }: FtrProviderContext): void => { expect(body).to.eql({ status_code: 400, message: - 'To import a list item, the index must exist first. Index ".lists-default" does not exist', + 'To import a list item, the data steam must exist first. Data stream ".lists-default" does not exist', }); }); }); @@ -110,6 +112,36 @@ export default ({ getService }: FtrProviderContext): void => { }; expect(bodyToCompare).to.eql(outputtedList); }); + + describe('legacy index (before migration to data streams)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should import list to legacy index and migrate it', async () => { + // create legacy indices + await createListsIndices(es); + + const { body } = await supertest + .post(`${LIST_ITEM_URL}/_import?type=ip`) + .set('kbn-xsrf', 'true') + .attach('file', getImportListItemAsBuffer(['127.0.0.1', '127.0.0.2']), 'list_items.txt') + .expect('Content-Type', 'application/json; charset=utf-8') + .expect(200); + + const bodyToCompare = removeListServerGeneratedProperties(body); + const outputtedList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'list_items.txt', + description: 'File uploaded from file system of list_items.txt', + }; + expect(bodyToCompare).to.eql(outputtedList); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts index 02b63c732d229..79217043b36bb 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/index.ts @@ -13,6 +13,8 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_lists')); loadTestFile(require.resolve('./create_lists_index')); loadTestFile(require.resolve('./create_list_items')); + loadTestFile(require.resolve('./patch_lists')); + loadTestFile(require.resolve('./patch_list_items')); loadTestFile(require.resolve('./read_lists')); loadTestFile(require.resolve('./read_list_items')); loadTestFile(require.resolve('./update_lists')); diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts new file mode 100644 index 0000000000000..0827b5813b6c4 --- /dev/null +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_list_items.ts @@ -0,0 +1,281 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { + PatchListItemSchema, + CreateListItemSchema, + ListItemSchema, +} from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; +import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; +import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; +import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; +import { getUpdateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_item_schema.mock'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +import { + createListsIndex, + deleteListsIndex, + removeListItemServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const retry = getService('retry'); + const es = getService('es'); + + describe('patch_list_items', () => { + describe('patch list items', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should patch a single list item property of value using an id', async () => { + const listItemId = getCreateMinimalListItemSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListItemSchemaMock()) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: listItemId, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload); + + const outputListItem: Partial = { + ...getListItemResponseMockWithoutAutoGeneratedValues(), + value: '192.168.0.2', + }; + const bodyToCompare = removeListItemServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(removeListItemServerGeneratedProperties(listItemBody)).to.eql(outputListItem); + return true; + }); + }); + + it('should patch a single list item of value using an auto-generated id of both list and list item', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: createListItemBody.id, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload) + .expect(200); + + const outputListItem: Partial = { + ...getListItemResponseMockWithoutAutoGeneratedValues(), + value: '192.168.0.2', + }; + const bodyToCompare = { + ...removeListItemServerGeneratedProperties(body), + list_id: outputListItem.list_id, + }; + expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: createListItemBody.id }) + .set('kbn-xsrf', 'true'); + const listItemBodyToCompare = { + ...removeListItemServerGeneratedProperties(listItemBody), + list_id: outputListItem.list_id, + }; + expect(listItemBodyToCompare).to.eql(outputListItem); + return true; + }); + }); + + it('should not remove unspecified in patch payload meta property', async () => { + const listItemId = getCreateMinimalListItemSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateMinimalListItemSchemaMock(), meta: { test: true } }) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + id: listItemId, + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload); + + expect(body.meta).to.eql({ test: true }); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(listItemBody.meta).to.eql({ test: true }); + return true; + }); + }); + + it('should give a 404 if it is given a fake id', async () => { + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListItemSchemaMock()) + .expect(200); + + // patch a simple list item's value + const patchListItemPayload: PatchListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: 'some-other-id', + value: '192.168.0.2', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchListItemPayload) + .expect(404); + + expect(body).to.eql({ + status_code: 404, + message: 'list item id: "some-other-id" not found', + }); + }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should patch list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should patch list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const patchPayload: PatchListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .patch(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts new file mode 100644 index 0000000000000..87076851bd34c --- /dev/null +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/patch_lists.ts @@ -0,0 +1,271 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import type { PatchListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; + +import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; +import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; +import { getUpdateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/update_list_schema.mock'; +import { + createListsIndex, + deleteListsIndex, + removeListServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, +} from '../../utils'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const log = getService('log'); + const retry = getService('retry'); + const es = getService('es'); + + describe('patch_lists', () => { + describe('patch lists', () => { + beforeEach(async () => { + await createListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + + it('should patch a single list property of name using an id', async () => { + const listId = getCreateMinimalListSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: listId }) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should patch a single list property of name using an auto-generated id', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: createListBody.id, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should not remove unspecified fields in payload', async () => { + const listId = getCreateMinimalListSchemaMock().id as string; + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list's name + const patchedListPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchedListPayload) + .expect(200); + + const outputList: Partial = { + ...getListResponseMockWithoutAutoGeneratedValues(), + name: 'some other name', + version: 2, + }; + const bodyToCompare = removeListServerGeneratedProperties(body); + expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: getUpdateMinimalListSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + const persistentBodyToCompare = removeListServerGeneratedProperties(list); + + expect(persistentBodyToCompare).to.eql(outputList); + return true; + }); + }); + + it('should change the version of a list when it patches a property', async () => { + // create a simple list + const { body: createdList } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // patch a simple list property of name and description + const patchPayload: PatchListSchema = { + id: createdList.id, + name: 'some other name', + description: 'some other description', + }; + + const { body: patchedList } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload); + + expect(createdList.version).to.be(1); + expect(patchedList.version).to.be(2); + + await retry.waitFor('patches should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: patchedList.id }) + .set('kbn-xsrf', 'true'); + + expect(list.version).to.be(2); + return true; + }); + }); + + it('should give a 404 if it is given a fake id', async () => { + const simpleList: PatchListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: '5096dec6-b6b9-4d8d-8f93-6c2602079d9d', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(simpleList) + .expect(404); + + expect(body).to.eql({ + status_code: 404, + message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', + }); + }); + + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // patch a simple list's name + const patchPayload: PatchListSchema = { + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .patch(LIST_URL) + .set('kbn-xsrf', 'true') + .send(patchPayload) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + }); + }); + }); +}; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts index c84bc80bb0231..9af6143b2152f 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/read_list_privileges.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import { LIST_PRIVILEGES_URL } from '@kbn/securitysolution-list-constants'; -import { getReadPrivilegeMock } from '@kbn/lists-plugin/server/routes/read_privileges_route.mock'; +import { getReadPrivilegeMock } from '@kbn/lists-plugin/server/routes/list_privileges/read_list_privileges_route.mock'; import { FtrProviderContext } from '../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts index e34ce8ba35465..e2bcddeb24841 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_list_items.ts @@ -12,7 +12,7 @@ import type { CreateListItemSchema, ListItemSchema, } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_ITEM_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getListItemResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_item_schema.mock'; import { getCreateMinimalListItemSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_item_schema.mock'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; @@ -23,12 +23,17 @@ import { createListsIndex, deleteListsIndex, removeListItemServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, + createListItemBypassingChecks, } from '../../utils'; // eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); + const es = getService('es'); describe('update_list_items', () => { describe('update list items', () => { @@ -64,8 +69,7 @@ export default ({ getService }: FtrProviderContext) => { const { body } = await supertest .put(LIST_ITEM_URL) .set('kbn-xsrf', 'true') - .send(updatedListItem) - .expect(200); + .send(updatedListItem); const outputListItem: Partial = { ...getListItemResponseMockWithoutAutoGeneratedValues(), @@ -73,6 +77,16 @@ export default ({ getService }: FtrProviderContext) => { }; const bodyToCompare = removeListItemServerGeneratedProperties(body); expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(removeListItemServerGeneratedProperties(listItemBody)).to.eql(outputListItem); + return true; + }); }); it('should update a single list item of value using an auto-generated id of both list and list item', async () => { @@ -116,6 +130,58 @@ export default ({ getService }: FtrProviderContext) => { list_id: outputListItem.list_id, }; expect(bodyToCompare).to.eql(outputListItem); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: createListItemBody.id }) + .set('kbn-xsrf', 'true'); + const listItemBodyToCompare = { + ...removeListItemServerGeneratedProperties(listItemBody), + list_id: outputListItem.list_id, + }; + expect(listItemBodyToCompare).to.eql(outputListItem); + return true; + }); + }); + + it('should remove unspecified in update payload meta property', async () => { + // create a simple list + await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(getCreateMinimalListSchemaMock()) + .expect(200); + + // create a simple list item + await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...getCreateMinimalListItemSchemaMock(), meta: { test: true } }) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + value: '192.168.0.2', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem); + + expect(body.meta).to.eql(undefined); + + await retry.waitFor('updates should be persistent', async () => { + const { body: listItemBody } = await supertest + .get(LIST_ITEM_URL) + .query({ id: getCreateMinimalListItemSchemaMock().id }) + .set('kbn-xsrf', 'true'); + + expect(listItemBody.meta).to.eql(undefined); + return true; + }); }); it('should give a 404 if it is given a fake id', async () => { @@ -151,6 +217,151 @@ export default ({ getService }: FtrProviderContext) => { message: 'list item id: "some-other-id" not found', }); }); + + describe('version control OCC', () => { + it('should return error if _version in payload mismatched', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: createListItemBody.id, + value: '192.168.0.2', + _version: createListItemBody._version, + }; + await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + // next update with the same _version should return 409 + const { body: errorBody } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(409); + + expect(errorBody.message).to.equal( + 'Conflict: versions mismatch. Provided versions:{"if_primary_term":1,"if_seq_no":0} does not match {"if_primary_term":1,"if_seq_no":1}' + ); + }); + + it('should return updated _version', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // create a simple list item also with an auto-generated id using the list's auto-generated id + const listItem: CreateListItemSchema = { + ...getCreateMinimalListItemSchemaMock(), + list_id: createListBody.id, + }; + const { body: createListItemBody } = await supertest + .post(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(listItem) + .expect(200); + + // update a simple list item's value + const updatedListItem: UpdateListItemSchema = { + ...getUpdateMinimalListItemSchemaMock(), + id: createListItemBody.id, + value: '192.168.0.2', + _version: createListItemBody._version, + }; + const { body: updatedListItemBody } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + // next update with the new version should be successful + await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send({ ...updatedListItem, _version: updatedListItemBody._version }) + .expect(200); + }); + }); + + describe('legacy list items index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list item that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + + it('should update list item that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + const listItemId = 'random-list-item'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + await createListItemBypassingChecks({ es, listId, id: listItemId, value: 'random' }); + + const updatedListItem: UpdateListItemSchema = { + id: listItemId, + value: 'new one', + }; + + const { body } = await supertest + .put(LIST_ITEM_URL) + .set('kbn-xsrf', 'true') + .send(updatedListItem) + .expect(200); + + expect(body.value).to.be('new one'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts index 7861c9fa2aa91..d9fc0bbe38bd3 100644 --- a/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts +++ b/x-pack/test/lists_api_integration/security_and_spaces/tests/update_lists.ts @@ -8,7 +8,7 @@ import expect from '@kbn/expect'; import type { UpdateListSchema, ListSchema } from '@kbn/securitysolution-io-ts-list-types'; -import { LIST_URL } from '@kbn/securitysolution-list-constants'; +import { LIST_URL, LIST_INDEX } from '@kbn/securitysolution-list-constants'; import { getCreateMinimalListSchemaMock } from '@kbn/lists-plugin/common/schemas/request/create_list_schema.mock'; import { getListResponseMockWithoutAutoGeneratedValues } from '@kbn/lists-plugin/common/schemas/response/list_schema.mock'; @@ -17,6 +17,8 @@ import { createListsIndex, deleteListsIndex, removeListServerGeneratedProperties, + createListsIndices, + createListBypassingChecks, } from '../../utils'; import { FtrProviderContext } from '../../common/ftr_provider_context'; @@ -24,6 +26,8 @@ import { FtrProviderContext } from '../../common/ftr_provider_context'; export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); + const retry = getService('retry'); + const es = getService('es'); describe('update_lists', () => { describe('update lists', () => { @@ -92,6 +96,79 @@ export default ({ getService }: FtrProviderContext) => { }; const bodyToCompare = removeListServerGeneratedProperties(body); expect(bodyToCompare).to.eql(outputList); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.version).to.be(2); + expect(list.name).to.be('some other name'); + return true; + }); + }); + + it('should remove unspecified meta field', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...listNoId, meta: { test: true } }); + + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList); + + expect(updatedListBody.meta).to.eql(undefined); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.meta).to.eql(undefined); + return true; + }); + }); + + it('should update meta field', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...listNoId, meta: { test: true } }); + + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + meta: { foo: 'some random value' }, + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList); + + expect(updatedListBody.meta).to.eql({ foo: 'some random value' }); + + await retry.waitFor('updates should be persistent', async () => { + const { body: list } = await supertest + .get(LIST_URL) + .query({ id: createListBody.id }) + .set('kbn-xsrf', 'true'); + + expect(list.meta).to.eql({ foo: 'some random value' }); + return true; + }); }); it('should change the version of a list when it updates a property', async () => { @@ -138,6 +215,126 @@ export default ({ getService }: FtrProviderContext) => { message: 'list id: "5096dec6-b6b9-4d8d-8f93-6c2602079d9d" not found', }); }); + + describe('version control OCC', () => { + it('should return error if _version in payload mismatched', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + _version: createListBody._version, + }; + await supertest.put(LIST_URL).set('kbn-xsrf', 'true').send(updatedList).expect(200); + + // next update with the same _version should return 409 + const { body: errorBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(409); + + expect(errorBody.message).to.equal( + 'Conflict: versions mismatch. Provided versions:{"if_primary_term":1,"if_seq_no":0} does not match {"if_primary_term":1,"if_seq_no":1}' + ); + }); + + it('should return updated _version', async () => { + const { id, ...listNoId } = getCreateMinimalListSchemaMock(); + // create a simple list with no id which will use an auto-generated id + const { body: createListBody } = await supertest + .post(LIST_URL) + .set('kbn-xsrf', 'true') + .send(listNoId) + .expect(200); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: createListBody.id, + name: 'some other name', + _version: createListBody._version, + }; + const { body: updatedListBody } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + + // version should be different + expect(updatedListBody._version).not.to.be(createListBody._version); + + // next update with the new version should be successful + await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send({ ...updatedList, _version: updatedListBody._version }) + .expect(200); + }); + }); + + describe('legacy list index (list created before migration to data stream)', () => { + beforeEach(async () => { + await deleteListsIndex(supertest, log); + }); + + afterEach(async () => { + await deleteListsIndex(supertest, log); + }); + it('should update list container that was created in legacy index and migrated through LIST_INDEX request', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // migrates old indices to data streams + await supertest.post(LIST_INDEX).set('kbn-xsrf', 'true'); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + + expect(body.name).to.be('some other name'); + }); + + it('should update list container that was created in legacy index and not yet migrated', async () => { + const listId = 'random-list'; + // create legacy indices + await createListsIndices(es); + // create a simple list + await createListBypassingChecks({ es, id: listId }); + + // update a simple list's name + const updatedList: UpdateListSchema = { + ...getUpdateMinimalListSchemaMock(), + id: listId, + name: 'some other name', + }; + const { body } = await supertest + .put(LIST_URL) + .set('kbn-xsrf', 'true') + .send(updatedList) + .expect(200); + expect(body.name).to.be('some other name'); + }); + }); }); }); }; diff --git a/x-pack/test/lists_api_integration/utils.ts b/x-pack/test/lists_api_integration/utils.ts index 13ea80e20b0fd..780042a293dcc 100644 --- a/x-pack/test/lists_api_integration/utils.ts +++ b/x-pack/test/lists_api_integration/utils.ts @@ -6,6 +6,7 @@ */ import type SuperTest from 'supertest'; +import { v4 as uuidv4 } from 'uuid'; import type { Type, @@ -21,10 +22,16 @@ import { LIST_INDEX, LIST_ITEM_URL, } from '@kbn/securitysolution-list-constants'; -import { setPolicy, setTemplate, createBootstrapIndex } from '@kbn/securitysolution-es-utils'; +import { + setPolicy, + setTemplate, + setIndexTemplate, + createBootstrapIndex, +} from '@kbn/securitysolution-es-utils'; import { Client } from '@elastic/elasticsearch'; import { ToolingLog } from '@kbn/tooling-log'; import { getImportListItemAsBuffer } from '@kbn/lists-plugin/common/schemas/request/import_list_item_schema.mock'; +import { encodeHitVersion } from '@kbn/securitysolution-es-utils'; import { countDownTest } from '../detection_engine_api_integration/utils'; @@ -98,8 +105,8 @@ export const removeListServerGeneratedProperties = ( list: Partial ): Partial => { /* eslint-disable-next-line @typescript-eslint/naming-convention */ - const { created_at, updated_at, id, tie_breaker_id, _version, ...removedProperties } = list; - return removedProperties; + const { created_at, updated_at, id, tie_breaker_id, _version, '@timestamp': _t, ...props } = list; + return props; }; /** @@ -110,8 +117,8 @@ export const removeListItemServerGeneratedProperties = ( list: Partial ): Partial => { /* eslint-disable-next-line @typescript-eslint/naming-convention */ - const { created_at, updated_at, id, tie_breaker_id, _version, ...removedProperties } = list; - return removedProperties; + const { created_at, updated_at, id, tie_breaker_id, _version, '@timestamp': _t, ...props } = list; + return props; }; /** @@ -418,87 +425,118 @@ export const waitForTextListItems = async ( await Promise.all(itemValues.map((item) => waitForTextListItem(supertest, log, item, fileName))); }; +const testPolicy = { + policy: { + phases: { + hot: { + min_age: '0ms', + actions: { + rollover: { + max_size: '50gb', + }, + }, + }, + }, + }, +}; + +const listsMappings = { + dynamic: 'strict', + properties: { + name: { + type: 'keyword', + }, + deserializer: { + type: 'keyword', + }, + serializer: { + type: 'keyword', + }, + description: { + type: 'keyword', + }, + type: { + type: 'keyword', + }, + tie_breaker_id: { + type: 'keyword', + }, + meta: { + enabled: 'false', + type: 'object', + }, + created_at: { + type: 'date', + }, + updated_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + immutable: { + type: 'boolean', + }, + }, +}; + +const itemsMappings = { + dynamic: 'strict', + properties: { + tie_breaker_id: { + type: 'keyword', + }, + list_id: { + type: 'keyword', + }, + deserializer: { + type: 'keyword', + }, + serializer: { + type: 'keyword', + }, + meta: { + enabled: 'false', + type: 'object', + }, + created_at: { + type: 'date', + }, + updated_at: { + type: 'date', + }, + created_by: { + type: 'keyword', + }, + updated_by: { + type: 'keyword', + }, + ip: { + type: 'ip', + }, + keyword: { + type: 'keyword', + }, + }, +}; + /** * Convenience function for creating legacy index templates to * test out logic updating to new index templates * @param es es client */ export const createLegacyListsIndices = async (es: Client) => { - await setPolicy(es, '.lists-default', { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_size: '50gb', - }, - }, - }, - }, - }, - }); - await setPolicy(es, '.items-default', { - policy: { - phases: { - hot: { - min_age: '0ms', - actions: { - rollover: { - max_size: '50gb', - }, - }, - }, - }, - }, - }); + await setPolicy(es, '.lists-default', testPolicy); + await setPolicy(es, '.items-default', testPolicy); await setTemplate(es, '.lists-default', { index_patterns: [`.lists-default-*`], - mappings: { - dynamic: 'strict', - properties: { - name: { - type: 'keyword', - }, - deserializer: { - type: 'keyword', - }, - serializer: { - type: 'keyword', - }, - description: { - type: 'keyword', - }, - type: { - type: 'keyword', - }, - tie_breaker_id: { - type: 'keyword', - }, - meta: { - enabled: 'false', - type: 'object', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - created_by: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - version: { - type: 'keyword', - }, - immutable: { - type: 'boolean', - }, - }, - }, + mappings: listsMappings, settings: { index: { lifecycle: { @@ -510,42 +548,7 @@ export const createLegacyListsIndices = async (es: Client) => { }); await setTemplate(es, '.items-default', { index_patterns: [`.items-default-*`], - mappings: { - dynamic: 'strict', - properties: { - tie_breaker_id: { - type: 'keyword', - }, - list_id: { - type: 'keyword', - }, - deserializer: { - type: 'keyword', - }, - serializer: { - type: 'keyword', - }, - meta: { - enabled: 'false', - type: 'object', - }, - created_at: { - type: 'date', - }, - updated_at: { - type: 'date', - }, - created_by: { - type: 'keyword', - }, - updated_by: { - type: 'keyword', - }, - ip: { - type: 'ip', - }, - }, - }, + mappings: itemsMappings, settings: { index: { lifecycle: { @@ -558,3 +561,126 @@ export const createLegacyListsIndices = async (es: Client) => { await createBootstrapIndex(es, '.lists-default'); await createBootstrapIndex(es, '.items-default'); }; + +/** + * Utility to create list indices, before they were migrated to data streams + * @param es ES client + */ +export const createListsIndices = async (es: Client) => { + await setPolicy(es, '.lists-default', testPolicy); + await setPolicy(es, '.items-default', testPolicy); + await setIndexTemplate(es, '.lists-default', { + index_patterns: [`.lists-default-*`], + template: { + mappings: listsMappings, + settings: { + index: { + lifecycle: { + name: '.lists-default', + rollover_alias: '.lists-default', + }, + }, + mapping: { + total_fields: { + limit: 10000, + }, + }, + }, + }, + }); + await setIndexTemplate(es, '.items-default', { + index_patterns: [`.items-default-*`], + template: { + mappings: itemsMappings, + settings: { + index: { + lifecycle: { + name: '.items-default', + rollover_alias: '.items-default', + }, + }, + mapping: { + total_fields: { + limit: 10000, + }, + }, + }, + }, + }); + await createBootstrapIndex(es, '.lists-default'); + await createBootstrapIndex(es, '.items-default'); +}; + +/** + * utility to create list directly by using ES, bypassing all checks + * useful, to create list in legacy indices + */ +export const createListBypassingChecks = async ({ es, id }: { es: Client; id: string }) => { + const createdAt = new Date().toISOString(); + const body = { + created_at: createdAt, + created_by: 'mock-user', + description: 'mock-description', + name: 'mock-name', + tie_breaker_id: uuidv4(), + type: 'keyword', + updated_at: createdAt, + updated_by: 'mock-user', + immutable: false, + version: 1, + }; + + const response = await es.create({ + body, + id, + index: '.lists-default', + refresh: 'wait_for', + }); + + return { + _version: encodeHitVersion(response), + id: response._id, + ...body, + }; +}; + +/** + * utility to create list item directly by using ES, bypassing all checks + * useful, to create list item in legacy indices + * supports keyword only + */ +export const createListItemBypassingChecks = async ({ + es, + listId, + id, + value, +}: { + es: Client; + listId: string; + id: string; + value: string; +}) => { + const createdAt = new Date().toISOString(); + const body = { + created_at: createdAt, + created_by: 'mock-user', + tie_breaker_id: uuidv4(), + updated_at: createdAt, + updated_by: 'mock-user', + list_id: listId, + keyword: value, + }; + + const response = await es.create({ + body, + id, + index: '.items-default', + refresh: 'wait_for', + }); + + return { + _version: encodeHitVersion(response), + id: response._id, + ...body, + }; +};