[7.x] [SecuritySolution][Detections] Adds SavedObject persistence to Signals Migrations (#85690)

x-pack/plugins/lists/public/lists/api.ts

@@ -123,7 +123,7 @@ const importListWithValidation = async ({
     ),
     chain((payload) => tryCatch(() => importList({ http, signal, ...payload }), toError)),
     chain((response) => fromEither(validateEither(listSchema, response))),
-    flow(toPromise)
+    toPromise
   );
 
 export { importListWithValidation as importList };

x-pack/plugins/security_solution/common/detection_engine/schemas/request/delete_signals_migration_schema.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+export const deleteSignalsMigrationSchema = t.exact(
+  t.type({
+    migration_ids: t.array(t.string),
+  })
+);
+
+export type DeleteSignalsMigrationSchema = t.TypeOf<typeof deleteSignalsMigrationSchema>;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.mock.ts

@@ -7,6 +7,5 @@
 import { FinalizeSignalsMigrationSchema } from './finalize_signals_migration_schema';
 
 export const getFinalizeSignalsMigrationSchemaMock = (): FinalizeSignalsMigrationSchema => ({
-  migration_token:
-    'eyJkZXN0aW5hdGlvbkluZGV4IjoiZGVzdGluYXRpb25JbmRleCIsInNvdXJjZUluZGV4Ijoic291cmNlSW5kZXgiLCJ0YXNrSWQiOiJteS10YXNrLWlkIn0=',
+  migration_ids: ['migrationSOIdentifier'],
 });

x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.ts

@@ -6,13 +6,9 @@
 
 import * as t from 'io-ts';
 
-import { NonEmptyString } from '../types';
-
-const migrationToken = NonEmptyString;
-
 export const finalizeSignalsMigrationSchema = t.exact(
   t.type({
-    migration_token: migrationToken,
+    migration_ids: t.array(t.string),
   })
 );
 

x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_signals_migration_status_schema.mock.ts

@@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GetSignalsMigrationStatusSchema } from './get_signals_migration_status_schema';
+
+export const getSignalsMigrationStatusSchemaMock = (): GetSignalsMigrationStatusSchema => ({
+  from: 'now-30d',
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_migration_status_schema.ts → x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_signals_migration_status_schema.ts

@@ -8,10 +8,10 @@ import * as t from 'io-ts';
 
 import { from } from '../common/schemas';
 
-export const getMigrationStatusSchema = t.exact(
+export const getSignalsMigrationStatusSchema = t.exact(
   t.type({
     from,
   })
 );
 
-export type GetMigrationStatusSchema = t.TypeOf<typeof getMigrationStatusSchema>;
+export type GetSignalsMigrationStatusSchema = t.TypeOf<typeof getSignalsMigrationStatusSchema>;

x-pack/plugins/security_solution/common/validate.ts

@@ -6,6 +6,7 @@
 
 import { fold, Either, mapLeft } from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/pipeable';
+import { fromEither, TaskEither } from 'fp-ts/lib/TaskEither';
 import * as t from 'io-ts';
 import { exactCheck } from './exact_check';
 import { formatErrors } from './format_errors';
@@ -33,3 +34,8 @@ export const validateEither = <T extends t.Mixed, A extends unknown>(
     (a) => schema.validate(a, t.getDefaultContext(schema.asDecoder())),
     mapLeft((errors) => new Error(formatErrors(errors).join(',')))
   );
+
+export const validateTaskEither = <T extends t.Mixed, A extends unknown>(
+  schema: T,
+  obj: A
+): TaskEither<Error, A> => fromEither(validateEither(schema, obj));

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration.test.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
+import { createMigrationIndex } from './create_migration_index';
+import { createMigration } from './create_migration';
+
+jest.mock('./create_migration_index');
+
+describe('createMigration', () => {
+  let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
+
+  beforeEach(() => {
+    esClient = elasticsearchServiceMock.createElasticsearchClient();
+  });
+
+  it('passes reindex options to the reindex call', async () => {
+    const reindexOptions = {
+      requests_per_second: 3,
+      size: 10,
+      slices: 2,
+    };
+    await createMigration({
+      esClient,
+      index: 'my-signals-index',
+      reindexOptions,
+      version: 12,
+    });
+
+    expect(esClient.reindex).toHaveBeenCalledWith(
+      expect.objectContaining({
+        body: expect.objectContaining({
+          source: {
+            index: 'my-signals-index',
+            size: reindexOptions.size,
+          },
+        }),
+        requests_per_second: reindexOptions.requests_per_second,
+        slices: reindexOptions.slices,
+      })
+    );
+  });
+
+  it('returns info about the created migration', async () => {
+    (createMigrationIndex as jest.Mock).mockResolvedValueOnce('destinationIndex');
+    // @ts-expect-error minimum stub for our reindex response
+    esClient.reindex.mockResolvedValueOnce({ body: { task: 'reindexTaskId' } });
+
+    const migration = await createMigration({
+      esClient,
+      index: 'my-signals-index',
+      reindexOptions: {},
+      version: 12,
+    });
+
+    expect(migration).toEqual({
+      destinationIndex: 'destinationIndex',
+      sourceIndex: 'my-signals-index',
+      taskId: 'reindexTaskId',
+      version: 12,
+    });
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migrate_signals.ts → x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration.ts

@@ -6,8 +6,14 @@
 
 import { ElasticsearchClient } from 'src/core/server';
 import { SignalsReindexOptions } from '../../../../common/detection_engine/schemas/request/create_signals_migration_schema';
-import { createSignalsMigrationIndex } from './create_signals_migration_index';
-import { MigrationDetails } from './types';
+import { createMigrationIndex } from './create_migration_index';
+
+export interface CreatedMigration {
+  destinationIndex: string;
+  sourceIndex: string;
+  taskId: string;
+  version: number;
+}
 
 /**
  * Migrates signals for a given concrete index. Signals are reindexed into a
@@ -19,10 +25,10 @@ import { MigrationDetails } from './types';
  * @param version version of the current signals template/mappings
  * @param reindexOptions object containing reindex options {@link SignalsReindexOptions}
  *
- * @returns identifying information representing the {@link MigrationDetails}
+ * @returns identifying information representing the {@link MigrationInfo}
  * @throws if elasticsearch returns an error
  */
-export const migrateSignals = async ({
+export const createMigration = async ({
   esClient,
   index,
   reindexOptions,
@@ -32,8 +38,8 @@ export const migrateSignals = async ({
   index: string;
   reindexOptions: SignalsReindexOptions;
   version: number;
-}): Promise<MigrationDetails> => {
-  const migrationIndex = await createSignalsMigrationIndex({
+}): Promise<CreatedMigration> => {
+  const migrationIndex = await createMigrationIndex({
     esClient,
     index,
     version,
@@ -67,5 +73,6 @@ export const migrateSignals = async ({
     destinationIndex: migrationIndex,
     sourceIndex: index,
     taskId: response.body.task,
+    version,
   };
 };

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.test.ts → x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.test.ts

@@ -4,19 +4,18 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { ElasticsearchClient } from 'src/core/server';
 import { elasticsearchServiceMock } from 'src/core/server/mocks';
-import { createSignalsMigrationIndex } from './create_signals_migration_index';
+import { createMigrationIndex } from './create_migration_index';
 
-describe('getMigrationStatus', () => {
-  let esClient: ElasticsearchClient;
+describe('createMigrationIndex', () => {
+  let esClient: ReturnType<typeof elasticsearchServiceMock.createElasticsearchClient>;
 
   beforeEach(() => {
     esClient = elasticsearchServiceMock.createElasticsearchClient();
   });
 
   it('creates an index suffixed with the template version', async () => {
-    await createSignalsMigrationIndex({ esClient, index: 'my-signals-index', version: 4 });
+    await createMigrationIndex({ esClient, index: 'my-signals-index', version: 4 });
 
     expect(esClient.indices.create).toHaveBeenCalledWith(
       expect.objectContaining({ index: 'my-signals-index-r000004' })

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.ts → x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_index.ts

@@ -19,7 +19,7 @@ import { ElasticsearchClient } from 'src/core/server';
  *
  * @returns the name of the created index
  */
-export const createSignalsMigrationIndex = async ({
+export const createMigrationIndex = async ({
   esClient,
   index,
   version,

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_saved_object.test.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { savedObjectsClientMock } from 'src/core/server/mocks';
+import { getSignalsMigrationSavedObjectMock } from './saved_objects_schema.mock';
+import { createMigrationSavedObject } from './create_migration_saved_object';
+
+describe('createMigrationSavedObjects', () => {
+  let soClient: ReturnType<typeof savedObjectsClientMock.create>;
+
+  beforeEach(() => {
+    soClient = savedObjectsClientMock.create();
+  });
+
+  it('returns the SavedObject if valid', () => {
+    // @ts-expect-error response mock is missing a few fields
+    soClient.create.mockResolvedValue(getSignalsMigrationSavedObjectMock());
+    const { attributes } = getSignalsMigrationSavedObjectMock();
+
+    return expect(
+      createMigrationSavedObject({ attributes, soClient, username: 'username' })
+    ).resolves.toEqual(getSignalsMigrationSavedObjectMock());
+  });
+
+  it('rejects if response is invalid', () => {
+    const { attributes } = getSignalsMigrationSavedObjectMock();
+    // @ts-expect-error stubbing our SO creation
+    soClient.create.mockResolvedValue({ ...getSignalsMigrationSavedObjectMock(), id: null });
+
+    return expect(
+      createMigrationSavedObject({ attributes, soClient, username: 'username' })
+    ).rejects.toThrow('Invalid value "null" supplied to "id"');
+  });
+
+  it('does not pass excess fields', async () => {
+    // @ts-expect-error response mock is missing a few fields
+    soClient.create.mockResolvedValue(getSignalsMigrationSavedObjectMock());
+    const { attributes } = getSignalsMigrationSavedObjectMock();
+    const attributesWithExtra = { ...attributes, extra: true };
+
+    const result = await createMigrationSavedObject({
+      attributes: attributesWithExtra,
+      soClient,
+      username: 'username',
+    });
+    expect(result).toEqual(getSignalsMigrationSavedObjectMock());
+
+    const [call] = soClient.create.mock.calls;
+    const attrs = call[1] as Record<string, unknown>;
+
+    expect(Object.keys(attrs)).not.toContain('extra');
+  });
+
+  it('rejects if attributes are invalid', () => {
+    const { attributes } = getSignalsMigrationSavedObjectMock();
+    // @ts-expect-error intentionally breaking the type
+    attributes.destinationIndex = null;
+
+    return expect(
+      createMigrationSavedObject({ attributes, soClient, username: 'username' })
+    ).rejects.toThrow('Invalid value "null" supplied to "destinationIndex"');
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_migration_saved_object.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { chain, tryCatch } from 'fp-ts/lib/TaskEither';
+import { pipe } from 'fp-ts/lib/pipeable';
+
+import { SavedObjectsClientContract } from 'src/core/server';
+import { validateTaskEither } from '../../../../common/validate';
+import { signalsMigrationSOClient } from './saved_objects_client';
+import {
+  signalsMigrationSO,
+  SignalsMigrationSO,
+  signalsMigrationSOCreateAttributes,
+  SignalsMigrationSOCreateAttributes,
+} from './saved_objects_schema';
+import { getIsoDateString, toError, toPromise } from './helpers';
+
+const generateAttributes = (username: string) => {
+  const now = getIsoDateString();
+  return { created: now, createdBy: username, updated: now, updatedBy: username };
+};
+
+export const createMigrationSavedObject = async ({
+  attributes,
+  soClient,
+  username,
+}: {
+  attributes: SignalsMigrationSOCreateAttributes;
+  soClient: SavedObjectsClientContract;
+  username: string;
+}): Promise<SignalsMigrationSO> => {
+  const client = signalsMigrationSOClient(soClient);
+
+  return pipe(
+    attributes,
+    (attrs) => validateTaskEither(signalsMigrationSOCreateAttributes, attrs),
+    chain((validAttrs) =>
+      tryCatch(() => client.create({ ...validAttrs, ...generateAttributes(username) }), toError)
+    ),
+    chain((so) => validateTaskEither(signalsMigrationSO, so)),
+    toPromise
+  );
+};