[Security Solution][Detections] Signals Migration API

x-pack/plugins/security_solution/common/constants.ts

@@ -136,9 +136,13 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged`;
  * Default signals index key for kibana.dev.yml
  */
 export const SIGNALS_INDEX_KEY = 'signalsIndex';
+
 export const DETECTION_ENGINE_SIGNALS_URL = `${DETECTION_ENGINE_URL}/signals`;
 export const DETECTION_ENGINE_SIGNALS_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/status`;
 export const DETECTION_ENGINE_QUERY_SIGNALS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/search`;
+export const DETECTION_ENGINE_SIGNALS_MIGRATION_URL = `${DETECTION_ENGINE_SIGNALS_URL}/migration`;
+export const DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/migration_status`;
+export const DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL = `${DETECTION_ENGINE_SIGNALS_URL}/finalize_migration`;
 
 /**
  * Common naming convention for an unauthenticated user

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.mock.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CreateSignalsMigrationSchema } from './create_signals_migration_schema';
+
+export const getCreateSignalsMigrationSchemaMock = (
+  index: string = 'signals-index'
+): CreateSignalsMigrationSchema => ({
+  index: [index],
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { index } from '../common/schemas';
+import { PositiveInteger, PositiveIntegerGreaterThanZero } from '../types';
+
+export const signalsReindexOptions = t.partial({
+  requests_per_second: t.number,
+  size: PositiveIntegerGreaterThanZero,
+  slices: PositiveInteger,
+});
+
+export type SignalsReindexOptions = t.TypeOf<typeof signalsReindexOptions>;
+
+export const createSignalsMigrationSchema = t.intersection([
+  t.exact(
+    t.type({
+      index,
+    })
+  ),
+  t.exact(signalsReindexOptions),
+]);
+
+export type CreateSignalsMigrationSchema = t.TypeOf<typeof createSignalsMigrationSchema>;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.mock.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { FinalizeSignalsMigrationSchema } from './finalize_signals_migration_schema';
+
+export const getFinalizeSignalsMigrationSchemaMock = (): FinalizeSignalsMigrationSchema => ({
+  migration_token:
+    'eyJkZXN0aW5hdGlvbkluZGV4IjoiZGVzdGluYXRpb25JbmRleCIsInNvdXJjZUluZGV4Ijoic291cmNlSW5kZXgiLCJ0YXNrSWQiOiJteS10YXNrLWlkIn0=',
+});

x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.ts

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { NonEmptyString } from '../types';
+
+const migrationToken = NonEmptyString;
+
+export const finalizeSignalsMigrationSchema = t.exact(
+  t.type({
+    migration_token: migrationToken,
+  })
+);
+
+export type FinalizeSignalsMigrationSchema = t.TypeOf<typeof finalizeSignalsMigrationSchema>;

x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_migration_status_schema.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as t from 'io-ts';
+
+import { from } from '../common/schemas';
+
+export const getMigrationStatusSchema = t.exact(
+  t.type({
+    from,
+  })
+);
+
+export type GetMigrationStatusSchema = t.TypeOf<typeof getMigrationStatusSchema>;

x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_aliases.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchClient } from 'src/core/server';
+
+interface AliasesResponse {
+  [indexName: string]: {
+    aliases: {
+      [aliasName: string]: {
+        is_write_index: boolean;
+      };
+    };
+  };
+}
+
+interface IndexAlias {
+  alias: string;
+  index: string;
+  isWriteIndex: boolean;
+}
+
+/**
+ * Retrieves all index aliases for a given alias name
+ *
+ * @param esClient An {@link ElasticsearchClient}
+ * @param alias alias name used to filter results
+ *
+ * @returns an array of {@link IndexAlias} objects
+ */
+export const getIndexAliases = async ({
+  esClient,
+  alias,
+}: {
+  esClient: ElasticsearchClient;
+  alias: string;
+}): Promise<IndexAlias[]> => {
+  const response = await esClient.indices.getAlias<AliasesResponse>({
+    name: alias,
+  });
+
+  return Object.keys(response.body).map((index) => ({
+    alias,
+    index,
+    isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true,
+  }));
+};

x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_count.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchClient } from 'src/core/server';
+
+/**
+ * Retrieves the count of documents in a given index
+ *
+ * @param esClient An {@link ElasticsearchClient}
+ * @param index index whose documents will be counted
+ *
+ * @returns the document count
+ */
+export const getIndexCount = async ({
+  esClient,
+  index,
+}: {
+  esClient: ElasticsearchClient;
+  index: string;
+}): Promise<number> => {
+  const response = await esClient.count<{ count: number }>({
+    index,
+  });
+
+  return response.body.count;
+};

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.test.ts

@@ -0,0 +1,25 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchClient } from 'src/core/server';
+import { elasticsearchServiceMock } from 'src/core/server/mocks';
+import { createSignalsMigrationIndex } from './create_signals_migration_index';
+
+describe('getMigrationStatus', () => {
+  let esClient: ElasticsearchClient;
+
+  beforeEach(() => {
+    esClient = elasticsearchServiceMock.createElasticsearchClient();
+  });
+
+  it('creates an index suffixed with the template version', async () => {
+    await createSignalsMigrationIndex({ esClient, index: 'my-signals-index', version: 4 });
+
+    expect(esClient.indices.create).toHaveBeenCalledWith(
+      expect.objectContaining({ index: 'my-signals-index-r000004' })
+    );
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.ts

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { ElasticsearchClient } from 'src/core/server';
+
+/**
+ * Creates the destination index to be used during the migration of a
+ * given signals index.
+ *
+ * The destination index's name is determined by adding a suffix of
+ * `-r${templateVersion}` to the source index name
+ *
+ * @param esClient An {@link ElasticsearchClient}
+ * @param index name of the source signals index
+ * @param version version of the current signals template/mappings
+ *
+ * @returns the name of the created index
+ */
+export const createSignalsMigrationIndex = async ({
+  esClient,
+  index,
+  version,
+}: {
+  esClient: ElasticsearchClient;
+  index: string;
+  version: number;
+}): Promise<string> => {
+  const paddedVersion = `${version}`.padStart(6, '0');
+  const destinationIndexName = `${index}-r${paddedVersion}`;
+
+  const response = await esClient.indices.create<{ index: string }>({
+    index: destinationIndexName,
+    body: {
+      settings: {
+        index: {
+          lifecycle: {
+            indexing_complete: true,
+          },
+        },
+      },
+    },
+  });
+
+  return response.body.index;
+};

x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.mock.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IndexMappingsResponse, MigrationStatusSearchResponse } from './types';
+
+export const getMigrationStatusSearchResponseMock = (
+  indices: string[] = ['signals-index'],
+  signalVersions: number[] = [-1]
+): MigrationStatusSearchResponse => ({
+  aggregations: {
+    signals_indices: {
+      buckets: indices.map((index) => ({
+        key: index,
+        signal_versions: {
+          buckets: signalVersions.map((version) => ({
+            key: version,
+            doc_count: 4,
+          })),
+        },
+      })),
+    },
+  },
+});
+
+export const getIndexMappingsResponseMock = (
+  index: string = 'signals-index'
+): IndexMappingsResponse => ({
+  [index]: { mappings: { _meta: { version: -1 } } },
+});