[Endpoint] Fix endpoint tests with data streams

src/es_archiver/actions/load.ts

@@ -49,13 +49,15 @@ const pipeline = (...streams: Readable[]) =>
 export async function loadAction({
   name,
   skipExisting,
+  useCreate,
   client,
   dataDir,
   log,
   kbnClient,
 }: {
   name: string;
   skipExisting: boolean;
+  useCreate: boolean;
   client: Client;
   dataDir: string;
   log: ToolingLog;
@@ -87,7 +89,7 @@ export async function loadAction({
   await createPromiseFromStreams([
     recordStream,
     createCreateIndexStream({ client, stats, skipExisting, log }),
-    createIndexDocRecordsStream(client, stats, progress),
+    createIndexDocRecordsStream(client, stats, progress, useCreate),
   ]);
 
   progress.deactivate();

src/es_archiver/es_archiver.ts

@@ -80,12 +80,20 @@ export class EsArchiver {
    *  @param {Object} options
    *  @property {Boolean} options.skipExisting - should existing indices
    *                                           be ignored or overwritten
+   *  @property {Boolean} options.useCreate - use a create operation instead of index for documents
    *  @return Promise<Stats>
    */
-  async load(name: string, { skipExisting = false }: { skipExisting?: boolean } = {}) {
+  async load(
+    name: string,
+    {
+      skipExisting = false,
+      useCreate = false,
+    }: { skipExisting?: boolean; useCreate?: boolean } = {}
+  ) {
     return await loadAction({
       name,
       skipExisting: !!skipExisting,
+      useCreate: !!useCreate,
       client: this.client,
       dataDir: this.dataDir,
       log: this.log,

src/es_archiver/lib/docs/index_doc_records_stream.ts

@@ -22,15 +22,20 @@ import { Writable } from 'stream';
 import { Stats } from '../stats';
 import { Progress } from '../progress';
 
-export function createIndexDocRecordsStream(client: Client, stats: Stats, progress: Progress) {
+export function createIndexDocRecordsStream(
+  client: Client,
+  stats: Stats,
+  progress: Progress,
+  useCreate: boolean
+) {
   async function indexDocs(docs: any[]) {
     const body: any[] = [];
-
+    const operation = useCreate === true ? 'create' : 'index';
     docs.forEach((doc) => {
       stats.indexedDoc(doc.index);
       body.push(
         {
-          index: {
+          [operation]: {
             _index: doc.index,
             _id: doc.id,
           },

x-pack/test/api_integration/apis/endpoint/alerts/index.ts

@@ -7,6 +7,7 @@ import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../../ftr_provider_context';
 import { AlertData } from '../../../../../plugins/security_solution/common/endpoint_alerts/types';
 import { AlertId } from '../../../../../plugins/security_solution/server/endpoint/alerts/handlers/lib/index';
+import { deleteEventsStream, deleteMetadataStream } from '../data_stream_helper';
 
 /**
  * The number of alert documents in the es archive.
@@ -66,26 +67,26 @@ export default function ({ getService }: FtrProviderContext) {
   const nextPrevPrefixOrder = 'order=desc';
   const nextPrevPrefixPageSize = 'page_size=10';
   const nextPrevPrefix = `${nextPrevPrefixQuery}&${nextPrevPrefixDateRange}&${nextPrevPrefixSort}&${nextPrevPrefixOrder}&${nextPrevPrefixPageSize}`;
-  const alertIndex = 'events-endpoint-1';
+  const alertIndex = '.ds-events-endpoint-1-000001';
 
   let nullableEventId = '';
 
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68613
-  describe.skip('Endpoint alert API', () => {
+  describe('Endpoint alert API', () => {
     describe('when data is in elasticsearch', () => {
       before(async () => {
-        await esArchiver.load('endpoint/alerts/api_feature');
-        await esArchiver.load('endpoint/alerts/host_api_feature');
+        await esArchiver.load('endpoint/alerts/api_feature', { useCreate: true });
+        await esArchiver.load('endpoint/alerts/host_api_feature', { useCreate: true });
         const res = await es.search({
-          index: alertIndex,
+          index: 'events-endpoint-*',
           body: ES_QUERY_MISSING,
         });
         nullableEventId = res.hits.hits[0]._source.event.id;
       });
 
       after(async () => {
-        await esArchiver.unload('endpoint/alerts/api_feature');
-        await esArchiver.unload('endpoint/alerts/host_api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await Promise.all([deleteEventsStream(getService), deleteMetadataStream(getService)]);
       });
 
       it('should not support POST requests', async () => {

x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+
+export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
+  const client = getService('es');
+  await client.transport.request(
+    {
+      method: 'DELETE',
+      path: `_data_stream/${index}`,
+    },
+    {
+      ignore: [404],
+    }
+  );
+}
+
+export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.metadata-*');
+}
+
+export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'events-endpoint-*');
+}
+
+export async function deletePolicyStream(getService: (serviceName: 'es') => Client) {
+  await deleteDataStream(getService, 'metrics-endpoint.policy-*');
+}

x-pack/test/api_integration/apis/endpoint/metadata.ts

@@ -5,6 +5,7 @@
  */
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deleteMetadataStream } from './data_stream_helper';
 
 /**
  * The number of host documents in the es archive.
@@ -14,11 +15,12 @@ const numberOfHostsInFixture = 3;
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing on ES PROMOTION: https://github.com/elastic/kibana/issues/68584
-  describe.skip('test metadata api', () => {
+  describe('test metadata api', () => {
     describe('POST /api/endpoint/metadata when index is empty', () => {
       it('metadata api should return empty result when index is empty', async () => {
-        await esArchiver.unload('endpoint/metadata/api_feature');
+        // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+        // to do it manually
+        await deleteMetadataStream(getService);
         const { body } = await supertest
           .post('/api/endpoint/metadata')
           .set('kbn-xsrf', 'xxx')
@@ -32,8 +34,12 @@ export default function ({ getService }: FtrProviderContext) {
     });
 
     describe('POST /api/endpoint/metadata when index is not empty', () => {
-      before(() => esArchiver.load('endpoint/metadata/api_feature'));
-      after(() => esArchiver.unload('endpoint/metadata/api_feature'));
+      before(
+        async () => await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true })
+      );
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deleteMetadataStream(getService));
       it('metadata api should return one entry for each host with default paging', async () => {
         const { body } = await supertest
           .post('/api/endpoint/metadata')

x-pack/test/api_integration/apis/endpoint/policy.ts

@@ -6,16 +6,18 @@
 
 import expect from '@kbn/expect/expect.js';
 import { FtrProviderContext } from '../../ftr_provider_context';
+import { deletePolicyStream } from './data_stream_helper';
 
 export default function ({ getService }: FtrProviderContext) {
   const esArchiver = getService('esArchiver');
   const supertest = getService('supertest');
-  // SKIPPED as it is failing ES PROMOTION: https://github.com/elastic/kibana/issues/68638
-  describe.skip('Endpoint policy api', () => {
+  describe('Endpoint policy api', () => {
     describe('GET /api/endpoint/policy_response', () => {
-      before(async () => await esArchiver.load('endpoint/policy'));
+      before(async () => await esArchiver.load('endpoint/policy', { useCreate: true }));
 
-      after(async () => await esArchiver.unload('endpoint/policy'));
+      // the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
+      // to do it manually
+      after(async () => await deletePolicyStream(getService));
 
       it('should return one policy response for host', async () => {
         const expectedHostId = '4f3b9858-a96d-49d8-a326-230d7763d767';

x-pack/test/api_integration/apis/endpoint/resolver.ts

@@ -216,6 +216,7 @@ export default function resolverAPIIntegrationTests({ getService }: FtrProviderC
     });
     after(async () => {
       await resolver.deleteTrees(resolverTrees);
+      // this unload is for an endgame-* index so it does not use data streams
       await esArchiver.unload('endpoint/resolver/api_feature');
     });