Merge branch 'master' into skip-ci-backportrc

src/plugins/data/server/saved_objects/migrations/to_v7_12_0.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { SavedObjectMigrationFn } from 'kibana/server';
+
+/**
+ * Drop the previous document's attributes, which report `averageDuration` incorrectly.
+ * @param doc
+ */
+export const migrate712: SavedObjectMigrationFn = (doc) => {
+  return { ...doc, attributes: {} };
+};

src/plugins/data/server/saved_objects/search_telemetry.ts

@@ -7,6 +7,7 @@
  */
 
 import { SavedObjectsType } from 'kibana/server';
+import { migrate712 } from './migrations/to_v7_12_0';
 
 export const searchTelemetry: SavedObjectsType = {
   name: 'search-telemetry',
@@ -16,4 +17,7 @@ export const searchTelemetry: SavedObjectsType = {
     dynamic: false,
     properties: {},
   },
+  migrations: {
+    '7.12.0': migrate712,
+  },
 };

src/plugins/data/server/search/collectors/fetch.ts

@@ -11,14 +11,14 @@ import { first } from 'rxjs/operators';
 import { SharedGlobalConfig } from 'kibana/server';
 import { SearchResponse } from 'elasticsearch';
 import { CollectorFetchContext } from 'src/plugins/usage_collection/server';
-import { Usage } from './register';
+import { CollectedUsage, ReportedUsage } from './register';
 interface SearchTelemetry {
-  'search-telemetry': Usage;
+  'search-telemetry': CollectedUsage;
 }
 type ESResponse = SearchResponse<SearchTelemetry>;
 
 export function fetchProvider(config$: Observable<SharedGlobalConfig>) {
-  return async ({ esClient }: CollectorFetchContext): Promise<Usage> => {
+  return async ({ esClient }: CollectorFetchContext): Promise<ReportedUsage> => {
     const config = await config$.pipe(first()).toPromise();
     const { body: esResponse } = await esClient.search<ESResponse>(
       {
@@ -37,6 +37,10 @@ export function fetchProvider(config$: Observable<SharedGlobalConfig>) {
         averageDuration: null,
       };
     }
-    return esResponse.hits.hits[0]._source['search-telemetry'];
+    const { successCount, errorCount, totalDuration } = esResponse.hits.hits[0]._source[
+      'search-telemetry'
+    ];
+    const averageDuration = totalDuration / successCount;
+    return { successCount, errorCount, averageDuration };
   };
 }

src/plugins/data/server/search/collectors/register.ts

@@ -10,7 +10,13 @@ import { PluginInitializerContext } from 'kibana/server';
 import { UsageCollectionSetup } from '../../../../usage_collection/server';
 import { fetchProvider } from './fetch';
 
-export interface Usage {
+export interface CollectedUsage {
+  successCount: number;
+  errorCount: number;
+  totalDuration: number;
+}
+
+export interface ReportedUsage {
   successCount: number;
   errorCount: number;
   averageDuration: number | null;
@@ -21,7 +27,7 @@ export async function registerUsageCollector(
   context: PluginInitializerContext
 ) {
   try {
-    const collector = usageCollection.makeUsageCollector<Usage>({
+    const collector = usageCollection.makeUsageCollector<ReportedUsage>({
       type: 'search',
       isReady: () => true,
       fetch: fetchProvider(context.config.legacy.globalConfig$),

src/plugins/data/server/search/collectors/usage.ts

@@ -6,63 +6,56 @@
  * Side Public License, v 1.
  */
 
+import { once } from 'lodash';
 import type { CoreSetup, Logger } from 'kibana/server';
+import { SavedObjectsErrorHelpers } from '../../../../../core/server';
 import type { IEsSearchResponse } from '../../../common';
-import type { Usage } from './register';
 
 const SAVED_OBJECT_ID = 'search-telemetry';
+const MAX_RETRY_COUNT = 3;
 
 export interface SearchUsage {
   trackError(): Promise<void>;
   trackSuccess(duration: number): Promise<void>;
 }
 
 export function usageProvider(core: CoreSetup): SearchUsage {
-  const getTracker = (eventType: keyof Usage) => {
-    return async (duration?: number) => {
-      const repository = await core
-        .getStartServices()
-        .then(([coreStart]) => coreStart.savedObjects.createInternalRepository());
+  const getRepository = once(async () => {
+    const [coreStart] = await core.getStartServices();
+    return coreStart.savedObjects.createInternalRepository();
+  });
 
-      let attributes: Usage;
-      let doesSavedObjectExist: boolean = true;
-
-      try {
-        const response = await repository.get<Usage>(SAVED_OBJECT_ID, SAVED_OBJECT_ID);
-        attributes = response.attributes;
-      } catch (e) {
-        doesSavedObjectExist = false;
-        attributes = {
-          successCount: 0,
-          errorCount: 0,
-          averageDuration: 0,
-        };
-      }
-
-      attributes[eventType]++;
-
-      // Only track the average duration for successful requests
-      if (eventType === 'successCount') {
-        attributes.averageDuration =
-          ((duration ?? 0) + (attributes.averageDuration ?? 0)) / (attributes.successCount ?? 1);
+  const trackSuccess = async (duration: number, retryCount = 0) => {
+    const repository = await getRepository();
+    try {
+      await repository.incrementCounter(SAVED_OBJECT_ID, SAVED_OBJECT_ID, [
+        { fieldName: 'successCount' },
+        {
+          fieldName: 'totalDuration',
+          incrementBy: duration,
+        },
+      ]);
+    } catch (e) {
+      if (SavedObjectsErrorHelpers.isConflictError(e) && retryCount < MAX_RETRY_COUNT) {
+        setTimeout(() => trackSuccess(duration, retryCount + 1), 1000);
       }
+    }
+  };
 
-      try {
-        if (doesSavedObjectExist) {
-          await repository.update(SAVED_OBJECT_ID, SAVED_OBJECT_ID, attributes);
-        } else {
-          await repository.create(SAVED_OBJECT_ID, attributes, { id: SAVED_OBJECT_ID });
-        }
-      } catch (e) {
-        // Version conflict error, swallow
+  const trackError = async (retryCount = 0) => {
+    const repository = await getRepository();
+    try {
+      await repository.incrementCounter(SAVED_OBJECT_ID, SAVED_OBJECT_ID, [
+        { fieldName: 'errorCount' },
+      ]);
+    } catch (e) {
+      if (SavedObjectsErrorHelpers.isConflictError(e) && retryCount < MAX_RETRY_COUNT) {
+        setTimeout(() => trackError(retryCount + 1), 1000);
       }
-    };
+    }
   };
 
-  return {
-    trackError: () => getTracker('errorCount')(),
-    trackSuccess: getTracker('successCount'),
-  };
+  return { trackSuccess, trackError };
 }
 
 /**

x-pack/plugins/fleet/common/constants/epm.ts

@@ -15,6 +15,7 @@ export const FLEET_SERVER_PACKAGE = 'fleet_server';
 export const requiredPackages = {
   System: 'system',
   Endpoint: 'endpoint',
+  ElasticAgent: 'elastic_agent',
 } as const;
 
 // these are currently identical. we can separate if they later diverge

x-pack/plugins/fleet/public/applications/fleet/sections/overview/components/agent_policy_section.tsx

@@ -31,7 +31,7 @@ export const OverviewPolicySection: React.FC<{ agentPolicies: AgentPolicy[] }> =
   });
 
   return (
-    <EuiFlexItem component="section">
+    <EuiFlexItem component="section" data-test-subj="fleet-agent-policy-section">
       <OverviewPanel
         title={i18n.translate('xpack.fleet.overviewPagePoliciesPanelTitle', {
           defaultMessage: 'Agent policies',

x-pack/plugins/fleet/public/applications/fleet/sections/overview/components/agent_section.tsx

@@ -24,7 +24,7 @@ export const OverviewAgentSection = () => {
   const agentStatusRequest = useGetAgentStatus({});
 
   return (
-    <EuiFlexItem component="section">
+    <EuiFlexItem component="section" data-test-subj="fleet-agent-section">
       <OverviewPanel
         title={i18n.translate('xpack.fleet.overviewPageAgentsPanelTitle', {
           defaultMessage: 'Agents',

x-pack/plugins/fleet/public/applications/fleet/sections/overview/components/datastream_section.tsx

@@ -45,7 +45,7 @@ export const OverviewDatastreamSection: React.FC = () => {
   }
 
   return (
-    <EuiFlexItem component="section">
+    <EuiFlexItem component="section" data-test-subj="fleet-datastream-section">
       <OverviewPanel
         title={i18n.translate('xpack.fleet.overviewPageDataStreamsPanelTitle', {
           defaultMessage: 'Data streams',

x-pack/plugins/fleet/public/applications/fleet/sections/overview/components/integration_section.tsx

@@ -31,7 +31,7 @@ export const OverviewIntegrationSection: React.FC = () => {
       (item) => 'savedObject' in item && item.version > item.savedObject.attributes.version
     )?.length ?? 0;
   return (
-    <EuiFlexItem component="section">
+    <EuiFlexItem component="section" data-test-subj="fleet-integrations-section">
       <OverviewPanel
         title={i18n.translate('xpack.fleet.overviewPageIntegrationsPanelTitle', {
           defaultMessage: 'Integrations',

x-pack/plugins/fleet/server/services/epm/elasticsearch/ingest_pipeline/remove.ts

@@ -8,6 +8,7 @@
 import { SavedObjectsClientContract } from 'src/core/server';
 import { appContextService } from '../../../';
 import { CallESAsCurrentUser, ElasticsearchAssetType } from '../../../../types';
+import { IngestManagerError } from '../../../../errors';
 import { getInstallation } from '../../packages/get';
 import { PACKAGES_SAVED_OBJECT_TYPE, EsAssetReference } from '../../../../../common';
 
@@ -61,7 +62,11 @@ export async function deletePipeline(callCluster: CallESAsCurrentUser, id: strin
     try {
       await callCluster('ingest.deletePipeline', { id });
     } catch (err) {
-      throw new Error(`error deleting pipeline ${id}`);
+      // Only throw if error is not a 404 error. Sometimes the pipeline is already deleted, but we have
+      // duplicate references to them, see https://github.com/elastic/kibana/issues/91192
+      if (err.statusCode !== 404) {
+        throw new IngestManagerError(`error deleting pipeline ${id}: ${err}`);
+      }
     }
   }
 }

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/install.ts

@@ -42,9 +42,13 @@ export const installTransform = async (
     previousInstalledTransformEsAssets = installation.installed_es.filter(
       ({ type, id }) => type === ElasticsearchAssetType.transform
     );
-    logger.info(
-      `Found previous transform references:\n ${JSON.stringify(previousInstalledTransformEsAssets)}`
-    );
+    if (previousInstalledTransformEsAssets.length) {
+      logger.info(
+        `Found previous transform references:\n ${JSON.stringify(
+          previousInstalledTransformEsAssets
+        )}`
+      );
+    }
   }
 
   // delete all previous transform

x-pack/plugins/fleet/server/services/epm/elasticsearch/transform/remove.ts

@@ -26,7 +26,9 @@ export const deleteTransforms = async (
   transformIds: string[]
 ) => {
   const logger = appContextService.getLogger();
-  logger.info(`Deleting currently installed transform ids ${transformIds}`);
+  if (transformIds.length) {
+    logger.info(`Deleting currently installed transform ids ${transformIds}`);
+  }
   await Promise.all(
     transformIds.map(async (transformId) => {
       // get the index the transform

x-pack/test/fleet_api_integration/apis/fleet_setup.ts

@@ -105,5 +105,19 @@ export default function (providerContext: FtrProviderContext) {
         transient_metadata: { enabled: true },
       });
     });
+
+    it('should install default packages', async () => {
+      await supertest.post(`/api/fleet/setup`).set('kbn-xsrf', 'xxxx').expect(200);
+
+      const { body: apiResponse } = await supertest
+        .get(`/api/fleet/epm/packages?experimental=true`)
+        .expect(200);
+      const installedPackages = apiResponse.response
+        .filter((p: any) => p.status === 'installed')
+        .map((p: any) => p.name)
+        .sort();
+
+      expect(installedPackages).to.eql(['elastic_agent', 'endpoint', 'system']);
+    });
   });
 }

x-pack/test/fleet_api_integration/config.ts

@@ -15,7 +15,7 @@ import { defineDockerServersConfig } from '@kbn/test';
 // example: https://beats-ci.elastic.co/blue/organizations/jenkins/Ingest-manager%2Fpackage-storage/detail/snapshot/74/pipeline/257#step-302-log-1.
 // It should be updated any time there is a new Docker image published for the Snapshot Distribution of the Package Registry.
 export const dockerImage =
-  'docker.elastic.co/package-registry/distribution:5314869e2f6bc01d37b8652f7bda89248950b3a4';
+  'docker.elastic.co/package-registry/distribution:99dadb957d76b704637150d34a7219345cc0aeef';
 
 export default async function ({ readConfigFile }: FtrConfigProviderContext) {
   const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));

x-pack/test/fleet_functional/apps/fleet/index.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function (providerContext: FtrProviderContext) {
+  const { loadTestFile } = providerContext;
+
+  describe('endpoint', function () {
+    this.tags('ciGroup7');
+    loadTestFile(require.resolve('./overview_page'));
+  });
+}

x-pack/test/fleet_functional/apps/fleet/overview_page.ts

@@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const { overviewPage } = getPageObjects(['overviewPage']);
+
+  describe('When in the Fleet application', function () {
+    this.tags(['ciGroup7']);
+
+    describe('and on the Overview page', () => {
+      before(async () => {
+        await overviewPage.navigateToOverview();
+      });
+
+      it('should show the Integrations section', async () => {
+        await overviewPage.integrationsSectionExistsOrFail();
+      });
+
+      it('should show the Agents section', async () => {
+        await overviewPage.agentSectionExistsOrFail();
+      });
+
+      it('should show the Agent policies section', async () => {
+        await overviewPage.agentPolicySectionExistsOrFail();
+      });
+
+      it('should show the Data streams section', async () => {
+        await overviewPage.datastreamSectionExistsOrFail();
+      });
+    });
+  });
+}