[Fleet] Rely on package content during installation

x-pack/plugins/apm/server/routes/fleet/get_latest_apm_package.ts

@@ -20,7 +20,7 @@ export async function getLatestApmPackage({
   const { name, version } = await packageClient.fetchFindLatestPackage(
     APM_PACKAGE_NAME
   );
-  const registryPackage = await packageClient.getRegistryPackage(name, version);
+  const registryPackage = await packageClient.getPackage(name, version);
   const { title, policy_templates: policyTemplates } =
     registryPackage.packageInfo;
   const firstTemplate = policyTemplates?.[0];

x-pack/plugins/fleet/server/integration_tests/validate_bundled_packages.test.ts

@@ -50,7 +50,7 @@ describe('validate bundled packages', () => {
       bundledPackages
         .filter((pkg) => !EXCLUDED_PACKAGES.includes(pkg.name))
         .map(async (bundledPackage) => {
-          const registryPackage = await Registry.getRegistryPackage(
+          const registryPackage = await Registry.getPackage(
             bundledPackage.name,
             bundledPackage.version
           );

x-pack/plugins/fleet/server/services/epm/package_service.mock.ts

@@ -11,7 +11,7 @@ const createClientMock = (): jest.Mocked<PackageClient> => ({
   getInstallation: jest.fn(),
   ensureInstalledPackage: jest.fn(),
   fetchFindLatestPackage: jest.fn(),
-  getRegistryPackage: jest.fn(),
+  getPackage: jest.fn(),
   reinstallEsAssets: jest.fn(),
 });
 

x-pack/plugins/fleet/server/services/epm/package_service.test.ts

@@ -31,7 +31,7 @@ const testKeys = [
   'getInstallation',
   'ensureInstalledPackage',
   'fetchFindLatestPackage',
-  'getRegistryPackage',
+  'getPackage',
   'reinstallEsAssets',
 ];
 
@@ -99,16 +99,16 @@ function getTest(
       break;
     case testKeys[3]:
       test = {
-        method: mocks.packageClient.getRegistryPackage.bind(mocks.packageClient),
+        method: mocks.packageClient.getPackage.bind(mocks.packageClient),
         args: ['package name', '8.0.0'],
-        spy: jest.spyOn(epmRegistry, 'getRegistryPackage'),
+        spy: jest.spyOn(epmRegistry, 'getPackage'),
         spyArgs: ['package name', '8.0.0', undefined],
         spyResponse: {
-          packageInfo: { name: 'getRegistryPackage test' },
+          packageInfo: { name: 'getPackage test' },
           paths: ['/some/test/path'],
         },
         expectedReturnValue: {
-          packageInfo: { name: 'getRegistryPackage test' },
+          packageInfo: { name: 'getPackage test' },
           paths: ['/some/test/path'],
         },
       };

x-pack/plugins/fleet/server/services/epm/package_service.ts

@@ -19,13 +19,14 @@ import type {
   InstallablePackage,
   Installation,
   RegistryPackage,
+  ArchivePackage,
   BundledPackage,
 } from '../../types';
 import { checkSuperuser } from '../../routes/security';
 import { FleetUnauthorizedError } from '../../errors';
 
 import { installTransforms, isTransform } from './elasticsearch/transform/install';
-import { fetchFindLatestPackageOrThrow, getRegistryPackage } from './registry';
+import { fetchFindLatestPackageOrThrow, getPackage } from './registry';
 import { ensureInstalledPackage, getInstallation } from './packages';
 
 export type InstalledAssetType = EsAssetReference;
@@ -46,10 +47,10 @@ export interface PackageClient {
 
   fetchFindLatestPackage(packageName: string): Promise<RegistryPackage | BundledPackage>;
 
-  getRegistryPackage(
+  getPackage(
     packageName: string,
     packageVersion: string
-  ): Promise<{ packageInfo: RegistryPackage; paths: string[] }>;
+  ): Promise<{ packageInfo: ArchivePackage; paths: string[] }>;
 
   reinstallEsAssets(
     packageInfo: InstallablePackage,
@@ -120,13 +121,13 @@ class PackageClientImpl implements PackageClient {
     return fetchFindLatestPackageOrThrow(packageName);
   }
 
-  public async getRegistryPackage(
+  public async getPackage(
     packageName: string,
     packageVersion: string,
-    options?: Parameters<typeof getRegistryPackage>['2']
+    options?: Parameters<typeof getPackage>['2']
   ) {
     await this.#runPreflight();
-    return getRegistryPackage(packageName, packageVersion, options);
+    return getPackage(packageName, packageVersion, options);
   }
 
   public async reinstallEsAssets(

x-pack/plugins/fleet/server/services/epm/packages/get.test.ts

@@ -188,7 +188,7 @@ describe('When using EPM `get` services', () => {
         name: 'my-package',
         version: '1.0.0',
       } as RegistryPackage);
-      MockRegistry.getRegistryPackage.mockResolvedValue({
+      MockRegistry.getPackage.mockResolvedValue({
         paths: [],
         packageInfo: {
           name: 'my-package',
@@ -366,7 +366,7 @@ describe('When using EPM `get` services', () => {
           status: 'not_installed',
         });
 
-        expect(MockRegistry.getRegistryPackage).not.toHaveBeenCalled();
+        expect(MockRegistry.getPackage).not.toHaveBeenCalled();
       });
 
       // when calling the get package endpoint without a package version we
@@ -392,7 +392,7 @@ describe('When using EPM `get` services', () => {
           status: 'not_installed',
         });
 
-        expect(MockRegistry.getRegistryPackage).not.toHaveBeenCalled();
+        expect(MockRegistry.getPackage).not.toHaveBeenCalled();
       });
     });
   });

x-pack/plugins/fleet/server/services/epm/packages/get.ts

@@ -170,7 +170,6 @@ export async function getPackageInfo({
       pkgVersion: resolvedPkgVersion,
       savedObjectsClient,
       installedPkg: savedObject?.attributes,
-      getPkgInfoFromArchive: packageInfo?.type === 'input',
       ignoreUnverified,
     }));
   }
@@ -235,13 +234,12 @@ interface PackageResponse {
 }
 type GetPackageResponse = PackageResponse | undefined;
 
-// gets package from install_source if it exists otherwise gets from registry
+// gets package from install_source
 export async function getPackageFromSource(options: {
   pkgName: string;
   pkgVersion: string;
   installedPkg?: Installation;
   savedObjectsClient: SavedObjectsClientContract;
-  getPkgInfoFromArchive?: boolean;
   ignoreUnverified?: boolean;
 }): Promise<PackageResponse> {
   const logger = appContextService.getLogger();
@@ -250,7 +248,6 @@ export async function getPackageFromSource(options: {
     pkgVersion,
     installedPkg,
     savedObjectsClient,
-    getPkgInfoFromArchive = true,
     ignoreUnverified = false,
   } = options;
   let res: GetPackageResponse;
@@ -280,11 +277,12 @@ export async function getPackageFromSource(options: {
         logger.debug(`retrieved installed package ${pkgName}-${pkgVersion} from ES`);
       }
     }
-    // for packages not in cache or package storage and installed from registry, check registry
+    // install source is now archive in all cases
+    // See https://github.com/elastic/kibana/issues/115032
     if (!res && pkgInstallSource === 'registry') {
       try {
-        res = await Registry.getRegistryPackage(pkgName, pkgVersion);
-        logger.debug(`retrieved installed package ${pkgName}-${pkgVersion} from registry`);
+        res = await Registry.getPackage(pkgName, pkgVersion);
+        logger.debug(`retrieved installed package ${pkgName}-${pkgVersion}`);
       } catch (error) {
         if (error instanceof PackageFailedVerificationError) {
           throw error;
@@ -294,12 +292,8 @@ export async function getPackageFromSource(options: {
       }
     }
   } else {
-    // else package is not installed or installed and missing from cache and storage and installed from registry
-    res = await Registry.getRegistryPackage(pkgName, pkgVersion, {
-      getPkgInfoFromArchive,
-      ignoreUnverified,
-    });
-    logger.debug(`retrieved uninstalled package ${pkgName}-${pkgVersion} from registry`);
+    res = await Registry.getPackage(pkgName, pkgVersion, { ignoreUnverified });
+    logger.debug(`retrieved uninstalled package ${pkgName}-${pkgVersion}`);
   }
   if (!res) {
     throw new FleetError(`package info for ${pkgName}-${pkgVersion} does not exist`);

x-pack/plugins/fleet/server/services/epm/packages/install.test.ts

@@ -84,7 +84,7 @@ describe('install', () => {
       .spyOn(Registry, 'fetchFindLatestPackageOrThrow')
       .mockImplementation(() => Promise.resolve({ version: '1.3.0' } as any));
     jest
-      .spyOn(Registry, 'getRegistryPackage')
+      .spyOn(Registry, 'getPackage')
       .mockImplementation(() => Promise.resolve({ packageInfo: { license: 'basic' } } as any));
 
     mockGetBundledPackages.mockReset();

x-pack/plugins/fleet/server/services/epm/packages/install.ts

@@ -302,7 +302,7 @@ async function installPackageFromRegistry({
       Registry.fetchFindLatestPackageOrThrow(pkgName, {
         ignoreConstraints,
       }),
-      Registry.getRegistryPackage(pkgName, pkgVersion, {
+      Registry.getPackage(pkgName, pkgVersion, {
         ignoreUnverified: force && !neverIgnoreVerificationError,
       }),
     ]);

x-pack/plugins/fleet/server/services/epm/registry/index.ts

@@ -250,35 +250,59 @@ export async function getInfo(name: string, version: string) {
   });
 }
 
-export async function getRegistryPackage(
+// Check that the packageInfo exists in cache
+// If not, retrieve it from the archive
+async function getPackageInfoFromArchiveOrCache(
   name: string,
   version: string,
-  options?: { ignoreUnverified?: boolean; getPkgInfoFromArchive?: boolean }
+  archiveBuffer: Buffer,
+  archivePath: string
+): Promise<ArchivePackage> {
+  const cachedInfo = getPackageInfo({ name, version });
+
+  if (!cachedInfo) {
+    const { packageInfo } = await generatePackageInfoFromArchiveBuffer(
+      archiveBuffer,
+      ensureContentType(archivePath)
+    );
+    setPackageInfo({ packageInfo, name, version });
+    return packageInfo;
+  } else {
+    return cachedInfo;
+  }
+}
+
+export async function getPackage(
+  name: string,
+  version: string,
+  options?: { ignoreUnverified?: boolean }
 ): Promise<{
   paths: string[];
-  packageInfo: RegistryPackage;
+  packageInfo: ArchivePackage;
   verificationResult?: PackageVerificationResult;
 }> {
   const verifyPackage = appContextService.getExperimentalFeatures().packageVerification;
   let paths = getArchiveFilelist({ name, version });
   let verificationResult = verifyPackage ? getVerificationResult({ name, version }) : undefined;
+
+  const {
+    archiveBuffer,
+    archivePath,
+    verificationResult: latestVerificationResult,
+  } = await withPackageSpan('Fetch package archive from archive buffer', () =>
+    fetchArchiveBuffer({
+      pkgName: name,
+      pkgVersion: version,
+      shouldVerify: verifyPackage,
+      ignoreUnverified: options?.ignoreUnverified,
+    })
+  );
+
+  if (latestVerificationResult) {
+    verificationResult = latestVerificationResult;
+    setVerificationResult({ name, version }, latestVerificationResult);
+  }
   if (!paths || paths.length === 0) {
-    const {
-      archiveBuffer,
-      archivePath,
-      verificationResult: latestVerificationResult,
-    } = await withPackageSpan('Fetch package archive from registry', () =>
-      fetchArchiveBuffer({
-        pkgName: name,
-        pkgVersion: version,
-        shouldVerify: verifyPackage,
-        ignoreUnverified: options?.ignoreUnverified,
-      })
-    );
-    if (latestVerificationResult) {
-      verificationResult = latestVerificationResult;
-      setVerificationResult({ name, version }, latestVerificationResult);
-    }
     paths = await withPackageSpan('Unpack archive', () =>
       unpackBufferToCache({
         name,
@@ -287,17 +311,14 @@ export async function getRegistryPackage(
         contentType: ensureContentType(archivePath),
       })
     );
-    const cachedInfo = getPackageInfo({ name, version });
-    if (options?.getPkgInfoFromArchive && !cachedInfo) {
-      const { packageInfo } = await generatePackageInfoFromArchiveBuffer(
-        archiveBuffer,
-        ensureContentType(archivePath)
-      );
-      setPackageInfo({ packageInfo, name, version });
-    }
   }
 
-  const packageInfo = await getInfo(name, version);
+  const packageInfo = await getPackageInfoFromArchiveOrCache(
+    name,
+    version,
+    archiveBuffer,
+    archivePath
+  );
   return { paths, packageInfo, verificationResult };
 }
 

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts

@@ -277,7 +277,7 @@ describe('check metadata transforms task', () => {
     });
 
     describe('transforms reinstall', () => {
-      let getRegistryPackageSpy: jest.SpyInstance;
+      let getPackageSpy: jest.SpyInstance;
       let reinstallEsAssetsSpy: jest.SpyInstance;
       let mockPackageClient: jest.Mocked<PackageClient>;
 
@@ -291,7 +291,7 @@ describe('check metadata transforms task', () => {
 
         mockPackageClient = mockEndpointAppContext.service.getInternalFleetServices()
           .packages as jest.Mocked<PackageClient>;
-        getRegistryPackageSpy = jest.spyOn(mockPackageClient, 'getRegistryPackage');
+        getPackageSpy = jest.spyOn(mockPackageClient, 'getPackage');
         reinstallEsAssetsSpy = jest.spyOn(mockPackageClient, 'reinstallEsAssets');
 
         const transformStatsResponseMock = {
@@ -313,7 +313,7 @@ describe('check metadata transforms task', () => {
           packageInfo: { name: 'package name' },
           paths: ['some/test/transform/path'],
         };
-        getRegistryPackageSpy.mockResolvedValue(expectedArgs);
+        getPackageSpy.mockResolvedValue(expectedArgs);
         reinstallEsAssetsSpy.mockResolvedValue([{}]);
         await runTask();
 
@@ -338,7 +338,7 @@ describe('check metadata transforms task', () => {
       });
 
       it('should return correct runAt', async () => {
-        getRegistryPackageSpy.mockResolvedValue({
+        getPackageSpy.mockResolvedValue({
           packageInfo: { name: 'package name' },
           paths: ['some/test/transform/path'],
         });

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts

@@ -256,7 +256,7 @@ export class CheckMetadataTransformsTask {
 
       const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
 
-      const { packageInfo, paths } = await packageClient.getRegistryPackage(
+      const { packageInfo, paths } = await packageClient.getPackage(
         FLEET_ENDPOINT_PACKAGE,
         pkgVersion
       );

x-pack/plugins/security_solution/server/lib/detection_engine/routes/fleet/get_installed_integrations/get_installed_integrations_route.ts

@@ -50,7 +50,7 @@ export const getInstalledIntegrationsRoute = (
           concurrency: MAX_CONCURRENT_REQUESTS_TO_PACKAGE_REGISTRY,
           items: set.getPackages(),
           executor: async (packageInfo) => {
-            const registryPackage = await fleet.packages.getRegistryPackage(
+            const registryPackage = await fleet.packages.getPackage(
               packageInfo.package_name,
               packageInfo.package_version
             );

x-pack/plugins/security_solution/server/lib/detection_engine/routes/fleet/get_installed_integrations/installed_integration_set.ts

@@ -6,7 +6,7 @@
  */
 
 import { capitalize, flatten } from 'lodash';
-import type { PackagePolicy, RegistryPackage } from '@kbn/fleet-plugin/common';
+import type { PackagePolicy, ArchivePackage } from '@kbn/fleet-plugin/common';
 import type {
   InstalledIntegration,
   InstalledIntegrationArray,
@@ -18,7 +18,7 @@ import type {
 
 export interface IInstalledIntegrationSet {
   addPackagePolicy(policy: PackagePolicy): void;
-  addRegistryPackage(registryPackage: RegistryPackage): void;
+  addRegistryPackage(registryPackage: ArchivePackage): void;
 
   getPackages(): InstalledPackageArray;
   getIntegrations(): InstalledIntegrationArray;
@@ -56,7 +56,7 @@ export const createInstalledIntegrationSet = (): IInstalledIntegrationSet => {
     }
   };
 
-  const addRegistryPackage = (registryPackage: RegistryPackage): void => {
+  const addRegistryPackage = (registryPackage: ArchivePackage): void => {
     const policyTemplates = registryPackage.policy_templates ?? [];
     const packageKey = `${registryPackage.name}:${registryPackage.version}`;
     const existingPackageInfo = packageMap.get(packageKey);