Merge branch 'main' into fix-keep-up-to-date

x-pack/plugins/fleet/common/types/models/epm.ts

@@ -125,6 +125,14 @@ export type DataType = typeof dataTypes;
 export type MonitoringType = typeof monitoringTypes;
 export type InstallablePackage = RegistryPackage | ArchivePackage;
 
+export type AssetsMap = Map<string, Buffer | undefined>;
+
+export interface PackageInstallContext {
+  packageInfo: InstallablePackage;
+  assetsMap: AssetsMap;
+  paths: string[];
+}
+
 export type ArchivePackage = PackageSpecManifest &
   // should an uploaded package be able to specify `internal`?
   Pick<RegistryPackage, 'readme' | 'assets' | 'data_streams' | 'internal' | 'elasticsearch'>;

x-pack/plugins/fleet/server/routes/epm/file_handler.test.ts

@@ -15,7 +15,7 @@ import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_p
 import { getFile, getInstallation } from '../../services/epm/packages/get';
 import type { FleetRequestHandlerContext } from '../..';
 import { appContextService } from '../../services';
-import { unpackBufferEntries, getArchiveEntry } from '../../services/epm/archive';
+import { unpackBufferEntries } from '../../services/epm/archive';
 import { getAsset } from '../../services/epm/archive/storage';
 
 import { getFileHandler } from './file_handler';
@@ -29,7 +29,6 @@ jest.mock('../../services/epm/packages/get');
 const mockedGetBundledPackageByPkgKey = jest.mocked(getBundledPackageByPkgKey);
 const mockedGetInstallation = jest.mocked(getInstallation);
 const mockedGetFile = jest.mocked(getFile);
-const mockedGetArchiveEntry = jest.mocked(getArchiveEntry);
 const mockedUnpackBufferEntries = jest.mocked(unpackBufferEntries);
 const mockedGetAsset = jest.mocked(getAsset);
 
@@ -61,7 +60,6 @@ describe('getFileHandler', () => {
     mockedUnpackBufferEntries.mockReset();
     mockedGetFile.mockReset();
     mockedGetInstallation.mockReset();
-    mockedGetArchiveEntry.mockReset();
     mockedGetAsset.mockReset();
   });
 
@@ -207,7 +205,15 @@ describe('getFileHandler', () => {
     const context = mockContext();
 
     mockedGetInstallation.mockResolvedValue({ version: '1.0.0' } as any);
-    mockedGetArchiveEntry.mockReturnValue(Buffer.from('test'));
+    mockedGetAsset.mockResolvedValue({
+      asset_path: '/test/1.0.0/README.md',
+      data_utf8: 'test',
+      data_base64: '',
+      media_type: 'text/markdown; charset=utf-8',
+      package_name: 'test',
+      package_version: '1.0.0',
+      install_source: 'registry',
+    });
 
     await getFileHandler(context, request, response);
 

x-pack/plugins/fleet/server/routes/epm/file_handler.ts

@@ -14,7 +14,6 @@ import type { ResponseHeaders, KnownHeaders, HttpResponseOptions } from '@kbn/co
 import type { GetFileRequestSchema, FleetRequestHandler } from '../../types';
 import { getFile, getInstallation } from '../../services/epm/packages';
 import { defaultFleetErrorHandler } from '../../errors';
-import { getArchiveEntry } from '../../services/epm/archive';
 import { getAsset } from '../../services/epm/archive/storage';
 import { getBundledPackageByPkgKey } from '../../services/epm/packages/bundled_packages';
 import { pkgToPkgKey } from '../../services/epm/registry';
@@ -31,35 +30,23 @@ export const getFileHandler: FleetRequestHandler<
     const savedObjectsClient = (await context.fleet).internalSoClient;
 
     const installation = await getInstallation({ savedObjectsClient, pkgName });
-    const useLocalFile = pkgVersion === installation?.version;
+    const isPackageInstalled = pkgVersion === installation?.version;
     const assetPath = `${pkgName}-${pkgVersion}/${filePath}`;
 
-    if (useLocalFile) {
-      const fileBuffer = getArchiveEntry(assetPath);
-      // only pull local installation if we don't have it cached
-      const storedAsset = !fileBuffer && (await getAsset({ savedObjectsClient, path: assetPath }));
+    if (isPackageInstalled) {
+      const storedAsset = await getAsset({ savedObjectsClient, path: assetPath });
 
-      // error, if neither is available
-      if (!fileBuffer && !storedAsset) {
+      if (!storedAsset) {
         return response.custom({
           body: `installed package file not found: ${filePath}`,
           statusCode: 404,
         });
       }
 
-      // if storedAsset is not available, fileBuffer *must* be
-      // b/c we error if we don't have at least one, and storedAsset is the least likely
-      const { buffer, contentType } = storedAsset
-        ? {
-            contentType: storedAsset.media_type,
-            buffer: storedAsset.data_utf8
-              ? Buffer.from(storedAsset.data_utf8, 'utf8')
-              : Buffer.from(storedAsset.data_base64, 'base64'),
-          }
-        : {
-            contentType: mime.contentType(path.extname(assetPath)),
-            buffer: fileBuffer,
-          };
+      const contentType = storedAsset.media_type;
+      const buffer = storedAsset.data_utf8
+        ? Buffer.from(storedAsset.data_utf8, 'utf8')
+        : Buffer.from(storedAsset.data_base64, 'base64');
 
       if (!contentType) {
         return response.custom({

x-pack/plugins/fleet/server/services/epm/archive/cache.ts

@@ -8,19 +8,10 @@
 import type { ArchivePackage, RegistryPackage, PackageVerificationResult } from '../../../types';
 import { appContextService } from '../..';
 
-import type { ArchiveEntry } from '.';
-
 type SharedKeyString = string;
 
 const sharedKey = ({ name, version }: SharedKey): SharedKeyString => `${name}-${version}`;
 
-const archiveEntryCache: Map<ArchiveEntry['path'], ArchiveEntry['buffer']> = new Map();
-export const getArchiveEntry = (key: string) => archiveEntryCache.get(key);
-export const setArchiveEntry = (key: string, value: Buffer) => archiveEntryCache.set(key, value);
-export const hasArchiveEntry = (key: string) => archiveEntryCache.has(key);
-export const clearArchiveEntries = () => archiveEntryCache.clear();
-export const deleteArchiveEntry = (key: string) => archiveEntryCache.delete(key);
-
 const verificationResultCache: Map<SharedKeyString, PackageVerificationResult> = new Map();
 export const getVerificationResult = (key: SharedKey) =>
   verificationResultCache.get(sharedKey(key));
@@ -37,36 +28,12 @@ export interface SharedKey {
   version: string;
 }
 
-const archiveFilelistCache: Map<SharedKeyString, string[]> = new Map();
-export const getArchiveFilelist = (keyArgs: SharedKey) =>
-  archiveFilelistCache.get(sharedKey(keyArgs));
-
-export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) => {
-  const logger = appContextService.getLogger();
-  logger.debug(`Setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`);
-  logger.trace(JSON.stringify(paths));
-  return archiveFilelistCache.set(sharedKey(keyArgs), paths);
-};
-
-export const deleteArchiveFilelist = (keyArgs: SharedKey) =>
-  archiveFilelistCache.delete(sharedKey(keyArgs));
-
 const packageInfoCache: Map<SharedKeyString, ArchivePackage | RegistryPackage> = new Map();
 
 export const getPackageInfo = (args: SharedKey) => {
   return packageInfoCache.get(sharedKey(args));
 };
 
-export const getArchivePackage = (args: SharedKey) => {
-  const packageInfo = getPackageInfo(args);
-  const paths = getArchiveFilelist(args);
-  if (!paths || !packageInfo) return undefined;
-  return {
-    paths,
-    packageInfo,
-  };
-};
-
 /*
  * This cache should only be used to store "full" package info generated from the package archive.
  * NOT package info from the EPR API. This is because we parse extra fields from the archive
@@ -85,11 +52,3 @@ export const setPackageInfo = ({
 };
 
 export const deletePackageInfo = (args: SharedKey) => packageInfoCache.delete(sharedKey(args));
-
-export const clearPackageFileCache = (args: SharedKey) => {
-  const fileList = getArchiveFilelist(args) ?? [];
-  fileList.forEach((filePath) => {
-    deleteArchiveEntry(filePath);
-  });
-  deleteArchiveFilelist(args);
-};

x-pack/plugins/fleet/server/services/epm/archive/index.ts

@@ -5,23 +5,14 @@
  * 2.0.
  */
 
-import type { AssetParts } from '../../../../common/types';
+import type { AssetParts, AssetsMap } from '../../../../common/types';
 import {
   PackageInvalidArchiveError,
   PackageUnsupportedMediaTypeError,
   PackageNotFoundError,
 } from '../../../errors';
 
-import {
-  getArchiveEntry,
-  setArchiveEntry,
-  deleteArchiveEntry,
-  getArchiveFilelist,
-  setArchiveFilelist,
-  deleteArchiveFilelist,
-  deletePackageInfo,
-  clearPackageFileCache,
-} from './cache';
+import { deletePackageInfo } from './cache';
 import type { SharedKey } from './cache';
 import { getBufferExtractor } from './extract';
 
@@ -34,7 +25,7 @@ export interface ArchiveEntry {
   buffer?: Buffer;
 }
 
-export async function unpackBufferToCache({
+export async function unpackBufferToAssetsMap({
   name,
   version,
   contentType,
@@ -44,22 +35,20 @@ export async function unpackBufferToCache({
   version: string;
   contentType: string;
   archiveBuffer: Buffer;
-}): Promise<string[]> {
+}): Promise<{ paths: string[]; assetsMap: AssetsMap }> {
+  const assetsMap = new Map<string, Buffer | undefined>();
+  const paths: string[] = [];
   const entries = await unpackBufferEntries(archiveBuffer, contentType);
-  // Make sure any buffers from previous installations from registry or upload are deleted first
-  clearPackageFileCache({ name, version });
 
-  const paths: string[] = [];
   entries.forEach((entry) => {
     const { path, buffer } = entry;
     if (buffer) {
-      setArchiveEntry(path, buffer);
+      assetsMap.set(path, buffer);
       paths.push(path);
     }
   });
-  setArchiveFilelist({ name, version }, paths);
 
-  return paths;
+  return { assetsMap, paths };
 }
 
 export async function unpackBufferEntries(
@@ -94,16 +83,6 @@ export async function unpackBufferEntries(
 }
 
 export const deletePackageCache = ({ name, version }: SharedKey) => {
-  // get cached archive filelist
-  const paths = getArchiveFilelist({ name, version });
-
-  // delete cached archive filelist
-  deleteArchiveFilelist({ name, version });
-
-  // delete cached archive files
-  // this has been populated in unpackBufferToCache()
-  paths?.forEach(deleteArchiveEntry);
-
   deletePackageInfo({ name, version });
 };
 
@@ -151,8 +130,8 @@ export function getPathParts(path: string): AssetParts {
   } as AssetParts;
 }
 
-export function getAsset(key: string) {
-  const buffer = getArchiveEntry(key);
+export function getAssetFromAssetsMap(assetsMap: AssetsMap, key: string) {
+  const buffer = assetsMap.get(key);
   if (buffer === undefined) throw new PackageNotFoundError(`Cannot find asset ${key}`);
 
   return buffer;

x-pack/plugins/fleet/server/services/epm/archive/storage.ts

@@ -23,7 +23,7 @@ import { PackageInvalidArchiveError, PackageNotFoundError } from '../../../error
 
 import { appContextService } from '../../app_context';
 
-import { getArchiveEntry, setArchiveEntry, setArchiveFilelist, setPackageInfo } from '.';
+import { setPackageInfo } from '.';
 import type { ArchiveEntry } from '.';
 import { filterAssetPathForParseAndVerifyArchive, parseAndVerifyArchive } from './parse';
 
@@ -104,16 +104,17 @@ export async function removeArchiveEntries(opts: {
   );
 }
 
-export async function saveArchiveEntries(opts: {
+export async function saveArchiveEntriesFromAssetsMap(opts: {
   savedObjectsClient: SavedObjectsClientContract;
   paths: string[];
+  assetsMap: Map<string, Buffer | undefined>;
   packageInfo: InstallablePackage;
   installSource: InstallSource;
 }) {
-  const { savedObjectsClient, paths, packageInfo, installSource } = opts;
+  const { savedObjectsClient, paths, packageInfo, assetsMap, installSource } = opts;
   const bulkBody = await Promise.all(
     paths.map((path) => {
-      const buffer = getArchiveEntry(path);
+      const buffer = assetsMap.get(path);
       if (!buffer) throw new PackageNotFoundError(`Could not find ArchiveEntry at ${path}`);
       const { name, version } = packageInfo;
       return archiveEntryToBulkCreateObject({ path, buffer, name, version, installSource });
@@ -208,26 +209,26 @@ export const getEsPackage = async (
     return undefined;
   }
 
-  const assetsMap: Record<string, Buffer> = {};
+  const parseAndVerifyAssetsMap: Record<string, Buffer> = {};
+  const assetsMap = new Map<string, Buffer | undefined>();
   const entries: ArchiveEntry[] = assets.map(packageAssetToArchiveEntry);
   const paths: string[] = [];
   entries.forEach(({ path, buffer }) => {
     if (path && buffer) {
-      setArchiveEntry(path, buffer);
+      assetsMap.set(path, buffer);
       paths.push(path);
     }
     if (buffer && filterAssetPathForParseAndVerifyArchive(path)) {
-      assetsMap[path] = buffer;
+      parseAndVerifyAssetsMap[path] = buffer;
     }
   });
-  // Add asset references to cache
-  setArchiveFilelist({ name: pkgName, version: pkgVersion }, paths);
 
-  const packageInfo = parseAndVerifyArchive(paths, assetsMap);
+  const packageInfo = parseAndVerifyArchive(paths, parseAndVerifyAssetsMap);
   setPackageInfo({ name: pkgName, version: pkgVersion, packageInfo });
 
   return {
     packageInfo,
     paths,
+    assetsMap,
   };
 };

x-pack/plugins/fleet/server/services/epm/elasticsearch/datastream_ilm/install.ts

@@ -7,14 +7,13 @@
 
 import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server';
 
-import { ElasticsearchAssetType } from '../../../../../common/types/models';
-import type {
-  EsAssetReference,
-  InstallablePackage,
-  RegistryDataStream,
+import {
+  ElasticsearchAssetType,
+  type PackageInstallContext,
 } from '../../../../../common/types/models';
+import type { EsAssetReference, RegistryDataStream } from '../../../../../common/types/models';
 import { updateEsAssetReferences } from '../../packages/install';
-import { getAsset } from '../transform/common';
+import { getAssetFromAssetsMap } from '../../archive';
 
 import { getESAssetMetadata } from '../meta';
 import { retryTransientEsErrors } from '../retry';
@@ -32,13 +31,13 @@ interface IlmPathDataset {
 }
 
 export const installIlmForDataStream = async (
-  registryPackage: InstallablePackage,
-  paths: string[],
+  packageInstallContext: PackageInstallContext,
   esClient: ElasticsearchClient,
   savedObjectsClient: SavedObjectsClientContract,
   logger: Logger,
   esReferences: EsAssetReference[]
 ) => {
+  const { packageInfo: registryPackage, paths, assetsMap } = packageInstallContext;
   const previousInstalledIlmEsAssets = esReferences.filter(
     ({ type }) => type === ElasticsearchAssetType.dataStreamIlmPolicy
   );
@@ -100,7 +99,9 @@ export const installIlmForDataStream = async (
 
     const ilmInstallations: IlmInstallation[] = ilmPathDatasets.map(
       (ilmPathDataset: IlmPathDataset) => {
-        const content = JSON.parse(getAsset(ilmPathDataset.path).toString('utf-8'));
+        const content = JSON.parse(
+          getAssetFromAssetsMap(assetsMap, ilmPathDataset.path).toString('utf-8')
+        );
         content.policy._meta = getESAssetMetadata({ packageName: registryPackage.name });
 
         return {