[Security Solution][Endpoint] Fix base64 download bug and adopt new user artifact/manifest format

x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts

@@ -19,10 +19,10 @@ import {
 export const manifestEntrySchema = t.exact(
   t.type({
     relative_url: relativeUrl,
-    precompress_sha256: sha256,
-    precompress_size: size,
-    postcompress_sha256: sha256,
-    postcompress_size: size,
+    decoded_sha256: sha256,
+    decoded_size: size,
+    encoded_sha256: sha256,
+    encoded_size: size,
     compression_algorithm: compressionAlgorithm,
     encryption_algorithm: encryptionAlgorithm,
   })

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -6,12 +6,12 @@
 
 export const ArtifactConstants = {
   GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
-  SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2',
   SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
   SCHEMA_VERSION: '1.0.0',
 };
 
 export const ManifestConstants = {
-  SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
+  SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2',
   SCHEMA_VERSION: '1.0.0',
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts

@@ -21,7 +21,7 @@ describe('buildEventTypeSignal', () => {
 
   test('it should convert the exception lists response to the proper endpoint format', async () => {
     const expectedEndpointExceptions = {
-      exceptions_list: [
+      entries: [
         {
           entries: [
             {
@@ -57,7 +57,7 @@ describe('buildEventTypeSignal', () => {
     ];
 
     const expectedEndpointExceptions = {
-      exceptions_list: [
+      entries: [
         {
           field: 'server.domain',
           operator: 'included',
@@ -100,7 +100,7 @@ describe('buildEventTypeSignal', () => {
     ];
 
     const expectedEndpointExceptions = {
-      exceptions_list: [
+      entries: [
         {
           field: 'server.domain',
           operator: 'included',
@@ -147,7 +147,7 @@ describe('buildEventTypeSignal', () => {
     ];
 
     const expectedEndpointExceptions = {
-      exceptions_list: [
+      entries: [
         {
           field: 'server.domain',
           operator: 'included',
@@ -182,7 +182,7 @@ describe('buildEventTypeSignal', () => {
       .mockReturnValueOnce(second)
       .mockReturnValueOnce(third);
     const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
-    expect(resp.exceptions_list.length).toEqual(6);
+    expect(resp.entries.length).toEqual(6);
   });
 
   test('it should handle no exceptions', async () => {
@@ -191,6 +191,6 @@ describe('buildEventTypeSignal', () => {
     exceptionsResponse.total = 0;
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse);
     const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
-    expect(resp.exceptions_list.length).toEqual(0);
+    expect(resp.entries.length).toEqual(0);
   });
 });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -14,7 +14,7 @@ import {
   InternalArtifactSchema,
   TranslatedEntry,
   WrappedTranslatedExceptionList,
-  wrappedExceptionList,
+  wrappedTranslatedExceptionList,
   TranslatedEntryNestedEntry,
   translatedEntryNestedEntry,
   translatedEntry as translatedEntryType,
@@ -36,10 +36,10 @@ export async function buildArtifact(
     identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
     compressionAlgorithm: 'none',
     encryptionAlgorithm: 'none',
-    decompressedSha256: sha256,
-    compressedSha256: sha256,
-    decompressedSize: exceptionsBuffer.byteLength,
-    compressedSize: exceptionsBuffer.byteLength,
+    decodedSha256: sha256,
+    encodedSha256: sha256,
+    decodedSize: exceptionsBuffer.byteLength,
+    encodedSize: exceptionsBuffer.byteLength,
     created: Date.now(),
     body: exceptionsBuffer.toString('base64'),
   };
@@ -50,7 +50,7 @@ export async function getFullEndpointExceptionList(
   os: string,
   schemaVersion: string
 ): Promise<WrappedTranslatedExceptionList> {
-  const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] };
+  const exceptions: WrappedTranslatedExceptionList = { entries: [] };
   let numResponses = 0;
   let page = 1;
 
@@ -68,7 +68,7 @@ export async function getFullEndpointExceptionList(
     if (response?.data !== undefined) {
       numResponses = response.data.length;
 
-      exceptions.exceptions_list = exceptions.exceptions_list.concat(
+      exceptions.entries = exceptions.entries.concat(
         translateToEndpointExceptions(response, schemaVersion)
       );
 
@@ -78,7 +78,7 @@ export async function getFullEndpointExceptionList(
     }
   } while (numResponses > 0);
 
-  const [validated, errors] = validate(exceptions, wrappedExceptionList);
+  const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList);
   if (errors != null) {
     throw new Error(errors);
   }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts

@@ -57,30 +57,30 @@ describe('manifest', () => {
           'endpoint-exceptionlist-linux-1.0.0': {
             compression_algorithm: 'none',
             encryption_algorithm: 'none',
-            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            precompress_size: 268,
-            postcompress_size: 268,
+            decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            decoded_size: 268,
+            encoded_size: 268,
             relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-macos-1.0.0': {
             compression_algorithm: 'none',
             encryption_algorithm: 'none',
-            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            precompress_size: 268,
-            postcompress_size: 268,
+            decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            decoded_size: 268,
+            encoded_size: 268,
             relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
           'endpoint-exceptionlist-windows-1.0.0': {
             compression_algorithm: 'none',
             encryption_algorithm: 'none',
-            precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-            precompress_size: 268,
-            postcompress_size: 268,
+            decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+            decoded_size: 268,
+            encoded_size: 268,
             relative_url:
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
           },
@@ -119,7 +119,7 @@ describe('manifest', () => {
 
     test('Manifest returns data for given artifact', async () => {
       const artifact = artifacts[0];
-      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`);
+      const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`);
       expect(returned).toEqual(artifact);
     });
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts

@@ -33,17 +33,17 @@ describe('manifest_entry', () => {
     });
 
     test('Correct sha256 is returned', () => {
-      expect(manifestEntry.getCompressedSha256()).toEqual(
+      expect(manifestEntry.getEncodedSha256()).toEqual(
         '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
       );
-      expect(manifestEntry.getDecompressedSha256()).toEqual(
+      expect(manifestEntry.getDecodedSha256()).toEqual(
         '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
       );
     });
 
     test('Correct size is returned', () => {
-      expect(manifestEntry.getCompressedSize()).toEqual(268);
-      expect(manifestEntry.getDecompressedSize()).toEqual(268);
+      expect(manifestEntry.getEncodedSize()).toEqual(268);
+      expect(manifestEntry.getDecodedSize()).toEqual(268);
     });
 
     test('Correct url is returned', () => {
@@ -60,10 +60,10 @@ describe('manifest_entry', () => {
       expect(manifestEntry.getRecord()).toEqual({
         compression_algorithm: 'none',
         encryption_algorithm: 'none',
-        precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-        postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
-        precompress_size: 268,
-        postcompress_size: 268,
+        decoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        encoded_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
+        decoded_size: 268,
+        encoded_size: 268,
         relative_url:
           '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
       });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts

@@ -15,31 +15,31 @@ export class ManifestEntry {
   }
 
   public getDocId(): string {
-    return `${this.getIdentifier()}-${this.getCompressedSha256()}`;
+    return `${this.getIdentifier()}-${this.getEncodedSha256()}`;
   }
 
   public getIdentifier(): string {
     return this.artifact.identifier;
   }
 
-  public getCompressedSha256(): string {
-    return this.artifact.compressedSha256;
+  public getEncodedSha256(): string {
+    return this.artifact.encodedSha256;
   }
 
-  public getDecompressedSha256(): string {
-    return this.artifact.decompressedSha256;
+  public getDecodedSha256(): string {
+    return this.artifact.decodedSha256;
   }
 
-  public getCompressedSize(): number {
-    return this.artifact.compressedSize;
+  public getEncodedSize(): number {
+    return this.artifact.encodedSize;
   }
 
-  public getDecompressedSize(): number {
-    return this.artifact.decompressedSize;
+  public getDecodedSize(): number {
+    return this.artifact.decodedSize;
   }
 
   public getUrl(): string {
-    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`;
+    return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getEncodedSha256()}`;
   }
 
   public getArtifact(): InternalArtifactSchema {
@@ -50,10 +50,10 @@ export class ManifestEntry {
     return {
       compression_algorithm: 'none',
       encryption_algorithm: 'none',
-      precompress_sha256: this.getDecompressedSha256(),
-      precompress_size: this.getDecompressedSize(),
-      postcompress_sha256: this.getCompressedSha256(),
-      postcompress_size: this.getCompressedSize(),
+      decoded_sha256: this.getDecodedSha256(),
+      decoded_size: this.getDecodedSize(),
+      encoded_sha256: this.getEncodedSha256(),
+      encoded_size: this.getEncodedSize(),
       relative_url: this.getUrl(),
     };
   }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts

@@ -24,18 +24,18 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings']
       type: 'keyword',
       index: false,
     },
-    compressedSha256: {
+    encodedSha256: {
       type: 'keyword',
     },
-    compressedSize: {
+    encodedSize: {
       type: 'long',
       index: false,
     },
-    decompressedSha256: {
+    decodedSha256: {
       type: 'keyword',
       index: false,
     },
-    decompressedSize: {
+    decodedSize: {
       type: 'long',
       index: false,
     },

x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts

@@ -72,9 +72,9 @@ export const translatedExceptionList = t.exact(
 );
 export type TranslatedExceptionList = t.TypeOf<typeof translatedExceptionList>;
 
-export const wrappedExceptionList = t.exact(
+export const wrappedTranslatedExceptionList = t.exact(
   t.type({
-    exceptions_list: t.array(translatedEntry),
+    entries: t.array(translatedEntry),
   })
 );
-export type WrappedTranslatedExceptionList = t.TypeOf<typeof wrappedExceptionList>;
+export type WrappedTranslatedExceptionList = t.TypeOf<typeof wrappedTranslatedExceptionList>;

x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts

@@ -19,10 +19,10 @@ export const internalArtifactSchema = t.exact(
     identifier,
     compressionAlgorithm,
     encryptionAlgorithm,
-    decompressedSha256: sha256,
-    decompressedSize: size,
-    compressedSha256: sha256,
-    compressedSize: size,
+    decodedSha256: sha256,
+    decodedSize: size,
+    encodedSha256: sha256,
+    encodedSize: size,
     created,
     body,
   })

x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts

@@ -16,7 +16,7 @@ export class ArtifactClient {
   }
 
   public getArtifactId(artifact: InternalArtifactSchema) {
-    return `${artifact.identifier}-${artifact.compressedSha256}`;
+    return `${artifact.identifier}-${artifact.encodedSha256}`;
   }
 
   public async getArtifact(id: string): Promise<SavedObject<InternalArtifactSchema>> {

x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts

@@ -84,9 +84,15 @@ export class ManifestManagerMock extends ManifestManager {
 }
 
 export const getManifestManagerMock = (opts?: {
+  cache?: ExceptionsCache;
   packageConfigService?: PackageConfigServiceMock;
   savedObjectsClient?: ReturnType<typeof savedObjectsClientMock.create>;
 }): ManifestManagerMock => {
+  let cache = new ExceptionsCache(5);
+  if (opts?.cache !== undefined) {
+    cache = opts.cache;
+  }
+
   let packageConfigService = getPackageConfigServiceMock();
   if (opts?.packageConfigService !== undefined) {
     packageConfigService = opts.packageConfigService;
@@ -99,7 +105,7 @@ export const getManifestManagerMock = (opts?: {
 
   const manifestManager = new ManifestManagerMock({
     artifactClient: getArtifactClientMock(savedObjectsClient),
-    cache: new ExceptionsCache(5),
+    cache,
     // @ts-ignore
     packageConfigService,
     exceptionListClient: listMock.getExceptionListClient(),