From bd75a1137064aeaf5eb4de213ae63fce25ee68ca Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 7 Jul 2020 21:52:08 -0400 Subject: [PATCH] [Security Solution][Endpoint] Fix base64 download bug and adopt new user artifact/manifest format (#70998) * Fix base64 download bug * Add test for artifact download * Add more tests to ensure cached versions of artifacts are correct * Convert to new format * missed some refs * partial fix to wrapper format * update fixtures and integration test * Fixing unit tests Co-authored-by: Alex Kahan Co-authored-by: Elastic Machine --- .../common/endpoint/schema/manifest.ts | 8 +- .../server/endpoint/lib/artifacts/common.ts | 4 +- .../endpoint/lib/artifacts/lists.test.ts | 32 ++++-- .../server/endpoint/lib/artifacts/lists.ts | 50 +++++---- .../endpoint/lib/artifacts/manifest.test.ts | 56 +++++----- .../lib/artifacts/manifest_entry.test.ts | 26 ++--- .../endpoint/lib/artifacts/manifest_entry.ts | 28 ++--- .../lib/artifacts/saved_object_mappings.ts | 8 +- .../artifacts/download_exception_list.test.ts | 30 ++++-- .../endpoint/schemas/artifacts/lists.mock.ts | 30 ++++-- .../endpoint/schemas/artifacts/lists.ts | 10 +- .../schemas/artifacts/saved_objects.mock.ts | 2 +- .../schemas/artifacts/saved_objects.ts | 8 +- .../services/artifacts/artifact_client.ts | 2 +- .../manifest_manager/manifest_manager.mock.ts | 8 +- .../manifest_manager/manifest_manager.test.ts | 61 +++++++++-- .../manifest_manager/manifest_manager.ts | 3 +- .../apis/endpoint/artifacts/index.ts | 100 +++++++++++++++++- .../endpoint/artifacts/api_feature/data.json | 28 ++--- 19 files changed, 349 insertions(+), 145 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts index 2f03895d91c74..1c8916dfdd5bb 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts @@ -19,10 +19,10 @@ import { export const manifestEntrySchema = t.exact( t.type({ relative_url: relativeUrl, - precompress_sha256: sha256, - precompress_size: size, - postcompress_sha256: sha256, - postcompress_size: size, + decoded_sha256: sha256, + decoded_size: size, + encoded_sha256: sha256, + encoded_size: size, compression_algorithm: compressionAlgorithm, encryption_algorithm: encryptionAlgorithm, }) diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index b6a5bed9078ab..cf38147522083 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -6,12 +6,12 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', - SAVED_OBJECT_TYPE: 'endpoint:user-artifact', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], SCHEMA_VERSION: '1.0.0', }; export const ManifestConstants = { - SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest', + SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2', SCHEMA_VERSION: '1.0.0', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 738890fb4038f..0a1cd556e6e91 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -21,7 +21,8 @@ describe('buildEventTypeSignal', () => { test('it should convert the exception lists response to the proper endpoint format', async () => { const expectedEndpointExceptions = { - exceptions_list: [ + type: 'simple', + entries: [ { entries: [ { @@ -46,7 +47,9 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert simple fields', async () => { @@ -57,7 +60,8 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + type: 'simple', + entries: [ { field: 'server.domain', operator: 'included', @@ -84,7 +88,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert fields case sensitive', async () => { @@ -100,7 +106,8 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + type: 'simple', + entries: [ { field: 'server.domain', operator: 'included', @@ -127,7 +134,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should ignore unsupported entries', async () => { @@ -147,7 +156,8 @@ describe('buildEventTypeSignal', () => { ]; const expectedEndpointExceptions = { - exceptions_list: [ + type: 'simple', + entries: [ { field: 'server.domain', operator: 'included', @@ -162,7 +172,9 @@ describe('buildEventTypeSignal', () => { mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp).toEqual(expectedEndpointExceptions); + expect(resp).toEqual({ + entries: [expectedEndpointExceptions], + }); }); test('it should convert the exception lists response to the proper endpoint format while paging', async () => { @@ -182,7 +194,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(second) .mockReturnValueOnce(third); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.exceptions_list.length).toEqual(6); + expect(resp.entries.length).toEqual(3); }); test('it should handle no exceptions', async () => { @@ -191,6 +203,6 @@ describe('buildEventTypeSignal', () => { exceptionsResponse.total = 0; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); - expect(resp.exceptions_list.length).toEqual(0); + expect(resp.entries.length).toEqual(0); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index 2abb72234fecd..a13781519b508 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; +import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { validate } from '../../../../common/validate'; import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries'; @@ -14,13 +15,14 @@ import { InternalArtifactSchema, TranslatedEntry, WrappedTranslatedExceptionList, - wrappedExceptionList, + wrappedTranslatedExceptionList, TranslatedEntryNestedEntry, translatedEntryNestedEntry, translatedEntry as translatedEntryType, TranslatedEntryMatcher, translatedEntryMatchMatcher, translatedEntryMatchAnyMatcher, + TranslatedExceptionListItem, } from '../../schemas'; import { ArtifactConstants } from './common'; @@ -36,10 +38,10 @@ export async function buildArtifact( identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, compressionAlgorithm: 'none', encryptionAlgorithm: 'none', - decompressedSha256: sha256, - compressedSha256: sha256, - decompressedSize: exceptionsBuffer.byteLength, - compressedSize: exceptionsBuffer.byteLength, + decodedSha256: sha256, + encodedSha256: sha256, + decodedSize: exceptionsBuffer.byteLength, + encodedSize: exceptionsBuffer.byteLength, created: Date.now(), body: exceptionsBuffer.toString('base64'), }; @@ -50,7 +52,7 @@ export async function getFullEndpointExceptionList( os: string, schemaVersion: string ): Promise { - const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] }; + const exceptions: WrappedTranslatedExceptionList = { entries: [] }; let numResponses = 0; let page = 1; @@ -68,7 +70,7 @@ export async function getFullEndpointExceptionList( if (response?.data !== undefined) { numResponses = response.data.length; - exceptions.exceptions_list = exceptions.exceptions_list.concat( + exceptions.entries = exceptions.entries.concat( translateToEndpointExceptions(response, schemaVersion) ); @@ -78,7 +80,7 @@ export async function getFullEndpointExceptionList( } } while (numResponses > 0); - const [validated, errors] = validate(exceptions, wrappedExceptionList); + const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList); if (errors != null) { throw new Error(errors); } @@ -92,19 +94,11 @@ export async function getFullEndpointExceptionList( export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string -): TranslatedEntry[] { +): TranslatedExceptionListItem[] { if (schemaVersion === '1.0.0') { - return exc.data - .flatMap((list) => { - return list.entries; - }) - .reduce((entries: TranslatedEntry[], entry) => { - const translatedEntry = translateEntry(schemaVersion, entry); - if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { - entries.push(translatedEntry); - } - return entries; - }, []); + return exc.data.map((item) => { + return translateItem(schemaVersion, item); + }); } else { throw new Error('unsupported schemaVersion'); } @@ -124,6 +118,22 @@ function normalizeFieldName(field: string): string { return field.endsWith('.text') ? field.substring(0, field.length - 5) : field; } +function translateItem( + schemaVersion: string, + item: ExceptionListItemSchema +): TranslatedExceptionListItem { + return { + type: item.type, + entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => { + const translatedEntry = translateEntry(schemaVersion, entry); + if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) { + translatedEntries.push(translatedEntry); + } + return translatedEntries; + }, []), + }; +} + function translateEntry( schemaVersion: string, entry: Entry | EntryNested diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index da8a449e1b026..3e5fdbf9484ca 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -57,32 +57,32 @@ describe('manifest', () => { 'endpoint-exceptionlist-linux-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, 'endpoint-exceptionlist-macos-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, 'endpoint-exceptionlist-windows-1.0.0': { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, }, manifest_version: 'abcd', @@ -94,9 +94,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ], }); }); @@ -106,12 +106,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1', + 'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', type: 'add', }, ]); @@ -119,7 +119,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`); + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`); expect(returned).toEqual(artifact); }); @@ -127,15 +127,15 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); expect(found).toEqual(true); }); @@ -144,17 +144,17 @@ describe('manifest', () => { const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index c8cbdfc2fc5f4..a52114ad90258 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -24,7 +24,7 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -33,22 +33,22 @@ describe('manifest_entry', () => { }); test('Correct sha256 is returned', () => { - expect(manifestEntry.getCompressedSha256()).toEqual( - '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + expect(manifestEntry.getEncodedSha256()).toEqual( + '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); - expect(manifestEntry.getDecompressedSha256()).toEqual( - '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + expect(manifestEntry.getDecodedSha256()).toEqual( + '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); test('Correct size is returned', () => { - expect(manifestEntry.getCompressedSize()).toEqual(268); - expect(manifestEntry.getDecompressedSize()).toEqual(268); + expect(manifestEntry.getEncodedSize()).toEqual(430); + expect(manifestEntry.getDecodedSize()).toEqual(430); }); test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -60,12 +60,12 @@ describe('manifest_entry', () => { expect(manifestEntry.getRecord()).toEqual({ compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', - precompress_size: 268, - postcompress_size: 268, + decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + decoded_size: 430, + encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index 860c2d7d704b2..c23258c4c3ba4 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -15,31 +15,31 @@ export class ManifestEntry { } public getDocId(): string { - return `${this.getIdentifier()}-${this.getCompressedSha256()}`; + return `${this.getIdentifier()}-${this.getEncodedSha256()}`; } public getIdentifier(): string { return this.artifact.identifier; } - public getCompressedSha256(): string { - return this.artifact.compressedSha256; + public getEncodedSha256(): string { + return this.artifact.encodedSha256; } - public getDecompressedSha256(): string { - return this.artifact.decompressedSha256; + public getDecodedSha256(): string { + return this.artifact.decodedSha256; } - public getCompressedSize(): number { - return this.artifact.compressedSize; + public getEncodedSize(): number { + return this.artifact.encodedSize; } - public getDecompressedSize(): number { - return this.artifact.decompressedSize; + public getDecodedSize(): number { + return this.artifact.decodedSize; } public getUrl(): string { - return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getCompressedSha256()}`; + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getEncodedSha256()}`; } public getArtifact(): InternalArtifactSchema { @@ -50,10 +50,10 @@ export class ManifestEntry { return { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: this.getDecompressedSha256(), - precompress_size: this.getDecompressedSize(), - postcompress_sha256: this.getCompressedSha256(), - postcompress_size: this.getCompressedSize(), + decoded_sha256: this.getDecodedSha256(), + decoded_size: this.getDecodedSize(), + encoded_sha256: this.getEncodedSha256(), + encoded_size: this.getEncodedSize(), relative_url: this.getUrl(), }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts index 5e61b278e87e4..89e974a3d5fd3 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/saved_object_mappings.ts @@ -24,18 +24,18 @@ export const exceptionsArtifactSavedObjectMappings: SavedObjectsType['mappings'] type: 'keyword', index: false, }, - compressedSha256: { + encodedSha256: { type: 'keyword', }, - compressedSize: { + encodedSize: { type: 'long', index: false, }, - decompressedSha256: { + decodedSha256: { type: 'keyword', index: false, }, - decompressedSize: { + decodedSize: { type: 'long', index: false, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index 863a1d5037756..fbcd3bd130dfd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -31,9 +31,22 @@ import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; const expectedEndpointExceptions: WrappedTranslatedExceptionList = { - exceptions_list: [ + entries: [ { + type: 'simple', entries: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, { field: 'some.not.nested.field', operator: 'included', @@ -41,14 +54,17 @@ const expectedEndpointExceptions: WrappedTranslatedExceptionList = { value: 'some value', }, ], - field: 'some.field', - type: 'nested', }, { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', + type: 'simple', + entries: [ + { + field: 'some.other.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some other value', + }, + ], }, ], }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts index 7354b5fd0ec4d..343b192163479 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.mock.ts @@ -8,9 +8,22 @@ import { WrappedTranslatedExceptionList } from './lists'; export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList => { return { - exceptions_list: [ + entries: [ { + type: 'simple', entries: [ + { + entries: [ + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.field', + type: 'nested', + }, { field: 'some.not.nested.field', operator: 'included', @@ -18,14 +31,17 @@ export const getTranslatedExceptionListMock = (): WrappedTranslatedExceptionList value: 'some value', }, ], - field: 'some.field', - type: 'nested', }, { - field: 'some.not.nested.field', - operator: 'included', - type: 'exact_cased', - value: 'some value', + type: 'simple', + entries: [ + { + field: 'some.other.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some other value', + }, + ], }, ], }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts index d071896c537bf..b7f99fe6fe297 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/lists.ts @@ -64,17 +64,17 @@ export const translatedEntry = t.union([ ]); export type TranslatedEntry = t.TypeOf; -export const translatedExceptionList = t.exact( +export const translatedExceptionListItem = t.exact( t.type({ type: t.string, entries: t.array(translatedEntry), }) ); -export type TranslatedExceptionList = t.TypeOf; +export type TranslatedExceptionListItem = t.TypeOf; -export const wrappedExceptionList = t.exact( +export const wrappedTranslatedExceptionList = t.exact( t.type({ - exceptions_list: t.array(translatedEntry), + entries: t.array(translatedExceptionListItem), }) ); -export type WrappedTranslatedExceptionList = t.TypeOf; +export type WrappedTranslatedExceptionList = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts index 1a9cc55ca5725..183a819807ed2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.mock.ts @@ -20,7 +20,7 @@ export const getInternalArtifactMockWithDiffs = async ( schemaVersion: string ): Promise => { const mock = getTranslatedExceptionListMock(); - mock.exceptions_list.pop(); + mock.entries.pop(); return buildArtifact(mock, os, schemaVersion); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index fe032586dda56..e4cd7f48a2901 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -19,10 +19,10 @@ export const internalArtifactSchema = t.exact( identifier, compressionAlgorithm, encryptionAlgorithm, - decompressedSha256: sha256, - decompressedSize: size, - compressedSha256: sha256, - compressedSize: size, + decodedSha256: sha256, + decodedSize: size, + encodedSha256: sha256, + encodedSize: size, created, body, }) diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index 00ae802ba6f32..e899905602c8d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -16,7 +16,7 @@ export class ArtifactClient { } public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.compressedSha256}`; + return `${artifact.identifier}-${artifact.encodedSha256}`; } public async getArtifact(id: string): Promise> { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index cd70b11aef305..483b3434d63f2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -84,9 +84,15 @@ export class ManifestManagerMock extends ManifestManager { } export const getManifestManagerMock = (opts?: { + cache?: ExceptionsCache; packageConfigService?: PackageConfigServiceMock; savedObjectsClient?: ReturnType; }): ManifestManagerMock => { + let cache = new ExceptionsCache(5); + if (opts?.cache !== undefined) { + cache = opts.cache; + } + let packageConfigService = getPackageConfigServiceMock(); if (opts?.packageConfigService !== undefined) { packageConfigService = opts.packageConfigService; @@ -99,7 +105,7 @@ export const getManifestManagerMock = (opts?: { const manifestManager = new ManifestManagerMock({ artifactClient: getArtifactClientMock(savedObjectsClient), - cache: new ExceptionsCache(5), + cache, // @ts-ignore packageConfigService, exceptionListClient: listMock.getExceptionListClient(), diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index ef4f921cb537e..1d6dffadde61a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -5,7 +5,12 @@ */ import { savedObjectsClientMock } from 'src/core/server/mocks'; -import { ArtifactConstants, ManifestConstants, Manifest } from '../../../lib/artifacts'; +import { + ArtifactConstants, + ManifestConstants, + Manifest, + ExceptionsCache, +} from '../../../lib/artifacts'; import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_manager.mock'; describe('manifest_manager', () => { @@ -16,13 +21,55 @@ describe('manifest_manager', () => { expect(manifestWrapper!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-d34a1f6659bd86fc2023d7477aa2e5d2055c9c0fb0a0f10fae76bf8b94bebe49', + 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); }); + test('ManifestManager populates cache properly', async () => { + const cache = new ExceptionsCache(5); + const manifestManager = getManifestManagerMock({ cache }); + const manifestWrapper = await manifestManager.refresh(); + expect(manifestWrapper!.diffs).toEqual([ + { + id: + 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + type: 'add', + }, + ]); + const diff = manifestWrapper!.diffs[0]; + const entry = JSON.parse(cache.get(diff!.id)!); + expect(entry).toEqual({ + entries: [ + { + type: 'simple', + entries: [ + { + entries: [ + { + field: 'nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + field: 'some.parentField', + type: 'nested', + }, + { + field: 'some.not.nested.field', + operator: 'included', + type: 'exact_cased', + value: 'some value', + }, + ], + }, + ], + }); + }); + test('ManifestManager can dispatch manifest', async () => { const packageConfigService = getPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); @@ -40,11 +87,11 @@ describe('manifest_manager', () => { [artifact.identifier]: { compression_algorithm: 'none', encryption_algorithm: 'none', - precompress_sha256: artifact.decompressedSha256, - postcompress_sha256: artifact.compressedSha256, - precompress_size: artifact.decompressedSize, - postcompress_size: artifact.compressedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.compressedSha256}`, + decoded_sha256: artifact.decodedSha256, + encoded_sha256: artifact.encodedSha256, + decoded_size: artifact.decodedSize, + encoded_size: artifact.encodedSize, + relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.encodedSha256}`, }, }, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index e47a23b893b71..f7bc711d4bd05 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -139,8 +139,9 @@ export class ManifestManager { const artifact = newManifest.getArtifact(diff.id); try { await this.artifactClient.createArtifact(artifact); + // Cache the body of the artifact - this.cache.set(diff.id, artifact.body); + this.cache.set(diff.id, Buffer.from(artifact.body, 'base64').toString()); } catch (err) { if (err.status === 409) { // This artifact already existed... diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 2721592ba3350..1f1c6f27b636a 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -71,12 +71,108 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() - .expect(200); + .expect(200) + .expect((response) => { + const artifactJson = JSON.parse(response.text); + expect(artifactJson).to.eql({ + entries: [ + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Elastic, N.V.', + }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + ], + }); + }); + }); + + it('should download an artifact with correct hash from cache', async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200) + .expect((response) => { + JSON.parse(response.text); + }) + .then(async () => { + await supertestWithoutAuth + .get( + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + ) + .set('kbn-xsrf', 'xxx') + .set('authorization', `ApiKey ${agentAccessAPIKey}`) + .send() + .expect(200) + .expect((response) => { + const artifactJson = JSON.parse(response.text); + expect(artifactJson).to.eql({ + entries: [ + { + type: 'simple', + entries: [ + { + field: 'actingProcess.file.signer', + operator: 'included', + type: 'exact_cased', + value: 'Elastic, N.V.', + }, + { + entries: [ + { + field: 'signer', + operator: 'included', + type: 'exact_cased', + value: 'Evil', + }, + { + field: 'trusted', + operator: 'included', + type: 'exact_cased', + value: 'true', + }, + ], + field: 'file.signature', + type: 'nested', + }, + ], + }, + ], + }); + }); + }); }); it('should fail on invalid api key', async () => { diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index b156f2f6cc7bf..3433070c08009 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,23 +1,23 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", + "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "index": ".kibana", "source": { "references": [ ], - "endpoint:user-artifact": { - "body": "eyJleGNlcHRpb25zX2xpc3QiOltdfQ==", + "endpoint:user-artifact:v2": { + "body": "eyJlbnRyaWVzIjpbeyJ0eXBlIjoic2ltcGxlIiwiZW50cmllcyI6W3siZmllbGQiOiJhY3RpbmdQcm9jZXNzLmZpbGUuc2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRWxhc3RpYywgTi5WLiJ9LHsiZW50cmllcyI6W3siZmllbGQiOiJzaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFdmlsIn0seyJmaWVsZCI6InRydXN0ZWQiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJ0cnVlIn1dLCJmaWVsZCI6ImZpbGUuc2lnbmF0dXJlIiwidHlwZSI6Im5lc3RlZCJ9XX1dfQ==", "created": 1593016187465, "compressionAlgorithm": "none", "encryptionAlgorithm": "none", "identifier": "endpoint-exceptionlist-linux-1.0.0", - "compressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "compressedSize": 22, - "decompressedSha256": "a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "decompressedSize": 22 + "encodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "encodedSize": 358, + "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "decodedSize": 358 }, - "type": "endpoint:user-artifact", + "type": "endpoint:user-artifact:v2", "updated_at": "2020-06-24T16:29:47.584Z" } } @@ -26,20 +26,20 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact-manifest:endpoint-manifest-1.0.0", + "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-1.0.0", "index": ".kibana", "source": { "references": [ ], - "endpoint:user-artifact-manifest": { + "endpoint:user-artifact-manifest:v2": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "endpoint-exceptionlist-macos-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d", - "endpoint-exceptionlist-windows-1.0.0-a4e4586e895fcb46dd25a25358b446f9a425279452afa3ef9a98bca39c39122d" + "endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "endpoint-exceptionlist-macos-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "endpoint-exceptionlist-windows-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" ] }, - "type": "endpoint:user-artifact-manifest", + "type": "endpoint:user-artifact-manifest:v2", "updated_at": "2020-06-26T15:01:39.704Z" } }