From 83906292d892cdcd4001386e6089d965da208f69 Mon Sep 17 00:00:00 2001 From: Paul Tavares <56442535+paul-tavares@users.noreply.github.com> Date: Mon, 15 Mar 2021 16:49:11 -0400 Subject: [PATCH] [SECURITY_SOLUTION][ENDPOINT] Create artifact manifests with new relative URL (if fleet-server is enabled) (#94499) * When xpack.securitySolution.fleetServerEnabled is true, then Endpoint artifact manifest will use a fleet-server relative url for the artifacts generated (note: this flag is temporary until we ship v7.13) * Refactors the security solution fleet integration extension point callbacks so that some action handlers can be executed in parallel (the creation of detection engine prepackaged rules can sometime take some time to complete --- x-pack/plugins/fleet/server/index.ts | 2 + .../endpoint/endpoint_app_context_services.ts | 2 +- .../server/endpoint/ingest_integration.ts | 195 ------------------ .../endpoint/lib/artifacts/manifest.test.ts | 47 +++++ .../server/endpoint/lib/artifacts/manifest.ts | 11 +- .../endpoint/lib/artifacts/manifest_entry.ts | 10 +- .../endpoint/lib/artifacts/task.test.ts | 27 ++- .../manifest_manager/manifest_manager.test.ts | 2 +- .../manifest_manager/manifest_manager.ts | 46 +++-- .../fleet_integration.test.ts} | 22 +- .../fleet_integration/fleet_integration.ts | 121 +++++++++++ .../handlers/create_default_policy.ts | 22 ++ .../create_policy_artifact_manifest.ts | 65 ++++++ .../handlers/install_prepackaged_rules.ts | 80 +++++++ .../validate_policy_against_license.ts | 25 +++ .../server/fleet_integration/index.ts | 8 + .../security_solution/server/plugin.ts | 19 +- 17 files changed, 457 insertions(+), 247 deletions(-) delete mode 100644 x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts rename x-pack/plugins/security_solution/server/{endpoint/ingest_integration.test.ts => fleet_integration/fleet_integration.test.ts} (94%) create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_artifact_manifest.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/install_prepackaged_rules.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/handlers/validate_policy_against_license.ts create mode 100644 x-pack/plugins/security_solution/server/fleet_integration/index.ts diff --git a/x-pack/plugins/fleet/server/index.ts b/x-pack/plugins/fleet/server/index.ts index 719a7dba599e9..73a8b419a869d 100644 --- a/x-pack/plugins/fleet/server/index.ts +++ b/x-pack/plugins/fleet/server/index.ts @@ -79,6 +79,8 @@ export type FleetConfigType = TypeOf<typeof config.schema>; export { PackagePolicyServiceInterface } from './services/package_policy'; +export { relativeDownloadUrlFromArtifact } from './services/artifacts/mappings'; + export const plugin = (initializerContext: PluginInitializerContext) => { return new FleetPlugin(initializerContext); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 2de4ac626e8c9..f4a5d6add4f41 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -24,7 +24,7 @@ import { PluginStartContract as AlertsPluginStartContract } from '../../../alert import { getPackagePolicyCreateCallback, getPackagePolicyUpdateCallback, -} from './ingest_integration'; +} from '../fleet_integration/fleet_integration'; import { ManifestManager } from './services/artifacts'; import { MetadataQueryStrategy } from './types'; import { MetadataQueryStrategyVersions } from '../../common/endpoint/types'; diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts deleted file mode 100644 index a70ec4b215a81..0000000000000 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { ExceptionListClient } from '../../../lists/server'; -import { PluginStartContract as AlertsStartContract } from '../../../alerting/server'; -import { SecurityPluginSetup } from '../../../security/server'; -import { ExternalCallback } from '../../../fleet/server'; -import { KibanaRequest, Logger, RequestHandlerContext } from '../../../../../src/core/server'; -import { NewPackagePolicy, UpdatePackagePolicy } from '../../../fleet/common/types/models'; -import { - policyFactory as policyConfigFactory, - policyFactoryWithoutPaidFeatures as policyConfigFactoryWithoutPaidFeatures, -} from '../../common/endpoint/models/policy_config'; -import { NewPolicyData } from '../../common/endpoint/types'; -import { ManifestManager } from './services/artifacts'; -import { Manifest } from './lib/artifacts'; -import { reportErrors } from './lib/artifacts/common'; -import { InternalArtifactCompleteSchema } from './schemas/artifacts'; -import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest'; -import { AppClientFactory } from '../client'; -import { createDetectionIndex } from '../lib/detection_engine/routes/index/create_index_route'; -import { createPrepackagedRules } from '../lib/detection_engine/routes/rules/add_prepackaged_rules_route'; -import { buildFrameworkRequest } from '../lib/timeline/routes/utils/common'; -import { isEndpointPolicyValidForLicense } from '../../common/license/policy_config'; -import { isAtLeast, LicenseService } from '../../common/license/license'; - -const getManifest = async (logger: Logger, manifestManager: ManifestManager): Promise<Manifest> => { - let manifest: Manifest | null = null; - - try { - manifest = await manifestManager.getLastComputedManifest(); - - // If we have not yet computed a manifest, then we have to do so now. This should only happen - // once. - if (manifest == null) { - // New computed manifest based on current state of exception list - const newManifest = await manifestManager.buildNewManifest(); - - // Persist new artifacts - const persistErrors = await manifestManager.pushArtifacts( - newManifest.getAllArtifacts() as InternalArtifactCompleteSchema[] - ); - if (persistErrors.length) { - reportErrors(logger, persistErrors); - throw new Error('Unable to persist new artifacts.'); - } - - // Commit the manifest state - await manifestManager.commit(newManifest); - - manifest = newManifest; - } - } catch (err) { - logger.error(err); - } - - return manifest ?? Manifest.getDefault(); -}; - -/** - * Callback to handle creation of PackagePolicies in Fleet - */ -export const getPackagePolicyCreateCallback = ( - logger: Logger, - manifestManager: ManifestManager, - appClientFactory: AppClientFactory, - maxTimelineImportExportSize: number, - securitySetup: SecurityPluginSetup, - alerts: AlertsStartContract, - licenseService: LicenseService, - exceptionsClient: ExceptionListClient | undefined -): ExternalCallback[1] => { - return async ( - newPackagePolicy: NewPackagePolicy, - context: RequestHandlerContext, - request: KibanaRequest - ): Promise<NewPackagePolicy> => { - // We only care about Endpoint package policies - if (newPackagePolicy.package?.name !== 'endpoint') { - return newPackagePolicy; - } - - // prep for detection rules creation - const appClient = appClientFactory.create(request); - // This callback is called by fleet plugin. - // It doesn't have access to SecuritySolutionRequestHandlerContext in runtime. - // Muting the error to have green CI. - // @ts-expect-error - const frameworkRequest = await buildFrameworkRequest(context, securitySetup, request); - - // Create detection index & rules (if necessary). move past any failure, this is just a convenience - try { - // @ts-expect-error - await createDetectionIndex(context, appClient); - } catch (err) { - if (err.statusCode !== 409) { - // 409 -> detection index already exists, which is fine - logger.warn( - `Possible problem creating detection signals index (${err.statusCode}): ${err.message}` - ); - } - } - try { - // this checks to make sure index exists first, safe to try in case of failure above - // may be able to recover from minor errors - await createPrepackagedRules( - // @ts-expect-error - context, - appClient, - alerts.getAlertsClientWithRequest(request), - frameworkRequest, - maxTimelineImportExportSize, - exceptionsClient - ); - } catch (err) { - logger.error( - `Unable to create detection rules automatically (${err.statusCode}): ${err.message}` - ); - } - - // Get most recent manifest - const manifest = await getManifest(logger, manifestManager); - const serializedManifest = manifest.toPackagePolicyManifest(); - if (!manifestDispatchSchema.is(serializedManifest)) { - // This should not happen. - // But if it does, we log it and return it anyway. - logger.error('Invalid manifest'); - } - - // We cast the type here so that any changes to the Endpoint specific data - // follow the types/schema expected - let updatedPackagePolicy = newPackagePolicy as NewPolicyData; - - // Until we get the Default Policy Configuration in the Endpoint package, - // we will add it here manually at creation time. - - // generate the correct default policy depending on the license - const defaultPolicy = isAtLeast(licenseService.getLicenseInformation(), 'platinum') - ? policyConfigFactory() - : policyConfigFactoryWithoutPaidFeatures(); - - updatedPackagePolicy = { - ...newPackagePolicy, - inputs: [ - { - type: 'endpoint', - enabled: true, - streams: [], - config: { - artifact_manifest: { - value: serializedManifest, - }, - policy: { - value: defaultPolicy, - }, - }, - }, - ], - }; - - return updatedPackagePolicy; - }; -}; - -export const getPackagePolicyUpdateCallback = ( - logger: Logger, - licenseService: LicenseService -): ExternalCallback[1] => { - return async ( - newPackagePolicy: NewPackagePolicy, - context: RequestHandlerContext, - request: KibanaRequest - ): Promise<UpdatePackagePolicy> => { - if (newPackagePolicy.package?.name !== 'endpoint') { - return newPackagePolicy; - } - - if ( - !isEndpointPolicyValidForLicense( - newPackagePolicy.inputs[0].config?.policy?.value, - licenseService.getLicenseInformation() - ) - ) { - logger.warn('Incorrect license tier for paid policy fields'); - const licenseError: Error & { statusCode?: number } = new Error('Requires Platinum license'); - licenseError.statusCode = 403; - throw licenseError; - } - return newPackagePolicy; - }; -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index beaf0c06299fa..cc1dda05f6738 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -10,6 +10,7 @@ import { InternalArtifactCompleteSchema } from '../../schemas'; import { getArtifactId } from './common'; import { isEmptyManifestDiff, Manifest } from './manifest'; import { getMockArtifacts, toArtifactRecords } from './mocks'; +import { cloneDeepWith, CloneDeepWithCustomizer } from 'lodash'; describe('manifest', () => { const TEST_POLICY_ID_1 = 'c6d16e42-c32d-4dce-8a88-113cfe276ad1'; @@ -694,4 +695,50 @@ describe('manifest', () => { expect(isEmptyManifestDiff(diff)).toBe(false); }); }); + + describe('and Fleet Server is enabled', () => { + const convertToFleetServerRelativeUrl: CloneDeepWithCustomizer<unknown> = (value, key) => { + if (key === 'relative_url') { + return value.replace('/api/endpoint/artifacts/download/', '/api/fleet/artifacts/'); + } + }; + let manifest: Manifest; + + beforeEach(() => { + manifest = new Manifest({ schemaVersion: 'v1', semanticVersion: '1.0.0' }, true); + + manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); + manifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); + manifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS, TEST_POLICY_ID_1); + }); + + test('should write manifest for global artifacts with fleet-server relative url', () => { + expect(manifest.toPackagePolicyManifest()).toStrictEqual({ + schema_version: 'v1', + manifest_version: '1.0.0', + artifacts: cloneDeepWith( + toArtifactRecords({ + 'endpoint-exceptionlist-windows-v1': ARTIFACT_EXCEPTIONS_WINDOWS, + 'endpoint-exceptionlist-macos-v1': ARTIFACT_EXCEPTIONS_MACOS, + }), + convertToFleetServerRelativeUrl + ), + }); + }); + + test('should write policy specific manifest with fleet-server relative url', () => { + expect(manifest.toPackagePolicyManifest(TEST_POLICY_ID_1)).toStrictEqual({ + schema_version: 'v1', + manifest_version: '1.0.0', + artifacts: cloneDeepWith( + toArtifactRecords({ + 'endpoint-exceptionlist-windows-v1': ARTIFACT_EXCEPTIONS_WINDOWS, + 'endpoint-trustlist-macos-v1': ARTIFACT_TRUSTED_APPS_MACOS, + }), + convertToFleetServerRelativeUrl + ), + }); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index 7e1accac37cf0..aefda4cf1f88d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -56,7 +56,10 @@ export class Manifest { private readonly policySpecificEntries: Map<string, Map<string, ManifestEntry>>; private version: ManifestVersion; - constructor(version?: Partial<ManifestVersion>) { + constructor( + version?: Partial<ManifestVersion>, + private readonly isFleetServerEnabled: boolean = false + ) { this.allEntries = new Map(); this.defaultEntries = new Map(); this.policySpecificEntries = new Map(); @@ -75,8 +78,8 @@ export class Manifest { this.version = validated; } - public static getDefault(schemaVersion?: ManifestSchemaVersion) { - return new Manifest({ schemaVersion, semanticVersion: '1.0.0' }); + public static getDefault(schemaVersion?: ManifestSchemaVersion, isFleetServerEnabled?: boolean) { + return new Manifest({ schemaVersion, semanticVersion: '1.0.0' }, isFleetServerEnabled); } public bumpSemanticVersion() { @@ -104,7 +107,7 @@ export class Manifest { const descriptor = { isDefaultEntry: existingDescriptor?.isDefaultEntry || policyId === undefined, specificTargetPolicies: addValueToSet(existingDescriptor?.specificTargetPolicies, policyId), - entry: existingDescriptor?.entry || new ManifestEntry(artifact), + entry: existingDescriptor?.entry || new ManifestEntry(artifact, this.isFleetServerEnabled), }; this.allEntries.set(descriptor.entry.getDocId(), descriptor); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index be38e52e94e9e..4dcdfa23e0d63 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -9,11 +9,12 @@ import { InternalArtifactSchema } from '../../schemas/artifacts'; import { CompressionAlgorithm } from '../../../../common/endpoint/schema/common'; import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; import { getArtifactId } from './common'; +import { relativeDownloadUrlFromArtifact } from '../../../../../fleet/server'; export class ManifestEntry { private artifact: InternalArtifactSchema; - constructor(artifact: InternalArtifactSchema) { + constructor(artifact: InternalArtifactSchema, private isFleetServerEnabled: boolean = false) { this.artifact = artifact; } @@ -46,6 +47,13 @@ export class ManifestEntry { } public getUrl(): string { + if (this.isFleetServerEnabled) { + return relativeDownloadUrlFromArtifact({ + identifier: this.getIdentifier(), + decodedSha256: this.getDecodedSha256(), + }); + } + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getDecodedSha256()}`; } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts index 9fac617f1f06d..1bf4802152ad2 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.test.ts @@ -16,7 +16,6 @@ import { ManifestManager } from '../../services/artifacts/manifest_manager'; import { buildManifestManagerMock } from '../../services/artifacts/manifest_manager/manifest_manager.mock'; import { InternalArtifactCompleteSchema } from '../../schemas/artifacts'; import { getMockArtifacts } from './mocks'; -import { Manifest } from './manifest'; describe('task', () => { const MOCK_TASK_INSTANCE = { @@ -129,7 +128,7 @@ describe('task', () => { test('Should stop the process when no building new manifest throws error', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); manifestManager.getLastComputedManifest = jest.fn().mockReturnValue(lastManifest); manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error()); @@ -147,11 +146,11 @@ describe('task', () => { test('Should not bump version and commit manifest when no diff in the manifest', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); @@ -176,9 +175,9 @@ describe('task', () => { test('Should stop the process when there are errors pushing new artifacts', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); @@ -204,9 +203,9 @@ describe('task', () => { test('Should stop the process when there are errors committing manifest', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); @@ -233,9 +232,9 @@ describe('task', () => { test('Should stop the process when there are errors dispatching manifest', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); @@ -263,11 +262,11 @@ describe('task', () => { test('Should succeed the process and delete old artifacts', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); @@ -293,11 +292,11 @@ describe('task', () => { test('Should succeed the process but not add or delete artifacts when there are only transitions', async () => { const manifestManager = buildManifestManagerMock(); - const lastManifest = Manifest.getDefault(); + const lastManifest = ManifestManager.createDefaultManifest(); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); lastManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_1); - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); newManifest.addEntry(ARTIFACT_EXCEPTIONS_WINDOWS, TEST_POLICY_ID_2); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 26db49be459fa..a4efdbc75fb16 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -581,7 +581,7 @@ describe('ManifestManager', () => { test('Creates new saved object if no saved object version', async () => { const context = buildManifestManagerContextMock({}); const manifestManager = new ManifestManager(context); - const manifest = Manifest.getDefault(); + const manifest = ManifestManager.createDefaultManifest(); manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); manifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS, TEST_POLICY_ID_1); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index 8c8ea34acfb8b..e219da38931da 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -100,7 +100,10 @@ export class ManifestManager { protected cache: LRU<string, Buffer>; protected schemaVersion: ManifestSchemaVersion; - constructor(context: ManifestManagerContext) { + constructor( + context: ManifestManagerContext, + private readonly isFleetServerEnabled: boolean = false + ) { this.artifactClient = context.artifactClient; this.exceptionListClient = context.exceptionListClient; this.packagePolicyService = context.packagePolicyService; @@ -277,11 +280,14 @@ export class ManifestManager { throw new Error('No version returned for manifest.'); } - const manifest = new Manifest({ - schemaVersion: this.schemaVersion, - semanticVersion: manifestSo.attributes.semanticVersion, - soVersion: manifestSo.version, - }); + const manifest = new Manifest( + { + schemaVersion: this.schemaVersion, + semanticVersion: manifestSo.attributes.semanticVersion, + soVersion: manifestSo.version, + }, + this.isFleetServerEnabled + ); for (const entry of manifestSo.attributes.artifacts) { manifest.addEntry( @@ -299,6 +305,16 @@ export class ManifestManager { } } + /** + * creates a new default Manifest + */ + public static createDefaultManifest( + schemaVersion?: ManifestSchemaVersion, + isFleetServerEnabled?: boolean + ): Manifest { + return Manifest.getDefault(schemaVersion, isFleetServerEnabled); + } + /** * Builds a new manifest based on the current user exception list. * @@ -306,18 +322,24 @@ export class ManifestManager { * @returns {Promise<Manifest>} A new Manifest object reprenting the current exception list. */ public async buildNewManifest( - baselineManifest: Manifest = Manifest.getDefault(this.schemaVersion) + baselineManifest: Manifest = ManifestManager.createDefaultManifest( + this.schemaVersion, + this.isFleetServerEnabled + ) ): Promise<Manifest> { const results = await Promise.all([ this.buildExceptionListArtifacts(), this.buildTrustedAppsArtifacts(), ]); - const manifest = new Manifest({ - schemaVersion: this.schemaVersion, - semanticVersion: baselineManifest.getSemanticVersion(), - soVersion: baselineManifest.getSavedObjectVersion(), - }); + const manifest = new Manifest( + { + schemaVersion: this.schemaVersion, + semanticVersion: baselineManifest.getSemanticVersion(), + soVersion: baselineManifest.getSavedObjectVersion(), + }, + this.isFleetServerEnabled + ); for (const result of results) { await iterateArtifactsBuildResult(result, async (artifact, policyId) => { diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts similarity index 94% rename from x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts rename to x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts index f16f1f6402818..5d0f51d52c2cf 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.test.ts @@ -11,15 +11,15 @@ import { policyFactory, policyFactoryWithoutPaidFeatures, } from '../../common/endpoint/models/policy_config'; -import { buildManifestManagerMock } from './services/artifacts/manifest_manager/manifest_manager.mock'; +import { buildManifestManagerMock } from '../endpoint/services/artifacts/manifest_manager/manifest_manager.mock'; import { getPackagePolicyCreateCallback, getPackagePolicyUpdateCallback, -} from './ingest_integration'; +} from './fleet_integration'; import { KibanaRequest } from 'kibana/server'; import { createMockConfig, requestContextMock } from '../lib/detection_engine/routes/__mocks__'; -import { EndpointAppContextServiceStartContract } from './endpoint_app_context_services'; -import { createMockEndpointAppContextServiceStartContract } from './mocks'; +import { EndpointAppContextServiceStartContract } from '../endpoint/endpoint_app_context_services'; +import { createMockEndpointAppContextServiceStartContract } from '../endpoint/mocks'; import { licenseMock } from '../../../licensing/common/licensing.mock'; import { LicenseService } from '../../common/license/license'; import { Subject } from 'rxjs'; @@ -29,10 +29,10 @@ import { ProtectionModes } from '../../common/endpoint/types'; import type { SecuritySolutionRequestHandlerContext } from '../types'; import { getExceptionListClientMock } from '../../../lists/server/services/exception_lists/exception_list_client.mock'; import { ExceptionListClient } from '../../../lists/server'; -import { InternalArtifactCompleteSchema } from './schemas/artifacts'; -import { ManifestManager } from './services/artifacts/manifest_manager'; -import { getMockArtifacts, toArtifactRecords } from './lib/artifacts/mocks'; -import { Manifest } from './lib/artifacts'; +import { InternalArtifactCompleteSchema } from '../endpoint/schemas/artifacts'; +import { ManifestManager } from '../endpoint/services/artifacts/manifest_manager'; +import { getMockArtifacts, toArtifactRecords } from '../endpoint/lib/artifacts/mocks'; +import { Manifest } from '../endpoint/lib/artifacts'; import { NewPackagePolicy } from '../../../fleet/common/types/models'; import { ManifestSchema } from '../../common/endpoint/schema/manifest'; @@ -130,7 +130,7 @@ describe('ingest_integration tests ', () => { }); test('default manifest is taken when there is none and there are errors pushing artifacts', async () => { - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); const manifestManager = buildManifestManagerMock(); @@ -152,7 +152,7 @@ describe('ingest_integration tests ', () => { }); test('default manifest is taken when there is none and there are errors commiting manifest', async () => { - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); const manifestManager = buildManifestManagerMock(); @@ -175,7 +175,7 @@ describe('ingest_integration tests ', () => { }); test('manifest is created successfuly when there is none', async () => { - const newManifest = Manifest.getDefault(); + const newManifest = ManifestManager.createDefaultManifest(); newManifest.addEntry(ARTIFACT_EXCEPTIONS_MACOS); newManifest.addEntry(ARTIFACT_TRUSTED_APPS_MACOS); diff --git a/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts new file mode 100644 index 0000000000000..c9939a163c977 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/fleet_integration.ts @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest, Logger, RequestHandlerContext } from 'kibana/server'; +import { ExceptionListClient } from '../../../lists/server'; +import { PluginStartContract as AlertsStartContract } from '../../../alerting/server'; +import { SecurityPluginSetup } from '../../../security/server'; +import { ExternalCallback } from '../../../fleet/server'; +import { NewPackagePolicy, UpdatePackagePolicy } from '../../../fleet/common'; +import { NewPolicyData, PolicyConfig } from '../../common/endpoint/types'; +import { ManifestManager } from '../endpoint/services'; +import { AppClientFactory } from '../client'; +import { LicenseService } from '../../common/license/license'; +import { installPrepackagedRules } from './handlers/install_prepackaged_rules'; +import { createPolicyArtifactManifest } from './handlers/create_policy_artifact_manifest'; +import { createDefaultPolicy } from './handlers/create_default_policy'; +import { validatePolicyAgainstLicense } from './handlers/validate_policy_against_license'; + +const isEndpointPackagePolicy = <T extends { package?: { name: string } }>( + packagePolicy: T +): boolean => { + return packagePolicy.package?.name === 'endpoint'; +}; + +/** + * Callback to handle creation of PackagePolicies in Fleet + */ +export const getPackagePolicyCreateCallback = ( + logger: Logger, + manifestManager: ManifestManager, + appClientFactory: AppClientFactory, + maxTimelineImportExportSize: number, + securitySetup: SecurityPluginSetup, + alerts: AlertsStartContract, + licenseService: LicenseService, + exceptionsClient: ExceptionListClient | undefined +): ExternalCallback[1] => { + return async ( + newPackagePolicy: NewPackagePolicy, + context: RequestHandlerContext, + request: KibanaRequest + ): Promise<NewPackagePolicy> => { + // We only care about Endpoint package policies + if (!isEndpointPackagePolicy(newPackagePolicy)) { + return newPackagePolicy; + } + + // perform these operations in parallel in order to help in not delaying the API response too much + const [, manifestValue] = await Promise.all([ + // Install Detection Engine prepackaged rules + exceptionsClient && + installPrepackagedRules({ + logger, + appClientFactory, + context, + request, + securitySetup, + alerts, + maxTimelineImportExportSize, + exceptionsClient, + }), + + // create the Artifact Manifest for this policy + createPolicyArtifactManifest(logger, manifestManager), + ]); + + // Add the default endpoint security policy + const defaultPolicyValue = createDefaultPolicy(licenseService); + + return { + // We cast the type here so that any changes to the Endpoint + // specific data follow the types/schema expected + ...(newPackagePolicy as NewPolicyData), + inputs: [ + { + type: 'endpoint', + enabled: true, + streams: [], + config: { + artifact_manifest: { + value: manifestValue, + }, + policy: { + value: defaultPolicyValue, + }, + }, + }, + ], + }; + }; +}; + +export const getPackagePolicyUpdateCallback = ( + logger: Logger, + licenseService: LicenseService +): ExternalCallback[1] => { + return async ( + newPackagePolicy: NewPackagePolicy + // context: RequestHandlerContext, + // request: KibanaRequest + ): Promise<UpdatePackagePolicy> => { + if (!isEndpointPackagePolicy(newPackagePolicy)) { + return newPackagePolicy; + } + + // Validate that Endpoint Security policy is valid against current license + validatePolicyAgainstLicense( + // The cast below is needed in order to ensure proper typing for + // the policy configuration specific for endpoint + newPackagePolicy.inputs[0].config?.policy?.value as PolicyConfig, + licenseService, + logger + ); + + return newPackagePolicy; + }; +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts new file mode 100644 index 0000000000000..461d82ff7c9d1 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_default_policy.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + policyFactory as policyConfigFactory, + policyFactoryWithoutPaidFeatures as policyConfigFactoryWithoutPaidFeatures, +} from '../../../common/endpoint/models/policy_config'; +import { isAtLeast, LicenseService } from '../../../common/license/license'; +import { PolicyConfig } from '../../../common/endpoint/types'; + +/** + * Create the default endpoint policy based on the current license + */ +export const createDefaultPolicy = (licenseService: LicenseService): PolicyConfig => { + return isAtLeast(licenseService.getLicenseInformation(), 'platinum') + ? policyConfigFactory() + : policyConfigFactoryWithoutPaidFeatures(); +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_artifact_manifest.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_artifact_manifest.ts new file mode 100644 index 0000000000000..8c2d612709ff3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/create_policy_artifact_manifest.ts @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { ManifestManager } from '../../endpoint/services'; +import { Manifest, reportErrors } from '../../endpoint/lib/artifacts'; +import { InternalArtifactCompleteSchema } from '../../endpoint/schemas'; +import { manifestDispatchSchema, ManifestSchema } from '../../../common/endpoint/schema/manifest'; + +const getManifest = async (logger: Logger, manifestManager: ManifestManager): Promise<Manifest> => { + let manifest: Manifest | null = null; + + try { + manifest = await manifestManager.getLastComputedManifest(); + + // If we have not yet computed a manifest, then we have to do so now. This should only happen + // once. + if (manifest == null) { + // New computed manifest based on current state of exception list + const newManifest = await manifestManager.buildNewManifest(); + + // Persist new artifacts + const persistErrors = await manifestManager.pushArtifacts( + newManifest.getAllArtifacts() as InternalArtifactCompleteSchema[] + ); + if (persistErrors.length) { + reportErrors(logger, persistErrors); + throw new Error('Unable to persist new artifacts.'); + } + + // Commit the manifest state + await manifestManager.commit(newManifest); + + manifest = newManifest; + } + } catch (err) { + logger.error(err); + } + + return manifest ?? ManifestManager.createDefaultManifest(); +}; + +/** + * Creates the initial manifest to be included in a policy when it is first created in fleet + */ +export const createPolicyArtifactManifest = async ( + logger: Logger, + manifestManager: ManifestManager +): Promise<ManifestSchema> => { + // Get most recent manifest + const manifest = await getManifest(logger, manifestManager); + const serializedManifest = manifest.toPackagePolicyManifest(); + + if (!manifestDispatchSchema.is(serializedManifest)) { + // This should not happen. + // But if it does, we log it and return it anyway. + logger.error('Invalid manifest'); + } + + return serializedManifest; +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/install_prepackaged_rules.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/install_prepackaged_rules.ts new file mode 100644 index 0000000000000..b317b91b0ccea --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/install_prepackaged_rules.ts @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { KibanaRequest, Logger, RequestHandlerContext } from 'kibana/server'; +import { ExceptionListClient } from '../../../../lists/server'; +import { PluginStartContract as AlertsStartContract } from '../../../../alerting/server'; +import { SecurityPluginSetup } from '../../../../security/server'; +import { AppClientFactory } from '../../client'; +import { createDetectionIndex } from '../../lib/detection_engine/routes/index/create_index_route'; +import { createPrepackagedRules } from '../../lib/detection_engine/routes/rules/add_prepackaged_rules_route'; +import { buildFrameworkRequest } from '../../lib/timeline/routes/utils/common'; + +export interface InstallPrepackagedRulesProps { + logger: Logger; + appClientFactory: AppClientFactory; + context: RequestHandlerContext; + request: KibanaRequest; + securitySetup: SecurityPluginSetup; + alerts: AlertsStartContract; + maxTimelineImportExportSize: number; + exceptionsClient: ExceptionListClient; +} + +/** + * As part of a user taking advantage of Endpoint Security from within fleet, we attempt to install + * the pre-packaged rules from the detection engine, which includes an Endpoint Rule enabled by default + */ +export const installPrepackagedRules = async ({ + logger, + appClientFactory, + context, + request, + securitySetup, + alerts, + maxTimelineImportExportSize, + exceptionsClient, +}: InstallPrepackagedRulesProps): Promise<void> => { + // prep for detection rules creation + const appClient = appClientFactory.create(request); + + // This callback is called by fleet plugin. + // It doesn't have access to SecuritySolutionRequestHandlerContext in runtime. + // Muting the error to have green CI. + // @ts-expect-error + const frameworkRequest = await buildFrameworkRequest(context, securitySetup, request); + + // Create detection index & rules (if necessary). move past any failure, this is just a convenience + try { + // @ts-expect-error + await createDetectionIndex(context, appClient); + } catch (err) { + if (err.statusCode !== 409) { + // 409 -> detection index already exists, which is fine + logger.warn( + `Possible problem creating detection signals index (${err.statusCode}): ${err.message}` + ); + } + } + try { + // this checks to make sure index exists first, safe to try in case of failure above + // may be able to recover from minor errors + await createPrepackagedRules( + // @ts-expect-error + context, + appClient, + alerts.getAlertsClientWithRequest(request), + frameworkRequest, + maxTimelineImportExportSize, + exceptionsClient + ); + } catch (err) { + logger.error( + `Unable to create detection rules automatically (${err.statusCode}): ${err.message}` + ); + } +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/handlers/validate_policy_against_license.ts b/x-pack/plugins/security_solution/server/fleet_integration/handlers/validate_policy_against_license.ts new file mode 100644 index 0000000000000..50dff2ac89f49 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/handlers/validate_policy_against_license.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Logger } from 'kibana/server'; +import { isEndpointPolicyValidForLicense } from '../../../common/license/policy_config'; +import { PolicyConfig } from '../../../common/endpoint/types'; +import { LicenseService } from '../../../common/license/license'; + +export const validatePolicyAgainstLicense = ( + policyConfig: PolicyConfig, + licenseService: LicenseService, + logger: Logger +): void => { + if (!isEndpointPolicyValidForLicense(policyConfig, licenseService.getLicenseInformation())) { + logger.warn('Incorrect license tier for paid policy fields'); + // The `statusCode` below is used by Fleet API handler to ensure that the proper HTTP code is used in the API response + const licenseError: Error & { statusCode?: number } = new Error('Requires Platinum license'); + licenseError.statusCode = 403; + throw licenseError; + } +}; diff --git a/x-pack/plugins/security_solution/server/fleet_integration/index.ts b/x-pack/plugins/security_solution/server/fleet_integration/index.ts new file mode 100644 index 0000000000000..36855b74e48a9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/fleet_integration/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export * from './fleet_integration'; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 04a5a525d24bf..5ce1029951563 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -348,14 +348,17 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S plugins.fleet.createArtifactsClient('endpoint') ) as unknown) as ArtifactClient; - manifestManager = new ManifestManager({ - savedObjectsClient, - artifactClient, - exceptionListClient, - packagePolicyService: plugins.fleet.packagePolicyService, - logger: this.logger, - cache: this.artifactsCache, - }); + manifestManager = new ManifestManager( + { + savedObjectsClient, + artifactClient, + exceptionListClient, + packagePolicyService: plugins.fleet.packagePolicyService, + logger: this.logger, + cache: this.artifactsCache, + }, + this.config.fleetServerEnabled + ); if (this.manifestTask) { this.manifestTask.start({