diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts index 135c870f20727..43122ba49519a 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/index.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/index.ts @@ -73,8 +73,8 @@ export class RuleDataClient implements IRuleDataClient { return clusterClient.bulk(requestWithDefaultParameters).then((response) => { if (response.body.errors) { if ( - response.body.items.length === 1 && - response.body.items[0]?.index?.error?.type === 'index_not_found_exception' + response.body.items.length > 0 && + response.body.items?.[0]?.index?.error?.type === 'index_not_found_exception' ) { return this.createOrUpdateWriteTarget({ namespace }).then(() => { return clusterClient.bulk(requestWithDefaultParameters); diff --git a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts index 2d0b4e0ae33c3..0e244fbaa2ee3 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_persistence_rule_type_factory.ts @@ -80,6 +80,7 @@ export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory ...event, 'event.kind': 'signal', 'kibana.rac.alert.id': '???', + 'kibana.rac.alert.status': 'open', 'kibana.rac.alert.uuid': v4(), 'kibana.rac.alert.ancestors': isAlert ? ((event['kibana.rac.alert.ancestors'] as string[]) ?? []).concat([ @@ -101,12 +102,7 @@ export const createPersistenceRuleTypeFactory: CreatePersistenceRuleTypeFactory if (ruleDataClient && numAlerts) { await ruleDataClient.getWriter().bulk({ - body: currentAlerts.map((alert) => ({ - create: { - _index: {}, - }, - alert, - })), + body: currentAlerts.flatMap((event) => [{ index: {} }, event]), }); } diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index dddfafc7ddf05..91b48afdc4ed1 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -25,7 +25,7 @@ export const DEFAULT_TIME_RANGE = 'timepicker:timeDefaults'; export const DEFAULT_REFRESH_RATE_INTERVAL = 'timepicker:refreshIntervalDefaults'; export const DEFAULT_APP_TIME_RANGE = 'securitySolution:timeDefaults'; export const DEFAULT_APP_REFRESH_INTERVAL = 'securitySolution:refreshIntervalDefaults'; -export const DEFAULT_ALERTS_INDEX = '*alerts-security-solution*'; +export const DEFAULT_ALERTS_INDEX = '.alerts-security-solution'; export const DEFAULT_SIGNALS_INDEX = '.siem-signals'; export const DEFAULT_LISTS_INDEX = '.lists'; export const DEFAULT_ITEMS_INDEX = '.items'; diff --git a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx index eb922ffeb3361..02a815bc59f3b 100644 --- a/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/alerts_table/default_config.tsx @@ -167,30 +167,30 @@ export const buildShowBuildingBlockFilterRuleRegistry = ( }, ]; -export const requiredFieldsForActionsRuleRegistry = [ - 'alert.id', - '@timestamp', - 'event.kind', - 'alert.start', - 'alert.uuid', - 'event.action', - 'alert.status', - 'alert.duration.us', - 'rule.uuid', - 'rule.id', - 'rule.name', - 'rule.category', - 'producer', - 'tags', -]; +export const requiredFieldMappingsForActionsRuleRegistry = { + '@timestamp': '@timestamp', + 'alert.id': 'kibana.rac.alert.id', + 'event.kind': 'event.kind', + 'alert.start': 'kibana.rac.alert.start', + 'alert.uuid': 'kibana.rac.alert.uuid', + 'event.action': 'event.action', + 'alert.status': 'kibana.rac.alert.status', + 'alert.duration.us': 'kibana.rac.alert.duration.us', + 'rule.uuid': 'rule.uuid', + 'rule.id': 'rule.id', + 'rule.name': 'rule.name', + 'rule.category': 'rule.category', + producer: 'kibana.rac.alert.producer', + tags: 'tags', +}; -export const alertsHeadersRuleRegistry: ColumnHeaderOptions[] = requiredFieldsForActionsRuleRegistry.map( - (field) => ({ - columnHeaderType: defaultColumnHeaderType, - id: field, - width: 120, - }) -); +export const alertsHeadersRuleRegistry: ColumnHeaderOptions[] = Object.entries( + requiredFieldMappingsForActionsRuleRegistry +).map(([alias, field]) => ({ + columnHeaderType: defaultColumnHeaderType, + displayAsText: alias, + id: field, +})); export const alertsDefaultModelRuleRegistry: SubsetTimelineModel = { ...timelineDefaults, diff --git a/x-pack/plugins/security_solution/public/timelines/containers/helpers.ts b/x-pack/plugins/security_solution/public/timelines/containers/helpers.ts index 068f813e0fc40..2f815cc44b3ac 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/helpers.ts +++ b/x-pack/plugins/security_solution/public/timelines/containers/helpers.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { DEFAULT_ALERTS_INDEX } from '../../../common/constants'; import { TimelineId } from '../../../common/types/timeline'; export const detectionsTimelineIds = [ @@ -12,7 +13,14 @@ export const detectionsTimelineIds = [ TimelineId.detectionsRulesDetailsPage, ]; -export const skipQueryForDetectionsPage = (id: string, defaultIndex: string[]) => +// TODO: Once we are past experimental phase `useRuleRegistry` should be removed +export const skipQueryForDetectionsPage = ( + id: string, + defaultIndex: string[], + useRuleRegistry = false +) => id != null && detectionsTimelineIds.some((timelineId) => timelineId === id) && - !defaultIndex.some((di) => di.toLowerCase().startsWith('.siem-signals')); + !defaultIndex.some((di) => + di.toLowerCase().startsWith(useRuleRegistry ? DEFAULT_ALERTS_INDEX : '.siem-signals') + ); diff --git a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx index 92199336b978c..17c107899d85a 100644 --- a/x-pack/plugins/security_solution/public/timelines/containers/index.tsx +++ b/x-pack/plugins/security_solution/public/timelines/containers/index.tsx @@ -13,6 +13,7 @@ import { Subscription } from 'rxjs'; import { ESQuery } from '../../../common/typed_json'; import { isCompleteResponse, isErrorResponse } from '../../../../../../src/plugins/data/public'; +import { useIsExperimentalFeatureEnabled } from '../../common/hooks/use_experimental_features'; import { inputsModel, KueryFilterQueryKind } from '../../common/store'; import { useKibana } from '../../common/lib/kibana'; import { createFilter } from '../../common/containers/helpers'; @@ -197,6 +198,9 @@ export const useTimelineEvents = ({ }); const { addError, addWarning } = useAppToasts(); + // TODO: Once we are past experimental phase this code should be removed + const ruleRegistryEnabled = useIsExperimentalFeatureEnabled('ruleRegistryEnabled'); + const timelineSearch = useCallback( (request: TimelineRequest | null) => { if (request == null || pageName === '' || skip) { @@ -305,7 +309,10 @@ export const useTimelineEvents = ({ ); useEffect(() => { - if (skipQueryForDetectionsPage(id, indexNames) || indexNames.length === 0) { + if ( + skipQueryForDetectionsPage(id, indexNames, ruleRegistryEnabled) || + indexNames.length === 0 + ) { return; } @@ -364,7 +371,10 @@ export const useTimelineEvents = ({ activeTimeline.setActivePage(newActivePage); } } - if (!skipQueryForDetectionsPage(id, indexNames) && !deepEqual(prevRequest, currentRequest)) { + if ( + !skipQueryForDetectionsPage(id, indexNames, ruleRegistryEnabled) && + !deepEqual(prevRequest, currentRequest) + ) { return currentRequest; } return prevRequest; @@ -380,6 +390,7 @@ export const useTimelineEvents = ({ id, language, limit, + ruleRegistryEnabled, startDate, sort, fields, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts index dd4d4fb54afba..3911dcabc34de 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/reference_rules/query.ts @@ -48,36 +48,41 @@ export const createQueryAlertType = (ruleDataClient: RuleDataClient, logger: Log services: { alertWithPersistence, findAlerts }, params: { indexPatterns, customQuery }, }) { - const indexPattern: IIndexPattern = { - fields: [], - title: indexPatterns.join(), - }; + try { + const indexPattern: IIndexPattern = { + fields: [], + title: indexPatterns.join(), + }; - // TODO: kql or lucene? + // TODO: kql or lucene? - const esQuery = buildEsQuery( - indexPattern, - { query: customQuery, language: 'kuery' }, - [] - ) as QueryContainer; - const query: ESSearchRequest = { - body: { - query: esQuery, - fields: ['*'], - sort: { - '@timestamp': 'asc' as const, + const esQuery = buildEsQuery( + indexPattern, + { query: customQuery, language: 'kuery' }, + [] + ) as QueryContainer; + const query: ESSearchRequest = { + body: { + query: esQuery, + fields: ['*'], + sort: { + '@timestamp': 'asc' as const, + }, }, - }, - }; + }; - const alerts = await findAlerts(query); - alertWithPersistence(alerts).forEach((alert) => { - alert.scheduleActions('default', { server: 'server-test' }); - }); + const alerts = await findAlerts(query); + // console.log('alerts', alerts); + alertWithPersistence(alerts).forEach((alert) => { + alert.scheduleActions('default', { server: 'server-test' }); + }); - return { - lastChecked: new Date(), - }; + return { + lastChecked: new Date(), + }; + } catch (error) { + logger.error(error); + } }, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index 6af4397a4193a..3527e43c03d52 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -6,15 +6,17 @@ */ import { transformError, getIndexExists } from '@kbn/securitysolution-es-utils'; +import { parseExperimentalConfigValue } from '../../../../../common/experimental_features'; +import { ConfigType } from '../../../../config'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; +import { DEFAULT_ALERTS_INDEX, DETECTION_ENGINE_INDEX_URL } from '../../../../../common/constants'; import { buildSiemResponse } from '../utils'; import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; import { getIndexVersion } from './get_index_version'; import { isOutdated } from '../../migrations/helpers'; -export const readIndexRoute = (router: SecuritySolutionPluginRouter) => { +export const readIndexRoute = (router: SecuritySolutionPluginRouter, config: ConfigType) => { router.get( { path: DETECTION_ENGINE_INDEX_URL, @@ -34,8 +36,16 @@ export const readIndexRoute = (router: SecuritySolutionPluginRouter) => { return siemResponse.error({ statusCode: 404 }); } + // TODO: Once we are past experimental phase this code should be removed + const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental); + if (ruleRegistryEnabled) { + return response.ok({ + body: { name: DEFAULT_ALERTS_INDEX, index_mapping_outdated: false }, + }); + } + const index = siemClient.getSignalsIndex(); - const indexExists = await getIndexExists(esClient, index); + const indexExists = ruleRegistryEnabled ? true : await getIndexExists(esClient, index); if (indexExists) { let mappingOutdated: boolean | null = null; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.test.ts index 909c94f145528..d6b998e314234 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.test.ts @@ -14,7 +14,7 @@ import { getSignalsAggsAndQueryRequest, getEmptySignalsResponse, } from '../__mocks__/request_responses'; -import { requestContextMock, serverMock, requestMock } from '../__mocks__'; +import { requestContextMock, serverMock, requestMock, createMockConfig } from '../__mocks__'; import { querySignalsRoute } from './query_signals_route'; describe('query for signal', () => { @@ -27,7 +27,7 @@ describe('query for signal', () => { clients.clusterClient.callAsCurrentUser.mockResolvedValue(getEmptySignalsResponse()); - querySignalsRoute(server.router); + querySignalsRoute(server.router, createMockConfig()); }); describe('query and agg on signals index', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.ts index 91172a277bf54..770c1a5da344f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/query_signals_route.ts @@ -6,8 +6,13 @@ */ import { transformError } from '@kbn/securitysolution-es-utils'; +import { parseExperimentalConfigValue } from '../../../../../common/experimental_features'; +import { ConfigType } from '../../../../config'; import type { SecuritySolutionPluginRouter } from '../../../../types'; -import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants'; +import { + DEFAULT_ALERTS_INDEX, + DETECTION_ENGINE_QUERY_SIGNALS_URL, +} from '../../../../../common/constants'; import { buildSiemResponse } from '../utils'; import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; @@ -16,7 +21,7 @@ import { QuerySignalsSchemaDecoded, } from '../../../../../common/detection_engine/schemas/request/query_signals_index_schema'; -export const querySignalsRoute = (router: SecuritySolutionPluginRouter) => { +export const querySignalsRoute = (router: SecuritySolutionPluginRouter, config: ConfigType) => { router.post( { path: DETECTION_ENGINE_QUERY_SIGNALS_URL, @@ -48,9 +53,12 @@ export const querySignalsRoute = (router: SecuritySolutionPluginRouter) => { const clusterClient = context.core.elasticsearch.legacy.client; const siemClient = context.securitySolution!.getAppClient(); + // TODO: Once we are past experimental phase this code should be removed + const { ruleRegistryEnabled } = parseExperimentalConfigValue(config.enableExperimental); + try { const result = await clusterClient.callAsCurrentUser('search', { - index: siemClient.getSignalsIndex(), + index: ruleRegistryEnabled ? DEFAULT_ALERTS_INDEX : siemClient.getSignalsIndex(), body: { query, aggs, _source, track_total_hits, size }, ignoreUnavailable: true, }); diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 14119a8796368..2507475592e88 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -5,6 +5,7 @@ * 2.0. */ +import { once } from 'lodash'; import { Observable } from 'rxjs'; import { i18n } from '@kbn/i18n'; import LRU from 'lru-cache'; @@ -27,9 +28,18 @@ import { PluginSetupContract as AlertingSetup, PluginStartContract as AlertPluginStartContract, } from '../../alerting/server'; + import { PluginStartContract as CasesPluginStartContract } from '../../cases/server'; +import { + ECS_COMPONENT_TEMPLATE_NAME, + TECHNICAL_COMPONENT_TEMPLATE_NAME, +} from '../../rule_registry/common/assets'; import { SecurityPluginSetup as SecuritySetup, SecurityPluginStart } from '../../security/server'; -import { RuleDataClient, RuleRegistryPluginSetupContract } from '../../rule_registry/server'; +import { + RuleDataClient, + RuleRegistryPluginSetupContract, + RuleRegistryPluginStartContract, +} from '../../rule_registry/server'; import { PluginSetupContract as FeaturesSetup } from '../../features/server'; import { MlPluginSetup as MlSetup } from '../../ml/server'; import { ListPluginSetup } from '../../lists/server'; @@ -106,7 +116,7 @@ export interface StartPlugins { data: DataPluginStart; fleet?: FleetStartContract; licensing: LicensingPluginStart; - ruleRegistry: RuleRegistryPluginSetupContract; + ruleRegistry: RuleRegistryPluginStartContract; taskManager?: TaskManagerStartContract; telemetry?: TelemetryPluginStart; security: SecurityPluginStart; @@ -208,16 +218,54 @@ export class Plugin implements IPlugin core.getStartServices().then(([coreStart]) => coreStart); + const ready = once(async () => { + const componentTemplateName = ruleDataService.getFullAssetName( + 'security-solution-mappings' + ); + + if (!ruleDataService.isWriteEnabled()) { + return; + } + + await ruleDataService.createOrUpdateComponentTemplate({ + name: componentTemplateName, + body: { + template: { + settings: { + number_of_shards: 1, + }, + mappings: {}, // TODO: Add mappings here via `mappingFromFieldMap()` + }, + }, + }); + + await ruleDataService.createOrUpdateIndexTemplate({ + name: ruleDataService.getFullAssetName('security-solution-index-template'), + body: { + index_patterns: [ruleDataService.getFullAssetName('security-solution*')], + composed_of: [ + ruleDataService.getFullAssetName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + ruleDataService.getFullAssetName(ECS_COMPONENT_TEMPLATE_NAME), + componentTemplateName, + ], + }, + }); + }); + + ready().catch((err) => { + this.logger!.error(err); + }); + ruleDataClient = new RuleDataClient({ + alias: plugins.ruleRegistry.ruleDataService.getFullAssetName('security-solution'), getClusterClient: async () => { const coreStart = await start(); return coreStart.elasticsearch.client.asInternalUser; }, - ready: () => Promise.resolve(), - alias: plugins.ruleRegistry.getFullAssetName(), + ready, }); // Register reference rule types via rule-registry diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 0ea7809bd1a15..0245d4cb99cc0 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -107,7 +107,7 @@ export const initRoutes = ( // POST /api/detection_engine/signals/status // Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals setSignalsStatusRoute(router); - querySignalsRoute(router); + querySignalsRoute(router, config); getSignalsMigrationStatusRoute(router); createSignalsMigrationRoute(router, security); finalizeSignalsMigrationRoute(router, security); @@ -116,7 +116,7 @@ export const initRoutes = ( // Detection Engine index routes that have the REST endpoints of /api/detection_engine/index // All REST index creation, policy management for spaces createIndexRoute(router); - readIndexRoute(router); + readIndexRoute(router, config); deleteIndexRoute(router); // Detection Engine tags routes that have the REST endpoints of /api/detection_engine/tags