From a12d8551a1c89fc674653c315e2947ff8d4492e5 Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Wed, 11 Dec 2019 11:09:36 -0500 Subject: [PATCH] [SIEM] [Detection Engine] Search signals index (#52661) * adds route for querying signals index, also updates signal status type names * first pass at happy path tests * fixes stuff after rebase with master * utilizes removes search_query from payload and replaces it with just query, adds aggs to signals search api, updates route and validation tests * removes _headers parameter from route handler and updates comment for aggs script --- .../legacy/plugins/siem/common/constants.ts | 1 + .../plugins/siem/server/kibana.index.ts | 2 + .../routes/__mocks__/request_responses.ts | 29 +++- .../query_signals_index_schema.test.ts | 39 ++++++ .../schemas/query_signals_index_schema.ts | 12 ++ .../schemas/set_signal_status_schema.test.ts | 14 +- .../signals/open_close_signals_route.ts | 4 +- .../signals/query_signals_route.test.ts | 129 ++++++++++++++++++ .../routes/signals/query_signals_route.ts | 47 +++++++ .../scripts/signals/aggs_signals.sh | 19 +++ .../scripts/signals/query_signals.sh | 19 +++ .../lib/detection_engine/signals/types.ts | 25 +++- 12 files changed, 323 insertions(+), 17 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.ts create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/aggs_signals.sh create mode 100755 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/query_signals.sh diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts index 0924b6c6eb5e6..c3494c0969900 100644 --- a/x-pack/legacy/plugins/siem/common/constants.ts +++ b/x-pack/legacy/plugins/siem/common/constants.ts @@ -53,3 +53,4 @@ export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`; export const SIGNALS_INDEX_KEY = 'signalsIndex'; export const DETECTION_ENGINE_SIGNALS_URL = `${DETECTION_ENGINE_URL}/signals`; export const DETECTION_ENGINE_SIGNALS_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/status`; +export const DETECTION_ENGINE_QUERY_SIGNALS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/search`; diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts index f56e6b3c3f550..65b673e1c72a5 100644 --- a/x-pack/legacy/plugins/siem/server/kibana.index.ts +++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts @@ -15,6 +15,7 @@ import { findRulesRoute } from './lib/detection_engine/routes/rules/find_rules_r import { deleteRulesRoute } from './lib/detection_engine/routes/rules/delete_rules_route'; import { updateRulesRoute } from './lib/detection_engine/routes/rules/update_rules_route'; import { setSignalsStatusRoute } from './lib/detection_engine/routes/signals/open_close_signals_route'; +import { querySignalsRoute } from './lib/detection_engine/routes/signals/query_signals_route'; import { ServerFacade } from './types'; import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route'; import { isAlertExecutor } from './lib/detection_engine/signals/types'; @@ -44,6 +45,7 @@ export const initServerWithKibana = (context: PluginInitializerContext, __legacy // POST /api/detection_engine/signals/status // Example usage can be found in siem/server/lib/detection_engine/scripts/signals setSignalsStatusRoute(__legacy); + querySignalsRoute(__legacy); // Detection Engine index routes that have the REST endpoints of /api/detection_engine/index // All REST index creation, policy management for spaces diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 978434859ef95..86726187c4fbd 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -6,10 +6,11 @@ import { ServerInjectOptions } from 'hapi'; import { ActionResult } from '../../../../../../actions/server/types'; -import { SignalsRestParams } from '../../signals/types'; +import { SignalsStatusRestParams, SignalsQueryRestParams } from '../../signals/types'; import { DETECTION_ENGINE_RULES_URL, DETECTION_ENGINE_SIGNALS_STATUS_URL, + DETECTION_ENGINE_QUERY_SIGNALS_URL, } from '../../../../../common/constants'; import { RuleAlertType } from '../../rules/types'; import { RuleAlertParamsRest } from '../../types'; @@ -40,17 +41,25 @@ export const typicalPayload = (): Partial> = ], }); -export const typicalSetStatusSignalByIdsPayload = (): Partial => ({ +export const typicalSetStatusSignalByIdsPayload = (): Partial => ({ signal_ids: ['somefakeid1', 'somefakeid2'], status: 'closed', }); -export const typicalSetStatusSignalByQueryPayload = (): Partial => ({ +export const typicalSetStatusSignalByQueryPayload = (): Partial => ({ query: { range: { '@timestamp': { gte: 'now-2M', lte: 'now/M' } } }, status: 'closed', }); -export const setStatusSignalMissingIdsAndQueryPayload = (): Partial => ({ +export const typicalSignalsQuery = (): Partial => ({ + query: { match_all: {} }, +}); + +export const typicalSignalsQueryAggs = (): Partial => ({ + aggs: { statuses: { terms: { field: 'signal.status', size: 10 } } }, +}); + +export const setStatusSignalMissingIdsAndQueryPayload = (): Partial => ({ status: 'closed', }); @@ -134,6 +143,18 @@ export const getSetSignalStatusByQueryRequest = (): ServerInjectOptions => ({ }, }); +export const getSignalsQueryRequest = (): ServerInjectOptions => ({ + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: { ...typicalSignalsQuery() }, +}); + +export const getSignalsAggsQueryRequest = (): ServerInjectOptions => ({ + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: { ...typicalSignalsQueryAggs() }, +}); + export const createActionResult = (): ActionResult => ({ id: 'result-1', actionTypeId: 'action-id-1', diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.test.ts new file mode 100644 index 0000000000000..4f0dbf10f4559 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.test.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { querySignalsSchema } from './query_signals_index_schema'; +import { SignalsQueryRestParams } from '../../signals/types'; + +describe('query and aggs on signals index', () => { + test('query and aggs simultaneously', () => { + expect( + querySignalsSchema.validate>({ + query: {}, + aggs: {}, + }).error + ).toBeFalsy(); + }); + + test('query only', () => { + expect( + querySignalsSchema.validate>({ + query: {}, + }).error + ).toBeFalsy(); + }); + + test('aggs only', () => { + expect( + querySignalsSchema.validate>({ + aggs: {}, + }).error + ).toBeFalsy(); + }); + + test('missing query and aggs is invalid', () => { + expect(querySignalsSchema.validate>({}).error).toBeTruthy(); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.ts new file mode 100644 index 0000000000000..53ce50692e84a --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Joi from 'joi'; + +export const querySignalsSchema = Joi.object({ + query: Joi.object(), + aggs: Joi.object(), +}).min(1); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/set_signal_status_schema.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/set_signal_status_schema.test.ts index b586b4666bfee..792c7afad05b1 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/set_signal_status_schema.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/set_signal_status_schema.test.ts @@ -5,12 +5,12 @@ */ import { setSignalsStatusSchema } from './set_signal_status_schema'; -import { SignalsRestParams } from '../../signals/types'; +import { SignalsStatusRestParams } from '../../signals/types'; describe('set signal status schema', () => { test('signal_ids and status is valid', () => { expect( - setSignalsStatusSchema.validate>({ + setSignalsStatusSchema.validate>({ signal_ids: ['somefakeid'], status: 'open', }).error @@ -19,7 +19,7 @@ describe('set signal status schema', () => { test('query and status is valid', () => { expect( - setSignalsStatusSchema.validate>({ + setSignalsStatusSchema.validate>({ query: {}, status: 'open', }).error @@ -28,7 +28,7 @@ describe('set signal status schema', () => { test('signal_ids and missing status is invalid', () => { expect( - setSignalsStatusSchema.validate>({ + setSignalsStatusSchema.validate>({ signal_ids: ['somefakeid'], }).error ).toBeTruthy(); @@ -36,7 +36,7 @@ describe('set signal status schema', () => { test('query and missing status is invalid', () => { expect( - setSignalsStatusSchema.validate>({ + setSignalsStatusSchema.validate>({ query: {}, }).error ).toBeTruthy(); @@ -44,7 +44,7 @@ describe('set signal status schema', () => { test('status is present but query or signal_ids is missing is invalid', () => { expect( - setSignalsStatusSchema.validate>({ + setSignalsStatusSchema.validate>({ status: 'closed', }).error ).toBeTruthy(); @@ -54,7 +54,7 @@ describe('set signal status schema', () => { expect( setSignalsStatusSchema.validate< Partial< - Omit & { + Omit & { status: string; } > diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals_route.ts index b342cc5cd14ef..7c49a1942ee91 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals_route.ts @@ -6,7 +6,7 @@ import Hapi from 'hapi'; import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '../../../../../common/constants'; -import { SignalsRequest } from '../../signals/types'; +import { SignalsStatusRequest } from '../../signals/types'; import { setSignalsStatusSchema } from '../schemas/set_signal_status_schema'; import { ServerFacade } from '../../../../types'; import { transformError, getIndex } from '../utils'; @@ -24,7 +24,7 @@ export const setSignalsStatusRouteDef = (server: ServerFacade): Hapi.ServerRoute payload: setSignalsStatusSchema, }, }, - async handler(request: SignalsRequest, headers) { + async handler(request: SignalsStatusRequest) { const { signal_ids: signalIds, query, status } = request.payload; const index = getIndex(request, server); const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts new file mode 100644 index 0000000000000..1b990e8c1ff57 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { createMockServer } from '../__mocks__/_mock_server'; +import { querySignalsRoute } from './query_signals_route'; +import * as myUtils from '../utils'; +import { ServerInjectOptions } from 'hapi'; +import { + getSignalsQueryRequest, + getSignalsAggsQueryRequest, + typicalSignalsQuery, + typicalSignalsQueryAggs, +} from '../__mocks__/request_responses'; +import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants'; + +describe('query for signal', () => { + let { server, elasticsearch } = createMockServer(); + + beforeEach(() => { + jest.resetAllMocks(); + jest.spyOn(myUtils, 'getIndex').mockReturnValue('fakeindex'); + ({ server, elasticsearch } = createMockServer()); + elasticsearch.getCluster = jest.fn(() => ({ + callWithRequest: jest.fn(() => true), + })); + querySignalsRoute(server); + }); + + describe('query and agg on signals index', () => { + test('returns 200 when using single query', async () => { + elasticsearch.getCluster = jest.fn(() => ({ + callWithRequest: jest.fn( + (_req, _type: string, queryBody: { index: string; body: object }) => { + expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() }); + return true; + } + ), + })); + const { statusCode } = await server.inject(getSignalsAggsQueryRequest()); + expect(statusCode).toBe(200); + expect(myUtils.getIndex).toHaveReturnedWith('fakeindex'); + }); + + test('returns 200 when using single agg', async () => { + elasticsearch.getCluster = jest.fn(() => ({ + callWithRequest: jest.fn( + (_req, _type: string, queryBody: { index: string; body: object }) => { + expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() }); + return true; + } + ), + })); + const { statusCode } = await server.inject(getSignalsAggsQueryRequest()); + expect(statusCode).toBe(200); + expect(myUtils.getIndex).toHaveReturnedWith('fakeindex'); + }); + + test('returns 200 when using aggs and query together', async () => { + const allTogether = getSignalsQueryRequest(); + allTogether.payload = { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() }; + elasticsearch.getCluster = jest.fn(() => ({ + callWithRequest: jest.fn( + (_req, _type: string, queryBody: { index: string; body: object }) => { + expect(queryBody.body).toMatchObject({ + ...typicalSignalsQueryAggs(), + ...typicalSignalsQuery(), + }); + return true; + } + ), + })); + const { statusCode } = await server.inject(allTogether); + expect(statusCode).toBe(200); + expect(myUtils.getIndex).toHaveReturnedWith('fakeindex'); + }); + + test('returns 400 when missing aggs and query', async () => { + const allTogether = getSignalsQueryRequest(); + allTogether.payload = {}; + const { statusCode } = await server.inject(allTogether); + expect(statusCode).toBe(400); + }); + }); + + describe('validation', () => { + test('returns 200 if query present', async () => { + const request: ServerInjectOptions = { + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: typicalSignalsQuery(), + }; + const { statusCode } = await server.inject(request); + expect(statusCode).toBe(200); + }); + + test('returns 200 if aggs is present', async () => { + const request: ServerInjectOptions = { + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: typicalSignalsQueryAggs(), + }; + const { statusCode } = await server.inject(request); + expect(statusCode).toBe(200); + }); + + test('returns 200 if aggs and query are present', async () => { + const request: ServerInjectOptions = { + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() }, + }; + const { statusCode } = await server.inject(request); + expect(statusCode).toBe(200); + }); + + test('returns 400 if aggs and query are NOT present', async () => { + const request: ServerInjectOptions = { + method: 'POST', + url: DETECTION_ENGINE_QUERY_SIGNALS_URL, + payload: {}, + }; + const { statusCode } = await server.inject(request); + expect(statusCode).toBe(400); + }); + }); +}); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.ts new file mode 100644 index 0000000000000..89ffed259cf77 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Hapi from 'hapi'; +import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants'; +import { SignalsQueryRequest } from '../../signals/types'; +import { querySignalsSchema } from '../schemas/query_signals_index_schema'; +import { ServerFacade } from '../../../../types'; +import { transformError, getIndex } from '../utils'; + +export const querySignalsRouteDef = (server: ServerFacade): Hapi.ServerRoute => { + return { + method: 'POST', + path: DETECTION_ENGINE_QUERY_SIGNALS_URL, + options: { + tags: ['access:siem'], + validate: { + options: { + abortEarly: false, + }, + payload: querySignalsSchema, + }, + }, + async handler(request: SignalsQueryRequest) { + const { query, aggs } = request.payload; + const body = { query, aggs }; + const index = getIndex(request, server); + const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); + try { + return callWithRequest(request, 'search', { + index, + body, + }); + } catch (exc) { + // error while getting or updating signal with id: id in signal index .siem-signals + return transformError(exc); + } + }, + }; +}; + +export const querySignalsRoute = (server: ServerFacade) => { + server.route(querySignalsRouteDef(server)); +}; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/aggs_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/aggs_signals.sh new file mode 100755 index 0000000000000..27186a14af902 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/aggs_signals.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signals/aggs_signal.sh + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/search \ + -d '{"aggs": {"statuses": {"terms": {"field": "signal.status", "size": 10 }}}}' \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/query_signals.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/query_signals.sh new file mode 100755 index 0000000000000..2fc76406ec0f6 --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/query_signals.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signals/query_signals.sh + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/search \ + -d '{ "query": { "match_all": {} } } ' \ + | jq . diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts index 213ceb29a6e25..a30182c537884 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts @@ -15,12 +15,29 @@ export interface SignalsParams { status: 'open' | 'closed'; } -export type SignalsRestParams = Omit & { - signal_ids: SignalsParams['signalIds']; +export interface SignalsStatusParams { + signalIds: string[] | undefined | null; + query: object | undefined | null; + status: 'open' | 'closed'; +} + +export interface SignalQueryParams { + query: object | undefined | null; + aggs: object | undefined | null; +} + +export type SignalsStatusRestParams = Omit & { + signal_ids: SignalsStatusParams['signalIds']; }; -export interface SignalsRequest extends RequestFacade { - payload: SignalsRestParams; +export type SignalsQueryRestParams = SignalQueryParams; + +export interface SignalsStatusRequest extends RequestFacade { + payload: SignalsStatusRestParams; +} + +export interface SignalsQueryRequest extends RequestFacade { + payload: SignalsQueryRestParams; } export type SearchTypes =