From 36ab99e546ead4f0a3e688d2575512172f94470b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 26 Nov 2020 14:01:53 +0100 Subject: [PATCH 01/11] [Logs UI] Limit the height of the "view in context" container (#83178) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../infra/public/pages/logs/stream/page_view_log_in_context.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx index 4ac3d15a82222..8a4081288b283 100644 --- a/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx +++ b/x-pack/plugins/infra/public/pages/logs/stream/page_view_log_in_context.tsx @@ -84,6 +84,7 @@ const LogInContextWrapper = euiStyled.div<{ width: number | string; height: numb padding: 16px; width: ${(props) => (typeof props.width === 'number' ? `${props.width}px` : props.width)}; height: ${(props) => (typeof props.height === 'number' ? `${props.height}px` : props.height)}; + max-height: 75vh; // Same as EuiModal `; const LogEntryContext: React.FC<{ context: LogEntry['context'] }> = ({ context }) => { From b246701d39a582b4ea3c32f9a3ca8ed8fcdb2480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Fern=C3=A1ndez=20G=C3=B3mez?= Date: Thu, 26 Nov 2020 14:18:42 +0100 Subject: [PATCH 02/11] [Logs UI] Polish the UI for the log entry examples in the anomaly table (#82139) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../log_entry_examples/log_entry_examples.tsx | 1 + .../log_entry_examples_empty_indicator.tsx | 2 +- .../log_entry_examples_failure_indicator.tsx | 2 +- .../sections/anomalies/expanded_row.tsx | 12 +++++++----- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx index 2ec9922d94555..2d15068e51da5 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples.tsx @@ -46,4 +46,5 @@ const Wrapper = euiStyled.div` flex-direction: column; flex: 1 0 0%; overflow: hidden; + padding-top: 1px; // Buffer for the "Reload" buttons' hover state `; diff --git a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx index 1d6028ed032a2..d3d309948529f 100644 --- a/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_entry_examples/log_entry_examples_empty_indicator.tsx @@ -10,7 +10,7 @@ import React from 'react'; export const LogEntryExampleMessagesEmptyIndicator: React.FunctionComponent<{ onReload: () => void; }> = ({ onReload }) => ( - + void; }> = ({ onRetry }) => ( - + - +

{examplesTitle}

 + + Date: Thu, 26 Nov 2020 15:45:21 +0100 Subject: [PATCH 03/11] [Discover] Fix navigating back when changing index pattern (#84061) --- .../public/application/angular/discover.js | 6 +++--- .../discover/_indexpattern_without_timefield.ts | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/plugins/discover/public/application/angular/discover.js b/src/plugins/discover/public/application/angular/discover.js index 272c2f2ca6187..7059593c0c4e7 100644 --- a/src/plugins/discover/public/application/angular/discover.js +++ b/src/plugins/discover/public/application/angular/discover.js @@ -252,7 +252,8 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise if (!_.isEqual(newStatePartial, oldStatePartial)) { $scope.$evalAsync(async () => { if (oldStatePartial.index !== newStatePartial.index) { - //in case of index switch the route has currently to be reloaded, legacy + //in case of index pattern switch the route has currently to be reloaded, legacy + $route.reload(); return; } @@ -289,8 +290,7 @@ function discoverController($element, $route, $scope, $timeout, $window, Promise $scope.state.sort, config.get(MODIFY_COLUMNS_ON_SWITCH) ); - await replaceUrlAppState(nextAppState); - $route.reload(); + await setAppState(nextAppState); } }; diff --git a/test/functional/apps/discover/_indexpattern_without_timefield.ts b/test/functional/apps/discover/_indexpattern_without_timefield.ts index 677b27c31bd86..20d783690277f 100644 --- a/test/functional/apps/discover/_indexpattern_without_timefield.ts +++ b/test/functional/apps/discover/_indexpattern_without_timefield.ts @@ -20,6 +20,7 @@ import { FtrProviderContext } from '../../ftr_provider_context'; export default function ({ getService, getPageObjects }: FtrProviderContext) { + const browser = getService('browser'); const esArchiver = getService('esArchiver'); const kibanaServer = getService('kibanaServer'); const security = getService('security'); @@ -50,5 +51,21 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { throw new Error('Expected timepicker to exist'); } }); + it('should switch between with and without timefield using the browser back button', async () => { + await PageObjects.discover.selectIndexPattern('without-timefield'); + if (await PageObjects.timePicker.timePickerExists()) { + throw new Error('Expected timepicker not to exist'); + } + + await PageObjects.discover.selectIndexPattern('with-timefield'); + if (!(await PageObjects.timePicker.timePickerExists())) { + throw new Error('Expected timepicker to exist'); + } + // Navigating back to discover + await browser.goBack(); + if (await PageObjects.timePicker.timePickerExists()) { + throw new Error('Expected timepicker not to exist'); + } + }); }); } From 93b0273ccddb89aed744ef1d64eafd390dcf9090 Mon Sep 17 00:00:00 2001 From: Alexey Antonov Date: Thu, 26 Nov 2020 18:04:27 +0300 Subject: [PATCH 04/11] TSVB offsets (#83051) Closes: #40299 Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../response_processors/series/time_shift.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js index 14de6aa18f872..c00b0894073d2 100644 --- a/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js +++ b/src/plugins/vis_type_timeseries/server/lib/vis_data/response_processors/series/time_shift.js @@ -17,7 +17,7 @@ * under the License. */ -import _ from 'lodash'; +import { startsWith } from 'lodash'; import moment from 'moment'; export function timeShift(resp, panel, series) { @@ -26,13 +26,15 @@ export function timeShift(resp, panel, series) { const matches = series.offset_time.match(/^([+-]?[\d]+)([shmdwMy]|ms)$/); if (matches) { - const offsetValue = Number(matches[1]); + const offsetValue = matches[1]; const offsetUnit = matches[2]; - const offset = moment.duration(offsetValue, offsetUnit).valueOf(); results.forEach((item) => { - if (_.startsWith(item.id, series.id)) { - item.data = item.data.map(([time, value]) => [time + offset, value]); + if (startsWith(item.id, series.id)) { + item.data = item.data.map((row) => [ + moment.utc(row[0]).add(offsetValue, offsetUnit).valueOf(), + row[1], + ]); } }); } From 4d8b7f9084f46bc4694ea8d78aee2b13a049a80c Mon Sep 17 00:00:00 2001 From: Joe Portner <5295965+jportner@users.noreply.github.com> Date: Thu, 26 Nov 2020 10:51:10 -0500 Subject: [PATCH 05/11] Improve short-url redirect validation (#84366) --- .../routes/lib/short_url_assert_valid.test.ts | 45 ++++++++++--------- .../routes/lib/short_url_assert_valid.ts | 12 +++-- 2 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts index 02a5e123b6481..232429ac4ec33 100644 --- a/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts +++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.test.ts @@ -19,36 +19,39 @@ import { shortUrlAssertValid } from './short_url_assert_valid'; +const PROTOCOL_ERROR = /^Short url targets cannot have a protocol/; +const HOSTNAME_ERROR = /^Short url targets cannot have a hostname/; +const PATH_ERROR = /^Short url target path must be in the format/; + describe('shortUrlAssertValid()', () => { const invalid = [ - ['protocol', 'http://localhost:5601/app/kibana'], - ['protocol', 'https://localhost:5601/app/kibana'], - ['protocol', 'mailto:foo@bar.net'], - ['protocol', 'javascript:alert("hi")'], // eslint-disable-line no-script-url - ['hostname', 'localhost/app/kibana'], - ['hostname and port', 'local.host:5601/app/kibana'], - ['hostname and auth', 'user:pass@localhost.net/app/kibana'], - ['path traversal', '/app/../../not-kibana'], - ['deep path', '/app/kibana/foo'], - ['deep path', '/app/kibana/foo/bar'], - ['base path', '/base/app/kibana'], + ['protocol', 'http://localhost:5601/app/kibana', PROTOCOL_ERROR], + ['protocol', 'https://localhost:5601/app/kibana', PROTOCOL_ERROR], + ['protocol', 'mailto:foo@bar.net', PROTOCOL_ERROR], + ['protocol', 'javascript:alert("hi")', PROTOCOL_ERROR], // eslint-disable-line no-script-url + ['hostname', 'localhost/app/kibana', PATH_ERROR], // according to spec, this is not a valid URL -- you cannot specify a hostname without a protocol + ['hostname and port', 'local.host:5601/app/kibana', PROTOCOL_ERROR], // parser detects 'local.host' as the protocol + ['hostname and auth', 'user:pass@localhost.net/app/kibana', PROTOCOL_ERROR], // parser detects 'user' as the protocol + ['path traversal', '/app/../../not-kibana', PATH_ERROR], // fails because there are >2 path parts + ['path traversal', '/../not-kibana', PATH_ERROR], // fails because first path part is not 'app' + ['deep path', '/app/kibana/foo', PATH_ERROR], // fails because there are >2 path parts + ['deeper path', '/app/kibana/foo/bar', PATH_ERROR], // fails because there are >2 path parts + ['base path', '/base/app/kibana', PATH_ERROR], // fails because there are >2 path parts + ['path with an extra leading slash', '//foo/app/kibana', HOSTNAME_ERROR], // parser detects 'foo' as the hostname + ['path with an extra leading slash', '///app/kibana', HOSTNAME_ERROR], // parser detects '' as the hostname + ['path without app', '/foo/kibana', PATH_ERROR], // fails because first path part is not 'app' + ['path without appId', '/app/', PATH_ERROR], // fails because there is only one path part (leading and trailing slashes are trimmed) ]; - invalid.forEach(([desc, url]) => { - it(`fails when url has ${desc}`, () => { - try { - shortUrlAssertValid(url); - throw new Error(`expected assertion to throw`); - } catch (err) { - if (!err || !err.isBoom) { - throw err; - } - } + invalid.forEach(([desc, url, error]) => { + it(`fails when url has ${desc as string}`, () => { + expect(() => shortUrlAssertValid(url as string)).toThrowError(error); }); }); const valid = [ '/app/kibana', + '/app/kibana/', // leading and trailing slashes are trimmed '/app/monitoring#angular/route', '/app/text#document-id', '/app/some?with=query', diff --git a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts index 581410359322f..773e3acdcb902 100644 --- a/src/plugins/share/server/routes/lib/short_url_assert_valid.ts +++ b/src/plugins/share/server/routes/lib/short_url_assert_valid.ts @@ -22,18 +22,22 @@ import { trim } from 'lodash'; import Boom from '@hapi/boom'; export function shortUrlAssertValid(url: string) { - const { protocol, hostname, pathname } = parse(url); + const { protocol, hostname, pathname } = parse( + url, + false /* parseQueryString */, + true /* slashesDenoteHost */ + ); - if (protocol) { + if (protocol !== null) { throw Boom.notAcceptable(`Short url targets cannot have a protocol, found "${protocol}"`); } - if (hostname) { + if (hostname !== null) { throw Boom.notAcceptable(`Short url targets cannot have a hostname, found "${hostname}"`); } const pathnameParts = trim(pathname === null ? undefined : pathname, '/').split('/'); - if (pathnameParts.length !== 2) { + if (pathnameParts.length !== 2 || pathnameParts[0] !== 'app' || !pathnameParts[1]) { throw Boom.notAcceptable( `Short url target path must be in the format "/app/{{appId}}", found "${pathname}"` ); From 51f75a5655224d77c97b944f8b603ff34ec92714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yulia=20=C4=8Cech?= <6585477+yuliacech@users.noreply.github.com> Date: Thu, 26 Nov 2020 17:04:24 +0100 Subject: [PATCH 06/11] Added data streams privileges to better control delete actions in UI (#83573) * Added data streams privileges to better control delete actions in UI * Fix type check issues * Change data streams privileges request * Fixed type check issue * Fixed api integration test * Cleaned up not needed code * Renamed some data streams and added a default value for stats find Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../helpers/test_subjects.ts | 2 + .../home/data_streams_tab.helpers.ts | 12 + .../home/data_streams_tab.test.ts | 73 ++++++ .../common/lib/data_stream_serialization.ts | 2 + .../common/types/data_streams.ts | 16 +- .../data_stream_detail_panel.tsx | 2 +- .../data_stream_list/data_stream_list.tsx | 9 +- .../data_stream_table/data_stream_table.tsx | 6 +- .../server/client/elasticsearch.ts | 22 -- .../plugins/index_management/server/plugin.ts | 3 +- .../component_templates/privileges.test.ts | 2 + .../api/data_streams/register_get_route.ts | 221 ++++++++++++------ .../index_management/server/shared_imports.ts | 2 +- .../plugins/index_management/server/types.ts | 3 +- .../index_management/data_streams.ts | 9 + 15 files changed, 271 insertions(+), 113 deletions(-) diff --git a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts index 04843cae6a57e..e8105ac2937c0 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/helpers/test_subjects.ts @@ -13,6 +13,8 @@ export type TestSubjects = | 'createTemplateButton' | 'dataStreamsEmptyPromptTemplateLink' | 'dataStreamTable' + | 'deleteDataStreamsButton' + | 'deleteDataStreamButton' | 'deleteSystemTemplateCallOut' | 'deleteTemplateButton' | 'deleteTemplatesConfirmation' diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts index 4e0486e55720d..9c92af30097a2 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.helpers.ts @@ -24,6 +24,7 @@ export interface DataStreamsTabTestBed extends TestBed { clickNameAt: (index: number) => void; clickIndicesAt: (index: number) => void; clickDeleteActionAt: (index: number) => void; + selectDataStream: (name: string, selected: boolean) => void; clickConfirmDelete: () => void; clickDeleteDataStreamButton: () => void; clickDetailPanelIndexTemplateLink: () => void; @@ -125,6 +126,13 @@ export const setup = async (overridingDependencies: any = {}): Promise { + const { + form: { selectCheckBox }, + } = testBed; + selectCheckBox(`checkboxSelectRow-${name}`, selected); + }; + const findDeleteConfirmationModal = () => { const { find } = testBed; return find('deleteDataStreamsConfirmation'); @@ -194,6 +202,7 @@ export const setup = async (overridingDependencies: any = {}): Promise): DataSt indexTemplateName: 'indexTemplate', storageSize: '1b', maxTimeStamp: 420, + privileges: { + delete_index: true, + }, ...dataStream, }); diff --git a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts index 8ce307c103f4c..91502621d50c5 100644 --- a/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts +++ b/x-pack/plugins/index_management/__jest__/client_integration/home/data_streams_tab.test.ts @@ -449,4 +449,77 @@ describe('Data Streams tab', () => { expect(tableCellsValues).toEqual([['', 'non-managed-data-stream', 'green', '1', 'Delete']]); }); }); + + describe('data stream privileges', () => { + describe('delete', () => { + const { setLoadDataStreamsResponse, setLoadDataStreamResponse } = httpRequestsMockHelpers; + + const dataStreamWithDelete = createDataStreamPayload({ + name: 'dataStreamWithDelete', + privileges: { delete_index: true }, + }); + const dataStreamNoDelete = createDataStreamPayload({ + name: 'dataStreamNoDelete', + privileges: { delete_index: false }, + }); + + beforeEach(async () => { + setLoadDataStreamsResponse([dataStreamWithDelete, dataStreamNoDelete]); + + testBed = await setup({ history: createMemoryHistory() }); + await act(async () => { + testBed.actions.goToDataStreamsList(); + }); + testBed.component.update(); + }); + + test('displays/hides delete button depending on data streams privileges', async () => { + const { table } = testBed; + const { tableCellsValues } = table.getMetaData('dataStreamTable'); + + expect(tableCellsValues).toEqual([ + ['', 'dataStreamNoDelete', 'green', '1', ''], + ['', 'dataStreamWithDelete', 'green', '1', 'Delete'], + ]); + }); + + test('displays/hides delete action depending on data streams privileges', async () => { + const { + actions: { selectDataStream }, + find, + } = testBed; + + selectDataStream('dataStreamNoDelete', true); + expect(find('deleteDataStreamsButton').exists()).toBeFalsy(); + + selectDataStream('dataStreamWithDelete', true); + expect(find('deleteDataStreamsButton').exists()).toBeFalsy(); + + selectDataStream('dataStreamNoDelete', false); + expect(find('deleteDataStreamsButton').exists()).toBeTruthy(); + }); + + test('displays delete button in detail panel', async () => { + const { + actions: { clickNameAt }, + find, + } = testBed; + setLoadDataStreamResponse(dataStreamWithDelete); + await clickNameAt(1); + + expect(find('deleteDataStreamButton').exists()).toBeTruthy(); + }); + + test('hides delete button in detail panel', async () => { + const { + actions: { clickNameAt }, + find, + } = testBed; + setLoadDataStreamResponse(dataStreamNoDelete); + await clickNameAt(0); + + expect(find('deleteDataStreamButton').exists()).toBeFalsy(); + }); + }); + }); }); diff --git a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts index 2d8e038d2a60f..fe7db99c98db1 100644 --- a/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts +++ b/x-pack/plugins/index_management/common/lib/data_stream_serialization.ts @@ -18,6 +18,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS store_size: storageSize, maximum_timestamp: maxTimeStamp, _meta, + privileges, } = dataStreamFromEs; return { @@ -37,6 +38,7 @@ export function deserializeDataStream(dataStreamFromEs: DataStreamFromEs): DataS storageSize, maxTimeStamp, _meta, + privileges, }; } diff --git a/x-pack/plugins/index_management/common/types/data_streams.ts b/x-pack/plugins/index_management/common/types/data_streams.ts index adb7104043fbb..fdfe6278eb985 100644 --- a/x-pack/plugins/index_management/common/types/data_streams.ts +++ b/x-pack/plugins/index_management/common/types/data_streams.ts @@ -10,13 +10,19 @@ interface TimestampFieldFromEs { type TimestampField = TimestampFieldFromEs; -interface MetaFieldFromEs { +interface MetaFromEs { managed_by: string; package: any; managed: boolean; } -type MetaField = MetaFieldFromEs; +type Meta = MetaFromEs; + +interface PrivilegesFromEs { + delete_index: boolean; +} + +type Privileges = PrivilegesFromEs; export type HealthFromEs = 'GREEN' | 'YELLOW' | 'RED'; @@ -25,12 +31,13 @@ export interface DataStreamFromEs { timestamp_field: TimestampFieldFromEs; indices: DataStreamIndexFromEs[]; generation: number; - _meta?: MetaFieldFromEs; + _meta?: MetaFromEs; status: HealthFromEs; template: string; ilm_policy?: string; store_size?: string; maximum_timestamp?: number; + privileges: PrivilegesFromEs; } export interface DataStreamIndexFromEs { @@ -50,7 +57,8 @@ export interface DataStream { ilmPolicyName?: string; storageSize?: string; maxTimeStamp?: number; - _meta?: MetaField; + _meta?: Meta; + privileges: Privileges; } export interface DataStreamIndex { diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx index 05d7e97745b9e..ec47b2c062aa9 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_detail_panel/data_stream_detail_panel.tsx @@ -290,7 +290,7 @@ export const DataStreamDetailPanel: React.FunctionComponent = ({ - {!isLoading && !error ? ( + {!isLoading && !error && dataStream?.privileges.delete_index ? ( { const { isDeepLink } = extractQueryParams(search); + const decodedDataStreamName = attemptToURIDecode(dataStreamName); const { core: { getUrlForApp }, @@ -241,8 +242,8 @@ export const DataStreamList: React.FunctionComponent { history.push(`/${Section.DataStreams}`); diff --git a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx index c1fd33a39569c..7a3e719d013c8 100644 --- a/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx +++ b/x-pack/plugins/index_management/public/application/sections/home/data_stream_list/data_stream_table/data_stream_table.tsx @@ -162,6 +162,7 @@ export const DataStreamTable: React.FunctionComponent = ({ }, isPrimary: true, 'data-test-subj': 'deleteDataStream', + available: ({ privileges: { delete_index: deleteIndex } }: DataStream) => deleteIndex, }, ], }); @@ -188,9 +189,10 @@ export const DataStreamTable: React.FunctionComponent = ({ incremental: true, }, toolsLeft: - selection.length > 0 ? ( + selection.length > 0 && + selection.every((dataStream: DataStream) => dataStream.privileges.delete_index) ? ( setDataStreamsToDelete(selection.map(({ name }: DataStream) => name))} color="danger" > diff --git a/x-pack/plugins/index_management/server/client/elasticsearch.ts b/x-pack/plugins/index_management/server/client/elasticsearch.ts index ed5ede07479ca..8b7749335131e 100644 --- a/x-pack/plugins/index_management/server/client/elasticsearch.ts +++ b/x-pack/plugins/index_management/server/client/elasticsearch.ts @@ -11,28 +11,6 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) const dataManagement = Client.prototype.dataManagement.prototype; // Data streams - dataManagement.getDataStreams = ca({ - urls: [ - { - fmt: '/_data_stream', - }, - ], - method: 'GET', - }); - - dataManagement.getDataStream = ca({ - urls: [ - { - fmt: '/_data_stream/<%=name%>', - req: { - name: { - type: 'string', - }, - }, - }, - ], - method: 'GET', - }); // We don't allow the user to create a data stream in the UI or API. We're just adding this here // to enable the API integration tests. diff --git a/x-pack/plugins/index_management/server/plugin.ts b/x-pack/plugins/index_management/server/plugin.ts index ae9633f3e22b9..3d70140fa60b7 100644 --- a/x-pack/plugins/index_management/server/plugin.ts +++ b/x-pack/plugins/index_management/server/plugin.ts @@ -24,7 +24,7 @@ import { PLUGIN } from '../common'; import { Dependencies } from './types'; import { ApiRoutes } from './routes'; import { License, IndexDataEnricher } from './services'; -import { isEsError } from './shared_imports'; +import { isEsError, handleEsError } from './shared_imports'; import { elasticsearchJsPlugin } from './client/elasticsearch'; export interface DataManagementContext { @@ -110,6 +110,7 @@ export class IndexMgmtServerPlugin implements Plugin { indexDataEnricher: mockedIndexDataEnricher, lib: { isEsError: jest.fn(), + handleEsError: jest.fn(), }, }); @@ -123,6 +124,7 @@ describe('GET privileges', () => { indexDataEnricher: mockedIndexDataEnricher, lib: { isEsError: jest.fn(), + handleEsError: jest.fn(), }, }); diff --git a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts index fa93d9bd0c563..d19383d892cbd 100644 --- a/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts +++ b/x-pack/plugins/index_management/server/routes/api/data_streams/register_get_route.ts @@ -6,122 +6,189 @@ import { schema, TypeOf } from '@kbn/config-schema'; +import { ElasticsearchClient } from 'kibana/server'; import { deserializeDataStream, deserializeDataStreamList } from '../../../../common/lib'; +import { DataStreamFromEs } from '../../../../common/types'; import { RouteDependencies } from '../../../types'; import { addBasePath } from '../index'; -const querySchema = schema.object({ - includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])), -}); +interface PrivilegesFromEs { + username: string; + has_all_requested: boolean; + cluster: Record; + index: Record>; + application: Record; +} + +interface StatsFromEs { + data_stream: string; + store_size: string; + maximum_timestamp: number; +} -export function registerGetAllRoute({ router, license, lib: { isEsError } }: RouteDependencies) { +const enhanceDataStreams = ({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, +}: { + dataStreams: DataStreamFromEs[]; + dataStreamsStats?: StatsFromEs[]; + dataStreamsPrivileges?: PrivilegesFromEs; +}): DataStreamFromEs[] => { + return dataStreams.map((dataStream: DataStreamFromEs) => { + let enhancedDataStream = { ...dataStream }; + + if (dataStreamsStats) { + // eslint-disable-next-line @typescript-eslint/naming-convention + const { store_size, maximum_timestamp } = + dataStreamsStats.find( + ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name + ) || {}; + + enhancedDataStream = { + ...enhancedDataStream, + store_size, + maximum_timestamp, + }; + } + + enhancedDataStream = { + ...enhancedDataStream, + privileges: { + delete_index: dataStreamsPrivileges + ? dataStreamsPrivileges.index[dataStream.name].delete_index + : true, + }, + }; + + return enhancedDataStream; + }); +}; + +const getDataStreamsStats = (client: ElasticsearchClient, name = '*') => { + return client.transport.request({ + path: `/_data_stream/${encodeURIComponent(name)}/_stats`, + method: 'GET', + querystring: { + human: true, + }, + }); +}; + +const getDataStreamsPrivileges = (client: ElasticsearchClient, names: string[]) => { + return client.security.hasPrivileges({ + body: { + index: [ + { + names, + privileges: ['delete_index'], + }, + ], + }, + }); +}; + +export function registerGetAllRoute({ + router, + license, + lib: { handleEsError }, + config, +}: RouteDependencies) { + const querySchema = schema.object({ + includeStats: schema.maybe(schema.oneOf([schema.literal('true'), schema.literal('false')])), + }); router.get( { path: addBasePath('/data_streams'), validate: { query: querySchema } }, - license.guardApiRoute(async (ctx, req, res) => { - const { callAsCurrentUser } = ctx.dataManagement!.client; + license.guardApiRoute(async (ctx, req, response) => { + const { asCurrentUser } = ctx.core.elasticsearch.client; const includeStats = (req.query as TypeOf).includeStats === 'true'; try { - const { data_streams: dataStreams } = await callAsCurrentUser( - 'dataManagement.getDataStreams' - ); - - if (includeStats) { - const { - data_streams: dataStreamsStats, - } = await ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', { - path: '/_data_stream/*/_stats', - method: 'GET', - query: { - human: true, - }, - }); + let { + body: { data_streams: dataStreams }, + } = await asCurrentUser.indices.getDataStream(); - // Merge stats into data streams. - for (let i = 0; i < dataStreams.length; i++) { - const dataStream = dataStreams[i]; + let dataStreamsStats; + let dataStreamsPrivileges; - // eslint-disable-next-line @typescript-eslint/naming-convention - const { store_size, maximum_timestamp } = dataStreamsStats.find( - ({ data_stream: statsName }: { data_stream: string }) => statsName === dataStream.name - ); - - dataStreams[i] = { - ...dataStream, - store_size, - maximum_timestamp, - }; - } + if (includeStats) { + ({ + body: { data_streams: dataStreamsStats }, + } = await getDataStreamsStats(asCurrentUser)); } - return res.ok({ body: deserializeDataStreamList(dataStreams) }); - } catch (error) { - if (isEsError(error)) { - return res.customError({ - statusCode: error.statusCode, - body: error, - }); + if (config.isSecurityEnabled() && dataStreams.length > 0) { + ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges( + asCurrentUser, + dataStreams.map((dataStream: DataStreamFromEs) => dataStream.name) + )); } - return res.internalError({ body: error }); + dataStreams = enhanceDataStreams({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, + }); + + return response.ok({ body: deserializeDataStreamList(dataStreams) }); + } catch (error) { + return handleEsError({ error, response }); } }) ); } -export function registerGetOneRoute({ router, license, lib: { isEsError } }: RouteDependencies) { +export function registerGetOneRoute({ + router, + license, + lib: { handleEsError }, + config, +}: RouteDependencies) { const paramsSchema = schema.object({ name: schema.string(), }); - router.get( { path: addBasePath('/data_streams/{name}'), validate: { params: paramsSchema }, }, - license.guardApiRoute(async (ctx, req, res) => { + license.guardApiRoute(async (ctx, req, response) => { const { name } = req.params as TypeOf; - const { callAsCurrentUser } = ctx.dataManagement!.client; + const { asCurrentUser } = ctx.core.elasticsearch.client; try { const [ - { data_streams: dataStream }, - { data_streams: dataStreamsStats }, + { + body: { data_streams: dataStreams }, + }, + { + body: { data_streams: dataStreamsStats }, + }, ] = await Promise.all([ - callAsCurrentUser('dataManagement.getDataStream', { - name, - }), - ctx.core.elasticsearch.legacy.client.callAsCurrentUser('transport.request', { - path: `/_data_stream/${encodeURIComponent(name)}/_stats`, - method: 'GET', - query: { - human: true, - }, - }), + asCurrentUser.indices.getDataStream({ name }), + getDataStreamsStats(asCurrentUser, name), ]); - if (dataStream[0]) { - // eslint-disable-next-line @typescript-eslint/naming-convention - const { store_size, maximum_timestamp } = dataStreamsStats[0]; - dataStream[0] = { - ...dataStream[0], - store_size, - maximum_timestamp, - }; - const body = deserializeDataStream(dataStream[0]); - return res.ok({ body }); - } + if (dataStreams[0]) { + let dataStreamsPrivileges; + if (config.isSecurityEnabled()) { + ({ body: dataStreamsPrivileges } = await getDataStreamsPrivileges(asCurrentUser, [ + dataStreams[0].name, + ])); + } - return res.notFound(); - } catch (e) { - if (isEsError(e)) { - return res.customError({ - statusCode: e.statusCode, - body: e, + const enhancedDataStreams = enhanceDataStreams({ + dataStreams, + dataStreamsStats, + dataStreamsPrivileges, }); + const body = deserializeDataStream(enhancedDataStreams[0]); + return response.ok({ body }); } - // Case: default - return res.internalError({ body: e }); + + return response.notFound(); + } catch (error) { + return handleEsError({ error, response }); } }) ); diff --git a/x-pack/plugins/index_management/server/shared_imports.ts b/x-pack/plugins/index_management/server/shared_imports.ts index 454beda5394c7..0606f474897b5 100644 --- a/x-pack/plugins/index_management/server/shared_imports.ts +++ b/x-pack/plugins/index_management/server/shared_imports.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { isEsError } from '../../../../src/plugins/es_ui_shared/server'; +export { isEsError, handleEsError } from '../../../../src/plugins/es_ui_shared/server'; diff --git a/x-pack/plugins/index_management/server/types.ts b/x-pack/plugins/index_management/server/types.ts index 7aa91629f0a47..177dedeb87bb4 100644 --- a/x-pack/plugins/index_management/server/types.ts +++ b/x-pack/plugins/index_management/server/types.ts @@ -8,7 +8,7 @@ import { PluginSetupContract as FeaturesPluginSetup } from '../../features/serve import { LicensingPluginSetup } from '../../licensing/server'; import { SecurityPluginSetup } from '../../security/server'; import { License, IndexDataEnricher } from './services'; -import { isEsError } from './shared_imports'; +import { isEsError, handleEsError } from './shared_imports'; export interface Dependencies { security: SecurityPluginSetup; @@ -25,6 +25,7 @@ export interface RouteDependencies { indexDataEnricher: IndexDataEnricher; lib: { isEsError: typeof isEsError; + handleEsError: typeof handleEsError; }; } diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index f4b947336e044..6cf1a40a4d5a1 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -77,6 +77,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreams).to.eql([ { name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { @@ -105,6 +108,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreams.length).to.be(1); expect(dataStreamWithoutStorageSize).to.eql({ name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { @@ -132,6 +138,9 @@ export default function ({ getService }: FtrProviderContext) { expect(dataStreamWithoutStorageSize).to.eql({ name: testDataStreamName, + privileges: { + delete_index: true, + }, timeStampField: { name: '@timestamp' }, indices: [ { From ee5c9bceebf1b0ddd3f4da371e8009d455e7b3f6 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Thu, 26 Nov 2020 20:34:06 +0100 Subject: [PATCH 07/11] Upgrade fp-ts to 2.8.6 (#83866) * Upgrade fp-ts to 2.8.6 * reduce import size from io-ts * removed unused imports * remove usage of fpts from alerts Co-authored-by: Gidi Meir Morris Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/alerts/public/alert_api.test.ts | 66 +------------------ x-pack/plugins/alerts/public/alert_api.ts | 41 ++---------- .../public/application/lib/alert_api.ts | 6 +- yarn.lock | 6 +- 4 files changed, 14 insertions(+), 105 deletions(-) diff --git a/x-pack/plugins/alerts/public/alert_api.test.ts b/x-pack/plugins/alerts/public/alert_api.test.ts index 3ee67b79b7bda..0fa2e7f25323b 100644 --- a/x-pack/plugins/alerts/public/alert_api.test.ts +++ b/x-pack/plugins/alerts/public/alert_api.test.ts @@ -6,7 +6,7 @@ import { AlertType } from '../common'; import { httpServiceMock } from '../../../../src/core/public/mocks'; -import { loadAlert, loadAlertState, loadAlertType, loadAlertTypes } from './alert_api'; +import { loadAlert, loadAlertType, loadAlertTypes } from './alert_api'; import uuid from 'uuid'; const http = httpServiceMock.createStartContract(); @@ -114,67 +114,3 @@ describe('loadAlert', () => { expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}`); }); }); - -describe('loadAlertState', () => { - test('should call get API with base parameters', async () => { - const alertId = uuid.v4(); - const resolvedValue = { - alertTypeState: { - some: 'value', - }, - alertInstances: { - first_instance: {}, - second_instance: {}, - }, - }; - http.get.mockResolvedValueOnce(resolvedValue); - - expect(await loadAlertState({ http, alertId })).toEqual(resolvedValue); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); - - test('should parse AlertInstances', async () => { - const alertId = uuid.v4(); - const resolvedValue = { - alertTypeState: { - some: 'value', - }, - alertInstances: { - first_instance: { - state: {}, - meta: { - lastScheduledActions: { - group: 'first_group', - date: '2020-02-09T23:15:41.941Z', - }, - }, - }, - }, - }; - http.get.mockResolvedValueOnce(resolvedValue); - - expect(await loadAlertState({ http, alertId })).toEqual({ - ...resolvedValue, - alertInstances: { - first_instance: { - state: {}, - meta: { - lastScheduledActions: { - group: 'first_group', - date: new Date('2020-02-09T23:15:41.941Z'), - }, - }, - }, - }, - }); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); - - test('should handle empty response from api', async () => { - const alertId = uuid.v4(); - http.get.mockResolvedValueOnce(''); - - expect(await loadAlertState({ http, alertId })).toEqual({}); - expect(http.get).toHaveBeenCalledWith(`/api/alerts/alert/${alertId}/state`); - }); -}); diff --git a/x-pack/plugins/alerts/public/alert_api.ts b/x-pack/plugins/alerts/public/alert_api.ts index 5b7cd2128f386..753158ed1ed42 100644 --- a/x-pack/plugins/alerts/public/alert_api.ts +++ b/x-pack/plugins/alerts/public/alert_api.ts @@ -5,15 +5,9 @@ */ import { HttpSetup } from 'kibana/public'; -import * as t from 'io-ts'; -import { pipe } from 'fp-ts/lib/pipeable'; -import { fold } from 'fp-ts/lib/Either'; -import { findFirst } from 'fp-ts/lib/Array'; -import { isNone } from 'fp-ts/lib/Option'; - import { i18n } from '@kbn/i18n'; -import { BASE_ALERT_API_PATH, alertStateSchema } from '../common'; -import { Alert, AlertType, AlertTaskState } from '../common'; +import { BASE_ALERT_API_PATH } from '../common'; +import type { Alert, AlertType } from '../common'; export async function loadAlertTypes({ http }: { http: HttpSetup }): Promise { return await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`); @@ -26,10 +20,10 @@ export async function loadAlertType({ http: HttpSetup; id: AlertType['id']; }): Promise { - const maybeAlertType = findFirst((type) => type.id === id)( - await http.get(`${BASE_ALERT_API_PATH}/list_alert_types`) - ); - if (isNone(maybeAlertType)) { + const maybeAlertType = ((await http.get( + `${BASE_ALERT_API_PATH}/list_alert_types` + )) as AlertType[]).find((type) => type.id === id); + if (!maybeAlertType) { throw new Error( i18n.translate('xpack.alerts.loadAlertType.missingAlertTypeError', { defaultMessage: 'Alert type "{id}" is not registered.', @@ -39,7 +33,7 @@ export async function loadAlertType({ }) ); } - return maybeAlertType.value; + return maybeAlertType; } export async function loadAlert({ @@ -51,24 +45,3 @@ export async function loadAlert({ }): Promise { return await http.get(`${BASE_ALERT_API_PATH}/alert/${alertId}`); } - -type EmptyHttpResponse = ''; -export async function loadAlertState({ - http, - alertId, -}: { - http: HttpSetup; - alertId: string; -}): Promise { - return await http - .get(`${BASE_ALERT_API_PATH}/alert/${alertId}/state`) - .then((state: AlertTaskState | EmptyHttpResponse) => (state ? state : {})) - .then((state: AlertTaskState) => { - return pipe( - alertStateSchema.decode(state), - fold((e: t.Errors) => { - throw new Error(`Alert "${alertId}" has invalid state`); - }, t.identity) - ); - }); -} diff --git a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts index 7c2f50211d4af..d34481850ca4a 100644 --- a/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts +++ b/x-pack/plugins/triggers_actions_ui/public/application/lib/alert_api.ts @@ -5,7 +5,7 @@ */ import { HttpSetup } from 'kibana/public'; -import * as t from 'io-ts'; +import { Errors, identity } from 'io-ts'; import { pipe } from 'fp-ts/lib/pipeable'; import { fold } from 'fp-ts/lib/Either'; import { pick } from 'lodash'; @@ -48,9 +48,9 @@ export async function loadAlertState({ .then((state: AlertTaskState) => { return pipe( alertStateSchema.decode(state), - fold((e: t.Errors) => { + fold((e: Errors) => { throw new Error(`Alert "${alertId}" has invalid state`); - }, t.identity) + }, identity) ); }); } diff --git a/yarn.lock b/yarn.lock index 1cde1266ca38f..1f5eba54fb835 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14045,9 +14045,9 @@ fp-ts@^1.0.0: integrity sha512-fWwnAgVlTsV26Ruo9nx+fxNHIm6l1puE1VJ/C0XJ3nRQJJJIgRHYw6sigB3MuNFZL1o4fpGlhwFhcbxHK0RsOA== fp-ts@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.3.1.tgz#8068bfcca118227932941101e062134d7ecd9119" - integrity sha512-KevPBnYt0aaJiuUzmU9YIxjrhC9AgJ8CLtLlXmwArovlNTeYM5NtEoKd86B0wHd7FIbzeE8sNXzCoYIOr7e6Iw== + version "2.8.6" + resolved "https://registry.yarnpkg.com/fp-ts/-/fp-ts-2.8.6.tgz#1a0e6c3f29f5b0fbfa3120f034ea266aa73c811b" + integrity sha512-fGGpKf/Jy3UT4s16oM+hr/8F5QXFcZ+20NAvaZXH5Y5jsiLPMDCaNqffXq0z1Kr6ZUJj0346cH9tq+cI2SoJ4w== fragment-cache@^0.2.1: version "0.2.1" From f99766e7ab838c1424f7633a3c5182705abf0eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Fri, 27 Nov 2020 10:19:11 +0100 Subject: [PATCH 08/11] [APM] Fix missing `service.node.name` (#84269) --- .../app/service_node_metrics/index.tsx | 25 ++++------------- .../public/hooks/useServiceMetricCharts.ts | 6 ++-- .../server/lib/metrics/by_agent/default.ts | 4 +-- .../by_agent/java/gc/get_gc_rate_chart.ts | 16 +++++++---- .../by_agent/java/gc/get_gc_time_chart.ts | 16 +++++++---- .../by_agent/java/heap_memory/index.ts | 14 ++++++---- .../server/lib/metrics/by_agent/java/index.ts | 28 +++++++++++-------- .../by_agent/java/non_heap_memory/index.ts | 14 ++++++---- .../by_agent/java/thread_count/index.ts | 14 ++++++---- .../lib/metrics/by_agent/shared/cpu/index.ts | 14 ++++++---- .../metrics/by_agent/shared/memory/index.ts | 14 ++++++---- .../get_metrics_chart_data_by_agent.ts | 2 +- .../apm/server/lib/metrics/queries.test.ts | 10 +++---- 13 files changed, 98 insertions(+), 79 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx index a74ff574bc0c8..a886c3f29d57c 100644 --- a/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_node_metrics/index.tsx @@ -59,7 +59,11 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { const { urlParams, uiFilters } = useUrlParams(); const { serviceName, serviceNodeName } = match.params; const { agentName } = useAgentName(); - const { data } = useServiceMetricCharts(urlParams, agentName); + const { data } = useServiceMetricCharts( + urlParams, + agentName, + serviceNodeName + ); const { start, end } = urlParams; const { data: { host, containerId } = INITIAL_DATA, status } = useFetcher( @@ -177,25 +181,6 @@ export function ServiceNodeMetrics({ match }: ServiceNodeMetricsProps) { )} - {agentName && ( - - - {data.charts.map((chart) => ( - - - - - - ))} - - - - )} {agentName && ( diff --git a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts index d264ad6069db3..7a54c6ffc6dbe 100644 --- a/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts +++ b/x-pack/plugins/apm/public/hooks/useServiceMetricCharts.ts @@ -17,7 +17,8 @@ const INITIAL_DATA: MetricsChartsByAgentAPIResponse = { export function useServiceMetricCharts( urlParams: IUrlParams, - agentName?: string + agentName?: string, + serviceNodeName?: string ) { const { serviceName } = useParams<{ serviceName?: string }>(); const { start, end } = urlParams; @@ -30,6 +31,7 @@ export function useServiceMetricCharts( params: { path: { serviceName }, query: { + serviceNodeName, start, end, agentName, @@ -39,7 +41,7 @@ export function useServiceMetricCharts( }); } }, - [serviceName, start, end, agentName, uiFilters] + [serviceName, start, end, agentName, serviceNodeName, uiFilters] ); return { diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts index fbcbc9f12791f..71bc1018b619b 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/default.ts @@ -13,8 +13,8 @@ export async function getDefaultMetricsCharts( serviceName: string ) { const charts = await Promise.all([ - getCPUChartData(setup, serviceName), - getMemoryChartData(setup, serviceName), + getCPUChartData({ setup, serviceName }), + getMemoryChartData({ setup, serviceName }), ]); return { charts }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts index 7cedeb828e3b7..8e68c229c555c 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts @@ -30,11 +30,15 @@ const chartBase: ChartBase = { series, }; -const getGcRateChart = ( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) => { +function getGcRateChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformGcMetrics({ setup, serviceName, @@ -42,6 +46,6 @@ const getGcRateChart = ( chartBase, fieldName: METRIC_JAVA_GC_COUNT, }); -}; +} export { getGcRateChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts index f21d3d8e7c056..e7a83e71a22e9 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts @@ -30,11 +30,15 @@ const chartBase: ChartBase = { series, }; -const getGcTimeChart = ( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) => { +function getGcTimeChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformGcMetrics({ setup, serviceName, @@ -42,6 +46,6 @@ const getGcTimeChart = ( chartBase, fieldName: METRIC_JAVA_GC_TIME, }); -}; +} export { getGcTimeChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts index eb79897f9f055..4f087594acc21 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts @@ -50,11 +50,15 @@ const chartBase: ChartBase = { series, }; -export async function getHeapMemoryChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getHeapMemoryChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts index d4084701f0f49..4a1f3f572b3a5 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/index.ts @@ -13,19 +13,23 @@ import { getMemoryChartData } from '../shared/memory'; import { getGcRateChart } from './gc/get_gc_rate_chart'; import { getGcTimeChart } from './gc/get_gc_time_chart'; -export async function getJavaMetricsCharts( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getJavaMetricsCharts({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const charts = await Promise.all([ - getCPUChartData(setup, serviceName, serviceNodeName), - getMemoryChartData(setup, serviceName, serviceNodeName), - getHeapMemoryChart(setup, serviceName, serviceNodeName), - getNonHeapMemoryChart(setup, serviceName, serviceNodeName), - getThreadCountChart(setup, serviceName, serviceNodeName), - getGcRateChart(setup, serviceName, serviceNodeName), - getGcTimeChart(setup, serviceName, serviceNodeName), + getCPUChartData({ setup, serviceName, serviceNodeName }), + getMemoryChartData({ setup, serviceName, serviceNodeName }), + getHeapMemoryChart({ setup, serviceName, serviceNodeName }), + getNonHeapMemoryChart({ setup, serviceName, serviceNodeName }), + getThreadCountChart({ setup, serviceName, serviceNodeName }), + getGcRateChart({ setup, serviceName, serviceNodeName }), + getGcTimeChart({ setup, serviceName, serviceNodeName }), ]); return { charts }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts index 50cc449da3c15..dc2684f707d17 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts @@ -47,11 +47,15 @@ const chartBase: ChartBase = { series, }; -export async function getNonHeapMemoryChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getNonHeapMemoryChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts index 0062f0a423970..b42ab61f5c766 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts @@ -39,11 +39,15 @@ const chartBase: ChartBase = { series, }; -export async function getThreadCountChart( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getThreadCountChart({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { return fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts index ca642aa12fff1..83311caf99516 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts @@ -51,11 +51,15 @@ const chartBase: ChartBase = { series, }; -export async function getCPUChartData( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getCPUChartData({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const metricsChart = await fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts index e6ee47cc815ef..8fd234903d58e 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts @@ -68,11 +68,15 @@ export const percentCgroupMemoryUsedScript = { `, }; -export async function getMemoryChartData( - setup: Setup & SetupTimeRange, - serviceName: string, - serviceNodeName?: string -) { +export async function getMemoryChartData({ + setup, + serviceName, + serviceNodeName, +}: { + setup: Setup & SetupTimeRange; + serviceName: string; + serviceNodeName?: string; +}) { const cgroupResponse = await fetchAndTransformMetrics({ setup, serviceName, diff --git a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts index 72cd65deebff6..1fc510b77c856 100644 --- a/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts +++ b/x-pack/plugins/apm/server/lib/metrics/get_metrics_chart_data_by_agent.ts @@ -25,7 +25,7 @@ export async function getMetricsChartDataByAgent({ }): Promise { switch (agentName) { case 'java': { - return getJavaMetricsCharts(setup, serviceName, serviceNodeName); + return getJavaMetricsCharts({ setup, serviceName, serviceNodeName }); } default: { diff --git a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts index fc24377ebf390..0bf3d3daaad66 100644 --- a/x-pack/plugins/apm/server/lib/metrics/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/metrics/queries.test.ts @@ -21,7 +21,7 @@ describe('metrics queries', () => { const createTests = (serviceNodeName?: string) => { it('fetches cpu chart data', async () => { mock = await inspectSearchParams((setup) => - getCPUChartData(setup, 'foo', serviceNodeName) + getCPUChartData({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -29,7 +29,7 @@ describe('metrics queries', () => { it('fetches memory chart data', async () => { mock = await inspectSearchParams((setup) => - getMemoryChartData(setup, 'foo', serviceNodeName) + getMemoryChartData({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -37,7 +37,7 @@ describe('metrics queries', () => { it('fetches heap memory chart data', async () => { mock = await inspectSearchParams((setup) => - getHeapMemoryChart(setup, 'foo', serviceNodeName) + getHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -45,7 +45,7 @@ describe('metrics queries', () => { it('fetches non heap memory chart data', async () => { mock = await inspectSearchParams((setup) => - getNonHeapMemoryChart(setup, 'foo', serviceNodeName) + getNonHeapMemoryChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); @@ -53,7 +53,7 @@ describe('metrics queries', () => { it('fetches thread count chart data', async () => { mock = await inspectSearchParams((setup) => - getThreadCountChart(setup, 'foo', serviceNodeName) + getThreadCountChart({ setup, serviceName: 'foo', serviceNodeName }) ); expect(mock.params).toMatchSnapshot(); From 31c1ca0026ffd17db16ff46214cb367e4f2fa513 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Fri, 27 Nov 2020 15:14:42 +0000 Subject: [PATCH 09/11] skip flaky suite (#84445) --- test/api_integration/apis/saved_objects/migrations.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts index 99a58620b17f5..b5fa16558514a 100644 --- a/test/api_integration/apis/saved_objects/migrations.ts +++ b/test/api_integration/apis/saved_objects/migrations.ts @@ -54,7 +54,8 @@ function getLogMock() { export default ({ getService }: FtrProviderContext) => { const esClient = getService('es'); - describe('Kibana index migration', () => { + // FLAKY: https://github.com/elastic/kibana/issues/84445 + describe.skip('Kibana index migration', () => { before(() => esClient.indices.delete({ index: '.migrate-*' })); it('Migrates an existing index that has never been migrated before', async () => { From d21e4f5af41369289023d2ead593b7a6d0bec436 Mon Sep 17 00:00:00 2001 From: Tiago Costa Date: Sat, 28 Nov 2020 03:49:04 +0000 Subject: [PATCH 10/11] skip flaky suite (#84440) --- x-pack/test/functional/apps/grok_debugger/grok_debugger.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js index 771ca2088c055..8513429639e58 100644 --- a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js +++ b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js @@ -11,7 +11,8 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['grokDebugger']); - describe('grok debugger app', function () { + // FLAKY: https://github.com/elastic/kibana/issues/84440 + describe.skip('grok debugger app', function () { this.tags('includeFirefox'); before(async () => { await esArchiver.load('empty_kibana'); From bdf7b88b45329b29d8affcb5a0dc5452ed97210e Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sun, 29 Nov 2020 22:10:23 -0500 Subject: [PATCH 11/11] [Security Solution][Detections] Handle dupes when processing threshold rules (#83062) * Fix threshold rule synthetic signal generation * Use top_hits aggregation * Find signals and aggregate over search terms * Exclude dupes * Fixes to algorithm * Sync timestamps with events/signals * Add timestampOverride * Revert changes in signal creation * Simplify query, return 10k buckets * Account for when threshold.field is not supplied * Ensure we're getting the last event when threshold.field is not provided * Add missing import * Handle case where threshold field not supplied * Fix type errors * Handle non-ECS fields * Regorganize * Address comments * Fix type error * Add unit test for buildBulkBody on threshold results * Add threshold_count back to mapping (and deprecate) * Timestamp fixes Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../routes/index/get_signals_template.ts | 2 +- .../routes/index/signals_mapping.json | 10 ++ .../signals/build_bulk_body.test.ts | 109 ++++++++++++++++++ .../signals/build_bulk_body.ts | 1 + .../detection_engine/signals/build_signal.ts | 4 +- .../signals/bulk_create_threshold_signals.ts | 10 +- .../find_previous_threshold_signals.ts | 87 ++++++++++++++ .../signals/find_threshold_signals.ts | 1 + .../signals/signal_rule_alert_type.ts | 45 +++++++- .../lib/detection_engine/signals/types.ts | 14 ++- .../endpoint/resolver/signals/mappings.json | 10 ++ .../es_archives/export_rule/mappings.json | 10 ++ 12 files changed, 296 insertions(+), 7 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index d1a9b701b2c9d..8ea1faa84cfba 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,7 +7,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 2; +export const SIGNALS_TEMPLATE_VERSION = 3; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 4e9477f3f2f73..96868e62ea978 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -337,6 +337,16 @@ "threshold_count": { "type": "float" }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } + }, "depth": { "type": "integer" } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index ad060a9304e84..cd61fbcfd0fc7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -123,6 +123,115 @@ describe('buildBulkBody', () => { expect(fakeSignalSourceHit).toEqual(expected); }); + test('bulk body builds well-defined body with threshold results', () => { + const sampleParams = sampleRuleAlertParams(); + const baseDoc = sampleDocNoSortId(); + const doc: SignalSourceHit = { + ...baseDoc, + _source: { + ...baseDoc._source, + threshold_result: { + count: 5, + value: 'abcd', + }, + }, + }; + delete doc._source.source; + const fakeSignalSourceHit = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + // Timestamp will potentially always be different so remove it for the test + // @ts-expect-error + delete fakeSignalSourceHit['@timestamp']; + const expected: Omit & { someKey: 'someValue' } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + throttle: 'no_actions', + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + exceptions_list: getListArrayMock(), + }, + threshold_result: { + count: 5, + value: 'abcd', + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + test('bulk body builds original_event if it exists on the event to begin with', () => { const sampleParams = sampleRuleAlertParams(); const doc = sampleDocNoSortId(); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index e50956e9ef752..e6e6d2e0f5fa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -71,6 +71,7 @@ export const buildBulkBody = ({ ...buildSignal([doc], rule), ...additionalSignalFields(doc), }; + delete doc._source.threshold_result; const event = buildEventTypeSignal(doc); const signalHit: SignalHit = { ...doc._source, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index b36a1cbb4a6b3..9cf2462526cfc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -96,9 +96,9 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => export const additionalSignalFields = (doc: BaseSignalHit) => { return { parent: buildParent(removeClashes(doc)), - original_time: doc._source['@timestamp'], + original_time: doc._source['@timestamp'], // This field has already been replaced with timestampOverride, if provided. original_event: doc._source.event ?? undefined, - threshold_count: doc._source.threshold_count ?? undefined, + threshold_result: doc._source.threshold_result, original_signal: doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts index edaaa345d8a69..a98aae4ec8107 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts @@ -149,7 +149,10 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: totalResults, + threshold_result: { + count: totalResults, + value: ruleId, + }, ...getThresholdSignalQueryFields(hit, filter), }; @@ -176,7 +179,10 @@ const getTransformedHits = ( const source = { '@timestamp': get(timestampOverride ?? '@timestamp', hit._source), - threshold_count: docCount, + threshold_result: { + count: docCount, + value: get(threshold.field, hit._source), + }, ...getThresholdSignalQueryFields(hit, filter), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts new file mode 100644 index 0000000000000..960693bc703d6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_previous_threshold_signals.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { TimestampOverrideOrUndefined } from '../../../../common/detection_engine/schemas/common/schemas'; +import { singleSearchAfter } from './single_search_after'; + +import { AlertServices } from '../../../../../alerts/server'; +import { Logger } from '../../../../../../../src/core/server'; +import { SignalSearchResponse } from './types'; +import { BuildRuleMessage } from './rule_messages'; + +interface FindPreviousThresholdSignalsParams { + from: string; + to: string; + indexPattern: string[]; + services: AlertServices; + logger: Logger; + ruleId: string; + bucketByField: string; + timestampOverride: TimestampOverrideOrUndefined; + buildRuleMessage: BuildRuleMessage; +} + +export const findPreviousThresholdSignals = async ({ + from, + to, + indexPattern, + services, + logger, + ruleId, + bucketByField, + timestampOverride, + buildRuleMessage, +}: FindPreviousThresholdSignalsParams): Promise<{ + searchResult: SignalSearchResponse; + searchDuration: string; + searchErrors: string[]; +}> => { + const aggregations = { + threshold: { + terms: { + field: 'signal.threshold_result.value', + }, + aggs: { + lastSignalTimestamp: { + max: { + field: 'signal.original_time', // timestamp of last event captured by bucket + }, + }, + }, + }, + }; + + const filter = { + bool: { + must: [ + { + term: { + 'signal.rule.rule_id': ruleId, + }, + }, + { + term: { + 'signal.rule.threshold.field': bucketByField, + }, + }, + ], + }, + }; + + return singleSearchAfter({ + aggregations, + searchAfterSortId: undefined, + timestampOverride, + index: indexPattern, + from, + to, + services, + logger, + filter, + pageSize: 0, + buildRuleMessage, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts index 01e4812b9c8bf..7141b61a23e6e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts @@ -51,6 +51,7 @@ export const findThresholdSignals = async ({ terms: { field: threshold.field, min_doc_count: threshold.value, + size: 10000, // max 10k buckets }, aggs: { // Get the most recent hit per bucket diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 003626e319007..f0b1825c7cc99 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -8,6 +8,7 @@ import { Logger, KibanaRequest } from 'src/core/server'; +import { Filter } from 'src/plugins/data/common'; import { SIGNALS_ID, DEFAULT_SEARCH_AFTER_PAGE_SIZE, @@ -29,6 +30,7 @@ import { RuleAlertAttributes, EqlSignalSearchResponse, BaseSignalHit, + ThresholdQueryBucket, } from './types'; import { getGapBetweenRuns, @@ -46,6 +48,7 @@ import { signalParamsSchema } from './signal_params_schema'; import { siemRuleActionGroups } from './siem_rule_action_groups'; import { findMlSignals } from './find_ml_signals'; import { findThresholdSignals } from './find_threshold_signals'; +import { findPreviousThresholdSignals } from './find_previous_threshold_signals'; import { bulkCreateMlSignals } from './bulk_create_ml_signals'; import { bulkCreateThresholdSignals } from './bulk_create_threshold_signals'; import { @@ -300,6 +303,46 @@ export const signalRulesAlertType = ({ lists: exceptionItems ?? [], }); + const { + searchResult: previousSignals, + searchErrors: previousSearchErrors, + } = await findPreviousThresholdSignals({ + indexPattern: [outputIndex], + from, + to, + services, + logger, + ruleId, + bucketByField: threshold.field, + timestampOverride, + buildRuleMessage, + }); + + previousSignals.aggregations.threshold.buckets.forEach((bucket: ThresholdQueryBucket) => { + esFilter.bool.filter.push(({ + bool: { + must_not: { + bool: { + must: [ + { + term: { + [threshold.field ?? 'signal.rule.rule_id']: bucket.key, + }, + }, + { + range: { + [timestampOverride ?? '@timestamp']: { + lte: bucket.lastSignalTimestamp.value_as_string, + }, + }, + }, + ], + }, + }, + }, + } as unknown) as Filter); + }); + const { searchResult: thresholdResults, searchErrors } = await findThresholdSignals({ inputIndexPattern: inputIndex, from, @@ -349,7 +392,7 @@ export const signalRulesAlertType = ({ }), createSearchAfterReturnType({ success, - errors: [...errors, ...searchErrors], + errors: [...errors, ...previousSearchErrors, ...searchErrors], createdSignalsCount: createdItemsCount, bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], }), diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index cda3c97c08531..9e81797b14731 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -46,6 +46,11 @@ export interface SignalsStatusParams { status: Status; } +export interface ThresholdResult { + count: number; + value: string; +} + export interface SignalSource { [key: string]: SearchTypes; // TODO: SignalSource is being used as the type for documents matching detection engine queries, but they may not @@ -67,6 +72,7 @@ export interface SignalSource { // signal.depth doesn't exist on pre-7.10 signals depth?: number; }; + threshold_result?: ThresholdResult; } export interface BulkItem { @@ -156,7 +162,7 @@ export interface Signal { original_time?: string; original_event?: SearchTypes; status: Status; - threshold_count?: SearchTypes; + threshold_result?: ThresholdResult; original_signal?: SearchTypes; depth: number; } @@ -239,3 +245,9 @@ export interface SearchAfterAndBulkCreateReturnType { export interface ThresholdAggregationBucket extends TermAggregationBucket { top_threshold_hits: BaseSearchResponse; } + +export interface ThresholdQueryBucket extends TermAggregationBucket { + lastSignalTimestamp: { + value_as_string: string; + }; +} diff --git a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json index ad77961a41445..3c6042e8efd24 100644 --- a/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json +++ b/x-pack/test/functional/es_archives/endpoint/resolver/signals/mappings.json @@ -2582,6 +2582,16 @@ }, "threshold_count": { "type": "float" + }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } }, diff --git a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json index 757121df53d44..f701f811b244b 100644 --- a/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json +++ b/x-pack/test/security_solution_cypress/es_archives/export_rule/mappings.json @@ -5131,6 +5131,16 @@ }, "threshold_count": { "type": "float" + }, + "threshold_result": { + "properties": { + "count": { + "type": "long" + }, + "value": { + "type": "keyword" + } + } } } },