From 28d8e2da273d040fcb212201305b79aff344ed7e Mon Sep 17 00:00:00 2001 From: Uladzislau Lasitsa Date: Mon, 15 Nov 2021 19:35:07 +0300 Subject: [PATCH 01/82] [Lens] Move reporting data- attributes from within the renderer to the embeddable (#116950) * Move logic about data-attrs for reporting to embeddable layer * Fix test * Fix CI * Fix test * Add onRender method so that correctly handle dispatchComplete * Update src/plugins/expressions/public/react_expression_renderer.tsx Co-authored-by: Tim Sullivan Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> Co-authored-by: Tim Sullivan --- .../public/react_expression_renderer.tsx | 3 + .../__snapshots__/table_basic.test.tsx.snap | 3 - .../components/table_basic.tsx | 12 +--- .../lens/public/embeddable/embeddable.tsx | 10 +++ .../public/embeddable/expression_wrapper.tsx | 3 + .../heatmap_visualization/chart_component.tsx | 19 +----- .../metric_visualization/expression.test.tsx | 14 ----- .../metric_visualization/expression.tsx | 14 +---- .../pie_visualization/render_function.tsx | 22 +------ .../public/visualization_container.test.tsx | 62 ++----------------- .../lens/public/visualization_container.tsx | 30 +-------- .../public/xy_visualization/expression.tsx | 17 +---- 12 files changed, 35 insertions(+), 174 deletions(-) diff --git a/src/plugins/expressions/public/react_expression_renderer.tsx b/src/plugins/expressions/public/react_expression_renderer.tsx index b42ea3f3fd149..abb6b01e77feb 100644 --- a/src/plugins/expressions/public/react_expression_renderer.tsx +++ b/src/plugins/expressions/public/react_expression_renderer.tsx @@ -38,6 +38,7 @@ export interface ReactExpressionRendererProps extends IExpressionLoaderParams { * An observable which can be used to re-run the expression without destroying the component */ reload$?: Observable; + onRender$?: (item: number) => void; debounce?: number; } @@ -66,6 +67,7 @@ export default function ReactExpressionRenderer({ expression, onEvent, onData$, + onRender$, reload$, debounce, ...expressionLoaderOptions @@ -155,6 +157,7 @@ export default function ReactExpressionRenderer({ ...defaultState, isEmpty: false, })); + onRender$?.(item); }) ); diff --git a/x-pack/plugins/lens/public/datatable_visualization/components/__snapshots__/table_basic.test.tsx.snap b/x-pack/plugins/lens/public/datatable_visualization/components/__snapshots__/table_basic.test.tsx.snap index bf8497e686e96..40eb546dfc208 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/components/__snapshots__/table_basic.test.tsx.snap +++ b/x-pack/plugins/lens/public/datatable_visualization/components/__snapshots__/table_basic.test.tsx.snap @@ -3,7 +3,6 @@ exports[`DatatableComponent it renders actions column when there are row actions 1`] = ` { if (isEmpty) { return ( - + ); @@ -339,11 +335,7 @@ export const DatatableComponent = (props: DatatableRenderProps) => { }); return ( - + { + this.renderComplete.dispatchComplete(); + }; + /** * * @param {HTMLElement} domNode @@ -371,6 +375,7 @@ export class Embeddable */ render(domNode: HTMLElement | Element) { this.domNode = domNode; + super.render(domNode as HTMLElement); if (!this.savedVis || !this.isInitialized || this.isDestroyed) { return; } @@ -378,6 +383,10 @@ export class Embeddable this.input.onLoad(true); } + this.domNode.setAttribute('data-shared-item', ''); + + this.renderComplete.dispatchInProgress(); + const executionContext = { type: 'lens', name: this.savedVis.visualizationType ?? '', @@ -400,6 +409,7 @@ export class Embeddable searchSessionId={this.externalSearchContext.searchSessionId} handleEvent={this.handleEvent} onData$={this.updateActiveData} + onRender$={this.onRender} interactive={!input.disableTriggers} renderMode={input.renderMode} syncColors={input.syncColors} diff --git a/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx b/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx index 3de914d13d69d..4c57bf7e7a53a 100644 --- a/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx +++ b/x-pack/plugins/lens/public/embeddable/expression_wrapper.tsx @@ -35,6 +35,7 @@ export interface ExpressionWrapperProps { data: unknown, inspectorAdapters?: Partial | undefined ) => void; + onRender$: () => void; renderMode?: RenderMode; syncColors?: boolean; hasCompatibleActions?: ReactExpressionRendererProps['hasCompatibleActions']; @@ -106,6 +107,7 @@ export function ExpressionWrapper({ interactive, searchSessionId, onData$, + onRender$, renderMode, syncColors, hasCompatibleActions, @@ -132,6 +134,7 @@ export function ExpressionWrapper({ searchContext={searchContext} searchSessionId={searchSessionId} onData$={onData$} + onRender$={onRender$} inspectorAdapters={lensInspector.adapters} renderMode={renderMode} syncColors={syncColors} diff --git a/x-pack/plugins/lens/public/heatmap_visualization/chart_component.tsx b/x-pack/plugins/lens/public/heatmap_visualization/chart_component.tsx index 5e4c2f2684062..4300208109b76 100644 --- a/x-pack/plugins/lens/public/heatmap_visualization/chart_component.tsx +++ b/x-pack/plugins/lens/public/heatmap_visualization/chart_component.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { FC, useEffect, useMemo, useState } from 'react'; +import React, { FC, useMemo } from 'react'; import { Chart, ElementClickListener, @@ -396,23 +396,8 @@ export const HeatmapComponent: FC = ({ const MemoizedChart = React.memo(HeatmapComponent); export function HeatmapChartReportable(props: HeatmapRenderProps) { - const [state, setState] = useState({ - isReady: false, - }); - - // It takes a cycle for the chart to render. This prevents - // reporting from printing a blank chart placeholder. - useEffect(() => { - setState({ isReady: true }); - }, [setState]); - return ( - + ); diff --git a/x-pack/plugins/lens/public/metric_visualization/expression.test.tsx b/x-pack/plugins/lens/public/metric_visualization/expression.test.tsx index db70a7c8508e5..baa0a5adc3b70 100644 --- a/x-pack/plugins/lens/public/metric_visualization/expression.test.tsx +++ b/x-pack/plugins/lens/public/metric_visualization/expression.test.tsx @@ -87,8 +87,6 @@ describe('metric_expression', () => { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` { ).toMatchInlineSnapshot(` ( - + ); @@ -84,11 +80,7 @@ export function MetricChart({ : Number(Number(row[accessor]).toFixed(3)).toString(); return ( - +
{value} diff --git a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx index 070448978f4ef..05b9ca9c34168 100644 --- a/x-pack/plugins/lens/public/pie_visualization/render_function.tsx +++ b/x-pack/plugins/lens/public/pie_visualization/render_function.tsx @@ -6,7 +6,7 @@ */ import { uniq } from 'lodash'; -import React, { useEffect, useState } from 'react'; +import React from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText } from '@elastic/eui'; import { @@ -224,13 +224,6 @@ export function PieComponent( }, }); - const [isReady, setIsReady] = useState(false); - // It takes a cycle for the chart to render. This prevents - // reporting from printing a blank chart placeholder. - useEffect(() => { - setIsReady(true); - }, []); - const hasNegative = firstTable.rows.some((row) => { const value = row[metricColumn.id]; return typeof value === 'number' && value < 0; @@ -247,11 +240,7 @@ export function PieComponent( if (isEmpty) { return ( - + ); @@ -278,12 +267,7 @@ export function PieComponent( }; return ( - + { - test('renders reporting data attributes when ready', () => { - const component = mount(Hello!); - - const reportingEl = component.find('[data-shared-item]').first(); - expect(reportingEl.prop('data-render-complete')).toBeTruthy(); - expect(reportingEl.prop('data-shared-item')).toBeTruthy(); - }); - - test('does not render data attributes when not ready', () => { - const component = mount( - Hello! - ); - - const reportingEl = component.find('[data-shared-item]').first(); - expect(reportingEl.prop('data-render-complete')).toBeFalsy(); - expect(reportingEl.prop('data-shared-item')).toBeTruthy(); - }); - - test('increments counter in data attribute for each render', () => { - const component = mount(Hello!); - - let reportingEl = component.find('[data-shared-item]').first(); - expect(reportingEl.prop('data-rendering-count')).toEqual(1); - - component.setProps({ children: 'Hello2!' }); - - reportingEl = component.find('[data-shared-item]').first(); - expect(reportingEl.prop('data-rendering-count')).toEqual(2); - }); - test('renders child content', () => { - const component = mount( - Hello! - ); - - expect(component.text()).toEqual('Hello!'); - }); - - test('defaults to rendered', () => { const component = mount(Hello!); - const reportingEl = component.find('[data-shared-item]').first(); - expect(reportingEl.prop('data-render-complete')).toBeTruthy(); - expect(reportingEl.prop('data-shared-item')).toBeTruthy(); - }); - - test('renders title and description for reporting, if provided', () => { - const component = mount( - - Hello! - - ); - const reportingEl = component.find('[data-shared-item]').first(); - - expect(reportingEl.prop('data-title')).toEqual('shazam!'); - expect(reportingEl.prop('data-description')).toEqual('Description'); + expect(component.text()).toEqual('Hello!'); }); test('renders style', () => { const component = mount( Hello! ); - const reportingEl = component.find('[data-shared-item]').first(); + const el = component.find('.lnsVisualizationContainer').first(); - expect(reportingEl.prop('style')).toEqual({ color: 'blue' }); + expect(el.prop('style')).toEqual({ color: 'blue' }); }); test('combines class names with container class', () => { const component = mount( Hello! ); - const reportingEl = component.find('[data-shared-item]').first(); + const el = component.find('.lnsVisualizationContainer').first(); - expect(reportingEl.prop('className')).toEqual('myClass lnsVisualizationContainer'); + expect(el.prop('className')).toEqual('myClass lnsVisualizationContainer'); }); }); diff --git a/x-pack/plugins/lens/public/visualization_container.tsx b/x-pack/plugins/lens/public/visualization_container.tsx index 89f7f3eb0d61e..a9020278db235 100644 --- a/x-pack/plugins/lens/public/visualization_container.tsx +++ b/x-pack/plugins/lens/public/visualization_container.tsx @@ -7,44 +7,18 @@ import './visualization_container.scss'; -import React, { useRef } from 'react'; +import React from 'react'; import classNames from 'classnames'; -interface Props extends React.HTMLAttributes { - isReady?: boolean; - reportTitle?: string; - reportDescription?: string; -} - -/** - * This is a convenience component that wraps rendered Lens visualizations. It adds reporting - * attributes (data-shared-item, data-render-complete, and data-title). - */ export function VisualizationContainer({ - isReady = true, - reportTitle, - reportDescription, children, className, ...rest -}: Props) { - const counterRef = useRef(0); - counterRef.current++; - const attributes: Partial<{ 'data-title': string; 'data-description': string }> = {}; - if (reportTitle) { - attributes['data-title'] = reportTitle; - } - if (reportDescription) { - attributes['data-description'] = reportDescription; - } +}: React.HTMLAttributes) { return (
{children} diff --git a/x-pack/plugins/lens/public/xy_visualization/expression.tsx b/x-pack/plugins/lens/public/xy_visualization/expression.tsx index a16537d3aae94..60d0fe85ed546 100644 --- a/x-pack/plugins/lens/public/xy_visualization/expression.tsx +++ b/x-pack/plugins/lens/public/xy_visualization/expression.tsx @@ -7,7 +7,7 @@ import './expression.scss'; -import React, { useState, useEffect, useRef } from 'react'; +import React, { useRef } from 'react'; import ReactDOM from 'react-dom'; import { Chart, @@ -207,21 +207,8 @@ function getIconForSeriesType(seriesType: SeriesType): IconType { const MemoizedChart = React.memo(XYChart); export function XYChartReportable(props: XYChartRenderProps) { - const [isReady, setIsReady] = useState(false); - - // It takes a cycle for the XY chart to render. This prevents - // reporting from printing a blank chart placeholder. - useEffect(() => { - setIsReady(true); - }, [setIsReady]); - return ( - + ); From 729830ff9a68e5cd2a6e69555959829ea27ef5c3 Mon Sep 17 00:00:00 2001 From: Dominique Clarke Date: Mon, 15 Nov 2021 11:40:44 -0500 Subject: [PATCH 02/82] [Elastic Synthetics Integration] Add Beta disclaimer in monitor type dropdown (#117918) * add Beta disclaimer in monitor type dropdown * adjust types Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../fleet_package/browser/source_field.tsx | 2 +- .../fleet_package/custom_fields.test.tsx | 4 ++-- .../fleet_package/custom_fields.tsx | 22 ++++++++++++++----- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/uptime/public/components/fleet_package/browser/source_field.tsx b/x-pack/plugins/uptime/public/components/fleet_package/browser/source_field.tsx index e4b03e53432dd..f0e06d17340b5 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/browser/source_field.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/browser/source_field.tsx @@ -121,7 +121,7 @@ export const SourceField = ({ onChange, defaultConfig = defaultValues }: Props) label={ } labelAppend={} diff --git a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.test.tsx b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.test.tsx index 5c24883d588e2..805b4aa0d7e0d 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.test.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.test.tsx @@ -233,7 +233,7 @@ describe('', () => { ).toBeInTheDocument(); // expect tls options to be available for browser - expect(queryByLabelText('Zip Proxy URL')).toBeInTheDocument(); + expect(queryByLabelText('Proxy Zip URL')).toBeInTheDocument(); expect(queryByLabelText('Enable TLS configuration for Zip URL')).toBeInTheDocument(); // ensure at least one browser advanced option is present @@ -316,7 +316,7 @@ describe('', () => { expect(getByText('HTTP')).toBeInTheDocument(); expect(getByText('TCP')).toBeInTheDocument(); expect(getByText('ICMP')).toBeInTheDocument(); - expect(queryByText('Browser')).not.toBeInTheDocument(); + expect(queryByText('Browser (Beta)')).not.toBeInTheDocument(); }); }); }); diff --git a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx index 98eac21a42076..0bdb2d62a7367 100644 --- a/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx +++ b/x-pack/plugins/uptime/public/components/fleet_package/custom_fields.tsx @@ -5,6 +5,7 @@ * 2.0. */ import React, { useMemo, memo } from 'react'; +import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, @@ -34,6 +35,21 @@ interface Props { dataStreams?: DataStream[]; } +const dataStreamToString = [ + { value: DataStream.HTTP, text: 'HTTP' }, + { value: DataStream.TCP, text: 'TCP' }, + { value: DataStream.ICMP, text: 'ICMP' }, + { + value: DataStream.BROWSER, + text: i18n.translate( + 'xpack.uptime.createPackagePolicy.stepConfigure.monitorIntegrationSettingsSection.browserLabel', + { + defaultMessage: 'Browser (Beta)', + } + ), + }, +]; + export const CustomFields = memo(({ validate, dataStreams = [] }) => { const { monitorType, setMonitorType, isTLSEnabled, setIsTLSEnabled, isEditable } = usePolicyConfigContext(); @@ -43,12 +59,6 @@ export const CustomFields = memo(({ validate, dataStreams = [] }) => { const isBrowser = monitorType === DataStream.BROWSER; const dataStreamOptions = useMemo(() => { - const dataStreamToString = [ - { value: DataStream.HTTP, text: 'HTTP' }, - { value: DataStream.TCP, text: 'TCP' }, - { value: DataStream.ICMP, text: 'ICMP' }, - { value: DataStream.BROWSER, text: 'Browser' }, - ]; return dataStreamToString.filter((dataStream) => dataStreams.includes(dataStream.value)); }, [dataStreams]); From 2cb9503416a75592909e7b75153200733b0ffcc3 Mon Sep 17 00:00:00 2001 From: Kyle Pollich Date: Mon, 15 Nov 2021 11:54:25 -0500 Subject: [PATCH 03/82] Add note on Storybook to Fleet README (#118540) --- x-pack/plugins/fleet/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugins/fleet/README.md b/x-pack/plugins/fleet/README.md index 80d13b28c8265..29436440fac8b 100644 --- a/x-pack/plugins/fleet/README.md +++ b/x-pack/plugins/fleet/README.md @@ -120,3 +120,12 @@ You need to have `docker` to run ingest manager api integration tests FLEET_PACKAGE_REGISTRY_DOCKER_IMAGE='docker.elastic.co/package-registry/distribution:production' FLEET_PACKAGE_REGISTRY_PORT=12345 yarn test:ftr:runner ``` +### Storybook + +Fleet contains [Storybook](https://storybook.js.org/) stories for developing UI components in isolation. To start the Storybook environment for Fleet, run the following from your `kibana` project root: + +```sh +$ yarn storybook fleet +``` + +Write stories by creating `.stories.tsx` files colocated with the components you're working on. Consult the [Storybook docs](https://storybook.js.org/docs/react/get-started/introduction) for more information. From 13dedff79212fcd6ebc72d3af75405881e817f05 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 15 Nov 2021 18:11:54 +0100 Subject: [PATCH 04/82] [Exploratory view] Fix embeddable example (#117495) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../exploratory_view_example/public/app.tsx | 2 +- .../embeddable/embeddable.tsx | 3 +- .../exploratory_view/embeddable/index.tsx | 31 +++++++---- .../utils/observability_index_patterns.ts | 52 +++++++++++++------ 4 files changed, 59 insertions(+), 29 deletions(-) diff --git a/x-pack/examples/exploratory_view_example/public/app.tsx b/x-pack/examples/exploratory_view_example/public/app.tsx index 9ad37b6fdbfef..4b8ed22723f89 100644 --- a/x-pack/examples/exploratory_view_example/public/app.tsx +++ b/x-pack/examples/exploratory_view_example/public/app.tsx @@ -40,7 +40,7 @@ export const App = (props: { reportDefinitions: { 'monitor.id': ['ALL_VALUES'], }, - breakdown: 'observer.geo.name', + breakdown: 'monitor.type', operationType: 'average', dataType: 'synthetics', seriesType: 'line', diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/embeddable.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/embeddable.tsx index 35ce2fc6c1a47..497435ec69a50 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/embeddable.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/embeddable.tsx @@ -10,7 +10,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiText, EuiTitle } from '@elastic/eui'; import styled from 'styled-components'; import { AllSeries, useTheme } from '../../../..'; import { LayerConfig, LensAttributes } from '../configurations/lens_attributes'; -import { ReportViewType } from '../types'; +import { AppDataType, ReportViewType } from '../types'; import { getLayerConfigs } from '../hooks/use_lens_attributes'; import { LensPublicStart, XYState } from '../../../../../../lens/public'; import { OperationTypeComponent } from '../series_editor/columns/operation_type_select'; @@ -24,6 +24,7 @@ export interface ExploratoryEmbeddableProps { showCalculationMethod?: boolean; axisTitlesVisibility?: XYState['axisTitlesVisibilitySettings']; legendIsVisible?: boolean; + dataTypesIndexPatterns?: Record; } export interface ExploratoryEmbeddableComponentProps extends ExploratoryEmbeddableProps { diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/index.tsx b/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/index.tsx index ad84880de5eb1..e68ddfe55e6f5 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/index.tsx +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/embeddable/index.tsx @@ -13,6 +13,7 @@ import { ObservabilityIndexPatterns } from '../utils/observability_index_pattern import { ObservabilityPublicPluginsStart } from '../../../../plugin'; import type { IndexPatternState } from '../hooks/use_app_index_pattern'; import { EuiThemeProvider } from '../../../../../../../../src/plugins/kibana_react/common'; +import type { AppDataType } from '../types'; const Embeddable = React.lazy(() => import('./embeddable')); @@ -36,18 +37,26 @@ export function getExploratoryViewEmbeddable( const isDarkMode = core.uiSettings.get('theme:darkMode'); - const loadIndexPattern = useCallback(async ({ dataType }) => { - setLoading(true); - try { - const obsvIndexP = new ObservabilityIndexPatterns(plugins.data); - const indPattern = await obsvIndexP.getIndexPattern(dataType, 'heartbeat-*'); - setIndexPatterns((prevState) => ({ ...(prevState ?? {}), [dataType]: indPattern })); + const loadIndexPattern = useCallback( + async ({ dataType }: { dataType: AppDataType }) => { + const dataTypesIndexPatterns = props.dataTypesIndexPatterns; - setLoading(false); - } catch (e) { - setLoading(false); - } - }, []); + setLoading(true); + try { + const obsvIndexP = new ObservabilityIndexPatterns(plugins.data); + const indPattern = await obsvIndexP.getIndexPattern( + dataType, + dataTypesIndexPatterns?.[dataType] + ); + setIndexPatterns((prevState) => ({ ...(prevState ?? {}), [dataType]: indPattern })); + + setLoading(false); + } catch (e) { + setLoading(false); + } + }, + [props.dataTypesIndexPatterns] + ); useEffect(() => { loadIndexPattern({ dataType: series.dataType }); diff --git a/x-pack/plugins/observability/public/components/shared/exploratory_view/utils/observability_index_patterns.ts b/x-pack/plugins/observability/public/components/shared/exploratory_view/utils/observability_index_patterns.ts index f591ef63a61fb..1c14ffe3d13c9 100644 --- a/x-pack/plugins/observability/public/components/shared/exploratory_view/utils/observability_index_patterns.ts +++ b/x-pack/plugins/observability/public/components/shared/exploratory_view/utils/observability_index_patterns.ts @@ -16,6 +16,7 @@ import { rumFieldFormats } from '../configurations/rum/field_formats'; import { syntheticsFieldFormats } from '../configurations/synthetics/field_formats'; import { AppDataType, FieldFormat, FieldFormatParams } from '../types'; import { apmFieldFormats } from '../configurations/apm/field_formats'; +import { getDataHandler } from '../../../../data_handler'; const appFieldFormats: Record = { infra_logs: null, @@ -125,28 +126,47 @@ export class ObservabilityIndexPatterns { return fieldFormatMap; } - async getIndexPattern(app: AppDataType, indices: string): Promise { + async getDataTypeIndices(dataType: AppDataType) { + switch (dataType) { + case 'ux': + case 'synthetics': + const resultUx = await getDataHandler(dataType)?.hasData(); + return resultUx?.indices; + case 'apm': + case 'mobile': + const resultApm = await getDataHandler('apm')?.hasData(); + return resultApm?.indices.transaction; + } + } + + async getIndexPattern(app: AppDataType, indices?: string): Promise { if (!this.data) { throw new Error('data is not defined'); } + let appIndices = indices; + if (!appIndices) { + appIndices = await this.getDataTypeIndices(app); + } - try { - const indexPatternId = getAppIndexPatternId(app, indices); - const indexPatternTitle = getAppIndicesWithPattern(app, indices); - // we will get index pattern by id - const indexPattern = await this.data?.indexPatterns.get(indexPatternId); + if (appIndices) { + try { + const indexPatternId = getAppIndexPatternId(app, appIndices); + const indexPatternTitle = getAppIndicesWithPattern(app, appIndices); + // we will get index pattern by id + const indexPattern = await this.data?.indexPatterns.get(indexPatternId); - // and make sure title matches, otherwise, we will need to create it - if (indexPattern.title !== indexPatternTitle) { - return await this.createIndexPattern(app, indices); - } + // and make sure title matches, otherwise, we will need to create it + if (indexPattern.title !== indexPatternTitle) { + return await this.createIndexPattern(app, appIndices); + } - // this is intentional a non blocking call, so no await clause - this.validateFieldFormats(app, indexPattern); - return indexPattern; - } catch (e: unknown) { - if (e instanceof SavedObjectNotFound) { - return await this.createIndexPattern(app, indices); + // this is intentional a non blocking call, so no await clause + this.validateFieldFormats(app, indexPattern); + return indexPattern; + } catch (e: unknown) { + if (e instanceof SavedObjectNotFound) { + return await this.createIndexPattern(app, appIndices); + } } } } From f564268c40fb4e64fd97055385ea4ea9bcce753b Mon Sep 17 00:00:00 2001 From: Gloria Hornero Date: Mon, 15 Nov 2021 18:16:37 +0100 Subject: [PATCH 05/82] fixes (#118089) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .github/CODEOWNERS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d5c569ac9d552..e9ac2965d0a6f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -416,10 +416,10 @@ /x-pack/test/security_solution_endpoint/apps/endpoint/ @elastic/security-onboarding-and-lifecycle-mgt ## Security Solution sub teams - security-engineering-productivity -x-pack/plugins/security_solution/cypress/ccs_integration -x-pack/plugins/security_solution/cypress/upgrade_integration -x-pack/plugins/security_solution/cypress/README.md -x-pack/test/security_solution_cypress +x-pack/plugins/security_solution/cypress/ccs_integration @elastic/security-engineering-productivity +x-pack/plugins/security_solution/cypress/upgrade_integration @elastic/security-engineering-productivity +x-pack/plugins/security_solution/cypress/README.md @elastic/security-engineering-productivity +x-pack/test/security_solution_cypress @elastic/security-engineering-productivity # Security Intelligence And Analytics /x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics From 75b56c4ab4dc1368ba03810b287a199a6ef45aaa Mon Sep 17 00:00:00 2001 From: Kylie Geller Date: Mon, 15 Nov 2021 12:18:57 -0500 Subject: [PATCH 06/82] [Ingest Node Pipelines] Adding create from csv (ecs mapper) functionality (#101216) --- .../helpers/http_requests.ts | 12 + .../client_integration/helpers/index.ts | 2 + .../pipelines_create_from_csv.helpers.ts | 81 ++++++ .../helpers/pipelines_list.helpers.ts | 2 +- .../helpers/setup_environment.tsx | 9 + .../ingest_pipelines_create_from_csv.test.tsx | 188 +++++++++++++ .../ingest_pipelines_list.test.ts | 4 +- .../plugins/ingest_pipelines/common/types.ts | 5 + x-pack/plugins/ingest_pipelines/kibana.json | 2 +- .../public/application/app.tsx | 9 +- .../public/application/index.tsx | 11 +- .../application/mount_management_section.ts | 10 +- .../public/application/sections/index.ts | 2 + .../sections/pipelines_create/index.ts | 1 + .../pipelines_create/pipelines_create.tsx | 14 +- .../error_display.tsx | 30 +++ .../pipelines_create_from_csv/index.ts | 8 + .../instructions.tsx | 50 ++++ .../pipelines_create_from_csv/main.tsx | 218 +++++++++++++++ .../pipelines_csv_uploader.tsx | 126 +++++++++ .../pipelines_preview.tsx | 163 +++++++++++ .../sections/pipelines_list/empty_list.tsx | 78 +++++- .../sections/pipelines_list/main.tsx | 2 +- .../sections/pipelines_list/table.tsx | 66 ++++- .../public/application/services/api.ts | 11 +- .../application/services/file_reader.ts | 38 +++ .../public/application/services/index.ts | 2 + .../public/application/services/navigation.ts | 9 + .../plugins/ingest_pipelines/public/types.ts | 2 + .../ingest_pipelines/server/lib/index.ts | 8 + .../server/lib/mapper.test.ts | 251 +++++++++++++++++ .../ingest_pipelines/server/lib/mapper.ts | 253 ++++++++++++++++++ .../server/routes/api/index.ts | 2 + .../server/routes/api/parse_csv.ts | 40 +++ .../ingest_pipelines/server/routes/index.ts | 2 + x-pack/plugins/ingest_pipelines/tsconfig.json | 1 + .../apps/ingest_node_pipelines.ts | 1 + .../ingest_pipelines/ingest_pipelines.ts | 25 ++ .../exports/example_mapping.csv | 2 + .../apps/ingest_pipelines/ingest_pipelines.ts | 59 ++-- .../page_objects/ingest_pipelines_page.ts | 33 ++- 41 files changed, 1780 insertions(+), 52 deletions(-) create mode 100644 x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_create_from_csv.helpers.ts create mode 100644 x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create_from_csv.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/error_display.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/index.ts create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/instructions.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/main.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_csv_uploader.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_preview.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/services/file_reader.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/lib/index.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/lib/mapper.test.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/lib/mapper.ts create mode 100644 x-pack/plugins/ingest_pipelines/server/routes/api/parse_csv.ts create mode 100644 x-pack/test/functional/apps/ingest_pipelines/exports/example_mapping.csv diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/http_requests.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/http_requests.ts index 67adb15f7cd85..e5c0e0a5e3673 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/http_requests.ts +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/http_requests.ts @@ -52,11 +52,23 @@ const registerHttpRequestMockHelpers = (server: SinonFakeServer) => { ]); }; + const setParseCsvResponse = (response?: object, error?: any) => { + const status = error ? error.status || 400 : 200; + const body = error ? JSON.stringify(error.body) : JSON.stringify(response); + + server.respondWith('POST', `${API_BASE_PATH}/parse_csv`, [ + status, + { 'Content-Type': 'application/json' }, + body, + ]); + }; + return { setLoadPipelinesResponse, setLoadPipelineResponse, setDeletePipelineResponse, setCreatePipelineResponse, + setParseCsvResponse, }; }; diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/index.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/index.ts index 8b3d91dac0f51..ed88156b740bf 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/index.ts +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/index.ts @@ -9,6 +9,7 @@ import { setup as pipelinesListSetup } from './pipelines_list.helpers'; import { setup as pipelinesCreateSetup } from './pipelines_create.helpers'; import { setup as pipelinesCloneSetup } from './pipelines_clone.helpers'; import { setup as pipelinesEditSetup } from './pipelines_edit.helpers'; +import { setup as pipelinesCreateFromCsvSetup } from './pipelines_create_from_csv.helpers'; export { nextTick, getRandomString, findTestSubject } from '@kbn/test/jest'; @@ -19,4 +20,5 @@ export const pageHelpers = { pipelinesCreate: { setup: pipelinesCreateSetup }, pipelinesClone: { setup: pipelinesCloneSetup }, pipelinesEdit: { setup: pipelinesEditSetup }, + pipelinesCreateFromCsv: { setup: pipelinesCreateFromCsvSetup }, }; diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_create_from_csv.helpers.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_create_from_csv.helpers.ts new file mode 100644 index 0000000000000..e7de57de0e948 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_create_from_csv.helpers.ts @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; + +import { registerTestBed, TestBed, AsyncTestBedConfig } from '@kbn/test/jest'; +import { PipelinesCreateFromCsv } from '../../../public/application/sections/pipelines_create_from_csv'; +import { WithAppDependencies } from './setup_environment'; +import { getCreateFromCsvPath, ROUTES } from '../../../public/application/services/navigation'; + +const testBedConfig: AsyncTestBedConfig = { + memoryRouter: { + initialEntries: [getCreateFromCsvPath()], + componentRoutePath: ROUTES.createFromCsv, + }, + doMountAsync: true, +}; + +const initTestBed = registerTestBed(WithAppDependencies(PipelinesCreateFromCsv), testBedConfig); + +export type PipelineCreateFromCsvTestBed = TestBed & { + actions: ReturnType; +}; +const createFromCsvActions = (testBed: TestBed) => { + // User Actions + + const selectCsvForUpload = (file?: File) => { + const { find } = testBed; + const csv = [file ? file : 'foo'] as any; + + act(() => { + find('csvFilePicker').simulate('change', { files: csv }); + }); + }; + + const clickProcessCsv = async () => { + const { component, find } = testBed; + + await act(async () => { + find('processFileButton').simulate('click'); + }); + + component.update(); + }; + + const uploadFile = async (file?: File) => { + selectCsvForUpload(file); + await clickProcessCsv(); + }; + + return { + selectCsvForUpload, + clickProcessCsv, + uploadFile, + }; +}; + +export const setup = async (): Promise => { + const testBed = await initTestBed(); + + return { + ...testBed, + actions: createFromCsvActions(testBed), + }; +}; + +export type PipelineCreateFromCsvTestSubjects = + | 'pageTitle' + | 'documentationLink' + | 'processFileButton' + | 'csvFilePicker' + | 'errorCallout' + | 'errorDetailsMessage' + | 'pipelineMappingsJSONEditor' + | 'continueToCreate' + | 'copyToClipboard' + | 'downloadJson'; diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_list.helpers.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_list.helpers.ts index 3cd768104203a..e2d9f1f8bf5f9 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_list.helpers.ts +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/pipelines_list.helpers.ts @@ -96,7 +96,7 @@ export const setup = async (): Promise => { export type PipelineListTestSubjects = | 'appTitle' | 'documentationLink' - | 'createPipelineButton' + | 'createPipelineDropdown' | 'pipelinesTable' | 'pipelineDetails' | 'pipelineDetails.title' diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx index 7ba5e44cddf61..a2c36e204cbea 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/helpers/setup_environment.tsx @@ -16,6 +16,7 @@ import { notificationServiceMock, docLinksServiceMock, scopedHistoryMock, + uiSettingsServiceMock, } from '../../../../../../src/core/public/mocks'; import { usageCollectionPluginMock } from '../../../../../../src/plugins/usage_collection/public/mocks'; @@ -41,13 +42,21 @@ const appServices = { metric: uiMetricService, documentation: documentationService, api: apiService, + fileReader: { + readFile: jest.fn((file) => file.text()), + }, notifications: notificationServiceMock.createSetupContract(), history, + uiSettings: uiSettingsServiceMock.createSetupContract(), urlGenerators: { getUrlGenerator: jest.fn().mockReturnValue({ createUrl: jest.fn(), }), }, + fileUpload: { + getMaxBytes: jest.fn().mockReturnValue(100), + getMaxBytesFormatted: jest.fn().mockReturnValue('100'), + }, }; export const setupEnvironment = () => { diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create_from_csv.test.tsx b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create_from_csv.test.tsx new file mode 100644 index 0000000000000..5f1230f004684 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_create_from_csv.test.tsx @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { act } from 'react-dom/test-utils'; + +import { API_BASE_PATH } from '../../common/constants'; + +import { setupEnvironment, pageHelpers } from './helpers'; +import { PipelineCreateFromCsvTestBed } from './helpers/pipelines_create_from_csv.helpers'; + +const { setup } = pageHelpers.pipelinesCreateFromCsv; + +jest.mock('@elastic/eui', () => { + const original = jest.requireActual('@elastic/eui'); + + return { + ...original, + EuiFilePicker: (props: any) => ( + { + props.onChange(syntheticEvent.files); + }} + /> + ), + }; +}); + +jest.mock('../../../../../src/plugins/kibana_react/public', () => { + const original = jest.requireActual('../../../../../src/plugins/kibana_react/public'); + + return { + ...original, + CodeEditorField: (props: any) => ( +

{props.value}

+ ), + }; +}); + +describe('', () => { + const { server, httpRequestsMockHelpers } = setupEnvironment(); + let testBed: PipelineCreateFromCsvTestBed; + + afterAll(() => { + server.restore(); + }); + + beforeEach(async () => { + await act(async () => { + testBed = await setup(); + }); + + testBed.component.update(); + }); + + describe('on component mount', () => { + test('should render the correct page header and documentation link', () => { + const { exists, find } = testBed; + + expect(exists('pageTitle')).toBe(true); + expect(find('pageTitle').text()).toEqual('Create pipeline from CSV'); + + expect(exists('documentationLink')).toBe(true); + expect(find('documentationLink').text()).toBe('Create pipeline docs'); + }); + + describe('form validation', () => { + test('should prevent form submission if file for upload is missing', async () => { + const { component, find, actions } = testBed; + + expect(find('processFileButton').props().disabled).toEqual(true); + + actions.selectCsvForUpload(); + component.update(); + + expect(find('processFileButton').props().disabled).toEqual(false); + }); + }); + + describe('form submission', () => { + const fileContent = 'Mock file content'; + + const mockFile = { + name: 'foo.csv', + text: () => Promise.resolve(fileContent), + size: fileContent.length, + } as File; + + const parsedCsv = { + processors: [ + { + set: { + field: 'foo', + if: 'ctx.bar != null', + value: '{{bar}}', + }, + }, + ], + }; + + beforeEach(async () => { + await act(async () => { + testBed = await setup(); + }); + + testBed.component.update(); + + testBed.actions.selectCsvForUpload(mockFile); + + testBed.component.update(); + + httpRequestsMockHelpers.setParseCsvResponse(parsedCsv, undefined); + }); + + test('should parse csv from file upload', async () => { + const { actions, find } = testBed; + const totalRequests = server.requests.length; + + await actions.clickProcessCsv(); + + expect(server.requests.length).toBe(totalRequests + 1); + + const lastRequest = server.requests[server.requests.length - 1]; + expect(lastRequest.url).toBe(`${API_BASE_PATH}/parse_csv`); + expect(JSON.parse(JSON.parse(lastRequest.requestBody).body)).toEqual({ + copyAction: 'copy', + file: fileContent, + }); + + expect(JSON.parse(find('pipelineMappingsJSONEditor').text())).toEqual(parsedCsv); + }); + + test('should render an error message if error mapping pipeline', async () => { + const { actions, find, exists } = testBed; + + const errorTitle = 'title'; + const errorDetails = 'helpful description'; + + const error = { + status: 400, + error: 'Bad Request', + message: `${errorTitle}:${errorDetails}`, + }; + + httpRequestsMockHelpers.setParseCsvResponse(undefined, { body: error }); + + actions.selectCsvForUpload(mockFile); + await actions.clickProcessCsv(); + + expect(exists('errorCallout')).toBe(true); + expect(find('errorCallout').text()).toContain(errorTitle); + expect(find('errorCallout').text()).toContain(errorDetails); + }); + + describe('results', () => { + beforeEach(async () => { + await act(async () => { + testBed = await setup(); + }); + + testBed.component.update(); + }); + + test('result buttons', async () => { + const { exists, find } = testBed; + + await testBed.actions.uploadFile(mockFile); + + expect(exists('pipelineMappingsJSONEditor')).toBe(true); + + expect(exists('continueToCreate')).toBe(true); + expect(find('continueToCreate').text()).toContain('Continue to create pipeline'); + + expect(exists('copyToClipboard')).toBe(true); + expect(find('copyToClipboard').text()).toContain('Copy JSON'); + + expect(exists('downloadJson')).toBe(true); + expect(find('downloadJson').text()).toContain('Download JSON'); + }); + }); + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_list.test.ts b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_list.test.ts index 19a2abb5a5a52..3f6a0f57bac34 100644 --- a/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_list.test.ts +++ b/x-pack/plugins/ingest_pipelines/__jest__/client_integration/ingest_pipelines_list.test.ts @@ -58,8 +58,8 @@ describe('', () => { expect(exists('documentationLink')).toBe(true); expect(find('documentationLink').text()).toBe('Ingest Pipelines docs'); - // Verify create button exists - expect(exists('createPipelineButton')).toBe(true); + // Verify create dropdown exists + expect(exists('createPipelineDropdown')).toBe(true); // Verify table content const { tableCellsValues } = table.getMetaData('pipelinesTable'); diff --git a/x-pack/plugins/ingest_pipelines/common/types.ts b/x-pack/plugins/ingest_pipelines/common/types.ts index 303db8423d401..2053d3e22dc74 100644 --- a/x-pack/plugins/ingest_pipelines/common/types.ts +++ b/x-pack/plugins/ingest_pipelines/common/types.ts @@ -33,3 +33,8 @@ export interface PipelinesByName { on_failure?: Processor[]; }; } + +export enum FieldCopyAction { + Copy = 'copy', + Rename = 'rename', +} diff --git a/x-pack/plugins/ingest_pipelines/kibana.json b/x-pack/plugins/ingest_pipelines/kibana.json index 889559826f1f1..958aa729ccf0d 100644 --- a/x-pack/plugins/ingest_pipelines/kibana.json +++ b/x-pack/plugins/ingest_pipelines/kibana.json @@ -7,7 +7,7 @@ "name": "Stack Management", "githubTeam": "kibana-stack-management" }, - "requiredPlugins": ["management", "features", "share"], + "requiredPlugins": ["management", "features", "share", "fileUpload"], "optionalPlugins": ["security", "usageCollection"], "configPath": ["xpack", "ingest_pipelines"], "requiredBundles": ["esUiShared", "kibanaReact"] diff --git a/x-pack/plugins/ingest_pipelines/public/application/app.tsx b/x-pack/plugins/ingest_pipelines/public/application/app.tsx index da8f74e1efae5..971a52d0b25b7 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/app.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/app.tsx @@ -21,7 +21,13 @@ import { SectionLoading, } from '../shared_imports'; -import { PipelinesList, PipelinesCreate, PipelinesEdit, PipelinesClone } from './sections'; +import { + PipelinesList, + PipelinesCreate, + PipelinesEdit, + PipelinesClone, + PipelinesCreateFromCsv, +} from './sections'; import { ROUTES } from './services/navigation'; export const AppWithoutRouter = () => ( @@ -30,6 +36,7 @@ export const AppWithoutRouter = () => ( + {/* Catch all */} diff --git a/x-pack/plugins/ingest_pipelines/public/application/index.tsx b/x-pack/plugins/ingest_pipelines/public/application/index.tsx index 18da9bbdc5d06..fb185537b1aca 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/index.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/index.tsx @@ -11,6 +11,7 @@ import { render, unmountComponentAtNode } from 'react-dom'; import { NotificationsSetup, IUiSettingsClient } from 'kibana/public'; import { ManagementAppMountParams } from 'src/plugins/management/public'; import { SharePluginStart } from 'src/plugins/share/public'; +import type { FileUploadPluginStart } from '../../../file_upload/public'; import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; import { API_BASE_PATH } from '../../common/constants'; @@ -18,17 +19,25 @@ import { API_BASE_PATH } from '../../common/constants'; import { AuthorizationProvider } from '../shared_imports'; import { App } from './app'; -import { DocumentationService, UiMetricService, ApiService, BreadcrumbService } from './services'; +import { + DocumentationService, + UiMetricService, + ApiService, + BreadcrumbService, + FileReaderService, +} from './services'; export interface AppServices { breadcrumbs: BreadcrumbService; metric: UiMetricService; documentation: DocumentationService; api: ApiService; + fileReader: FileReaderService; notifications: NotificationsSetup; history: ManagementAppMountParams['history']; uiSettings: IUiSettingsClient; urlGenerators: SharePluginStart['urlGenerators']; + fileUpload: FileUploadPluginStart; } export interface CoreServices { diff --git a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts index b662f0a99de91..025a661477a24 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/mount_management_section.ts @@ -9,7 +9,13 @@ import { CoreSetup } from 'src/core/public'; import { ManagementAppMountParams } from 'src/plugins/management/public'; import { StartDependencies } from '../types'; -import { documentationService, uiMetricService, apiService, breadcrumbService } from './services'; +import { + documentationService, + uiMetricService, + apiService, + breadcrumbService, + fileReaderService, +} from './services'; import { renderApp } from '.'; export async function mountManagementSection( @@ -31,10 +37,12 @@ export async function mountManagementSection( metric: uiMetricService, documentation: documentationService, api: apiService, + fileReader: fileReaderService, notifications, history, uiSettings: coreStart.uiSettings, urlGenerators: depsStart.share.urlGenerators, + fileUpload: depsStart.fileUpload, }; return renderApp(element, I18nContext, services, { http }); diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/index.ts b/x-pack/plugins/ingest_pipelines/public/application/sections/index.ts index 215fbd032932c..bd3ab41936b29 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/index.ts @@ -12,3 +12,5 @@ export { PipelinesCreate } from './pipelines_create'; export { PipelinesEdit } from './pipelines_edit'; export { PipelinesClone } from './pipelines_clone'; + +export { PipelinesCreateFromCsv } from './pipelines_create_from_csv'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/index.ts b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/index.ts index b44fdfa77c06a..1962ce4f30738 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/index.ts @@ -6,3 +6,4 @@ */ export { PipelinesCreate } from './pipelines_create'; +export { PipelinesCreateFromCsv } from '../pipelines_create_from_csv'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/pipelines_create.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/pipelines_create.tsx index 5aa9205e1e1e5..a8068a6521406 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/pipelines_create.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create/pipelines_create.tsx @@ -5,7 +5,7 @@ * 2.0. */ -import React, { useState, useEffect } from 'react'; +import React, { useState, useEffect, useMemo } from 'react'; import { RouteComponentProps } from 'react-router-dom'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiPageHeader, EuiButtonEmpty, EuiSpacer } from '@elastic/eui'; @@ -55,6 +55,16 @@ export const PipelinesCreate: React.FunctionComponent { + if (sourcePipeline) { + return sourcePipeline; + } + + if (history.location.state?.sourcePipeline) { + return history.location.state.sourcePipeline as Pipeline; + } + }, [sourcePipeline, history]); + return ( <> = ({ error }) => { + return ( + +

+ +

+ + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/index.ts b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/index.ts new file mode 100644 index 0000000000000..8549d39e85c07 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { PipelinesCreateFromCsv } from './main'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/instructions.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/instructions.tsx new file mode 100644 index 0000000000000..42c6c3d348e3f --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/instructions.tsx @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { FC } from 'react'; +import { EuiCode, EuiFlexGroup, EuiFlexItem, EuiLink, EuiSpacer, EuiText } from '@elastic/eui'; + +export const Instructions: FC = () => { + return ( +
+ + + +

+ + sample mappings + + ), + source: source_field, + destination: destination_field, + }} + /> +

+ + + +

+ +

+ + + + +
+ ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/main.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/main.tsx new file mode 100644 index 0000000000000..0c2f3316661a5 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/main.tsx @@ -0,0 +1,218 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { RouteComponentProps } from 'react-router-dom'; +import { i18n } from '@kbn/i18n'; +import { EuiButtonEmpty, EuiPageHeader, EuiSpacer } from '@elastic/eui'; +import fileSaver from 'file-saver'; + +import { FieldCopyAction, Processor } from '../../../../common/types'; +import { useKibana } from '../../../shared_imports'; +import { PipelinesCsvUploader } from './pipelines_csv_uploader'; +import { PipelinesPreview } from './pipelines_preview'; +import { Error } from './error_display'; +import { Instructions } from './instructions'; + +export const PipelinesCreateFromCsv: React.FunctionComponent = ({ + history, +}) => { + const [isLoading, setIsLoading] = useState(false); + const [isUploaded, setIsUploaded] = useState(false); + const [pipelineProcessors, setPipelineProcessors] = useState([]); + const [errorInfo, setErrorInfo] = useState<{ title: string; message: string } | null>(null); + const [file, setFile] = useState(null); + + const hasError = errorInfo !== null; + + const { services } = useKibana(); + + useEffect(() => { + services.breadcrumbs.setBreadcrumbs('create'); + }, [services]); + + const onFilePickerChange = (files: FileList) => { + setErrorInfo(null); + setFile(files); + }; + + const onFileUpload = async (action: FieldCopyAction) => { + if (file != null && file.length > 0) { + await processFile(file[0], action); + } + }; + + const onUpdateProcessors = (updatedProcessors: Processor[]) => { + setPipelineProcessors(updatedProcessors); + }; + + const onDownload = () => { + const jsonBlob = new Blob([JSON.stringify(pipelineProcessors)], { type: 'application/json' }); + fileSaver.saveAs(jsonBlob, `my-mappings.json`); + }; + + const onClickToCreatePipeline = () => { + history.push({ pathname: '/create', state: { sourcePipeline: pipelineProcessors } }); + }; + + const processFile = async (csv: File, action: FieldCopyAction) => { + const maxBytes = services.fileUpload.getMaxBytes(); + + if (csv.size === 0) { + setErrorInfo({ + title: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.processFile.emptyFileErrorTitle', + { + defaultMessage: 'File is empty', + } + ), + message: i18n.translate('xpack.ingestPipelines.createFromCsv.processFile.emptyFileError', { + defaultMessage: 'The file provided is empty.', + }), + }); + } else if (csv.size > maxBytes) { + setErrorInfo({ + title: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.processFile.fileTooLargeErrorTitle', + { + defaultMessage: 'File too large', + } + ), + message: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.processFile.fileTooLargeError', + { + defaultMessage: 'File is greater than allowed size of {maxBytes} bytes.', + values: { maxBytes }, + } + ), + }); + } else { + try { + setIsLoading(true); + setIsUploaded(false); + + const fileContents = await services.fileReader.readFile(csv, maxBytes); + const success = await fetchPipelineFromMapping(fileContents, action); + + setIsLoading(false); + if (success) { + setIsUploaded(true); + } + } catch (e) { + setIsLoading(false); + setErrorInfo({ + title: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.processFile.unexpectedErrorTitle', + { + defaultMessage: 'Error reading file', + } + ), + message: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.processFile.unexpectedError', + { + defaultMessage: 'The file provided could not be read.', + } + ), + }); + } + } + }; + + const fetchPipelineFromMapping = async (fileContents: string, action: FieldCopyAction) => { + const { error, data: processors } = await services.api.parseCsv({ + file: fileContents, + copyAction: action, + }); + setPipelineProcessors(processors); + + if (!!error) { + try { + const errorParts = error.message.split(':'); + setErrorInfo({ title: errorParts[0], message: errorParts[1] }); + } catch (e) { + setErrorInfo({ + title: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.fetchPipeline.unexpectedErrorTitle', + { + defaultMessage: 'Something went wrong', + } + ), + message: i18n.translate( + 'xpack.ingestPipelines.createFromCsv.fetchPipeline.unexpectedErrorDetails', + { + defaultMessage: 'Unexpected error', + } + ), + }); + } + } + + return error === null; + }; + + return ( + <> + + + + } + rightSideItems={[ + + + , + ]} + /> + + + + + + {hasError && } + + + + {(!isUploaded || hasError) && ( + 0} + /> + )} + + {isUploaded && ( + + )} + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_csv_uploader.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_csv_uploader.tsx new file mode 100644 index 0000000000000..75d249ff8c423 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_csv_uploader.tsx @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { FC, useState } from 'react'; +import { + EuiSpacer, + EuiFilePicker, + EuiButton, + EuiFormRow, + EuiIconTip, + EuiRadioGroup, +} from '@elastic/eui'; + +import { useKibana } from '../../../shared_imports'; +import { FieldCopyAction } from '../../../../common/types'; + +interface Props { + actionOptions: FieldCopyAction[]; + onFilePickerChange(files: FileList | null): void; + onFileUpload(action: string | null): void; + isLoading: boolean; + isUploaded: boolean; + hasError: boolean; + hasFile: boolean; +} + +function getOptions(actions: FieldCopyAction[]) { + return actions.map((action) => ({ + id: action, + label: action === FieldCopyAction.Copy ? 'Copy field name' : 'Rename field', + })); +} + +export const PipelinesCsvUploader: FC = ({ + actionOptions, + onFilePickerChange, + onFileUpload, + isLoading, + isUploaded, + hasError, + hasFile, +}) => { + const [action, setAction] = useState(FieldCopyAction.Copy); + const { services } = useKibana(); + + const maxFileSize = services.fileUpload.getMaxBytesFormatted(); + + const options = getOptions(actionOptions); + + return ( + <> + + } + > + + + + + + + Default action + + } + /> +

+ } + > + setAction(id as FieldCopyAction)} + /> + + + + +
+ onFileUpload(action)} + isLoading={isLoading} + isDisabled={!hasFile || isUploaded || hasError} + data-test-subj="processFileButton" + fill + > + + +
+ + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_preview.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_preview.tsx new file mode 100644 index 0000000000000..dda616bdec4d0 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_create_from_csv/pipelines_preview.tsx @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FC, Fragment, useState, useEffect } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiSpacer, + EuiFlexGroup, + EuiFlexItem, + EuiFormRow, + EuiCopy, + EuiCallOut, + EuiText, + EuiButton, + EuiButtonEmpty, +} from '@elastic/eui'; +import { XJsonLang } from '@kbn/monaco'; +import { i18n } from '@kbn/i18n'; + +import { CodeEditorField } from '../../../../../../../src/plugins/kibana_react/public'; + +interface Props { + processors: object[]; + onDownload(): void; + onClickToCreatePipeline(): void; + onUpdateProcessors(processors: object[]): void; + hasError: boolean; +} + +export const PipelinesPreview: FC = ({ + processors, + onDownload, + onClickToCreatePipeline, + onUpdateProcessors, + hasError, +}) => { + const [isValidJson, setIsValidJson] = useState(true); + const [processorsJson, setProcessorsJson] = useState(''); + + useEffect(() => { + const jsonString = JSON.stringify(processors, null, 2); + setProcessorsJson(jsonString); + }, [processors]); + + const onUpdate = (updated: string) => { + setProcessorsJson(updated); + + try { + setIsValidJson(true); + const parsedJson = JSON.parse(updated); + onUpdateProcessors(parsedJson); + } catch (e) { + setIsValidJson(false); + } + }; + + return ( + + + {!hasError && ( + + +

+ +

+ + + )} + + + + + + + + + + + + + + + + + + + + + {(copy: () => void) => ( + + + + )} + + + + + + + + + + + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/empty_list.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/empty_list.tsx index 9f401bca5431f..65950870e7d4a 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/empty_list.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/empty_list.tsx @@ -5,19 +5,47 @@ * 2.0. */ -import React, { FunctionComponent } from 'react'; +import React, { FunctionComponent, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import { EuiEmptyPrompt, EuiLink, EuiPageContent, EuiButton } from '@elastic/eui'; +import { + EuiEmptyPrompt, + EuiLink, + EuiPageContent, + EuiButton, + EuiPopover, + EuiContextMenu, +} from '@elastic/eui'; import { useHistory } from 'react-router-dom'; import { ScopedHistory } from 'kibana/public'; import { reactRouterNavigate } from '../../../../../../../src/plugins/kibana_react/public'; import { useKibana } from '../../../shared_imports'; -import { getCreatePath } from '../../services/navigation'; +import { getCreateFromCsvPath, getCreatePath } from '../../services/navigation'; export const EmptyList: FunctionComponent = () => { const { services } = useKibana(); const history = useHistory() as ScopedHistory; + const [showPopover, setShowPopover] = useState(false); + + const createMenuItems = [ + { + name: i18n.translate('xpack.ingestPipelines.list.table.emptyPrompt.createButtonLabel', { + defaultMessage: 'New pipeline', + }), + ...reactRouterNavigate(history, getCreatePath()), + 'data-test-subj': `emptyStateCreatePipelineButton`, + }, + { + name: i18n.translate( + 'xpack.ingestPipelines.list.table.emptyPrompt.createButtonLabel.createPipelineFromCsvButtonLabel', + { + defaultMessage: 'New pipeline from CSV', + } + ), + ...reactRouterNavigate(history, getCreateFromCsvPath()), + 'data-test-subj': `emptyStatecreatePipelineFromCsvButton`, + }, + ]; return ( @@ -35,7 +63,7 @@ export const EmptyList: FunctionComponent = () => {

{' '} {i18n.translate('xpack.ingestPipelines.list.table.emptyPromptDocumentionLink', { @@ -45,16 +73,40 @@ export const EmptyList: FunctionComponent = () => {

} actions={ - setShowPopover(false)} + button={ + setShowPopover((previousBool) => !previousBool)} + > + {i18n.translate( + 'xpack.ingestPipelines.list.table.emptyCreatePipelineDropdownLabel', + { + defaultMessage: 'Create pipeline', + } + )} + + } + panelPaddingSize="none" + repositionOnScroll > - {i18n.translate('xpack.ingestPipelines.list.table.emptyPrompt.createButtonLabel', { - defaultMessage: 'Create a pipeline', - })} - + + } /> diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx index 95621601011f9..835ebc23666e2 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/main.tsx @@ -160,7 +160,7 @@ export const PipelinesList: React.FunctionComponent = ({ description={ } rightSideItems={[ diff --git a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx index 9cf807052fcb9..1001bf96a53d9 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/sections/pipelines_list/table.tsx @@ -14,6 +14,8 @@ import { EuiButton, EuiInMemoryTableProps, EuiTableFieldDataColumnType, + EuiPopover, + EuiContextMenu, } from '@elastic/eui'; import { reactRouterNavigate } from '../../../../../../../src/plugins/kibana_react/public'; @@ -37,6 +39,30 @@ export const PipelineTable: FunctionComponent = ({ }) => { const { history } = useKibana().services; const [selection, setSelection] = useState([]); + const [showPopover, setShowPopover] = useState(false); + + const createMenuItems = [ + /** + * Create pipeline + */ + { + name: i18n.translate('xpack.ingestPipelines.list.table.createPipelineButtonLabel', { + defaultMessage: 'New pipeline', + }), + ...reactRouterNavigate(history, '/create'), + 'data-test-subj': `createNewPipeline`, + }, + /** + * Create pipeline from CSV + */ + { + name: i18n.translate('xpack.ingestPipelines.list.table.createPipelineFromCsvButtonLabel', { + defaultMessage: 'New pipeline from CSV', + }), + ...reactRouterNavigate(history, '/csv_create'), + 'data-test-subj': `createPipelineFromCsv`, + }, + ]; const tableProps: EuiInMemoryTableProps = { itemId: 'name', @@ -83,17 +109,37 @@ export const PipelineTable: FunctionComponent = ({ defaultMessage: 'Reload', })} , - setShowPopover(false)} + button={ + setShowPopover((previousBool) => !previousBool)} + > + {i18n.translate('xpack.ingestPipelines.list.table.createPipelineDropdownLabel', { + defaultMessage: 'Create pipeline', + })} + + } + panelPaddingSize="none" + repositionOnScroll > - {i18n.translate('xpack.ingestPipelines.list.table.createPipelineButtonLabel', { - defaultMessage: 'Create pipeline', - })} - , + + , ], box: { incremental: true, diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts index 6db30dd283048..e1a9bdbd4e026 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/services/api.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/services/api.ts @@ -7,7 +7,7 @@ import { HttpSetup } from 'src/core/public'; -import { Pipeline } from '../../../common/types'; +import { FieldCopyAction, Pipeline } from '../../../common/types'; import { API_BASE_PATH } from '../../../common/constants'; import { UseRequestConfig, @@ -131,6 +131,15 @@ export class ApiService { return result; } + + public async parseCsv(reqBody: { file: string; copyAction: FieldCopyAction }) { + const result = await this.sendRequest({ + path: `${API_BASE_PATH}/parse_csv`, + method: 'post', + body: JSON.stringify(reqBody), + }); + return result; + } } export const apiService = new ApiService(); diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/file_reader.ts b/x-pack/plugins/ingest_pipelines/public/application/services/file_reader.ts new file mode 100644 index 0000000000000..cebb0658f31ec --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/services/file_reader.ts @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export class FileReaderService { + public readFile(file: File, maxFileSizeBytes: any): Promise { + return new Promise((resolve, reject) => { + if (file && file.size) { + const reader = new FileReader(); + reader.readAsArrayBuffer(file); + + reader.onload = (() => { + return () => { + const decoder = new TextDecoder(); + const data = reader.result; + if (data === null || typeof data === 'string') { + return reject(); + } + const fileContents = decoder.decode(data.slice(0, maxFileSizeBytes)); + + if (fileContents === '') { + reject(); + } else { + resolve(fileContents); + } + }; + })(); + } else { + reject(); + } + }); + } +} + +export const fileReaderService = new FileReaderService(); diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/index.ts b/x-pack/plugins/ingest_pipelines/public/application/services/index.ts index 3672af5d181fd..1f1bc29677375 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/services/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/services/index.ts @@ -12,3 +12,5 @@ export { uiMetricService, UiMetricService } from './ui_metric'; export { apiService, ApiService } from './api'; export { breadcrumbService, BreadcrumbService } from './breadcrumbs'; + +export { fileReaderService, FileReaderService } from './file_reader'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/services/navigation.ts b/x-pack/plugins/ingest_pipelines/public/application/services/navigation.ts index a0f2667a43c71..7d3e11fea3d89 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/services/navigation.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/services/navigation.ts @@ -11,6 +11,8 @@ const EDIT_PATH = 'edit'; const CREATE_PATH = 'create'; +const CREATE_FROM_CSV_PATH = 'csv_create'; + const _getEditPath = (name: string, encode = true): string => { return `${BASE_PATH}${EDIT_PATH}/${encode ? encodeURIComponent(name) : name}`; }; @@ -22,15 +24,21 @@ const _getCreatePath = (): string => { const _getClonePath = (name: string, encode = true): string => { return `${BASE_PATH}${CREATE_PATH}/${encode ? encodeURIComponent(name) : name}`; }; + const _getListPath = (name?: string): string => { return `${BASE_PATH}${name ? `?pipeline=${encodeURIComponent(name)}` : ''}`; }; +const _getCreateFromCsvPath = (): string => { + return `${BASE_PATH}${CREATE_FROM_CSV_PATH}`; +}; + export const ROUTES = { list: _getListPath(), edit: _getEditPath(':name', false), create: _getCreatePath(), clone: _getClonePath(':sourceName', false), + createFromCsv: _getCreateFromCsvPath(), }; export const getListPath = ({ @@ -43,3 +51,4 @@ export const getEditPath = ({ pipelineName }: { pipelineName: string }): string export const getCreatePath = (): string => _getCreatePath(); export const getClonePath = ({ clonedPipelineName }: { clonedPipelineName: string }): string => _getClonePath(clonedPipelineName, true); +export const getCreateFromCsvPath = (): string => _getCreateFromCsvPath(); diff --git a/x-pack/plugins/ingest_pipelines/public/types.ts b/x-pack/plugins/ingest_pipelines/public/types.ts index 784eccd462af4..e135f8fed58a7 100644 --- a/x-pack/plugins/ingest_pipelines/public/types.ts +++ b/x-pack/plugins/ingest_pipelines/public/types.ts @@ -8,6 +8,7 @@ import { ManagementSetup } from 'src/plugins/management/public'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/public'; import { SharePluginStart, SharePluginSetup } from 'src/plugins/share/public'; +import type { FileUploadPluginStart } from '../../file_upload/public'; export interface SetupDependencies { management: ManagementSetup; @@ -17,4 +18,5 @@ export interface SetupDependencies { export interface StartDependencies { share: SharePluginStart; + fileUpload: FileUploadPluginStart; } diff --git a/x-pack/plugins/ingest_pipelines/server/lib/index.ts b/x-pack/plugins/ingest_pipelines/server/lib/index.ts new file mode 100644 index 0000000000000..7aa564f5e9645 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/lib/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { csvToIngestPipeline } from './mapper'; diff --git a/x-pack/plugins/ingest_pipelines/server/lib/mapper.test.ts b/x-pack/plugins/ingest_pipelines/server/lib/mapper.test.ts new file mode 100644 index 0000000000000..0c2f52d3c3bdf --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/lib/mapper.test.ts @@ -0,0 +1,251 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { csvToIngestPipeline } from './mapper'; +import { FieldCopyAction } from '../../common/types'; + +describe('mapper', () => { + describe('csvToIngestPipeline()', () => { + it('empty file returns empty mapping', () => { + expect(() => { + csvToIngestPipeline('', FieldCopyAction.Copy); + }).toThrow(new Error('Error reading file: The file provided is empty.')); + }); + + it('file parsing error for invalid csv', () => { + const invalidCsv = `name,number + one|1 + two.2 + fourty two,42`; + + expect(() => { + csvToIngestPipeline(invalidCsv, FieldCopyAction.Copy); + }).toThrow( + new Error( + 'Error reading file: An unexpected issue has occured during the processing of the file.' + ) + ); + }); + + it('missing the required headers errors', () => { + const noHeadersCsv = 'srcip,,,,source.address,Copying srcip to source.address'; + + expect(() => { + csvToIngestPipeline(noHeadersCsv, FieldCopyAction.Copy); + }).toThrow( + new Error( + 'Missing required headers: Include [source_field, destination_field] header(s) in the CSV file.' + ) + ); + }); + + it('unacceptable format action errors', () => { + const badFormatCsv = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nsrcport,,invalid_action,,source.port,\n'; + + expect(() => { + csvToIngestPipeline(badFormatCsv, FieldCopyAction.Copy); + }).toThrow( + new Error( + 'Invalid format action [invalid_action]. The valid actions are uppercase, lowercase, to_boolean, to_integer, to_float, to_array, to_string, parse_timestamp' + ) + ); + }); + + describe('successful mapping', () => { + it('duplicate row handled', () => { + const duplciateRow = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nsrcip,,,,source.address,Copying srcip to source.address\nsrcip,,,,source.address,Copying srcip to source.address\n'; + const expectedJson = { + processors: [ + { + set: { + field: 'source.address', + if: 'ctx.srcip != null', + value: '{{srcip}}', + }, + }, + ], + }; + expect(csvToIngestPipeline(duplciateRow, FieldCopyAction.Copy)).toEqual(expectedJson); + }); + + describe('timestamp formatting', () => { + it('default handling', () => { + const defaultTimestamp = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nsome_timestamp,,,,@timestamp,\n'; + + const expectedJson = { + processors: [ + { + date: { + field: 'some_timestamp', + formats: ['UNIX_MS'], + timezone: 'UTC', + target_field: '@timestamp', + ignore_failure: true, + }, + }, + ], + }; + expect(csvToIngestPipeline(defaultTimestamp, FieldCopyAction.Copy)).toEqual(expectedJson); + }); + + it('specified handling', () => { + const timestampSpecifics = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nsome_timestamp,,parse_timestamp,UNIX,destination_timestamp,\n'; + + const expectedJson = { + processors: [ + { + date: { + field: 'some_timestamp', + formats: ['UNIX'], + timezone: 'UTC', + target_field: 'destination_timestamp', + ignore_failure: true, + }, + }, + ], + }; + expect(csvToIngestPipeline(timestampSpecifics, FieldCopyAction.Copy)).toEqual( + expectedJson + ); + }); + }); + + describe('field copy action', () => { + it('copy', () => { + const copyFile = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nts,copy,,,timestamp,\n'; + + const expectedJson = { + processors: [ + { + set: { + field: 'timestamp', + value: '{{ts}}', + if: 'ctx.ts != null', + }, + }, + ], + }; + expect(csvToIngestPipeline(copyFile, FieldCopyAction.Rename)).toEqual(expectedJson); + }); + it('rename', () => { + const renameFile = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nhostip,,to_array,,host.ip,\n'; + + const expectedJson = { + processors: [ + { + rename: { + field: 'hostip', + target_field: 'host.ip', + ignore_missing: true, + }, + }, + { + append: { + field: 'host.ip', + value: [], + ignore_failure: true, + if: 'ctx.host?.ip != null', + }, + }, + ], + }; + expect(csvToIngestPipeline(renameFile, FieldCopyAction.Rename)).toEqual(expectedJson); + }); + }); + + it('successful mapping example file', () => { + const validCsv = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\n' + + 'srcip,,,,source.address,Copying srcip to source.address\n' + + 'new_event.srcip,,,,source.ip,\n' + + 'some_timestamp_field,,parse_timestamp,,@timestamp,\n' + + 'srcport,rename,to_integer,,source.port,\n' + + 'log_level,rename,uppercase,,log.level,\n' + + 'hostip,,to_array,,host.ip,\n'; + + const expectedJson = { + processors: [ + { + set: { + field: 'source.address', + if: 'ctx.srcip != null', + value: '{{srcip}}', + }, + }, + { + set: { + field: 'source.ip', + value: '{{new_event.srcip}}', + if: 'ctx.new_event?.srcip != null', + }, + }, + { + date: { + field: 'some_timestamp_field', + target_field: '@timestamp', + formats: ['UNIX_MS'], + timezone: 'UTC', + ignore_failure: true, + }, + }, + { + rename: { + field: 'srcport', + target_field: 'source.port', + ignore_missing: true, + }, + }, + { + convert: { + field: 'source.port', + type: 'long', + ignore_missing: true, + ignore_failure: true, + }, + }, + { + rename: { + field: 'log_level', + target_field: 'log.level', + ignore_missing: true, + }, + }, + { + uppercase: { + field: 'log.level', + ignore_missing: true, + ignore_failure: true, + }, + }, + { + set: { + field: 'host.ip', + value: '{{hostip}}', + if: 'ctx.hostip != null', + }, + }, + { + append: { + field: 'host.ip', + value: [], + ignore_failure: true, + if: 'ctx.host?.ip != null', + }, + }, + ], + }; + expect(csvToIngestPipeline(validCsv, FieldCopyAction.Copy)).toEqual(expectedJson); + }); + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/server/lib/mapper.ts b/x-pack/plugins/ingest_pipelines/server/lib/mapper.ts new file mode 100644 index 0000000000000..86a2a886156d8 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/lib/mapper.ts @@ -0,0 +1,253 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { i18n } from '@kbn/i18n'; +import Papa from 'papaparse'; + +import { FieldCopyAction, Pipeline, Processor } from '../../common/types'; + +const REQUIRED_CSV_HEADERS = ['source_field', 'destination_field']; + +const FORMAT_ACTIONS = [ + 'uppercase', + 'lowercase', + 'to_boolean', + 'to_integer', + 'to_float', + 'to_array', + 'to_string', + 'parse_timestamp', +] as const; + +type FormatAction = typeof FORMAT_ACTIONS[number]; +type TimeStampFormat = 'UNIX' | 'UNIX_MS' | 'ISO8601' | 'TAI64N'; + +interface Mapping { + source_field: string; + destination_field?: string; + copy_action?: FieldCopyAction; + format_action?: FormatAction; + timestamp_format?: TimeStampFormat; +} + +interface Row extends Mapping { + notes?: string; + [key: string]: unknown; // allow unknown columns +} + +export function csvToIngestPipeline(file: string, copyAction: FieldCopyAction) { + if (file.trim().length === 0) { + throw new Error( + i18n.translate('xpack.ingestPipelines.csvToIngestPipeline.error.emptyFileErrors', { + defaultMessage: 'Error reading file: The file provided is empty.', + }) + ); + } + + const fileData = parseAndValidate(file); + const mapping = convertCsvToMapping(fileData, copyAction); + return generatePipeline(mapping); +} + +function parseAndValidate(file: string) { + const config: Papa.ParseConfig = { + header: true, + skipEmptyLines: true, + }; + + const { data, errors, meta } = Papa.parse(file, config); + if (errors.length > 0) { + throw new Error( + i18n.translate('xpack.ingestPipelines.mapToIngestPipeline.error.parseErrors', { + defaultMessage: + 'Error reading file: An unexpected issue has occured during the processing of the file.', + }) + ); + } + + const missingHeaders = REQUIRED_CSV_HEADERS.reduce((acc, header) => { + if (meta.fields.includes(header)) { + return acc; + } + return [...acc, header]; + }, []); + + if (missingHeaders.length > 0) { + throw new Error( + i18n.translate('xpack.ingestPipelines.mapToIngestPipeline.error.missingHeaders', { + defaultMessage: 'Missing required headers: Include [{missing}] header(s) in the CSV file.', + values: { missing: missingHeaders.join(', ') }, + }) + ); + } + + return data; +} + +function convertCsvToMapping(rows: Row[], copyFieldAction: FieldCopyAction) { + const mapping = new Map(); + + if (rows.length < 1) { + return mapping; + } + + for (const row of rows) { + if (!row.source_field || !row.source_field.trim()) { + // Skip rows that don't have a source field + continue; + } + if ( + (!row.destination_field || !row.destination_field.trim()) && + (!row.format_action || !row.format_action.trim()) + ) { + // Skip if no destination field and no format field provided since it's possible to reformat a source field by itself + continue; + } + + const source = row.source_field.trim(); + let destination = row.destination_field && row.destination_field.trim(); + const copyAction = (row.copy_action && row.copy_action.trim()) || copyFieldAction; + let formatAction = row.format_action && (row.format_action.trim() as FormatAction); + let timestampFormat = row.timestamp_format && (row.timestamp_format.trim() as TimeStampFormat); + + if (destination === '@timestamp' && !Boolean(timestampFormat)) { + // If @timestamp is the destination and the user does not specify how to format the conversion, convert it to UNIX_MS + formatAction = 'parse_timestamp'; + timestampFormat = 'UNIX_MS'; + } else if (!destination && formatAction) { + // If the destination field is empty but a format action is provided, then assume we're formating the source field. + destination = source; + } + + if (formatAction && !FORMAT_ACTIONS.includes(formatAction)) { + const formatActions = FORMAT_ACTIONS.join(', '); + throw new Error( + i18n.translate('xpack.ingestPipelines.mapToIngestPipeline.error.invalidFormatAction', { + defaultMessage: + 'Invalid format action [{ formatAction }]. The valid actions are {formatActions}', + values: { formatAction, formatActions }, + }) + ); + } + + mapping.set(`${source}+${destination}`, { + source_field: source, + destination_field: destination, + copy_action: copyAction as FieldCopyAction, + format_action: formatAction as FormatAction, + timestamp_format: timestampFormat as TimeStampFormat, + }); + } + + return mapping; +} + +function generatePipeline(mapping: Map) { + const processors: Processor[] = []; + for (const [, row] of mapping) { + if (hasSameName(row) && !row.format_action) continue; + + const source = row.source_field; + const dest = row.destination_field; + + // Copy/Rename + if (dest && `parse_timestamp` !== row.format_action) { + let processor = {}; + if ('copy' === row.copy_action) { + processor = { + set: { + field: dest, + value: `{{${source}}}`, + if: fieldPresencePredicate(source), + }, + }; + } else { + processor = { + rename: { + field: source, + target_field: dest, + ignore_missing: true, + }, + }; + } + processors.push(processor); + } + + if (row.format_action) { + // Modify the source_field if there's no destination_field (no rename, just a type change) + const affectedField = dest || source; + + let type = ''; + if ('to_boolean' === row.format_action) type = 'boolean'; + else if ('to_integer' === row.format_action) type = 'long'; + else if ('to_string' === row.format_action) type = 'string'; + else if ('to_float' === row.format_action) type = 'float'; + + let processor: Processor | undefined; + + if (type) { + processor = { + convert: { + field: affectedField, + type, + ignore_missing: true, + ignore_failure: true, + }, + }; + } else if ('uppercase' === row.format_action || 'lowercase' === row.format_action) { + processor = { + [row.format_action]: { + field: affectedField, + ignore_missing: true, + ignore_failure: true, + }, + }; + } else if ('to_array' === row.format_action) { + processor = { + append: { + field: affectedField, + value: [], + ignore_failure: true, + if: fieldPresencePredicate(affectedField), + }, + }; + } else if ('parse_timestamp' === row.format_action) { + processor = { + date: { + field: source, + target_field: dest, + formats: [row.timestamp_format], + timezone: 'UTC', + ignore_failure: true, + }, + }; + } + + if (processor) { + processors.push(processor!); + } + } + } + return { processors } as Pipeline; +} + +function fieldPresencePredicate(field: string) { + if ('@timestamp' === field) { + return "ctx.containsKey('@timestamp')"; + } + + const fieldPath = field.split('.'); + if (fieldPath.length === 1) { + return `ctx.${field} != null`; + } + + return `ctx.${fieldPath.join('?.')} != null`; +} + +function hasSameName(row: Mapping) { + return row.source_field === row.destination_field; +} diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts index 6601bc3b4e7f8..aec90d2c3a2eb 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/index.ts @@ -18,3 +18,5 @@ export { registerDeleteRoute } from './delete'; export { registerSimulateRoute } from './simulate'; export { registerDocumentsRoute } from './documents'; + +export { registerParseCsvRoute } from './parse_csv'; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/api/parse_csv.ts b/x-pack/plugins/ingest_pipelines/server/routes/api/parse_csv.ts new file mode 100644 index 0000000000000..43bd94a52c056 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/server/routes/api/parse_csv.ts @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { schema, TypeOf, Type } from '@kbn/config-schema'; + +import { API_BASE_PATH } from '../../../common/constants'; +import { FieldCopyAction } from '../../../common/types'; +import { csvToIngestPipeline } from '../../lib'; +import { RouteDependencies } from '../../types'; + +const bodySchema = schema.object({ + file: schema.string(), + copyAction: schema.string() as Type, +}); + +type ReqBody = TypeOf; + +export const registerParseCsvRoute = ({ router }: RouteDependencies): void => { + router.post( + { + path: `${API_BASE_PATH}/parse_csv`, + validate: { + body: bodySchema, + }, + }, + async (contxt, req, res) => { + const { file, copyAction } = req.body; + try { + const result = csvToIngestPipeline(file, copyAction); + return res.ok({ body: result }); + } catch (error) { + return res.badRequest({ body: error.message }); + } + } + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/server/routes/index.ts b/x-pack/plugins/ingest_pipelines/server/routes/index.ts index 9f6175736c414..d3d74b31c1013 100644 --- a/x-pack/plugins/ingest_pipelines/server/routes/index.ts +++ b/x-pack/plugins/ingest_pipelines/server/routes/index.ts @@ -15,6 +15,7 @@ import { registerDeleteRoute, registerSimulateRoute, registerDocumentsRoute, + registerParseCsvRoute, } from './api'; export class ApiRoutes { @@ -26,5 +27,6 @@ export class ApiRoutes { registerDeleteRoute(dependencies); registerSimulateRoute(dependencies); registerDocumentsRoute(dependencies); + registerParseCsvRoute(dependencies); } } diff --git a/x-pack/plugins/ingest_pipelines/tsconfig.json b/x-pack/plugins/ingest_pipelines/tsconfig.json index de9a8362e8c6b..0bb8031adcf77 100644 --- a/x-pack/plugins/ingest_pipelines/tsconfig.json +++ b/x-pack/plugins/ingest_pipelines/tsconfig.json @@ -18,6 +18,7 @@ { "path": "../licensing/tsconfig.json" }, { "path": "../features/tsconfig.json" }, { "path": "../security/tsconfig.json" }, + { "path": "../file_upload/tsconfig.json" }, { "path": "../../../src/plugins/es_ui_shared/tsconfig.json"}, { "path": "../../../src/plugins/kibana_react/tsconfig.json"}, { "path": "../../../src/plugins/management/tsconfig.json"}, diff --git a/x-pack/test/accessibility/apps/ingest_node_pipelines.ts b/x-pack/test/accessibility/apps/ingest_node_pipelines.ts index 9e92446fe4b5e..a09a07bb01267 100644 --- a/x-pack/test/accessibility/apps/ingest_node_pipelines.ts +++ b/x-pack/test/accessibility/apps/ingest_node_pipelines.ts @@ -49,6 +49,7 @@ export default function ({ getService, getPageObjects }: any) { }); it('Create Pipeline Wizard', async () => { + await testSubjects.click('emptyStateCreatePipelineDropdown'); await testSubjects.click('emptyStateCreatePipelineButton'); await retry.waitFor('Create pipeline page one to be visible', async () => { return testSubjects.isDisplayed('pageTitle') ? true : false; diff --git a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts index 80790a6df400f..9695527b3557f 100644 --- a/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts +++ b/x-pack/test/api_integration/apis/management/ingest_pipelines/ingest_pipelines.ts @@ -495,5 +495,30 @@ export default function ({ getService }: FtrProviderContext) { }); }); }); + + describe('Map CSV to pipeline', () => { + it('should map to a pipeline', async () => { + const validCsv = + 'source_field,copy_action,format_action,timestamp_format,destination_field,Notes\nsrcip,,,,source.address,Copying srcip to source.address'; + const { body } = await supertest + .post(`${API_BASE_PATH}/parse_csv`) + .set('kbn-xsrf', 'xxx') + .send({ + copyAction: 'copy', + file: validCsv, + }) + .expect(200); + + expect(body.processors).to.eql([ + { + set: { + field: 'source.address', + value: '{{srcip}}', + if: 'ctx.srcip != null', + }, + }, + ]); + }); + }); }); } diff --git a/x-pack/test/functional/apps/ingest_pipelines/exports/example_mapping.csv b/x-pack/test/functional/apps/ingest_pipelines/exports/example_mapping.csv new file mode 100644 index 0000000000000..ffbacdbe86b47 --- /dev/null +++ b/x-pack/test/functional/apps/ingest_pipelines/exports/example_mapping.csv @@ -0,0 +1,2 @@ +source_field,copy_action,format_action,timestamp_format,destination_field,Notes +srcip,,,,source.address,Copying srcip to source.address \ No newline at end of file diff --git a/x-pack/test/functional/apps/ingest_pipelines/ingest_pipelines.ts b/x-pack/test/functional/apps/ingest_pipelines/ingest_pipelines.ts index 026cea52e8102..ffc0a5636161a 100644 --- a/x-pack/test/functional/apps/ingest_pipelines/ingest_pipelines.ts +++ b/x-pack/test/functional/apps/ingest_pipelines/ingest_pipelines.ts @@ -6,6 +6,7 @@ */ import expect from '@kbn/expect'; +import path from 'path'; import { FtrProviderContext } from '../../ftr_provider_context'; const PIPELINE = { @@ -14,14 +15,17 @@ const PIPELINE = { version: 1, }; +const PIPELINE_CSV = { + name: 'test_pipeline', +}; + export default ({ getPageObjects, getService }: FtrProviderContext) => { - const pageObjects = getPageObjects(['common', 'ingestPipelines']); + const pageObjects = getPageObjects(['common', 'ingestPipelines', 'savedObjects']); const log = getService('log'); const es = getService('es'); const security = getService('security'); - // FAILING ES PROMOTION: https://github.com/elastic/kibana/issues/113439 - describe.skip('Ingest Pipelines', function () { + describe('Ingest Pipelines', function () { this.tags('smoke'); before(async () => { await security.testUser.setRoles(['ingest_pipelines_user']); @@ -31,25 +35,46 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { it('Loads the app', async () => { log.debug('Checking for section heading to say Ingest Pipelines.'); - const headingText = await pageObjects.ingestPipelines.sectionHeadingText(); - expect(headingText).to.be('Ingest Pipelines'); + const headingText = await pageObjects.ingestPipelines.emptyStateHeaderText(); + expect(headingText).to.be('Start by creating a pipeline'); }); - it('Creates a pipeline', async () => { - await pageObjects.ingestPipelines.createNewPipeline(PIPELINE); + describe('create pipeline', () => { + it('Creates a pipeline', async () => { + await pageObjects.ingestPipelines.createNewPipeline(PIPELINE); - const pipelinesList = await pageObjects.ingestPipelines.getPipelinesList(); - const newPipelineExists = Boolean( - pipelinesList.find((pipelineName) => pipelineName === PIPELINE.name) - ); + const pipelinesList = await pageObjects.ingestPipelines.getPipelinesList(); + const newPipelineExists = Boolean( + pipelinesList.find((pipelineName) => pipelineName === PIPELINE.name) + ); - expect(newPipelineExists).to.be(true); - }); + expect(newPipelineExists).to.be(true); + }); + + it('Creates a pipeline from CSV', async () => { + await pageObjects.ingestPipelines.navigateToCreateFromCsv(); + + await pageObjects.common.setFileInputPath( + path.join(__dirname, 'exports', 'example_mapping.csv') + ); + + await pageObjects.ingestPipelines.createPipelineFromCsv(PIPELINE_CSV); + + const pipelinesList = await pageObjects.ingestPipelines.getPipelinesList(); + const newPipelineExists = Boolean( + pipelinesList.find((pipelineName) => pipelineName === PIPELINE.name) + ); + + expect(newPipelineExists).to.be(true); + }); - after(async () => { - // Delete the pipeline that was created - await es.ingest.deletePipeline({ id: PIPELINE.name }); - await security.testUser.restoreDefaults(); + afterEach(async () => { + // Close details flyout + await pageObjects.ingestPipelines.closePipelineDetailsFlyout(); + // Delete the pipeline that was created + await es.ingest.deletePipeline({ id: PIPELINE.name }); + await security.testUser.restoreDefaults(); + }); }); }); }; diff --git a/x-pack/test/functional/page_objects/ingest_pipelines_page.ts b/x-pack/test/functional/page_objects/ingest_pipelines_page.ts index beda5d4410dd0..0de17f9d943ac 100644 --- a/x-pack/test/functional/page_objects/ingest_pipelines_page.ts +++ b/x-pack/test/functional/page_objects/ingest_pipelines_page.ts @@ -35,7 +35,9 @@ export function IngestPipelinesPageProvider({ getService, getPageObjects }: FtrP processors?: string; onFailureProcessors?: string; }) { - await testSubjects.click('createPipelineButton'); + await testSubjects.click('emptyStateCreatePipelineDropdown'); + await testSubjects.click('emptyStateCreatePipelineButton'); + await testSubjects.exists('pipelineForm'); await testSubjects.setValue('nameField > input', name); @@ -69,5 +71,34 @@ export function IngestPipelinesPageProvider({ getService, getPageObjects }: FtrP return await Promise.all(pipelines.map((pipeline) => getPipelineName(pipeline))); }, + + async navigateToCreateFromCsv() { + await testSubjects.click('emptyStateCreatePipelineDropdown'); + await testSubjects.click('emptyStatecreatePipelineFromCsvButton'); + + await testSubjects.exists('createFromCsvInstructions'); + }, + + async createPipelineFromCsv({ name }: { name: string }) { + await testSubjects.click('processFileButton'); + + await testSubjects.exists('pipelineMappingsJSONEditor'); + + await testSubjects.exists('copyToClipboard'); + await testSubjects.exists('downloadJson'); + + await testSubjects.click('continueToCreate'); + + await testSubjects.exists('pipelineForm'); + + await testSubjects.setValue('nameField > input', name); + await testSubjects.click('submitButton'); + + await pageObjects.header.waitUntilLoadingHasFinished(); + }, + + async closePipelineDetailsFlyout() { + await testSubjects.click('euiFlyoutCloseButton'); + }, }; } From 9900702f68c1cc156a4e1eef6eda0f67a5a1dd19 Mon Sep 17 00:00:00 2001 From: Angela Chuang <6295984+angorayc@users.noreply.github.com> Date: Mon, 15 Nov 2021 17:28:36 +0000 Subject: [PATCH 07/82] [Security Solution] Add Security Data View badges on Kibana settings page (#115176) * security data view badge in kibana settings * clean up * add badges on edit kibana index page * update gutter size * review * review * lint Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../edit_index_pattern/edit_index_pattern.tsx | 16 +++++++++++++++- .../index_pattern_table/index_pattern_table.tsx | 7 ++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/src/plugins/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx b/src/plugins/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx index e6c9b7bc5a061..75fb3a6114c6c 100644 --- a/src/plugins/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx +++ b/src/plugins/index_pattern_management/public/components/edit_index_pattern/edit_index_pattern.tsx @@ -54,6 +54,15 @@ const confirmModalOptionsDelete = { }), }; +const securityDataView = i18n.translate( + 'indexPatternManagement.editIndexPattern.badge.securityDataViewTitle', + { + defaultMessage: 'Security Data View', + } +); + +const securitySolution = 'security-solution'; + export const EditIndexPattern = withRouter( ({ indexPattern, history, location }: EditIndexPatternProps) => { const { application, uiSettings, overlays, chrome, data } = @@ -145,12 +154,17 @@ export const EditIndexPattern = withRouter( defaultIndex={defaultIndex} > {showTagsSection && ( - + {Boolean(indexPattern.timeFieldName) && ( {timeFilterHeader} )} + {indexPattern.id && indexPattern.id.indexOf(securitySolution) === 0 && ( + + {securityDataView} + + )} {tags.map((tag: any) => ( {tag.name} diff --git a/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx b/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx index 89230ae03a923..e8ce4b468f22f 100644 --- a/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx +++ b/src/plugins/index_pattern_management/public/components/index_pattern_table/index_pattern_table.tsx @@ -57,6 +57,8 @@ const securityDataView = i18n.translate( } ); +const securitySolution = 'security-solution'; + interface Props extends RouteComponentProps { canSave: boolean; showCreateDialog?: boolean; @@ -123,10 +125,9 @@ export const IndexPatternTable = ({   - {index.id && index.id === 'security-solution' && ( - {securityDataView} + {index.id && index.id.indexOf(securitySolution) === 0 && ( + {securityDataView} )} - {index.tags && index.tags.map(({ key: tagKey, name: tagName }) => ( {tagName} From e4814b91cacf16a8b257cc64de691ec391f30a0d Mon Sep 17 00:00:00 2001 From: Andrew Goldstein Date: Mon, 15 Nov 2021 10:35:25 -0700 Subject: [PATCH 08/82] [Security Solution] Adjusts the width of the `Actions` column and action icon buttons (#118454) ## [Security Solution] Adjusts the width of the `Actions` column and action icon buttons This PR adjusts the width of the `Actions` column, and normalizes the action icon button sizes throughout the Security Solution, per https://github.com/elastic/kibana/issues/115726 ### Before / after screenshots This section provides before / after screenshots for the following views: - Alerts - Alerts > Event rendered - Rules > Details - Rules > Details > Event rendered - Host > Events - Host > External alerts - Network > External alerts - Timeline > Query tab - Timeline > Correlation tab - Timeline > Pinned tab - Observability > alerts (no change) #### Alerts (before) ![01-security_alerts_before](https://user-images.githubusercontent.com/4459398/141429498-a6040f8b-5bfb-468e-aa1a-993caa7f179c.png) #### Alerts (after) ![01a-security_alerts_after](https://user-images.githubusercontent.com/4459398/141429618-8ad313e1-fabc-424e-9e7d-c24240861c1d.png) #### Alerts > Event rendered (before) ![02-security_alerts_event_rendered_before](https://user-images.githubusercontent.com/4459398/141430881-2bfeb57a-9881-47f1-99e4-cc7eadcfff69.png) #### Alerts > Event rendered (after) ![02a-security_alerts_event_rendered_after](https://user-images.githubusercontent.com/4459398/141430976-88f8099a-81b1-4f1c-99a2-26f86218f909.png) #### Rules > Details (before) ![03-security_rules_details_before](https://user-images.githubusercontent.com/4459398/141431149-a308f171-a170-4ce9-9616-77e5c08dc406.png) #### Rules > Details (after) ![03a-security_rules_details_after](https://user-images.githubusercontent.com/4459398/141431221-06701540-97bb-400a-97bf-f2d22cd65caf.png) #### Rules > Details > Event rendered (before) ![04-security_rule_details_event_rendered_before](https://user-images.githubusercontent.com/4459398/141431394-12b29689-41c8-44b6-b69f-7796f99c5424.png) #### Rules > Details > Event rendered (after) ![04a-security_rule_details_event_rendered_after](https://user-images.githubusercontent.com/4459398/141431477-049804c0-1455-4216-a241-a44df5c9d398.png) #### Host > Events (before) ![05-host_events_before](https://user-images.githubusercontent.com/4459398/141431858-31116980-47f7-4779-af26-3b3785638137.png) #### Host > Events (after) ![05a-host_events_after](https://user-images.githubusercontent.com/4459398/141431956-664f86b9-2ad7-4281-bf82-8278fa23c755.png) #### Host > External alerts (before) ![06-host_external_alerts_before](https://user-images.githubusercontent.com/4459398/141432103-8cc9c10e-4d2d-42ec-a62c-a1e5867bf2d8.png) #### Host > External alerts (after) ![06a-host_external_alerts_after](https://user-images.githubusercontent.com/4459398/141432185-4d7e4007-dea9-47f3-af4b-1719f338a5ba.png) #### Network > External alerts (before) ![07-network_external_alerts_before](https://user-images.githubusercontent.com/4459398/141432331-2bb5a714-f733-4c97-91dc-73ff76633daa.png) #### Network > External alerts (after) ![07a-network_external_alerts_after](https://user-images.githubusercontent.com/4459398/141432428-b7b20450-db87-44ab-8014-cf4d6032dfe3.png) #### Timeline > Query tab (before) ![08-timeline_query_tab_before](https://user-images.githubusercontent.com/4459398/141432638-e484813b-275d-4eff-aa38-1705f913ce59.png) #### Timeline > Query tab (after) ![08a-timeline_query_tab_after](https://user-images.githubusercontent.com/4459398/141434461-1d36bba5-8fd1-484a-bacd-733aede95815.png) #### Timeline > Correlation tab (before) ![09-timeline_correlation_tab_before](https://user-images.githubusercontent.com/4459398/141434637-33f05447-e3d3-4eac-b38a-3612945e8379.png) #### Timeline > Correlation tab (after) ![09a-timeline_correlation_tab_after](https://user-images.githubusercontent.com/4459398/141434751-250fd26b-25fc-48cc-8a06-dbb17e53dce7.png) #### Timeline > Pinned tab (before) ![10-timeline_pinned_tab_before](https://user-images.githubusercontent.com/4459398/141434893-3f2b3d17-7e4b-4e0c-9096-ab1ee57f096f.png) #### Timeline > Pinned tab (after) ![10a-timeline_pinned_tab_after](https://user-images.githubusercontent.com/4459398/141435431-26eac065-bce4-4a25-99fd-095d447fb6f3.png) #### Observability > alerts (before) ![11-observability_alerts_before](https://user-images.githubusercontent.com/4459398/141435607-da059e9c-af03-4a21-bb1b-e47d44d61dde.png) #### Observability > alerts (after / no change) ![11a-observability_alerts_after_no_change](https://user-images.githubusercontent.com/4459398/141435696-52bcc5e1-6823-4b6a-b2da-32e3f8733dc8.png) ### Additional details - Per [this comment](https://github.com/elastic/kibana/issues/115726#issuecomment-962077067) from @monina-n , the size of all action buttons have been normalized match the size off the `...` overflow button (`28 x 32` at the time of this writing) via the `EuiButtonIcon` `size` prop: ``` size="s" ``` - The horizontal alignment of the `Analyze event` icon was updated by the EUI team in the following PR: https://github.com/elastic/eui/pull/5365 --- .../components/alerts_viewer/alerts_table.tsx | 5 + .../events_viewer/events_viewer.test.tsx | 485 ------------------ .../events_viewer/events_viewer.tsx | 395 -------------- .../components/events_viewer/index.test.tsx | 3 + .../common/components/events_viewer/index.tsx | 125 ++--- .../components/alerts_table/index.tsx | 5 + .../timeline_actions/alert_context_menu.tsx | 4 +- .../navigation/events_query_tab_body.tsx | 8 +- .../body/actions/action_icon_item.tsx | 5 +- .../timeline/body/actions/header_actions.tsx | 10 +- .../timeline/body/actions/index.tsx | 10 +- .../body/actions/pin_event_action.tsx | 4 +- .../__snapshots__/index.test.tsx.snap | 4 +- .../body/column_headers/helpers.test.ts | 31 +- .../timeline/body/column_headers/helpers.ts | 26 +- .../body/column_headers/index.test.tsx | 37 +- .../components/timeline/body/constants.ts | 14 - .../body/control_columns/index.test.tsx | 31 ++ .../timeline/body/control_columns/index.tsx | 21 +- .../body/data_driven_columns/index.test.tsx | 6 +- .../body/events/event_column_view.test.tsx | 12 +- .../components/timeline/body/index.test.tsx | 6 +- .../components/timeline/body/index.tsx | 21 +- .../cell_rendering/default_cell_renderer.tsx | 2 +- .../timeline/eql_tab_content/index.tsx | 18 +- .../timelines/components/timeline/helpers.tsx | 2 - .../components/timeline/pin/index.tsx | 7 +- .../timeline/pinned_tab_content/index.tsx | 16 +- .../timeline/properties/helpers.tsx | 1 + .../timeline/query_tab_content/index.tsx | 16 +- .../common/types/timeline/actions/index.ts | 2 - .../components/actions/action_icon_item.tsx | 4 +- .../__snapshots__/index.test.tsx.snap | 2 +- .../body/column_headers/helpers.test.tsx | 58 ++- .../t_grid/body/column_headers/helpers.tsx | 42 +- .../t_grid/body/column_headers/index.test.tsx | 19 +- .../components/t_grid/body/constants.ts | 28 +- .../body/events/event_column_view.test.tsx | 5 +- .../public/components/t_grid/body/index.tsx | 23 +- .../public/components/t_grid/helpers.tsx | 2 - x-pack/plugins/timelines/public/index.ts | 2 + 41 files changed, 311 insertions(+), 1206 deletions(-) delete mode 100644 x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx delete mode 100644 x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx create mode 100644 x-pack/plugins/security_solution/public/timelines/components/timeline/body/control_columns/index.test.tsx diff --git a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx index fc440197e8349..fff5b465956de 100644 --- a/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx +++ b/x-pack/plugins/security_solution/public/common/components/alerts_viewer/alerts_table.tsx @@ -21,6 +21,7 @@ import { SourcererScopeName } from '../../store/sourcerer/model'; import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features'; import { DEFAULT_COLUMN_MIN_WIDTH } from '../../../timelines/components/timeline/body/constants'; import type { EntityType } from '../../../../../timelines/common'; +import { getDefaultControlColumn } from '../../../timelines/components/timeline/body/control_columns'; export interface OwnProps { end: string; @@ -79,6 +80,7 @@ const AlertsTableComponent: React.FC = ({ const dispatch = useDispatch(); const alertsFilter = useMemo(() => [...defaultAlertsFilters, ...pageFilters], [pageFilters]); const { filterManager } = useKibana().services.data.query; + const ACTION_BUTTON_COUNT = 3; const tGridEnabled = useIsExperimentalFeatureEnabled('tGridEnabled'); @@ -104,6 +106,8 @@ const AlertsTableComponent: React.FC = ({ ); }, [dispatch, filterManager, tGridEnabled, timelineId]); + const leadingControlColumns = useMemo(() => getDefaultControlColumn(ACTION_BUTTON_COUNT), []); + return ( = ({ end={endDate} entityType={entityType} id={timelineId} + leadingControlColumns={leadingControlColumns} renderCellValue={DefaultCellRenderer} rowRenderers={defaultRowRenderers} scopeId={SourcererScopeName.default} diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx deleted file mode 100644 index cc94f24d04024..0000000000000 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.test.tsx +++ /dev/null @@ -1,485 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import React from 'react'; -import { waitFor, act } from '@testing-library/react'; -import useResizeObserver from 'use-resize-observer/polyfilled'; - -import '../../mock/match_media'; -import { mockIndexNames, mockIndexPattern, TestProviders } from '../../mock'; - -import { mockEventViewerResponse, mockEventViewerResponseWithEvents } from './mock'; -import { StatefulEventsViewer } from '.'; -import { EventsViewer } from './events_viewer'; -import { defaultHeaders } from './default_headers'; -import { useSourcererDataView } from '../../containers/sourcerer'; -import { - mockBrowserFields, - mockDocValueFields, - mockRuntimeMappings, -} from '../../containers/source/mock'; -import { eventsDefaultModel } from './default_model'; -import { useMountAppended } from '../../utils/use_mount_appended'; -import { inputsModel } from '../../store/inputs'; -import { TimelineId, SortDirection } from '../../../../common/types/timeline'; -import { KqlMode } from '../../../timelines/store/timeline/model'; -import { EntityType } from '../../../../../timelines/common'; -import { AlertsTableFilterGroup } from '../../../detections/components/alerts_table/alerts_filter_group'; -import { SourcererScopeName } from '../../store/sourcerer/model'; -import { defaultRowRenderers } from '../../../timelines/components/timeline/body/renderers'; -import { DefaultCellRenderer } from '../../../timelines/components/timeline/cell_rendering/default_cell_renderer'; -import { useTimelineEvents } from '../../../timelines/containers'; -import { useIsExperimentalFeatureEnabled } from '../../hooks/use_experimental_features'; -import { defaultCellActions } from '../../lib/cell_actions/default_cell_actions'; -import { mockTimelines } from '../../mock/mock_timelines_plugin'; - -jest.mock('../../lib/kibana', () => ({ - useKibana: () => ({ - services: { - application: { - navigateToApp: jest.fn(), - getUrlForApp: jest.fn(), - capabilities: { - siem: { crud_alerts: true, read_alerts: true }, - }, - }, - uiSettings: { - get: jest.fn(), - }, - savedObjects: { - client: {}, - }, - timelines: { ...mockTimelines }, - }, - }), - useToasts: jest.fn().mockReturnValue({ - addError: jest.fn(), - addSuccess: jest.fn(), - addWarning: jest.fn(), - }), - useGetUserCasesPermissions: jest.fn(), - useDateFormat: jest.fn(), - useTimeZone: jest.fn(), -})); - -jest.mock('../../hooks/use_experimental_features'); -const useIsExperimentalFeatureEnabledMock = useIsExperimentalFeatureEnabled as jest.Mock; - -jest.mock('../../../timelines/components/graph_overlay', () => ({ - GraphOverlay: jest.fn(() =>
), -})); - -const mockDispatch = jest.fn(); -jest.mock('react-redux', () => { - const original = jest.requireActual('react-redux'); - return { - ...original, - useDispatch: () => mockDispatch, - }; -}); - -jest.mock('@elastic/eui', () => { - const original = jest.requireActual('@elastic/eui'); - return { - ...original, - useDataGridColumnSorting: jest.fn(), - }; -}); -jest.mock('../../../timelines/containers', () => ({ - useTimelineEvents: jest.fn(), -})); - -jest.mock('../../components/url_state/normalize_time_range.ts'); - -const mockUseSourcererDataView: jest.Mock = useSourcererDataView as jest.Mock; -jest.mock('../../containers/sourcerer'); - -const mockUseResizeObserver: jest.Mock = useResizeObserver as jest.Mock; -jest.mock('use-resize-observer/polyfilled'); -mockUseResizeObserver.mockImplementation(() => ({})); - -const mockUseTimelineEvents: jest.Mock = useTimelineEvents as jest.Mock; -jest.mock('../../../timelines/containers'); - -const from = '2019-08-26T22:10:56.791Z'; -const to = '2019-08-27T22:10:56.794Z'; - -const defaultMocks = { - browserFields: mockBrowserFields, - docValueFields: mockDocValueFields, - runtimeMappings: mockRuntimeMappings, - indexPattern: mockIndexPattern, - loading: false, - selectedPatterns: mockIndexNames, -}; - -const utilityBar = (refetch: inputsModel.Refetch, totalCount: number) => ( -
-); - -const eventsViewerDefaultProps = { - browserFields: {}, - columns: [], - dataProviders: [], - deletedEventIds: [], - docValueFields: [], - end: to, - entityType: EntityType.ALERTS, - filters: [], - id: TimelineId.detectionsPage, - indexNames: mockIndexNames, - indexPattern: mockIndexPattern, - isLive: false, - isLoadingIndexPattern: false, - itemsPerPage: 10, - itemsPerPageOptions: [], - kqlMode: 'filter' as KqlMode, - query: { - query: '', - language: 'kql', - }, - renderCellValue: DefaultCellRenderer, - rowRenderers: defaultRowRenderers, - runtimeMappings: {}, - start: from, - sort: [ - { - columnId: 'foo', - columnType: 'number', - sortDirection: 'asc' as SortDirection, - }, - ], - scopeId: SourcererScopeName.timeline, - utilityBar, -}; - -describe('EventsViewer', () => { - const mount = useMountAppended(); - - let testProps = { - defaultCellActions, - defaultModel: eventsDefaultModel, - end: to, - entityType: EntityType.ALERTS, - id: TimelineId.test, - renderCellValue: DefaultCellRenderer, - rowRenderers: defaultRowRenderers, - start: from, - scopeId: SourcererScopeName.timeline, - }; - beforeEach(() => { - mockUseTimelineEvents.mockReset(); - }); - beforeAll(() => { - mockUseSourcererDataView.mockImplementation(() => defaultMocks); - }); - - describe('event details', () => { - useIsExperimentalFeatureEnabledMock.mockReturnValue(false); - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponseWithEvents]); - }); - - test('call the right reduce action to show event details', async () => { - const wrapper = mount( - - - - ); - - act(() => { - wrapper.find(`[data-test-subj="expand-event"]`).first().simulate('click'); - }); - - await waitFor(() => { - expect(mockDispatch).toBeCalledTimes(3); - expect(mockDispatch.mock.calls[1][0]).toEqual({ - payload: { - id: 'test', - isLoading: false, - }, - type: 'x-pack/timelines/t-grid/UPDATE_LOADING', - }); - }); - }); - }); - - describe('rendering', () => { - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]); - }); - - test('it renders the "Showing..." subtitle with the expected event count by default', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="header-section-subtitle"]`).first().text()).toEqual( - 'Showing: 12 events' - ); - }); - - test('should not render the "Showing..." subtitle with the expected event count if showTotalCount is set to false ', () => { - const disableSubTitle = { - ...eventsViewerDefaultProps, - showTotalCount: false, - }; - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="header-section-subtitle"]`).first().text()).toEqual(''); - }); - - test('it renders the Fields Browser as a settings gear', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="field-browser"]`).first().exists()).toBe(true); - }); - - test('it renders the footer containing the pagination', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="timeline-pagination"]`).first().exists()).toBe(true); - }); - - defaultHeaders.forEach((header) => { - test(`it renders the ${header.id} default EventsViewer column header`, () => { - testProps = { - ...testProps, - // Update with a new id, to force columns back to default. - id: TimelineId.alternateTest, - }; - const wrapper = mount( - - - - ); - - defaultHeaders.forEach((h) => { - expect(wrapper.find(`[data-test-subj="header-text-${header.id}"]`).first().exists()).toBe( - true - ); - }); - }); - }); - }); - - describe('loading', () => { - beforeAll(() => { - mockUseSourcererDataView.mockImplementation(() => ({ ...defaultMocks, loading: true })); - }); - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]); - }); - - test('it does NOT render fetch index pattern is loading', () => { - const wrapper = mount( - - - - ); - - expect(wrapper.find(`[data-test-subj="header-section-subtitle"]`).first().exists()).toBe( - false - ); - }); - - test('it does NOT render when start is empty', () => { - testProps = { - ...testProps, - start: '', - }; - const wrapper = mount( - - - - ); - - expect(wrapper.find(`[data-test-subj="header-section-subtitle"]`).first().exists()).toBe( - false - ); - }); - - test('it does NOT render when end is empty', () => { - testProps = { - ...testProps, - end: '', - }; - const wrapper = mount( - - - - ); - - expect(wrapper.find(`[data-test-subj="header-section-subtitle"]`).first().exists()).toBe( - false - ); - }); - }); - - describe('headerFilterGroup', () => { - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]); - }); - - test('it renders the provided headerFilterGroup', () => { - const wrapper = mount( - - - } - /> - - ); - expect(wrapper.find(`[data-test-subj="alerts-table-filter-group"]`).exists()).toBe(true); - }); - - test('it has a visible HeaderFilterGroupWrapper when Resolver is NOT showing, because graphEventId is undefined', () => { - const wrapper = mount( - - - } - /> - - ); - expect( - wrapper.find(`[data-test-subj="header-filter-group-wrapper"]`).first() - ).not.toHaveStyleRule('visibility', 'hidden'); - }); - - test('it has a visible HeaderFilterGroupWrapper when Resolver is NOT showing, because graphEventId is an empty string', () => { - const wrapper = mount( - - - } - /> - - ); - expect( - wrapper.find(`[data-test-subj="header-filter-group-wrapper"]`).first() - ).not.toHaveStyleRule('visibility', 'hidden'); - }); - - test('it does NOT have a visible HeaderFilterGroupWrapper when Resolver is showing, because graphEventId is a valid id', () => { - const wrapper = mount( - - - } - /> - - ); - expect( - wrapper.find(`[data-test-subj="header-filter-group-wrapper"]`).first() - ).toHaveStyleRule('visibility', 'hidden'); - }); - - test('it (still) renders an invisible headerFilterGroup (to maintain state while hidden) when Resolver is showing, because graphEventId is a valid id', () => { - const wrapper = mount( - - - } - /> - - ); - expect(wrapper.find(`[data-test-subj="alerts-table-filter-group"]`).exists()).toBe(true); - }); - }); - - describe('utilityBar', () => { - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]); - }); - - test('it renders the provided utilityBar when Resolver is NOT showing, because graphEventId is undefined', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="mock-utility-bar"]`).exists()).toBe(true); - }); - - test('it renders the provided utilityBar when Resolver is NOT showing, because graphEventId is an empty string', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="mock-utility-bar"]`).exists()).toBe(true); - }); - - test('it does NOT render the provided utilityBar when Resolver is showing, because graphEventId is a valid id', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="mock-utility-bar"]`).exists()).toBe(false); - }); - }); - - describe('header inspect button', () => { - beforeEach(() => { - mockUseTimelineEvents.mockReturnValue([false, mockEventViewerResponse]); - }); - - test('it renders the inspect button when Resolver is NOT showing, because graphEventId is undefined', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="inspect-icon-button"]`).exists()).toBe(true); - }); - - test('it renders the inspect button when Resolver is NOT showing, because graphEventId is an empty string', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="inspect-icon-button"]`).exists()).toBe(true); - }); - - test('it does NOT render the inspect button when Resolver is showing, because graphEventId is a valid id', () => { - const wrapper = mount( - - - - ); - expect(wrapper.find(`[data-test-subj="inspect-icon-button"]`).exists()).toBe(false); - }); - }); -}); diff --git a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx b/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx deleted file mode 100644 index 5a3aa2e6dc38a..0000000000000 --- a/x-pack/plugins/security_solution/public/common/components/events_viewer/events_viewer.tsx +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -import { EuiFlexGroup, EuiFlexItem, EuiPanel } from '@elastic/eui'; -import { isEmpty } from 'lodash/fp'; -import React, { useEffect, useMemo, useState } from 'react'; -import styled from 'styled-components'; -import deepEqual from 'fast-deep-equal'; - -import { useDispatch } from 'react-redux'; -import { DataViewBase, Filter, Query } from '@kbn/es-query'; -import { MappingRuntimeFields } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { Direction } from '../../../../common/search_strategy'; -import { BrowserFields, DocValueFields } from '../../containers/source'; -import { useTimelineEvents } from '../../../timelines/containers'; -import { useKibana } from '../../lib/kibana'; -import { KqlMode } from '../../../timelines/store/timeline/model'; -import { HeaderSection } from '../header_section'; -import { defaultHeaders } from '../../../timelines/components/timeline/body/column_headers/default_headers'; -import { Sort } from '../../../timelines/components/timeline/body/sort'; -import { StatefulBody } from '../../../timelines/components/timeline/body'; -import { DataProvider } from '../../../timelines/components/timeline/data_providers/data_provider'; -import { Footer, footerHeight } from '../../../timelines/components/timeline/footer'; -import { - calculateTotalPages, - combineQueries, - resolverIsShowing, -} from '../../../timelines/components/timeline/helpers'; -import { TimelineRefetch } from '../../../timelines/components/timeline/refetch_timeline'; -import { EventDetailsWidthProvider } from './event_details_width_context'; -import * as i18n from './translations'; -import { esQuery } from '../../../../../../../src/plugins/data/public'; -import { inputsModel } from '../../store'; -import { ExitFullScreen } from '../exit_full_screen'; -import { useGlobalFullScreen } from '../../containers/use_full_screen'; -import { - ColumnHeaderOptions, - ControlColumnProps, - RowRenderer, - TimelineId, - TimelineTabs, -} from '../../../../common/types/timeline'; -import { GraphOverlay } from '../../../timelines/components/graph_overlay'; -import { CellValueElementProps } from '../../../timelines/components/timeline/cell_rendering'; -import { SELECTOR_TIMELINE_GLOBAL_CONTAINER } from '../../../timelines/components/timeline/styles'; -import { timelineSelectors, timelineActions } from '../../../timelines/store/timeline'; -import { useDeepEqualSelector } from '../../hooks/use_selector'; -import { defaultControlColumn } from '../../../timelines/components/timeline/body/control_columns'; -import { TimelineContext } from '../../../../../timelines/public'; - -export const EVENTS_VIEWER_HEADER_HEIGHT = 90; // px -const UTILITY_BAR_HEIGHT = 19; // px -const COMPACT_HEADER_HEIGHT = EVENTS_VIEWER_HEADER_HEIGHT - UTILITY_BAR_HEIGHT; // px - -const UtilityBar = styled.div` - height: ${UTILITY_BAR_HEIGHT}px; -`; - -const TitleText = styled.span` - margin-right: 12px; -`; - -const StyledEuiPanel = styled(EuiPanel)<{ $isFullScreen: boolean }>` - display: flex; - flex-direction: column; - - ${({ $isFullScreen }) => - $isFullScreen && - ` - border: 0; - box-shadow: none; - padding-top: 0; - padding-bottom: 0; - `} -`; - -const TitleFlexGroup = styled(EuiFlexGroup)` - margin-top: 8px; -`; - -const EventsContainerLoading = styled.div.attrs(({ className = '' }) => ({ - className: `${SELECTOR_TIMELINE_GLOBAL_CONTAINER} ${className}`, -}))` - width: 100%; - overflow: hidden; - flex: 1; - display: flex; - flex-direction: column; -`; - -const FullWidthFlexGroup = styled(EuiFlexGroup)<{ $visible: boolean }>` - overflow: hidden; - margin: 0; - display: ${({ $visible }) => ($visible ? 'flex' : 'none')}; -`; - -const ScrollableFlexItem = styled(EuiFlexItem)` - overflow: auto; -`; - -/** - * Hides stateful headerFilterGroup implementations, but prevents the component - * from being unmounted, to preserve the state of the component - */ -const HeaderFilterGroupWrapper = styled.header<{ show: boolean }>` - ${({ show }) => (show ? '' : 'visibility: hidden;')} -`; - -interface Props { - browserFields: BrowserFields; - columns: ColumnHeaderOptions[]; - dataProviders: DataProvider[]; - deletedEventIds: Readonly; - docValueFields: DocValueFields[]; - end: string; - filters: Filter[]; - headerFilterGroup?: React.ReactNode; - id: TimelineId; - indexNames: string[]; - indexPattern: DataViewBase; - isLive: boolean; - isLoadingIndexPattern: boolean; - itemsPerPage: number; - itemsPerPageOptions: number[]; - kqlMode: KqlMode; - query: Query; - onRuleChange?: () => void; - renderCellValue: (props: CellValueElementProps) => React.ReactNode; - rowRenderers: RowRenderer[]; - runtimeMappings: MappingRuntimeFields; - start: string; - sort: Sort[]; - showTotalCount?: boolean; - utilityBar?: (refetch: inputsModel.Refetch, totalCount: number) => React.ReactNode; - // If truthy, the graph viewer (Resolver) is showing - graphEventId: string | undefined; -} - -const EventsViewerComponent: React.FC = ({ - browserFields, - columns, - dataProviders, - deletedEventIds, - docValueFields, - end, - filters, - headerFilterGroup, - id, - indexNames, - indexPattern, - isLive, - isLoadingIndexPattern, - itemsPerPage, - itemsPerPageOptions, - kqlMode, - onRuleChange, - query, - renderCellValue, - rowRenderers, - runtimeMappings, - start, - sort, - showTotalCount = true, - utilityBar, - graphEventId, -}) => { - const dispatch = useDispatch(); - const { globalFullScreen, setGlobalFullScreen } = useGlobalFullScreen(); - const columnsHeader = isEmpty(columns) ? defaultHeaders : columns; - const kibana = useKibana(); - const [isQueryLoading, setIsQueryLoading] = useState(false); - - useEffect(() => { - dispatch(timelineActions.updateIsLoading({ id, isLoading: isQueryLoading })); - }, [dispatch, id, isQueryLoading]); - - const getManageTimeline = useMemo(() => timelineSelectors.getManageTimelineById(), []); - const unit = useMemo(() => (n: number) => i18n.UNIT(n), []); - const { queryFields, title } = useDeepEqualSelector((state) => getManageTimeline(state, id)); - - const justTitle = useMemo(() => {title}, [title]); - - const titleWithExitFullScreen = useMemo( - () => ( - - {justTitle} - - - - - ), - [globalFullScreen, justTitle, setGlobalFullScreen] - ); - - const combinedQueries = combineQueries({ - config: esQuery.getEsQueryConfig(kibana.services.uiSettings), - dataProviders, - indexPattern, - browserFields, - filters, - kqlQuery: query, - kqlMode, - isEventViewer: true, - }); - - const canQueryTimeline = useMemo( - () => - combinedQueries != null && - isLoadingIndexPattern != null && - !isLoadingIndexPattern && - !isEmpty(start) && - !isEmpty(end), - [isLoadingIndexPattern, combinedQueries, start, end] - ); - - const fields = useMemo( - () => [...columnsHeader.map((c) => c.id), ...(queryFields ?? [])], - [columnsHeader, queryFields] - ); - - const sortField = useMemo( - () => - sort.map(({ columnId, columnType, sortDirection }) => ({ - field: columnId, - type: columnType, - direction: sortDirection as Direction, - })), - [sort] - ); - - const [loading, { events, updatedAt, inspect, loadPage, pageInfo, refetch, totalCount = 0 }] = - useTimelineEvents({ - docValueFields, - fields, - filterQuery: combinedQueries?.filterQuery, - id, - indexNames, - limit: itemsPerPage, - runtimeMappings, - sort: sortField, - startDate: start, - endDate: end, - skip: !canQueryTimeline || combinedQueries?.filterQuery === undefined, // When the filterQuery comes back as undefined, it means an error has been thrown and the request should be skipped - }); - - const totalCountMinusDeleted = useMemo( - () => (totalCount > 0 ? totalCount - deletedEventIds.length : 0), - [deletedEventIds.length, totalCount] - ); - - const subtitle = useMemo( - () => - showTotalCount - ? `${i18n.SHOWING}: ${totalCountMinusDeleted.toLocaleString()} ${unit( - totalCountMinusDeleted - )}` - : null, - [showTotalCount, totalCountMinusDeleted, unit] - ); - - const nonDeletedEvents = useMemo( - () => events.filter((e) => !deletedEventIds.includes(e._id)), - [deletedEventIds, events] - ); - - const HeaderSectionContent = useMemo( - () => - headerFilterGroup && ( - - {headerFilterGroup} - - ), - [graphEventId, headerFilterGroup] - ); - - useEffect(() => { - setIsQueryLoading(loading); - }, [loading]); - - const leadingControlColumns: ControlColumnProps[] = [defaultControlColumn]; - const trailingControlColumns: ControlColumnProps[] = []; - const timelineContext = useMemo(() => ({ timelineId: id }), [id]); - return ( - - {canQueryTimeline ? ( - - <> - - {HeaderSectionContent} - - {utilityBar && !resolverIsShowing(graphEventId) && ( - {utilityBar?.(refetch, totalCountMinusDeleted)} - )} - - - - - {graphEventId && } - - - -