From eb4b573994285a61f6284ef363aa3d8b4f9beb7d Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Tue, 17 Nov 2020 18:55:28 -0500 Subject: [PATCH 01/12] [Lens] Implement types for reference-based operations --- .../visualization.test.tsx | 4 +- .../editor_frame/state_helpers.ts | 2 +- .../workspace_panel/workspace_panel.tsx | 4 +- .../dimension_panel/dimension_editor.tsx | 2 +- .../dimension_panel/dimension_panel.test.tsx | 1 + .../dimension_panel/operation_support.ts | 2 + .../indexpattern.test.ts | 116 +++- .../indexpattern_datasource/indexpattern.tsx | 120 ++-- .../indexpattern_suggestions.ts | 6 +- .../public/indexpattern_datasource/mocks.ts | 2 +- .../operations/__mocks__/index.ts | 6 + .../operations/definitions/cardinality.tsx | 2 + .../operations/definitions/column_types.ts | 24 +- .../operations/definitions/count.tsx | 1 + .../operations/definitions/date_histogram.tsx | 2 + .../definitions/filters/filters.tsx | 1 + .../operations/definitions/index.ts | 105 +++- .../operations/definitions/metrics.tsx | 2 + .../operations/definitions/ranges/ranges.tsx | 4 +- .../operations/definitions/terms/index.tsx | 19 +- .../operations/index.ts | 9 +- .../operations/layer_helpers.test.ts | 572 +++++++++++++++++- .../operations/layer_helpers.ts | 274 ++++++++- .../operations/mocks.ts | 39 ++ .../operations/operations.ts | 9 + .../indexpattern_datasource/to_expression.ts | 35 +- .../public/indexpattern_datasource/types.ts | 4 +- .../public/indexpattern_datasource/utils.ts | 8 +- 28 files changed, 1227 insertions(+), 148 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts diff --git a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx index 0af8e01d7290d..cf3752e649600 100644 --- a/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx +++ b/x-pack/plugins/lens/public/datatable_visualization/visualization.test.tsx @@ -410,7 +410,7 @@ describe('Datatable Visualization', () => { const error = datatableVisualization.getErrorMessages({ layers: [layer] }, frame); - expect(error).not.toBeDefined(); + expect(error).toBeUndefined(); }); it('returns undefined if the metric dimension is defined', () => { @@ -427,7 +427,7 @@ describe('Datatable Visualization', () => { const error = datatableVisualization.getErrorMessages({ layers: [layer] }, frame); - expect(error).not.toBeDefined(); + expect(error).toBeUndefined(); }); }); }); diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts index 647c0f3ac9cca..0c96fc45de128 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/state_helpers.ts @@ -134,7 +134,7 @@ export const validateDatasourceAndVisualization = ( ? currentVisualization?.getErrorMessages(currentVisualizationState, frameAPI) : undefined; - if (datasourceValidationErrors || visualizationValidationErrors) { + if (datasourceValidationErrors?.length || visualizationValidationErrors?.length) { return [...(datasourceValidationErrors || []), ...(visualizationValidationErrors || [])]; } return undefined; diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 00cb932a6d4e2..95aeedbd857ca 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -385,7 +385,7 @@ export const InnerVisualizationWrapper = ({ [dispatch] ); - if (localState.configurationValidationError) { + if (localState.configurationValidationError?.length) { let showExtraErrors = null; if (localState.configurationValidationError.length > 1) { if (localState.expandError) { @@ -445,7 +445,7 @@ export const InnerVisualizationWrapper = ({ ); } - if (localState.expressionBuildError) { + if (localState.expressionBuildError?.length) { return ( diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index cd196745f3315..e5c05a1cf8c7a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -419,7 +419,7 @@ export function DimensionEditor(props: DimensionEditorProps) { function getErrorMessage( selectedColumn: IndexPatternColumn | undefined, incompatibleSelectedOperationType: boolean, - input: 'none' | 'field' | undefined, + input: 'none' | 'field' | 'fullReference' | undefined, fieldInvalid: boolean ) { if (selectedColumn && incompatibleSelectedOperationType) { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index b2edc61a56736..2e57ecee86033 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -1054,6 +1054,7 @@ describe('IndexPatternDimensionEditorPanel', () => { indexPatternId: '1', columns: {}, columnOrder: [], + incompleteColumns: {}, }, }, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/operation_support.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/operation_support.ts index 31fb5277d53ec..817fdf637f001 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/operation_support.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/operation_support.ts @@ -21,6 +21,8 @@ type Props = Pick< 'layerId' | 'columnId' | 'state' | 'filterOperations' >; +// TODO: the support matrix should be available outside of the dimension panel + // TODO: This code has historically been memoized, as a potentially performance // sensitive task. If we can add memoization without breaking the behavior, we should. export const getOperationSupportMatrix = (props: Props): OperationSupportMatrix => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 51d95245adb25..3cf9bdc3a92f1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -13,9 +13,15 @@ import { dataPluginMock } from '../../../../../src/plugins/data/public/mocks'; import { Ast } from '@kbn/interpreter/common'; import { chartPluginMock } from '../../../../../src/plugins/charts/public/mocks'; import { getFieldByNameFactory } from './pure_helpers'; +import { + operationDefinitionMap, + getErrorMessages, + createMockedReferenceOperation, +} from './operations'; jest.mock('./loader'); jest.mock('../id_generator'); +jest.mock('./operations'); const fieldsOne = [ { @@ -489,6 +495,56 @@ describe('IndexPattern Data Source', () => { expect(ast.chain[0].arguments.timeFields).toEqual(['timestamp']); expect(ast.chain[0].arguments.timeFields).not.toContain('timefield'); }); + + describe('references', () => { + beforeEach(() => { + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.testReference = createMockedReferenceOperation(); + + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.testReference.toExpression.mockReturnValue(['mock']); + }); + + afterEach(() => { + delete operationDefinitionMap.testReference; + }); + + it('should collect expression references and append them', async () => { + const queryBaseState: IndexPatternBaseState = { + currentIndexPatternId: '1', + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Count of records', + dataType: 'date', + isBucketed: false, + sourceField: 'timefield', + operationType: 'cardinality', + }, + col2: { + label: 'Reference', + dataType: 'number', + isBucketed: false, + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }, + }, + }; + + const state = enrichBaseState(queryBaseState); + + const ast = indexPatternDatasource.toExpression(state, 'first') as Ast; + // @ts-expect-error we can't isolate just the reference type + expect(operationDefinitionMap.testReference.toExpression).toHaveBeenCalled(); + expect(ast.chain[2]).toEqual('mock'); + }); + }); }); describe('#insertLayer', () => { @@ -599,11 +655,33 @@ describe('IndexPattern Data Source', () => { describe('getTableSpec', () => { it('should include col1', () => { - expect(publicAPI.getTableSpec()).toEqual([ - { - columnId: 'col1', + expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col1' }]); + }); + + it('should skip columns that are being referenced', () => { + publicAPI = indexPatternDatasource.getPublicAPI({ + state: { + layers: { + first: { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + // @ts-ignore this is too little information for a real column + col1: { + dataType: 'number', + }, + col2: { + // @ts-expect-error update once we have a reference operation outside tests + references: ['col1'], + }, + }, + }, + }, }, - ]); + layerId: 'first', + }); + + expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col2' }]); }); }); @@ -764,7 +842,7 @@ describe('IndexPattern Data Source', () => { dataType: 'number', isBucketed: false, label: 'Foo', - operationType: 'document', + operationType: 'avg', sourceField: 'bytes', }, }, @@ -774,7 +852,7 @@ describe('IndexPattern Data Source', () => { }; expect( indexPatternDatasource.getErrorMessages(state as IndexPatternPrivateState) - ).not.toBeDefined(); + ).toBeUndefined(); }); it('should return no errors with layers with no columns', () => { @@ -792,7 +870,31 @@ describe('IndexPattern Data Source', () => { }, currentIndexPatternId: '1', }; - expect(indexPatternDatasource.getErrorMessages(state)).not.toBeDefined(); + expect(indexPatternDatasource.getErrorMessages(state)).toBeUndefined(); + }); + + it('should bubble up invalid configuration from operations', () => { + (getErrorMessages as jest.Mock).mockClear(); + (getErrorMessages as jest.Mock).mockReturnValueOnce(['error 1', 'error 2']); + const state: IndexPatternPrivateState = { + indexPatternRefs: [], + existingFields: {}, + isFirstExistenceFetch: false, + indexPatterns: expectedIndexPatterns, + layers: { + first: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + }, + }, + currentIndexPatternId: '1', + }; + expect(indexPatternDatasource.getErrorMessages(state)).toEqual([ + { shortMessage: 'error 1', longMessage: '' }, + { shortMessage: 'error 2', longMessage: '' }, + ]); + expect(getErrorMessages).toHaveBeenCalledTimes(1); }); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 94f240058d618..2c64431867df0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -40,13 +40,13 @@ import { } from './indexpattern_suggestions'; import { - getInvalidFieldReferencesForLayer, - getInvalidReferences, + getInvalidFieldsForLayer, + getInvalidLayers, isDraggedField, normalizeOperationDataType, } from './utils'; import { LayerPanel } from './layerpanel'; -import { IndexPatternColumn } from './operations'; +import { IndexPatternColumn, getErrorMessages } from './operations'; import { IndexPatternField, IndexPatternPrivateState, IndexPatternPersistedState } from './types'; import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/public'; import { DataPublicPluginStart } from '../../../../../src/plugins/data/public'; @@ -54,7 +54,7 @@ import { VisualizeFieldContext } from '../../../../../src/plugins/ui_actions/pub import { mergeLayer } from './state_helpers'; import { Datasource, StateSetter } from '../index'; import { ChartsPluginSetup } from '../../../../../src/plugins/charts/public'; -import { deleteColumn } from './operations'; +import { deleteColumn, isReferenced } from './operations'; import { FieldBasedIndexPatternColumn } from './operations/definitions/column_types'; import { Dragging } from '../drag_drop/providers'; @@ -325,7 +325,9 @@ export function getIndexPatternDatasource({ datasourceId: 'indexpattern', getTableSpec: () => { - return state.layers[layerId].columnOrder.map((colId) => ({ columnId: colId })); + return state.layers[layerId].columnOrder + .filter((colId) => !isReferenced(state.layers[layerId], colId)) + .map((colId) => ({ columnId: colId })); }, getOperationForColumnId: (columnId: string) => { const layer = state.layers[layerId]; @@ -349,10 +351,17 @@ export function getIndexPatternDatasource({ if (!state) { return; } - const invalidLayers = getInvalidReferences(state); + const invalidLayers = getInvalidLayers(state); + + const layerErrors = Object.values(state.layers).flatMap((layer) => + (getErrorMessages(layer) ?? []).map((message) => ({ + shortMessage: message, + longMessage: '', + })) + ); if (invalidLayers.length === 0) { - return; + return layerErrors.length ? layerErrors : undefined; } const realIndex = Object.values(state.layers) @@ -363,64 +372,69 @@ export function getIndexPatternDatasource({ } }) .filter(Boolean) as Array<[number, number]>; - const invalidFieldsPerLayer: string[][] = getInvalidFieldReferencesForLayer( + const invalidFieldsPerLayer: string[][] = getInvalidFieldsForLayer( invalidLayers, state.indexPatterns ); const originalLayersList = Object.keys(state.layers); - return realIndex.map(([filteredIndex, layerIndex]) => { - const fieldsWithBrokenReferences: string[] = invalidFieldsPerLayer[filteredIndex].map( - (columnId) => { - const column = invalidLayers[filteredIndex].columns[ - columnId - ] as FieldBasedIndexPatternColumn; - return column.sourceField; - } - ); - - if (originalLayersList.length === 1) { - return { - shortMessage: i18n.translate( - 'xpack.lens.indexPattern.dataReferenceFailureShortSingleLayer', - { - defaultMessage: 'Invalid {fields, plural, one {reference} other {references}}.', + if (layerErrors.length || realIndex.length) { + return [ + ...layerErrors, + ...realIndex.map(([filteredIndex, layerIndex]) => { + const fieldsWithBrokenReferences: string[] = invalidFieldsPerLayer[filteredIndex].map( + (columnId) => { + const column = invalidLayers[filteredIndex].columns[ + columnId + ] as FieldBasedIndexPatternColumn; + return column.sourceField; + } + ); + + if (originalLayersList.length === 1) { + return { + shortMessage: i18n.translate( + 'xpack.lens.indexPattern.dataReferenceFailureShortSingleLayer', + { + defaultMessage: 'Invalid {fields, plural, one {reference} other {references}}.', + values: { + fields: fieldsWithBrokenReferences.length, + }, + } + ), + longMessage: i18n.translate( + 'xpack.lens.indexPattern.dataReferenceFailureLongSingleLayer', + { + defaultMessage: `{fieldsLength, plural, one {Field} other {Fields}} "{fields}" {fieldsLength, plural, one {has an} other {have}} invalid reference.`, + values: { + fields: fieldsWithBrokenReferences.join('", "'), + fieldsLength: fieldsWithBrokenReferences.length, + }, + } + ), + }; + } + return { + shortMessage: i18n.translate('xpack.lens.indexPattern.dataReferenceFailureShort', { + defaultMessage: + 'Invalid {fieldsLength, plural, one {reference} other {references}} on Layer {layer}.', values: { - fields: fieldsWithBrokenReferences.length, + layer: layerIndex, + fieldsLength: fieldsWithBrokenReferences.length, }, - } - ), - longMessage: i18n.translate( - 'xpack.lens.indexPattern.dataReferenceFailureLongSingleLayer', - { - defaultMessage: `{fieldsLength, plural, one {Field} other {Fields}} "{fields}" {fieldsLength, plural, one {has an} other {have}} invalid reference.`, + }), + longMessage: i18n.translate('xpack.lens.indexPattern.dataReferenceFailureLong', { + defaultMessage: `Layer {layer} has {fieldsLength, plural, one {an invalid} other {invalid}} {fieldsLength, plural, one {reference} other {references}} in {fieldsLength, plural, one {field} other {fields}} "{fields}".`, values: { + layer: layerIndex, fields: fieldsWithBrokenReferences.join('", "'), fieldsLength: fieldsWithBrokenReferences.length, }, - } - ), - }; - } - return { - shortMessage: i18n.translate('xpack.lens.indexPattern.dataReferenceFailureShort', { - defaultMessage: - 'Invalid {fieldsLength, plural, one {reference} other {references}} on Layer {layer}.', - values: { - layer: layerIndex, - fieldsLength: fieldsWithBrokenReferences.length, - }, + }), + }; }), - longMessage: i18n.translate('xpack.lens.indexPattern.dataReferenceFailureLong', { - defaultMessage: `Layer {layer} has {fieldsLength, plural, one {an invalid} other {invalid}} {fieldsLength, plural, one {reference} other {references}} in {fieldsLength, plural, one {field} other {fields}} "{fields}".`, - values: { - layer: layerIndex, - fields: fieldsWithBrokenReferences.join('", "'), - fieldsLength: fieldsWithBrokenReferences.length, - }, - }), - }; - }); + ]; + } }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts index ccdefee62ad5c..263b4646c9feb 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.ts @@ -18,7 +18,7 @@ import { IndexPatternColumn, OperationType, } from './operations'; -import { hasField, hasInvalidReference } from './utils'; +import { hasField, hasInvalidFields } from './utils'; import { IndexPattern, IndexPatternPrivateState, @@ -90,7 +90,7 @@ export function getDatasourceSuggestionsForField( indexPatternId: string, field: IndexPatternField ): IndexPatternSugestion[] { - if (hasInvalidReference(state)) return []; + if (hasInvalidFields(state)) return []; const layers = Object.keys(state.layers); const layerIds = layers.filter((id) => state.layers[id].indexPatternId === indexPatternId); @@ -331,7 +331,7 @@ function createNewLayerWithMetricAggregation( export function getDatasourceSuggestionsFromCurrentState( state: IndexPatternPrivateState ): Array> { - if (hasInvalidReference(state)) return []; + if (hasInvalidFields(state)) return []; const layers = Object.entries(state.layers || {}); if (layers.length > 1) { // Return suggestions that reduce the data to each layer individually diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts index 2c6f42668d863..d0cbcee61db6f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/mocks.ts @@ -6,7 +6,7 @@ import { DragContextState } from '../drag_drop'; import { getFieldByNameFactory } from './pure_helpers'; -import { IndexPattern } from './types'; +import type { IndexPattern } from './types'; export const createMockedIndexPattern = (): IndexPattern => { const fields = [ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts index 72dfe85dfc0e9..f27fb8d4642f6 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/__mocks__/index.ts @@ -6,12 +6,14 @@ const actualOperations = jest.requireActual('../operations'); const actualHelpers = jest.requireActual('../layer_helpers'); +const actualMocks = jest.requireActual('../mocks'); jest.spyOn(actualOperations.operationDefinitionMap.date_histogram, 'paramEditor'); jest.spyOn(actualOperations.operationDefinitionMap.terms, 'onOtherColumnChanged'); jest.spyOn(actualHelpers, 'insertOrReplaceColumn'); jest.spyOn(actualHelpers, 'insertNewColumn'); jest.spyOn(actualHelpers, 'replaceColumn'); +jest.spyOn(actualHelpers, 'getErrorMessages'); export const { getAvailableOperationsByMetadata, @@ -35,4 +37,8 @@ export const { updateLayerIndexPattern, mergeLayer, isColumnTransferable, + getErrorMessages, + isReferenced, } = actualHelpers; + +export const { createMockedReferenceOperation } = actualMocks; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx index bd8c4b4683396..fd3ca4319669e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cardinality.tsx @@ -52,6 +52,8 @@ export const cardinalityOperation: OperationDefinition + ofName(indexPattern.getFieldByName(column.sourceField)!.displayName), buildColumn({ field, previousColumn }) { return { label: ofName(field.displayName), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts index bd4b452a49e1d..13bddc0c2ec26 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts @@ -4,13 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Operation } from '../../../types'; +import type { Operation } from '../../../types'; -/** - * This is the root type of a column. If you are implementing a new - * operation, extend your column type on `BaseIndexPatternColumn` to make - * sure it's matching all the basic requirements. - */ export interface BaseIndexPatternColumn extends Operation { // Private operationType: string; @@ -18,7 +13,8 @@ export interface BaseIndexPatternColumn extends Operation { } // Formatting can optionally be added to any column -export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { +// export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { +export type FormattedIndexPatternColumn = BaseIndexPatternColumn & { params?: { format: { id: string; @@ -27,8 +23,20 @@ export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { }; }; }; -} +}; export interface FieldBasedIndexPatternColumn extends BaseIndexPatternColumn { sourceField: string; } + +export interface ReferenceBasedIndexPatternColumn + extends BaseIndexPatternColumn, + FormattedIndexPatternColumn { + references: string[]; +} + +// Used to store the temporary invalid state +export interface IncompleteColumn { + operationType?: string; + sourceField?: string; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index e33fc681b2579..30f64929fc1af 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -41,6 +41,7 @@ export const countOperation: OperationDefinition countLabel, buildColumn({ field, previousColumn }) { return { label: countLabel, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx index 659390a42f261..efac9c151a435 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.tsx @@ -59,6 +59,8 @@ export const dateHistogramOperation: OperationDefinition< }; } }, + getDefaultLabel: (column, indexPattern) => + indexPattern.getFieldByName(column.sourceField)!.displayName, buildColumn({ field }) { let interval = autoInterval; let timeZone: string | undefined; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.tsx index 522e951bfba34..1b0452d18a79c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/filters/filters.tsx @@ -75,6 +75,7 @@ export const filtersOperation: OperationDefinition true, + getDefaultLabel: () => filtersLabel, buildColumn({ previousColumn }) { let params = { filters: [defaultFilter] }; if (previousColumn?.operationType === 'terms') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 5c067ebaf21e9..564a4ca340b40 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { ExpressionFunctionAST } from '@kbn/interpreter/common'; import { IUiSettingsClient, SavedObjectsClientContract, HttpSetup } from 'kibana/public'; import { IStorageWrapper } from 'src/plugins/kibana_utils/public'; import { termsOperation, TermsIndexPatternColumn } from './terms'; @@ -24,8 +25,13 @@ import { import { dateHistogramOperation, DateHistogramIndexPatternColumn } from './date_histogram'; import { countOperation, CountIndexPatternColumn } from './count'; import { StateSetter, OperationMetadata } from '../../../types'; -import { BaseIndexPatternColumn } from './column_types'; -import { IndexPatternPrivateState, IndexPattern, IndexPatternField } from '../../types'; +import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; +import { + IndexPatternPrivateState, + IndexPattern, + IndexPatternField, + IndexPatternLayer, +} from '../../types'; import { DateRange } from '../../../../common'; import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; import { RangeIndexPatternColumn, rangeOperation } from './ranges'; @@ -50,6 +56,8 @@ export type IndexPatternColumn = export type FieldBasedIndexPatternColumn = Extract; +export { IncompleteColumn } from './column_types'; + // List of all operation definitions registered to this data source. // If you want to implement a new operation, add the definition to this array and // the column type to the `IndexPatternColumn` union type below. @@ -104,6 +112,14 @@ interface BaseOperationDefinitionProps { * Should be i18n-ified. */ displayName: string; + /** + * The default label is assigned by the editor + */ + getDefaultLabel: ( + column: C, + indexPattern: IndexPattern, + columns: Record + ) => string; /** * This function is called if another column in the same layer changed or got removed. * Can be used to update references to other columns (e.g. for sorting). @@ -118,11 +134,6 @@ interface BaseOperationDefinitionProps { * React component for operation specific settings shown in the popover editor */ paramEditor?: React.ComponentType>; - /** - * Function turning a column into an agg config passed to the `esaggs` function - * together with the agg configs returned from other columns. - */ - toEsAggsConfig: (column: C, columnId: string, indexPattern: IndexPattern) => unknown; /** * Returns true if the `column` can also be used on `newIndexPattern`. * If this function returns false, the column is removed when switching index pattern @@ -138,7 +149,7 @@ interface BaseOperationDefinitionProps { } interface BaseBuildColumnArgs { - columns: Partial>; + columns: Record; indexPattern: IndexPattern; } @@ -156,7 +167,12 @@ interface FieldlessOperationDefinition { * Returns the meta data of the operation if applied. Undefined * if the field is not applicable. */ - getPossibleOperation: () => OperationMetadata | undefined; + getPossibleOperation: () => OperationMetadata; + /** + * Function turning a column into an agg config passed to the `esaggs` function + * together with the agg configs returned from other columns. + */ + toEsAggsConfig: (column: C, columnId: string, indexPattern: IndexPattern) => unknown; } interface FieldBasedOperationDefinition { @@ -167,7 +183,7 @@ interface FieldBasedOperationDefinition { */ getPossibleOperationForField: (field: IndexPatternField) => OperationMetadata | undefined; /** - * Builds the column object for the given parameters. Should include default p + * Builds the column object for the given parameters. */ buildColumn: ( arg: BaseBuildColumnArgs & { @@ -191,11 +207,77 @@ interface FieldBasedOperationDefinition { * @param field The field that the user changed to. */ onFieldChange: (oldColumn: C, field: IndexPatternField) => C; + /** + * Function turning a column into an agg config passed to the `esaggs` function + * together with the agg configs returned from other columns. + */ + toEsAggsConfig: (column: C, columnId: string, indexPattern: IndexPattern) => unknown; +} + +export interface RequiredReference { + // Limit the input types, usually used to prevent other references from being used + input: Array; + // Function which is used to determine if the reference is bucketed, or if it's a number + validateMetadata: (metadata: OperationMetadata) => boolean; + // Do not use specificOperations unless you need to limit to only one or two exact + // operation types. The main use case is Cumulative Sum, where we need to only take the + // sum of Count or sum of Sum. + specificOperations?: OperationType[]; +} + +// Full reference uses one or more reference operations which are visible to the user +// Partial reference is similar except that it uses the field selector +interface FullReferenceOperationDefinition { + input: 'fullReference'; + /** + * The filters provided here are used to construct the UI, transition correctly + * between operations, and validate the configuration. + */ + requiredReferences: RequiredReference[]; + + /** + * The type of UI that is shown in the editor for this function: + * - full: List of sub-functions and fields + * - field: List of fields, selects first operation per field + */ + selectionStyle: 'full' | 'field'; + + /** + * Builds the column object for the given parameters. Should include default p + */ + buildColumn: ( + arg: BaseBuildColumnArgs & { + columnOrder: string[]; + referenceIds: string[]; + previousColumn?: IndexPatternColumn; + } + ) => ReferenceBasedIndexPatternColumn & C; + /** + * Returns the meta data of the operation if applied. Undefined + * if the field is not applicable. + */ + getPossibleOperation: () => OperationMetadata; + /** + * A chain of expression functions which will transform the table + */ + toExpression: ( + layer: IndexPatternLayer, + columnId: string, + indexPattern: IndexPattern + ) => ExpressionFunctionAST[]; + /** + * Validate that the operation has the right preconditions in the state. For example: + * + * - Requires a date histogram operation somewhere before it in order + * - Missing references + */ + getErrorMessage?: (layer: IndexPatternLayer, columnId: string) => string[] | undefined; } interface OperationDefinitionMap { field: FieldBasedOperationDefinition; none: FieldlessOperationDefinition; + fullReference: FullReferenceOperationDefinition; } /** @@ -220,7 +302,8 @@ export type OperationType = typeof internalOperationDefinitions[number]['type']; */ export type GenericOperationDefinition = | OperationDefinition - | OperationDefinition; + | OperationDefinition + | OperationDefinition; /** * List of all available operation definitions diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx index 37a7ef8ee2563..96df72ba8b7c1 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/metrics.tsx @@ -52,6 +52,8 @@ function buildMetricOperation>({ (!newField.aggregationRestrictions || newField.aggregationRestrictions![type]) ); }, + getDefaultLabel: (column, indexPattern, columns) => + ofName(indexPattern.getFieldByName(column.sourceField)!.displayName), buildColumn: ({ field, previousColumn }) => ({ label: ofName(field.displayName), dataType: 'number', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx index b1cb2312d5bb8..d2456e1c8d375 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/ranges/ranges.tsx @@ -122,9 +122,11 @@ export const rangeOperation: OperationDefinition + indexPattern.getFieldByName(column.sourceField)!.displayName, buildColumn({ field }) { return { - label: field.name, + label: field.displayName, dataType: 'number', // string for Range operationType: 'range', sourceField: field.name, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index ddc473a5c588d..e11329b8e50a9 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -17,7 +17,7 @@ import { EuiText, } from '@elastic/eui'; import { IndexPatternColumn } from '../../../indexpattern'; -import { updateColumnParam } from '../../layer_helpers'; +import { updateColumnParam, isReferenced } from '../../layer_helpers'; import { DataType } from '../../../../types'; import { OperationDefinition } from '../index'; import { FieldBasedIndexPatternColumn } from '../column_types'; @@ -84,7 +84,20 @@ export const termsOperation: OperationDefinition column && isSortableByColumn(column)) + .filter( + ([columnId, column]) => + column && + !column.isBucketed && + !isReferenced( + // Fake layer, but works with real columns + { + columns, + columnOrder: [], + indexPatternId: '', + }, + columnId + ) + ) .map(([id]) => id)[0]; const previousBucketsLength = Object.values(columns).filter((col) => col && col.isBucketed) @@ -131,6 +144,8 @@ export const termsOperation: OperationDefinition + ofName(indexPattern.getFieldByName(column.sourceField)!.displayName), onFieldChange: (oldColumn, field) => { const newParams = { ...oldColumn.params }; if ('format' in newParams && field.type !== 'number') { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts index f0e02c7ff0faf..3ad9a1e5b3674 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/index.ts @@ -6,4 +6,11 @@ export * from './operations'; export * from './layer_helpers'; -export { OperationType, IndexPatternColumn, FieldBasedIndexPatternColumn } from './definitions'; +export { + OperationType, + IndexPatternColumn, + FieldBasedIndexPatternColumn, + IncompleteColumn, +} from './definitions'; + +export { createMockedReferenceOperation } from './mocks'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index e1a31dc274837..f983a6a334d53 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import type { OperationMetadata } from '../../types'; import { insertNewColumn, replaceColumn, @@ -11,16 +12,20 @@ import { getColumnOrder, deleteColumn, updateLayerIndexPattern, + getErrorMessages, } from './layer_helpers'; import { operationDefinitionMap, OperationType } from '../operations'; import { TermsIndexPatternColumn } from './definitions/terms'; import { DateHistogramIndexPatternColumn } from './definitions/date_histogram'; import { AvgIndexPatternColumn } from './definitions/metrics'; -import { IndexPattern, IndexPatternPrivateState, IndexPatternLayer } from '../types'; +import type { IndexPattern, IndexPatternPrivateState, IndexPatternLayer } from '../types'; import { documentField } from '../document_field'; import { getFieldByNameFactory } from '../pure_helpers'; +import { generateId } from '../../id_generator'; +import { createMockedReferenceOperation } from './mocks'; jest.mock('../operations'); +jest.mock('../../id_generator'); const indexPatternFields = [ { @@ -74,10 +79,22 @@ const indexPattern = { timeFieldName: 'timestamp', hasRestrictions: false, fields: indexPatternFields, - getFieldByName: getFieldByNameFactory(indexPatternFields), + getFieldByName: getFieldByNameFactory([...indexPatternFields, documentField]), }; describe('state_helpers', () => { + beforeEach(() => { + let count = 0; + (generateId as jest.Mock).mockImplementation(() => `id${++count}`); + + // @ts-expect-error we are inserting an invalid type + operationDefinitionMap.testReference = createMockedReferenceOperation(); + }); + + afterEach(() => { + delete operationDefinitionMap.testReference; + }); + describe('insertNewColumn', () => { it('should throw for invalid operations', () => { expect(() => { @@ -315,6 +332,110 @@ describe('state_helpers', () => { }) ).toEqual(expect.objectContaining({ columnOrder: ['col1', 'col2', 'col3'] })); }); + + describe('inserting a new reference', () => { + it('should throw if the required references are impossible to match', () => { + // @ts-expect-error this function is not valid + operationDefinitionMap.testReference.requiredReferences = [ + { + input: ['none', 'field'], + validateMetadata: () => false, + specificOperations: [], + }, + ]; + const layer: IndexPatternLayer = { indexPatternId: '1', columnOrder: [], columns: {} }; + expect(() => { + insertNewColumn({ + layer, + indexPattern, + columnId: 'col2', + op: 'testReference' as OperationType, + }); + }).toThrow(); + }); + + it('should leave the references empty if too ambiguous', () => { + const layer: IndexPatternLayer = { indexPatternId: '1', columnOrder: [], columns: {} }; + const result = insertNewColumn({ + layer, + indexPattern, + columnId: 'col2', + op: 'testReference' as OperationType, + }); + + expect(operationDefinitionMap.testReference.buildColumn).toHaveBeenCalledWith( + expect.objectContaining({ + referenceIds: ['id1'], + }) + ); + expect(result).toEqual( + expect.objectContaining({ + columns: { + col2: expect.objectContaining({ references: ['id1'] }), + }, + }) + ); + }); + + it('should create an operation if there is exactly one possible match', () => { + // There is only one operation with `none` as the input type + // @ts-expect-error this function is not valid + operationDefinitionMap.testReference.requiredReferences = [ + { + input: ['none'], + validateMetadata: () => true, + }, + ]; + const layer: IndexPatternLayer = { indexPatternId: '1', columnOrder: [], columns: {} }; + const result = insertNewColumn({ + layer, + indexPattern, + columnId: 'col1', + // @ts-expect-error invalid type + op: 'testReference', + }); + expect(result.columnOrder).toEqual(['id1', 'col1']); + expect(result.columns).toEqual( + expect.objectContaining({ + id1: expect.objectContaining({ operationType: 'filters' }), + col1: expect.objectContaining({ references: ['id1'] }), + }) + ); + }); + + it('should create a referenced column if the ID is being used as a reference', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + dataType: 'number', + isBucketed: false, + + // @ts-expect-error only in test + operationType: 'testReference', + references: ['ref1'], + }, + }, + }; + expect( + insertNewColumn({ + layer, + indexPattern, + columnId: 'ref1', + op: 'count', + field: documentField, + }) + ).toEqual( + expect.objectContaining({ + columns: { + col1: expect.objectContaining({ references: ['ref1'] }), + ref1: expect.objectContaining({}), + }, + }) + ); + }); + }); }); describe('replaceColumn', () => { @@ -655,10 +776,301 @@ describe('state_helpers', () => { }), }); }); + + it('should not wrap the previous operation when switching to reference', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + label: 'Count', + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + sourceField: 'Records', + operationType: 'count' as const, + }, + }, + }; + const result = replaceColumn({ + layer, + indexPattern, + columnId: 'col1', + op: 'testReference' as OperationType, + }); + + expect(operationDefinitionMap.testReference.buildColumn).toHaveBeenCalledWith( + expect.objectContaining({ + referenceIds: ['id1'], + }) + ); + expect(result.columns).toEqual( + expect.objectContaining({ + col1: expect.objectContaining({ operationType: 'testReference' }), + }) + ); + }); + + it('should delete the previous references and reset to default values when going from reference to no-input', () => { + // @ts-expect-error this function is not valid + operationDefinitionMap.testReference.requiredReferences = [ + { + input: ['none'], + validateMetadata: () => true, + }, + ]; + const expectedCol = { + dataType: 'string' as const, + isBucketed: true, + + operationType: 'filters' as const, + params: { + // These filters are reset + filters: [{ input: { query: 'field: true', language: 'kuery' }, label: 'Custom label' }], + }, + }; + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + ...expectedCol, + label: 'Custom label', + customLabel: true, + }, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }; + expect( + replaceColumn({ + layer, + indexPattern, + columnId: 'col2', + op: 'filters', + }) + ).toEqual( + expect.objectContaining({ + columnOrder: ['col2'], + columns: { + col2: { + ...expectedCol, + label: 'Filters', + scale: 'ordinal', // added in buildColumn + params: { + filters: [{ input: { query: '', language: 'kuery' }, label: '' }], + }, + }, + }, + }) + ); + }); + + it('should delete the inner references when switching away from reference to field-based operation', () => { + const expectedCol = { + label: 'Count of records', + dataType: 'number' as const, + isBucketed: false, + + operationType: 'count' as const, + sourceField: 'Records', + }; + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: expectedCol, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }; + expect( + replaceColumn({ + layer, + indexPattern, + columnId: 'col2', + op: 'count', + field: documentField, + }) + ).toEqual( + expect.objectContaining({ + columnOrder: ['col2'], + columns: { + col2: expect.objectContaining(expectedCol), + }, + }) + ); + }); + + it('should reset when switching from one reference to another', () => { + operationDefinitionMap.secondTest = { + input: 'fullReference', + displayName: 'Reference test 2', + // @ts-expect-error this type is not statically available + type: 'secondTest', + requiredReferences: [ + { + // Any numeric metric that isn't also a reference + input: ['none', 'field'], + validateMetadata: (meta: OperationMetadata) => + meta.dataType === 'number' && !meta.isBucketed, + }, + ], + // @ts-expect-error don't want to define valid arguments + buildColumn: jest.fn((args) => { + return { + label: 'Test reference', + isBucketed: false, + dataType: 'number', + + operationType: 'secondTest', + references: args.referenceIds, + }; + }), + isTransferable: jest.fn(), + toExpression: jest.fn().mockReturnValue([]), + getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), + }; + + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Count', + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + + operationType: 'count' as const, + sourceField: 'Records', + }, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }; + expect( + replaceColumn({ + layer, + indexPattern, + columnId: 'col2', + // @ts-expect-error not statically available + op: 'secondTest', + }) + ).toEqual( + expect.objectContaining({ + columnOrder: ['col2'], + columns: { + col2: expect.objectContaining({ references: ['id1'] }), + }, + incompleteColumns: {}, + }) + ); + + delete operationDefinitionMap.secondTest; + }); + + it('should allow making a replacement on an operation that is being referenced, even if it ends up invalid', () => { + // @ts-expect-error this function is not valid + operationDefinitionMap.testReference.requiredReferences = [ + { + input: ['field'], + validateMetadata: (meta: OperationMetadata) => meta.dataType === 'number', + specificOperations: ['sum'], + }, + ]; + + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Asdf', + customLabel: true, + dataType: 'number' as const, + isBucketed: false, + + operationType: 'sum' as const, + sourceField: 'bytes', + }, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }; + expect( + replaceColumn({ + layer, + indexPattern, + columnId: 'col1', + op: 'count', + field: documentField, + }) + ).toEqual( + expect.objectContaining({ + columnOrder: ['col1', 'col2'], + columns: { + col1: expect.objectContaining({ + sourceField: 'Records', + operationType: 'count', + }), + col2: expect.objectContaining({ references: ['col1'] }), + }, + }) + ); + }); }); describe('deleteColumn', () => { - it('should remove column', () => { + it('should clear incomplete columns when column is already empty', () => { + expect( + deleteColumn({ + layer: { + indexPatternId: '1', + columnOrder: [], + columns: {}, + incompleteColumns: { + col1: { sourceField: 'test' }, + }, + }, + columnId: 'col1', + }) + ).toEqual({ + indexPatternId: '1', + columnOrder: [], + columns: {}, + incompleteColumns: {}, + }); + }); + + it('should remove column and any incomplete state', () => { const termsColumn: TermsIndexPatternColumn = { label: 'Top values of source', dataType: 'string', @@ -682,25 +1094,33 @@ describe('state_helpers', () => { columns: { col1: termsColumn, col2: { - label: 'Count', + label: 'Count of records', dataType: 'number', isBucketed: false, sourceField: 'Records', operationType: 'count', }, }, + incompleteColumns: { + col2: { sourceField: 'other' }, + }, }, columnId: 'col2', - }).columns + }) ).toEqual({ - col1: { - ...termsColumn, - params: { - ...termsColumn.params, - orderBy: { type: 'alphabetical' }, - orderDirection: 'asc', + indexPatternId: '1', + columnOrder: ['col1'], + columns: { + col1: { + ...termsColumn, + params: { + ...termsColumn.params, + orderBy: { type: 'alphabetical' }, + orderDirection: 'asc', + }, }, }, + incompleteColumns: {}, }); }); @@ -742,6 +1162,73 @@ describe('state_helpers', () => { col1: termsColumn, }); }); + + it('should delete the column and all of its references', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2'], + columns: { + col1: { + label: 'Count', + dataType: 'number', + isBucketed: false, + + operationType: 'count', + sourceField: 'Records', + }, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + }, + }; + expect(deleteColumn({ layer, columnId: 'col2' })).toEqual( + expect.objectContaining({ columnOrder: [], columns: {} }) + ); + }); + + it('should recursively delete references', () => { + const layer: IndexPatternLayer = { + indexPatternId: '1', + columnOrder: ['col1', 'col2', 'col3'], + columns: { + col1: { + label: 'Count', + dataType: 'number', + isBucketed: false, + + operationType: 'count', + sourceField: 'Records', + }, + col2: { + label: 'Test reference', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col1'], + }, + col3: { + label: 'Test reference 2', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error not a valid type + operationType: 'testReference', + references: ['col2'], + }, + }, + }; + expect(deleteColumn({ layer, columnId: 'col3' })).toEqual( + expect.objectContaining({ columnOrder: [], columns: {} }) + ); + }); }); describe('updateColumnParam', () => { @@ -1141,4 +1628,67 @@ describe('state_helpers', () => { }); }); }); + + describe('getErrorMessages', () => { + it('should collect errors from the operation definitions', () => { + const mock = jest.fn().mockReturnValue(['error 1']); + // @ts-expect-error not statically analyzed + operationDefinitionMap.testReference.getErrorMessage = mock; + const errors = getErrorMessages({ + indexPatternId: '1', + columnOrder: [], + columns: { + col1: + // @ts-expect-error not statically analyzed + { operationType: 'testReference', references: [] }, + }, + }); + expect(mock).toHaveBeenCalled(); + expect(errors).toHaveLength(1); + }); + + it('should identify missing references', () => { + const errors = getErrorMessages({ + indexPatternId: '1', + columnOrder: [], + columns: { + col1: + // @ts-expect-error not statically analyzed yet + { operationType: 'testReference', references: ['ref1', 'ref2'] }, + }, + }); + expect(errors).toHaveLength(2); + }); + + it('should identify references that are no longer valid', () => { + // There is only one operation with `none` as the input type + // @ts-expect-error this function is not valid + operationDefinitionMap.testReference.requiredReferences = [ + { + input: ['none'], + validateMetadata: () => true, + }, + ]; + + const errors = getErrorMessages({ + indexPatternId: '1', + columnOrder: [], + columns: { + // @ts-expect-error incomplete operation + ref1: { + dataType: 'string', + isBucketed: true, + operationType: 'terms', + }, + col1: { + label: '', + references: ['ref1'], + // @ts-expect-error tests only + operationType: 'testReference', + }, + }, + }); + expect(errors).toHaveLength(1); + }); + }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index f071df1542147..2d532b36d0625 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -5,13 +5,15 @@ */ import _, { partition } from 'lodash'; +import { i18n } from '@kbn/i18n'; import { operationDefinitionMap, operationDefinitions, OperationType, IndexPatternColumn, + RequiredReference, } from './definitions'; -import { +import type { IndexPattern, IndexPatternField, IndexPatternLayer, @@ -19,6 +21,7 @@ import { } from '../types'; import { getSortScoreByPriority } from './operations'; import { mergeLayer } from '../state_helpers'; +import { generateId } from '../../id_generator'; interface ColumnChange { op: OperationType; @@ -54,13 +57,8 @@ export function insertNewColumn({ previousColumn: layer.columns[columnId], }; - // TODO: Reference based operations require more setup to create the references - if (operationDefinition.input === 'none') { const possibleOperation = operationDefinition.getPossibleOperation(); - if (!possibleOperation) { - throw new Error('Tried to create an invalid operation'); - } const isBucketed = Boolean(possibleOperation.isBucketed); if (isBucketed) { return addBucket(layer, operationDefinition.buildColumn(baseOptions), columnId); @@ -69,6 +67,80 @@ export function insertNewColumn({ } } + if (operationDefinition.input === 'fullReference') { + let tempLayer = { ...layer }; + const referenceIds = operationDefinition.requiredReferences.map((validation) => { + const validOperations = Object.values(operationDefinitionMap).filter(({ type }) => + isOperationAllowedAsReference({ validation, operationType: type }) + ); + + if (!validOperations.length) { + throw new Error( + `Can't create reference, ${op} has a validation function which doesn't allow any operations` + ); + } + + const newId = generateId(); + if (validOperations.length === 1) { + const def = validOperations[0]; + + const validFields = + def.input === 'field' ? indexPattern.fields.filter(def.getPossibleOperationForField) : []; + + if (def.input === 'none') { + tempLayer = insertNewColumn({ + layer: tempLayer, + columnId: newId, + op: def.type, + indexPattern, + }); + } else if (validFields.length === 1) { + // Recursively update the layer for each new reference + tempLayer = insertNewColumn({ + layer: tempLayer, + columnId: newId, + op: def.type, + indexPattern, + field: validFields[0], + }); + } else { + tempLayer = { + ...tempLayer, + incompleteColumns: { + ...tempLayer.incompleteColumns, + [newId]: { operationType: def.type }, + }, + }; + } + } + return newId; + }); + + const possibleOperation = operationDefinition.getPossibleOperation(); + const isBucketed = Boolean(possibleOperation.isBucketed); + if (isBucketed) { + return addBucket( + tempLayer, + operationDefinition.buildColumn({ + ...baseOptions, + columnOrder: layer.columnOrder, + referenceIds, + }), + columnId + ); + } else { + return addMetric( + tempLayer, + operationDefinition.buildColumn({ + ...baseOptions, + columnOrder: layer.columnOrder, + referenceIds, + }), + columnId + ); + } + } + if (!field) { throw new Error(`Invariant error: ${operationDefinition.type} operation requires field`); } @@ -99,8 +171,9 @@ export function replaceColumn({ throw new Error(`Can't replace column because there is no prior column`); } - const isNewOperation = Boolean(op) && op !== previousColumn.operationType; - const operationDefinition = operationDefinitionMap[op || previousColumn.operationType]; + const isNewOperation = op !== previousColumn.operationType; + const operationDefinition = operationDefinitionMap[op]; + const previousDefinition = operationDefinitionMap[previousColumn.operationType]; if (!operationDefinition) { throw new Error('No suitable operation found for given parameters'); @@ -113,22 +186,48 @@ export function replaceColumn({ }; if (isNewOperation) { - // TODO: Reference based operations require more setup to create the references + let tempLayer = { ...layer }; + + if (previousDefinition.input === 'fullReference') { + // @ts-expect-error references are not statically analyzed + previousColumn.references.forEach((id: string) => { + tempLayer = deleteColumn({ layer: tempLayer, columnId: id }); + }); + } + + if (operationDefinition.input === 'fullReference') { + const referenceIds = operationDefinition.requiredReferences.map(() => generateId()); + + const incompleteColumns = { ...(tempLayer.incompleteColumns || {}) }; + delete incompleteColumns[columnId]; + return { + ...tempLayer, + columns: { + ...tempLayer.columns, + [columnId]: operationDefinition.buildColumn({ + ...baseOptions, + columns: tempLayer.columns, + columnOrder: tempLayer.columnOrder, + referenceIds, + previousColumn, + }), + }, + incompleteColumns, + }; + } if (operationDefinition.input === 'none') { const newColumn = operationDefinition.buildColumn(baseOptions); - if (previousColumn.customLabel) { newColumn.customLabel = true; newColumn.label = previousColumn.label; } + const newColumns = { ...tempLayer.columns, [columnId]: newColumn }; return { - ...layer, - columns: adjustColumnReferencesForChangedColumn( - { ...layer.columns, [columnId]: newColumn }, - columnId - ), + ...tempLayer, + columnOrder: getColumnOrder({ ...tempLayer, columns: newColumns }), + columns: adjustColumnReferencesForChangedColumn(newColumns, columnId), }; } @@ -143,10 +242,10 @@ export function replaceColumn({ newColumn.label = previousColumn.label; } - const newColumns = { ...layer.columns, [columnId]: newColumn }; + const newColumns = { ...tempLayer.columns, [columnId]: newColumn }; return { - ...layer, - columnOrder: getColumnOrder({ ...layer, columns: newColumns }), + ...tempLayer, + columnOrder: getColumnOrder({ ...tempLayer, columns: newColumns }), columns: adjustColumnReferencesForChangedColumn(newColumns, columnId), }; } else if ( @@ -294,14 +393,36 @@ export function deleteColumn({ layer: IndexPatternLayer; columnId: string; }): IndexPatternLayer { + const column = layer.columns[columnId]; + if (!column) { + const newIncomplete = { ...(layer.incompleteColumns || {}) }; + delete newIncomplete[columnId]; + return { + ...layer, + columnOrder: layer.columnOrder.filter((id) => id !== columnId), + incompleteColumns: newIncomplete, + }; + } + + // @ts-expect-error this fails statically because there are no references added + const extraDeletions: string[] = 'references' in column ? column.references : []; + const hypotheticalColumns = { ...layer.columns }; delete hypotheticalColumns[columnId]; - const newLayer = { + let newLayer = { ...layer, columns: adjustColumnReferencesForChangedColumn(hypotheticalColumns, columnId), }; - return { ...newLayer, columnOrder: getColumnOrder(newLayer) }; + + extraDeletions.forEach((id) => { + newLayer = deleteColumn({ layer: newLayer, columnId: id }); + }); + + const newIncomplete = { ...(newLayer.incompleteColumns || {}) }; + delete newIncomplete[columnId]; + + return { ...newLayer, columnOrder: getColumnOrder(newLayer), incompleteColumns: newIncomplete }; } export function getColumnOrder(layer: IndexPatternLayer): string[] { @@ -342,3 +463,116 @@ export function updateLayerIndexPattern( columnOrder: newColumnOrder, }; } + +/** + * Collects all errors from the columns in the layer, for display in the workspace. This includes: + * + * - All columns have complete references + * - All column references are valid + * - All prerequisites are met + */ +export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined { + const errors: string[] = []; + + Object.entries(layer.columns).forEach(([columnId, column]) => { + const def = operationDefinitionMap[column.operationType]; + if (def.input === 'fullReference' && def.getErrorMessage) { + errors.push(...(def.getErrorMessage(layer, columnId) ?? [])); + } + + if ('references' in column) { + // @ts-expect-error references are not statically analyzed yet + column.references.forEach((referenceId, index) => { + if (!layer.columns[referenceId]) { + errors.push( + i18n.translate('xpack.lens.indexPattern.missingReferenceError', { + defaultMessage: 'Dimension {dimensionLabel} is incomplete', + values: { + // @ts-expect-error references are not statically analyzed yet + dimensionLabel: column.label, + }, + }) + ); + } else { + const referenceColumn = layer.columns[referenceId]!; + const requirements = + // @ts-expect-error not statically analyzed + operationDefinitionMap[column.operationType].requiredReferences[index]; + const isValid = isColumnValidAsReference({ + validation: requirements, + column: referenceColumn, + }); + + if (!isValid) { + errors.push( + i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { + defaultMessage: 'Dimension {dimensionLabel} does not have a valid configuration', + values: { + // @ts-expect-error references are not statically analyzed yet + dimensionLabel: column.label, + }, + }) + ); + } + } + }); + } + }); + + return errors.length ? errors : undefined; +} + +export function isReferenced(layer: IndexPatternLayer, columnId: string): boolean { + const allReferences = Object.values(layer.columns).flatMap((col) => + 'references' in col + ? // @ts-expect-error not statically analyzed + col.references + : [] + ); + return allReferences.includes(columnId); +} + +function isColumnValidAsReference({ + column, + validation, +}: { + column: IndexPatternColumn; + validation: RequiredReference; +}): boolean { + if (!column) return false; + const operationType = column.operationType; + const operationDefinition = operationDefinitionMap[operationType]; + return ( + validation.input.includes(operationDefinition.input) && + (!validation.specificOperations || validation.specificOperations.includes(operationType)) && + validation.validateMetadata(column) + ); +} + +function isOperationAllowedAsReference({ + operationType, + validation, + field, +}: { + operationType: OperationType; + validation: RequiredReference; + field?: IndexPatternField; +}): boolean { + const operationDefinition = operationDefinitionMap[operationType]; + + let hasValidMetadata = true; + if (field && operationDefinition.input === 'field') { + const metadata = operationDefinition.getPossibleOperationForField(field); + hasValidMetadata = Boolean(metadata) && validation.validateMetadata(metadata!); + } else if (operationDefinition.input !== 'field') { + const metadata = operationDefinition.getPossibleOperation(); + hasValidMetadata = Boolean(metadata) && validation.validateMetadata(metadata!); + } else { + // TODO: How can we validate the metadata without a specific field? + } + return ( + validation.input.includes(operationDefinition.input) && + (!validation.specificOperations || validation.specificOperations.includes(operationType)) && + hasValidMetadata + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts new file mode 100644 index 0000000000000..c3f7dac03ada3 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/mocks.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import type { OperationMetadata } from '../../types'; +import type { OperationType } from './definitions'; + +export const createMockedReferenceOperation = () => { + return { + input: 'fullReference', + displayName: 'Reference test', + type: 'testReference' as OperationType, + selectionStyle: 'full', + requiredReferences: [ + { + // Any numeric metric that isn't also a reference + input: ['none', 'field'], + validateMetadata: (meta: OperationMetadata) => + meta.dataType === 'number' && !meta.isBucketed, + }, + ], + buildColumn: jest.fn((args) => { + return { + label: 'Test reference', + isBucketed: false, + dataType: 'number', + + operationType: 'testReference', + references: args.referenceIds, + }; + }), + isTransferable: jest.fn(), + toExpression: jest.fn().mockReturnValue([]), + getPossibleOperation: jest.fn().mockReturnValue({ dataType: 'number', isBucketed: false }), + getDefaultLabel: jest.fn().mockReturnValue('Default label'), + }; +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts index 8d489df366088..58685fa494a04 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.ts @@ -87,6 +87,10 @@ type OperationFieldTuple = | { type: 'none'; operationType: OperationType; + } + | { + type: 'fullReference'; + operationType: OperationType; }; /** @@ -162,6 +166,11 @@ export function getAvailableOperationsByMetadata(indexPattern: IndexPattern) { }, operationDefinition.getPossibleOperation() ); + } else if (operationDefinition.input === 'fullReference') { + addToMap( + { type: 'fullReference', operationType: operationDefinition.type }, + operationDefinition.getPossibleOperation() + ); } }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts index ea7aa62054e5c..5b66d4aae77ab 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/to_expression.ts @@ -7,32 +7,29 @@ import { Ast, ExpressionFunctionAST } from '@kbn/interpreter/common'; import { IndexPatternColumn } from './indexpattern'; import { operationDefinitionMap } from './operations'; -import { IndexPattern, IndexPatternPrivateState } from './types'; +import { IndexPattern, IndexPatternPrivateState, IndexPatternLayer } from './types'; import { OriginalColumn } from './rename_columns'; import { dateHistogramOperation } from './operations/definitions'; -function getExpressionForLayer( - indexPattern: IndexPattern, - columns: Record, - columnOrder: string[] -): Ast | null { +function getExpressionForLayer(layer: IndexPatternLayer, indexPattern: IndexPattern): Ast | null { + const { columns, columnOrder } = layer; + if (columnOrder.length === 0) { return null; } - function getEsAggsConfig(column: C, columnId: string) { - return operationDefinitionMap[column.operationType].toEsAggsConfig( - column, - columnId, - indexPattern - ); - } - const columnEntries = columnOrder.map((colId) => [colId, columns[colId]] as const); if (columnEntries.length) { - const aggs = columnEntries.map(([colId, col]) => { - return getEsAggsConfig(col, colId); + const aggs: unknown[] = []; + const expressions: ExpressionFunctionAST[] = []; + columnEntries.forEach(([colId, col]) => { + const def = operationDefinitionMap[col.operationType]; + if (def.input === 'fullReference') { + expressions.push(...def.toExpression(layer, colId, indexPattern)); + } else { + aggs.push(def.toEsAggsConfig(col, colId, indexPattern)); + } }); const idMap = columnEntries.reduce((currentIdMap, [colId, column], index) => { @@ -119,6 +116,7 @@ function getExpressionForLayer( }, }, ...formatterOverrides, + ...expressions, ], }; } @@ -129,9 +127,8 @@ function getExpressionForLayer( export function toExpression(state: IndexPatternPrivateState, layerId: string) { if (state.layers[layerId]) { return getExpressionForLayer( - state.indexPatterns[state.layers[layerId].indexPatternId], - state.layers[layerId].columns, - state.layers[layerId].columnOrder + state.layers[layerId], + state.indexPatterns[state.layers[layerId].indexPatternId] ); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts index 1e6fc5a5806b5..e4958da471417 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/types.ts @@ -5,7 +5,7 @@ */ import { IFieldType } from 'src/plugins/data/common'; -import { IndexPatternColumn } from './operations'; +import { IndexPatternColumn, IncompleteColumn } from './operations'; import { IndexPatternAggRestrictions } from '../../../../../src/plugins/data/public'; export interface IndexPattern { @@ -35,6 +35,8 @@ export interface IndexPatternLayer { columns: Record; // Each layer is tied to the index pattern that created it indexPatternId: string; + // Partial columns represent the temporary invalid states + incompleteColumns?: Record; } export interface IndexPatternPersistedState { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts index d0ea81d135156..01b834610eb1a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/utils.ts @@ -42,11 +42,11 @@ export function isDraggedField(fieldCandidate: unknown): fieldCandidate is Dragg ); } -export function hasInvalidReference(state: IndexPatternPrivateState) { - return getInvalidReferences(state).length > 0; +export function hasInvalidFields(state: IndexPatternPrivateState) { + return getInvalidLayers(state).length > 0; } -export function getInvalidReferences(state: IndexPatternPrivateState) { +export function getInvalidLayers(state: IndexPatternPrivateState) { return Object.values(state.layers).filter((layer) => { return layer.columnOrder.some((columnId) => { const column = layer.columns[columnId]; @@ -62,7 +62,7 @@ export function getInvalidReferences(state: IndexPatternPrivateState) { }); } -export function getInvalidFieldReferencesForLayer( +export function getInvalidFieldsForLayer( layers: IndexPatternLayer[], indexPatternMap: Record ) { From 104595f16ad5c2295f2cf81aeb16669a735a8dd6 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Wed, 18 Nov 2020 15:59:54 -0500 Subject: [PATCH 02/12] Update from review feedback --- .../definitions/date_histogram.test.tsx | 6 +- .../operations/definitions/index.ts | 3 +- .../operations/definitions/terms/index.tsx | 22 ++--- .../definitions/terms/terms.test.tsx | 28 +++--- .../operations/layer_helpers.test.ts | 54 ++++++++++++ .../operations/layer_helpers.ts | 88 +++++++++++++------ 6 files changed, 142 insertions(+), 59 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx index 7d50c28b7465a..558fab02ad084 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/date_histogram.test.tsx @@ -188,7 +188,7 @@ describe('date_histogram', () => { describe('buildColumn', () => { it('should create column object with auto interval for primary time field', () => { const column = dateHistogramOperation.buildColumn({ - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, indexPattern: createMockedIndexPattern(), field: { name: 'timestamp', @@ -204,7 +204,7 @@ describe('date_histogram', () => { it('should create column object with auto interval for non-primary time fields', () => { const column = dateHistogramOperation.buildColumn({ - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, indexPattern: createMockedIndexPattern(), field: { name: 'start_date', @@ -220,7 +220,7 @@ describe('date_histogram', () => { it('should create column object with restrictions', () => { const column = dateHistogramOperation.buildColumn({ - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, indexPattern: createMockedIndexPattern(), field: { name: 'timestamp', diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 564a4ca340b40..0e7e125944e71 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -149,7 +149,7 @@ interface BaseOperationDefinitionProps { } interface BaseBuildColumnArgs { - columns: Record; + layer: IndexPatternLayer; indexPattern: IndexPattern; } @@ -247,7 +247,6 @@ interface FullReferenceOperationDefinition { */ buildColumn: ( arg: BaseBuildColumnArgs & { - columnOrder: string[]; referenceIds: string[]; previousColumn?: IndexPatternColumn; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx index e11329b8e50a9..7c69a70c09351 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/index.tsx @@ -82,26 +82,16 @@ export const termsOperation: OperationDefinition - column && - !column.isBucketed && - !isReferenced( - // Fake layer, but works with real columns - { - columns, - columnOrder: [], - indexPatternId: '', - }, - columnId - ) + ([columnId, column]) => column && !column.isBucketed && !isReferenced(layer, columnId) ) .map(([id]) => id)[0]; - const previousBucketsLength = Object.values(columns).filter((col) => col && col.isBucketed) - .length; + const previousBucketsLength = Object.values(layer.columns).filter( + (col) => col && col.isBucketed + ).length; return { label: ofName(field.displayName), diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index bba7bda308b72..e43c7bbd2f72e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -270,7 +270,7 @@ describe('terms', () => { name: 'test', displayName: 'test', }, - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, }); expect(termsColumn.dataType).toEqual('boolean'); }); @@ -285,7 +285,7 @@ describe('terms', () => { name: 'test', displayName: 'test', }, - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, }); expect(termsColumn.params.otherBucket).toEqual(true); }); @@ -300,7 +300,7 @@ describe('terms', () => { name: 'test', displayName: 'test', }, - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, }); expect(termsColumn.params.otherBucket).toEqual(false); }); @@ -308,14 +308,18 @@ describe('terms', () => { it('should use existing metric column as order column', () => { const termsColumn = termsOperation.buildColumn({ indexPattern: createMockedIndexPattern(), - columns: { - col1: { - label: 'Count', - dataType: 'number', - isBucketed: false, - sourceField: 'Records', - operationType: 'count', + layer: { + columns: { + col1: { + label: 'Count', + dataType: 'number', + isBucketed: false, + sourceField: 'Records', + operationType: 'count', + }, }, + columnOrder: [], + indexPatternId: '', }, field: { aggregatable: true, @@ -335,7 +339,7 @@ describe('terms', () => { it('should use the default size when there is an existing bucket', () => { const termsColumn = termsOperation.buildColumn({ indexPattern: createMockedIndexPattern(), - columns: state.layers.first.columns, + layer: state.layers.first, field: { aggregatable: true, searchable: true, @@ -350,7 +354,7 @@ describe('terms', () => { it('should use a size of 5 when there are no other buckets', () => { const termsColumn = termsOperation.buildColumn({ indexPattern: createMockedIndexPattern(), - columns: {}, + layer: { columns: {}, columnOrder: [], indexPatternId: '' }, field: { aggregatable: true, searchable: true, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts index f983a6a334d53..0d103a766c23a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.test.ts @@ -1400,6 +1400,60 @@ describe('state_helpers', () => { }) ).toEqual(['col1', 'col3', 'col2']); }); + + it('should correctly sort references to other references', () => { + expect( + getColumnOrder({ + columnOrder: [], + indexPatternId: '', + columns: { + bucket: { + label: 'Top values of category', + dataType: 'string', + isBucketed: true, + + // Private + operationType: 'terms', + sourceField: 'category', + params: { + size: 5, + orderBy: { + type: 'alphabetical', + }, + orderDirection: 'asc', + }, + }, + metric: { + label: 'Average of bytes', + dataType: 'number', + isBucketed: false, + + // Private + operationType: 'avg', + sourceField: 'bytes', + }, + ref2: { + label: 'Ref2', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error only for testing + operationType: 'testReference', + references: ['ref1'], + }, + ref1: { + label: 'Ref', + dataType: 'number', + isBucketed: false, + + // @ts-expect-error only for testing + operationType: 'testReference', + references: ['bucket'], + }, + }, + }) + ).toEqual(['bucket', 'metric', 'ref1', 'ref2']); + }); }); describe('updateLayerIndexPattern', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 2d532b36d0625..1495a876a2c8e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -38,6 +38,8 @@ export function insertOrReplaceColumn(args: ColumnChange): IndexPatternLayer { return insertNewColumn(args); } +// Insert a column into an empty ID. The field parameter is required when constructing +// a field-based operation, but will cause the function to fail for any other type of operation. export function insertNewColumn({ op, layer, @@ -51,25 +53,34 @@ export function insertNewColumn({ throw new Error('No suitable operation found for given parameters'); } - const baseOptions = { - columns: layer.columns, - indexPattern, - previousColumn: layer.columns[columnId], - }; + if (layer.columns[columnId]) { + throw new Error(`Can't insert a column with an ID that is already in use`); + } + + const baseOptions = { indexPattern, previousColumn: layer.columns[columnId] }; if (operationDefinition.input === 'none') { + if (field) { + throw new Error(`Can't create operation ${op} with the provided field ${field.name}`); + } const possibleOperation = operationDefinition.getPossibleOperation(); const isBucketed = Boolean(possibleOperation.isBucketed); if (isBucketed) { - return addBucket(layer, operationDefinition.buildColumn(baseOptions), columnId); + return addBucket(layer, operationDefinition.buildColumn({ ...baseOptions, layer }), columnId); } else { - return addMetric(layer, operationDefinition.buildColumn(baseOptions), columnId); + return addMetric(layer, operationDefinition.buildColumn({ ...baseOptions, layer }), columnId); } } if (operationDefinition.input === 'fullReference') { + if (field) { + throw new Error(`Reference-based operations can't take a field as input when creating`); + } let tempLayer = { ...layer }; const referenceIds = operationDefinition.requiredReferences.map((validation) => { + // TODO: This logic is too simple because it's not using fields. Once we have + // access to the operationSupportMatrix, we should validate the metadata against + // the possible fields const validOperations = Object.values(operationDefinitionMap).filter(({ type }) => isOperationAllowedAsReference({ validation, operationType: type }) ); @@ -123,7 +134,7 @@ export function insertNewColumn({ tempLayer, operationDefinition.buildColumn({ ...baseOptions, - columnOrder: layer.columnOrder, + layer: tempLayer, referenceIds, }), columnId @@ -133,7 +144,7 @@ export function insertNewColumn({ tempLayer, operationDefinition.buildColumn({ ...baseOptions, - columnOrder: layer.columnOrder, + layer: tempLayer, referenceIds, }), columnId @@ -153,9 +164,17 @@ export function insertNewColumn({ } const isBucketed = Boolean(possibleOperation.isBucketed); if (isBucketed) { - return addBucket(layer, operationDefinition.buildColumn({ ...baseOptions, field }), columnId); + return addBucket( + layer, + operationDefinition.buildColumn({ ...baseOptions, layer, field }), + columnId + ); } else { - return addMetric(layer, operationDefinition.buildColumn({ ...baseOptions, field }), columnId); + return addMetric( + layer, + operationDefinition.buildColumn({ ...baseOptions, layer, field }), + columnId + ); } } @@ -200,24 +219,25 @@ export function replaceColumn({ const incompleteColumns = { ...(tempLayer.incompleteColumns || {}) }; delete incompleteColumns[columnId]; + const newColumns = { + ...tempLayer.columns, + [columnId]: operationDefinition.buildColumn({ + ...baseOptions, + layer: tempLayer, + referenceIds, + previousColumn, + }), + }; return { ...tempLayer, - columns: { - ...tempLayer.columns, - [columnId]: operationDefinition.buildColumn({ - ...baseOptions, - columns: tempLayer.columns, - columnOrder: tempLayer.columnOrder, - referenceIds, - previousColumn, - }), - }, + columnOrder: getColumnOrder({ ...tempLayer, columns: newColumns }), + columns: newColumns, incompleteColumns, }; } if (operationDefinition.input === 'none') { - const newColumn = operationDefinition.buildColumn(baseOptions); + const newColumn = operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer }); if (previousColumn.customLabel) { newColumn.customLabel = true; newColumn.label = previousColumn.label; @@ -235,7 +255,7 @@ export function replaceColumn({ throw new Error(`Invariant error: ${operationDefinition.type} operation requires field`); } - const newColumn = operationDefinition.buildColumn({ ...baseOptions, field }); + const newColumn = operationDefinition.buildColumn({ ...baseOptions, layer: tempLayer, field }); if (previousColumn.customLabel) { newColumn.customLabel = true; @@ -426,12 +446,28 @@ export function deleteColumn({ } export function getColumnOrder(layer: IndexPatternLayer): string[] { - const [aggregations, metrics] = _.partition( + const [direct, referenceBased] = _.partition( Object.entries(layer.columns), - ([id, col]) => col.isBucketed + ([id, col]) => operationDefinitionMap[col.operationType].input !== 'fullReference' ); + // If a reference has another reference as input, put it last in sort order + referenceBased.sort(([idA, a], [idB, b]) => { + // @ts-expect-error not statically analyzed + if ('references' in a && a.references.includes(idB)) { + return 1; + } + // @ts-expect-error not statically analyzed + if ('references' in b && b.references.includes(idA)) { + return -1; + } + return 0; + }); + const [aggregations, metrics] = _.partition(direct, ([, col]) => col.isBucketed); - return aggregations.map(([id]) => id).concat(metrics.map(([id]) => id)); + return aggregations + .map(([id]) => id) + .concat(metrics.map(([id]) => id)) + .concat(referenceBased.map(([id]) => id)); } /** From 963cfc1d0b55a735731391a996c2b3f6bd99683c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 18 Nov 2020 11:51:04 +0100 Subject: [PATCH 03/12] wip --- .../workspace_panel/workspace_panel.tsx | 2 +- .../operations/definitions/cumulative_sum.tsx | 105 ++++++++++++++++++ .../operations/definitions/index.ts | 5 +- 3 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx diff --git a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx index 95aeedbd857ca..6c2c01d944cd9 100644 --- a/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx +++ b/x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx @@ -161,7 +161,7 @@ export function WorkspacePanel({ const expression = useMemo( () => { - if (!configurationValidationError) { + if (!configurationValidationError || configurationValidationError.length === 0) { try { return buildExpression({ visualization: activeVisualization, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx new file mode 100644 index 0000000000000..9980d1af5758f --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { OperationDefinition } from './index'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; +import { IndexPatternLayer } from '../../types'; + +const ofName = (name: string) => { + return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { + defaultMessage: 'Cumulative sum of {name}', + values: { name }, + }); +}; + +export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'cumulative_sum'; + }; + +export const cumulativeSumOperation: OperationDefinition< + CumulativeSumIndexPatternColumn, + 'fullReference' +> = { + type: 'cumulative_sum', + priority: 2, + displayName: i18n.translate('xpack.lens.indexPattern.cumulativeSum', { + defaultMessage: 'Cumulative sum', + }), + input: 'fullReference', + requiredReferences: [ + { + input: ['field'], + specificOperations: ['count', 'sum'], + validateMetadata: (metadata) => + metadata.dataType === 'number' && metadata.isBucketed === false, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: () => { + return i18n.translate('xpack.lens.indexPattern.cumulativeSum', { + defaultMessage: 'Cumulative sum', + }); + }, + toExpression: (layer, columnId) => { + const currentColumn = (layer.columns[columnId] as unknown) as CumulativeSumIndexPatternColumn; + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const dateColumnIndex = buckets.findIndex( + (colId) => layer.columns[colId].operationType === 'date_histogram' + )!; + buckets.splice(dateColumnIndex, 1); + + return [ + { + type: 'function', + function: 'cumulative_sum', + arguments: { + by: buckets, + inputColumnId: [currentColumn.references[0]], + outputColumnId: [columnId], + }, + }, + ]; + }, + buildColumn: ({ referenceIds, previousColumn, columns }) => { + const metric = columns[referenceIds[0]]; + return { + label: ofName(metric.label), + dataType: 'number', + operationType: 'cumulative_sum', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: () => { + return true; + }, + getErrorMessage: (layer: IndexPatternLayer) => { + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const hasDateHistogram = buckets.some( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (hasDateHistogram) { + return undefined; + } + return ['Needs a date histogram to work']; + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index 0e7e125944e71..e0dfb6b248390 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -23,6 +23,7 @@ import { MedianIndexPatternColumn, } from './metrics'; import { dateHistogramOperation, DateHistogramIndexPatternColumn } from './date_histogram'; +import { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum'; import { countOperation, CountIndexPatternColumn } from './count'; import { StateSetter, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -52,7 +53,8 @@ export type IndexPatternColumn = | CardinalityIndexPatternColumn | SumIndexPatternColumn | MedianIndexPatternColumn - | CountIndexPatternColumn; + | CountIndexPatternColumn + | CumulativeSumIndexPatternColumn; export type FieldBasedIndexPatternColumn = Extract; @@ -73,6 +75,7 @@ const internalOperationDefinitions = [ medianOperation, countOperation, rangeOperation, + cumulativeSumOperation, ]; export { termsOperation } from './terms'; From ad6f522379abb32450a0b080f604622252304f4b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 19 Nov 2020 13:39:44 +0100 Subject: [PATCH 04/12] fix problem --- .../operations/definitions/cumulative_sum.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx index 9980d1af5758f..3676de4db33b3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx @@ -31,6 +31,7 @@ export const cumulativeSumOperation: OperationDefinition< defaultMessage: 'Cumulative sum', }), input: 'fullReference', + selectionStyle: 'field', requiredReferences: [ { input: ['field'], @@ -71,8 +72,8 @@ export const cumulativeSumOperation: OperationDefinition< }, ]; }, - buildColumn: ({ referenceIds, previousColumn, columns }) => { - const metric = columns[referenceIds[0]]; + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; return { label: ofName(metric.label), dataType: 'number', @@ -100,6 +101,10 @@ export const cumulativeSumOperation: OperationDefinition< if (hasDateHistogram) { return undefined; } - return ['Needs a date histogram to work']; + return [ + i18n.translate('xpack.lens.indexPattern.cumulativeSum.dateHistogramErrorMessage', { + defaultMessage: 'Needs a date histogram to work', + }), + ]; }, }; From 1c1f69da8ccce78bafb0ee47c803e1193427e4d7 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 19 Nov 2020 14:53:58 +0100 Subject: [PATCH 05/12] fix types --- .../indexpattern_datasource/indexpattern.test.ts | 14 ++++++++++---- .../operations/layer_helpers.ts | 11 +---------- .../operations/operations.test.ts | 4 ++++ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 3cf9bdc3a92f1..106692b73d59a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -660,24 +660,30 @@ describe('IndexPattern Data Source', () => { it('should skip columns that are being referenced', () => { publicAPI = indexPatternDatasource.getPublicAPI({ - state: { + state: ({ layers: { first: { indexPatternId: '1', columnOrder: ['col1', 'col2'], columns: { - // @ts-ignore this is too little information for a real column col1: { dataType: 'number', + operationType: 'sum', + label: 'Sum', + isBucketed: false, + sourceField: 'test', }, col2: { - // @ts-expect-error update once we have a reference operation outside tests + operationType: 'cumulative_sum', + dataType: 'number', + isBucketed: false, + label: 'Cumulative sum', references: ['col1'], }, }, }, }, - }, + } as unknown) as IndexPatternPrivateState, layerId: 'first', }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts index 1495a876a2c8e..58a066c81a1a7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts @@ -424,7 +424,6 @@ export function deleteColumn({ }; } - // @ts-expect-error this fails statically because there are no references added const extraDeletions: string[] = 'references' in column ? column.references : []; const hypotheticalColumns = { ...layer.columns }; @@ -452,11 +451,9 @@ export function getColumnOrder(layer: IndexPatternLayer): string[] { ); // If a reference has another reference as input, put it last in sort order referenceBased.sort(([idA, a], [idB, b]) => { - // @ts-expect-error not statically analyzed if ('references' in a && a.references.includes(idB)) { return 1; } - // @ts-expect-error not statically analyzed if ('references' in b && b.references.includes(idA)) { return -1; } @@ -517,14 +514,12 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined } if ('references' in column) { - // @ts-expect-error references are not statically analyzed yet column.references.forEach((referenceId, index) => { if (!layer.columns[referenceId]) { errors.push( i18n.translate('xpack.lens.indexPattern.missingReferenceError', { defaultMessage: 'Dimension {dimensionLabel} is incomplete', values: { - // @ts-expect-error references are not statically analyzed yet dimensionLabel: column.label, }, }) @@ -544,7 +539,6 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined i18n.translate('xpack.lens.indexPattern.invalidReferenceConfiguration', { defaultMessage: 'Dimension {dimensionLabel} does not have a valid configuration', values: { - // @ts-expect-error references are not statically analyzed yet dimensionLabel: column.label, }, }) @@ -560,10 +554,7 @@ export function getErrorMessages(layer: IndexPatternLayer): string[] | undefined export function isReferenced(layer: IndexPatternLayer, columnId: string): boolean { const allReferences = Object.values(layer.columns).flatMap((col) => - 'references' in col - ? // @ts-expect-error not statically analyzed - col.references - : [] + 'references' in col ? col.references : [] ); return allReferences.includes(columnId); } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index d6f5b10cf64e1..6ea4c49facdff 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -242,6 +242,10 @@ describe('getOperationTypesForField', () => { "operationType": "avg", "type": "field", }, + Object { + "operationType": "cumulative_sum", + "type": "fullReference", + }, Object { "field": "bytes", "operationType": "sum", From 7ef2558b5505f6fbf1ee4680c257626afa74596d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 20 Nov 2020 14:48:17 +0100 Subject: [PATCH 06/12] add other operations --- .../definitions/calculations/counter_rate.tsx | 83 +++++++++++++++++++ .../{ => calculations}/cumulative_sum.tsx | 39 ++------- .../definitions/calculations/derivative.tsx | 82 ++++++++++++++++++ .../definitions/calculations/index.ts | 10 +++ .../calculations/moving_average.tsx | 82 ++++++++++++++++++ .../definitions/calculations/utils.ts | 60 ++++++++++++++ .../operations/definitions/index.ts | 19 ++++- 7 files changed, 340 insertions(+), 35 deletions(-) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx rename x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/{ => calculations}/cumulative_sum.tsx (65%) create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx create mode 100644 x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx new file mode 100644 index 0000000000000..9573968ffc4ad --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name: string) => { + return i18n.translate('xpack.lens.indexPattern.CounterRateOf', { + defaultMessage: 'Counter rate of {name}', + values: { name }, + }); +}; + +export type CounterRateIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'counter_rate'; + }; + +export const counterRateOperation: OperationDefinition< + CounterRateIndexPatternColumn, + 'fullReference' +> = { + type: 'counter_rate', + priority: 2, + displayName: i18n.translate('xpack.lens.indexPattern.counterRate', { + defaultMessage: 'Counter rate', + }), + input: 'fullReference', + selectionStyle: 'field', + requiredReferences: [ + { + input: ['field'], + specificOperations: ['max'], + validateMetadata: (metadata) => + metadata.dataType === 'number' && metadata.isBucketed === false, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: () => { + return i18n.translate('xpack.lens.indexPattern.counterRate', { + defaultMessage: 'Counter rate', + }); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric.label), + dataType: 'number', + operationType: 'counter_rate', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram(layer); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx similarity index 65% rename from x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx rename to x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index 3676de4db33b3..c130549b0593c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -5,9 +5,10 @@ */ import { i18n } from '@kbn/i18n'; -import { OperationDefinition } from './index'; -import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; -import { IndexPatternLayer } from '../../types'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression } from './utils'; +import { OperationDefinition } from '..'; const ofName = (name: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { @@ -53,24 +54,7 @@ export const cumulativeSumOperation: OperationDefinition< }); }, toExpression: (layer, columnId) => { - const currentColumn = (layer.columns[columnId] as unknown) as CumulativeSumIndexPatternColumn; - const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); - const dateColumnIndex = buckets.findIndex( - (colId) => layer.columns[colId].operationType === 'date_histogram' - )!; - buckets.splice(dateColumnIndex, 1); - - return [ - { - type: 'function', - function: 'cumulative_sum', - arguments: { - by: buckets, - inputColumnId: [currentColumn.references[0]], - outputColumnId: [columnId], - }, - }, - ]; + return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum'); }, buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; @@ -94,17 +78,6 @@ export const cumulativeSumOperation: OperationDefinition< return true; }, getErrorMessage: (layer: IndexPatternLayer) => { - const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); - const hasDateHistogram = buckets.some( - (colId) => layer.columns[colId].operationType === 'date_histogram' - ); - if (hasDateHistogram) { - return undefined; - } - return [ - i18n.translate('xpack.lens.indexPattern.cumulativeSum.dateHistogramErrorMessage', { - defaultMessage: 'Needs a date histogram to work', - }), - ]; + return checkForDateHistogram(layer); }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx new file mode 100644 index 0000000000000..b9c65f9633138 --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name: string) => { + return i18n.translate('xpack.lens.indexPattern.derivativeOf', { + defaultMessage: 'Derivative of {name}', + values: { name }, + }); +}; + +export type DerivativeIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'derivative'; + }; + +export const derivativeOperation: OperationDefinition< + DerivativeIndexPatternColumn, + 'fullReference' +> = { + type: 'derivative', + priority: 2, + displayName: i18n.translate('xpack.lens.indexPattern.derivative', { + defaultMessage: 'Derivative', + }), + input: 'fullReference', + selectionStyle: 'full', + requiredReferences: [ + { + input: ['field'], + validateMetadata: (metadata) => + metadata.dataType === 'number' && metadata.isBucketed === false, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: () => { + return i18n.translate('xpack.lens.indexPattern.derivative', { + defaultMessage: 'Derivative', + }); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'derivative'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric.label), + dataType: 'number', + operationType: 'derivative', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram(layer); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts new file mode 100644 index 0000000000000..30e87aef46a0d --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { counterRateOperation, CounterRateIndexPatternColumn } from './counter_rate'; +export { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum'; +export { derivativeOperation, DerivativeIndexPatternColumn } from './derivative'; +export { movingAverageOperation, MovingAverageIndexPatternColumn } from './moving_average'; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx new file mode 100644 index 0000000000000..76e0ee0e4481e --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; +import { IndexPatternLayer } from '../../../types'; +import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; +import { OperationDefinition } from '..'; + +const ofName = (name: string) => { + return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { + defaultMessage: 'Moving average of {name}', + values: { name }, + }); +}; + +export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn & + ReferenceBasedIndexPatternColumn & { + operationType: 'moving_average'; + }; + +export const movingAverageOperation: OperationDefinition< + MovingAverageIndexPatternColumn, + 'fullReference' +> = { + type: 'moving_average', + priority: 2, + displayName: i18n.translate('xpack.lens.indexPattern.movingAverage', { + defaultMessage: 'Moving Average', + }), + input: 'fullReference', + selectionStyle: 'full', + requiredReferences: [ + { + input: ['field'], + validateMetadata: (metadata) => + metadata.dataType === 'number' && metadata.isBucketed === false, + }, + ], + getPossibleOperation: () => { + return { + dataType: 'number', + isBucketed: false, + scale: 'ratio', + }; + }, + getDefaultLabel: () => { + return i18n.translate('xpack.lens.indexPattern.movingAverage', { + defaultMessage: 'Moving Average', + }); + }, + toExpression: (layer, columnId) => { + return dateBasedOperationToExpression(layer, columnId, 'moving_average'); + }, + buildColumn: ({ referenceIds, previousColumn, layer }) => { + const metric = layer.columns[referenceIds[0]]; + return { + label: ofName(metric.label), + dataType: 'number', + operationType: 'moving_average', + isBucketed: false, + scale: 'ratio', + references: referenceIds, + params: + previousColumn?.dataType === 'number' && + previousColumn.params && + 'format' in previousColumn.params && + previousColumn.params.format + ? { format: previousColumn.params.format } + : undefined, + }; + }, + isTransferable: (column, newIndexPattern) => { + return hasDateField(newIndexPattern); + }, + getErrorMessage: (layer: IndexPatternLayer) => { + return checkForDateHistogram(layer); + }, +}; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts new file mode 100644 index 0000000000000..cbad98acedd9a --- /dev/null +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionAST } from '@kbn/interpreter/common'; +import { IndexPattern, IndexPatternLayer } from '../../../types'; +import { ReferenceBasedIndexPatternColumn } from '../column_types'; + +/** + * Checks whether the current layer includes a date histogram and returns an error otherwise + */ +export function checkForDateHistogram(layer: IndexPatternLayer) { + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const hasDateHistogram = buckets.some( + (colId) => layer.columns[colId].operationType === 'date_histogram' + ); + if (hasDateHistogram) { + return undefined; + } + return [ + i18n.translate('xpack.lens.indexPattern.calculations.dateHistogramErrorMessage', { + defaultMessage: 'Needs a date histogram to work', + }), + ]; +} + +export function hasDateField(indexPattern: IndexPattern) { + return indexPattern.fields.some((field) => field.type === 'date'); +} + +/** + * Creates an expression ast for a date based operation (cumulative sum, derivative, moving average, counter rate) + */ +export function dateBasedOperationToExpression( + layer: IndexPatternLayer, + columnId: string, + functionName: string +): ExpressionFunctionAST[] { + const currentColumn = (layer.columns[columnId] as unknown) as ReferenceBasedIndexPatternColumn; + const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); + const dateColumnIndex = buckets.findIndex( + (colId) => layer.columns[colId].operationType === 'date_histogram' + )!; + buckets.splice(dateColumnIndex, 1); + + return [ + { + type: 'function', + function: functionName, + arguments: { + by: buckets, + inputColumnId: [currentColumn.references[0]], + outputColumnId: [columnId], + }, + }, + ]; +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts index e0dfb6b248390..392377234d76d 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/index.ts @@ -23,7 +23,16 @@ import { MedianIndexPatternColumn, } from './metrics'; import { dateHistogramOperation, DateHistogramIndexPatternColumn } from './date_histogram'; -import { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum'; +import { + cumulativeSumOperation, + CumulativeSumIndexPatternColumn, + counterRateOperation, + CounterRateIndexPatternColumn, + derivativeOperation, + DerivativeIndexPatternColumn, + movingAverageOperation, + MovingAverageIndexPatternColumn, +} from './calculations'; import { countOperation, CountIndexPatternColumn } from './count'; import { StateSetter, OperationMetadata } from '../../../types'; import type { BaseIndexPatternColumn, ReferenceBasedIndexPatternColumn } from './column_types'; @@ -54,7 +63,10 @@ export type IndexPatternColumn = | SumIndexPatternColumn | MedianIndexPatternColumn | CountIndexPatternColumn - | CumulativeSumIndexPatternColumn; + | CumulativeSumIndexPatternColumn + | CounterRateIndexPatternColumn + | DerivativeIndexPatternColumn + | MovingAverageIndexPatternColumn; export type FieldBasedIndexPatternColumn = Extract; @@ -76,6 +88,9 @@ const internalOperationDefinitions = [ countOperation, rangeOperation, cumulativeSumOperation, + counterRateOperation, + derivativeOperation, + movingAverageOperation, ]; export { termsOperation } from './terms'; From 724e7c38b98203f40f379b2c40c1f862b34e23d4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 20 Nov 2020 15:05:23 +0100 Subject: [PATCH 07/12] add window param --- .../calculations/moving_average.tsx | 65 +++++++++++++++++-- .../definitions/calculations/utils.ts | 4 +- .../operations/definitions/column_types.ts | 2 +- 3 files changed, 65 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 76e0ee0e4481e..8727cdaf22d10 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -5,10 +5,16 @@ */ import { i18n } from '@kbn/i18n'; +import { useState } from 'react'; +import React from 'react'; +import { EuiFormRow } from '@elastic/eui'; +import { EuiFieldNumber } from '@elastic/eui'; import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPatternLayer } from '../../../types'; import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; -import { OperationDefinition } from '..'; +import { updateColumnParam } from '../..'; +import { useDebounceWithOptions } from '../helpers'; +import { OperationDefinition, ParamEditorProps } from '..'; const ofName = (name: string) => { return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { @@ -20,6 +26,9 @@ const ofName = (name: string) => { export type MovingAverageIndexPatternColumn = FormattedIndexPatternColumn & ReferenceBasedIndexPatternColumn & { operationType: 'moving_average'; + params: { + window: number; + }; }; export const movingAverageOperation: OperationDefinition< @@ -53,7 +62,9 @@ export const movingAverageOperation: OperationDefinition< }); }, toExpression: (layer, columnId) => { - return dateBasedOperationToExpression(layer, columnId, 'moving_average'); + return dateBasedOperationToExpression(layer, columnId, 'moving_average', { + window: [(layer.columns[columnId] as MovingAverageIndexPatternColumn).params.window], + }); }, buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; @@ -69,10 +80,11 @@ export const movingAverageOperation: OperationDefinition< previousColumn.params && 'format' in previousColumn.params && previousColumn.params.format - ? { format: previousColumn.params.format } - : undefined, + ? { format: previousColumn.params.format, window: 5 } + : { window: 5 }, }; }, + paramEditor: MovingAverageParamEditor, isTransferable: (column, newIndexPattern) => { return hasDateField(newIndexPattern); }, @@ -80,3 +92,48 @@ export const movingAverageOperation: OperationDefinition< return checkForDateHistogram(layer); }, }; + +function MovingAverageParamEditor({ + state, + setState, + currentColumn, + layerId, +}: ParamEditorProps) { + const [inputValue, setInputValue] = useState(String(currentColumn.params.window)); + + useDebounceWithOptions( + () => { + if (inputValue === '') { + return; + } + const inputNumber = Number(inputValue); + setState( + updateColumnParam({ + state, + layerId, + currentColumn, + paramName: 'window', + value: inputNumber, + }) + ); + }, + { skipFirstRender: true }, + 256, + [inputValue] + ); + return ( + + ) => setInputValue(e.target.value)} + /> + + ); +} diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts index cbad98acedd9a..afdcf06aa43fc 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -37,7 +37,8 @@ export function hasDateField(indexPattern: IndexPattern) { export function dateBasedOperationToExpression( layer: IndexPatternLayer, columnId: string, - functionName: string + functionName: string, + additionalArgs: Record = {} ): ExpressionFunctionAST[] { const currentColumn = (layer.columns[columnId] as unknown) as ReferenceBasedIndexPatternColumn; const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); @@ -54,6 +55,7 @@ export function dateBasedOperationToExpression( by: buckets, inputColumnId: [currentColumn.references[0]], outputColumnId: [columnId], + ...additionalArgs, }, }, ]; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts index 13bddc0c2ec26..aef9bb7731d4c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/column_types.ts @@ -16,7 +16,7 @@ export interface BaseIndexPatternColumn extends Operation { // export interface FormattedIndexPatternColumn extends BaseIndexPatternColumn { export type FormattedIndexPatternColumn = BaseIndexPatternColumn & { params?: { - format: { + format?: { id: string; params?: { decimals: number; From cc902e2a2d322a5e896f1782972cf51df9599c12 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 20 Nov 2020 17:13:49 +0100 Subject: [PATCH 08/12] fix circular import --- .../definitions/calculations/moving_average.tsx | 4 ++-- .../operations/operations.test.ts | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 8727cdaf22d10..ffe9cff11bfb2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -12,9 +12,9 @@ import { EuiFieldNumber } from '@elastic/eui'; import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types'; import { IndexPatternLayer } from '../../../types'; import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; -import { updateColumnParam } from '../..'; +import { updateColumnParam } from '../../layer_helpers'; import { useDebounceWithOptions } from '../helpers'; -import { OperationDefinition, ParamEditorProps } from '..'; +import type { OperationDefinition, ParamEditorProps } from '..'; const ofName = (name: string) => { return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 6ea4c49facdff..c4482fa272e94 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -246,6 +246,18 @@ describe('getOperationTypesForField', () => { "operationType": "cumulative_sum", "type": "fullReference", }, + Object { + "operationType": "counter_rate", + "type": "fullReference", + }, + Object { + "operationType": "derivative", + "type": "fullReference", + }, + Object { + "operationType": "moving_average", + "type": "fullReference", + }, Object { "field": "bytes", "operationType": "sum", From 26db7b8f4ceef6541b3c3bec0eaad3970ef1313a Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 20 Nov 2020 15:26:40 -0500 Subject: [PATCH 09/12] Change types in test --- .../indexpattern.test.ts | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts index 106692b73d59a..c3247b251d88a 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts @@ -660,30 +660,35 @@ describe('IndexPattern Data Source', () => { it('should skip columns that are being referenced', () => { publicAPI = indexPatternDatasource.getPublicAPI({ - state: ({ + state: { + ...enrichBaseState(baseState), layers: { first: { indexPatternId: '1', columnOrder: ['col1', 'col2'], columns: { col1: { - dataType: 'number', - operationType: 'sum', label: 'Sum', + dataType: 'number', isBucketed: false, + + operationType: 'sum', sourceField: 'test', - }, + params: {}, + } as IndexPatternColumn, col2: { - operationType: 'cumulative_sum', + label: 'Cumulative sum', dataType: 'number', isBucketed: false, - label: 'Cumulative sum', + + operationType: 'cumulative_sum', references: ['col1'], - }, + params: {}, + } as IndexPatternColumn, }, }, }, - } as unknown) as IndexPatternPrivateState, + }, layerId: 'first', }); From c208fdef689aa3be76e5d0c552b5152153552011 Mon Sep 17 00:00:00 2001 From: Wylie Conlon Date: Fri, 20 Nov 2020 16:32:39 -0500 Subject: [PATCH 10/12] Fix labeling --- .../definitions/calculations/counter_rate.tsx | 12 ++++++++--- .../calculations/cumulative_sum.tsx | 20 +++++++++++-------- .../definitions/calculations/derivative.tsx | 18 ++++++++++------- .../calculations/moving_average.tsx | 18 ++++++++++------- 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 9573968ffc4ad..92b655e2c0de5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -10,10 +10,16 @@ import { IndexPatternLayer } from '../../../types'; import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; import { OperationDefinition } from '..'; -const ofName = (name: string) => { +const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.CounterRateOf', { defaultMessage: 'Counter rate of {name}', - values: { name }, + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, }); }; @@ -59,7 +65,7 @@ export const counterRateOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; return { - label: ofName(metric.label), + label: ofName(metric?.label), dataType: 'number', operationType: 'counter_rate', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index c130549b0593c..cadc74c088295 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -10,10 +10,16 @@ import { IndexPatternLayer } from '../../../types'; import { checkForDateHistogram, dateBasedOperationToExpression } from './utils'; import { OperationDefinition } from '..'; -const ofName = (name: string) => { +const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', { - defaultMessage: 'Cumulative sum of {name}', - values: { name }, + defaultMessage: 'Cumulative sum rate of {name}', + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, }); }; @@ -48,10 +54,8 @@ export const cumulativeSumOperation: OperationDefinition< scale: 'ratio', }; }, - getDefaultLabel: () => { - return i18n.translate('xpack.lens.indexPattern.cumulativeSum', { - defaultMessage: 'Cumulative sum', - }); + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum'); @@ -59,7 +63,7 @@ export const cumulativeSumOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; return { - label: ofName(metric.label), + label: ofName(metric?.label), dataType: 'number', operationType: 'cumulative_sum', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx index b9c65f9633138..79432494ca9c2 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx @@ -10,10 +10,16 @@ import { IndexPatternLayer } from '../../../types'; import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils'; import { OperationDefinition } from '..'; -const ofName = (name: string) => { +const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.derivativeOf', { defaultMessage: 'Derivative of {name}', - values: { name }, + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, }); }; @@ -47,10 +53,8 @@ export const derivativeOperation: OperationDefinition< scale: 'ratio', }; }, - getDefaultLabel: () => { - return i18n.translate('xpack.lens.indexPattern.derivative', { - defaultMessage: 'Derivative', - }); + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'derivative'); @@ -58,7 +62,7 @@ export const derivativeOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; return { - label: ofName(metric.label), + label: ofName(metric?.label), dataType: 'number', operationType: 'derivative', isBucketed: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index ffe9cff11bfb2..607ae3f8aeacd 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -16,10 +16,16 @@ import { updateColumnParam } from '../../layer_helpers'; import { useDebounceWithOptions } from '../helpers'; import type { OperationDefinition, ParamEditorProps } from '..'; -const ofName = (name: string) => { +const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.movingAverageOf', { defaultMessage: 'Moving average of {name}', - values: { name }, + values: { + name: + name ?? + i18n.translate('xpack.lens.indexPattern.incompleteOperation', { + defaultMessage: '(incomplete)', + }), + }, }); }; @@ -56,10 +62,8 @@ export const movingAverageOperation: OperationDefinition< scale: 'ratio', }; }, - getDefaultLabel: () => { - return i18n.translate('xpack.lens.indexPattern.movingAverage', { - defaultMessage: 'Moving Average', - }); + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'moving_average', { @@ -69,7 +73,7 @@ export const movingAverageOperation: OperationDefinition< buildColumn: ({ referenceIds, previousColumn, layer }) => { const metric = layer.columns[referenceIds[0]]; return { - label: ofName(metric.label), + label: ofName(metric?.label), dataType: 'number', operationType: 'moving_average', isBucketed: false, From 4751e992ce88e21cebe67df8c5445a892eed300b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 23 Nov 2020 14:30:03 +0100 Subject: [PATCH 11/12] fix stuff --- .../definitions/calculations/counter_rate.tsx | 18 ++++++++++-------- .../calculations/cumulative_sum.tsx | 12 ++++++++---- .../definitions/calculations/derivative.tsx | 16 ++++++++++------ .../calculations/moving_average.tsx | 12 ++++++++---- .../definitions/calculations/utils.ts | 9 +++++++-- .../operations/operations.test.ts | 10 +++++----- 6 files changed, 48 insertions(+), 29 deletions(-) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx index 92b655e2c0de5..d256b74696a4c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx @@ -33,7 +33,7 @@ export const counterRateOperation: OperationDefinition< 'fullReference' > = { type: 'counter_rate', - priority: 2, + priority: 1, displayName: i18n.translate('xpack.lens.indexPattern.counterRate', { defaultMessage: 'Counter rate', }), @@ -43,8 +43,7 @@ export const counterRateOperation: OperationDefinition< { input: ['field'], specificOperations: ['max'], - validateMetadata: (metadata) => - metadata.dataType === 'number' && metadata.isBucketed === false, + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], getPossibleOperation: () => { @@ -54,10 +53,8 @@ export const counterRateOperation: OperationDefinition< scale: 'ratio', }; }, - getDefaultLabel: () => { - return i18n.translate('xpack.lens.indexPattern.counterRate', { - defaultMessage: 'Counter rate', - }); + getDefaultLabel: (column, indexPattern, columns) => { + return ofName(columns[column.references[0]]?.label); }, toExpression: (layer, columnId) => { return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate'); @@ -84,6 +81,11 @@ export const counterRateOperation: OperationDefinition< return hasDateField(newIndexPattern); }, getErrorMessage: (layer: IndexPatternLayer) => { - return checkForDateHistogram(layer); + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.counterRate', { + defaultMessage: 'Counter rate', + }) + ); }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx index cadc74c088295..9244aaaf90ab7 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx @@ -33,7 +33,7 @@ export const cumulativeSumOperation: OperationDefinition< 'fullReference' > = { type: 'cumulative_sum', - priority: 2, + priority: 1, displayName: i18n.translate('xpack.lens.indexPattern.cumulativeSum', { defaultMessage: 'Cumulative sum', }), @@ -43,8 +43,7 @@ export const cumulativeSumOperation: OperationDefinition< { input: ['field'], specificOperations: ['count', 'sum'], - validateMetadata: (metadata) => - metadata.dataType === 'number' && metadata.isBucketed === false, + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], getPossibleOperation: () => { @@ -82,6 +81,11 @@ export const cumulativeSumOperation: OperationDefinition< return true; }, getErrorMessage: (layer: IndexPatternLayer) => { - return checkForDateHistogram(layer); + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.cumulativeSum', { + defaultMessage: 'Cumulative sum', + }) + ); }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx index 79432494ca9c2..7398f7e07ea4e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx @@ -12,7 +12,7 @@ import { OperationDefinition } from '..'; const ofName = (name?: string) => { return i18n.translate('xpack.lens.indexPattern.derivativeOf', { - defaultMessage: 'Derivative of {name}', + defaultMessage: 'Differences of {name}', values: { name: name ?? @@ -33,17 +33,16 @@ export const derivativeOperation: OperationDefinition< 'fullReference' > = { type: 'derivative', - priority: 2, + priority: 1, displayName: i18n.translate('xpack.lens.indexPattern.derivative', { - defaultMessage: 'Derivative', + defaultMessage: 'Differences', }), input: 'fullReference', selectionStyle: 'full', requiredReferences: [ { input: ['field'], - validateMetadata: (metadata) => - metadata.dataType === 'number' && metadata.isBucketed === false, + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], getPossibleOperation: () => { @@ -81,6 +80,11 @@ export const derivativeOperation: OperationDefinition< return hasDateField(newIndexPattern); }, getErrorMessage: (layer: IndexPatternLayer) => { - return checkForDateHistogram(layer); + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.derivative', { + defaultMessage: 'Differences', + }) + ); }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx index 607ae3f8aeacd..795281d0fd994 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/moving_average.tsx @@ -42,7 +42,7 @@ export const movingAverageOperation: OperationDefinition< 'fullReference' > = { type: 'moving_average', - priority: 2, + priority: 1, displayName: i18n.translate('xpack.lens.indexPattern.movingAverage', { defaultMessage: 'Moving Average', }), @@ -51,8 +51,7 @@ export const movingAverageOperation: OperationDefinition< requiredReferences: [ { input: ['field'], - validateMetadata: (metadata) => - metadata.dataType === 'number' && metadata.isBucketed === false, + validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed, }, ], getPossibleOperation: () => { @@ -93,7 +92,12 @@ export const movingAverageOperation: OperationDefinition< return hasDateField(newIndexPattern); }, getErrorMessage: (layer: IndexPatternLayer) => { - return checkForDateHistogram(layer); + return checkForDateHistogram( + layer, + i18n.translate('xpack.lens.indexPattern.movingAverage', { + defaultMessage: 'Moving Average', + }) + ); }, }; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts index afdcf06aa43fc..c64a292280603 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/utils.ts @@ -12,7 +12,7 @@ import { ReferenceBasedIndexPatternColumn } from '../column_types'; /** * Checks whether the current layer includes a date histogram and returns an error otherwise */ -export function checkForDateHistogram(layer: IndexPatternLayer) { +export function checkForDateHistogram(layer: IndexPatternLayer, name: string) { const buckets = layer.columnOrder.filter((colId) => layer.columns[colId].isBucketed); const hasDateHistogram = buckets.some( (colId) => layer.columns[colId].operationType === 'date_histogram' @@ -22,7 +22,11 @@ export function checkForDateHistogram(layer: IndexPatternLayer) { } return [ i18n.translate('xpack.lens.indexPattern.calculations.dateHistogramErrorMessage', { - defaultMessage: 'Needs a date histogram to work', + defaultMessage: + '{name} requires a date histogram to work. Choose a different function or add a date histogram.', + values: { + name, + }, }), ]; } @@ -55,6 +59,7 @@ export function dateBasedOperationToExpression( by: buckets, inputColumnId: [currentColumn.references[0]], outputColumnId: [columnId], + outputColumnName: [currentColumn.label], ...additionalArgs, }, }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index c4482fa272e94..63d0fd3d4e5c5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -242,6 +242,11 @@ describe('getOperationTypesForField', () => { "operationType": "avg", "type": "field", }, + Object { + "field": "bytes", + "operationType": "sum", + "type": "field", + }, Object { "operationType": "cumulative_sum", "type": "fullReference", @@ -258,11 +263,6 @@ describe('getOperationTypesForField', () => { "operationType": "moving_average", "type": "fullReference", }, - Object { - "field": "bytes", - "operationType": "sum", - "type": "field", - }, Object { "field": "bytes", "operationType": "min", From 3f9fb7d19af047198905153694521c22c1344b01 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Tue, 24 Nov 2020 15:50:13 +0100 Subject: [PATCH 12/12] register counter rate function --- x-pack/plugins/lens/public/indexpattern_datasource/index.ts | 2 ++ .../lens/public/indexpattern_datasource/indexpattern.tsx | 1 + 2 files changed, 3 insertions(+) diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts index 793f3387e707d..5f7eddd807c93 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/index.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/index.ts @@ -37,12 +37,14 @@ export class IndexPatternDatasource { getIndexPatternDatasource, renameColumns, formatColumn, + counterRate, getTimeScaleFunction, getSuffixFormatter, } = await import('../async_services'); return core.getStartServices().then(([coreStart, { data }]) => { data.fieldFormats.register([getSuffixFormatter(data.fieldFormats.deserialize)]); expressions.registerFunction(getTimeScaleFunction(data)); + expressions.registerFunction(counterRate); expressions.registerFunction(renameColumns); expressions.registerFunction(formatColumn); return getIndexPatternDatasource({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx index 2c64431867df0..289b6bbe3f25b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx @@ -78,6 +78,7 @@ export function columnToOperation(column: IndexPatternColumn, uniqueLabel?: stri export * from './rename_columns'; export * from './format_column'; export * from './time_scale'; +export * from './counter_rate'; export * from './suffix_formatter'; export function getIndexPatternDatasource({