diff --git a/src/plugins/data/common/search/aggs/agg_types.ts b/src/plugins/data/common/search/aggs/agg_types.ts index 396ca49edaf4f..43e3df6100a75 100644 --- a/src/plugins/data/common/search/aggs/agg_types.ts +++ b/src/plugins/data/common/search/aggs/agg_types.ts @@ -36,6 +36,7 @@ export const getAggTypes = () => ({ { name: METRIC_TYPES.MAX, fn: metrics.getMaxMetricAgg }, { name: METRIC_TYPES.STD_DEV, fn: metrics.getStdDeviationMetricAgg }, { name: METRIC_TYPES.CARDINALITY, fn: metrics.getCardinalityMetricAgg }, + { name: METRIC_TYPES.VALUE_COUNT, fn: metrics.getValueCountMetricAgg }, { name: METRIC_TYPES.PERCENTILES, fn: metrics.getPercentilesMetricAgg }, { name: METRIC_TYPES.PERCENTILE_RANKS, fn: metrics.getPercentileRanksMetricAgg }, { name: METRIC_TYPES.TOP_HITS, fn: metrics.getTopHitMetricAgg }, @@ -97,6 +98,7 @@ export const getAggTypesFunctions = () => [ metrics.aggBucketSum, metrics.aggFilteredMetric, metrics.aggCardinality, + metrics.aggValueCount, metrics.aggCount, metrics.aggCumulativeSum, metrics.aggDerivative, diff --git a/src/plugins/data/common/search/aggs/aggs_service.test.ts b/src/plugins/data/common/search/aggs/aggs_service.test.ts index ee40a1d76d1aa..f0425e460ae0f 100644 --- a/src/plugins/data/common/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/common/search/aggs/aggs_service.test.ts @@ -84,6 +84,7 @@ describe('Aggs service', () => { "max", "std_dev", "cardinality", + "value_count", "percentiles", "percentile_ranks", "top_hits", @@ -136,6 +137,7 @@ describe('Aggs service', () => { "max", "std_dev", "cardinality", + "value_count", "percentiles", "percentile_ranks", "top_hits", diff --git a/src/plugins/data/common/search/aggs/metrics/index.ts b/src/plugins/data/common/search/aggs/metrics/index.ts index 55af141b8fcb7..492f76ccd3a08 100644 --- a/src/plugins/data/common/search/aggs/metrics/index.ts +++ b/src/plugins/data/common/search/aggs/metrics/index.ts @@ -20,6 +20,8 @@ export * from './filtered_metric_fn'; export * from './filtered_metric'; export * from './cardinality_fn'; export * from './cardinality'; +export * from './value_count_fn'; +export * from './value_count'; export * from './count'; export * from './count_fn'; export * from './cumulative_sum_fn'; diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts index 59bbe377ba28a..6af5e18b0709e 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_type.ts @@ -77,9 +77,11 @@ export class MetricAggType { // Metric types where an empty set equals `zero` - const isSettableToZero = [METRIC_TYPES.CARDINALITY, METRIC_TYPES.SUM].includes( - agg.type.name as METRIC_TYPES - ); + const isSettableToZero = [ + METRIC_TYPES.CARDINALITY, + METRIC_TYPES.VALUE_COUNT, + METRIC_TYPES.SUM, + ].includes(agg.type.name as METRIC_TYPES); // Return proper values when no buckets are present // `Count` handles empty sets properly diff --git a/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts b/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts index 4174808892a16..151f18110a301 100644 --- a/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts +++ b/src/plugins/data/common/search/aggs/metrics/metric_agg_types.ts @@ -10,6 +10,7 @@ export enum METRIC_TYPES { AVG = 'avg', FILTERED_METRIC = 'filtered_metric', CARDINALITY = 'cardinality', + VALUE_COUNT = 'value_count', AVG_BUCKET = 'avg_bucket', MAX_BUCKET = 'max_bucket', MIN_BUCKET = 'min_bucket', diff --git a/src/plugins/data/common/search/aggs/metrics/value_count.ts b/src/plugins/data/common/search/aggs/metrics/value_count.ts new file mode 100644 index 0000000000000..c5cea76c88bb5 --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/value_count.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { aggValueCountFnName } from './value_count_fn'; +import { MetricAggType, IMetricAggConfig } from './metric_agg_type'; +import { METRIC_TYPES } from './metric_agg_types'; +import { BaseAggParams } from '../types'; + +const valueCountTitle = i18n.translate('data.search.aggs.metrics.valueCountTitle', { + defaultMessage: 'Value Count', +}); + +export interface AggParamsValueCount extends BaseAggParams { + field: string; + emptyAsNull?: boolean; +} + +export const getValueCountMetricAgg = () => + new MetricAggType({ + name: METRIC_TYPES.VALUE_COUNT, + valueType: 'number', + expressionName: aggValueCountFnName, + title: valueCountTitle, + enableEmptyAsNull: true, + makeLabel(aggConfig: IMetricAggConfig) { + return i18n.translate('data.search.aggs.metrics.valueCountLabel', { + defaultMessage: 'Value count of {field}', + values: { field: aggConfig.getFieldDisplayName() }, + }); + }, + getSerializedFormat(agg) { + return { + id: 'number', + }; + }, + params: [ + { + name: 'field', + type: 'field', + }, + ], + }); diff --git a/src/plugins/data/common/search/aggs/metrics/value_count_fn.test.ts b/src/plugins/data/common/search/aggs/metrics/value_count_fn.test.ts new file mode 100644 index 0000000000000..db590244659fa --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/value_count_fn.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { functionWrapper } from '../test_helpers'; +import { aggValueCount } from './value_count_fn'; + +describe('agg_expression_functions', () => { + describe('aggValueCount', () => { + const fn = functionWrapper(aggValueCount()); + + test('required args are provided', () => { + const actual = fn({ + field: 'machine.os.keyword', + }); + expect(actual).toMatchInlineSnapshot(` + Object { + "type": "agg_type", + "value": Object { + "enabled": true, + "id": undefined, + "params": Object { + "customLabel": undefined, + "emptyAsNull": undefined, + "field": "machine.os.keyword", + "json": undefined, + "timeShift": undefined, + }, + "schema": undefined, + "type": "value_count", + }, + } + `); + }); + + test('correctly parses json string argument', () => { + const actual = fn({ + field: 'machine.os.keyword', + json: '{ "foo": true }', + }); + + expect(actual.value.params.json).toEqual('{ "foo": true }'); + }); + }); +}); diff --git a/src/plugins/data/common/search/aggs/metrics/value_count_fn.ts b/src/plugins/data/common/search/aggs/metrics/value_count_fn.ts new file mode 100644 index 0000000000000..3acf2da345d27 --- /dev/null +++ b/src/plugins/data/common/search/aggs/metrics/value_count_fn.ts @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { i18n } from '@kbn/i18n'; +import { ExpressionFunctionDefinition } from '@kbn/expressions-plugin/common'; +import { AggExpressionType, AggExpressionFunctionArgs, METRIC_TYPES } from '..'; + +export const aggValueCountFnName = 'aggValueCount'; + +type Input = any; +type AggArgs = AggExpressionFunctionArgs; +type Output = AggExpressionType; +type FunctionDefinition = ExpressionFunctionDefinition< + typeof aggValueCountFnName, + Input, + AggArgs, + Output +>; + +export const aggValueCount = (): FunctionDefinition => ({ + name: aggValueCountFnName, + help: i18n.translate('data.search.aggs.function.metrics.valueCount.help', { + defaultMessage: 'Generates a serialized agg config for a value count agg', + }), + type: 'agg_type', + args: { + id: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.value_count.id.help', { + defaultMessage: 'ID for this aggregation', + }), + }, + enabled: { + types: ['boolean'], + default: true, + help: i18n.translate('data.search.aggs.metrics.value_count.enabled.help', { + defaultMessage: 'Specifies whether this aggregation should be enabled', + }), + }, + schema: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.value_count.schema.help', { + defaultMessage: 'Schema to use for this aggregation', + }), + }, + field: { + types: ['string'], + required: true, + help: i18n.translate('data.search.aggs.metrics.value_count.field.help', { + defaultMessage: 'Field to use for this aggregation', + }), + }, + json: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.value_count.json.help', { + defaultMessage: 'Advanced json to include when the agg is sent to Elasticsearch', + }), + }, + customLabel: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.value_count.customLabel.help', { + defaultMessage: 'Represents a custom label for this aggregation', + }), + }, + timeShift: { + types: ['string'], + help: i18n.translate('data.search.aggs.metrics.timeShift.help', { + defaultMessage: + 'Shift the time range for the metric by a set time, for example 1h or 7d. "previous" will use the closest time range from the date histogram or time range filter.', + }), + }, + emptyAsNull: { + types: ['boolean'], + help: i18n.translate('data.search.aggs.metrics.emptyAsNull.help', { + defaultMessage: + 'If set to true, a missing value is treated as null in the resulting data table. If set to false, a "zero" is filled in', + }), + }, + }, + fn: (input, args) => { + const { id, enabled, schema, ...rest } = args; + + return { + type: 'agg_type', + value: { + id, + enabled, + schema, + type: METRIC_TYPES.VALUE_COUNT, + params: { + ...rest, + }, + }, + }; + }, +}); diff --git a/src/plugins/data/common/search/aggs/types.ts b/src/plugins/data/common/search/aggs/types.ts index 7e72365f06af0..16fac531fad94 100644 --- a/src/plugins/data/common/search/aggs/types.ts +++ b/src/plugins/data/common/search/aggs/types.ts @@ -15,6 +15,7 @@ import { aggBucketMin, aggBucketSum, aggCardinality, + aggValueCount, AggConfigs, AggConfigSerialized, aggCount, @@ -41,6 +42,7 @@ import { AggParamsBucketSum, AggParamsFilteredMetric, AggParamsCardinality, + AggParamsValueCount, AggParamsCumulativeSum, AggParamsDateHistogram, AggParamsDateRange, @@ -174,6 +176,7 @@ export interface AggParamsMapping { [METRIC_TYPES.AVG]: AggParamsAvg; [METRIC_TYPES.CARDINALITY]: AggParamsCardinality; [METRIC_TYPES.COUNT]: AggParamsCount; + [METRIC_TYPES.VALUE_COUNT]: AggParamsValueCount; [METRIC_TYPES.GEO_BOUNDS]: AggParamsGeoBounds; [METRIC_TYPES.GEO_CENTROID]: AggParamsGeoCentroid; [METRIC_TYPES.MAX]: AggParamsMax; @@ -222,6 +225,7 @@ export interface AggFunctionsMapping { aggBucketSum: ReturnType; aggFilteredMetric: ReturnType; aggCardinality: ReturnType; + aggValueCount: ReturnType; aggCount: ReturnType; aggCumulativeSum: ReturnType; aggDerivative: ReturnType; diff --git a/src/plugins/data/public/search/aggs/aggs_service.test.ts b/src/plugins/data/public/search/aggs/aggs_service.test.ts index 40cc3590a32e2..0c8067df5c7f7 100644 --- a/src/plugins/data/public/search/aggs/aggs_service.test.ts +++ b/src/plugins/data/public/search/aggs/aggs_service.test.ts @@ -53,7 +53,7 @@ describe('AggsService - public', () => { service.setup(setupDeps); const start = service.start(startDeps); expect(start.types.getAll().buckets.length).toBe(16); - expect(start.types.getAll().metrics.length).toBe(25); + expect(start.types.getAll().metrics.length).toBe(26); }); test('registers custom agg types', () => { @@ -70,7 +70,7 @@ describe('AggsService - public', () => { const start = service.start(startDeps); expect(start.types.getAll().buckets.length).toBe(17); expect(start.types.getAll().buckets.some(({ name }) => name === 'foo')).toBe(true); - expect(start.types.getAll().metrics.length).toBe(26); + expect(start.types.getAll().metrics.length).toBe(27); expect(start.types.getAll().metrics.some(({ name }) => name === 'bar')).toBe(true); }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx index 413f40279a3ce..559c3ef7fff44 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_editor.tsx @@ -45,6 +45,7 @@ import { Filtering } from './filtering'; import { AdvancedOptions } from './advanced_options'; import { TimeShift } from './time_shift'; import type { LayerType } from '../../../common'; +import { DOCUMENT_FIELD_NAME } from '../../../common'; import { quickFunctionsName, staticValueOperationName, @@ -62,6 +63,7 @@ import { ParamEditorProps } from '../operations/definitions'; import { WrappingHelpPopover } from '../help_popover'; import { isColumn } from '../operations/definitions/helpers'; import { FieldChoiceWithOperationType } from './field_select'; +import { documentField } from '../document_field'; export interface DimensionEditorProps extends IndexPatternDimensionEditorProps { selectedColumn?: GenericIndexPatternColumn; @@ -449,7 +451,8 @@ export function DimensionEditor(props: DimensionEditorProps) { indexPattern: currentIndexPattern, columnId, op: operationType, - field: undefined, + // if document field can be used, default to it + field: possibleFields.has(DOCUMENT_FIELD_NAME) ? documentField : undefined, visualizationGroups: dimensionGroups, targetGroup: props.groupId, }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx index a2d83da5d89e2..291ccb1ede76b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/dimension_panel.test.tsx @@ -986,26 +986,16 @@ describe('IndexPatternDimensionEditorPanel', () => { }); }); - it('should select the Records field when count is selected', () => { + it('should select the Records field when count is selected on non-existing column', () => { wrapper = mount( ); - wrapper - .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') - .simulate('click'); + wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); const newColumnState = setState.mock.calls[0][0](state).layers.first.columns.col2; expect(newColumnState.operationType).toEqual('count'); @@ -1175,9 +1165,7 @@ describe('IndexPatternDimensionEditorPanel', () => { label: 'Sum of bytes per hour', }); wrapper = mount(); - wrapper - .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') - .simulate('click'); + wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1188,7 +1176,7 @@ describe('IndexPatternDimensionEditorPanel', () => { ...props.state.layers.first.columns, col2: expect.objectContaining({ timeScale: 'h', - label: 'Count of records per hour', + label: 'Value count of bytes per hour', }), }, }, @@ -1385,9 +1373,7 @@ describe('IndexPatternDimensionEditorPanel', () => { label: 'Sum of bytes per hour', }); wrapper = mount(); - wrapper - .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') - .simulate('click'); + wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); expect((props.setState as jest.Mock).mock.calls[0][0](props.state)).toEqual({ ...props.state, layers: { @@ -1523,9 +1509,7 @@ describe('IndexPatternDimensionEditorPanel', () => { label: 'Sum of bytes per hour', }); wrapper = mount(); - wrapper - .find('button[data-test-subj="lns-indexPatternDimension-count incompatible"]') - .simulate('click'); + wrapper.find('button[data-test-subj="lns-indexPatternDimension-count"]').simulate('click'); expect(setState.mock.calls[0]).toEqual([expect.any(Function), { isDimensionComplete: true }]); expect(setState.mock.calls[0][0](props.state)).toEqual({ ...props.state, @@ -1764,10 +1748,6 @@ describe('IndexPatternDimensionEditorPanel', () => { .prop('options'); expect(options![0]['data-test-subj']).not.toContain('Incompatible'); - - options![1].options!.map((operation) => - expect(operation['data-test-subj']).toContain('Incompatible') - ); }); it('should not update when selecting the current field again', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/get_drop_props.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/get_drop_props.test.ts index f9afc9a00c98f..f48e4d897a637 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/get_drop_props.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/dimension_panel/droppable/get_drop_props.test.ts @@ -362,7 +362,7 @@ describe('IndexPatternDimensionEditorPanel#getDropProps', () => { }) ).toEqual({ dropTypes: ['replace_incompatible', 'replace_duplicate_incompatible'], - nextLabel: 'Minimum', + nextLabel: 'Count', }); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx index 812bb8413137a..f96a6dbd3340e 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/indexpattern_suggestions.test.tsx @@ -3030,7 +3030,7 @@ describe('IndexPattern Data Source suggestions', () => { operation: { dataType: 'number', isBucketed: false, - label: 'Cumulative sum of Records label', + label: 'Cumulative sum of Records', scale: undefined, isStaticValue: false, hasTimeShift: false, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts index 8ca21272a555e..d3affb5b32d8c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/loader.ts @@ -105,7 +105,7 @@ export function convertDataViewIntoLensIndexPattern(dataView: DataView): IndexPa ]) ), fields: newFields, - getFieldByName: getFieldByNameFactory(newFields), + getFieldByName: getFieldByNameFactory(newFields, false), hasRestrictions: !!typeMeta?.aggs, }; } diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions.test.ts index 1a0da126f28e3..dae677663d289 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions.test.ts @@ -211,7 +211,7 @@ describe('time scale transition', () => { ).toEqual( expect.objectContaining({ timeScale: undefined, - label: 'Count of records', + label: 'Value count of bytesLabel', }) ); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx index 888c992031430..d3c0b9700379c 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/count.tsx @@ -11,6 +11,7 @@ import { euiThemeVars } from '@kbn/ui-theme'; import { EuiSwitch } from '@elastic/eui'; import { AggFunctionsMapping } from '@kbn/data-plugin/public'; import { buildExpressionFunction } from '@kbn/expressions-plugin/public'; +import { TimeScaleUnit } from '../../../../common/expressions'; import { OperationDefinition, ParamEditorProps } from '.'; import { FieldBasedIndexPatternColumn, ValueFormatConfig } from './column_types'; import { IndexPatternField } from '../../types'; @@ -32,6 +33,39 @@ const countLabel = i18n.translate('xpack.lens.indexPattern.countOf', { defaultMessage: 'Count of records', }); +const supportedTypes = new Set([ + 'string', + 'boolean', + 'number', + 'number_range', + 'ip', + 'ip_range', + 'date', + 'date_range', + 'murmur3', +]); + +function ofName( + field: IndexPatternField | undefined, + timeShift: string | undefined, + timeScale: string | undefined +) { + return adjustTimeScaleLabelSuffix( + field?.type !== 'document' + ? i18n.translate('xpack.lens.indexPattern.valueCountOf', { + defaultMessage: 'Value count of {name}', + values: { + name: field?.displayName || '-', + }, + }) + : countLabel, + undefined, + timeScale as TimeScaleUnit, + undefined, + timeShift + ); +} + export type CountIndexPatternColumn = FieldBasedIndexPatternColumn & { operationType: 'count'; params?: { @@ -40,9 +74,11 @@ export type CountIndexPatternColumn = FieldBasedIndexPatternColumn & { }; }; +const SCALE = 'ratio'; +const IS_BUCKETED = false; + export const countOperation: OperationDefinition = { type: 'count', - priority: 2, displayName: i18n.translate('xpack.lens.indexPattern.count', { defaultMessage: 'Count', }), @@ -56,42 +92,27 @@ export const countOperation: OperationDefinition { return { ...oldColumn, - label: adjustTimeScaleLabelSuffix( - field.displayName, - undefined, - oldColumn.timeScale, - undefined, - oldColumn.timeShift - ), + label: ofName(field, oldColumn.timeShift, oldColumn.timeShift), sourceField: field.name, }; }, - getPossibleOperationForField: (field: IndexPatternField) => { - if (field.type === 'document') { - return { - dataType: 'number', - isBucketed: false, - scale: 'ratio', - }; + getPossibleOperationForField: ({ aggregationRestrictions, aggregatable, type }) => { + if ( + type === 'document' || + (aggregatable && + (!aggregationRestrictions || aggregationRestrictions.value_count) && + supportedTypes.has(type)) + ) { + return { dataType: 'number', isBucketed: IS_BUCKETED, scale: SCALE }; } }, - getDefaultLabel: (column) => - adjustTimeScaleLabelSuffix( - countLabel, - undefined, - column.timeScale, - undefined, - column.timeShift - ), + getDefaultLabel: (column, indexPattern) => { + const field = indexPattern.getFieldByName(column.sourceField); + return ofName(field, column.timeShift, column.timeScale); + }, buildColumn({ field, previousColumn }, columnParams) { return { - label: adjustTimeScaleLabelSuffix( - countLabel, - undefined, - previousColumn?.timeScale, - undefined, - previousColumn?.timeShift - ), + label: ofName(field, previousColumn?.timeShift, previousColumn?.timeScale), dataType: 'number', operationType: 'count', isBucketed: false, @@ -147,33 +168,58 @@ export const countOperation: OperationDefinition adjustTimeScaleOnOtherColumnChange(layer, thisColumnId), - toEsAggsFn: (column, columnId) => { - return buildExpressionFunction('aggCount', { - id: columnId, - enabled: true, - schema: 'metric', - // time shift is added to wrapping aggFilteredMetric if filter is set - timeShift: column.filter ? undefined : column.timeShift, - emptyAsNull: column.params?.emptyAsNull, - }).toAst(); + toEsAggsFn: (column, columnId, indexPattern) => { + const field = indexPattern.getFieldByName(column.sourceField); + if (field?.type === 'document') { + return buildExpressionFunction('aggCount', { + id: columnId, + enabled: true, + schema: 'metric', + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + emptyAsNull: column.params?.emptyAsNull, + }).toAst(); + } else { + return buildExpressionFunction('aggValueCount', { + id: columnId, + enabled: true, + schema: 'metric', + field: column.sourceField, + // time shift is added to wrapping aggFilteredMetric if filter is set + timeShift: column.filter ? undefined : column.timeShift, + emptyAsNull: column.params?.emptyAsNull, + }).toAst(); + } }, - isTransferable: () => { - return true; + isTransferable: (column, newIndexPattern) => { + const newField = newIndexPattern.getFieldByName(column.sourceField); + + return Boolean( + newField && + (newField.type === 'document' || + (supportedTypes.has(newField.type) && + newField.aggregatable && + (!newField.aggregationRestrictions || newField.aggregationRestrictions.cardinality))) + ); }, timeScalingMode: 'optional', filterable: true, documentation: { section: 'elasticsearch', - signature: '', + signature: i18n.translate('xpack.lens.indexPattern.count.signature', { + defaultMessage: '[field: string]', + }), description: i18n.translate('xpack.lens.indexPattern.count.documentation.markdown', { defaultMessage: ` -Calculates the number of documents. +The total number of documents. When you provide a field as the first argument, the total number of field values is counted. Use the count function for fields that have multiple values in a single document. + +#### Examples + +To calculate the total number of documents, use \`count()\`. -Example: Calculate the number of documents: -\`count()\` +To calculate the number of products in all orders, use \`count(products.id)\`. -Example: Calculate the number of documents matching a certain filter: -\`count(kql='price > 500')\` +To calculate the number of documents that match a specific filter, use \`count(kql='price > 500')\`. `, }), }, diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx index 3059318bd562d..bdc774f9d15f3 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/editor/formula_help.tsx @@ -483,7 +483,7 @@ export function getFunctionSignatureLabel( const def = operationDefinitionMap[name]; let extraArgs = ''; if (def.filterable) { - extraArgs += hasFunctionFieldArgument(name) || 'operationParams' in def ? ',' : ''; + extraArgs += ','; extraArgs += i18n.translate('xpack.lens.formula.kqlExtraArguments', { defaultMessage: '[kql]?: string, [lucene]?: string', }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx index c4402277148c7..1e16c27253d9b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/formula.test.tsx @@ -674,14 +674,6 @@ describe('formula', () => { } }); - it('returns no change but error if an argument is passed to count operation', () => { - const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; - - for (const formula of formulas) { - testIsBrokenFormula(formula); - } - }); - it('returns no change but error if a required parameter is not passed to the operation in formula', () => { const formula = 'moving_average(average(bytes))'; testIsBrokenFormula(formula); @@ -1124,19 +1116,15 @@ invalid: " } }); - it('returns an error if an argument is passed to count() operation', () => { - const formulas = ['count(7)', 'count("bytes")', 'count(bytes)']; - - for (const formula of formulas) { - expect( - formulaOperation.getErrorMessage!( - getNewLayerWithFormula(formula), - 'col1', - indexPattern, - operationDefinitionMap - ) - ).toEqual(['The operation count does not accept any field as argument']); - } + it('does not return an error if count() is called without a field', () => { + expect( + formulaOperation.getErrorMessage!( + getNewLayerWithFormula('count()'), + 'col1', + indexPattern, + operationDefinitionMap + ) + ).toEqual(undefined); }); it('returns an error if an operation with required parameters does not receive them', () => { diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts index 4be0df4f86c1b..01bae7d7511b0 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/generate.ts @@ -6,6 +6,7 @@ */ import { isObject } from 'lodash'; +import { DOCUMENT_FIELD_NAME } from '../../../../../common'; import { FieldBasedIndexPatternColumn, GenericOperationDefinition, @@ -25,7 +26,7 @@ export function getSafeFieldName({ operationType, }: FieldBasedIndexPatternColumn) { // return empty for the records field - if (!fieldName || operationType === 'count') { + if (!fieldName || (operationType === 'count' && fieldName === DOCUMENT_FIELD_NAME)) { return ''; } if (unquotedStringRegex.test(fieldName)) { @@ -66,7 +67,10 @@ export function generateFormula( ', ' + formulaNamedArgs.map(({ name, value }) => `${name}=${value}`).join(', '); } if (previousColumn.filter) { - if (previousColumn.operationType !== 'count') { + if ( + previousColumn.operationType !== 'count' || + ('sourceField' in previousColumn && previousColumn.sourceField !== DOCUMENT_FIELD_NAME) + ) { previousFormula += ', '; } previousFormula += @@ -74,7 +78,11 @@ export function generateFormula( `'${previousColumn.filter.query.replace(/'/g, `\\'`)}'`; // replace all } if (previousColumn.timeShift) { - if (previousColumn.operationType !== 'count' || previousColumn.filter) { + if ( + previousColumn.operationType !== 'count' || + ('sourceField' in previousColumn && previousColumn.sourceField !== DOCUMENT_FIELD_NAME) || + previousColumn.filter + ) { previousFormula += ', '; } previousFormula += `shift='${previousColumn.timeShift}'`; diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts index 371c7e268388b..63e0935a3425b 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/parse.ts @@ -103,9 +103,10 @@ function extractColumns( if (nodeOperation.input === 'field') { const [fieldName] = variables.filter((v): v is TinymathVariable => isObject(v)); // a validation task passed before executing this and checked already there's a field - const field = shouldHaveFieldArgument(node) - ? indexPattern.getFieldByName(fieldName.value)! - : documentField; + let field = fieldName ? indexPattern.getFieldByName(fieldName.value) : undefined; + if (!shouldHaveFieldArgument(node) && !field) { + field = documentField; + } const mappedParams = { ...mergeWithGlobalFilter( @@ -122,7 +123,8 @@ function extractColumns( { layer, indexPattern, - field, + // checked in the validation phase + field: field!, }, mappedParams ); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts index 5936b90c095ec..28e015e4fc0b5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/formula/validation.ts @@ -638,23 +638,23 @@ function runFullASTValidation( } } else { if (nodeOperation.input === 'field') { - if (shouldHaveFieldArgument(node)) { - if (!isArgumentValidType(firstArg, 'variable')) { - if (isMathNode(firstArg)) { - errors.push( - getMessageFromId({ - messageId: 'wrongFirstArgument', - values: { - operation: node.name, - type: i18n.translate('xpack.lens.indexPattern.formulaFieldValue', { - defaultMessage: 'field', - }), - argument: `math operation`, - }, - locations: node.location ? [node.location] : [], - }) - ); - } else { + if (!isArgumentValidType(firstArg, 'variable')) { + if (isMathNode(firstArg)) { + errors.push( + getMessageFromId({ + messageId: 'wrongFirstArgument', + values: { + operation: node.name, + type: i18n.translate('xpack.lens.indexPattern.formulaFieldValue', { + defaultMessage: 'field', + }), + argument: `math operation`, + }, + locations: node.location ? [node.location] : [], + }) + ); + } else { + if (shouldHaveFieldArgument(node)) { errors.push( getMessageFromId({ messageId: 'wrongFirstArgument', @@ -673,41 +673,28 @@ function runFullASTValidation( }) ); } - } else { - // If the first argument is valid proceed with the other arguments validation - const fieldErrors = validateFieldArguments(node, variables, { - isFieldOperation: true, - firstArg, - returnedType: getReturnedType(nodeOperation, indexPattern, firstArg), - }); - if (fieldErrors.length) { - errors.push(...fieldErrors); - } } - const functionErrors = validateFunctionArguments(node, functions, 0, { + } else { + // If the first argument is valid proceed with the other arguments validation + const fieldErrors = validateFieldArguments(node, variables, { isFieldOperation: true, - type: i18n.translate('xpack.lens.indexPattern.formulaFieldValue', { - defaultMessage: 'field', - }), - firstArgValidation: false, + firstArg, + returnedType: getReturnedType(nodeOperation, indexPattern, firstArg), }); - if (functionErrors.length) { - errors.push(...functionErrors); - } - } else { - // Named arguments only - if (functions?.length || variables?.length) { - errors.push( - getMessageFromId({ - messageId: 'shouldNotHaveField', - values: { - operation: node.name, - }, - locations: node.location ? [node.location] : [], - }) - ); + if (fieldErrors.length) { + errors.push(...fieldErrors); } } + const functionErrors = validateFunctionArguments(node, functions, 0, { + isFieldOperation: true, + type: i18n.translate('xpack.lens.indexPattern.formulaFieldValue', { + defaultMessage: 'field', + }), + firstArgValidation: false, + }); + if (functionErrors.length) { + errors.push(...functionErrors); + } if (!canHaveParams(nodeOperation) && namedArguments.length) { errors.push( getMessageFromId({ diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx index 335677b279132..c4661d6799df5 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/terms/terms.test.tsx @@ -2555,13 +2555,19 @@ describe('terms', () => { const functionComboBox = comboBoxes.filter( '[data-test-subj="indexPattern-reference-function"]' ); - const fieldComboBox = comboBoxes.filter('[data-test-subj="indexPattern-dimension-field"]'); const option = functionComboBox.prop('options')!.find(({ label }) => label === 'Average')!; act(() => { functionComboBox.prop('onChange')!([option]); }); + instance.update(); - expect(fieldComboBox.prop('isInvalid')).toBeTruthy(); + expect( + instance + .find('ReferenceEditor') + .find(EuiComboBox) + .filter('[data-test-subj="indexPattern-dimension-field"]') + .prop('isInvalid') + ).toBeTruthy(); expect(updateLayerSpy).not.toHaveBeenCalled(); }); diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts index 0f626cef850f9..4e3ee9b1ff66f 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/operations/operations.test.ts @@ -56,7 +56,7 @@ describe('getOperationTypesForField', () => { aggregatable: true, searchable: true, }) - ).toEqual(['terms', 'unique_count', 'last_value']); + ).toEqual(['terms', 'unique_count', 'last_value', 'count']); }); it('should return only bucketed operations on strings when passed proper filterOperations function', () => { @@ -96,6 +96,7 @@ describe('getOperationTypesForField', () => { 'percentile', 'percentile_rank', 'last_value', + 'count', ]); }); @@ -122,6 +123,7 @@ describe('getOperationTypesForField', () => { 'percentile', 'percentile_rank', 'last_value', + 'count', ]); }); @@ -388,6 +390,21 @@ describe('getOperationTypesForField', () => { "operationType": "last_value", "type": "field", }, + Object { + "field": "timestamp", + "operationType": "count", + "type": "field", + }, + Object { + "field": "bytes", + "operationType": "count", + "type": "field", + }, + Object { + "field": "source", + "operationType": "count", + "type": "field", + }, Object { "operationType": "math", "type": "managedReference", diff --git a/x-pack/plugins/lens/public/indexpattern_datasource/pure_helpers.ts b/x-pack/plugins/lens/public/indexpattern_datasource/pure_helpers.ts index e22fb30b23b27..6c81634fb4c09 100644 --- a/x-pack/plugins/lens/public/indexpattern_datasource/pure_helpers.ts +++ b/x-pack/plugins/lens/public/indexpattern_datasource/pure_helpers.ts @@ -7,6 +7,7 @@ import { keyBy } from 'lodash'; import { IndexPatternField, IndexPatternPrivateState } from './types'; +import { documentField } from './document_field'; export function fieldExists( existingFields: IndexPatternPrivateState['existingFields'], @@ -16,7 +17,13 @@ export function fieldExists( return existingFields[indexPatternTitle] && existingFields[indexPatternTitle][fieldName]; } -export function getFieldByNameFactory(newFields: IndexPatternField[]) { +export function getFieldByNameFactory( + newFields: IndexPatternField[], + addRecordsField: boolean = true +) { const fieldsLookup = keyBy(newFields, 'name'); + if (addRecordsField) { + fieldsLookup[documentField.name] = documentField; + } return (name: string) => fieldsLookup[name]; } diff --git a/x-pack/test/functional/apps/lens/group3/drag_and_drop.ts b/x-pack/test/functional/apps/lens/group3/drag_and_drop.ts index 5022852bf08ff..bfb573636383d 100644 --- a/x-pack/test/functional/apps/lens/group3/drag_and_drop.ts +++ b/x-pack/test/functional/apps/lens/group3/drag_and_drop.ts @@ -105,7 +105,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.getDimensionTriggersTexts('lnsXY_splitDimensionPanel') ).to.eql([]); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_yDimensionPanel')).to.eql([ - 'Unique count of @message.raw', + 'Value count of @message.raw', ]); }); it('should duplicate the column when dragging to empty dimension in the same group', async () => { @@ -118,9 +118,9 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { to: 'lnsXY_yDimensionPanel > lns-empty-dimension', }); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_yDimensionPanel')).to.eql([ - 'Unique count of @message.raw', - 'Unique count of @message.raw [1]', - 'Unique count of @message.raw [2]', + 'Value count of @message.raw', + 'Value count of @message.raw [1]', + 'Value count of @message.raw [2]', ]); }); it('should move duplicated column to non-compatible dimension group', async () => { @@ -129,8 +129,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { to: 'lnsXY_xDimensionPanel > lns-empty-dimension', }); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_yDimensionPanel')).to.eql([ - 'Unique count of @message.raw', - 'Unique count of @message.raw [1]', + 'Value count of @message.raw', + 'Value count of @message.raw [1]', ]); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_xDimensionPanel')).to.eql([ 'Top 5 values of @message.raw', @@ -160,7 +160,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { 'swap' ); expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_yDimensionPanel')).to.eql( - 'Unique count of @timestamp' + 'Value count of @timestamp' ); expect(await PageObjects.lens.getDimensionTriggerText('lnsXY_splitDimensionPanel')).to.eql( 'Top 3 values of @message.raw' @@ -297,14 +297,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.dimensionKeyboardDragDrop('lnsXY_xDimensionPanel', 0, 2); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_yDimensionPanel')).to.eql([ 'Count of records', - 'Unique count of @timestamp', + 'Value count of @timestamp', ]); - await PageObjects.lens.assertFocusedDimension('Unique count of @timestamp'); + await PageObjects.lens.assertFocusedDimension('Value count of @timestamp'); }); it('should reorder elements with keyboard', async () => { await PageObjects.lens.dimensionKeyboardReorder('lnsXY_yDimensionPanel', 0, 1); expect(await PageObjects.lens.getDimensionTriggersTexts('lnsXY_yDimensionPanel')).to.eql([ - 'Unique count of @timestamp', + 'Value count of @timestamp', 'Count of records', ]); await PageObjects.lens.assertFocusedDimension('Count of records'); diff --git a/x-pack/test/functional/apps/lens/group3/gauge.ts b/x-pack/test/functional/apps/lens/group3/gauge.ts index ea029793ddfc0..0a283c4fc62f7 100644 --- a/x-pack/test/functional/apps/lens/group3/gauge.ts +++ b/x-pack/test/functional/apps/lens/group3/gauge.ts @@ -53,7 +53,8 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.lens.configureDimension({ dimension: 'lnsGauge_metricDimensionPanel > lns-dimensionTrigger', operation: 'count', - isPreviousIncompatible: true, + field: 'Records', + isPreviousIncompatible: false, keepOpen: true, });