[Lens] Calculation operations (elastic#83789)

# Conflicts:
#	x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts
#	x-pack/plugins/lens/public/indexpattern_datasource/operations/layer_helpers.ts

x-pack/plugins/lens/public/editor_frame_service/editor_frame/workspace_panel/workspace_panel.tsx

@@ -161,7 +161,7 @@ export function WorkspacePanel({
 
   const expression = useMemo(
     () => {
-      if (!configurationValidationError) {
+      if (!configurationValidationError || configurationValidationError.length === 0) {
         try {
           return buildExpression({
             visualization: activeVisualization,

x-pack/plugins/lens/public/indexpattern_datasource/index.ts

@@ -37,12 +37,14 @@ export class IndexPatternDatasource {
         getIndexPatternDatasource,
         renameColumns,
         formatColumn,
+        counterRate,
         getTimeScaleFunction,
         getSuffixFormatter,
       } = await import('../async_services');
       return core.getStartServices().then(([coreStart, { data }]) => {
         data.fieldFormats.register([getSuffixFormatter(data.fieldFormats.deserialize)]);
         expressions.registerFunction(getTimeScaleFunction(data));
+        expressions.registerFunction(counterRate);
         expressions.registerFunction(renameColumns);
         expressions.registerFunction(formatColumn);
         return getIndexPatternDatasource({

x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.test.ts

@@ -599,9 +599,39 @@ describe('IndexPattern Data Source', () => {
 
     describe('getTableSpec', () => {
       it('should include col1', () => {
-        expect(publicAPI.getTableSpec()).toEqual([
-          {
-            columnId: 'col1',
+        expect(publicAPI.getTableSpec()).toEqual([{ columnId: 'col1' }]);
+      });
+
+      it('should skip columns that are being referenced', () => {
+        publicAPI = indexPatternDatasource.getPublicAPI({
+          state: {
+            ...enrichBaseState(baseState),
+            layers: {
+              first: {
+                indexPatternId: '1',
+                columnOrder: ['col1', 'col2'],
+                columns: {
+                  col1: {
+                    label: 'Sum',
+                    dataType: 'number',
+                    isBucketed: false,
+
+                    operationType: 'sum',
+                    sourceField: 'test',
+                    params: {},
+                  } as IndexPatternColumn,
+                  col2: {
+                    label: 'Cumulative sum',
+                    dataType: 'number',
+                    isBucketed: false,
+
+                    operationType: 'cumulative_sum',
+                    references: ['col1'],
+                    params: {},
+                  } as IndexPatternColumn,
+                },
+              },
+            },
           },
         ]);
       });

x-pack/plugins/lens/public/indexpattern_datasource/indexpattern.tsx

@@ -78,6 +78,7 @@ export function columnToOperation(column: IndexPatternColumn, uniqueLabel?: stri
 export * from './rename_columns';
 export * from './format_column';
 export * from './time_scale';
+export * from './counter_rate';
 export * from './suffix_formatter';
 
 export function getIndexPatternDatasource({

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/counter_rate.tsx

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types';
+import { IndexPatternLayer } from '../../../types';
+import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils';
+import { OperationDefinition } from '..';
+
+const ofName = (name?: string) => {
+  return i18n.translate('xpack.lens.indexPattern.CounterRateOf', {
+    defaultMessage: 'Counter rate of {name}',
+    values: {
+      name:
+        name ??
+        i18n.translate('xpack.lens.indexPattern.incompleteOperation', {
+          defaultMessage: '(incomplete)',
+        }),
+    },
+  });
+};
+
+export type CounterRateIndexPatternColumn = FormattedIndexPatternColumn &
+  ReferenceBasedIndexPatternColumn & {
+    operationType: 'counter_rate';
+  };
+
+export const counterRateOperation: OperationDefinition<
+  CounterRateIndexPatternColumn,
+  'fullReference'
+> = {
+  type: 'counter_rate',
+  priority: 1,
+  displayName: i18n.translate('xpack.lens.indexPattern.counterRate', {
+    defaultMessage: 'Counter rate',
+  }),
+  input: 'fullReference',
+  selectionStyle: 'field',
+  requiredReferences: [
+    {
+      input: ['field'],
+      specificOperations: ['max'],
+      validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
+    },
+  ],
+  getPossibleOperation: () => {
+    return {
+      dataType: 'number',
+      isBucketed: false,
+      scale: 'ratio',
+    };
+  },
+  getDefaultLabel: (column, indexPattern, columns) => {
+    return ofName(columns[column.references[0]]?.label);
+  },
+  toExpression: (layer, columnId) => {
+    return dateBasedOperationToExpression(layer, columnId, 'lens_counter_rate');
+  },
+  buildColumn: ({ referenceIds, previousColumn, layer }) => {
+    const metric = layer.columns[referenceIds[0]];
+    return {
+      label: ofName(metric?.label),
+      dataType: 'number',
+      operationType: 'counter_rate',
+      isBucketed: false,
+      scale: 'ratio',
+      references: referenceIds,
+      params:
+        previousColumn?.dataType === 'number' &&
+        previousColumn.params &&
+        'format' in previousColumn.params &&
+        previousColumn.params.format
+          ? { format: previousColumn.params.format }
+          : undefined,
+    };
+  },
+  isTransferable: (column, newIndexPattern) => {
+    return hasDateField(newIndexPattern);
+  },
+  getErrorMessage: (layer: IndexPatternLayer) => {
+    return checkForDateHistogram(
+      layer,
+      i18n.translate('xpack.lens.indexPattern.counterRate', {
+        defaultMessage: 'Counter rate',
+      })
+    );
+  },
+};

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/cumulative_sum.tsx

@@ -0,0 +1,91 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types';
+import { IndexPatternLayer } from '../../../types';
+import { checkForDateHistogram, dateBasedOperationToExpression } from './utils';
+import { OperationDefinition } from '..';
+
+const ofName = (name?: string) => {
+  return i18n.translate('xpack.lens.indexPattern.cumulativeSumOf', {
+    defaultMessage: 'Cumulative sum rate of {name}',
+    values: {
+      name:
+        name ??
+        i18n.translate('xpack.lens.indexPattern.incompleteOperation', {
+          defaultMessage: '(incomplete)',
+        }),
+    },
+  });
+};
+
+export type CumulativeSumIndexPatternColumn = FormattedIndexPatternColumn &
+  ReferenceBasedIndexPatternColumn & {
+    operationType: 'cumulative_sum';
+  };
+
+export const cumulativeSumOperation: OperationDefinition<
+  CumulativeSumIndexPatternColumn,
+  'fullReference'
+> = {
+  type: 'cumulative_sum',
+  priority: 1,
+  displayName: i18n.translate('xpack.lens.indexPattern.cumulativeSum', {
+    defaultMessage: 'Cumulative sum',
+  }),
+  input: 'fullReference',
+  selectionStyle: 'field',
+  requiredReferences: [
+    {
+      input: ['field'],
+      specificOperations: ['count', 'sum'],
+      validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
+    },
+  ],
+  getPossibleOperation: () => {
+    return {
+      dataType: 'number',
+      isBucketed: false,
+      scale: 'ratio',
+    };
+  },
+  getDefaultLabel: (column, indexPattern, columns) => {
+    return ofName(columns[column.references[0]]?.label);
+  },
+  toExpression: (layer, columnId) => {
+    return dateBasedOperationToExpression(layer, columnId, 'cumulative_sum');
+  },
+  buildColumn: ({ referenceIds, previousColumn, layer }) => {
+    const metric = layer.columns[referenceIds[0]];
+    return {
+      label: ofName(metric?.label),
+      dataType: 'number',
+      operationType: 'cumulative_sum',
+      isBucketed: false,
+      scale: 'ratio',
+      references: referenceIds,
+      params:
+        previousColumn?.dataType === 'number' &&
+        previousColumn.params &&
+        'format' in previousColumn.params &&
+        previousColumn.params.format
+          ? { format: previousColumn.params.format }
+          : undefined,
+    };
+  },
+  isTransferable: () => {
+    return true;
+  },
+  getErrorMessage: (layer: IndexPatternLayer) => {
+    return checkForDateHistogram(
+      layer,
+      i18n.translate('xpack.lens.indexPattern.cumulativeSum', {
+        defaultMessage: 'Cumulative sum',
+      })
+    );
+  },
+};

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/derivative.tsx

@@ -0,0 +1,90 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedIndexPatternColumn, ReferenceBasedIndexPatternColumn } from '../column_types';
+import { IndexPatternLayer } from '../../../types';
+import { checkForDateHistogram, dateBasedOperationToExpression, hasDateField } from './utils';
+import { OperationDefinition } from '..';
+
+const ofName = (name?: string) => {
+  return i18n.translate('xpack.lens.indexPattern.derivativeOf', {
+    defaultMessage: 'Differences of {name}',
+    values: {
+      name:
+        name ??
+        i18n.translate('xpack.lens.indexPattern.incompleteOperation', {
+          defaultMessage: '(incomplete)',
+        }),
+    },
+  });
+};
+
+export type DerivativeIndexPatternColumn = FormattedIndexPatternColumn &
+  ReferenceBasedIndexPatternColumn & {
+    operationType: 'derivative';
+  };
+
+export const derivativeOperation: OperationDefinition<
+  DerivativeIndexPatternColumn,
+  'fullReference'
+> = {
+  type: 'derivative',
+  priority: 1,
+  displayName: i18n.translate('xpack.lens.indexPattern.derivative', {
+    defaultMessage: 'Differences',
+  }),
+  input: 'fullReference',
+  selectionStyle: 'full',
+  requiredReferences: [
+    {
+      input: ['field'],
+      validateMetadata: (meta) => meta.dataType === 'number' && !meta.isBucketed,
+    },
+  ],
+  getPossibleOperation: () => {
+    return {
+      dataType: 'number',
+      isBucketed: false,
+      scale: 'ratio',
+    };
+  },
+  getDefaultLabel: (column, indexPattern, columns) => {
+    return ofName(columns[column.references[0]]?.label);
+  },
+  toExpression: (layer, columnId) => {
+    return dateBasedOperationToExpression(layer, columnId, 'derivative');
+  },
+  buildColumn: ({ referenceIds, previousColumn, layer }) => {
+    const metric = layer.columns[referenceIds[0]];
+    return {
+      label: ofName(metric?.label),
+      dataType: 'number',
+      operationType: 'derivative',
+      isBucketed: false,
+      scale: 'ratio',
+      references: referenceIds,
+      params:
+        previousColumn?.dataType === 'number' &&
+        previousColumn.params &&
+        'format' in previousColumn.params &&
+        previousColumn.params.format
+          ? { format: previousColumn.params.format }
+          : undefined,
+    };
+  },
+  isTransferable: (column, newIndexPattern) => {
+    return hasDateField(newIndexPattern);
+  },
+  getErrorMessage: (layer: IndexPatternLayer) => {
+    return checkForDateHistogram(
+      layer,
+      i18n.translate('xpack.lens.indexPattern.derivative', {
+        defaultMessage: 'Differences',
+      })
+    );
+  },
+};

x-pack/plugins/lens/public/indexpattern_datasource/operations/definitions/calculations/index.ts

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { counterRateOperation, CounterRateIndexPatternColumn } from './counter_rate';
+export { cumulativeSumOperation, CumulativeSumIndexPatternColumn } from './cumulative_sum';
+export { derivativeOperation, DerivativeIndexPatternColumn } from './derivative';
+export { movingAverageOperation, MovingAverageIndexPatternColumn } from './moving_average';