[ML] Data frame transform: Fix progress in wizard create step.

x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json

@@ -0,0 +1,130 @@
+{
+  "count" : 4,
+  "transforms" : [
+    {
+      "id" : "transform-created",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 0,
+        "documents_processed" : 0,
+        "documents_indexed" : 0,
+        "trigger_count" : 0,
+        "index_time_in_ms" : 0,
+        "index_total" : 0,
+        "index_failures" : 0,
+        "search_time_in_ms" : 0,
+        "search_total" : 0,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "operations_behind" : 0
+      }
+    },
+    {
+      "id" : "transform-created-started-stopped",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 2,
+        "documents_processed" : 2864381,
+        "documents_indexed" : 1000,
+        "trigger_count" : 1,
+        "index_time_in_ms" : 52,
+        "index_total" : 2,
+        "index_failures" : 0,
+        "search_time_in_ms" : 1859,
+        "search_total" : 3,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "operations_behind" : 21139495
+      }
+    },
+    {
+      "id" : "transform-running",
+      "state" : "indexing",
+      "node" : {
+        "id" : "xxx",
+        "name" : "node",
+        "ephemeral_id" : "xxx",
+        "transport_address" : "127.0.0.1:9300",
+        "attributes" : { }
+      },
+      "stats" : {
+        "pages_processed" : 3,
+        "documents_processed" : 4457902,
+        "documents_indexed" : 1500,
+        "trigger_count" : 2,
+        "index_time_in_ms" : 101,
+        "index_total" : 3,
+        "index_failures" : 0,
+        "search_time_in_ms" : 1909,
+        "search_total" : 4,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "next" : {
+          "checkpoint" : 1,
+          "position" : {
+            "indexer_position" : {
+              "@timestamp" : 1551282000000,
+              "agent.version" : "7.0.0"
+            }
+          },
+          "checkpoint_progress" : {
+            "docs_remaining" : 16681594,
+            "total_docs" : 21139496,
+            "percent_complete" : 21.08802404749858,
+            "docs_indexed" : 1500,
+            "docs_processed" : 4457902
+          },
+          "timestamp_millis" : 1568021396011
+        },
+        "operations_behind" : 21139495
+      }
+    },
+    {
+      "id" : "transform-completed",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 22,
+        "documents_processed" : 21139496,
+        "documents_indexed" : 10154,
+        "trigger_count" : 2,
+        "index_time_in_ms" : 530,
+        "index_total" : 21,
+        "index_failures" : 0,
+        "search_time_in_ms" : 8566,
+        "search_total" : 23,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 1,
+          "timestamp_millis" : 1568021396011
+        },
+        "operations_behind" : 0
+      }
+    }
+  ]
+}

x-pack/legacy/plugins/ml/public/data_frame/common/index.ts

@@ -29,6 +29,15 @@ export {
   IndexPattern,
   REFRESH_TRANSFORM_LIST_STATE,
 } from './transform';
+export { DATA_FRAME_TRANSFORM_LIST_COLUMN, DataFrameTransformListRow } from './transform_list';
+export {
+  getTransformProgress,
+  isCompletedBatchTransform,
+  isDataFrameTransformStats,
+  DataFrameTransformStats,
+  DATA_FRAME_MODE,
+  DATA_FRAME_TRANSFORM_STATE,
+} from './transform_stats';
 export { moveToDataFrameTransformList, moveToDataFrameWizard, moveToDiscover } from './navigation';
 export {
   getEsAggFromAggConfig,

x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataFrameTransformId, DataFrameTransformPivotConfig } from './transform';
+import { DataFrameTransformStats } from './transform_stats';
+
+// Used to pass on attribute names to table columns
+export enum DATA_FRAME_TRANSFORM_LIST_COLUMN {
+  CONFIG_DEST_INDEX = 'config.dest.index',
+  CONFIG_SOURCE_INDEX = 'config.source.index',
+  DESCRIPTION = 'config.description',
+  ID = 'id',
+}
+
+export interface DataFrameTransformListRow {
+  id: DataFrameTransformId;
+  config: DataFrameTransformPivotConfig;
+  mode?: string; // added property on client side to allow filtering by this field
+  stats: DataFrameTransformStats;
+}

x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import mockDataFrameTransformStats from './__mocks__/data_frame_transform_stats.json';
+
+import { DataFrameTransformListRow } from './transform_list';
+import { getTransformProgress, isCompletedBatchTransform } from './transform_stats';
+
+const getRow = (statsId: string) => {
+  return {
+    ...(mockDataFrameTransformListRow as DataFrameTransformListRow),
+    stats: {
+      ...mockDataFrameTransformStats.transforms.find(
+        (stats: DataFrameTransformListRow['stats']) => stats.id === statsId
+      ),
+    },
+  };
+};
+
+describe('Data Frame: Transform stats.', () => {
+  test('getTransformProgress()', () => {
+    // At the moment, any kind of stopped jobs don't include progress information.
+    // We cannot infer progress for now from an unfinished job that has been stopped for now.
+    expect(getTransformProgress(getRow('transform-created'))).toBe(undefined);
+    expect(getTransformProgress(getRow('transform-created-started-stopped'))).toBe(undefined);
+    expect(getTransformProgress(getRow('transform-running'))).toBe(21);
+    expect(getTransformProgress(getRow('transform-completed'))).toBe(100);
+  });
+
+  test('isCompletedBatchTransform()', () => {
+    expect(isCompletedBatchTransform(getRow('transform-created'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-created-started-stopped'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-running'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-completed'))).toBe(true);
+  });
+});

x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { idx } from '@kbn/elastic-idx';
+
+import { DataFrameTransformId } from './transform';
+import { DataFrameTransformListRow } from './transform_list';
+
+// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243
+export enum DATA_FRAME_TRANSFORM_STATE {
+  ABORTING = 'aborting',
+  FAILED = 'failed',
+  INDEXING = 'indexing',
+  STARTED = 'started',
+  STOPPED = 'stopped',
+  STOPPING = 'stopping',
+}
+
+export enum DATA_FRAME_MODE {
+  BATCH = 'batch',
+  CONTINUOUS = 'continuous',
+}
+
+export interface DataFrameTransformStats {
+  id: DataFrameTransformId;
+  checkpointing: {
+    last: {
+      checkpoint: number;
+      timestamp_millis?: number;
+    };
+    next?: {
+      checkpoint: number;
+      checkpoint_progress?: {
+        total_docs: number;
+        docs_remaining: number;
+        percent_complete: number;
+      };
+    };
+    operations_behind: number;
+  };
+  node?: {
+    id: string;
+    name: string;
+    ephemeral_id: string;
+    transport_address: string;
+    attributes: Record<string, any>;
+  };
+  stats: {
+    documents_indexed: number;
+    documents_processed: number;
+    index_failures: number;
+    index_time_in_ms: number;
+    index_total: number;
+    pages_processed: number;
+    search_failures: number;
+    search_time_in_ms: number;
+    search_total: number;
+    trigger_count: number;
+  };
+  reason?: string;
+  state: DATA_FRAME_TRANSFORM_STATE;
+}
+
+export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats {
+  return (
+    typeof arg === 'object' &&
+    arg !== null &&
+    {}.hasOwnProperty.call(arg, 'state') &&
+    Object.values(DATA_FRAME_TRANSFORM_STATE).includes(arg.state)
+  );
+}
+
+export function getTransformProgress(item: DataFrameTransformListRow) {
+  if (isCompletedBatchTransform(item)) {
+    return 100;
+  }
+
+  const progress = idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete);
+
+  return progress !== undefined ? Math.round(progress) : undefined;
+}
+
+export function isCompletedBatchTransform(item: DataFrameTransformListRow) {
+  // If `checkpoint=1`, `sync` is missing from the config and state is stopped,
+  // then this is a completed batch data frame transform.
+  return (
+    item.stats.checkpointing.last.checkpoint === 1 &&
+    item.config.sync === undefined &&
+    item.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED
+  );
+}

x-pack/legacy/plugins/ml/public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx

@@ -5,7 +5,6 @@
  */
 
 import React, { Fragment, SFC, useEffect, useState } from 'react';
-import { idx } from '@kbn/elastic-idx';
 import { i18n } from '@kbn/i18n';
 import { toastNotifications } from 'ui/notify';
 
@@ -35,7 +34,11 @@ import { useKibanaContext } from '../../../../../contexts/kibana/use_kibana_cont
 import { useUiChromeContext } from '../../../../../contexts/ui/use_ui_chrome_context';
 import { PROGRESS_JOBS_REFRESH_INTERVAL_MS } from '../../../../../../common/constants/jobs_list';
 
-import { moveToDataFrameTransformList, moveToDiscover } from '../../../../common';
+import {
+  getTransformProgress,
+  moveToDataFrameTransformList,
+  moveToDiscover,
+} from '../../../../common';
 
 export interface StepDetailsExposedState {
   created: boolean;
@@ -198,12 +201,12 @@ export const StepCreateForm: SFC<Props> = React.memo(
           try {
             const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId);
             if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) {
-              const percent = Math.round(
-                idx(
-                  stats,
-                  _ => _.transforms[0].checkpointing.next.checkpoint_progress.percent_complete
-                ) || 0
-              );
+              const percent =
+                getTransformProgress({
+                  id: transformConfig.id,
+                  config: transformConfig,
+                  stats: stats.transforms[0],
+                }) || 0;
               setProgressPercentComplete(percent);
               if (percent >= 100) {
                 clearInterval(interval);

x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx

@@ -7,10 +7,10 @@
 import { shallow } from 'enzyme';
 import React from 'react';
 
-import { DataFrameTransformListRow } from './common';
+import { DataFrameTransformListRow } from '../../../../common';
 import { DeleteAction } from './action_delete';
 
-import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';
 
 describe('Data Frame: Transform List Actions <DeleteAction />', () => {
   test('Minimal initialization', () => {

x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx

@@ -21,7 +21,7 @@ import {
   createPermissionFailureMessage,
 } from '../../../../../privilege/check_privilege';
 
-import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common';
+import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common';
 
 interface DeleteActionProps {
   items: DataFrameTransformListRow[];

x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx

@@ -7,10 +7,10 @@
 import { shallow } from 'enzyme';
 import React from 'react';
 
-import { DataFrameTransformListRow } from './common';
+import { DataFrameTransformListRow } from '../../../../common';
 import { StartAction } from './action_start';
 
-import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';
 
 describe('Data Frame: Transform List Actions <StartAction />', () => {
   test('Minimal initialization', () => {

x-pack/legacy/plugins/ml/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx

@@ -25,7 +25,7 @@ import {
   DataFrameTransformListRow,
   isCompletedBatchTransform,
   DATA_FRAME_TRANSFORM_STATE,
-} from './common';
+} from '../../../../common';
 
 interface StartActionProps {
   items: DataFrameTransformListRow[];