[ML] Data frame transform: Fix progress in wizard create step. (elast…

…ic#45116) Fixes a regression where the progress bar in the transform wizard would return to 0% after a transform finishes. The actual bugfix is just the code in 6033a4c to make sure the code in the wizard uses the same function getTransformProgress() like the transform management page. Also includes a fix for sorting on the transform list table related to progress. The rest of the diff is a) moving types and utility functions previously only used within the transform management page to the common shared code of data frame transforms so it's available for the wizard without cross-deep-importing (717d894) and b) adding tests with updated mock data (f14e231).
walterra · Sep 10, 2019 · 665fbc7 · 665fbc7
1 parent 978f153
commit 665fbc7
Show file tree

Hide file tree

Showing 28 changed files with 371 additions and 167 deletions.
diff --git a/...ocks__/data_frame_transform_list_row.json → ...ocks__/data_frame_transform_list_row.json b/...ocks__/data_frame_transform_list_row.json → ...ocks__/data_frame_transform_list_row.json
diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json b/x-pack/legacy/plugins/ml/public/data_frame/common/__mocks__/data_frame_transform_stats.json
@@ -0,0 +1,130 @@
+{
+  "count" : 4,
+  "transforms" : [
+    {
+      "id" : "transform-created",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 0,
+        "documents_processed" : 0,
+        "documents_indexed" : 0,
+        "trigger_count" : 0,
+        "index_time_in_ms" : 0,
+        "index_total" : 0,
+        "index_failures" : 0,
+        "search_time_in_ms" : 0,
+        "search_total" : 0,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "operations_behind" : 0
+      }
+    },
+    {
+      "id" : "transform-created-started-stopped",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 2,
+        "documents_processed" : 2864381,
+        "documents_indexed" : 1000,
+        "trigger_count" : 1,
+        "index_time_in_ms" : 52,
+        "index_total" : 2,
+        "index_failures" : 0,
+        "search_time_in_ms" : 1859,
+        "search_total" : 3,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "operations_behind" : 21139495
+      }
+    },
+    {
+      "id" : "transform-running",
+      "state" : "indexing",
+      "node" : {
+        "id" : "xxx",
+        "name" : "node",
+        "ephemeral_id" : "xxx",
+        "transport_address" : "127.0.0.1:9300",
+        "attributes" : { }
+      },
+      "stats" : {
+        "pages_processed" : 3,
+        "documents_processed" : 4457902,
+        "documents_indexed" : 1500,
+        "trigger_count" : 2,
+        "index_time_in_ms" : 101,
+        "index_total" : 3,
+        "index_failures" : 0,
+        "search_time_in_ms" : 1909,
+        "search_total" : 4,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 0
+        },
+        "next" : {
+          "checkpoint" : 1,
+          "position" : {
+            "indexer_position" : {
+              "@timestamp" : 1551282000000,
+              "agent.version" : "7.0.0"
+            }
+          },
+          "checkpoint_progress" : {
+            "docs_remaining" : 16681594,
+            "total_docs" : 21139496,
+            "percent_complete" : 21.08802404749858,
+            "docs_indexed" : 1500,
+            "docs_processed" : 4457902
+          },
+          "timestamp_millis" : 1568021396011
+        },
+        "operations_behind" : 21139495
+      }
+    },
+    {
+      "id" : "transform-completed",
+      "state" : "stopped",
+      "stats" : {
+        "pages_processed" : 22,
+        "documents_processed" : 21139496,
+        "documents_indexed" : 10154,
+        "trigger_count" : 2,
+        "index_time_in_ms" : 530,
+        "index_total" : 21,
+        "index_failures" : 0,
+        "search_time_in_ms" : 8566,
+        "search_total" : 23,
+        "search_failures" : 0,
+        "exponential_avg_checkpoint_duration_ms" : 0.0,
+        "exponential_avg_documents_indexed" : 0.0,
+        "exponential_avg_documents_processed" : 0.0
+      },
+      "checkpointing" : {
+        "last" : {
+          "checkpoint" : 1,
+          "timestamp_millis" : 1568021396011
+        },
+        "operations_behind" : 0
+      }
+    }
+  ]
+}
diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/index.ts
@@ -28,6 +28,15 @@ export {
   IndexPattern,
   REFRESH_TRANSFORM_LIST_STATE,
 } from './transform';
+export { DATA_FRAME_TRANSFORM_LIST_COLUMN, DataFrameTransformListRow } from './transform_list';
+export {
+  getTransformProgress,
+  isCompletedBatchTransform,
+  isDataFrameTransformStats,
+  DataFrameTransformStats,
+  DATA_FRAME_MODE,
+  DATA_FRAME_TRANSFORM_STATE,
+} from './transform_stats';
 export { moveToDataFrameTransformList, moveToDataFrameWizard, moveToDiscover } from './navigation';
 export {
   getEsAggFromAggConfig,

diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_list.ts
@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { DataFrameTransformId, DataFrameTransformPivotConfig } from './transform';
+import { DataFrameTransformStats } from './transform_stats';
+
+// Used to pass on attribute names to table columns
+export enum DATA_FRAME_TRANSFORM_LIST_COLUMN {
+  CONFIG_DEST_INDEX = 'config.dest.index',
+  CONFIG_SOURCE_INDEX = 'config.source.index',
+  DESCRIPTION = 'config.description',
+  ID = 'id',
+}
+
+export interface DataFrameTransformListRow {
+  id: DataFrameTransformId;
+  config: DataFrameTransformPivotConfig;
+  mode?: string; // added property on client side to allow filtering by this field
+  stats: DataFrameTransformStats;
+}
diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.test.ts
@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import mockDataFrameTransformStats from './__mocks__/data_frame_transform_stats.json';
+
+import { DataFrameTransformListRow } from './transform_list';
+import { getTransformProgress, isCompletedBatchTransform } from './transform_stats';
+
+const getRow = (statsId: string) => {
+  return {
+    ...(mockDataFrameTransformListRow as DataFrameTransformListRow),
+    stats: {
+      ...mockDataFrameTransformStats.transforms.find(
+        (stats: DataFrameTransformListRow['stats']) => stats.id === statsId
+      ),
+    },
+  };
+};
+
+describe('Data Frame: Transform stats.', () => {
+  test('getTransformProgress()', () => {
+    // At the moment, any kind of stopped jobs don't include progress information.
+    // We cannot infer progress for now from an unfinished job that has been stopped for now.
+    expect(getTransformProgress(getRow('transform-created'))).toBe(undefined);
+    expect(getTransformProgress(getRow('transform-created-started-stopped'))).toBe(undefined);
+    expect(getTransformProgress(getRow('transform-running'))).toBe(21);
+    expect(getTransformProgress(getRow('transform-completed'))).toBe(100);
+  });
+
+  test('isCompletedBatchTransform()', () => {
+    expect(isCompletedBatchTransform(getRow('transform-created'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-created-started-stopped'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-running'))).toBe(false);
+    expect(isCompletedBatchTransform(getRow('transform-completed'))).toBe(true);
+  });
+});
diff --git a/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts b/x-pack/legacy/plugins/ml/public/data_frame/common/transform_stats.ts
@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { idx } from '@kbn/elastic-idx';
+
+import { DataFrameTransformId } from './transform';
+import { DataFrameTransformListRow } from './transform_list';
+
+// reflects https://github.com/elastic/elasticsearch/blob/master/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/transforms/DataFrameTransformStats.java#L243
+export enum DATA_FRAME_TRANSFORM_STATE {
+  ABORTING = 'aborting',
+  FAILED = 'failed',
+  INDEXING = 'indexing',
+  STARTED = 'started',
+  STOPPED = 'stopped',
+  STOPPING = 'stopping',
+}
+
+export enum DATA_FRAME_MODE {
+  BATCH = 'batch',
+  CONTINUOUS = 'continuous',
+}
+
+export interface DataFrameTransformStats {
+  id: DataFrameTransformId;
+  checkpointing: {
+    last: {
+      checkpoint: number;
+      timestamp_millis?: number;
+    };
+    next?: {
+      checkpoint: number;
+      checkpoint_progress?: {
+        total_docs: number;
+        docs_remaining: number;
+        percent_complete: number;
+      };
+    };
+    operations_behind: number;
+  };
+  node?: {
+    id: string;
+    name: string;
+    ephemeral_id: string;
+    transport_address: string;
+    attributes: Record<string, any>;
+  };
+  stats: {
+    documents_indexed: number;
+    documents_processed: number;
+    index_failures: number;
+    index_time_in_ms: number;
+    index_total: number;
+    pages_processed: number;
+    search_failures: number;
+    search_time_in_ms: number;
+    search_total: number;
+    trigger_count: number;
+  };
+  reason?: string;
+  state: DATA_FRAME_TRANSFORM_STATE;
+}
+
+export function isDataFrameTransformStats(arg: any): arg is DataFrameTransformStats {
+  return (
+    typeof arg === 'object' &&
+    arg !== null &&
+    {}.hasOwnProperty.call(arg, 'state') &&
+    Object.values(DATA_FRAME_TRANSFORM_STATE).includes(arg.state)
+  );
+}
+
+export function getTransformProgress(item: DataFrameTransformListRow) {
+  if (isCompletedBatchTransform(item)) {
+    return 100;
+  }
+
+  const progress = idx(item, _ => _.stats.checkpointing.next.checkpoint_progress.percent_complete);
+
+  return progress !== undefined ? Math.round(progress) : undefined;
+}
+
+export function isCompletedBatchTransform(item: DataFrameTransformListRow) {
+  // If `checkpoint=1`, `sync` is missing from the config and state is stopped,
+  // then this is a completed batch data frame transform.
+  return (
+    item.stats.checkpointing.last.checkpoint === 1 &&
+    item.config.sync === undefined &&
+    item.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED
+  );
+}
diff --git a/.../public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx b/.../public/data_frame/pages/data_frame_new_pivot/components/step_create/step_create_form.tsx
@@ -5,7 +5,6 @@
  */
 
 import React, { Fragment, SFC, useEffect, useState } from 'react';
-import { idx } from '@kbn/elastic-idx';
 import { i18n } from '@kbn/i18n';
 import { toastNotifications } from 'ui/notify';
 
@@ -35,7 +34,11 @@ import { useKibanaContext } from '../../../../../contexts/kibana/use_kibana_cont
 import { useUiChromeContext } from '../../../../../contexts/ui/use_ui_chrome_context';
 import { PROGRESS_JOBS_REFRESH_INTERVAL_MS } from '../../../../../../common/constants/jobs_list';
 
-import { moveToDataFrameTransformList, moveToDiscover } from '../../../../common';
+import {
+  getTransformProgress,
+  moveToDataFrameTransformList,
+  moveToDiscover,
+} from '../../../../common';
 
 export interface StepDetailsExposedState {
   created: boolean;
@@ -198,12 +201,12 @@ export const StepCreateForm: SFC<Props> = React.memo(
           try {
             const stats = await ml.dataFrame.getDataFrameTransformsStats(transformId);
             if (stats && Array.isArray(stats.transforms) && stats.transforms.length > 0) {
-              const percent = Math.round(
-                idx(
-                  stats,
-                  _ => _.transforms[0].checkpointing.next.checkpoint_progress.percent_complete
-                ) || 0
-              );
+              const percent =
+                getTransformProgress({
+                  id: transformConfig.id,
+                  config: transformConfig,
+                  stats: stats.transforms[0],
+                }) || 0;
               setProgressPercentComplete(percent);
               if (percent >= 100) {
                 clearInterval(interval);

diff --git a/...ic/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx b/...ic/data_frame/pages/transform_management/components/transform_list/action_delete.test.tsx
@@ -7,10 +7,10 @@
 import { shallow } from 'enzyme';
 import React from 'react';
 
-import { DataFrameTransformListRow } from './common';
+import { DataFrameTransformListRow } from '../../../../common';
 import { DeleteAction } from './action_delete';
 
-import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';
 
 describe('Data Frame: Transform List Actions <DeleteAction />', () => {
   test('Minimal initialization', () => {

diff --git a/.../public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx b/.../public/data_frame/pages/transform_management/components/transform_list/action_delete.tsx
@@ -21,7 +21,7 @@ import {
   createPermissionFailureMessage,
 } from '../../../../../privilege/check_privilege';
 
-import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from './common';
+import { DataFrameTransformListRow, DATA_FRAME_TRANSFORM_STATE } from '../../../../common';
 
 interface DeleteActionProps {
   items: DataFrameTransformListRow[];

diff --git a/...lic/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx b/...lic/data_frame/pages/transform_management/components/transform_list/action_start.test.tsx
@@ -7,10 +7,10 @@
 import { shallow } from 'enzyme';
 import React from 'react';
 
-import { DataFrameTransformListRow } from './common';
+import { DataFrameTransformListRow } from '../../../../common';
 import { StartAction } from './action_start';
 
-import dataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
+import dataFrameTransformListRow from '../../../../common/__mocks__/data_frame_transform_list_row.json';
 
 describe('Data Frame: Transform List Actions <StartAction />', () => {
   test('Minimal initialization', () => {

diff --git a/...l/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx b/...l/public/data_frame/pages/transform_management/components/transform_list/action_start.tsx
@@ -25,7 +25,7 @@ import {
   DataFrameTransformListRow,
   isCompletedBatchTransform,
   DATA_FRAME_TRANSFORM_STATE,
-} from './common';
+} from '../../../../common';
 
 interface StartActionProps {
   items: DataFrameTransformListRow[];