Skip to content

Commit

Permalink
[ML] Updated tests with better mock data.
Browse files Browse the repository at this point in the history
  • Loading branch information
walterra committed Sep 9, 2019
1 parent 717d894 commit f14e231
Show file tree
Hide file tree
Showing 2 changed files with 157 additions and 14 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
{
"count" : 2,
"transforms" : [
{
"id" : "transform-created",
"state" : "stopped",
"stats" : {
"pages_processed" : 0,
"documents_processed" : 0,
"documents_indexed" : 0,
"trigger_count" : 0,
"index_time_in_ms" : 0,
"index_total" : 0,
"index_failures" : 0,
"search_time_in_ms" : 0,
"search_total" : 0,
"search_failures" : 0,
"exponential_avg_checkpoint_duration_ms" : 0.0,
"exponential_avg_documents_indexed" : 0.0,
"exponential_avg_documents_processed" : 0.0
},
"checkpointing" : {
"last" : {
"checkpoint" : 0
},
"operations_behind" : 0
}
},
{
"id" : "transform-created-started-stopped",
"state" : "stopped",
"stats" : {
"pages_processed" : 2,
"documents_processed" : 2864381,
"documents_indexed" : 1000,
"trigger_count" : 1,
"index_time_in_ms" : 52,
"index_total" : 2,
"index_failures" : 0,
"search_time_in_ms" : 1859,
"search_total" : 3,
"search_failures" : 0,
"exponential_avg_checkpoint_duration_ms" : 0.0,
"exponential_avg_documents_indexed" : 0.0,
"exponential_avg_documents_processed" : 0.0
},
"checkpointing" : {
"last" : {
"checkpoint" : 0
},
"operations_behind" : 21139495
}
},
{
"id" : "transform-running",
"state" : "indexing",
"node" : {
"id" : "xxx",
"name" : "node",
"ephemeral_id" : "xxx",
"transport_address" : "127.0.0.1:9300",
"attributes" : { }
},
"stats" : {
"pages_processed" : 3,
"documents_processed" : 4457902,
"documents_indexed" : 1500,
"trigger_count" : 2,
"index_time_in_ms" : 101,
"index_total" : 3,
"index_failures" : 0,
"search_time_in_ms" : 1909,
"search_total" : 4,
"search_failures" : 0,
"exponential_avg_checkpoint_duration_ms" : 0.0,
"exponential_avg_documents_indexed" : 0.0,
"exponential_avg_documents_processed" : 0.0
},
"checkpointing" : {
"last" : {
"checkpoint" : 0
},
"next" : {
"checkpoint" : 1,
"position" : {
"indexer_position" : {
"@timestamp" : 1551282000000,
"agent.version" : "7.0.0"
}
},
"checkpoint_progress" : {
"docs_remaining" : 16681594,
"total_docs" : 21139496,
"percent_complete" : 21.08802404749858,
"docs_indexed" : 1500,
"docs_processed" : 4457902
},
"timestamp_millis" : 1568021396011
},
"operations_behind" : 21139495
}
},
{
"id" : "transform-completed",
"state" : "stopped",
"stats" : {
"pages_processed" : 22,
"documents_processed" : 21139496,
"documents_indexed" : 10154,
"trigger_count" : 2,
"index_time_in_ms" : 530,
"index_total" : 21,
"index_failures" : 0,
"search_time_in_ms" : 8566,
"search_total" : 23,
"search_failures" : 0,
"exponential_avg_checkpoint_duration_ms" : 0.0,
"exponential_avg_documents_indexed" : 0.0,
"exponential_avg_documents_processed" : 0.0
},
"checkpointing" : {
"last" : {
"checkpoint" : 1,
"timestamp_millis" : 1568021396011
},
"operations_behind" : 0
}
}
]
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,36 @@
*/

import mockDataFrameTransformListRow from './__mocks__/data_frame_transform_list_row.json';
import mockDataFrameTransformStats from './__mocks__/data_frame_transform_stats.json';

import { DataFrameTransformListRow } from './transform_list';
import { isCompletedBatchTransform, DATA_FRAME_TRANSFORM_STATE } from './transform_stats';
import { getTransformProgress, isCompletedBatchTransform } from './transform_stats';

describe('Data Frame: isCompletedBatchTransform()', () => {
test('isCompletedBatchTransform()', () => {
// check the transform config/state against the conditions
// that will be used by isCompletedBatchTransform()
// followed by a call to isCompletedBatchTransform() itself
const row = mockDataFrameTransformListRow as DataFrameTransformListRow;
expect(row.stats.checkpointing.last.checkpoint === 1).toBe(true);
expect(row.config.sync === undefined).toBe(true);
expect(row.stats.state === DATA_FRAME_TRANSFORM_STATE.STOPPED).toBe(true);
expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(true);
const getRow = (statsId: string) => {
return {
...(mockDataFrameTransformListRow as DataFrameTransformListRow),
stats: {
...mockDataFrameTransformStats.transforms.find(
(stats: DataFrameTransformListRow['stats']) => stats.id === statsId
),
},
};
};

describe('Data Frame: Transform stats.', () => {
test('getTransformProgress()', () => {
// At the moment, any kind of stopped jobs don't include progress information.
// We cannot infer progress for now from an unfinished job that has been stopped for now.
expect(getTransformProgress(getRow('transform-created'))).toBe(undefined);
expect(getTransformProgress(getRow('transform-created-started-stopped'))).toBe(undefined);
expect(getTransformProgress(getRow('transform-running'))).toBe(21);
expect(getTransformProgress(getRow('transform-completed'))).toBe(100);
});

// adapt the mock config to resemble a non-completed transform.
row.stats.checkpointing.last.checkpoint = 0;
expect(isCompletedBatchTransform(mockDataFrameTransformListRow)).toBe(false);
test('isCompletedBatchTransform()', () => {
expect(isCompletedBatchTransform(getRow('transform-created'))).toBe(false);
expect(isCompletedBatchTransform(getRow('transform-created-started-stopped'))).toBe(false);
expect(isCompletedBatchTransform(getRow('transform-running'))).toBe(false);
expect(isCompletedBatchTransform(getRow('transform-completed'))).toBe(true);
});
});

0 comments on commit f14e231

Please sign in to comment.