From 03457cb5bdd4e66d15934d8baee08a42d48f8f92 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Mon, 27 Feb 2023 16:03:31 -0600 Subject: [PATCH 1/5] more efficient data view telemetry --- ...ter_index_pattern_usage_collection.test.ts | 87 ++++++++--- ...register_index_pattern_usage_collection.ts | 141 ++++++++++-------- 2 files changed, 141 insertions(+), 87 deletions(-) diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts index 01d3a574a58cb..72f5f29aec170 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts @@ -12,7 +12,7 @@ import { updateMax, getIndexPatternTelemetry, } from './register_index_pattern_usage_collection'; -import { DataViewsService } from '../common'; +import { SavedObjectsClient } from '@kbn/core/server'; const scriptA = 'emit(0);'; const scriptB = 'emit(1);\nemit(2);'; @@ -22,17 +22,36 @@ const scriptedFieldA = { script: scriptA }; const scriptedFieldB = { script: scriptB }; const scriptedFieldC = { script: scriptC }; +/* const runtimeFieldA = { runtimeField: { script: { source: scriptA } } }; const runtimeFieldB = { runtimeField: { script: { source: scriptB } } }; const runtimeFieldC = { runtimeField: { script: { source: scriptC } } }; - -const indexPatterns = { - getIds: async () => [1, 2, 3], - get: jest.fn().mockResolvedValue({ - getScriptedFields: () => [], - fields: [], +*/ + +const runtimeFieldA = { script: { source: scriptA } }; +const runtimeFieldB = { script: { source: scriptB } }; +const runtimeFieldC = { script: { source: scriptC } }; + +let returnedSavedObjects = [ + { + attributes: { + fields: '[]', + runtimeFieldMap: JSON.stringify({}), + }, + }, +]; + +const savedObjects = { + createPointInTimeFinder: jest.fn().mockReturnValue({ + find: jest.fn().mockImplementation(async function* () { + yield await Promise.resolve({ + total: 3, + saved_objects: returnedSavedObjects, + }); + }), + close: jest.fn(), }), -} as any as DataViewsService; +} as any as SavedObjectsClient; describe('index pattern usage collection', () => { it('minMaxAvgLoC calculates min, max, and average ', () => { @@ -59,7 +78,7 @@ describe('index pattern usage collection', () => { }; it('when there are no runtime fields or scripted fields', async () => { - expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({ + expect(await getIndexPatternTelemetry(savedObjects)).toEqual({ indexPatternsCount: 3, indexPatternsWithScriptedFieldCount: 0, indexPatternsWithRuntimeFieldCount: 0, @@ -75,12 +94,20 @@ describe('index pattern usage collection', () => { }); it('when there are both runtime fields or scripted fields', async () => { - indexPatterns.get = jest.fn().mockResolvedValue({ - getScriptedFields: () => [scriptedFieldA, scriptedFieldB, scriptedFieldC], - fields: [runtimeFieldA, runtimeFieldB, runtimeFieldC], - }); + const dataView = { + attributes: { + fields: JSON.stringify([scriptedFieldA, scriptedFieldB, scriptedFieldC]), + runtimeFieldMap: JSON.stringify({ + runtimeFieldA, + runtimeFieldB, + runtimeFieldC, + }), + }, + }; + + returnedSavedObjects = [dataView, dataView, dataView]; - expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({ + expect(await getIndexPatternTelemetry(savedObjects)).toEqual({ indexPatternsCount: 3, indexPatternsWithScriptedFieldCount: 3, indexPatternsWithRuntimeFieldCount: 3, @@ -96,12 +123,20 @@ describe('index pattern usage collection', () => { }); it('when there are only runtime fields', async () => { - indexPatterns.get = jest.fn().mockResolvedValue({ - getScriptedFields: () => [], - fields: [runtimeFieldA, runtimeFieldB, runtimeFieldC], - }); + const dataView = { + attributes: { + fields: JSON.stringify([]), + runtimeFieldMap: JSON.stringify({ + runtimeFieldA, + runtimeFieldB, + runtimeFieldC, + }), + }, + }; + + returnedSavedObjects = [dataView, dataView, dataView]; - expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({ + expect(await getIndexPatternTelemetry(savedObjects)).toEqual({ indexPatternsCount: 3, indexPatternsWithScriptedFieldCount: 0, indexPatternsWithRuntimeFieldCount: 3, @@ -117,12 +152,16 @@ describe('index pattern usage collection', () => { }); it('when there are only scripted fields', async () => { - indexPatterns.get = jest.fn().mockResolvedValue({ - getScriptedFields: () => [scriptedFieldA, scriptedFieldB, scriptedFieldC], - fields: [], - }); + const dataView = { + attributes: { + fields: JSON.stringify([scriptedFieldA, scriptedFieldB, scriptedFieldC]), + runtimeFieldMap: JSON.stringify({}), + }, + }; + + returnedSavedObjects = [dataView, dataView, dataView]; - expect(await getIndexPatternTelemetry(indexPatterns)).toEqual({ + expect(await getIndexPatternTelemetry(savedObjects)).toEqual({ indexPatternsCount: 3, indexPatternsWithScriptedFieldCount: 3, indexPatternsWithRuntimeFieldCount: 0, diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts index 484a1289a59f6..01b9c3a4ec1b8 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts @@ -8,8 +8,13 @@ import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { StartServicesAccessor } from '@kbn/core/server'; -import { SavedObjectsClient } from '@kbn/core/server'; -import { DataViewsContract } from '../common'; +import { SavedObjectsClient, SavedObjectsCreatePointInTimeFinderOptions } from '@kbn/core/server'; +import { + DATA_VIEW_SAVED_OBJECT_TYPE, + DataViewAttributes, + FieldSpec, + RuntimeField, +} from '../common'; import { DataViewsServerPluginStartDependencies, DataViewsServerPluginStart } from './types'; interface CountSummary { @@ -18,6 +23,8 @@ interface CountSummary { avg?: number; } +type DataViewFieldAttrs = Pick; + interface IndexPatternUsage { indexPatternsCount: number; indexPatternsWithScriptedFieldCount: number; @@ -57,9 +64,7 @@ export const updateMax = (currentMax: number | undefined, newVal: number): numbe } }; -export async function getIndexPatternTelemetry(indexPatterns: DataViewsContract) { - const ids = await indexPatterns.getIds(); - +export async function getIndexPatternTelemetry(savedObjectsService: SavedObjectsClient) { const countSummaryDefaults: CountSummary = { min: undefined, max: undefined, @@ -67,7 +72,7 @@ export async function getIndexPatternTelemetry(indexPatterns: DataViewsContract) }; const results = { - indexPatternsCount: ids.length, + indexPatternsCount: 0, indexPatternsWithScriptedFieldCount: 0, indexPatternsWithRuntimeFieldCount: 0, scriptedFieldCount: 0, @@ -80,60 +85,74 @@ export async function getIndexPatternTelemetry(indexPatterns: DataViewsContract) }, }; - await ids.reduce(async (col, id) => { - await col; - const ip = await indexPatterns.get(id); + const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { + type: DATA_VIEW_SAVED_OBJECT_TYPE, + perPage: 1000, + fields: ['attributes.fields', 'attributes.runtimeFieldMap'], + }; - const scriptedFields = ip.getScriptedFields(); - const runtimeFields = ip.fields.filter((fld) => !!fld.runtimeField); + const finder = savedObjectsService.createPointInTimeFinder(findOptions); - if (scriptedFields.length > 0) { - // increment counts - results.indexPatternsWithScriptedFieldCount++; - results.scriptedFieldCount += scriptedFields.length; + for await (const response of finder.find()) { + const { saved_objects: savedObjects, total } = response; + results.indexPatternsCount = total; - // calc LoC - results.perIndexPattern.scriptedFieldLineCount = minMaxAvgLoC( - scriptedFields.map((fld) => fld.script || '') - ); - - // calc field counts - results.perIndexPattern.scriptedFieldCount.min = updateMin( - results.perIndexPattern.scriptedFieldCount.min, - scriptedFields.length - ); - results.perIndexPattern.scriptedFieldCount.max = updateMax( - results.perIndexPattern.scriptedFieldCount.max, - scriptedFields.length - ); - results.perIndexPattern.scriptedFieldCount.avg = - results.scriptedFieldCount / results.indexPatternsWithScriptedFieldCount; - } - - if (runtimeFields.length > 0) { - // increment counts - results.indexPatternsWithRuntimeFieldCount++; - results.runtimeFieldCount += runtimeFields.length; + savedObjects.forEach((obj) => { + const fields = JSON.parse(obj.attributes.fields) || []; + const runtimeFieldsMap = obj.attributes.runtimeFieldMap + ? JSON.parse(obj.attributes.runtimeFieldMap) || {} + : {}; + const scriptedFields: FieldSpec[] = fields.filter((fld: FieldSpec) => !!fld.script); + // to array + const runtimeFields: RuntimeField[] = Object.values(runtimeFieldsMap); // calc LoC - const runtimeFieldScripts = runtimeFields.map( - (fld) => fld.runtimeField?.script?.source || '' - ); - results.perIndexPattern.runtimeFieldLineCount = minMaxAvgLoC(runtimeFieldScripts); - - // calc field counts - results.perIndexPattern.runtimeFieldCount.min = updateMin( - results.perIndexPattern.runtimeFieldCount.min, - runtimeFields.length - ); - results.perIndexPattern.runtimeFieldCount.max = updateMax( - results.perIndexPattern.runtimeFieldCount.max, - runtimeFields.length - ); - results.perIndexPattern.runtimeFieldCount.avg = - results.runtimeFieldCount / results.indexPatternsWithRuntimeFieldCount; - } - }, Promise.resolve()); + if (scriptedFields.length > 0) { + // increment counts + results.indexPatternsWithScriptedFieldCount++; + results.scriptedFieldCount += scriptedFields.length; + + // calc LoC + results.perIndexPattern.scriptedFieldLineCount = minMaxAvgLoC( + scriptedFields.map((fld) => fld.script || '') + ); + + // calc field counts + results.perIndexPattern.scriptedFieldCount.min = updateMin( + results.perIndexPattern.scriptedFieldCount.min, + scriptedFields.length + ); + results.perIndexPattern.scriptedFieldCount.max = updateMax( + results.perIndexPattern.scriptedFieldCount.max, + scriptedFields.length + ); + results.perIndexPattern.scriptedFieldCount.avg = + results.scriptedFieldCount / results.indexPatternsWithScriptedFieldCount; + } + + if (runtimeFields.length > 0) { + // increment counts + results.indexPatternsWithRuntimeFieldCount++; + results.runtimeFieldCount += runtimeFields.length; + + // calc LoC + const runtimeFieldScripts = runtimeFields.map((fld) => fld.script?.source || ''); + results.perIndexPattern.runtimeFieldLineCount = minMaxAvgLoC(runtimeFieldScripts); + + // calc field counts + results.perIndexPattern.runtimeFieldCount.min = updateMin( + results.perIndexPattern.runtimeFieldCount.min, + runtimeFields.length + ); + results.perIndexPattern.runtimeFieldCount.max = updateMax( + results.perIndexPattern.runtimeFieldCount.max, + runtimeFields.length + ); + results.perIndexPattern.runtimeFieldCount.avg = + results.runtimeFieldCount / results.indexPatternsWithRuntimeFieldCount; + } + }); + } return results; } @@ -153,14 +172,10 @@ export function registerIndexPatternsUsageCollector( type: 'index-patterns', isReady: () => true, fetch: async () => { - const [{ savedObjects, elasticsearch }, , { dataViewsServiceFactory }] = - await getStartServices(); - const indexPatternService = await dataViewsServiceFactory( - new SavedObjectsClient(savedObjects.createInternalRepository()), - elasticsearch.client.asInternalUser - ); - - return await getIndexPatternTelemetry(indexPatternService); + const [{ savedObjects }] = await getStartServices(); + + const savedObjectsService = new SavedObjectsClient(savedObjects.createInternalRepository()); + return await getIndexPatternTelemetry(savedObjectsService); }, schema: { indexPatternsCount: { type: 'long' }, From aa5b5596c84c4068b6a759d73bbf56c78d7003a9 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 28 Feb 2023 07:57:07 -0600 Subject: [PATCH 2/5] check for fields attribute --- .../server/register_index_pattern_usage_collection.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts index 01b9c3a4ec1b8..6d1462e321b27 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts @@ -98,8 +98,8 @@ export async function getIndexPatternTelemetry(savedObjectsService: SavedObjects results.indexPatternsCount = total; savedObjects.forEach((obj) => { - const fields = JSON.parse(obj.attributes.fields) || []; - const runtimeFieldsMap = obj.attributes.runtimeFieldMap + const fields = obj.attributes?.fields ? JSON.parse(obj.attributes.fields) || [] : []; + const runtimeFieldsMap = obj.attributes?.runtimeFieldMap ? JSON.parse(obj.attributes.runtimeFieldMap) || {} : {}; const scriptedFields: FieldSpec[] = fields.filter((fld: FieldSpec) => !!fld.script); From 63d88369aa54b887a32710838677892e45bf8198 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 28 Feb 2023 07:59:09 -0600 Subject: [PATCH 3/5] test without fields array and runtime field map --- .../server/register_index_pattern_usage_collection.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts index 72f5f29aec170..20894027f788c 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts @@ -34,10 +34,7 @@ const runtimeFieldC = { script: { source: scriptC } }; let returnedSavedObjects = [ { - attributes: { - fields: '[]', - runtimeFieldMap: JSON.stringify({}), - }, + attributes: {}, }, ]; From bcc4ab851a8c1819e1318cb2456a3a54d3985df7 Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Tue, 28 Feb 2023 08:22:08 -0600 Subject: [PATCH 4/5] remove comment --- .../server/register_index_pattern_usage_collection.test.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts index 20894027f788c..bbd3ef0688737 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.test.ts @@ -22,12 +22,6 @@ const scriptedFieldA = { script: scriptA }; const scriptedFieldB = { script: scriptB }; const scriptedFieldC = { script: scriptC }; -/* -const runtimeFieldA = { runtimeField: { script: { source: scriptA } } }; -const runtimeFieldB = { runtimeField: { script: { source: scriptB } } }; -const runtimeFieldC = { runtimeField: { script: { source: scriptC } } }; -*/ - const runtimeFieldA = { script: { source: scriptA } }; const runtimeFieldB = { script: { source: scriptB } }; const runtimeFieldC = { script: { source: scriptC } }; From 518aea8827e59955498444a17ba9bc950974243b Mon Sep 17 00:00:00 2001 From: Matt Kime Date: Wed, 1 Mar 2023 14:28:57 -0600 Subject: [PATCH 5/5] fix attribute references --- .../server/register_index_pattern_usage_collection.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts index 6d1462e321b27..7b59f95d571ab 100644 --- a/src/plugins/data_views/server/register_index_pattern_usage_collection.ts +++ b/src/plugins/data_views/server/register_index_pattern_usage_collection.ts @@ -88,7 +88,7 @@ export async function getIndexPatternTelemetry(savedObjectsService: SavedObjects const findOptions: SavedObjectsCreatePointInTimeFinderOptions = { type: DATA_VIEW_SAVED_OBJECT_TYPE, perPage: 1000, - fields: ['attributes.fields', 'attributes.runtimeFieldMap'], + fields: ['fields', 'runtimeFieldMap'], }; const finder = savedObjectsService.createPointInTimeFinder(findOptions); @@ -103,7 +103,6 @@ export async function getIndexPatternTelemetry(savedObjectsService: SavedObjects ? JSON.parse(obj.attributes.runtimeFieldMap) || {} : {}; const scriptedFields: FieldSpec[] = fields.filter((fld: FieldSpec) => !!fld.script); - // to array const runtimeFields: RuntimeField[] = Object.values(runtimeFieldsMap); // calc LoC