From 185ac9113e08e3f9b0bd8ea0cd594632bc03c2a8 Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Fri, 29 Oct 2021 15:02:25 -0700 Subject: [PATCH] fix csv missing fields issue and empty csv on _source fields --- .../server/routes/lib/createReport.ts | 6 +- .../__tests__/savedSearchReportHelper.test.ts | 103 +++++++++++++++--- .../__tests__/visualReportHelper.test.ts | 19 +--- .../server/routes/utils/dataReportHelpers.ts | 9 +- .../routes/utils/savedSearchReportHelper.ts | 18 ++- 5 files changed, 118 insertions(+), 37 deletions(-) diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts index 36709343..70058a87 100644 --- a/dashboards-reports/server/routes/lib/createReport.ts +++ b/dashboards-reports/server/routes/lib/createReport.ts @@ -69,7 +69,8 @@ export const createReport = async ( // @ts-ignore const timezone = request.query.timezone; // @ts-ignore - const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; + const dateFormat = + request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; const { basePath, serverInfo: { protocol, port, hostname }, @@ -97,7 +98,8 @@ export const createReport = async ( report, opensearchClient, dateFormat, - isScheduledTask + isScheduledTask, + logger ); } else { // report source can only be one of [saved search, visualization, dashboard, notebook] diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 6d3b7377..ea63ddfa 100644 --- a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -27,6 +27,7 @@ import 'regenerator-runtime/runtime'; import { createSavedSearchReport } from '../savedSearchReportHelper'; import { reportSchema } from '../../../model'; +import mockLogger from '../../../../test/__mocks__/loggerMock'; /** * The mock and sample input for saved search export function. @@ -54,7 +55,7 @@ const input = { configIds: [], title: 'title', textDescription: 'text description', - htmlDescription: 'html description' + htmlDescription: 'html description', }, trigger: { trigger_type: 'On demand', @@ -62,6 +63,8 @@ const input = { }, }; +const mockDateFormat = 'date_hour_minute_second_fraction'; + /** * Max result window size in OpenSearch index settings. */ @@ -86,14 +89,20 @@ describe('test create saved search report', () => { test('create report with expected file name extension', async () => { const csvReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + mockDateFormat, + undefined, + mockLogger ); expect(csvReport.fileName).toContain('.csv'); input.report_definition.report_params.core_params.report_format = 'xlsx'; const xlsxReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + mockDateFormat, + undefined, + mockLogger ); expect(xlsxReport.fileName).toContain('.xlsx'); }, 20000); @@ -101,7 +110,13 @@ describe('test create saved search report', () => { test('create report for empty data set', async () => { const hits: Array<{ _source: any }> = []; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual(''); }, 20000); @@ -114,7 +129,13 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -141,7 +162,13 @@ describe('test create saved search report', () => { hit({ category: 'c11', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -171,7 +198,13 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male'); }, 20000); @@ -193,7 +226,13 @@ describe('test create saved search report', () => { hit({ category: 'c10', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -219,7 +258,13 @@ describe('test create saved search report', () => { hit({ category: 'c6', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -239,7 +284,13 @@ describe('test create saved search report', () => { hit({ category: ',,c3', customer_gender: 'Male,,,' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -265,7 +316,13 @@ describe('test create saved search report', () => { hits, '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' ); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' + @@ -283,7 +340,13 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -307,7 +370,13 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -327,7 +396,13 @@ test('create report for data set contains null field value', async () => { hit({ category: 'c3', customer_gender: null }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + undefined, + mockLogger + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, ' diff --git a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts index ff2aca0b..5a97e0b9 100644 --- a/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/visualReportHelper.test.ts @@ -28,18 +28,9 @@ import 'regenerator-runtime/runtime'; import { createVisualReport } from '../visual_report/visualReportHelper'; import { Logger } from '../../../../../../src/core/server'; import { ReportParamsSchemaType, reportSchema } from '../../../model'; +import mockLogger from '../../../../test/__mocks__/loggerMock'; -const mockLogger: Logger = { - info: jest.fn(), - trace: jest.fn(), - warn: jest.fn(), - debug: jest.fn(), - error: jest.fn(), - fatal: jest.fn(), - log: jest.fn(), - get: jest.fn(), -}; - +const mockHeader = { mockKey: 'mockValue' }; const input = { query_url: '/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d', time_from: 1343576635300, @@ -84,7 +75,8 @@ describe('test create visual report', () => { const { dataUrl, fileName } = await createVisualReport( reportParams as ReportParamsSchemaType, mockHtmlPath, - mockLogger + mockLogger, + mockHeader ); expect(fileName).toContain(`${reportParams.report_name}`); expect(fileName).toContain('.png'); @@ -99,7 +91,8 @@ describe('test create visual report', () => { const { dataUrl, fileName } = await createVisualReport( reportParams as ReportParamsSchemaType, mockHtmlPath, - mockLogger + mockLogger, + mockHeader ); expect(fileName).toContain(`${reportParams.report_name}`); expect(fileName).toContain('.pdf'); diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index 7a3b6d19..a5ede76c 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -49,14 +49,17 @@ export var metaData = { // Get the selected columns by the user. export const getSelectedFields = async (columns) => { const selectedFields = []; + let fields_exist = false; for (let column of columns) { if (column !== '_source') { - metaData.fields_exist = true; + fields_exist = true; selectedFields.push(column); } else { + fields_exist = false; selectedFields.push('_source'); } } + metaData.fields_exist = fields_exist; metaData.selectedFields = selectedFields; }; @@ -191,7 +194,7 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) } delete data['fields']; if (report._source.fields_exist === true) { - let result = traverse(data._source, report._source.selectedFields); + let result = traverse(data, report._source.selectedFields); hits.push(params.excel ? sanitize(result) : result); } else { hits.push(params.excel ? sanitize(data) : data); @@ -229,7 +232,7 @@ function flattenHits(hits, result = {}, prefix = '') { ) { flattenHits(value, result, prefix + key + '.'); } else { - result[prefix + key] = value; + result[prefix.replace(/^_source\./, '') + key] = value; } } return result; diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index 76a5a9cf..f42836f9 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -34,6 +34,7 @@ import { import { ILegacyClusterClient, ILegacyScopedClusterClient, + Logger, } from '../../../../../src/core/server'; import { getFileName, callCluster } from './helpers'; import { CreateReportResultType } from './types'; @@ -49,18 +50,20 @@ export async function createSavedSearchReport( report: any, client: ILegacyClusterClient | ILegacyScopedClusterClient, dateFormat: string, - isScheduledTask: boolean = true + isScheduledTask: boolean = true, + logger: Logger ): Promise { const params = report.report_definition.report_params; const reportFormat = params.core_params.report_format; const reportName = params.report_name; - await populateMetaData(client, report, isScheduledTask); + await populateMetaData(client, report, isScheduledTask, logger); const data = await generateReportData( client, params.core_params, dateFormat, - isScheduledTask + isScheduledTask, + logger ); const curTime = new Date(); @@ -81,7 +84,8 @@ export async function createSavedSearchReport( async function populateMetaData( client: ILegacyClusterClient | ILegacyScopedClusterClient, report: any, - isScheduledTask: boolean + isScheduledTask: boolean, + logger: Logger ) { metaData.saved_search_id = report.report_definition.report_params.core_params.saved_search_id; @@ -144,7 +148,8 @@ async function generateReportData( client: ILegacyClusterClient | ILegacyScopedClusterClient, params: any, dateFormat: string, - isScheduledTask: boolean + isScheduledTask: boolean, + logger: Logger ) { let opensearchData: any = {}; const arrayHits: any = []; @@ -159,6 +164,9 @@ async function generateReportData( } const reqBody = buildRequestBody(buildQuery(report, 0)); + logger.info( + `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` + ); if (total > maxResultSize) { await getOpenSearchDataByScroll(); } else {