From 42776a02bee002dd0139bb9b3c425b98ac33ad8d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 31 Jan 2024 14:19:49 -0700 Subject: [PATCH 01/16] Use Data Stream for Reporting storage --- docs/settings/reporting-settings.asciidoc | 4 +- docs/user/reporting/script-example.asciidoc | 2 +- packages/kbn-reporting/common/types.ts | 3 +- packages/kbn-reporting/server/constants.ts | 8 +- .../plugins/reporting/server/config/index.ts | 1 + .../reporting/server/deprecations/index.ts | 2 +- ...igrate_existing_indices_ilm_policy.test.ts | 18 +-- .../migrate_existing_indices_ilm_policy.ts | 21 +-- .../check_ilm_migration_status.ts | 7 +- .../server/lib/deprecations/types.ts | 2 - .../server/lib/store/index_timestamp.ts | 41 ------ .../reporting/server/lib/store/mapping.ts | 93 ------------- .../reporting/server/lib/store/report.test.ts | 3 + .../reporting/server/lib/store/report.ts | 7 +- .../reporting/server/lib/store/store.test.ts | 64 --------- .../reporting/server/lib/store/store.ts | 104 ++------------ .../server/routes/common/jobs/jobs_query.ts | 14 +- .../internal/deprecations/deprecations.ts | 11 +- .../bwc_existing_indexes.ts | 64 --------- .../reporting_and_security/index.ts | 1 - .../job_apis_csv.ts | 127 ++---------------- .../services/scenarios.ts | 11 +- .../services/usage.ts | 38 ------ 23 files changed, 69 insertions(+), 577 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/lib/store/index_timestamp.ts delete mode 100644 x-pack/plugins/reporting/server/lib/store/mapping.ts delete mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts diff --git a/docs/settings/reporting-settings.asciidoc b/docs/settings/reporting-settings.asciidoc index e19065a533adc..1bf741b748576 100644 --- a/docs/settings/reporting-settings.asciidoc +++ b/docs/settings/reporting-settings.asciidoc @@ -70,7 +70,9 @@ reports, you might need to change the following settings. If capturing a report fails for any reason, {kib} will re-queue the report job for retry, as many times as this setting. Defaults to `3`. `xpack.reporting.queue.indexInterval`:: -How often the index that stores reporting jobs rolls over to a new index. Valid values are `year`, `month`, `week`, `day`, and `hour`. Defaults to `week`. +deprecated:[8.10.0,This setting has no effect.] How often Reporting creates a new index to store report jobs and file contents. +Valid values are `year`, `month`, `week`, `day`, and `hour`. Defaults to `week`. +*NOTE*: This setting exists for backwards compatibility, but is unused. Use the built-in ILM policy provided for the reporting plugin to customize the rollover of Reporting data. [[xpack-reportingQueue-pollEnabled]] `xpack.reporting.queue.pollEnabled` :: When `true`, enables the {kib} instance to poll {es} for pending jobs and claim them for diff --git a/docs/user/reporting/script-example.asciidoc b/docs/user/reporting/script-example.asciidoc index 937e140bd67a0..5445058ead03b 100644 --- a/docs/user/reporting/script-example.asciidoc +++ b/docs/user/reporting/script-example.asciidoc @@ -36,4 +36,4 @@ An example response for a successfully queued report: --------------------------------------------------------- <1> The relative path on the {kib} host for downloading the report. -<2> (Not included in the example) Internal representation of the reporting job, as found in the `.reporting-*` index. +<2> (Not included in the example) Internal representation of the reporting job, as found in the `.reporting-*` storage. diff --git a/packages/kbn-reporting/common/types.ts b/packages/kbn-reporting/common/types.ts index 92eb444880543..54cc8abf807fc 100644 --- a/packages/kbn-reporting/common/types.ts +++ b/packages/kbn-reporting/common/types.ts @@ -148,7 +148,8 @@ export interface ReportSource { }; migration_version: string; // for reminding the user to update their POST URL attempts: number; // initially populated as 0 - created_at: string; // timestamp in UTC + created_at: string; // creation timestamp in UTC + '@timestamp'?: string; // creation timestamp, only used for data streams compatibility status: JOB_STATUS; /* diff --git a/packages/kbn-reporting/server/constants.ts b/packages/kbn-reporting/server/constants.ts index a625aefcf3904..da124c371be70 100644 --- a/packages/kbn-reporting/server/constants.ts +++ b/packages/kbn-reporting/server/constants.ts @@ -12,7 +12,13 @@ export const PLUGIN_ID = 'reporting'; * Storage */ -export const REPORTING_SYSTEM_INDEX = '.reporting'; +// The name of the Reporting data stream is given to query across all data stored by Reporting. The data +// stream is an alias, automatically configured in the Kibana plugin of Elasticsearch, pointing to an unknown +// number of backing indices. +export const REPORTING_DATA_STREAM = '.kibana-reporting'; + +// Pattern to search for historical reports +export const REPORTING_DATA_STREAM_WILDCARD = '.reporting-*,.kibana-reporting*'; /* * Telemetry diff --git a/x-pack/plugins/reporting/server/config/index.ts b/x-pack/plugins/reporting/server/config/index.ts index 59f60f35cb717..a9c2f211cc883 100644 --- a/x-pack/plugins/reporting/server/config/index.ts +++ b/x-pack/plugins/reporting/server/config/index.ts @@ -24,6 +24,7 @@ export const config: PluginConfigDescriptor = { }, schema: ConfigSchema, deprecations: ({ unused }) => [ + unused('queue.indexInterval', { level: 'warning' }), // unused since 8.13 unused('capture.browser.chromium.maxScreenshotDimension', { level: 'warning' }), // unused since 7.8 unused('capture.browser.type', { level: 'warning' }), unused('poll.jobCompletionNotifier.intervalErrorMultiplier', { level: 'warning' }), // unused since 7.10 diff --git a/x-pack/plugins/reporting/server/deprecations/index.ts b/x-pack/plugins/reporting/server/deprecations/index.ts index 651defbb5d688..bca439532b3b0 100644 --- a/x-pack/plugins/reporting/server/deprecations/index.ts +++ b/x-pack/plugins/reporting/server/deprecations/index.ts @@ -20,7 +20,7 @@ export const registerDeprecations = ({ core.deprecations.registerDeprecations({ getDeprecations: async (ctx) => { return [ - ...(await getIlmPolicyDeprecationsInfo(ctx, { reportingCore })), + ...(await getIlmPolicyDeprecationsInfo(ctx)), ...(await getReportingRoleDeprecationsInfo(ctx, { reportingCore })), ]; }, diff --git a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts index 9f0a10b861adb..420809aba646e 100644 --- a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts +++ b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts @@ -7,10 +7,6 @@ import type { GetDeprecationsContext } from '@kbn/core/server'; import { elasticsearchServiceMock, savedObjectsClientMock } from '@kbn/core/server/mocks'; -import { createMockConfigSchema } from '@kbn/reporting-mocks-server'; - -import { ReportingCore } from '../core'; -import { createMockReportingCore } from '../test_helpers'; import { getDeprecationsInfo } from './migrate_existing_indices_ilm_policy'; @@ -21,12 +17,10 @@ type ScopedClusterClientMock = ReturnType< describe("Migrate existing indices' ILM policy deprecations", () => { let esClient: ScopedClusterClientMock; let deprecationsCtx: GetDeprecationsContext; - let reportingCore: ReportingCore; beforeEach(async () => { esClient = elasticsearchServiceMock.createScopedClusterClient(); deprecationsCtx = { esClient, savedObjectsClient: savedObjectsClientMock.create() }; - reportingCore = await createMockReportingCore(createMockConfigSchema()); }); const createIndexSettings = (lifecycleName: string) => ({ @@ -47,7 +41,7 @@ describe("Migrate existing indices' ILM policy deprecations", () => { indexB: createIndexSettings('kibana-reporting'), }); - expect(await getDeprecationsInfo(deprecationsCtx, { reportingCore })).toMatchInlineSnapshot(` + expect(await getDeprecationsInfo(deprecationsCtx)).toMatchInlineSnapshot(` Array [ Object { "correctiveActions": Object { @@ -60,7 +54,7 @@ describe("Migrate existing indices' ILM policy deprecations", () => { ], }, "level": "warning", - "message": "New reporting indices will be managed by the \\"kibana-reporting\\" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets all indices prefixed with \\".reporting-*\\".", + "message": "New reporting indices will be managed by the \\"kibana-reporting\\" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets the hidden system index pattern \\".reporting-*,.kibana-reporting*\\".", "title": "Found reporting indices managed by custom ILM policy.", }, ] @@ -73,14 +67,10 @@ describe("Migrate existing indices' ILM policy deprecations", () => { indexB: createIndexSettings('kibana-reporting'), }); - expect(await getDeprecationsInfo(deprecationsCtx, { reportingCore })).toMatchInlineSnapshot( - `Array []` - ); + expect(await getDeprecationsInfo(deprecationsCtx)).toMatchInlineSnapshot(`Array []`); esClient.asInternalUser.indices.getSettings.mockResponse({}); - expect(await getDeprecationsInfo(deprecationsCtx, { reportingCore })).toMatchInlineSnapshot( - `Array []` - ); + expect(await getDeprecationsInfo(deprecationsCtx)).toMatchInlineSnapshot(`Array []`); }); }); diff --git a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.ts b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.ts index 2fcba6c0ccf53..c5ef6ce0f710a 100644 --- a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.ts +++ b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.ts @@ -8,22 +8,13 @@ import { DeprecationsDetails, GetDeprecationsContext } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; import { ILM_POLICY_NAME, INTERNAL_ROUTES } from '@kbn/reporting-common'; -import { ReportingCore } from '../core'; +import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import { deprecations } from '../lib/deprecations'; -interface ExtraDependencies { - reportingCore: ReportingCore; -} - -export const getDeprecationsInfo = async ( - { esClient }: GetDeprecationsContext, - { reportingCore }: ExtraDependencies -): Promise => { - const store = await reportingCore.getStore(); - const indexPattern = store.getReportingIndexPattern(); - +export const getDeprecationsInfo = async ({ + esClient, +}: GetDeprecationsContext): Promise => { const migrationStatus = await deprecations.checkIlmMigrationStatus({ - reportingCore, elasticsearchClient: esClient.asInternalUser, }); @@ -35,10 +26,10 @@ export const getDeprecationsInfo = async ( }), level: 'warning', message: i18n.translate('xpack.reporting.deprecations.migrateIndexIlmPolicyActionMessage', { - defaultMessage: `New reporting indices will be managed by the "{reportingIlmPolicy}" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets all indices prefixed with "{indexPattern}".`, + defaultMessage: `New reporting indices will be managed by the "{reportingIlmPolicy}" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets the hidden system index pattern "{indexPattern}".`, values: { reportingIlmPolicy: ILM_POLICY_NAME, - indexPattern, + indexPattern: REPORTING_DATA_STREAM_WILDCARD, }, }), correctiveActions: { diff --git a/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts b/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts index aab4c1f0cecf4..a9a5bdaea721c 100644 --- a/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts +++ b/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts @@ -7,11 +7,11 @@ import { IlmPolicyMigrationStatus } from '@kbn/reporting-common/url'; import { ILM_POLICY_NAME } from '@kbn/reporting-common'; +import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import { IlmPolicyManager } from '../store/ilm_policy_manager'; import type { DeprecationsDependencies } from './types'; export const checkIlmMigrationStatus = async ({ - reportingCore, elasticsearchClient, }: DeprecationsDependencies): Promise => { const ilmPolicyManager = IlmPolicyManager.create({ client: elasticsearchClient }); @@ -19,11 +19,8 @@ export const checkIlmMigrationStatus = async ({ return 'policy-not-found'; } - const store = await reportingCore.getStore(); - const indexPattern = store.getReportingIndexPattern(); - const reportingIndicesSettings = await elasticsearchClient.indices.getSettings({ - index: indexPattern, + index: REPORTING_DATA_STREAM_WILDCARD, }); const hasUnmanagedIndices = Object.values(reportingIndicesSettings).some((settings) => { diff --git a/x-pack/plugins/reporting/server/lib/deprecations/types.ts b/x-pack/plugins/reporting/server/lib/deprecations/types.ts index 6df429d2b2ed9..5f572f89911ef 100644 --- a/x-pack/plugins/reporting/server/lib/deprecations/types.ts +++ b/x-pack/plugins/reporting/server/lib/deprecations/types.ts @@ -6,9 +6,7 @@ */ import type { ElasticsearchClient } from '@kbn/core/server'; -import type { ReportingCore } from '../../core'; export interface DeprecationsDependencies { - reportingCore: ReportingCore; elasticsearchClient: ElasticsearchClient; } diff --git a/x-pack/plugins/reporting/server/lib/store/index_timestamp.ts b/x-pack/plugins/reporting/server/lib/store/index_timestamp.ts deleted file mode 100644 index ade5fd19f4618..0000000000000 --- a/x-pack/plugins/reporting/server/lib/store/index_timestamp.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import moment, { unitOfTime } from 'moment'; - -export const intervals = ['year', 'month', 'week', 'day', 'hour', 'minute']; - -export function indexTimestamp(intervalStr: string, separator = '-') { - const startOf = intervalStr as unitOfTime.StartOf; - if (separator.match(/[a-z]/i)) throw new Error('Interval separator can not be a letter'); - - const index = intervals.indexOf(intervalStr); - if (index === -1) throw new Error('Invalid index interval: ' + intervalStr); - - const m = moment(); - m.startOf(startOf); - - let dateString; - switch (intervalStr) { - case 'year': - dateString = 'YYYY'; - break; - case 'month': - dateString = `YYYY${separator}MM`; - break; - case 'hour': - dateString = `YYYY${separator}MM${separator}DD${separator}HH`; - break; - case 'minute': - dateString = `YYYY${separator}MM${separator}DD${separator}HH${separator}mm`; - break; - default: - dateString = `YYYY${separator}MM${separator}DD`; - } - - return m.format(dateString); -} diff --git a/x-pack/plugins/reporting/server/lib/store/mapping.ts b/x-pack/plugins/reporting/server/lib/store/mapping.ts deleted file mode 100644 index 9accfd0c75184..0000000000000 --- a/x-pack/plugins/reporting/server/lib/store/mapping.ts +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export const mapping = { - meta: { - // We are indexing these properties with both text and keyword fields - // because that's what will be auto generated when an index already exists. - properties: { - // ID of the app this report: search, visualization or dashboard, etc - objectType: { - type: 'text', - fields: { - keyword: { - type: 'keyword', - ignore_above: 256, - }, - }, - }, - layout: { - type: 'text', - fields: { - keyword: { - type: 'keyword', - ignore_above: 256, - }, - }, - }, - isDeprecated: { - type: 'boolean', - }, - }, - }, - migration_version: { type: 'keyword' }, // new field (7.14) to distinguish reports that were scheduled with Task Manager - jobtype: { type: 'keyword' }, - payload: { type: 'object', enabled: false }, - priority: { type: 'byte' }, // TODO: remove: this is unused - timeout: { type: 'long' }, - process_expiration: { type: 'date' }, - created_by: { type: 'keyword' }, // `null` if security is disabled - created_at: { type: 'date' }, - started_at: { type: 'date' }, - completed_at: { type: 'date' }, - attempts: { type: 'short' }, - max_attempts: { type: 'short' }, - kibana_name: { type: 'keyword' }, - kibana_id: { type: 'keyword' }, - status: { type: 'keyword' }, - parent_id: { type: 'keyword' }, - output: { - type: 'object', - properties: { - error_code: { type: 'keyword' }, - chunk: { type: 'long' }, - content_type: { type: 'keyword' }, - size: { type: 'long' }, - content: { type: 'object', enabled: false }, - }, - }, - metrics: { - type: 'object', - properties: { - csv: { - type: 'object', - properties: { - rows: { type: 'long' }, - }, - }, - pdf: { - type: 'object', - properties: { - pages: { type: 'long' }, - cpu: { type: 'double' }, - cpuInPercentage: { type: 'double' }, - memory: { type: 'long' }, - memoryInMegabytes: { type: 'double' }, - }, - }, - png: { - type: 'object', - properties: { - cpu: { type: 'double' }, - cpuInPercentage: { type: 'double' }, - memory: { type: 'long' }, - memoryInMegabytes: { type: 'double' }, - }, - }, - }, - }, -} as const; diff --git a/x-pack/plugins/reporting/server/lib/store/report.test.ts b/x-pack/plugins/reporting/server/lib/store/report.test.ts index f6cbbade4df7b..ca1294f663da8 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.test.ts @@ -124,6 +124,9 @@ describe('Class Report', () => { it('throws error if converted to task JSON before being synced with ES storage', () => { const report = new Report({ jobtype: 'spam', payload: {} } as any); + // @ts-ignore null is not applicable to string + report._index = null; + expect(() => report.updateWithEsDoc(report)).toThrowErrorMatchingInlineSnapshot( `"Report object from ES has missing fields!"` ); diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts index e62f4f2f20b58..d46f44ca93549 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -10,6 +10,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import { JOB_STATUS } from '@kbn/reporting-common'; +import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; import { ReportApiJSON, ReportDocumentHead, @@ -25,7 +26,7 @@ export const MIGRATION_VERSION = '7.14.0'; * Class for an ephemeral report document: possibly is not saved in Elasticsearch */ export class Report implements Partial { - public _index?: string; + public _index: string; public _id: string; public _primary_term?: number; // set by ES public _seq_no?: number; // set by ES @@ -63,7 +64,7 @@ export class Report implements Partial { */ constructor(opts: Partial & Partial, fields?: ReportFields) { this._id = opts._id != null ? opts._id : uuidv4(); - this._index = opts._index; + this._index = opts._index ?? REPORTING_DATA_STREAM; // Sets the value to the data stream, unless it's a stored report and we know the name of the backing index this._primary_term = opts._primary_term; this._seq_no = opts._seq_no; @@ -167,7 +168,7 @@ export class Report implements Partial { toApiJSON(): ReportApiJSON { return { id: this._id, - index: this._index!, + index: this._index ?? REPORTING_DATA_STREAM, kibana_name: this.kibana_name, kibana_id: this.kibana_id, jobtype: this.jobtype, diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 946ceb4f105b0..6fb28913f7ae6 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -80,70 +80,6 @@ describe('ReportingStore', () => { ); }); - it('handles error creating the index', async () => { - // setup - mockEsClient.indices.exists.mockResponse(false); - mockEsClient.indices.create.mockRejectedValue(new Error('horrible error')); - - const store = new ReportingStore(mockCore, mockLogger); - const mockReport = new Report({ - _index: '.reporting-errortest', - jobtype: 'unknowntype', - payload: {}, - meta: {}, - } as any); - await expect(store.addReport(mockReport)).rejects.toMatchInlineSnapshot( - `[Error: horrible error]` - ); - }); - - /* Creating the index will fail, if there were multiple jobs staged in - * parallel and creation completed from another Kibana instance. Only the - * first request in line can successfully create it. - * In spite of that race condition, adding the new job in Elasticsearch is - * fine. - */ - it('ignores index creation error if the index already exists and continues adding the report', async () => { - // setup - mockEsClient.indices.exists.mockResponse(false); - mockEsClient.indices.create.mockRejectedValue(new Error('devastating error')); - - const store = new ReportingStore(mockCore, mockLogger); - const mockReport = new Report({ - _index: '.reporting-mock', - jobtype: 'unknowntype', - payload: {}, - meta: {}, - } as any); - await expect(store.addReport(mockReport)).rejects.toMatchInlineSnapshot( - `[Error: devastating error]` - ); - }); - - it('skips creating the index if already exists', async () => { - // setup - mockEsClient.indices.exists.mockResponse(false); - // will be triggered but ignored - mockEsClient.indices.create.mockRejectedValue(new Error('resource_already_exists_exception')); - - const store = new ReportingStore(mockCore, mockLogger); - const mockReport = new Report({ - created_by: 'user1', - jobtype: 'unknowntype', - payload: {}, - meta: {}, - } as any); - await expect(store.addReport(mockReport)).resolves.toMatchObject({ - _primary_term: undefined, - _seq_no: undefined, - attempts: 0, - created_by: 'user1', - jobtype: 'unknowntype', - payload: {}, - status: 'pending', - }); - }); - it('allows username string to be `false`', async () => { // setup mockEsClient.indices.exists.mockResponse(false); diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index 543045393a1b2..e2e5bed45ab7d 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -14,15 +14,13 @@ import type { ReportOutput, ReportSource, } from '@kbn/reporting-common/types'; -import { REPORTING_SYSTEM_INDEX } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; import moment from 'moment'; import type { Report } from '.'; import { SavedReport } from '.'; import type { ReportingCore } from '../..'; import type { ReportTaskParams } from '../tasks'; import { IlmPolicyManager } from './ilm_policy_manager'; -import { indexTimestamp } from './index_timestamp'; -import { mapping } from './mapping'; import { MIGRATION_VERSION } from './report'; type UpdateResponse = estypes.UpdateResponse; @@ -71,6 +69,7 @@ const sourceDoc = (doc: Partial): Partial => { return { ...doc, migration_version: MIGRATION_VERSION, + '@timestamp': new Date(0).toISOString(), // required for data streams compatibility }; }; @@ -103,16 +102,9 @@ const jobDebugMessage = (report: Report) => * - interface for downloading the report */ export class ReportingStore { - private readonly indexPrefix: string; // config setting of index prefix in system index name - private readonly indexInterval: string; // config setting of index prefix: how often to poll for pending work private client?: ElasticsearchClient; - config: ReportingCore['config']; constructor(private reportingCore: ReportingCore, private logger: Logger) { - this.config = reportingCore.getConfig(); - - this.indexPrefix = REPORTING_SYSTEM_INDEX; - this.indexInterval = this.config.queue.indexInterval; this.logger = logger.get('store'); } @@ -124,62 +116,23 @@ export class ReportingStore { return this.client; } - private async getIlmPolicyManager() { - const client = await this.getClient(); - return IlmPolicyManager.create({ client }); - } - - private async createIndex(indexName: string) { + private async createIlmPolicy() { const client = await this.getClient(); - const exists = await client.indices.exists({ index: indexName }); - - if (exists) { - return exists; - } - - const indexSettings = this.config.statefulSettings.enabled - ? { - settings: { - number_of_shards: 1, - auto_expand_replicas: '0-1', - lifecycle: { - name: ILM_POLICY_NAME, - }, - }, - } - : {}; - - try { - await client.indices.create({ - index: indexName, - body: { - ...indexSettings, - mappings: { - properties: mapping, - }, - }, - }); - - return true; - } catch (error) { - const isIndexExistsError = error.message.match(/resource_already_exists_exception/); - if (isIndexExistsError) { - // Do not fail a job if the job runner hits the race condition. - this.logger.warn(`Automatic index creation failed: index already exists: ${error}`); - return; - } - - this.logger.error(error); - - throw error; + const ilmPolicyManager = IlmPolicyManager.create({ client }); + if (await ilmPolicyManager.doesIlmPolicyExist()) { + this.logger.debug(`Found ILM policy ${ILM_POLICY_NAME}; skipping creation.`); + return; } + this.logger.info(`Creating ILM policy for managing reporting indices: ${ILM_POLICY_NAME}`); + await ilmPolicyManager.createIlmPolicy(); } private async indexReport(report: Report): Promise { const doc = { - index: report._index!, + index: REPORTING_DATA_STREAM, id: report._id, refresh: 'wait_for' as estypes.Refresh, + op_type: 'create' as const, body: { ...report.toReportSource(), ...sourceDoc({ @@ -193,31 +146,13 @@ export class ReportingStore { return await client.index(doc); } - /* - * Called from addReport, which handles any errors - */ - private async refreshIndex(index: string) { - const client = await this.getClient(); - - return client.indices.refresh({ index }); - } - /** * Function to be called during plugin start phase. This ensures the environment is correctly * configured for storage of reports. */ public async start() { - if (!this.config.statefulSettings.enabled) { - return; - } - const ilmPolicyManager = await this.getIlmPolicyManager(); try { - if (await ilmPolicyManager.doesIlmPolicyExist()) { - this.logger.debug(`Found ILM policy ${ILM_POLICY_NAME}; skipping creation.`); - return; - } - this.logger.info(`Creating ILM policy for managing reporting indices: ${ILM_POLICY_NAME}`); - await ilmPolicyManager.createIlmPolicy(); + await this.createIlmPolicy(); } catch (e) { this.logger.error('Error in start phase'); this.logger.error(e.body?.error); @@ -226,19 +161,8 @@ export class ReportingStore { } public async addReport(report: Report): Promise { - let index = report._index; - if (!index) { - const timestamp = indexTimestamp(this.indexInterval); - index = `${this.indexPrefix}-${timestamp}`; - report._index = index; - } - await this.createIndex(index); - try { report.updateWithEsDoc(await this.indexReport(report)); - - await this.refreshIndex(index); - return report as SavedReport; } catch (err) { this.reportingCore.getEventLogger(report).logError(err); @@ -402,8 +326,4 @@ export class ReportingStore { return body; } - - public getReportingIndexPattern(): string { - return `${this.indexPrefix}-*`; - } } diff --git a/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts b/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts index d371d49970a72..a6dc747130e3c 100644 --- a/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts +++ b/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts @@ -10,7 +10,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; import { JOB_STATUS } from '@kbn/reporting-common'; import type { ReportApiJSON, ReportSource } from '@kbn/reporting-common/types'; -import { REPORTING_SYSTEM_INDEX } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import type { ReportingCore } from '../../..'; import { Report } from '../../../lib/store'; import { runtimeFieldKeys, runtimeFields } from '../../../lib/store/runtime_fields'; @@ -54,10 +54,6 @@ export interface JobsQueryFactory { } export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory { - function getIndex() { - return `${REPORTING_SYSTEM_INDEX}-*`; - } - async function execQuery< T extends (client: ElasticsearchClient) => Promise> | undefined> >(callback: T): Promise> | undefined> { @@ -96,7 +92,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }); const response = (await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: getIndex() }) + elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) )) as estypes.SearchResponse; return ( @@ -127,7 +123,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }; const response = await execQuery((elasticsearchClient) => - elasticsearchClient.count({ body, index: getIndex() }) + elasticsearchClient.count({ body, index: REPORTING_DATA_STREAM_WILDCARD }) ); return response?.count ?? 0; @@ -156,7 +152,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }); const response = await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: getIndex() }) + elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) ); const result = response?.hits?.hits?.[0]; @@ -187,7 +183,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }; const response = await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: getIndex() }) + elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) ); const hits = response?.hits?.hits?.[0]; const status = hits?._source?.status; diff --git a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts index d35adf717cbb9..21156436ada68 100644 --- a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts +++ b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts @@ -7,6 +7,7 @@ import { errors } from '@elastic/elasticsearch'; import type { Logger, RequestHandler } from '@kbn/core/server'; import { ILM_POLICY_NAME, INTERNAL_ROUTES } from '@kbn/reporting-common'; +import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import type { IlmPolicyStatusResponse } from '@kbn/reporting-common/url'; import type { ReportingCore } from '../../../core'; import { IlmPolicyManager } from '../../../lib'; @@ -24,15 +25,13 @@ const getAuthzWrapper = const { elasticsearch } = await ctx.core; - const store = await reporting.getStore(); - try { const body = await elasticsearch.client.asCurrentUser.security.hasPrivileges({ body: { index: [ { privileges: ['manage'], // required to do anything with the reporting indices - names: [store.getReportingIndexPattern()], + names: [REPORTING_DATA_STREAM_WILDCARD], allow_restricted_indices: true, }, ], @@ -70,7 +69,6 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log } = await core; const checkIlmMigrationStatus = () => { return deprecations.checkIlmMigrationStatus({ - reportingCore: reporting, // We want to make the current status visible to all reporting users elasticsearchClient: scopedClient.asInternalUser, }); @@ -106,7 +104,6 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log authzWrapper(async ({ core }, req, res) => { const counters = getCounters(req.route.method, migrateApiPath, reporting.getUsageCounter()); - const store = await reporting.getStore(); const { client: { asCurrentUser: client }, } = (await core).elasticsearch; @@ -125,12 +122,10 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log return res.customError({ statusCode: e?.statusCode ?? 500, body: { message: e.message } }); } - const indexPattern = store.getReportingIndexPattern(); - // Second we migrate all of the existing indices to be managed by the reporting ILM policy try { await client.indices.putSettings({ - index: indexPattern, + index: REPORTING_DATA_STREAM_WILDCARD, body: { index: { lifecycle: { diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts deleted file mode 100644 index 168390bc6fc28..0000000000000 --- a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { FtrProviderContext } from '../ftr_provider_context'; - -/** - * This file tests the situation when a reporting index spans releases. By default reporting indexes are created - * on a weekly basis, but this is configurable so it is possible a user has this set to yearly. In that event, it - * is possible report data is getting posted to an index that was created by a very old version. We don't have a - * reporting index migration plan, so this test is important to ensure BWC, or that in the event we decide to make - * a major change in a major release, we handle it properly. - */ - -// eslint-disable-next-line import/no-default-export -export default function ({ getService }: FtrProviderContext) { - const esArchiver = getService('esArchiver'); - const reportingAPI = getService('reportingAPI'); - const kibanaServer = getService('kibanaServer'); - - describe('BWC report generation into existing indexes', () => { - let cleanupIndexAlias: () => Promise; - - describe('existing 6_2 index', () => { - before('load data and add index alias', async () => { - await reportingAPI.deleteAllReports(); - // data to report on - await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); - await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); - - // archive with reporting index mappings v6.2 - await esArchiver.load('x-pack/test/functional/es_archives/reporting/bwc/6_2'); - - // The index name in the reporting/bwc/6_2 archive. - const ARCHIVED_REPORTING_INDEX = '.reporting-2018.03.11'; - // causes reporting to assume the v6.2 index is the one to use for new jobs posted - cleanupIndexAlias = await reportingAPI.coerceReportsIntoExistingIndex( - ARCHIVED_REPORTING_INDEX - ); - }); - - after('remove index alias', async () => { - await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); - await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); - - await cleanupIndexAlias(); - await esArchiver.unload('x-pack/test/functional/es_archives/reporting/bwc/6_2'); - }); - - it('single job posted can complete in an index created with an older version', async () => { - const reportPaths = []; - reportPaths.push( - await reportingAPI.postJob( - '/api/reporting/generate/csv_searchsource?jobParams=%28browserTimezone%3AAmerica%2FPhoenix%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cfilter%3A%21%28%28meta%3A%28index%3A%27logstash-%2A%27%2Cparams%3A%28%29%29%2Crange%3A%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Cgte%3A%272015-09-20T16%3A00%3A56.290Z%27%2Clte%3A%272015-09-21T10%3A37%3A45.066Z%27%29%29%29%29%2Cindex%3A%27logstash-%2A%27%2Cparent%3A%28filter%3A%21%28%29%2Cindex%3A%27logstash-%2A%27%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%29%2Csort%3A%21%28%28%27%40timestamp%27%3Adesc%29%29%2CtrackTotalHits%3A%21t%2Cversion%3A%21t%29%2Ctitle%3A%27Discover%20search%20%5B2021-07-30T11%3A47%3A03.731-07%3A00%5D%27%29' - ) - ); - await reportingAPI.expectAllJobsToFinishSuccessfully(reportPaths); - }).timeout(1540000); - }); - }); -} diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index 2d968295f09be..a641c867ce235 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -19,7 +19,6 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await reportingAPI.createTestReportingUser(); }); - loadTestFile(require.resolve('./bwc_existing_indexes')); loadTestFile(require.resolve('./security_roles_privileges')); loadTestFile(require.resolve('./generate_csv_discover')); loadTestFile(require.resolve('./csv_v2')); diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts index 5aec831f70897..bee0d6fc44b13 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts @@ -8,20 +8,8 @@ import expect from '@kbn/expect'; import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import { ReportApiJSON } from '@kbn/reporting-common/types'; -import { pick } from 'lodash'; import { FtrProviderContext } from '../ftr_provider_context'; -const apiResponseFields = [ - 'attempts', - 'created_by', - 'jobtype', - 'meta', - 'payload.isDeprecated', - 'payload.title', - 'payload.type', - 'status', -]; - // eslint-disable-next-line import/no-default-export export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); @@ -54,31 +42,18 @@ export default function ({ getService }: FtrProviderContext) { await esArchiver.load('x-pack/test/functional/es_archives/logstash_functional'); }); + afterEach(async () => { + await reportingAPI.deleteAllReports(); + }); + after(async () => { await reportingAPI.teardownLogs(); await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); }); - afterEach(async () => { - await reportingAPI.deleteAllReports(); - }); - it('Posted CSV job is visible in the job count', async () => { const { job, path } = await postJobCSV(); - expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - } - `); + expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); // call the job count api const { text: countText } = await supertestNoAuth @@ -93,20 +68,7 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job is visible in the status check', async () => { // post a job const { job, path } = await postJobCSV(); - expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - } - `); + expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); // call the listing api const { text: listText } = await supertestNoAuth @@ -116,22 +78,6 @@ export default function ({ getService }: FtrProviderContext) { // verify the top item in the list const listingJobs: ReportApiJSON[] = JSON.parse(listText); expect(listingJobs[0].id).to.be(job.id); - expectSnapshot(listingJobs.map((j) => pick(j, apiResponseFields))).toMatchInline(` - Array [ - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - }, - ] - `); // clean up await reportingAPI.waitForJobToFinish(path); @@ -140,20 +86,7 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job is visible in the first page of jobs listing', async () => { // post a job const { job, path } = await postJobCSV(); - expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - } - `); + expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); // call the listing api const { text: listText, status } = await supertestNoAuth @@ -164,22 +97,6 @@ export default function ({ getService }: FtrProviderContext) { // verify the top item in the list const listingJobs: ReportApiJSON[] = JSON.parse(listText); expect(listingJobs[0].id).to.be(job.id); - expectSnapshot(listingJobs.map((j) => pick(j, apiResponseFields))).toMatchInline(` - Array [ - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - }, - ] - `); // clean up await reportingAPI.waitForJobToFinish(path); @@ -188,20 +105,7 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job details are visible in the info API', async () => { // post a job const { job, path } = await postJobCSV(); - expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - } - `); + expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); const { text: infoText, status } = await supertestNoAuth .get(`${INTERNAL_ROUTES.JOBS.INFO_PREFIX}/${job.id}`) @@ -209,20 +113,7 @@ export default function ({ getService }: FtrProviderContext) { expect(status).to.be(200); const info = JSON.parse(infoText); - expectSnapshot(pick(info, apiResponseFields)).toMatchInline(` - Object { - "attempts": 0, - "created_by": false, - "jobtype": "csv_searchsource", - "meta": Object { - "objectType": "search", - }, - "payload": Object { - "title": "A Saved Search With a DATE FILTER", - }, - "status": "pending", - } - `); + expect(info.payload.title).equal('A Saved Search With a DATE FILTER'); await reportingAPI.waitForJobToFinish(path); }); diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index a2bc7e57dc8d7..e6b95bce46272 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -5,12 +5,13 @@ * 2.0. */ +import type { LoadActionPerfOptions } from '@kbn/es-archiver'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; +import type { JobParamsCSV } from '@kbn/reporting-export-types-csv-common'; import type { JobParamsPDFDeprecated } from '@kbn/reporting-export-types-pdf-common'; import type { JobParamsPNGV2 } from '@kbn/reporting-export-types-png-common'; -import type { JobParamsCSV } from '@kbn/reporting-export-types-csv-common'; +import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import rison from '@kbn/rison'; -import { LoadActionPerfOptions } from '@kbn/es-archiver'; -import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import { FtrProviderContext } from '../ftr_provider_context'; function removeWhitespace(str: string) { @@ -211,7 +212,7 @@ export function createScenarios({ getService }: Pick { await esSupertest - .post('/.reporting*/_delete_by_query') + .post(`/${REPORTING_DATA_STREAM_WILDCARD}/_delete_by_query`) .send({ query: { match_all: {} } }) .expect(200); }); @@ -248,7 +249,7 @@ export function createScenarios({ getService }: Pick} A function to call to clean up the index alias that was added. - */ - async coerceReportsIntoExistingIndex(indexName: string) { - log.debug(`ReportingAPI.coerceReportsIntoExistingIndex(${indexName})`); - - // Adding an index alias coerces the report to be generated on an existing index which means any new - // index schema won't be applied. This is important if a point release updated the schema. Reports may still - // be inserted into an existing index before the new schema is applied. - const timestampForIndex = indexTimestamp('week', '.'); - await esSupertest - .post('/_aliases') - .send({ - actions: [ - { - add: { index: indexName, alias: `.reporting-${timestampForIndex}` }, - }, - ], - }) - .expect(200); - - return async () => { - await esSupertest - .post('/_aliases') - .send({ - actions: [ - { - remove: { index: indexName, alias: `.reporting-${timestampForIndex}` }, - }, - ], - }) - .expect(200); - }; - }, - async expectAllJobsToFinishSuccessfully(jobPaths: string[]) { await Promise.all( jobPaths.map(async (path) => { From 1c83747e576c7652816f008fa78a9f3f79a276b8 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 25 Mar 2024 12:52:05 -0700 Subject: [PATCH 02/16] Docs --- docs/user/reporting/index.asciidoc | 7 +++---- packages/kbn-reporting/common/constants.ts | 1 + 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user/reporting/index.asciidoc b/docs/user/reporting/index.asciidoc index 318a901b15e4c..338b2bc53a55a 100644 --- a/docs/user/reporting/index.asciidoc +++ b/docs/user/reporting/index.asciidoc @@ -67,10 +67,9 @@ NOTE: When you create a dashboard report that includes a data table or saved sea . To view and manage reports, open the main menu, then click *Stack Management > Reporting*. -NOTE: Reports are stored in {es} and managed by the `kibana-reporting` {ilm} -({ilm-init}) policy. By default, the policy stores reports forever. To learn -more about {ilm-init} policies, refer to the {es} -{ref}/index-lifecycle-management.html[{ilm-init} documentation]. +NOTE: In "stateful" deployments, reports are stored in {es} and managed by the `kibana-reporting` {ilm} +({ilm-init}) policy. By default, the policy stores reports forever. To learn more about {ilm-init} policies, refer +to the {es} {ref}/index-lifecycle-management.html[{ilm-init} documentation]. [float] [[csv-limitations]] diff --git a/packages/kbn-reporting/common/constants.ts b/packages/kbn-reporting/common/constants.ts index 4620b46e8cab6..e041291323439 100644 --- a/packages/kbn-reporting/common/constants.ts +++ b/packages/kbn-reporting/common/constants.ts @@ -55,6 +55,7 @@ export const REPORTING_MANAGEMENT_HOME = '/app/management/insightsAndAlerting/re * ILM */ +// The ILM policy manages stored reports only in stateful deployments. export const ILM_POLICY_NAME = 'kibana-reporting'; /* From c842af5a726b6a4a201f8a3d821651994a61a2ea Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 25 Mar 2024 13:14:14 -0700 Subject: [PATCH 03/16] Fix ILM tests --- packages/kbn-reporting/server/constants.ts | 2 +- .../lib/store/ilm_policy_manager/constants.ts | 13 ++++++++++++- .../ilm_policy_manager/ilm_policy_manager.ts | 16 ++++++++++++++-- .../server/lib/store/ilm_policy_manager/index.ts | 1 - .../routes/internal/deprecations/deprecations.ts | 4 ++-- .../reporting_and_security/ilm_migration_apis.ts | 3 ++- .../services/scenarios.ts | 5 ++--- 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/packages/kbn-reporting/server/constants.ts b/packages/kbn-reporting/server/constants.ts index da124c371be70..c0606eba027e9 100644 --- a/packages/kbn-reporting/server/constants.ts +++ b/packages/kbn-reporting/server/constants.ts @@ -17,7 +17,7 @@ export const PLUGIN_ID = 'reporting'; // number of backing indices. export const REPORTING_DATA_STREAM = '.kibana-reporting'; -// Pattern to search for historical reports +// Pattern to include historical reports in search for reports export const REPORTING_DATA_STREAM_WILDCARD = '.reporting-*,.kibana-reporting*'; /* diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts index cbbf21094d61f..4143fdca10a38 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts @@ -5,7 +5,10 @@ * 2.0. */ -import type { IlmPutLifecycleRequest } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { + IlmPutLifecycleRequest, + IndicesPutIndexTemplateRequest, +} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; export const reportingIlmPolicy: IlmPutLifecycleRequest['body'] = { policy: { @@ -16,3 +19,11 @@ export const reportingIlmPolicy: IlmPutLifecycleRequest['body'] = { }, }, }; + +export const reportingIndexTemplate: IndicesPutIndexTemplateRequest['body'] = { + index_patterns: ['.kibana-reporting*'], + template: { + settings: { 'index.lifecycle.name': 'kibana-reporting' }, + }, + data_stream: {}, +}; diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index 90bda652898f3..6ae3ec04c8ed6 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -7,10 +7,12 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { ILM_POLICY_NAME } from '@kbn/reporting-common'; -import { reportingIlmPolicy } from './constants'; +import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; +import { reportingIlmPolicy, reportingIndexTemplate } from './constants'; /** - * Responsible for detecting and provisioning the reporting ILM policy. + * Responsible for detecting and provisioning the reporting ILM policy in stateful deployments. + * Must be linked to the data stream once it is created. * * Uses the provided {@link ElasticsearchClient} to scope request privileges. */ @@ -42,4 +44,14 @@ export class IlmPolicyManager { body: reportingIlmPolicy, }); } + + /** + * Link the Reporting ILM policy to the Data Stream index template + */ + public async linkIlmPolicy(): Promise { + await this.client.indices.putIndexTemplate({ + name: REPORTING_DATA_STREAM, + body: reportingIndexTemplate, + }); + } } diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/index.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/index.ts index 045a9ecb59997..a10b1dbb26151 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/index.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/index.ts @@ -5,5 +5,4 @@ * 2.0. */ -export { reportingIlmPolicy } from './constants'; export { IlmPolicyManager } from './ilm_policy_manager'; diff --git a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts index 21156436ada68..1b10a59773ec0 100644 --- a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts +++ b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts @@ -7,7 +7,7 @@ import { errors } from '@elastic/elasticsearch'; import type { Logger, RequestHandler } from '@kbn/core/server'; import { ILM_POLICY_NAME, INTERNAL_ROUTES } from '@kbn/reporting-common'; -import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import type { IlmPolicyStatusResponse } from '@kbn/reporting-common/url'; import type { ReportingCore } from '../../../core'; import { IlmPolicyManager } from '../../../lib'; @@ -125,7 +125,7 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log // Second we migrate all of the existing indices to be managed by the reporting ILM policy try { await client.indices.putSettings({ - index: REPORTING_DATA_STREAM_WILDCARD, + index: REPORTING_DATA_STREAM, body: { index: { lifecycle: { diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/ilm_migration_apis.ts b/x-pack/test/reporting_api_integration/reporting_and_security/ilm_migration_apis.ts index f3e1223c26084..0442917012f49 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/ilm_migration_apis.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/ilm_migration_apis.ts @@ -42,7 +42,8 @@ export default function ({ getService }: FtrProviderContext) { cluster: ['manage_ilm'], indices: [ { names: ['ecommerce'], privileges: ['read'], allow_restricted_indices: false }, - { names: ['.reporting-*'], privileges: ['all'], allow_restricted_indices: true }, + { names: ['.reporting-*'], privileges: ['all'], allow_restricted_indices: true }, // plain indices (from old version) + { names: ['.kibana-reporting'], privileges: ['all'], allow_restricted_indices: true }, // data stream ], run_as: [], }, diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index e6b95bce46272..8e5d873372aa3 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -10,7 +10,7 @@ import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import type { JobParamsCSV } from '@kbn/reporting-export-types-csv-common'; import type { JobParamsPDFDeprecated } from '@kbn/reporting-export-types-pdf-common'; import type { JobParamsPNGV2 } from '@kbn/reporting-export-types-png-common'; -import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import rison from '@kbn/rison'; import { FtrProviderContext } from '../ftr_provider_context'; @@ -65,7 +65,6 @@ export function createScenarios({ getService }: Pick { await esArchiver.unload('x-pack/test/functional/es_archives/reporting/ecommerce'); await kibanaServer.importExport.unload(ecommerceSOPath); - await deleteAllReports(); }; const initLogs = async () => { @@ -249,7 +248,7 @@ export function createScenarios({ getService }: Pick Date: Tue, 2 Apr 2024 11:57:59 -0700 Subject: [PATCH 04/16] Consolidate code for checkIlmMigrationStatus --- packages/kbn-reporting/common/types.ts | 12 +++---- packages/kbn-reporting/common/url.ts | 6 ---- .../migrate_existing_indices_ilm_policy.ts | 7 ++-- .../check_ilm_migration_status.ts | 34 ------------------- .../server/lib/deprecations/index.ts | 2 -- .../ilm_policy_manager/ilm_policy_manager.ts | 24 ++++++++++++- .../internal/deprecations/deprecations.ts | 18 +++++----- 7 files changed, 40 insertions(+), 63 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts diff --git a/packages/kbn-reporting/common/types.ts b/packages/kbn-reporting/common/types.ts index 54cc8abf807fc..46a849820297d 100644 --- a/packages/kbn-reporting/common/types.ts +++ b/packages/kbn-reporting/common/types.ts @@ -114,12 +114,6 @@ export interface ReportingServerInfo { uuid: string; } -export type IlmPolicyMigrationStatus = 'policy-not-found' | 'indices-not-managed-by-policy' | 'ok'; - -export interface IlmPolicyStatusResponse { - status: IlmPolicyMigrationStatus; -} - export interface ReportDocumentHead { _id: string; _index: string; @@ -207,3 +201,9 @@ export interface LicenseCheckResults { showLinks: boolean; message: string; } + +export type IlmPolicyMigrationStatus = 'policy-not-found' | 'indices-not-managed-by-policy' | 'ok'; + +export interface IlmPolicyStatusResponse { + status: IlmPolicyMigrationStatus; +} diff --git a/packages/kbn-reporting/common/url.ts b/packages/kbn-reporting/common/url.ts index 14772b3a612aa..d7140bbd22044 100644 --- a/packages/kbn-reporting/common/url.ts +++ b/packages/kbn-reporting/common/url.ts @@ -29,12 +29,6 @@ export interface LocatorParams

=> { - const migrationStatus = await deprecations.checkIlmMigrationStatus({ - elasticsearchClient: esClient.asInternalUser, - }); + const ilmPolicyManager = IlmPolicyManager.create({ client: esClient.asInternalUser }); + const migrationStatus = await ilmPolicyManager.checkIlmMigrationStatus(); if (migrationStatus !== 'ok') { return [ diff --git a/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts b/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts deleted file mode 100644 index a9a5bdaea721c..0000000000000 --- a/x-pack/plugins/reporting/server/lib/deprecations/check_ilm_migration_status.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { IlmPolicyMigrationStatus } from '@kbn/reporting-common/url'; -import { ILM_POLICY_NAME } from '@kbn/reporting-common'; -import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; -import { IlmPolicyManager } from '../store/ilm_policy_manager'; -import type { DeprecationsDependencies } from './types'; - -export const checkIlmMigrationStatus = async ({ - elasticsearchClient, -}: DeprecationsDependencies): Promise => { - const ilmPolicyManager = IlmPolicyManager.create({ client: elasticsearchClient }); - if (!(await ilmPolicyManager.doesIlmPolicyExist())) { - return 'policy-not-found'; - } - - const reportingIndicesSettings = await elasticsearchClient.indices.getSettings({ - index: REPORTING_DATA_STREAM_WILDCARD, - }); - - const hasUnmanagedIndices = Object.values(reportingIndicesSettings).some((settings) => { - return ( - settings?.settings?.index?.lifecycle?.name !== ILM_POLICY_NAME && - settings?.settings?.['index.lifecycle']?.name !== ILM_POLICY_NAME - ); - }); - - return hasUnmanagedIndices ? 'indices-not-managed-by-policy' : 'ok'; -}; diff --git a/x-pack/plugins/reporting/server/lib/deprecations/index.ts b/x-pack/plugins/reporting/server/lib/deprecations/index.ts index 2ddc46663600f..a4d1ee919e1cc 100644 --- a/x-pack/plugins/reporting/server/lib/deprecations/index.ts +++ b/x-pack/plugins/reporting/server/lib/deprecations/index.ts @@ -9,7 +9,6 @@ import { errors } from '@elastic/elasticsearch'; import Boom from '@hapi/boom'; import { i18n } from '@kbn/i18n'; import { DeprecationsDetails, DocLinksServiceSetup } from '@kbn/core/server'; -import { checkIlmMigrationStatus } from './check_ilm_migration_status'; function deprecationError( title: string, @@ -83,7 +82,6 @@ function getDetailedErrorMessage(error: any): string { } export const deprecations = { - checkIlmMigrationStatus, deprecationError, getDetailedErrorMessage, getErrorStatusCode, diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index 6ae3ec04c8ed6..f3d4611fff8cb 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -7,7 +7,8 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { ILM_POLICY_NAME } from '@kbn/reporting-common'; -import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; +import { IlmPolicyMigrationStatus } from '@kbn/reporting-common/types'; +import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; import { reportingIlmPolicy, reportingIndexTemplate } from './constants'; /** @@ -35,6 +36,27 @@ export class IlmPolicyManager { } } + public async checkIlmMigrationStatus(): Promise { + if (!(await this.doesIlmPolicyExist())) { + return 'policy-not-found'; + } + + // FIXME: should also check that template has link to ILM policy + // FIXME: should also check legacy indices, if any exist + const reportingIndicesSettings = await this.client.indices.getSettings({ + index: REPORTING_DATA_STREAM_WILDCARD, + }); + + const hasUnmanagedIndices = Object.values(reportingIndicesSettings).some((settings) => { + return ( + settings?.settings?.index?.lifecycle?.name !== ILM_POLICY_NAME && + settings?.settings?.['index.lifecycle']?.name !== ILM_POLICY_NAME + ); + }); + + return hasUnmanagedIndices ? 'indices-not-managed-by-policy' : 'ok'; + } + /** * Create the Reporting ILM policy */ diff --git a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts index 1b10a59773ec0..8cef74b424b78 100644 --- a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts +++ b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts @@ -8,10 +8,9 @@ import { errors } from '@elastic/elasticsearch'; import type { Logger, RequestHandler } from '@kbn/core/server'; import { ILM_POLICY_NAME, INTERNAL_ROUTES } from '@kbn/reporting-common'; import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; -import type { IlmPolicyStatusResponse } from '@kbn/reporting-common/url'; +import type { IlmPolicyStatusResponse } from '@kbn/reporting-common/types'; import type { ReportingCore } from '../../../core'; import { IlmPolicyManager } from '../../../lib'; -import { deprecations } from '../../../lib/deprecations'; import { getCounters } from '../../common'; const getAuthzWrapper = @@ -64,14 +63,13 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log authzWrapper(async ({ core }, req, res) => { const counters = getCounters(req.route.method, getStatusPath, reporting.getUsageCounter()); - const { - elasticsearch: { client: scopedClient }, - } = await core; - const checkIlmMigrationStatus = () => { - return deprecations.checkIlmMigrationStatus({ - // We want to make the current status visible to all reporting users - elasticsearchClient: scopedClient.asInternalUser, - }); + const checkIlmMigrationStatus = async () => { + const { + elasticsearch: { client: scopedClient }, + } = await core; + + const ilmPolicyManager = IlmPolicyManager.create({ client: scopedClient.asInternalUser }); + return ilmPolicyManager.checkIlmMigrationStatus(); }; try { From 65b368e78372d8fd8f3ebb91af725f0ba1d32c2d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 25 Mar 2024 15:44:36 -0700 Subject: [PATCH 05/16] Use component template for reporting ILM settings --- docs/settings/reporting-settings.asciidoc | 2 +- packages/kbn-reporting/common/types.ts | 12 +- packages/kbn-reporting/server/constants.ts | 15 ++- .../plugins/reporting/server/config/index.ts | 2 +- ...igrate_existing_indices_ilm_policy.test.ts | 2 +- .../lib/store/ilm_policy_manager/constants.ts | 29 ---- .../ilm_policy_manager/ilm_policy_manager.ts | 70 ++++++++-- .../reporting/server/lib/store/report.ts | 6 +- .../reporting/server/lib/store/store.test.ts | 12 +- .../reporting/server/lib/store/store.ts | 11 +- .../server/routes/common/jobs/jobs_query.ts | 16 ++- .../internal/deprecations/deprecations.ts | 17 +-- .../bwc_existing_indexes.ts | 64 +++++++++ .../reporting_and_security/index.ts | 1 + .../job_apis_csv.ts | 127 ++++++++++++++++-- .../services/scenarios.ts | 9 +- .../services/usage.ts | 38 ++++++ x-pack/test/tsconfig.json | 1 + 18 files changed, 331 insertions(+), 103 deletions(-) delete mode 100644 x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts diff --git a/docs/settings/reporting-settings.asciidoc b/docs/settings/reporting-settings.asciidoc index 1bf741b748576..f871f72db22c0 100644 --- a/docs/settings/reporting-settings.asciidoc +++ b/docs/settings/reporting-settings.asciidoc @@ -70,7 +70,7 @@ reports, you might need to change the following settings. If capturing a report fails for any reason, {kib} will re-queue the report job for retry, as many times as this setting. Defaults to `3`. `xpack.reporting.queue.indexInterval`:: -deprecated:[8.10.0,This setting has no effect.] How often Reporting creates a new index to store report jobs and file contents. +deprecated:[8.15.0,This setting has no effect.] How often Reporting creates a new index to store report jobs and file contents. Valid values are `year`, `month`, `week`, `day`, and `hour`. Defaults to `week`. *NOTE*: This setting exists for backwards compatibility, but is unused. Use the built-in ILM policy provided for the reporting plugin to customize the rollover of Reporting data. diff --git a/packages/kbn-reporting/common/types.ts b/packages/kbn-reporting/common/types.ts index 46a849820297d..54cc8abf807fc 100644 --- a/packages/kbn-reporting/common/types.ts +++ b/packages/kbn-reporting/common/types.ts @@ -114,6 +114,12 @@ export interface ReportingServerInfo { uuid: string; } +export type IlmPolicyMigrationStatus = 'policy-not-found' | 'indices-not-managed-by-policy' | 'ok'; + +export interface IlmPolicyStatusResponse { + status: IlmPolicyMigrationStatus; +} + export interface ReportDocumentHead { _id: string; _index: string; @@ -201,9 +207,3 @@ export interface LicenseCheckResults { showLinks: boolean; message: string; } - -export type IlmPolicyMigrationStatus = 'policy-not-found' | 'indices-not-managed-by-policy' | 'ok'; - -export interface IlmPolicyStatusResponse { - status: IlmPolicyMigrationStatus; -} diff --git a/packages/kbn-reporting/server/constants.ts b/packages/kbn-reporting/server/constants.ts index c0606eba027e9..9d7c1ed225df7 100644 --- a/packages/kbn-reporting/server/constants.ts +++ b/packages/kbn-reporting/server/constants.ts @@ -12,13 +12,14 @@ export const PLUGIN_ID = 'reporting'; * Storage */ -// The name of the Reporting data stream is given to query across all data stored by Reporting. The data -// stream is an alias, automatically configured in the Kibana plugin of Elasticsearch, pointing to an unknown -// number of backing indices. -export const REPORTING_DATA_STREAM = '.kibana-reporting'; - -// Pattern to include historical reports in search for reports -export const REPORTING_DATA_STREAM_WILDCARD = '.reporting-*,.kibana-reporting*'; +// Used to index new documents +export const REPORTING_DATA_STREAM_ALIAS = '.kibana-reporting'; +// Used to retrieve settings +export const REPORTING_DATA_STREAM_WILDCARD = '.kibana-reporting*'; +// Used to search for all reports and check for managing privileges +export const REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY = '.reporting-*,.kibana-reporting*'; +// Name of component template which Kibana overrides for lifecycle settings +export const REPORTING_DATA_STREAM_COMPONENT_TEMPLATE = 'kibana-reporting@custom'; /* * Telemetry diff --git a/x-pack/plugins/reporting/server/config/index.ts b/x-pack/plugins/reporting/server/config/index.ts index a9c2f211cc883..4129717c6d791 100644 --- a/x-pack/plugins/reporting/server/config/index.ts +++ b/x-pack/plugins/reporting/server/config/index.ts @@ -24,7 +24,7 @@ export const config: PluginConfigDescriptor = { }, schema: ConfigSchema, deprecations: ({ unused }) => [ - unused('queue.indexInterval', { level: 'warning' }), // unused since 8.13 + unused('queue.indexInterval', { level: 'warning' }), // unused since 8.15 unused('capture.browser.chromium.maxScreenshotDimension', { level: 'warning' }), // unused since 7.8 unused('capture.browser.type', { level: 'warning' }), unused('poll.jobCompletionNotifier.intervalErrorMultiplier', { level: 'warning' }), // unused since 7.10 diff --git a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts index 420809aba646e..233626d1b7b4c 100644 --- a/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts +++ b/x-pack/plugins/reporting/server/deprecations/migrate_existing_indices_ilm_policy.test.ts @@ -54,7 +54,7 @@ describe("Migrate existing indices' ILM policy deprecations", () => { ], }, "level": "warning", - "message": "New reporting indices will be managed by the \\"kibana-reporting\\" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets the hidden system index pattern \\".reporting-*,.kibana-reporting*\\".", + "message": "New reporting indices will be managed by the \\"kibana-reporting\\" provisioned ILM policy. You must edit this policy to manage the report lifecycle. This change targets the hidden system index pattern \\".kibana-reporting*\\".", "title": "Found reporting indices managed by custom ILM policy.", }, ] diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts deleted file mode 100644 index 4143fdca10a38..0000000000000 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/constants.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type { - IlmPutLifecycleRequest, - IndicesPutIndexTemplateRequest, -} from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; - -export const reportingIlmPolicy: IlmPutLifecycleRequest['body'] = { - policy: { - phases: { - hot: { - actions: {}, - }, - }, - }, -}; - -export const reportingIndexTemplate: IndicesPutIndexTemplateRequest['body'] = { - index_patterns: ['.kibana-reporting*'], - template: { - settings: { 'index.lifecycle.name': 'kibana-reporting' }, - }, - data_stream: {}, -}; diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index f3d4611fff8cb..6f293fade53ef 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -5,15 +5,17 @@ * 2.0. */ +import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from '@kbn/core/server'; import { ILM_POLICY_NAME } from '@kbn/reporting-common'; import { IlmPolicyMigrationStatus } from '@kbn/reporting-common/types'; -import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; -import { reportingIlmPolicy, reportingIndexTemplate } from './constants'; +import { + REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, + REPORTING_DATA_STREAM_WILDCARD, +} from '@kbn/reporting-server'; /** * Responsible for detecting and provisioning the reporting ILM policy in stateful deployments. - * Must be linked to the data stream once it is created. * * Uses the provided {@link ElasticsearchClient} to scope request privileges. */ @@ -24,9 +26,15 @@ export class IlmPolicyManager { return new IlmPolicyManager(opts.client); } + /** + * Check that the ILM policy exists + */ public async doesIlmPolicyExist(): Promise { + const reportingIlmGetLifecycleRequest: estypes.IlmGetLifecycleRequest = { + name: ILM_POLICY_NAME, + }; try { - await this.client.ilm.getLifecycle({ name: ILM_POLICY_NAME }); + await this.client.ilm.getLifecycle(reportingIlmGetLifecycleRequest); return true; } catch (e) { if (e.statusCode === 404) { @@ -41,8 +49,7 @@ export class IlmPolicyManager { return 'policy-not-found'; } - // FIXME: should also check that template has link to ILM policy - // FIXME: should also check legacy indices, if any exist + // FIXME: should also check if legacy indices exist const reportingIndicesSettings = await this.client.indices.getSettings({ index: REPORTING_DATA_STREAM_WILDCARD, }); @@ -61,19 +68,54 @@ export class IlmPolicyManager { * Create the Reporting ILM policy */ public async createIlmPolicy(): Promise { - await this.client.ilm.putLifecycle({ + const reportingIlmPutLifecycleRequest: estypes.IlmPutLifecycleRequest = { name: ILM_POLICY_NAME, - body: reportingIlmPolicy, - }); + policy: { + phases: { + hot: { + actions: {}, + }, + }, + }, + }; + + await this.client.ilm.putLifecycle(reportingIlmPutLifecycleRequest); } /** - * Link the Reporting ILM policy to the Data Stream index template + * Update the Data Stream index template with a link to the Reporting ILM policy */ public async linkIlmPolicy(): Promise { - await this.client.indices.putIndexTemplate({ - name: REPORTING_DATA_STREAM, - body: reportingIndexTemplate, - }); + const reportingIndicesPutTemplateRequest: estypes.ClusterPutComponentTemplateRequest = { + name: REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, + template: { + settings: { + lifecycle: { + name: ILM_POLICY_NAME, + }, + }, + }, + create: false, + }; + + await this.client.cluster.putComponentTemplate(reportingIndicesPutTemplateRequest); + } + + /** + * Update existing index settings to use ILM policy + * + * FIXME: should also migrate legacy indices, if any exist + */ + public async migrateIndicesToIlmPolicy() { + const indicesPutSettingsRequest: estypes.IndicesPutSettingsRequest = { + index: REPORTING_DATA_STREAM_WILDCARD, + settings: { + lifecycle: { + name: ILM_POLICY_NAME, + }, + }, + }; + + await this.client.indices.putSettings(indicesPutSettingsRequest); } } diff --git a/x-pack/plugins/reporting/server/lib/store/report.ts b/x-pack/plugins/reporting/server/lib/store/report.ts index d46f44ca93549..6eb0960aedd93 100644 --- a/x-pack/plugins/reporting/server/lib/store/report.ts +++ b/x-pack/plugins/reporting/server/lib/store/report.ts @@ -10,7 +10,7 @@ import moment from 'moment'; import { v4 as uuidv4 } from 'uuid'; import { JOB_STATUS } from '@kbn/reporting-common'; -import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM_ALIAS } from '@kbn/reporting-server'; import { ReportApiJSON, ReportDocumentHead, @@ -64,7 +64,7 @@ export class Report implements Partial { */ constructor(opts: Partial & Partial, fields?: ReportFields) { this._id = opts._id != null ? opts._id : uuidv4(); - this._index = opts._index ?? REPORTING_DATA_STREAM; // Sets the value to the data stream, unless it's a stored report and we know the name of the backing index + this._index = opts._index ?? REPORTING_DATA_STREAM_ALIAS; // Sets the value to the data stream, unless it's a stored report and we know the name of the backing index this._primary_term = opts._primary_term; this._seq_no = opts._seq_no; @@ -168,7 +168,7 @@ export class Report implements Partial { toApiJSON(): ReportApiJSON { return { id: this._id, - index: this._index ?? REPORTING_DATA_STREAM, + index: this._index ?? REPORTING_DATA_STREAM_ALIAS, kibana_name: this.kibana_name, kibana_id: this.kibana_id, jobtype: this.jobtype, diff --git a/x-pack/plugins/reporting/server/lib/store/store.test.ts b/x-pack/plugins/reporting/server/lib/store/store.test.ts index 6fb28913f7ae6..826e13c0ac696 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.test.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.test.ts @@ -362,16 +362,14 @@ describe('ReportingStore', () => { expect(mockEsClient.ilm.getLifecycle).toHaveBeenCalledWith({ name: 'kibana-reporting' }); expect(mockEsClient.ilm.putLifecycle.mock.calls[0][0]).toMatchInlineSnapshot(` Object { - "body": Object { - "policy": Object { - "phases": Object { - "hot": Object { - "actions": Object {}, - }, + "name": "kibana-reporting", + "policy": Object { + "phases": Object { + "hot": Object { + "actions": Object {}, }, }, }, - "name": "kibana-reporting", } `); }); diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index e2e5bed45ab7d..a4d37ec7fad35 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -14,7 +14,7 @@ import type { ReportOutput, ReportSource, } from '@kbn/reporting-common/types'; -import { REPORTING_DATA_STREAM } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM_ALIAS } from '@kbn/reporting-server'; import moment from 'moment'; import type { Report } from '.'; import { SavedReport } from '.'; @@ -123,13 +123,16 @@ export class ReportingStore { this.logger.debug(`Found ILM policy ${ILM_POLICY_NAME}; skipping creation.`); return; } - this.logger.info(`Creating ILM policy for managing reporting indices: ${ILM_POLICY_NAME}`); + this.logger.info(`Creating ILM policy for reporting data stream: ${ILM_POLICY_NAME}`); await ilmPolicyManager.createIlmPolicy(); + + this.logger.info(`Linking ILM policy to reporting data stream: ${REPORTING_DATA_STREAM_ALIAS}`); + await ilmPolicyManager.linkIlmPolicy(); } private async indexReport(report: Report): Promise { const doc = { - index: REPORTING_DATA_STREAM, + index: REPORTING_DATA_STREAM_ALIAS, id: report._id, refresh: 'wait_for' as estypes.Refresh, op_type: 'create' as const, @@ -155,7 +158,7 @@ export class ReportingStore { await this.createIlmPolicy(); } catch (e) { this.logger.error('Error in start phase'); - this.logger.error(e.body?.error); + this.logger.error(e); throw e; } } diff --git a/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts b/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts index a6dc747130e3c..56b0ac4677449 100644 --- a/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts +++ b/x-pack/plugins/reporting/server/routes/common/jobs/jobs_query.ts @@ -10,7 +10,7 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { i18n } from '@kbn/i18n'; import { JOB_STATUS } from '@kbn/reporting-common'; import type { ReportApiJSON, ReportSource } from '@kbn/reporting-common/types'; -import { REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; +import { REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY } from '@kbn/reporting-server'; import type { ReportingCore } from '../../..'; import { Report } from '../../../lib/store'; import { runtimeFieldKeys, runtimeFields } from '../../../lib/store/runtime_fields'; @@ -92,7 +92,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }); const response = (await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) + elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY }) )) as estypes.SearchResponse; return ( @@ -123,7 +123,7 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }; const response = await execQuery((elasticsearchClient) => - elasticsearchClient.count({ body, index: REPORTING_DATA_STREAM_WILDCARD }) + elasticsearchClient.count({ body, index: REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY }) ); return response?.count ?? 0; @@ -152,7 +152,10 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }); const response = await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) + elasticsearchClient.search({ + body, + index: REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, + }) ); const result = response?.hits?.hits?.[0]; @@ -183,7 +186,10 @@ export function jobsQueryFactory(reportingCore: ReportingCore): JobsQueryFactory }; const response = await execQuery((elasticsearchClient) => - elasticsearchClient.search({ body, index: REPORTING_DATA_STREAM_WILDCARD }) + elasticsearchClient.search({ + body, + index: REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, + }) ); const hits = response?.hits?.hits?.[0]; const status = hits?._source?.status; diff --git a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts index 8cef74b424b78..653da6f535c8c 100644 --- a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts +++ b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts @@ -6,9 +6,9 @@ */ import { errors } from '@elastic/elasticsearch'; import type { Logger, RequestHandler } from '@kbn/core/server'; -import { ILM_POLICY_NAME, INTERNAL_ROUTES } from '@kbn/reporting-common'; -import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; +import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import type { IlmPolicyStatusResponse } from '@kbn/reporting-common/types'; +import { REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY } from '@kbn/reporting-server'; import type { ReportingCore } from '../../../core'; import { IlmPolicyManager } from '../../../lib'; import { getCounters } from '../../common'; @@ -30,7 +30,7 @@ const getAuthzWrapper = index: [ { privileges: ['manage'], // required to do anything with the reporting indices - names: [REPORTING_DATA_STREAM_WILDCARD], + names: [REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY], allow_restricted_indices: true, }, ], @@ -122,16 +122,7 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log // Second we migrate all of the existing indices to be managed by the reporting ILM policy try { - await client.indices.putSettings({ - index: REPORTING_DATA_STREAM, - body: { - index: { - lifecycle: { - name: ILM_POLICY_NAME, - }, - }, - }, - }); + await scopedIlmPolicyManager.migrateIndicesToIlmPolicy(); counters.usageCounter(); diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts new file mode 100644 index 0000000000000..168390bc6fc28 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/bwc_existing_indexes.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../ftr_provider_context'; + +/** + * This file tests the situation when a reporting index spans releases. By default reporting indexes are created + * on a weekly basis, but this is configurable so it is possible a user has this set to yearly. In that event, it + * is possible report data is getting posted to an index that was created by a very old version. We don't have a + * reporting index migration plan, so this test is important to ensure BWC, or that in the event we decide to make + * a major change in a major release, we handle it properly. + */ + +// eslint-disable-next-line import/no-default-export +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const reportingAPI = getService('reportingAPI'); + const kibanaServer = getService('kibanaServer'); + + describe('BWC report generation into existing indexes', () => { + let cleanupIndexAlias: () => Promise; + + describe('existing 6_2 index', () => { + before('load data and add index alias', async () => { + await reportingAPI.deleteAllReports(); + // data to report on + await esArchiver.load('test/functional/fixtures/es_archiver/logstash_functional'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); + + // archive with reporting index mappings v6.2 + await esArchiver.load('x-pack/test/functional/es_archives/reporting/bwc/6_2'); + + // The index name in the reporting/bwc/6_2 archive. + const ARCHIVED_REPORTING_INDEX = '.reporting-2018.03.11'; + // causes reporting to assume the v6.2 index is the one to use for new jobs posted + cleanupIndexAlias = await reportingAPI.coerceReportsIntoExistingIndex( + ARCHIVED_REPORTING_INDEX + ); + }); + + after('remove index alias', async () => { + await esArchiver.unload('test/functional/fixtures/es_archiver/logstash_functional'); + await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); + + await cleanupIndexAlias(); + await esArchiver.unload('x-pack/test/functional/es_archives/reporting/bwc/6_2'); + }); + + it('single job posted can complete in an index created with an older version', async () => { + const reportPaths = []; + reportPaths.push( + await reportingAPI.postJob( + '/api/reporting/generate/csv_searchsource?jobParams=%28browserTimezone%3AAmerica%2FPhoenix%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cfilter%3A%21%28%28meta%3A%28index%3A%27logstash-%2A%27%2Cparams%3A%28%29%29%2Crange%3A%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Cgte%3A%272015-09-20T16%3A00%3A56.290Z%27%2Clte%3A%272015-09-21T10%3A37%3A45.066Z%27%29%29%29%29%2Cindex%3A%27logstash-%2A%27%2Cparent%3A%28filter%3A%21%28%29%2Cindex%3A%27logstash-%2A%27%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%29%2Csort%3A%21%28%28%27%40timestamp%27%3Adesc%29%29%2CtrackTotalHits%3A%21t%2Cversion%3A%21t%29%2Ctitle%3A%27Discover%20search%20%5B2021-07-30T11%3A47%3A03.731-07%3A00%5D%27%29' + ) + ); + await reportingAPI.expectAllJobsToFinishSuccessfully(reportPaths); + }).timeout(1540000); + }); + }); +} diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index a641c867ce235..2d968295f09be 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -19,6 +19,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { await reportingAPI.createTestReportingUser(); }); + loadTestFile(require.resolve('./bwc_existing_indexes')); loadTestFile(require.resolve('./security_roles_privileges')); loadTestFile(require.resolve('./generate_csv_discover')); loadTestFile(require.resolve('./csv_v2')); diff --git a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts index bee0d6fc44b13..5aec831f70897 100644 --- a/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts +++ b/x-pack/test/reporting_api_integration/reporting_without_security/job_apis_csv.ts @@ -8,8 +8,20 @@ import expect from '@kbn/expect'; import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import { ReportApiJSON } from '@kbn/reporting-common/types'; +import { pick } from 'lodash'; import { FtrProviderContext } from '../ftr_provider_context'; +const apiResponseFields = [ + 'attempts', + 'created_by', + 'jobtype', + 'meta', + 'payload.isDeprecated', + 'payload.title', + 'payload.type', + 'status', +]; + // eslint-disable-next-line import/no-default-export export default function ({ getService }: FtrProviderContext) { const esArchiver = getService('esArchiver'); @@ -42,18 +54,31 @@ export default function ({ getService }: FtrProviderContext) { await esArchiver.load('x-pack/test/functional/es_archives/logstash_functional'); }); - afterEach(async () => { - await reportingAPI.deleteAllReports(); - }); - after(async () => { await reportingAPI.teardownLogs(); await esArchiver.unload('x-pack/test/functional/es_archives/logstash_functional'); }); + afterEach(async () => { + await reportingAPI.deleteAllReports(); + }); + it('Posted CSV job is visible in the job count', async () => { const { job, path } = await postJobCSV(); - expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); + expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + } + `); // call the job count api const { text: countText } = await supertestNoAuth @@ -68,7 +93,20 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job is visible in the status check', async () => { // post a job const { job, path } = await postJobCSV(); - expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); + expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + } + `); // call the listing api const { text: listText } = await supertestNoAuth @@ -78,6 +116,22 @@ export default function ({ getService }: FtrProviderContext) { // verify the top item in the list const listingJobs: ReportApiJSON[] = JSON.parse(listText); expect(listingJobs[0].id).to.be(job.id); + expectSnapshot(listingJobs.map((j) => pick(j, apiResponseFields))).toMatchInline(` + Array [ + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + }, + ] + `); // clean up await reportingAPI.waitForJobToFinish(path); @@ -86,7 +140,20 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job is visible in the first page of jobs listing', async () => { // post a job const { job, path } = await postJobCSV(); - expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); + expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + } + `); // call the listing api const { text: listText, status } = await supertestNoAuth @@ -97,6 +164,22 @@ export default function ({ getService }: FtrProviderContext) { // verify the top item in the list const listingJobs: ReportApiJSON[] = JSON.parse(listText); expect(listingJobs[0].id).to.be(job.id); + expectSnapshot(listingJobs.map((j) => pick(j, apiResponseFields))).toMatchInline(` + Array [ + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + }, + ] + `); // clean up await reportingAPI.waitForJobToFinish(path); @@ -105,7 +188,20 @@ export default function ({ getService }: FtrProviderContext) { it('Posted CSV job details are visible in the info API', async () => { // post a job const { job, path } = await postJobCSV(); - expect(job.payload.title).equal('A Saved Search With a DATE FILTER'); + expectSnapshot(pick(job, apiResponseFields)).toMatchInline(` + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + } + `); const { text: infoText, status } = await supertestNoAuth .get(`${INTERNAL_ROUTES.JOBS.INFO_PREFIX}/${job.id}`) @@ -113,7 +209,20 @@ export default function ({ getService }: FtrProviderContext) { expect(status).to.be(200); const info = JSON.parse(infoText); - expect(info.payload.title).equal('A Saved Search With a DATE FILTER'); + expectSnapshot(pick(info, apiResponseFields)).toMatchInline(` + Object { + "attempts": 0, + "created_by": false, + "jobtype": "csv_searchsource", + "meta": Object { + "objectType": "search", + }, + "payload": Object { + "title": "A Saved Search With a DATE FILTER", + }, + "status": "pending", + } + `); await reportingAPI.waitForJobToFinish(path); }); diff --git a/x-pack/test/reporting_api_integration/services/scenarios.ts b/x-pack/test/reporting_api_integration/services/scenarios.ts index 8e5d873372aa3..253c0d36354ce 100644 --- a/x-pack/test/reporting_api_integration/services/scenarios.ts +++ b/x-pack/test/reporting_api_integration/services/scenarios.ts @@ -10,7 +10,10 @@ import { INTERNAL_ROUTES } from '@kbn/reporting-common'; import type { JobParamsCSV } from '@kbn/reporting-export-types-csv-common'; import type { JobParamsPDFDeprecated } from '@kbn/reporting-export-types-pdf-common'; import type { JobParamsPNGV2 } from '@kbn/reporting-export-types-png-common'; -import { REPORTING_DATA_STREAM, REPORTING_DATA_STREAM_WILDCARD } from '@kbn/reporting-server'; +import { + REPORTING_DATA_STREAM_WILDCARD, + REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, +} from '@kbn/reporting-server'; import rison from '@kbn/rison'; import { FtrProviderContext } from '../ftr_provider_context'; @@ -211,7 +214,7 @@ export function createScenarios({ getService }: Pick { await esSupertest - .post(`/${REPORTING_DATA_STREAM_WILDCARD}/_delete_by_query`) + .post(`/${REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY}/_delete_by_query`) .send({ query: { match_all: {} } }) .expect(200); }); @@ -248,7 +251,7 @@ export function createScenarios({ getService }: Pick} A function to call to clean up the index alias that was added. + */ + async coerceReportsIntoExistingIndex(indexName: string) { + log.debug(`ReportingAPI.coerceReportsIntoExistingIndex(${indexName})`); + + // Adding an index alias coerces the report to be generated on an existing index which means any new + // index schema won't be applied. This is important if a point release updated the schema. Reports may still + // be inserted into an existing index before the new schema is applied. + const timestampForIndex = indexTimestamp('week', '.'); + await esSupertest + .post('/_aliases') + .send({ + actions: [ + { + add: { index: indexName, alias: `.reporting-${timestampForIndex}` }, + }, + ], + }) + .expect(200); + + return async () => { + await esSupertest + .post('/_aliases') + .send({ + actions: [ + { + remove: { index: indexName, alias: `.reporting-${timestampForIndex}` }, + }, + ], + }) + .expect(200); + }; + }, + async expectAllJobsToFinishSuccessfully(jobPaths: string[]) { await Promise.all( jobPaths.map(async (path) => { diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 4634b40c74518..d5f46bce279af 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -158,6 +158,7 @@ "@kbn/reporting-export-types-pdf-common", "@kbn/reporting-export-types-png-common", "@kbn/reporting-common", + "@kbn/reporting-server", "@kbn/observability-logs-explorer-plugin", "@kbn/io-ts-utils", "@kbn/logs-explorer-plugin", From d3e199de25ce38cd0fd6f4bc44505433ea5fb23d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 May 2024 13:20:54 -0700 Subject: [PATCH 06/16] Fix issue: policy would not be linked if was created in previous startup --- x-pack/plugins/reporting/server/lib/store/store.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index a4d37ec7fad35..46e3d2446a470 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -121,7 +121,9 @@ export class ReportingStore { const ilmPolicyManager = IlmPolicyManager.create({ client }); if (await ilmPolicyManager.doesIlmPolicyExist()) { this.logger.debug(`Found ILM policy ${ILM_POLICY_NAME}; skipping creation.`); - return; + } else { + this.logger.info(`Creating ILM policy for reporting data stream: ${ILM_POLICY_NAME}`); + await ilmPolicyManager.createIlmPolicy(); } this.logger.info(`Creating ILM policy for reporting data stream: ${ILM_POLICY_NAME}`); await ilmPolicyManager.createIlmPolicy(); From d7848c7156fa3e6958cdc9854eed080fdd72b906 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 May 2024 14:26:55 -0700 Subject: [PATCH 07/16] Fix migration: link template and update settings of backing indices --- packages/kbn-reporting/server/constants.ts | 2 + .../ilm_policy_manager/ilm_policy_manager.ts | 86 ++++++++++++++----- .../internal/deprecations/deprecations.ts | 29 ++++--- 3 files changed, 84 insertions(+), 33 deletions(-) diff --git a/packages/kbn-reporting/server/constants.ts b/packages/kbn-reporting/server/constants.ts index 9d7c1ed225df7..16f3992a6e409 100644 --- a/packages/kbn-reporting/server/constants.ts +++ b/packages/kbn-reporting/server/constants.ts @@ -16,6 +16,8 @@ export const PLUGIN_ID = 'reporting'; export const REPORTING_DATA_STREAM_ALIAS = '.kibana-reporting'; // Used to retrieve settings export const REPORTING_DATA_STREAM_WILDCARD = '.kibana-reporting*'; +// Index pattern of plain indices before Reporting used Data Stream storage +export const REPORTING_LEGACY_INDICES = '.reporting-*'; // Used to search for all reports and check for managing privileges export const REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY = '.reporting-*,.kibana-reporting*'; // Name of component template which Kibana overrides for lifecycle settings diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index 6f293fade53ef..c747052c3a43c 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -10,8 +10,10 @@ import type { ElasticsearchClient } from '@kbn/core/server'; import { ILM_POLICY_NAME } from '@kbn/reporting-common'; import { IlmPolicyMigrationStatus } from '@kbn/reporting-common/types'; import { + REPORTING_DATA_STREAM_ALIAS, REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, REPORTING_DATA_STREAM_WILDCARD, + REPORTING_LEGACY_INDICES, } from '@kbn/reporting-server'; /** @@ -49,19 +51,26 @@ export class IlmPolicyManager { return 'policy-not-found'; } - // FIXME: should also check if legacy indices exist - const reportingIndicesSettings = await this.client.indices.getSettings({ - index: REPORTING_DATA_STREAM_WILDCARD, - }); + const [reportingDataStreamSettings, reportingLegacyIndexSettings] = await Promise.all([ + this.client.indices.getSettings({ + index: REPORTING_DATA_STREAM_WILDCARD, + }), + this.client.indices.getSettings({ + index: REPORTING_LEGACY_INDICES, + }), + ]); - const hasUnmanagedIndices = Object.values(reportingIndicesSettings).some((settings) => { + const hasUnmanaged = (settings: estypes.IndicesIndexState) => { return ( settings?.settings?.index?.lifecycle?.name !== ILM_POLICY_NAME && settings?.settings?.['index.lifecycle']?.name !== ILM_POLICY_NAME ); - }); + }; - return hasUnmanagedIndices ? 'indices-not-managed-by-policy' : 'ok'; + const hasUnmanagedDataStream = Object.values(reportingDataStreamSettings).some(hasUnmanaged); + const hasUnmanagedIndices = Object.values(reportingLegacyIndexSettings).some(hasUnmanaged); + + return hasUnmanagedDataStream || hasUnmanagedIndices ? 'indices-not-managed-by-policy' : 'ok'; } /** @@ -85,7 +94,7 @@ export class IlmPolicyManager { /** * Update the Data Stream index template with a link to the Reporting ILM policy */ - public async linkIlmPolicy(): Promise { + public async linkIlmPolicy() { const reportingIndicesPutTemplateRequest: estypes.ClusterPutComponentTemplateRequest = { name: REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, template: { @@ -97,25 +106,62 @@ export class IlmPolicyManager { }, create: false, }; + const putTemplateAcknowledged = await this.client.cluster.putComponentTemplate( + reportingIndicesPutTemplateRequest + ); - await this.client.cluster.putComponentTemplate(reportingIndicesPutTemplateRequest); + let backingIndicesAcknowledged: { acknowledged: boolean | null } = { acknowledged: null }; + const backingIndicesExist = await this.client.indices.exists({ + index: REPORTING_DATA_STREAM_ALIAS, + expand_wildcards: ['hidden'], + }); + if (backingIndicesExist) { + const datastreamPutSettingsRequest: estypes.IndicesPutSettingsRequest = { + index: REPORTING_DATA_STREAM_ALIAS, + settings: { + lifecycle: { + name: ILM_POLICY_NAME, + }, + }, + }; + backingIndicesAcknowledged = await this.client.indices.putSettings( + datastreamPutSettingsRequest + ); + } + + return { putTemplateResponse: putTemplateAcknowledged, backingIndicesAcknowledged }; } /** - * Update existing index settings to use ILM policy - * - * FIXME: should also migrate legacy indices, if any exist + * Update datastream to use ILM policy. If legacy indices exist, this attempts to link + * the ILM policy to them as well. */ public async migrateIndicesToIlmPolicy() { - const indicesPutSettingsRequest: estypes.IndicesPutSettingsRequest = { - index: REPORTING_DATA_STREAM_WILDCARD, - settings: { - lifecycle: { - name: ILM_POLICY_NAME, + const { + putTemplateResponse: { acknowledged: putTemplateAcknowledged }, + backingIndicesAcknowledged: { acknowledged: backingIndicesAcknowledged }, + } = await this.linkIlmPolicy(); + + let legacyAcknowledged: boolean | null = null; + const legacyExists = await this.client.indices.exists({ + index: REPORTING_LEGACY_INDICES, + expand_wildcards: ['hidden'], + }); + if (legacyExists) { + const legacyIndicesPutSettingsRequest: estypes.IndicesPutSettingsRequest = { + index: REPORTING_LEGACY_INDICES, + settings: { + lifecycle: { + name: ILM_POLICY_NAME, + }, }, - }, - }; + }; + const { acknowledged } = await this.client.indices.putSettings( + legacyIndicesPutSettingsRequest + ); + legacyAcknowledged = acknowledged; + } - await this.client.indices.putSettings(indicesPutSettingsRequest); + return { putTemplateAcknowledged, backingIndicesAcknowledged, legacyAcknowledged }; } } diff --git a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts index 653da6f535c8c..ae8ff7fd55cf1 100644 --- a/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts +++ b/x-pack/plugins/reporting/server/routes/internal/deprecations/deprecations.ts @@ -64,9 +64,7 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log const counters = getCounters(req.route.method, getStatusPath, reporting.getUsageCounter()); const checkIlmMigrationStatus = async () => { - const { - elasticsearch: { client: scopedClient }, - } = await core; + const { client: scopedClient } = (await core).elasticsearch; const ilmPolicyManager = IlmPolicyManager.create({ client: scopedClient.asInternalUser }); return ilmPolicyManager.checkIlmMigrationStatus(); @@ -102,16 +100,15 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log authzWrapper(async ({ core }, req, res) => { const counters = getCounters(req.route.method, migrateApiPath, reporting.getUsageCounter()); - const { - client: { asCurrentUser: client }, - } = (await core).elasticsearch; - - const scopedIlmPolicyManager = IlmPolicyManager.create({ - client, - }); - // First we ensure that the reporting ILM policy exists in the cluster try { + const { + client: { asCurrentUser }, + } = (await core).elasticsearch; + const scopedIlmPolicyManager = IlmPolicyManager.create({ + client: asCurrentUser, + }); + // We don't want to overwrite an existing reporting policy because it may contain alterations made by users if (!(await scopedIlmPolicyManager.doesIlmPolicyExist())) { await scopedIlmPolicyManager.createIlmPolicy(); @@ -122,11 +119,17 @@ export const registerDeprecationsRoutes = (reporting: ReportingCore, logger: Log // Second we migrate all of the existing indices to be managed by the reporting ILM policy try { - await scopedIlmPolicyManager.migrateIndicesToIlmPolicy(); + const { + client: { asInternalUser }, + } = (await core).elasticsearch; + const unscopedIlmPolicyManager = IlmPolicyManager.create({ + client: asInternalUser, + }); + const response = await unscopedIlmPolicyManager.migrateIndicesToIlmPolicy(); counters.usageCounter(); - return res.ok(); + return res.ok({ body: response }); } catch (err) { logger.error(err); From 791480736553d64a1b6787b06fcc558ac4a7af86 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 May 2024 13:21:09 -0700 Subject: [PATCH 08/16] Improve logging --- .../store/ilm_policy_manager/ilm_policy_manager.ts | 4 ++++ x-pack/plugins/reporting/server/lib/store/store.ts | 11 +++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index c747052c3a43c..80ced460c664b 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -46,6 +46,10 @@ export class IlmPolicyManager { } } + /** + * This method is automatically called on the Stack Management > Reporting page, by the `` API for users with + * privilege to manage ILM, to notify them when attention is needed to update the policy for any reason. + */ public async checkIlmMigrationStatus(): Promise { if (!(await this.doesIlmPolicyExist())) { return 'policy-not-found'; diff --git a/x-pack/plugins/reporting/server/lib/store/store.ts b/x-pack/plugins/reporting/server/lib/store/store.ts index 46e3d2446a470..648230113780d 100644 --- a/x-pack/plugins/reporting/server/lib/store/store.ts +++ b/x-pack/plugins/reporting/server/lib/store/store.ts @@ -14,7 +14,10 @@ import type { ReportOutput, ReportSource, } from '@kbn/reporting-common/types'; -import { REPORTING_DATA_STREAM_ALIAS } from '@kbn/reporting-server'; +import { + REPORTING_DATA_STREAM_ALIAS, + REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, +} from '@kbn/reporting-server'; import moment from 'moment'; import type { Report } from '.'; import { SavedReport } from '.'; @@ -125,10 +128,10 @@ export class ReportingStore { this.logger.info(`Creating ILM policy for reporting data stream: ${ILM_POLICY_NAME}`); await ilmPolicyManager.createIlmPolicy(); } - this.logger.info(`Creating ILM policy for reporting data stream: ${ILM_POLICY_NAME}`); - await ilmPolicyManager.createIlmPolicy(); - this.logger.info(`Linking ILM policy to reporting data stream: ${REPORTING_DATA_STREAM_ALIAS}`); + this.logger.info( + `Linking ILM policy to reporting data stream: ${REPORTING_DATA_STREAM_ALIAS}, component template: ${REPORTING_DATA_STREAM_COMPONENT_TEMPLATE}` + ); await ilmPolicyManager.linkIlmPolicy(); } From 39a5a0c92bee1e3f677c6361aa6c820cd9821f83 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Mon, 13 May 2024 16:33:50 -0700 Subject: [PATCH 09/16] update syntax --- packages/kbn-reporting/common/types.ts | 2 +- .../ilm_policy_manager/ilm_policy_manager.ts | 60 ++++--------------- .../services/index_timestamp.ts | 41 +++++++++++++ 3 files changed, 55 insertions(+), 48 deletions(-) create mode 100644 x-pack/test/reporting_api_integration/services/index_timestamp.ts diff --git a/packages/kbn-reporting/common/types.ts b/packages/kbn-reporting/common/types.ts index 54cc8abf807fc..39d5a79e731c6 100644 --- a/packages/kbn-reporting/common/types.ts +++ b/packages/kbn-reporting/common/types.ts @@ -148,7 +148,7 @@ export interface ReportSource { }; migration_version: string; // for reminding the user to update their POST URL attempts: number; // initially populated as 0 - created_at: string; // creation timestamp in UTC + created_at: string; // timestamp in UTC '@timestamp'?: string; // creation timestamp, only used for data streams compatibility status: JOB_STATUS; diff --git a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts index 80ced460c664b..950d6ea5e84c6 100644 --- a/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts +++ b/x-pack/plugins/reporting/server/lib/store/ilm_policy_manager/ilm_policy_manager.ts @@ -32,11 +32,8 @@ export class IlmPolicyManager { * Check that the ILM policy exists */ public async doesIlmPolicyExist(): Promise { - const reportingIlmGetLifecycleRequest: estypes.IlmGetLifecycleRequest = { - name: ILM_POLICY_NAME, - }; try { - await this.client.ilm.getLifecycle(reportingIlmGetLifecycleRequest); + await this.client.ilm.getLifecycle({ name: ILM_POLICY_NAME }); return true; } catch (e) { if (e.statusCode === 404) { @@ -81,38 +78,21 @@ export class IlmPolicyManager { * Create the Reporting ILM policy */ public async createIlmPolicy(): Promise { - const reportingIlmPutLifecycleRequest: estypes.IlmPutLifecycleRequest = { + await this.client.ilm.putLifecycle({ name: ILM_POLICY_NAME, - policy: { - phases: { - hot: { - actions: {}, - }, - }, - }, - }; - - await this.client.ilm.putLifecycle(reportingIlmPutLifecycleRequest); + policy: { phases: { hot: { actions: {} } } }, + }); } /** * Update the Data Stream index template with a link to the Reporting ILM policy */ public async linkIlmPolicy() { - const reportingIndicesPutTemplateRequest: estypes.ClusterPutComponentTemplateRequest = { + const putTemplateAcknowledged = await this.client.cluster.putComponentTemplate({ name: REPORTING_DATA_STREAM_COMPONENT_TEMPLATE, - template: { - settings: { - lifecycle: { - name: ILM_POLICY_NAME, - }, - }, - }, + template: { settings: { lifecycle: { name: ILM_POLICY_NAME } } }, create: false, - }; - const putTemplateAcknowledged = await this.client.cluster.putComponentTemplate( - reportingIndicesPutTemplateRequest - ); + }); let backingIndicesAcknowledged: { acknowledged: boolean | null } = { acknowledged: null }; const backingIndicesExist = await this.client.indices.exists({ @@ -120,17 +100,10 @@ export class IlmPolicyManager { expand_wildcards: ['hidden'], }); if (backingIndicesExist) { - const datastreamPutSettingsRequest: estypes.IndicesPutSettingsRequest = { + backingIndicesAcknowledged = await this.client.indices.putSettings({ index: REPORTING_DATA_STREAM_ALIAS, - settings: { - lifecycle: { - name: ILM_POLICY_NAME, - }, - }, - }; - backingIndicesAcknowledged = await this.client.indices.putSettings( - datastreamPutSettingsRequest - ); + settings: { lifecycle: { name: ILM_POLICY_NAME } }, + }); } return { putTemplateResponse: putTemplateAcknowledged, backingIndicesAcknowledged }; @@ -152,17 +125,10 @@ export class IlmPolicyManager { expand_wildcards: ['hidden'], }); if (legacyExists) { - const legacyIndicesPutSettingsRequest: estypes.IndicesPutSettingsRequest = { + const { acknowledged } = await this.client.indices.putSettings({ index: REPORTING_LEGACY_INDICES, - settings: { - lifecycle: { - name: ILM_POLICY_NAME, - }, - }, - }; - const { acknowledged } = await this.client.indices.putSettings( - legacyIndicesPutSettingsRequest - ); + settings: { lifecycle: { name: ILM_POLICY_NAME } }, + }); legacyAcknowledged = acknowledged; } diff --git a/x-pack/test/reporting_api_integration/services/index_timestamp.ts b/x-pack/test/reporting_api_integration/services/index_timestamp.ts new file mode 100644 index 0000000000000..ade5fd19f4618 --- /dev/null +++ b/x-pack/test/reporting_api_integration/services/index_timestamp.ts @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import moment, { unitOfTime } from 'moment'; + +export const intervals = ['year', 'month', 'week', 'day', 'hour', 'minute']; + +export function indexTimestamp(intervalStr: string, separator = '-') { + const startOf = intervalStr as unitOfTime.StartOf; + if (separator.match(/[a-z]/i)) throw new Error('Interval separator can not be a letter'); + + const index = intervals.indexOf(intervalStr); + if (index === -1) throw new Error('Invalid index interval: ' + intervalStr); + + const m = moment(); + m.startOf(startOf); + + let dateString; + switch (intervalStr) { + case 'year': + dateString = 'YYYY'; + break; + case 'month': + dateString = `YYYY${separator}MM`; + break; + case 'hour': + dateString = `YYYY${separator}MM${separator}DD${separator}HH`; + break; + case 'minute': + dateString = `YYYY${separator}MM${separator}DD${separator}HH${separator}mm`; + break; + default: + dateString = `YYYY${separator}MM${separator}DD`; + } + + return m.format(dateString); +} From 161e45a62486b848669099a27c9432a6af856f90 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 14 May 2024 08:03:39 -0700 Subject: [PATCH 10/16] add index_timestamp --- x-pack/test/reporting_api_integration/services/usage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/test/reporting_api_integration/services/usage.ts b/x-pack/test/reporting_api_integration/services/usage.ts index 70f60846d0219..dc2f6c2efdd2c 100644 --- a/x-pack/test/reporting_api_integration/services/usage.ts +++ b/x-pack/test/reporting_api_integration/services/usage.ts @@ -7,8 +7,8 @@ import expect from '@kbn/expect'; import { INTERNAL_ROUTES, PUBLIC_ROUTES } from '@kbn/reporting-common'; -import { indexTimestamp } from '@kbn/reporting-plugin/server/lib/store/index_timestamp'; import { Response } from 'supertest'; +import { indexTimestamp } from './index_timestamp'; import { FtrProviderContext } from '../ftr_provider_context'; export function createUsageServices({ getService }: FtrProviderContext) { From 70436bd601d6f3ba196c3a5525480e16426babd5 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 14 May 2024 08:09:00 -0700 Subject: [PATCH 11/16] update tsconfig --- x-pack/test/tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index d5f46bce279af..4634b40c74518 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -158,7 +158,6 @@ "@kbn/reporting-export-types-pdf-common", "@kbn/reporting-export-types-png-common", "@kbn/reporting-common", - "@kbn/reporting-server", "@kbn/observability-logs-explorer-plugin", "@kbn/io-ts-utils", "@kbn/logs-explorer-plugin", From 7735d61a5c16cc770c415fa463a6e1cc00f75c1d Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 14 May 2024 08:03:39 -0700 Subject: [PATCH 12/16] add index_timestamp --- x-pack/test/reporting_api_integration/services/usage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/test/reporting_api_integration/services/usage.ts b/x-pack/test/reporting_api_integration/services/usage.ts index dc2f6c2efdd2c..3e9167e44e522 100644 --- a/x-pack/test/reporting_api_integration/services/usage.ts +++ b/x-pack/test/reporting_api_integration/services/usage.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import expect from '@kbn/expect'; import { INTERNAL_ROUTES, PUBLIC_ROUTES } from '@kbn/reporting-common'; import { Response } from 'supertest'; From 9142cf690f0345807a2913b108d37cdee3da1cc9 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 14 May 2024 14:38:34 -0700 Subject: [PATCH 13/16] update ContentStream write operation --- .../reporting/server/lib/content_stream.ts | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/content_stream.ts b/x-pack/plugins/reporting/server/lib/content_stream.ts index 17362516d2c9d..b3c113dd43c3b 100644 --- a/x-pack/plugins/reporting/server/lib/content_stream.ts +++ b/x-pack/plugins/reporting/server/lib/content_stream.ts @@ -8,9 +8,13 @@ import { Duplex } from 'stream'; import { v4 as uuidv4 } from 'uuid'; -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient, Logger } from '@kbn/core/server'; import type { ReportSource } from '@kbn/reporting-common/types'; +import { + REPORTING_DATA_STREAM_ALIAS, + REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, +} from '@kbn/reporting-server'; import type { ReportingCore } from '..'; const ONE_MB = 1024 * 1024; @@ -31,6 +35,7 @@ interface ChunkOutput { } interface ChunkSource { + '@timestamp': string; parent_id: string; output: ChunkOutput; } @@ -90,7 +95,7 @@ export class ContentStream extends Duplex { private async readHead() { const { id, index } = this.document; - const body: SearchRequest['body'] = { + const body: SearchRequest = { _source: { includes: ['output.content', 'output.size', 'jobtype'] }, query: { constant_score: { @@ -110,13 +115,14 @@ export class ContentStream extends Duplex { const hits = response?.hits?.hits?.[0]; this.jobSize = hits?._source?.output?.size; + this.logger.debug(`Reading job of size ${this.jobSize}`); return hits?._source?.output?.content; } private async readChunk() { - const { id, index } = this.document; - const body: SearchRequest['body'] = { + const { id } = this.document; + const body: SearchRequest = { _source: { includes: ['output.content'] }, query: { constant_score: { @@ -132,7 +138,10 @@ export class ContentStream extends Duplex { this.logger.debug(`Reading chunk #${this.chunksRead}.`); - const response = await this.client.search({ body, index }); + const response = await this.client.search({ + body, + index: REPORTING_DATA_STREAM_WILDCARD_WITH_LEGACY, + }); const hits = response?.hits?.hits?.[0]; return hits?._source?.output.content; @@ -179,10 +188,11 @@ export class ContentStream extends Duplex { } private async writeHead(content: string) { - this.logger.debug(`Updating report contents.`); + this.logger.debug(`Updating chunk #0 (${this.document.id}).`); const body = await this.client.update({ ...this.document, + refresh: 'wait_for', body: { doc: { output: { content }, @@ -194,16 +204,19 @@ export class ContentStream extends Duplex { } private async writeChunk(content: string) { - const { id: parentId, index } = this.document; + const { id: parentId } = this.document; const id = uuidv4(); this.logger.debug(`Writing chunk #${this.chunksWritten} (${id}).`); await this.client.index({ id, - index, + index: REPORTING_DATA_STREAM_ALIAS, + refresh: 'wait_for', + op_type: 'create', body: { parent_id: parentId, + '@timestamp': new Date(0).toISOString(), // required for data streams compatibility output: { content, chunk: this.chunksWritten, From ca209af37e87505b3fd31fc023340ba0b0e87f76 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Tue, 14 May 2024 16:07:34 -0700 Subject: [PATCH 14/16] Increase timeout for max size reached test --- x-pack/test/functional/apps/discover/reporting.ts | 6 +++--- x-pack/test/tsconfig.json | 1 + .../test_suites/common/reporting/generate_csv_discover.ts | 4 +++- .../test_suites/common/discover/x_pack/reporting.ts | 6 +++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/x-pack/test/functional/apps/discover/reporting.ts b/x-pack/test/functional/apps/discover/reporting.ts index cc5a198cb5416..61ddea54a7cb1 100644 --- a/x-pack/test/functional/apps/discover/reporting.ts +++ b/x-pack/test/functional/apps/discover/reporting.ts @@ -36,14 +36,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await browser.refresh(); }; - const getReport = async () => { + const getReport = async ({ timeout } = { timeout: 60 * 1000 }) => { // close any open notification toasts await toasts.dismissAll(); await PageObjects.reporting.openExportTab(); await PageObjects.reporting.clickGenerateReportButton(); - const url = await PageObjects.reporting.getReportURL(60000); + const url = await PageObjects.reporting.getReportURL(timeout); const res = await PageObjects.reporting.getResponse(url ?? ''); expect(res.status).to.equal(200); @@ -173,7 +173,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.saveSearch('large export'); // match file length, the beginning and the end of the csv file contents - const { text: csvFile } = await getReport(); + const { text: csvFile } = await getReport({ timeout: 80 * 1000 }); expect(csvFile.length).to.be(4826973); expectSnapshot(csvFile.slice(0, 5000)).toMatch(); expectSnapshot(csvFile.slice(-5000)).toMatch(); diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 4634b40c74518..2df784cd0c8ba 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -175,5 +175,6 @@ "@kbn/esql-utils", "@kbn/search-types", "@kbn/analytics-ftr-helpers-plugin", + "@kbn/reporting-server", ] } diff --git a/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts b/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts index 89e5f5a0de4a7..0450ec2a3ff30 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/reporting/generate_csv_discover.ts @@ -730,7 +730,9 @@ export default function ({ getService }: FtrProviderContext) { }, }) ); - await reportingAPI.waitForJobToFinish(res.path); + await reportingAPI.waitForJobToFinish(res.path, undefined, undefined, { + timeout: 80 * 1000, + }); const csvFile = await reportingAPI.getCompletedJobOutput(res.path); expect((csvFile as string).length).to.be(4826973); expectSnapshot(createPartialCsv(csvFile)).toMatch(); diff --git a/x-pack/test_serverless/functional/test_suites/common/discover/x_pack/reporting.ts b/x-pack/test_serverless/functional/test_suites/common/discover/x_pack/reporting.ts index 3a4cee55dad94..b5eb49519ce31 100644 --- a/x-pack/test_serverless/functional/test_suites/common/discover/x_pack/reporting.ts +++ b/x-pack/test_serverless/functional/test_suites/common/discover/x_pack/reporting.ts @@ -35,14 +35,14 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await browser.refresh(); }; - const getReport = async () => { + const getReport = async ({ timeout } = { timeout: 60 * 1000 }) => { // close any open notification toasts await toasts.dismissAll(); await PageObjects.reporting.openExportTab(); await PageObjects.reporting.clickGenerateReportButton(); - const url = await PageObjects.reporting.getReportURL(60000); + const url = await PageObjects.reporting.getReportURL(timeout); // TODO: Fetch CSV client side in Serverless since `PageObjects.reporting.getResponse()` // doesn't work because it relies on `SecurityService.testUserSupertest` const res: { status: number; contentType: string | null; text: string } = @@ -184,7 +184,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await PageObjects.discover.saveSearch('large export'); // match file length, the beginning and the end of the csv file contents - const { text: csvFile } = await getReport(); + const { text: csvFile } = await getReport({ timeout: 80 * 1000 }); expect(csvFile.length).to.be(4826973); expectSnapshot(csvFile.slice(0, 5000)).toMatch(); expectSnapshot(csvFile.slice(-5000)).toMatch(); From ad59e7fdc4b63317b472e5394a0029c6c27076d7 Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 15 May 2024 09:42:34 -0700 Subject: [PATCH 15/16] fix contentstream jest tests --- .../server/lib/content_stream.test.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/content_stream.test.ts b/x-pack/plugins/reporting/server/lib/content_stream.test.ts index d4f179c5c9359..165c06baa579b 100644 --- a/x-pack/plugins/reporting/server/lib/content_stream.test.ts +++ b/x-pack/plugins/reporting/server/lib/content_stream.test.ts @@ -122,12 +122,12 @@ describe('ContentStream', () => { 'body.query.constant_score.filter.bool.must.0.term._id', 'something' ); - expect(request2).toHaveProperty('index', 'somewhere'); + expect(request2).toHaveProperty('index', '.reporting-*,.kibana-reporting*'); expect(request2).toHaveProperty( 'body.query.constant_score.filter.bool.must.0.term.parent_id', 'something' ); - expect(request3).toHaveProperty('index', 'somewhere'); + expect(request3).toHaveProperty('index', '.reporting-*,.kibana-reporting*'); expect(request3).toHaveProperty( 'body.query.constant_score.filter.bool.must.0.term.parent_id', 'something' @@ -293,8 +293,11 @@ describe('ContentStream', () => { 1, expect.objectContaining({ id: expect.any(String), - index: 'somewhere', + index: '.kibana-reporting', + op_type: 'create', + refresh: 'wait_for', body: { + '@timestamp': '1970-01-01T00:00:00.000Z', parent_id: 'something', output: { content: '34', @@ -307,8 +310,11 @@ describe('ContentStream', () => { 2, expect.objectContaining({ id: expect.any(String), - index: 'somewhere', + index: '.kibana-reporting', + op_type: 'create', + refresh: 'wait_for', body: { + '@timestamp': '1970-01-01T00:00:00.000Z', parent_id: 'something', output: { content: '56', @@ -335,9 +341,12 @@ describe('ContentStream', () => { 1, expect.objectContaining({ id: expect.any(String), - index: 'somewhere', + index: '.kibana-reporting', + op_type: 'create', + refresh: 'wait_for', body: { parent_id: 'something', + '@timestamp': '1970-01-01T00:00:00.000Z', output: { content: Buffer.from('456').toString('base64'), chunk: 1, @@ -349,9 +358,12 @@ describe('ContentStream', () => { 2, expect.objectContaining({ id: expect.any(String), - index: 'somewhere', + index: '.kibana-reporting', + op_type: 'create', + refresh: 'wait_for', body: { parent_id: 'something', + '@timestamp': '1970-01-01T00:00:00.000Z', output: { content: Buffer.from('78').toString('base64'), chunk: 2, From 7204d46937ab35b40ec39595d42a4286d2b838da Mon Sep 17 00:00:00 2001 From: Timothy Sullivan Date: Wed, 15 May 2024 10:02:17 -0700 Subject: [PATCH 16/16] add integration test for datastream --- .../reporting_and_security/datastream.ts | 67 +++++++++++++++ .../reporting_and_security/index.ts | 1 + .../common/reporting/datastream.ts | 84 +++++++++++++++++++ .../test_suites/common/reporting/index.ts | 1 + 4 files changed, 153 insertions(+) create mode 100644 x-pack/test/reporting_api_integration/reporting_and_security/datastream.ts create mode 100644 x-pack/test_serverless/api_integration/test_suites/common/reporting/datastream.ts diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/datastream.ts b/x-pack/test/reporting_api_integration/reporting_and_security/datastream.ts new file mode 100644 index 0000000000000..f116110db78f1 --- /dev/null +++ b/x-pack/test/reporting_api_integration/reporting_and_security/datastream.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from 'expect'; +import { FtrProviderContext } from '../ftr_provider_context'; + +// eslint-disable-next-line import/no-default-export +export default function ({ getService }: FtrProviderContext) { + const reportingAPI = getService('reportingAPI'); + const supertest = getService('supertest'); + + describe('Data Stream', () => { + before(async () => { + await reportingAPI.initEcommerce(); + + // for this test, we don't need to wait for the job to finish or verify the result + await reportingAPI.postJob( + `/api/reporting/generate/csv_searchsource?jobParams=%28browserTimezone%3AUTC%2Ccolumns%3A%21%28%29%2CobjectType%3Asearch%2CsearchSource%3A%28fields%3A%21%28%28field%3A%27%2A%27%2Cinclude_unmapped%3Atrue%29%29%2Cfilter%3A%21%28%28meta%3A%28field%3A%27%40timestamp%27%2Cindex%3A%27logstash-%2A%27%2Cparams%3A%28%29%29%2Cquery%3A%28range%3A%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Cgte%3A%272015-09-22T09%3A17%3A53.728Z%27%2Clte%3A%272015-09-22T09%3A30%3A50.786Z%27%29%29%29%29%2C%28%27%24state%27%3A%28store%3AappState%29%2Cmeta%3A%28alias%3A%21n%2Cdisabled%3A%21f%2Cindex%3A%27logstash-%2A%27%2Ckey%3Aquery%2Cnegate%3A%21f%2Ctype%3Acustom%2Cvalue%3A%27%7B%22bool%22%3A%7B%22minimum_should_match%22%3A1%2C%22should%22%3A%5B%7B%22match_phrase%22%3A%7B%22%40tags%22%3A%22info%22%7D%7D%5D%7D%7D%27%29%2Cquery%3A%28bool%3A%28minimum_should_match%3A1%2Cshould%3A%21%28%28match_phrase%3A%28%27%40tags%27%3Ainfo%29%29%29%29%29%29%29%2Cindex%3A%27logstash-%2A%27%2Cquery%3A%28language%3Akuery%2Cquery%3A%27%27%29%2Csort%3A%21%28%28%27%40timestamp%27%3A%28format%3Astrict_date_optional_time%2Corder%3Adesc%29%29%29%29%2Ctitle%3A%27A%20saved%20search%20with%20match_phrase%20filter%20and%20no%20columns%20selected%27%2Cversion%3A%278.15.0%27%29` + ); + }); + + after(async () => { + await reportingAPI.deleteAllReports(); + await reportingAPI.teardownEcommerce(); + }); + + it('uses the datastream configuration without set policy', async () => { + const { body } = await supertest + .get(`/api/index_management/data_streams/.kibana-reporting`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(body).toEqual({ + _meta: { + description: 'default kibana reporting template installed by elasticsearch', + managed: true, + }, + name: '.kibana-reporting', + indexTemplateName: '.kibana-reporting', + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: expect.any(String), + uuid: expect.any(String), + managedBy: 'Index Lifecycle Management', + preferILM: true, + }, + ], + generation: 1, + health: 'green', + ilmPolicyName: 'kibana-reporting', + maxTimeStamp: 0, + privileges: { delete_index: true, manage_data_stream_lifecycle: true }, + hidden: true, + lifecycle: { enabled: true }, + nextGenerationManagedBy: 'Index Lifecycle Management', + storageSize: expect.any(String), + storageSizeBytes: expect.any(Number), + }); + }); + }); +} diff --git a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts index 2d968295f09be..14c33bced7601 100644 --- a/x-pack/test/reporting_api_integration/reporting_and_security/index.ts +++ b/x-pack/test/reporting_api_integration/reporting_and_security/index.ts @@ -22,6 +22,7 @@ export default function ({ getService, loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./bwc_existing_indexes')); loadTestFile(require.resolve('./security_roles_privileges')); loadTestFile(require.resolve('./generate_csv_discover')); + loadTestFile(require.resolve('./datastream')); loadTestFile(require.resolve('./csv_v2')); loadTestFile(require.resolve('./csv_v2_esql')); loadTestFile(require.resolve('./network_policy')); diff --git a/x-pack/test_serverless/api_integration/test_suites/common/reporting/datastream.ts b/x-pack/test_serverless/api_integration/test_suites/common/reporting/datastream.ts new file mode 100644 index 0000000000000..1d336ec504250 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/reporting/datastream.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { expect } from 'expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const reportingAPI = getService('svlReportingApi'); + const supertest = getService('supertest'); + + const archives: Record = { + ecommerce: { + data: 'x-pack/test/functional/es_archives/reporting/ecommerce', + savedObjects: 'x-pack/test/functional/fixtures/kbn_archiver/reporting/ecommerce', + }, + }; + + describe('Data Stream', () => { + before(async () => { + await esArchiver.load(archives.ecommerce.data); + await kibanaServer.importExport.load(archives.ecommerce.savedObjects); + + // for this test, we don't need to wait for the job to finish or verify the result + await reportingAPI.createReportJobInternal('csv_searchsource', { + browserTimezone: 'UTC', + objectType: 'search', + searchSource: { + index: '5193f870-d861-11e9-a311-0fa548c5f953', + query: { language: 'kuery', query: '' }, + version: true, + }, + title: 'Ecommerce Data', + version: '8.15.0', + }); + }); + + after(async () => { + await reportingAPI.deleteAllReports(); + await esArchiver.unload(archives.ecommerce.data); + await kibanaServer.importExport.unload(archives.ecommerce.savedObjects); + }); + + it('uses the datastream configuration with set ILM policy', async () => { + const { body } = await supertest + .get(`/api/index_management/data_streams/.kibana-reporting`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(body).toEqual({ + _meta: { + description: 'default kibana reporting template installed by elasticsearch', + managed: true, + }, + name: '.kibana-reporting', + indexTemplateName: '.kibana-reporting', + generation: 1, + health: 'green', + hidden: true, + indices: [ + { + name: expect.any(String), + uuid: expect.any(String), + managedBy: 'Data stream lifecycle', + preferILM: true, + }, + ], + lifecycle: { enabled: true }, + maxTimeStamp: 0, + nextGenerationManagedBy: 'Data stream lifecycle', + privileges: { delete_index: true, manage_data_stream_lifecycle: true }, + timeStampField: { name: '@timestamp' }, + storageSize: expect.any(String), + storageSizeBytes: expect.any(Number), + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/reporting/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/reporting/index.ts index 3d776ec490abd..790a98257552e 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/reporting/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/reporting/index.ts @@ -13,5 +13,6 @@ export default ({ loadTestFile }: FtrProviderContext) => { loadTestFile(require.resolve('./management')); loadTestFile(require.resolve('./generate_csv_discover')); + loadTestFile(require.resolve('./datastream')); }); };