From 0f4538195f5494f3292b8021a8726ff31df81cd4 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Tue, 20 Apr 2021 18:02:27 +0300 Subject: [PATCH] [Usage collection] Collect non-default kibana configs (#97368) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .telemetryrc.json | 4 +- ...-plugin-core-server.makeusagefromschema.md | 15 + .../core/server/kibana-plugin-core-server.md | 1 + ...er.pluginconfigdescriptor.exposetousage.md | 17 + ...ugin-core-server.pluginconfigdescriptor.md | 1 + .../core_usage_data_service.mock.ts | 1 + .../core_usage_data_service.test.ts | 478 +++++++++++++++++- .../core_usage_data_service.ts | 123 ++++- src/core/server/core_usage_data/index.ts | 2 +- src/core/server/core_usage_data/types.ts | 13 + src/core/server/index.ts | 3 + .../server/plugins/plugins_service.mock.ts | 1 + .../server/plugins/plugins_service.test.ts | 99 +++- src/core/server/plugins/plugins_service.ts | 13 +- src/core/server/plugins/types.ts | 33 ++ src/core/server/server.api.md | 21 +- src/core/server/server.ts | 1 + src/plugins/kibana_usage_collection/README.md | 6 +- .../server/collectors/config_usage/README.md | 64 +++ .../server/collectors/config_usage/index.ts | 9 + .../register_config_usage_collector.test.ts | 44 ++ .../register_config_usage_collector.ts | 39 ++ ...x.test.ts => core_usage_collector.test.ts} | 6 +- .../server/collectors/index.ts | 1 + .../server/plugin.test.ts | 4 + .../kibana_usage_collection/server/plugin.ts | 2 + src/plugins/telemetry/schema/oss_root.json | 4 +- src/plugins/usage_collection/server/config.ts | 5 + .../apis/telemetry/telemetry_local.ts | 30 ++ .../utils/schema_to_config_schema.ts | 12 +- .../apis/telemetry/telemetry_local.ts | 1 + 31 files changed, 1026 insertions(+), 27 deletions(-) create mode 100644 docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md create mode 100644 docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts create mode 100644 src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts rename src/plugins/kibana_usage_collection/server/collectors/core/{index.test.ts => core_usage_collector.test.ts} (89%) diff --git a/.telemetryrc.json b/.telemetryrc.json index a408a5e2842f9..3b404f98af5cc 100644 --- a/.telemetryrc.json +++ b/.telemetryrc.json @@ -2,6 +2,8 @@ { "output": "src/plugins/telemetry/schema/oss_plugins.json", "root": "src/plugins/", - "exclude": [] + "exclude": [ + "src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts" + ] } ] diff --git a/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md new file mode 100644 index 0000000000000..f47d01a2d09e8 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.makeusagefromschema.md @@ -0,0 +1,15 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) + +## MakeUsageFromSchema type + +List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to `true` then the actual value of these configs will be reoprted. If parent node or actual config path is set to `false` then the config will be reported as \[redacted\]. + +Signature: + +```typescript +export declare type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends Maybe ? false : T[Key] extends Maybe ? boolean : T[Key] extends Maybe ? MakeUsageFromSchema | boolean : boolean; +}; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.md b/docs/development/core/server/kibana-plugin-core-server.md index 3bbdf8c703ab1..e33e9472d42a9 100644 --- a/docs/development/core/server/kibana-plugin-core-server.md +++ b/docs/development/core/server/kibana-plugin-core-server.md @@ -272,6 +272,7 @@ The plugin integrates with the core system via lifecycle events: `setup` | [LegacyElasticsearchClientConfig](./kibana-plugin-core-server.legacyelasticsearchclientconfig.md) | | | [LifecycleResponseFactory](./kibana-plugin-core-server.lifecycleresponsefactory.md) | Creates an object containing redirection or error response with error details, HTTP headers, and other data transmitted to the client. | | [LoggerConfigType](./kibana-plugin-core-server.loggerconfigtype.md) | | +| [MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | List of configuration values that will be exposed to usage collection. If parent node or actual config path is set to true then the actual value of these configs will be reoprted. If parent node or actual config path is set to false then the config will be reported as \[redacted\]. | | [MetricsServiceStart](./kibana-plugin-core-server.metricsservicestart.md) | APIs to retrieves metrics gathered and exposed by the core platform. | | [MIGRATION\_ASSISTANCE\_INDEX\_ACTION](./kibana-plugin-core-server.migration_assistance_index_action.md) | | | [MIGRATION\_DEPRECATION\_LEVEL](./kibana-plugin-core-server.migration_deprecation_level.md) | | diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md new file mode 100644 index 0000000000000..8c50c2e339426 --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md @@ -0,0 +1,17 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginConfigDescriptor](./kibana-plugin-core-server.pluginconfigdescriptor.md) > [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) + +## PluginConfigDescriptor.exposeToUsage property + +Expose non-default configs to usage collection to be sent via telemetry. set a config to `true` to report the actual changed config value. set a config to `false` to report the changed config value as \[redacted\]. + +All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified. + +[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) + +Signature: + +```typescript +exposeToUsage?: MakeUsageFromSchema; +``` diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md index 5708c4f9a3f88..80e807a1361fd 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginconfigdescriptor.md @@ -46,5 +46,6 @@ export const config: PluginConfigDescriptor = { | --- | --- | --- | | [deprecations](./kibana-plugin-core-server.pluginconfigdescriptor.deprecations.md) | ConfigDeprecationProvider | Provider for the to apply to the plugin configuration. | | [exposeToBrowser](./kibana-plugin-core-server.pluginconfigdescriptor.exposetobrowser.md) | {
[P in keyof T]?: boolean;
} | List of configuration properties that will be available on the client-side plugin. | +| [exposeToUsage](./kibana-plugin-core-server.pluginconfigdescriptor.exposetousage.md) | MakeUsageFromSchema<T> | Expose non-default configs to usage collection to be sent via telemetry. set a config to true to report the actual changed config value. set a config to false to report the changed config value as \[redacted\].All changed configs except booleans and numbers will be reported as \[redacted\] unless otherwise specified.[MakeUsageFromSchema](./kibana-plugin-core-server.makeusagefromschema.md) | | [schema](./kibana-plugin-core-server.pluginconfigdescriptor.schema.md) | PluginConfigSchema<T> | Schema to use to validate the plugin configuration.[PluginConfigSchema](./kibana-plugin-core-server.pluginconfigschema.md) | diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts index 7fb15a921a413..e09f595747c30 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.mock.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.mock.ts @@ -139,6 +139,7 @@ const createStartContractMock = () => { }, }) ), + getConfigsUsageData: jest.fn(), }; return startContract; diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts index d1f047c129efe..dc74b65c8dcfc 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.test.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.test.ts @@ -35,7 +35,35 @@ describe('CoreUsageDataService', () => { }); let service: CoreUsageDataService; - const configService = configServiceMock.create(); + const mockConfig = { + unused_config: {}, + elasticsearch: { username: 'kibana_system', password: 'changeme' }, + plugins: { paths: ['pluginA', 'pluginAB', 'pluginB'] }, + server: { port: 5603, basePath: '/zvt', rewriteBasePath: true }, + logging: { json: false }, + pluginA: { + enabled: true, + objectConfig: { + debug: true, + username: 'some_user', + }, + arrayOfNumbers: [1, 2, 3], + }, + pluginAB: { + enabled: false, + }, + pluginB: { + arrayOfObjects: [ + { propA: 'a', propB: 'b' }, + { propA: 'a2', propB: 'b2' }, + ], + }, + }; + + const configService = configServiceMock.create({ + getConfig$: mockConfig, + }); + configService.atPath.mockImplementation((path) => { if (path === 'elasticsearch') { return new BehaviorSubject(RawElasticsearchConfig.schema.validate({})); @@ -146,6 +174,7 @@ describe('CoreUsageDataService', () => { const { getCoreUsageData } = service.start({ savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage: new Map(), elasticsearch, }); expect(getCoreUsageData()).resolves.toMatchInlineSnapshot(` @@ -281,6 +310,453 @@ describe('CoreUsageDataService', () => { `); }); }); + + describe('getConfigsUsageData', () => { + const elasticsearch = elasticsearchServiceMock.createStart(); + const typeRegistry = savedObjectsServiceMock.createTypeRegistryMock(); + let exposedConfigsToUsage: Map>; + beforeEach(() => { + exposedConfigsToUsage = new Map(); + }); + + it('loops over all used configs once each', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'logging.json', + ]); + + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, + }); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + const mockGetMarkedAsSafe = jest.fn().mockReturnValue({}); + // @ts-expect-error + service.getMarkedAsSafe = mockGetMarkedAsSafe; + await getConfigsUsageData(); + + expect(mockGetMarkedAsSafe).toBeCalledTimes(2); + expect(mockGetMarkedAsSafe.mock.calls).toMatchInlineSnapshot(` + Array [ + Array [ + Map { + "pluginA" => Object { + "objectConfig": true, + }, + }, + "pluginA.objectConfig.debug", + "pluginA", + ], + Array [ + Map { + "pluginA" => Object { + "objectConfig": true, + }, + }, + "logging.json", + undefined, + ], + ] + `); + }); + + it('plucks pluginId from config path correctly', async () => { + exposedConfigsToUsage.set('pluginA', { + enabled: false, + }); + exposedConfigsToUsage.set('pluginAB', { + enabled: false, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginA.enabled', 'pluginAB.enabled']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.enabled": "[redacted]", + "pluginAB.enabled": "[redacted]", + } + `); + }); + + it('returns an object of plugin config usage', async () => { + exposedConfigsToUsage.set('unused_config', { never_reported: true }); + exposedConfigsToUsage.set('server', { basePath: true }); + exposedConfigsToUsage.set('pluginA', { elasticsearch: false }); + exposedConfigsToUsage.set('plugins', { paths: false }); + exposedConfigsToUsage.set('pluginA', { arrayOfNumbers: false }); + + configService.getUsedPaths.mockResolvedValue([ + 'elasticsearch.username', + 'elasticsearch.password', + 'plugins.paths', + 'server.port', + 'server.basePath', + 'server.rewriteBasePath', + 'logging.json', + 'pluginA.enabled', + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', + 'pluginA.arrayOfNumbers', + 'pluginAB.enabled', + 'pluginB.arrayOfObjects', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "elasticsearch.password": "[redacted]", + "elasticsearch.username": "[redacted]", + "logging.json": false, + "pluginA.arrayOfNumbers": "[redacted]", + "pluginA.enabled": true, + "pluginA.objectConfig.debug": true, + "pluginA.objectConfig.username": "[redacted]", + "pluginAB.enabled": false, + "pluginB.arrayOfObjects": "[redacted]", + "plugins.paths": "[redacted]", + "server.basePath": "/zvt", + "server.port": 5603, + "server.rewriteBasePath": true, + } + `); + }); + + describe('config explicitly exposed to usage', () => { + it('returns [redacted] on unsafe complete match', async () => { + exposedConfigsToUsage.set('pluginA', { + 'objectConfig.debug': false, + }); + exposedConfigsToUsage.set('server', { + basePath: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'server.basePath', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.objectConfig.debug": "[redacted]", + "server.basePath": "[redacted]", + } + `); + }); + + it('returns config value on safe complete match', async () => { + exposedConfigsToUsage.set('server', { + basePath: true, + }); + + configService.getUsedPaths.mockResolvedValue(['server.basePath']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "server.basePath": "/zvt", + } + `); + }); + + it('returns [redacted] on unsafe parent match', async () => { + exposedConfigsToUsage.set('pluginA', { + objectConfig: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.objectConfig.debug": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", + } + `); + }); + + it('returns config value on safe parent match', async () => { + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.objectConfig.debug": true, + "pluginA.objectConfig.username": "some_user", + } + `); + }); + + it('returns [redacted] on explicitly marked as safe array of objects', async () => { + exposedConfigsToUsage.set('pluginB', { + arrayOfObjects: true, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginB.arrayOfObjects": "[redacted]", + } + `); + }); + + it('returns values on explicitly marked as safe array of numbers', async () => { + exposedConfigsToUsage.set('pluginA', { + arrayOfNumbers: true, + }); + + configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.arrayOfNumbers": Array [ + 1, + 2, + 3, + ], + } + `); + }); + + it('returns values on explicitly marked as safe array of strings', async () => { + exposedConfigsToUsage.set('plugins', { + paths: true, + }); + + configService.getUsedPaths.mockResolvedValue(['plugins.paths']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "plugins.paths": Array [ + "pluginA", + "pluginAB", + "pluginB", + ], + } + `); + }); + }); + + describe('config not explicitly exposed to usage', () => { + it('returns [redacted] for string configs', async () => { + exposedConfigsToUsage.set('pluginA', { + objectConfig: false, + }); + + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'pluginA.objectConfig.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.objectConfig.debug": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", + } + `); + }); + + it('returns config value on safe parent match', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'elasticsearch.password', + 'elasticsearch.username', + 'pluginA.objectConfig.username', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "elasticsearch.password": "[redacted]", + "elasticsearch.username": "[redacted]", + "pluginA.objectConfig.username": "[redacted]", + } + `); + }); + + it('returns [redacted] on implicit array of objects', async () => { + configService.getUsedPaths.mockResolvedValue(['pluginB.arrayOfObjects']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginB.arrayOfObjects": "[redacted]", + } + `); + }); + + it('returns values on implicit array of numbers', async () => { + configService.getUsedPaths.mockResolvedValue(['pluginA.arrayOfNumbers']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "pluginA.arrayOfNumbers": Array [ + 1, + 2, + 3, + ], + } + `); + }); + it('returns [redacted] on implicit array of strings', async () => { + configService.getUsedPaths.mockResolvedValue(['plugins.paths']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "plugins.paths": "[redacted]", + } + `); + }); + + it('returns config value for numbers', async () => { + configService.getUsedPaths.mockResolvedValue(['server.port']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "server.port": 5603, + } + `); + }); + + it('returns config value for booleans', async () => { + configService.getUsedPaths.mockResolvedValue([ + 'pluginA.objectConfig.debug', + 'logging.json', + ]); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "logging.json": false, + "pluginA.objectConfig.debug": true, + } + `); + }); + + it('ignores exposed to usage configs but not used', async () => { + exposedConfigsToUsage.set('pluginA', { + objectConfig: true, + }); + + configService.getUsedPaths.mockResolvedValue(['logging.json']); + + const { getConfigsUsageData } = service.start({ + savedObjects: savedObjectsServiceMock.createInternalStartContract(typeRegistry), + exposedConfigsToUsage, + elasticsearch, + }); + + await expect(getConfigsUsageData()).resolves.toMatchInlineSnapshot(` + Object { + "logging.json": false, + } + `); + }); + }); + }); }); describe('setup and stop', () => { diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts index 78ac977c31a7d..85abdca9ea5dc 100644 --- a/src/core/server/core_usage_data/core_usage_data_service.ts +++ b/src/core/server/core_usage_data/core_usage_data_service.ts @@ -7,7 +7,9 @@ */ import { Subject } from 'rxjs'; -import { takeUntil } from 'rxjs/operators'; +import { takeUntil, first } from 'rxjs/operators'; +import { get } from 'lodash'; +import { hasConfigPathIntersection } from '@kbn/config'; import { CoreService } from 'src/core/types'; import { Logger, SavedObjectsServiceStart, SavedObjectTypeRegistry } from 'src/core/server'; @@ -16,11 +18,12 @@ import { ElasticsearchConfigType } from '../elasticsearch/elasticsearch_config'; import { HttpConfigType, InternalHttpServiceSetup } from '../http'; import { LoggingConfigType } from '../logging'; import { SavedObjectsConfigType } from '../saved_objects/saved_objects_config'; -import { +import type { CoreServicesUsageData, CoreUsageData, CoreUsageDataStart, CoreUsageDataSetup, + ConfigUsageData, } from './types'; import { isConfigured } from './is_configured'; import { ElasticsearchServiceStart } from '../elasticsearch'; @@ -30,6 +33,8 @@ import { CORE_USAGE_STATS_TYPE } from './constants'; import { CoreUsageStatsClient } from './core_usage_stats_client'; import { MetricsServiceSetup, OpsMetrics } from '..'; +export type ExposedConfigsToUsage = Map>; + export interface SetupDeps { http: InternalHttpServiceSetup; metrics: MetricsServiceSetup; @@ -39,6 +44,7 @@ export interface SetupDeps { export interface StartDeps { savedObjects: SavedObjectsServiceStart; elasticsearch: ElasticsearchServiceStart; + exposedConfigsToUsage: ExposedConfigsToUsage; } /** @@ -266,6 +272,110 @@ export class CoreUsageDataService implements CoreService { + const fullPath = `${pluginId}.${exposeKey}`; + return hasConfigPathIntersection(usedPath, fullPath); + }); + + if (exposeKeyDetails) { + const explicitlyMarkedAsSafe = exposeDetails[exposeKeyDetails]; + + if (typeof explicitlyMarkedAsSafe === 'boolean') { + return { + explicitlyMarked: true, + isSafe: explicitlyMarkedAsSafe, + }; + } + } + } + + return { explicitlyMarked: false, isSafe: false }; + } + + private async getNonDefaultKibanaConfigs( + exposedConfigsToUsage: ExposedConfigsToUsage + ): Promise { + const config = await this.configService.getConfig$().pipe(first()).toPromise(); + const nonDefaultConfigs = config.toRaw(); + const usedPaths = await this.configService.getUsedPaths(); + const exposedConfigsKeys = [...exposedConfigsToUsage.keys()]; + + return usedPaths.reduce((acc, usedPath) => { + const rawConfigValue = get(nonDefaultConfigs, usedPath); + const pluginId = exposedConfigsKeys.find( + (exposedConfigsKey) => + usedPath === exposedConfigsKey || usedPath.startsWith(`${exposedConfigsKey}.`) + ); + + const { explicitlyMarked, isSafe } = this.getMarkedAsSafe( + exposedConfigsToUsage, + usedPath, + pluginId + ); + + // explicitly marked as safe + if (explicitlyMarked && isSafe) { + // report array of objects as redacted even if explicitly marked as safe. + // TS typings prevent explicitly marking arrays of objects as safe + // this makes sure to report redacted even if TS was bypassed. + if ( + Array.isArray(rawConfigValue) && + rawConfigValue.some((item) => typeof item === 'object') + ) { + acc[usedPath] = '[redacted]'; + } else { + acc[usedPath] = rawConfigValue; + } + } + + // explicitly marked as unsafe + if (explicitlyMarked && !isSafe) { + acc[usedPath] = '[redacted]'; + } + + /** + * not all types of values may contain sensitive values. + * Report boolean and number configs if not explicitly marked as unsafe. + */ + if (!explicitlyMarked) { + switch (typeof rawConfigValue) { + case 'number': + case 'boolean': + acc[usedPath] = rawConfigValue; + break; + case 'undefined': + acc[usedPath] = 'undefined'; + break; + case 'object': { + // non-array object types are already handled + if (Array.isArray(rawConfigValue)) { + if ( + rawConfigValue.every( + (item) => typeof item === 'number' || typeof item === 'boolean' + ) + ) { + acc[usedPath] = rawConfigValue; + break; + } + } + } + default: { + acc[usedPath] = '[redacted]'; + } + } + } + + return acc; + }, {} as Record); + } + setup({ http, metrics, savedObjectsStartPromise }: SetupDeps) { metrics .getOpsMetrics$() @@ -326,10 +436,13 @@ export class CoreUsageDataService implements CoreService { - return this.getCoreUsageData(savedObjects, elasticsearch); + getCoreUsageData: async () => { + return await this.getCoreUsageData(savedObjects, elasticsearch); + }, + getConfigsUsageData: async () => { + return await this.getNonDefaultKibanaConfigs(exposedConfigsToUsage); }, }; } diff --git a/src/core/server/core_usage_data/index.ts b/src/core/server/core_usage_data/index.ts index 4e0200ed1e4ea..638fc65522433 100644 --- a/src/core/server/core_usage_data/index.ts +++ b/src/core/server/core_usage_data/index.ts @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -export type { CoreUsageDataSetup, CoreUsageDataStart } from './types'; +export type { CoreUsageDataSetup, ConfigUsageData, CoreUsageDataStart } from './types'; export { CoreUsageDataService } from './core_usage_data_service'; export { CoreUsageStatsClient } from './core_usage_stats_client'; diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts index b29cf41da6826..1d5ef6d893f53 100644 --- a/src/core/server/core_usage_data/types.ts +++ b/src/core/server/core_usage_data/types.ts @@ -122,6 +122,18 @@ export interface CoreUsageData extends CoreUsageStats { environment: CoreEnvironmentUsageData; } +/** + * Type describing Core's usage data payload + * @internal + */ +export type ConfigUsageData = Record; + +/** + * Type describing Core's usage data payload + * @internal + */ +export type ExposedConfigsToUsage = Map>; + /** * Usage data from Core services * @internal @@ -270,4 +282,5 @@ export interface CoreUsageDataStart { * @internal * */ getCoreUsageData(): Promise; + getConfigsUsageData(): Promise; } diff --git a/src/core/server/index.ts b/src/core/server/index.ts index 2c6fa74cb54a0..6b7fa994e6a97 100644 --- a/src/core/server/index.ts +++ b/src/core/server/index.ts @@ -64,6 +64,7 @@ import { CoreUsageStats, CoreUsageData, CoreConfigUsageData, + ConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData, } from './core_usage_data'; @@ -74,6 +75,7 @@ export type { CoreConfigUsageData, CoreEnvironmentUsageData, CoreServicesUsageData, + ConfigUsageData, }; export { bootstrap } from './bootstrap'; @@ -256,6 +258,7 @@ export type { PluginManifest, PluginName, SharedGlobalConfig, + MakeUsageFromSchema, } from './plugins'; export { diff --git a/src/core/server/plugins/plugins_service.mock.ts b/src/core/server/plugins/plugins_service.mock.ts index 1d0ed7cb09299..f4f2263a1bdb0 100644 --- a/src/core/server/plugins/plugins_service.mock.ts +++ b/src/core/server/plugins/plugins_service.mock.ts @@ -19,6 +19,7 @@ const createStartContractMock = () => ({ contracts: new Map() }); const createServiceMock = (): PluginsServiceMock => ({ discover: jest.fn(), + getExposedPluginConfigsToUsage: jest.fn(), setup: jest.fn().mockResolvedValue(createSetupContractMock()), start: jest.fn().mockResolvedValue(createStartContractMock()), stop: jest.fn(), diff --git a/src/core/server/plugins/plugins_service.test.ts b/src/core/server/plugins/plugins_service.test.ts index 6bf7a1fadb4d3..5c50df07dc697 100644 --- a/src/core/server/plugins/plugins_service.test.ts +++ b/src/core/server/plugins/plugins_service.test.ts @@ -78,7 +78,7 @@ const createPlugin = ( manifest: { id, version, - configPath: `${configPath}${disabled ? '-disabled' : ''}`, + configPath: disabled ? configPath.concat('-disabled') : configPath, kibanaVersion, requiredPlugins, requiredBundles, @@ -374,7 +374,6 @@ describe('PluginsService', () => { expect(mockPluginSystem.addPlugin).toHaveBeenCalledTimes(2); expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(firstPlugin); expect(mockPluginSystem.addPlugin).toHaveBeenCalledWith(secondPlugin); - expect(mockDiscover).toHaveBeenCalledTimes(1); expect(mockDiscover).toHaveBeenCalledWith( { @@ -472,6 +471,88 @@ describe('PluginsService', () => { expect(pluginPaths).toEqual(['/plugin-A-path', '/plugin-B-path']); }); + + it('ppopulates pluginConfigUsageDescriptors with plugins exposeToUsage property', async () => { + const pluginA = createPlugin('plugin-with-expose-usage', { + path: 'plugin-with-expose-usage', + configPath: 'pathA', + }); + + jest.doMock( + join('plugin-with-expose-usage', 'server'), + () => ({ + config: { + exposeToUsage: { + test: true, + nested: { + prop: true, + }, + }, + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + const pluginB = createPlugin('plugin-with-array-configPath', { + path: 'plugin-with-array-configPath', + configPath: ['plugin', 'pathB'], + }); + + jest.doMock( + join('plugin-with-array-configPath', 'server'), + () => ({ + config: { + exposeToUsage: { + test: true, + }, + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + jest.doMock( + join('plugin-without-expose', 'server'), + () => ({ + config: { + schema: schema.maybe(schema.any()), + }, + }), + { + virtual: true, + } + ); + + const pluginC = createPlugin('plugin-without-expose', { + path: 'plugin-without-expose', + configPath: 'pathC', + }); + + mockDiscover.mockReturnValue({ + error$: from([]), + plugin$: from([pluginA, pluginB, pluginC]), + }); + + await pluginsService.discover({ environment: environmentSetup }); + + // eslint-disable-next-line dot-notation + expect(pluginsService['pluginConfigUsageDescriptors']).toMatchInlineSnapshot(` + Map { + "pathA" => Object { + "nested.prop": true, + "test": true, + }, + "plugin.pathB" => Object { + "test": true, + }, + } + `); + }); }); describe('#generateUiPluginsConfigs()', () => { @@ -624,6 +705,20 @@ describe('PluginsService', () => { }); }); + describe('#getExposedPluginConfigsToUsage', () => { + it('returns pluginConfigUsageDescriptors', () => { + // eslint-disable-next-line dot-notation + pluginsService['pluginConfigUsageDescriptors'].set('test', { enabled: true }); + expect(pluginsService.getExposedPluginConfigsToUsage()).toMatchInlineSnapshot(` + Map { + "test" => Object { + "enabled": true, + }, + } + `); + }); + }); + describe('#stop()', () => { it('`stop` stops plugins system', async () => { await pluginsService.stop(); diff --git a/src/core/server/plugins/plugins_service.ts b/src/core/server/plugins/plugins_service.ts index 09be40ecaf2a2..547fe00fdb1cf 100644 --- a/src/core/server/plugins/plugins_service.ts +++ b/src/core/server/plugins/plugins_service.ts @@ -9,7 +9,7 @@ import Path from 'path'; import { Observable } from 'rxjs'; import { filter, first, map, mergeMap, tap, toArray } from 'rxjs/operators'; -import { pick } from '@kbn/std'; +import { pick, getFlattenedObject } from '@kbn/std'; import { CoreService } from '../../types'; import { CoreContext } from '../core_context'; @@ -75,6 +75,7 @@ export class PluginsService implements CoreService; private readonly pluginConfigDescriptors = new Map(); private readonly uiPluginInternalInfo = new Map(); + private readonly pluginConfigUsageDescriptors = new Map>(); constructor(private readonly coreContext: CoreContext) { this.log = coreContext.logger.get('plugins-service'); @@ -109,6 +110,10 @@ export class PluginsService implements CoreService = T | undefined; + /** * Dedicated type for plugin configuration schema. * @@ -70,8 +72,39 @@ export interface PluginConfigDescriptor { * {@link PluginConfigSchema} */ schema: PluginConfigSchema; + /** + * Expose non-default configs to usage collection to be sent via telemetry. + * set a config to `true` to report the actual changed config value. + * set a config to `false` to report the changed config value as [redacted]. + * + * All changed configs except booleans and numbers will be reported + * as [redacted] unless otherwise specified. + * + * {@link MakeUsageFromSchema} + */ + exposeToUsage?: MakeUsageFromSchema; } +/** + * List of configuration values that will be exposed to usage collection. + * If parent node or actual config path is set to `true` then the actual value + * of these configs will be reoprted. + * If parent node or actual config path is set to `false` then the config + * will be reported as [redacted]. + * + * @public + */ +export type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends Maybe + ? // arrays of objects are always redacted + false + : T[Key] extends Maybe + ? boolean + : T[Key] extends Maybe + ? MakeUsageFromSchema | boolean + : boolean; +}; + /** * Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays * that use it as a key or value more obvious. diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md index e5804b3c9fc58..ccff20458f7e6 100644 --- a/src/core/server/server.api.md +++ b/src/core/server/server.api.md @@ -381,6 +381,9 @@ export { ConfigPath } export { ConfigService } +// @internal +export type ConfigUsageData = Record; + // @public export interface ContextSetup { createContextContainer(): IContextContainer; @@ -558,6 +561,8 @@ export interface CoreUsageData extends CoreUsageStats { // @internal export interface CoreUsageDataStart { + // (undocumented) + getConfigsUsageData(): Promise; getCoreUsageData(): Promise; } @@ -1662,6 +1667,13 @@ export { LogMeta } export { LogRecord } +// Warning: (ae-forgotten-export) The symbol "Maybe" needs to be exported by the entry point index.d.ts +// +// @public +export type MakeUsageFromSchema = { + [Key in keyof T]?: T[Key] extends Maybe ? false : T[Key] extends Maybe ? boolean : T[Key] extends Maybe ? MakeUsageFromSchema | boolean : boolean; +}; + // @public export interface MetricsServiceSetup { readonly collectionInterval: number; @@ -1848,6 +1860,7 @@ export interface PluginConfigDescriptor { exposeToBrowser?: { [P in keyof T]?: boolean; }; + exposeToUsage?: MakeUsageFromSchema; schema: PluginConfigSchema; } @@ -3234,9 +3247,9 @@ export const validBodyOutput: readonly ["data", "stream"]; // // src/core/server/elasticsearch/client/types.ts:94:7 - (ae-forgotten-export) The symbol "Explanation" needs to be exported by the entry point index.d.ts // src/core/server/http/router/response.ts:297:3 - (ae-forgotten-export) The symbol "KibanaResponse" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:293:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:296:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts -// src/core/server/plugins/types.ts:401:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" +// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "KibanaConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:326:3 - (ae-forgotten-export) The symbol "SharedGlobalConfigKeys" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:329:3 - (ae-forgotten-export) The symbol "SavedObjectsConfigType" needs to be exported by the entry point index.d.ts +// src/core/server/plugins/types.ts:434:5 - (ae-unresolved-link) The @link reference could not be resolved: The package "kibana" does not have an export "create" ``` diff --git a/src/core/server/server.ts b/src/core/server/server.ts index da2bcf220b718..fcfca3a5e0e2f 100644 --- a/src/core/server/server.ts +++ b/src/core/server/server.ts @@ -247,6 +247,7 @@ export class Server { const coreUsageDataStart = this.coreUsageData.start({ elasticsearch: elasticsearchStart, savedObjects: savedObjectsStart, + exposedConfigsToUsage: this.plugins.getExposedPluginConfigsToUsage(), }); this.coreStart = { diff --git a/src/plugins/kibana_usage_collection/README.md b/src/plugins/kibana_usage_collection/README.md index 9ad2bd987e1f4..9e9438b1b5fee 100644 --- a/src/plugins/kibana_usage_collection/README.md +++ b/src/plugins/kibana_usage_collection/README.md @@ -4,6 +4,7 @@ This plugin registers the basic usage collectors from Kibana: - [Application Usage](./server/collectors/application_usage/README.md) - Core Metrics +- [Config Usage](./server/collectors/config_usage/README.md) - CSP configuration - Kibana: Number of Saved Objects per type - Localization data @@ -11,8 +12,3 @@ This plugin registers the basic usage collectors from Kibana: - Ops stats - UI Counts - UI Metrics - - - - - diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md new file mode 100644 index 0000000000000..b476244e5082f --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/README.md @@ -0,0 +1,64 @@ +# Config Usage Collector + +The config usage collector reports non-default kibana configs. + +All non-default configs except booleans and numbers will be reported as `[redacted]` unless otherwise specified via `config.exposeToUsage` in the plugin config descriptor. + +```ts +import { schema, TypeOf } from '@kbn/config-schema'; +import { PluginConfigDescriptor } from 'src/core/server'; + +export const configSchema = schema.object({ + usageCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + retryCount: schema.number({ defaultValue: 1 }), + bufferDuration: schema.duration({ defaultValue: '5s' }), + }), + uiCounters: schema.object({ + enabled: schema.boolean({ defaultValue: true }), + debug: schema.boolean({ defaultValue: schema.contextRef('dev') }), + }), + maximumWaitTimeForAllCollectorsInS: schema.number({ + defaultValue: DEFAULT_MAXIMUM_WAIT_TIME_FOR_ALL_COLLECTORS_IN_S, + }), +}); + +export const config: PluginConfigDescriptor = { + schema: configSchema, + exposeToUsage: { + uiCounters: true, + usageCounters: { + bufferDuration: true, + }, + maximumWaitTimeForAllCollectorsInS: false, + }, +}; +``` + +In the above example setting `uiCounters: true` in the `exposeToUsage` property marks all configs +under the path `uiCounters` as safe. The collector will send the actual non-default config value +when setting an exact config or its parent path to `true`. + +Settings the config path or its parent path to `false` will explicitly mark this config as unsafe. +The collector will send `[redacted]` for non-default configs +when setting an exact config or its parent path to `false`. + +### Output of the collector + +```json +{ + "kibana_config_usage": { + "xpack.apm.serviceMapTraceIdBucketSize": 30, + "elasticsearch.username": "[redacted]", + "elasticsearch.password": "[redacted]", + "plugins.paths": "[redacted]", + "server.port": 5603, + "server.basePath": "[redacted]", + "server.rewriteBasePath": true, + "logging.json": false, + "usageCollection.uiCounters.debug": true + } +} +``` + +Note that arrays of objects will be reported as `[redacted]` and cannot be explicitly marked as safe. \ No newline at end of file diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts new file mode 100644 index 0000000000000..5d37cfe5957ab --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export { registerConfigUsageCollector } from './register_config_usage_collector'; diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts new file mode 100644 index 0000000000000..7d4f03fd30edf --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.test.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + Collector, + createUsageCollectionSetupMock, + createCollectorFetchContextMock, +} from '../../../../usage_collection/server/mocks'; +import { registerConfigUsageCollector } from './register_config_usage_collector'; +import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; +import type { ConfigUsageData } from '../../../../../core/server'; + +const logger = loggingSystemMock.createLogger(); + +describe('kibana_config_usage', () => { + let collector: Collector; + + const usageCollectionMock = createUsageCollectionSetupMock(); + usageCollectionMock.makeUsageCollector.mockImplementation((config) => { + collector = new Collector(logger, config); + return createUsageCollectionSetupMock().makeUsageCollector(config); + }); + + const collectorFetchContext = createCollectorFetchContextMock(); + const coreUsageDataStart = coreUsageDataServiceMock.createStartContract(); + const mockConfigUsage = (Symbol('config usage telemetry') as any) as ConfigUsageData; + coreUsageDataStart.getConfigsUsageData.mockResolvedValue(mockConfigUsage); + + beforeAll(() => registerConfigUsageCollector(usageCollectionMock, () => coreUsageDataStart)); + + test('registered collector is set', () => { + expect(collector).not.toBeUndefined(); + expect(collector.type).toBe('kibana_config_usage'); + }); + + test('fetch', async () => { + expect(await collector.fetch(collectorFetchContext)).toEqual(mockConfigUsage); + }); +}); diff --git a/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts new file mode 100644 index 0000000000000..ad7f570432abf --- /dev/null +++ b/src/plugins/kibana_usage_collection/server/collectors/config_usage/register_config_usage_collector.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { UsageCollectionSetup } from '../../../../usage_collection/server'; +import { ConfigUsageData, CoreUsageDataStart } from '../../../../../core/server'; + +export function registerConfigUsageCollector( + usageCollection: UsageCollectionSetup, + getCoreUsageDataService: () => CoreUsageDataStart +) { + const collector = usageCollection.makeUsageCollector({ + type: 'kibana_config_usage', + isReady: () => typeof getCoreUsageDataService() !== 'undefined', + /** + * No schema for this collector. + * This collector will collect non-default configs from all plugins. + * Mapping each config to the schema is inconvenient for developers + * and would result in 100's of extra field mappings. + * + * We'll experiment with flattened type and runtime fields before comitting to a schema. + */ + schema: {}, + fetch: async () => { + const coreUsageDataService = getCoreUsageDataService(); + if (!coreUsageDataService) { + return; + } + + return await coreUsageDataService.getConfigsUsageData(); + }, + }); + + usageCollection.registerCollector(collector); +} diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts similarity index 89% rename from src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts rename to src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts index cbc38129fdddf..b671a9f93d369 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/core/index.test.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.test.ts @@ -9,11 +9,11 @@ import { Collector, createUsageCollectionSetupMock, + createCollectorFetchContextMock, } from '../../../../usage_collection/server/mocks'; -import { createCollectorFetchContextMock } from 'src/plugins/usage_collection/server/mocks'; -import { registerCoreUsageCollector } from '.'; +import { registerCoreUsageCollector } from './core_usage_collector'; import { coreUsageDataServiceMock, loggingSystemMock } from '../../../../../core/server/mocks'; -import { CoreUsageData } from 'src/core/server/'; +import type { CoreUsageData } from '../../../../../core/server'; const logger = loggingSystemMock.createLogger(); diff --git a/src/plugins/kibana_usage_collection/server/collectors/index.ts b/src/plugins/kibana_usage_collection/server/collectors/index.ts index 522860e58918c..94ed0eefe7a06 100644 --- a/src/plugins/kibana_usage_collection/server/collectors/index.ts +++ b/src/plugins/kibana_usage_collection/server/collectors/index.ts @@ -15,6 +15,7 @@ export { registerCloudProviderUsageCollector } from './cloud'; export { registerCspCollector } from './csp'; export { registerCoreUsageCollector } from './core'; export { registerLocalizationUsageCollector } from './localization'; +export { registerConfigUsageCollector } from './config_usage'; export { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, diff --git a/src/plugins/kibana_usage_collection/server/plugin.test.ts b/src/plugins/kibana_usage_collection/server/plugin.test.ts index 86204ed30e656..450c610afc620 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.test.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.test.ts @@ -93,6 +93,10 @@ describe('kibana_usage_collection', () => { "isReady": false, "type": "core", }, + Object { + "isReady": false, + "type": "kibana_config_usage", + }, Object { "isReady": true, "type": "localization", diff --git a/src/plugins/kibana_usage_collection/server/plugin.ts b/src/plugins/kibana_usage_collection/server/plugin.ts index a27b8dff57b67..c144384e0882f 100644 --- a/src/plugins/kibana_usage_collection/server/plugin.ts +++ b/src/plugins/kibana_usage_collection/server/plugin.ts @@ -35,6 +35,7 @@ import { registerUiCountersUsageCollector, registerUiCounterSavedObjectType, registerUiCountersRollups, + registerConfigUsageCollector, registerUsageCountersRollups, registerUsageCountersUsageCollector, } from './collectors'; @@ -122,6 +123,7 @@ export class KibanaUsageCollectionPlugin implements Plugin { registerCloudProviderUsageCollector(usageCollection); registerCspCollector(usageCollection, coreSetup.http); registerCoreUsageCollector(usageCollection, getCoreUsageDataService); + registerConfigUsageCollector(usageCollection, getCoreUsageDataService); registerLocalizationUsageCollector(usageCollection, coreSetup.i18n); } } diff --git a/src/plugins/telemetry/schema/oss_root.json b/src/plugins/telemetry/schema/oss_root.json index 658f5ee4e66da..c4dd1096a6e98 100644 --- a/src/plugins/telemetry/schema/oss_root.json +++ b/src/plugins/telemetry/schema/oss_root.json @@ -183,8 +183,8 @@ }, "plugins": { "properties": { - "THIS_WILL_BE_REPLACED_BY_THE_PLUGINS_JSON": { - "type": "text" + "kibana_config_usage": { + "type": "pass_through" } } } diff --git a/src/plugins/usage_collection/server/config.ts b/src/plugins/usage_collection/server/config.ts index cd6f6b9d81396..faf8ce7535e8a 100644 --- a/src/plugins/usage_collection/server/config.ts +++ b/src/plugins/usage_collection/server/config.ts @@ -38,4 +38,9 @@ export const config: PluginConfigDescriptor = { exposeToBrowser: { uiCounters: true, }, + exposeToUsage: { + usageCounters: { + bufferDuration: true, + }, + }, }; diff --git a/test/api_integration/apis/telemetry/telemetry_local.ts b/test/api_integration/apis/telemetry/telemetry_local.ts index 9b92576c84b3a..c14fc658f2768 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/test/api_integration/apis/telemetry/telemetry_local.ts @@ -8,6 +8,7 @@ import expect from '@kbn/expect'; import supertestAsPromised from 'supertest-as-promised'; +import { omit } from 'lodash'; import { basicUiCounters } from './__fixtures__/ui_counters'; import { basicUsageCounters } from './__fixtures__/usage_counters'; import type { FtrProviderContext } from '../../ftr_provider_context'; @@ -86,6 +87,35 @@ export default function ({ getService }: FtrProviderContext) { expect(stats.stack_stats.kibana.plugins.csp.strict).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.warnLegacyBrowsers).to.be(true); expect(stats.stack_stats.kibana.plugins.csp.rulesChangedFromDefault).to.be(false); + expect(stats.stack_stats.kibana.plugins.kibana_config_usage).to.be.an('object'); + // non-default kibana configs. Configs set at 'test/api_integration/config.js'. + expect(omit(stats.stack_stats.kibana.plugins.kibana_config_usage, 'server.port')).to.eql({ + 'elasticsearch.username': '[redacted]', + 'elasticsearch.password': '[redacted]', + 'elasticsearch.hosts': '[redacted]', + 'elasticsearch.healthCheck.delay': 3600000, + 'plugins.paths': '[redacted]', + 'logging.json': false, + 'server.xsrf.disableProtection': true, + 'server.compression.referrerWhitelist': '[redacted]', + 'server.maxPayload': 1679958, + 'status.allowAnonymous': true, + 'home.disableWelcomeScreen': true, + 'data.search.aggs.shardDelay.enabled': true, + 'security.showInsecureClusterWarning': false, + 'telemetry.banner': false, + 'telemetry.url': '[redacted]', + 'telemetry.optInStatusUrl': '[redacted]', + 'telemetry.optIn': false, + 'newsfeed.service.urlRoot': '[redacted]', + 'newsfeed.service.pathTemplate': '[redacted]', + 'savedObjects.maxImportPayloadBytes': 10485760, + 'savedObjects.maxImportExportSize': 10001, + 'usageCollection.usageCounters.bufferDuration': 0, + }); + expect(stats.stack_stats.kibana.plugins.kibana_config_usage['server.port']).to.be.a( + 'number' + ); // Testing stack_stats.data expect(stats.stack_stats.data).to.be.an('object'); diff --git a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts index b45930682e3aa..ec44cec39c29a 100644 --- a/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts +++ b/test/api_integration/apis/telemetry/utils/schema_to_config_schema.ts @@ -8,8 +8,8 @@ import type { ObjectType, Type } from '@kbn/config-schema'; import { schema } from '@kbn/config-schema'; -import { get } from 'lodash'; import { set } from '@elastic/safer-lodash-set'; +import { get, merge } from 'lodash'; import type { AllowedSchemaTypes } from 'src/plugins/usage_collection/server'; /** @@ -125,11 +125,19 @@ export function assertTelemetryPayload( stats: unknown ): void { const fullSchema = telemetrySchema.root; + + const mergedPluginsSchema = merge( + {}, + get(fullSchema, 'properties.stack_stats.properties.kibana.properties.plugins'), + telemetrySchema.plugins + ); + set( fullSchema, 'properties.stack_stats.properties.kibana.properties.plugins', - telemetrySchema.plugins + mergedPluginsSchema ); + const ossTelemetryValidationSchema = convertSchemaToConfigSchema(fullSchema); // Run @kbn/config-schema validation to the entire payload diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts index a85e8ef82fc8c..2412b91e6ee68 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.ts @@ -53,6 +53,7 @@ export default function ({ getService }: FtrProviderContext) { it('should pass the schema validation', () => { const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema); const plugins = deepmerge(ossPluginsTelemetrySchema, xpackPluginsTelemetrySchema); + try { assertTelemetryPayload({ root, plugins }, stats); } catch (err) {