[Sustainable Architecture] Telemetry schemas (elastic#201760)

## Summary

Since we are moving code around, the schema extractor now needs to adapt
to the new directories.

This PR adds new per-solution schemas to the `.telemetryrc.json` files
and adapts FTRs accordingly.

Related elastic#201653

### Checklist

- [x]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios

(cherry picked from commit dac87ef)

.telemetryrc.json

@@ -10,5 +10,10 @@
     "output": "src/plugins/telemetry/schema/kbn_packages.json",
     "root": "packages/",
     "exclude": []
+  },
+  {
+    "output": "src/plugins/telemetry/schema/oss_platform.json",
+    "root": "src/platform/",
+    "exclude": []
   }
 ]

packages/kbn-telemetry-tools/src/tools/extract_collectors.ts

@@ -37,7 +37,8 @@ export async function getProgramPaths({
   );
 
   if (filePaths.length === 0) {
-    throw Error(`No files found in ${root}`);
+    return []; // Temporarily accept empty directories while https://github.com/elastic/kibana-team/issues/1066 is completed
+    // throw Error(`No files found in ${root}`);
   }
 
   const fullPaths = filePaths

src/plugins/telemetry/schema/README.md

@@ -2,16 +2,18 @@
 
 This list of `.json` files describes the format of the payloads sent to the Remote Telemetry Service. All the files should follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`, with the addition of the type `pass_through`. This additional `type` indicates Kibana sends the payload as-is from the output of an external ES query.
 
-There are currently 2 files:
+There are currently 4 files:
 
 - `oss_root.json`: Defines the schema for the payload from the root keys.    
   Manually maintained for now because the frequency it changes should be pretty low.
 - `oss_plugins.json`: The schema for the content that will be nested in `stack_stats.kibana.plugins`.    
   It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API.    
   More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs.
+- `kbn_packages.json`: Same as `oss_plugins` but for collectors defined in `<ROOT_DIR>/packages/*`.
+- `oss_platform.json`: Same as `oss_plugins` but for collectors defined in `<ROOT_DIR>/src/platform/*`.
 
 NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service.
 
 ## Testing
 
-Functional tests are defined at `test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files, and validates the actual output of the telemetry endpoint against the final schema.
+Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge all files and [the x-pack counterparts](../../../../x-pack/plugins/telemetry_collection_xpack/schema), and validates the actual output of the telemetry endpoint against the final schema.

src/plugins/telemetry/schema/oss_platform.json

@@ -0,0 +1,3 @@
+{
+  "properties": {}
+}

src/plugins/telemetry/tsconfig.json

@@ -9,9 +9,7 @@
     "server/**/**/*",
     "common/**/*",
     "../../../typings/**/*",
-    "schema/oss_plugins.json",
-    "schema/oss_root.json",
-    "schema/kbn_packages.json"
+    "schema/*.json",
   ],
   "kbn_references": [
     "@kbn/core",

x-pack/.telemetryrc.json

@@ -10,5 +10,25 @@
     "output": "plugins/telemetry_collection_xpack/schema/xpack_monitoring.json",
     "root": "plugins/monitoring/server/telemetry_collection/",
     "exclude": []
+  },
+  {
+    "output": "plugins/telemetry_collection_xpack/schema/xpack_platform.json",
+    "root": "platform/",
+    "exclude": []
+  },
+  {
+    "output": "plugins/telemetry_collection_xpack/schema/xpack_observability.json",
+    "root": "solutions/observability",
+    "exclude": []
+  },
+  {
+    "output": "plugins/telemetry_collection_xpack/schema/xpack_search.json",
+    "root": "solutions/search",
+    "exclude": []
+  },
+  {
+    "output": "plugins/telemetry_collection_xpack/schema/xpack_security.json",
+    "root": "solutions/security",
+    "exclude": []
   }
 ]

x-pack/plugins/telemetry_collection_xpack/schema/README.md

@@ -2,17 +2,21 @@
 
 This is an extension of the [OSS Telemetry Schemas](../../../../src/plugins/telemetry/schema) to add the X-Pack-related data. The payloads described in these `.json` files must be merged to the OSS ones to get the structure of the full payload sent to the Remote Telemetry Service. All the files follow the schema convention as defined in the `usage_collection` plugin and `@kbn/telemetry-tools`.
 
-There are currently 2 files:
+There are currently 7 files:
 
 - `xpack_root.json`: Defines the extra fields x-pack reports over the OSS payload defined in the `oss_root.json`.    
   Manually maintained for now because the frequency it changes is expected to be pretty low.
 - `xpack_plugins.json`: The X-Pack related schema for the content that will be nested in `stack_stats.kibana.plugins`.    
   It is automatically generated by `@kbn/telemetry-tools` based on the `schema` property provided by all the registered Usage Collectors via the `usageCollection.makeUsageCollector` API.    
   More details in the [Schema field](../../usage_collection/README.md#schema-field) chapter in the UsageCollection's docs.
+- `xpack_platform.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/platform/*`.
+- `xpack_observability.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/observability/*`.
+- `xpack_search.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/search/*`.
+- `xpack_security.json`: Same as `xpack_plugins.json` but for collectors defined in `<ROOT_DIR>/x-pack/solutions/security/*`.
 - `xpack_monitoring.json`: It declares the payload sent by the monitoring-sourced telemetry. The actual schema for the payload is declared under `properties.monitoringTelemetry.properties.stats.items`, but due to the general behaviour in the `@kbn/telemetry-tools`, it gets nested down in that path.
 
 NOTE: Despite its similarities to ES mappings, the intention of these files is not to define any index mappings. They should be considered as a tool to understand the format of the payload that will be sent when reporting telemetry to the Remote Service.
 
 ## Testing
 
-Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge both files (+ the OSS definitions), and validates the actual output of the telemetry endpoint against the final schema.
+Functional tests are defined at `x-pack/test/api_integration/apis/telemetry/telemetry_local.ts`. They merge these files (+ the [OSS definitions](../../../../src/plugins/telemetry/schema)), and validates the actual output of the telemetry endpoint against the final schema.

x-pack/plugins/telemetry_collection_xpack/schema/xpack_observability.json

@@ -0,0 +1,3 @@
+{
+  "properties": {}
+}

x-pack/plugins/telemetry_collection_xpack/schema/xpack_platform.json

@@ -0,0 +1,3 @@
+{
+  "properties": {}
+}

x-pack/plugins/telemetry_collection_xpack/schema/xpack_search.json

@@ -0,0 +1,3 @@
+{
+  "properties": {}
+}

x-pack/plugins/telemetry_collection_xpack/schema/xpack_security.json

@@ -0,0 +1,3 @@
+{
+  "properties": {}
+}

x-pack/plugins/telemetry_collection_xpack/tsconfig.json

@@ -8,9 +8,7 @@
     "common/**/*",
     "server/**/*",
     "../../../typings/*",
-    "schema/xpack_monitoring.json",
-    "schema/xpack_plugins.json",
-    "schema/xpack_root.json",
+    "schema/*.json",
   ],
   "kbn_references": [
     "@kbn/core",

x-pack/test/api_integration/apis/telemetry/telemetry.ts

@@ -14,8 +14,13 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
 import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
 import monitoringRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_monitoring.json';
 import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
+import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
 import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
 import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
+import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
+import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
+import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
 import type { UnencryptedTelemetryPayload } from '@kbn/telemetry-plugin/common/types';
 import type {
   UsageStatsPayload,
@@ -161,18 +166,25 @@ export default function ({ getService }: FtrProviderContext) {
           // It's nested because of the way it's collected and declared
           monitoringRootTelemetrySchema.properties.monitoringTelemetry.properties.stats.items
         );
-        const plugins = deepmerge(
-          deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
-          xpackPluginsTelemetrySchema
-        );
+
+        const plugins = [
+          ossPluginsTelemetrySchema,
+          ossPackagesTelemetrySchema,
+          ossPlatformTelemetrySchema,
+          xpackPluginsTelemetrySchema,
+          xpackPlatformTelemetrySchema,
+          xpackObservabilityTelemetrySchema,
+          xpackSearchTelemetrySchema,
+          xpackSecurityTelemetrySchema,
+        ].reduce((acc, schema) => deepmerge(acc, schema));
 
         try {
           assertTelemetryPayload({ root, plugins }, localXPack);
           monitoring.forEach((stats) => {
             assertTelemetryPayload({ root: monitoringRoot, plugins }, stats);
           });
         } catch (err) {
-          err.message = `The telemetry schemas in 'x-pack/plugins/telemetry_collection_xpack/schema/' are out-of-date, please update it as required: ${err.message}`;
+          err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
           throw err;
         }
       });

x-pack/test/api_integration/apis/telemetry/telemetry_local.ts

@@ -9,9 +9,14 @@ import expect from '@kbn/expect';
 import deepmerge from 'deepmerge';
 import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
 import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
+import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
 import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
 import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
 import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
+import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
+import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
+import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
 import { assertTelemetryPayload } from '@kbn/telemetry-tools';
 import {
   ELASTIC_HTTP_VERSION_HEADER,
@@ -57,15 +62,21 @@ export default function ({ getService }: FtrProviderContext) {
 
     it('should pass the schema validation', () => {
       const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
-      const plugins = deepmerge(
-        deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
-        xpackPluginsTelemetrySchema
-      );
+      const plugins = [
+        ossPluginsTelemetrySchema,
+        ossPackagesTelemetrySchema,
+        ossPlatformTelemetrySchema,
+        xpackPluginsTelemetrySchema,
+        xpackPlatformTelemetrySchema,
+        xpackObservabilityTelemetrySchema,
+        xpackSearchTelemetrySchema,
+        xpackSecurityTelemetrySchema,
+      ].reduce((acc, schema) => deepmerge(acc, schema));
 
       try {
         assertTelemetryPayload({ root, plugins }, stats);
       } catch (err) {
-        err.message = `The telemetry schemas in 'x-pack/plugins/telemetry_collection_xpack/schema/' are out-of-date, please update it as required: ${err.message}`;
+        err.message = `The telemetry schemas in are out-of-date. Please define the schema of your collector and run "node scripts/telemetry_check --fix" to update them: ${err.message}`;
         throw err;
       }
     });

x-pack/test_serverless/api_integration/test_suites/common/telemetry/snapshot_telemetry.ts

@@ -11,7 +11,12 @@ import ossRootTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_root.json';
 import xpackRootTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_root.json';
 import ossPluginsTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_plugins.json';
 import ossPackagesTelemetrySchema from '@kbn/telemetry-plugin/schema/kbn_packages.json';
+import ossPlatformTelemetrySchema from '@kbn/telemetry-plugin/schema/oss_platform.json';
 import xpackPluginsTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_plugins.json';
+import xpackPlatformTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_platform.json';
+import xpackObservabilityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_observability.json';
+import xpackSearchTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_search.json';
+import xpackSecurityTelemetrySchema from '@kbn/telemetry-collection-xpack-plugin/schema/xpack_security.json';
 import { assertTelemetryPayload } from '@kbn/telemetry-tools';
 import type { UsageStatsPayloadTestFriendly } from '@kbn/test-suites-xpack/api_integration/services/usage_api';
 import type { RoleCredentials } from '../../../../shared/services';
@@ -42,10 +47,16 @@ export default function ({ getService }: FtrProviderContext) {
 
     it('should pass the schema validation (ensures BWC with Classic offering)', () => {
       const root = deepmerge(ossRootTelemetrySchema, xpackRootTelemetrySchema);
-      const plugins = deepmerge(
-        deepmerge(ossPluginsTelemetrySchema, ossPackagesTelemetrySchema),
-        xpackPluginsTelemetrySchema
-      );
+      const plugins = [
+        ossPluginsTelemetrySchema,
+        ossPackagesTelemetrySchema,
+        ossPlatformTelemetrySchema,
+        xpackPluginsTelemetrySchema,
+        xpackPlatformTelemetrySchema,
+        xpackObservabilityTelemetrySchema,
+        xpackSearchTelemetrySchema,
+        xpackSecurityTelemetrySchema,
+      ].reduce((acc, schema) => deepmerge(acc, schema));
 
       try {
         assertTelemetryPayload({ root, plugins }, stats);