[kbn/optimizer] report limits with ci metrics

packages/kbn-dev-utils/src/ci_stats_reporter/README.md

@@ -8,10 +8,23 @@ This class integrates with the `ciStats.trackBuild {}` Jenkins Pipeline function
 
 To create an instance of the reporter, import the class and call `CiStatsReporter.fromEnv(log)` (passing it a tooling log).
 
-#### `CiStatsReporter#metrics(metrics: Array<{ group: string, id: string, value: number }>)`
+#### `CiStatsReporter#metrics(metrics: Metric[])`
 
 Use this method to record metrics in the Kibana CI Stats service.
 
+```ts
+interface Metric {
+  group: string,
+  id: string,
+  value: number,
+  // optional limit, values which exceed the limit will fail PRs
+  limit?: number
+  // optional path, relative to the root of the repo, where config values
+  // are defined. Will be linked to in PRs which have overages.
+  limitConfigPath?: string
+}
+```
+
 Example:
 
 ```ts

packages/kbn-dev-utils/src/ci_stats_reporter/ci_stats_reporter.ts

@@ -29,7 +29,13 @@ interface Config {
   buildId: string;
 }
 
-export type CiStatsMetrics = Array<{ group: string; id: string; value: number }>;
+export type CiStatsMetrics = Array<{
+  group: string;
+  id: string;
+  value: number;
+  limit?: number;
+  limitConfigPath?: string;
+}>;
 
 function parseConfig(log: ToolingLog) {
   const configJson = process.env.KIBANA_CI_STATS_CONFIG;

packages/kbn-optimizer/limits.yml

@@ -0,0 +1,99 @@
+pageLoadAssetSize:
+  advancedSettings: 17_596
+  alerts: 96_936
+  apm: 54_385
+  apmOss: 8_996
+  beatsManagement: 178_135
+  bfetch: 31_874
+  canvas: 1_056_461
+  charts: 149_211
+  cloud: 11_076
+  console: 36_091
+  core: 682_068
+  crossClusterReplication: 55_408
+  dashboard: 587_988
+  dashboardEnhanced: 55_646
+  dashboardMode: 12_716
+  data: 1_161_035
+  dataEnhanced: 40_192
+  devTools: 28_637
+  discover: 94_982
+  discoverEnhanced: 32_730
+  embeddable: 302_792
+  embeddableEnhanced: 31_145
+  enterpriseSearch: 25_484
+  esUiShared: 316_654
+  expressions: 214_136
+  features: 21_211
+  fileUpload: 14_717
+  globalSearch: 33_548
+  globalSearchBar: 52_888
+  globalSearchProviders: 15_554
+  graph: 21_215
+  grokdebugger: 16_779
+  home: 32_372
+  indexLifecycleManagement: 97_090
+  indexManagement: 130_608
+  indexPatternManagement: 144_222
+  infra: 187_873
+  ingestManager: 555_174
+  ingestPipelines: 48_003
+  inputControlVis: 162_675
+  inspector: 138_711
+  kibanaLegacy: 97_711
+  kibanaReact: 132_073
+  kibanaUtils: 188_829
+  lens: 86_624
+  licenseManagement: 31_817
+  licensing: 28_693
+  lists: 173_665
+  logstash: 43_548
+  management: 36_112
+  maps: 173_452
+  mapsLegacy: 106_817
+  mapsLegacyLicensing: 10_214
+  ml: 64_944
+  monitoring: 258_612
+  navigation: 27_269
+  newsfeed: 27_628
+  observability: 77_897
+  painlessLab: 169_748
+  regionMap: 56_098
+  remoteClusters: 41_327
+  reporting: 173_418
+  rollup: 87_204
+  savedObjects: 98_518
+  savedObjectsManagement: 90_503
+  searchprofiler: 57_080
+  security: 179_027
+  securityOss: 20_806
+  securitySolution: 606_237
+  share: 89_061
+  snapshotRestore: 69_032
+  spaces: 377_915
+  telemetry: 81_832
+  telemetryManagementSection: 42_443
+  tileMap: 55_337
+  timelion: 19_920
+  transform: 31_007
+  triggersActionsUi: 160_006
+  uiActions: 87_717
+  uiActionsEnhanced: 339_652
+  upgradeAssistant: 71_241
+  uptime: 30_825
+  urlDrilldown: 24_174
+  urlForwarding: 22_554
+  usageCollection: 29_762
+  visDefaultEditor: 40_178
+  visTypeMarkdown: 20_896
+  visTypeMetric: 32_790
+  visTypeTable: 84_934
+  visTypeTagcloud: 27_575
+  visTypeTimelion: 41_933
+  visTypeTimeseries: 145_203
+  visTypeVega: 143_573
+  visTypeVislib: 232_838
+  visTypeXy: 10_255
+  visualizations: 285_025
+  visualize: 47_431
+  watcher: 33_598

packages/kbn-optimizer/package.json

@@ -22,6 +22,7 @@
     "cpy": "^8.0.0",
     "core-js": "^3.6.5",
     "css-loader": "^3.4.2",
+    "dedent": "^0.7.0",
     "del": "^5.1.0",
     "execa": "^4.0.2",
     "file-loader": "^4.2.0",
@@ -38,6 +39,7 @@
     "postcss-loader": "^3.0.0",
     "raw-loader": "^3.1.0",
     "rxjs": "^6.5.5",
+    "js-yaml": "^3.14.0",
     "sass-loader": "^8.0.2",
     "source-map-support": "^0.5.19",
     "style-loader": "^1.1.3",

packages/kbn-optimizer/src/cli.ts

@@ -28,6 +28,7 @@ import { logOptimizerState } from './log_optimizer_state';
 import { OptimizerConfig } from './optimizer';
 import { reportOptimizerStats } from './report_optimizer_stats';
 import { runOptimizer } from './run_optimizer';
+import { validateLimitsForAllBundles, updateBundleLimits } from './limits';
 
 run(
   async ({ log, flags }) => {
@@ -93,21 +94,36 @@ run(
       throw createFlagError('expected --filter to be one or more strings');
     }
 
+    const validateLimits = flags['validate-limits'] ?? false;
+    if (typeof validateLimits !== 'boolean') {
+      throw createFlagError('expected --validate-limits to have no value');
+    }
+
+    const updateLimits = flags['update-limits'] ?? false;
+    if (typeof updateLimits !== 'boolean') {
+      throw createFlagError('expected --update-limits to have no value');
+    }
+
     const config = OptimizerConfig.create({
       repoRoot: REPO_ROOT,
       watch,
       maxWorkerCount,
-      oss,
-      dist,
+      oss: oss && !(validateLimits || updateLimits),
+      dist: dist || updateLimits,
       cache,
-      examples,
+      examples: examples && !(validateLimits || updateLimits),
       profileWebpack,
       extraPluginScanDirs,
       inspectWorkers,
       includeCoreBundle,
       filter,
     });
 
+    if (validateLimits) {
+      validateLimitsForAllBundles(log, config);
+      return;
+    }
+
     let update$ = runOptimizer(config);
 
     if (reportStats) {
@@ -121,6 +137,10 @@ run(
     }
 
     await update$.pipe(logOptimizerState(log, config)).toPromise();
+
+    if (updateLimits) {
+      updateBundleLimits(log, config);
+    }
   },
   {
     flags: {
@@ -134,6 +154,8 @@ run(
         'profile',
         'inspect-workers',
         'report-stats',
+        'validate-limits',
+        'update-limits',
       ],
       string: ['workers', 'scan-dir', 'filter'],
       default: {
@@ -152,10 +174,12 @@ run(
         --no-cache         disable the cache
         --filter           comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported
         --no-examples      don't build the example plugins
-        --dist             create bundles that are suitable for inclusion in the Kibana distributable
+        --dist             create bundles that are suitable for inclusion in the Kibana distributable, enabled when running with --update-limits
         --scan-dir         add a directory to the list of directories scanned for plugins (specify as many times as necessary)
         --no-inspect-workers  when inspecting the parent process, don't inspect the workers
         --report-stats     attempt to report stats about this execution of the build to the kibana-ci-stats service using this name
+        --validate-limits  validate the limits.yml config to ensure that there are limits defined for every bundle
+        --update-limits    run a build and rewrite the limits file to include the current bundle sizes +5kb
       `,
     },
   }

packages/kbn-optimizer/src/index.ts

@@ -22,3 +22,4 @@ export * from './run_optimizer';
 export * from './log_optimizer_state';
 export * from './report_optimizer_stats';
 export * from './node';
+export * from './limits';

packages/kbn-optimizer/src/integration_tests/basic_optimization.test.ts

@@ -27,7 +27,13 @@ import del from 'del';
 import { toArray, tap, filter } from 'rxjs/operators';
 import { REPO_ROOT } from '@kbn/utils';
 import { ToolingLog } from '@kbn/dev-utils';
-import { runOptimizer, OptimizerConfig, OptimizerUpdate, logOptimizerState } from '@kbn/optimizer';
+import {
+  runOptimizer,
+  OptimizerConfig,
+  OptimizerUpdate,
+  logOptimizerState,
+  readLimits,
+} from '@kbn/optimizer';
 
 const TMP_DIR = Path.resolve(__dirname, '../__fixtures__/__tmp__');
 const MOCK_REPO_SRC = Path.resolve(__dirname, '../__fixtures__/mock_repo');
@@ -72,6 +78,9 @@ it('builds expected bundles, saves bundle counts to metadata', async () => {
     dist: false,
   });
 
+  expect(config.limits).toEqual(readLimits());
+  (config as any).limits = '<Limits>';
+
   expect(config).toMatchSnapshot('OptimizerConfig');
 
   const msgs = await runOptimizer(config)

packages/kbn-optimizer/src/limits.ts

@@ -0,0 +1,81 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import dedent from 'dedent';
+import Yaml from 'js-yaml';
+import { createFailError, ToolingLog } from '@kbn/dev-utils';
+
+import { OptimizerConfig, getMetrics } from './optimizer';
+
+const LIMITS_PATH = require.resolve('../limits.yml');
+const diff = <T>(a: T[], b: T[]): T[] => a.filter((item) => !b.includes(item));
+
+export function readLimits() {
+  return Yaml.safeLoad(Fs.readFileSync(LIMITS_PATH, 'utf8'));
+}
+
+export function validateLimitsForAllBundles(log: ToolingLog, config: OptimizerConfig) {
+  const limitBundleIds = Object.keys(config.limits.pageLoadAssetSize);
+  const configBundleIds = config.bundles.map((b) => b.id);
+
+  const missingBundleIds = diff(configBundleIds, limitBundleIds);
+  const extraBundleIds = diff(limitBundleIds, configBundleIds);
+
+  const issues = [];
+  if (missingBundleIds.length) {
+    issues.push(`missing: ${missingBundleIds.join(', ')}`);
+  }
+  if (extraBundleIds.length) {
+    issues.push(`extra: ${extraBundleIds.join(', ')}`);
+  }
+  if (issues.length) {
+    throw createFailError(
+      dedent`
+        The limits defined in packages/kbn-optimizer/limits.yml are outdated. Please update
+        this file with a limit (in bytes) for every production bundle.
+
+          ${issues.join('\n          ')}
+
+        To validate your changes locally, run:
+
+          node scripts/build_kibana_platform_plugins.js --validate-limits
+      ` + '\n'
+    );
+  }
+
+  log.success('limits.yml file valid');
+}
+
+export function updateBundleLimits(log: ToolingLog, config: OptimizerConfig) {
+  const metrics = getMetrics(log, config);
+
+  const number = (input: number) => input.toLocaleString('en').split(',').join('_');
+
+  let yaml = `pageLoadAssetSize:\n`;
+  for (const metric of metrics.sort((a, b) => a.id.localeCompare(b.id))) {
+    if (metric.group === 'page load bundle size') {
+      yaml += `  ${metric.id}: ${number(metric.value + 5000)}\n`;
+    }
+  }
+
+  Fs.writeFileSync(LIMITS_PATH, yaml);
+  log.success(`wrote updated limits to ${LIMITS_PATH}`);
+}