diff --git a/.ci/packer_cache_for_branch.sh b/.ci/packer_cache_for_branch.sh index a9fbe781915b6..5b4a94be50fa2 100755 --- a/.ci/packer_cache_for_branch.sh +++ b/.ci/packer_cache_for_branch.sh @@ -46,7 +46,7 @@ echo "Creating bootstrap_cache archive" # archive cacheable directories mkdir -p "$HOME/.kibana/bootstrap_cache" tar -cf "$HOME/.kibana/bootstrap_cache/$branch.tar" \ - x-pack/plugins/reporting/.chromium \ + .chromium \ .es \ .chromedriver \ .geckodriver; diff --git a/.eslintignore b/.eslintignore index 9de2cc2872960..4b5e781c26971 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1,6 +1,7 @@ **/*.js.snap **/graphql/types.ts /.es +/.chromium /build /built_assets /config/apm.dev.js diff --git a/.gitignore b/.gitignore index 32377ec0f1ffe..716cea937f9c0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .signing-config.json .ackrc /.es +/.chromium .DS_Store .node_binaries .native_modules diff --git a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md index 0e2b9bd60ab67..b88a179c5c4b3 100644 --- a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md +++ b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.md @@ -19,5 +19,6 @@ export interface DiscoveredPlugin | [configPath](./kibana-plugin-core-server.discoveredplugin.configpath.md) | ConfigPath | Root configuration path used by the plugin, defaults to "id" in snake\_case format. | | [id](./kibana-plugin-core-server.discoveredplugin.id.md) | PluginName | Identifier of the plugin. | | [optionalPlugins](./kibana-plugin-core-server.discoveredplugin.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | +| [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md) | readonly PluginName[] | List of plugin ids that this plugin's UI code imports modules from that are not in requiredPlugins. | | [requiredPlugins](./kibana-plugin-core-server.discoveredplugin.requiredplugins.md) | readonly PluginName[] | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. | diff --git a/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.requiredbundles.md b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.requiredbundles.md new file mode 100644 index 0000000000000..6d54adb5236ea --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.discoveredplugin.requiredbundles.md @@ -0,0 +1,18 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [DiscoveredPlugin](./kibana-plugin-core-server.discoveredplugin.md) > [requiredBundles](./kibana-plugin-core-server.discoveredplugin.requiredbundles.md) + +## DiscoveredPlugin.requiredBundles property + +List of plugin ids that this plugin's UI code imports modules from that are not in `requiredPlugins`. + +Signature: + +```typescript +readonly requiredBundles: readonly PluginName[]; +``` + +## Remarks + +The plugins listed here will be loaded in the browser, even if the plugin is disabled. Required by `@kbn/optimizer` to support cross-plugin imports. "core" and plugins already listed in `requiredPlugins` do not need to be duplicated here. + diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md index 5edee51d6c523..6db2f89590149 100644 --- a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.md @@ -25,6 +25,7 @@ Should never be used in code outside of Core but is exported for documentation p | [id](./kibana-plugin-core-server.pluginmanifest.id.md) | PluginName | Identifier of the plugin. Must be a string in camelCase. Part of a plugin public contract. Other plugins leverage it to access plugin API, navigate to the plugin, etc. | | [kibanaVersion](./kibana-plugin-core-server.pluginmanifest.kibanaversion.md) | string | The version of Kibana the plugin is compatible with, defaults to "version". | | [optionalPlugins](./kibana-plugin-core-server.pluginmanifest.optionalplugins.md) | readonly PluginName[] | An optional list of the other plugins that if installed and enabled \*\*may be\*\* leveraged by this plugin for some additional functionality but otherwise are not required for this plugin to work properly. | +| [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) | readonly string[] | List of plugin ids that this plugin's UI code imports modules from that are not in requiredPlugins. | | [requiredPlugins](./kibana-plugin-core-server.pluginmanifest.requiredplugins.md) | readonly PluginName[] | An optional list of the other plugins that \*\*must be\*\* installed and enabled for this plugin to function properly. | | [server](./kibana-plugin-core-server.pluginmanifest.server.md) | boolean | Specifies whether plugin includes some server-side specific functionality. | | [ui](./kibana-plugin-core-server.pluginmanifest.ui.md) | boolean | Specifies whether plugin includes some client/browser specific functionality that should be included into client bundle via public/ui_plugin.js file. | diff --git a/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.requiredbundles.md b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.requiredbundles.md new file mode 100644 index 0000000000000..98505d07101fe --- /dev/null +++ b/docs/development/core/server/kibana-plugin-core-server.pluginmanifest.requiredbundles.md @@ -0,0 +1,18 @@ + + +[Home](./index.md) > [kibana-plugin-core-server](./kibana-plugin-core-server.md) > [PluginManifest](./kibana-plugin-core-server.pluginmanifest.md) > [requiredBundles](./kibana-plugin-core-server.pluginmanifest.requiredbundles.md) + +## PluginManifest.requiredBundles property + +List of plugin ids that this plugin's UI code imports modules from that are not in `requiredPlugins`. + +Signature: + +```typescript +readonly requiredBundles: readonly string[]; +``` + +## Remarks + +The plugins listed here will be loaded in the browser, even if the plugin is disabled. Required by `@kbn/optimizer` to support cross-plugin imports. "core" and plugins already listed in `requiredPlugins` do not need to be duplicated here. + diff --git a/docs/user/reporting/index.asciidoc b/docs/user/reporting/index.asciidoc index 4123912b79237..6acdbbe3f0a99 100644 --- a/docs/user/reporting/index.asciidoc +++ b/docs/user/reporting/index.asciidoc @@ -19,7 +19,7 @@ image::user/reporting/images/share-button.png["Share"] [float] == Setup -{reporting} is automatically enabled in {kib}. The first time {kib} runs, it extracts a custom build for the Chromium web browser, which +{reporting} is automatically enabled in {kib}. It runs a custom build of the Chromium web browser, which runs on the server in headless mode to load {kib} and capture the rendered {kib} charts as images. Chromium is an open-source project not related to Elastic, but the Chromium binary for {kib} has been custom-built by Elastic to ensure it diff --git a/examples/bfetch_explorer/kibana.json b/examples/bfetch_explorer/kibana.json index 0039e9647bf83..f32cdfc13a1fe 100644 --- a/examples/bfetch_explorer/kibana.json +++ b/examples/bfetch_explorer/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["bfetch", "developerExamples"], - "optionalPlugins": [] + "optionalPlugins": [], + "requiredBundles": ["kibanaReact"] } diff --git a/examples/dashboard_embeddable_examples/kibana.json b/examples/dashboard_embeddable_examples/kibana.json index bb2ced569edb5..807229fad9dcf 100644 --- a/examples/dashboard_embeddable_examples/kibana.json +++ b/examples/dashboard_embeddable_examples/kibana.json @@ -5,5 +5,6 @@ "server": false, "ui": true, "requiredPlugins": ["embeddable", "embeddableExamples", "dashboard", "developerExamples"], - "optionalPlugins": [] + "optionalPlugins": [], + "requiredBundles": ["esUiShared"] } diff --git a/examples/embeddable_examples/kibana.json b/examples/embeddable_examples/kibana.json index 8ae04c1f6c644..771c19cfdbd3d 100644 --- a/examples/embeddable_examples/kibana.json +++ b/examples/embeddable_examples/kibana.json @@ -6,5 +6,6 @@ "ui": true, "requiredPlugins": ["embeddable", "uiActions"], "optionalPlugins": [], - "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"] + "extraPublicDirs": ["public/todo", "public/hello_world", "public/todo/todo_ref_embeddable"], + "requiredBundles": ["kibanaReact"] } diff --git a/examples/state_containers_examples/kibana.json b/examples/state_containers_examples/kibana.json index 66da207cb4e77..58346af8f1d19 100644 --- a/examples/state_containers_examples/kibana.json +++ b/examples/state_containers_examples/kibana.json @@ -5,5 +5,6 @@ "server": true, "ui": true, "requiredPlugins": ["navigation", "data", "developerExamples"], - "optionalPlugins": [] + "optionalPlugins": [], + "requiredBundles": ["kibanaUtils", "kibanaReact"] } diff --git a/examples/ui_action_examples/kibana.json b/examples/ui_action_examples/kibana.json index cd12442daf61c..0e0b6b6830b95 100644 --- a/examples/ui_action_examples/kibana.json +++ b/examples/ui_action_examples/kibana.json @@ -5,5 +5,6 @@ "server": false, "ui": true, "requiredPlugins": ["uiActions"], - "optionalPlugins": [] + "optionalPlugins": [], + "requiredBundles": ["kibanaReact"] } diff --git a/examples/ui_actions_explorer/kibana.json b/examples/ui_actions_explorer/kibana.json index f57072e89b06d..0a55e60374710 100644 --- a/examples/ui_actions_explorer/kibana.json +++ b/examples/ui_actions_explorer/kibana.json @@ -5,5 +5,6 @@ "server": false, "ui": true, "requiredPlugins": ["uiActions", "uiActionsExamples", "developerExamples"], - "optionalPlugins": [] + "optionalPlugins": [], + "requiredBundles": ["kibanaReact"] } diff --git a/package.json b/package.json index 1a497a2ec8b10..a45f240ce13af 100644 --- a/package.json +++ b/package.json @@ -127,7 +127,7 @@ "@elastic/datemath": "5.0.3", "@elastic/elasticsearch": "7.8.0", "@elastic/ems-client": "7.9.3", - "@elastic/eui": "24.1.0", + "@elastic/eui": "26.3.1", "@elastic/filesaver": "1.1.2", "@elastic/good": "8.1.1-kibana2", "@elastic/numeral": "^2.5.0", diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json index 20c8046daa65e..33f53e336598d 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json @@ -1,4 +1,5 @@ { "id": "bar", - "ui": true + "ui": true, + "requiredBundles": ["foo"] } diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/_other_styles.scss b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/_other_styles.scss new file mode 100644 index 0000000000000..2c1b9562b9567 --- /dev/null +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/_other_styles.scss @@ -0,0 +1,3 @@ +p { + background-color: rebeccapurple; +} diff --git a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss index e71a2d485a2f8..1dc7bbe9daeb0 100644 --- a/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss +++ b/packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/public/legacy/styles.scss @@ -1,3 +1,5 @@ +@import "./other_styles.scss"; + body { width: $globalStyleConstant; background-image: url("ui/icon.svg"); diff --git a/packages/kbn-optimizer/src/cli.ts b/packages/kbn-optimizer/src/cli.ts index 0916f12a7110d..9d3f4b88a258f 100644 --- a/packages/kbn-optimizer/src/cli.ts +++ b/packages/kbn-optimizer/src/cli.ts @@ -87,6 +87,11 @@ run( throw createFlagError('expected --report-stats to have no value'); } + const filter = typeof flags.filter === 'string' ? [flags.filter] : flags.filter; + if (!Array.isArray(filter) || !filter.every((f) => typeof f === 'string')) { + throw createFlagError('expected --filter to be one or more strings'); + } + const config = OptimizerConfig.create({ repoRoot: REPO_ROOT, watch, @@ -99,6 +104,7 @@ run( extraPluginScanDirs, inspectWorkers, includeCoreBundle, + filter, }); let update$ = runOptimizer(config); @@ -128,12 +134,13 @@ run( 'inspect-workers', 'report-stats', ], - string: ['workers', 'scan-dir'], + string: ['workers', 'scan-dir', 'filter'], default: { core: true, examples: true, cache: true, 'inspect-workers': true, + filter: [], }, help: ` --watch run the optimizer in watch mode @@ -142,6 +149,7 @@ run( --profile profile the webpack builds and write stats.json files to build outputs --no-core disable generating the core bundle --no-cache disable the cache + --filter comma-separated list of bundle id filters, results from multiple flags are merged, * and ! are supported --no-examples don't build the example plugins --dist create bundles that are suitable for inclusion in the Kibana distributable --scan-dir add a directory to the list of directories scanned for plugins (specify as many times as necessary) diff --git a/packages/kbn-optimizer/src/common/bundle.test.ts b/packages/kbn-optimizer/src/common/bundle.test.ts index b209bbca25ac4..6197a08485854 100644 --- a/packages/kbn-optimizer/src/common/bundle.test.ts +++ b/packages/kbn-optimizer/src/common/bundle.test.ts @@ -50,6 +50,7 @@ it('creates cache keys', () => { "spec": Object { "contextDir": "/foo/bar", "id": "bar", + "manifestPath": undefined, "outputDir": "/foo/bar/target", "publicDirNames": Array [ "public", @@ -85,6 +86,7 @@ it('parses bundles from JSON specs', () => { }, "contextDir": "/foo/bar", "id": "bar", + "manifestPath": undefined, "outputDir": "/foo/bar/target", "publicDirNames": Array [ "public", diff --git a/packages/kbn-optimizer/src/common/bundle.ts b/packages/kbn-optimizer/src/common/bundle.ts index 80af94c30f8da..a354da7a21521 100644 --- a/packages/kbn-optimizer/src/common/bundle.ts +++ b/packages/kbn-optimizer/src/common/bundle.ts @@ -18,6 +18,7 @@ */ import Path from 'path'; +import Fs from 'fs'; import { BundleCache } from './bundle_cache'; import { UnknownVals } from './ts_helpers'; @@ -25,6 +26,11 @@ import { includes, ascending, entriesToObject } from './array_helpers'; const VALID_BUNDLE_TYPES = ['plugin' as const, 'entry' as const]; +const DEFAULT_IMPLICIT_BUNDLE_DEPS = ['core']; + +const isStringArray = (input: any): input is string[] => + Array.isArray(input) && input.every((x) => typeof x === 'string'); + export interface BundleSpec { readonly type: typeof VALID_BUNDLE_TYPES[0]; /** Unique id for this bundle */ @@ -37,6 +43,8 @@ export interface BundleSpec { readonly sourceRoot: string; /** Absolute path to the directory where output should be written */ readonly outputDir: string; + /** Absolute path to a kibana.json manifest file, if omitted we assume there are not dependenices */ + readonly manifestPath?: string; } export class Bundle { @@ -56,6 +64,12 @@ export class Bundle { public readonly sourceRoot: BundleSpec['sourceRoot']; /** Absolute path to the output directory for this bundle */ public readonly outputDir: BundleSpec['outputDir']; + /** + * Absolute path to a manifest file with "requiredBundles" which will be + * used to allow bundleRefs from this bundle to the exports of another bundle. + * Every bundle mentioned in the `requiredBundles` must be built together. + */ + public readonly manifestPath: BundleSpec['manifestPath']; public readonly cache: BundleCache; @@ -66,6 +80,7 @@ export class Bundle { this.contextDir = spec.contextDir; this.sourceRoot = spec.sourceRoot; this.outputDir = spec.outputDir; + this.manifestPath = spec.manifestPath; this.cache = new BundleCache(Path.resolve(this.outputDir, '.kbn-optimizer-cache')); } @@ -96,8 +111,54 @@ export class Bundle { contextDir: this.contextDir, sourceRoot: this.sourceRoot, outputDir: this.outputDir, + manifestPath: this.manifestPath, }; } + + readBundleDeps(): { implicit: string[]; explicit: string[] } { + if (!this.manifestPath) { + return { + implicit: [...DEFAULT_IMPLICIT_BUNDLE_DEPS], + explicit: [], + }; + } + + let json: string; + try { + json = Fs.readFileSync(this.manifestPath, 'utf8'); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + + json = '{}'; + } + + let parsedManifest: { requiredPlugins?: string[]; requiredBundles?: string[] }; + try { + parsedManifest = JSON.parse(json); + } catch (error) { + throw new Error( + `unable to parse manifest at [${this.manifestPath}], error: [${error.message}]` + ); + } + + if (typeof parsedManifest === 'object' && parsedManifest) { + const explicit = parsedManifest.requiredBundles || []; + const implicit = [...DEFAULT_IMPLICIT_BUNDLE_DEPS, ...(parsedManifest.requiredPlugins || [])]; + + if (isStringArray(explicit) && isStringArray(implicit)) { + return { + explicit, + implicit, + }; + } + } + + throw new Error( + `Expected "requiredBundles" and "requiredPlugins" in manifest file [${this.manifestPath}] to be arrays of strings` + ); + } } /** @@ -152,6 +213,13 @@ export function parseBundles(json: string) { throw new Error('`bundles[]` must have an absolute path `outputDir` property'); } + const { manifestPath } = spec; + if (manifestPath !== undefined) { + if (!(typeof manifestPath === 'string' && Path.isAbsolute(manifestPath))) { + throw new Error('`bundles[]` must have an absolute path `manifestPath` property'); + } + } + return new Bundle({ type, id, @@ -159,6 +227,7 @@ export function parseBundles(json: string) { contextDir, sourceRoot, outputDir, + manifestPath, }); } ); diff --git a/packages/kbn-optimizer/src/common/bundle_cache.ts b/packages/kbn-optimizer/src/common/bundle_cache.ts index 5ae3e4c28a201..7607e270b5b4f 100644 --- a/packages/kbn-optimizer/src/common/bundle_cache.ts +++ b/packages/kbn-optimizer/src/common/bundle_cache.ts @@ -24,6 +24,7 @@ export interface State { optimizerCacheKey?: unknown; cacheKey?: unknown; moduleCount?: number; + workUnits?: number; files?: string[]; bundleRefExportIds?: string[]; } @@ -96,6 +97,10 @@ export class BundleCache { return this.get().cacheKey; } + public getWorkUnits() { + return this.get().workUnits; + } + public getOptimizerCacheKey() { return this.get().optimizerCacheKey; } diff --git a/packages/kbn-optimizer/src/common/bundle_refs.ts b/packages/kbn-optimizer/src/common/bundle_refs.ts index a5c60f2031c0b..85731f32f8991 100644 --- a/packages/kbn-optimizer/src/common/bundle_refs.ts +++ b/packages/kbn-optimizer/src/common/bundle_refs.ts @@ -114,6 +114,10 @@ export class BundleRefs { constructor(private readonly refs: BundleRef[]) {} + public forBundleIds(bundleIds: string[]) { + return this.refs.filter((r) => bundleIds.includes(r.bundleId)); + } + public filterByExportIds(exportIds: string[]) { return this.refs.filter((r) => exportIds.includes(r.exportId)); } diff --git a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap index 211cfac3806ad..c52873ab7ec20 100644 --- a/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap +++ b/packages/kbn-optimizer/src/integration_tests/__snapshots__/basic_optimization.test.ts.snap @@ -10,6 +10,7 @@ OptimizerConfig { }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar, "id": "bar", + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json, "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/target/public, "publicDirNames": Array [ "public", @@ -24,6 +25,7 @@ OptimizerConfig { }, "contextDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, "id": "foo", + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json, "outputDir": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/target/public, "publicDirNames": Array [ "public", @@ -42,18 +44,21 @@ OptimizerConfig { "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo, "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/nested/baz, "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/nested/baz/kibana.json, }, ], "profileWebpack": false, @@ -66,7 +71,7 @@ OptimizerConfig { } `; -exports[`prepares assets for distribution: bar bundle 1`] = `"(function(modules){var installedModules={};function __webpack_require__(moduleId){if(installedModules[moduleId]){return installedModules[moduleId].exports}var module=installedModules[moduleId]={i:moduleId,l:false,exports:{}};modules[moduleId].call(module.exports,module,module.exports,__webpack_require__);module.l=true;return module.exports}__webpack_require__.m=modules;__webpack_require__.c=installedModules;__webpack_require__.d=function(exports,name,getter){if(!__webpack_require__.o(exports,name)){Object.defineProperty(exports,name,{enumerable:true,get:getter})}};__webpack_require__.r=function(exports){if(typeof Symbol!==\\"undefined\\"&&Symbol.toStringTag){Object.defineProperty(exports,Symbol.toStringTag,{value:\\"Module\\"})}Object.defineProperty(exports,\\"__esModule\\",{value:true})};__webpack_require__.t=function(value,mode){if(mode&1)value=__webpack_require__(value);if(mode&8)return value;if(mode&4&&typeof value===\\"object\\"&&value&&value.__esModule)return value;var ns=Object.create(null);__webpack_require__.r(ns);Object.defineProperty(ns,\\"default\\",{enumerable:true,value:value});if(mode&2&&typeof value!=\\"string\\")for(var key in value)__webpack_require__.d(ns,key,function(key){return value[key]}.bind(null,key));return ns};__webpack_require__.n=function(module){var getter=module&&module.__esModule?function getDefault(){return module[\\"default\\"]}:function getModuleExports(){return module};__webpack_require__.d(getter,\\"a\\",getter);return getter};__webpack_require__.o=function(object,property){return Object.prototype.hasOwnProperty.call(object,property)};__webpack_require__.p=\\"\\";return __webpack_require__(__webpack_require__.s=5)})([function(module,exports,__webpack_require__){\\"use strict\\";var isOldIE=function isOldIE(){var memo;return function memorize(){if(typeof memo===\\"undefined\\"){memo=Boolean(window&&document&&document.all&&!window.atob)}return memo}}();var getTarget=function getTarget(){var memo={};return function memorize(target){if(typeof memo[target]===\\"undefined\\"){var styleTarget=document.querySelector(target);if(window.HTMLIFrameElement&&styleTarget instanceof window.HTMLIFrameElement){try{styleTarget=styleTarget.contentDocument.head}catch(e){styleTarget=null}}memo[target]=styleTarget}return memo[target]}}();var stylesInDom=[];function getIndexByIdentifier(identifier){var result=-1;for(var i=0;i { expect(foo.cache.getModuleCount()).toBe(6); expect(foo.cache.getReferencedFiles()).toMatchInlineSnapshot(` Array [ + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/kibana.json, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/async_import.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/ext.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/foo/public/index.ts, @@ -160,12 +161,17 @@ it('builds expected bundles, saves bundle counts to metadata', async () => { Array [ /node_modules/css-loader/package.json, /node_modules/style-loader/package.json, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/kibana.json, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/index.ts, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/_other_styles.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/legacy/styles.scss, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/plugins/bar/public/lib.ts, /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/icon.svg, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7dark.scss, + /packages/kbn-optimizer/src/__fixtures__/__tmp__/mock_repo/src/legacy/ui/public/styles/_globals_v7light.scss, /packages/kbn-optimizer/target/worker/entry_point_creator.js, + /packages/kbn-optimizer/target/worker/postcss.config.js, /packages/kbn-ui-shared-deps/public_path_module_creator.js, ] `); diff --git a/packages/kbn-optimizer/src/log_optimizer_state.ts b/packages/kbn-optimizer/src/log_optimizer_state.ts index 23767be610da4..20d98f74dbe86 100644 --- a/packages/kbn-optimizer/src/log_optimizer_state.ts +++ b/packages/kbn-optimizer/src/log_optimizer_state.ts @@ -54,12 +54,18 @@ export function logOptimizerState(log: ToolingLog, config: OptimizerConfig) { if (event?.type === 'worker started') { let moduleCount = 0; + let workUnits = 0; for (const bundle of event.bundles) { moduleCount += bundle.cache.getModuleCount() ?? NaN; + workUnits += bundle.cache.getWorkUnits() ?? NaN; } - const mcString = isFinite(moduleCount) ? String(moduleCount) : '?'; - const bcString = String(event.bundles.length); - log.info(`starting worker [${bcString} bundles, ${mcString} modules]`); + + log.info( + `starting worker [${event.bundles.length} ${ + event.bundles.length === 1 ? 'bundle' : 'bundles' + }]` + ); + log.debug(`modules [${moduleCount}] work units [${workUnits}]`); } if (state.phase === 'reallocating') { diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts index ca50a49e26913..5443a88eb1a63 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.test.ts @@ -23,11 +23,11 @@ import { Bundle } from '../common'; import { assignBundlesToWorkers, Assignments } from './assign_bundles_to_workers'; -const hasModuleCount = (b: Bundle) => b.cache.getModuleCount() !== undefined; -const noModuleCount = (b: Bundle) => b.cache.getModuleCount() === undefined; +const hasWorkUnits = (b: Bundle) => b.cache.getWorkUnits() !== undefined; +const noWorkUnits = (b: Bundle) => b.cache.getWorkUnits() === undefined; const summarizeBundles = (w: Assignments) => [ - w.moduleCount ? `${w.moduleCount} known modules` : '', + w.workUnits ? `${w.workUnits} work units` : '', w.newBundles ? `${w.newBundles} new bundles` : '', ] .filter(Boolean) @@ -42,15 +42,15 @@ const assertReturnVal = (workers: Assignments[]) => { expect(workers).toBeInstanceOf(Array); for (const worker of workers) { expect(worker).toEqual({ - moduleCount: expect.any(Number), + workUnits: expect.any(Number), newBundles: expect.any(Number), bundles: expect.any(Array), }); - expect(worker.bundles.filter(noModuleCount).length).toBe(worker.newBundles); + expect(worker.bundles.filter(noWorkUnits).length).toBe(worker.newBundles); expect( - worker.bundles.filter(hasModuleCount).reduce((sum, b) => sum + b.cache.getModuleCount()!, 0) - ).toBe(worker.moduleCount); + worker.bundles.filter(hasWorkUnits).reduce((sum, b) => sum + b.cache.getWorkUnits()!, 0) + ).toBe(worker.workUnits); } }; @@ -76,7 +76,7 @@ const getBundles = ({ for (let i = 1; i <= withCounts; i++) { const id = `foo${i}`; const bundle = testBundle(id); - bundle.cache.set({ moduleCount: i % 5 === 0 ? i * 10 : i }); + bundle.cache.set({ workUnits: i % 5 === 0 ? i * 10 : i }); bundles.push(bundle); } @@ -95,8 +95,8 @@ it('creates less workers if maxWorkersCount is larger than bundle count', () => expect(workers.length).toBe(2); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (1 known modules) => foo1", - "worker 1 (2 known modules) => foo2", + "worker 0 (1 work units) => foo1", + "worker 1 (2 work units) => foo2", ] `); }); @@ -121,10 +121,10 @@ it('distributes bundles without module counts evenly after assigning modules wit assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (78 known modules, 3 new bundles) => foo5,foo11,foo8,foo6,foo2,foo1,bar9,bar5,bar1", - "worker 1 (78 known modules, 3 new bundles) => foo16,foo14,foo13,foo12,foo9,foo7,foo4,foo3,bar8,bar4,bar0", - "worker 2 (100 known modules, 2 new bundles) => foo10,bar7,bar3", - "worker 3 (150 known modules, 2 new bundles) => foo15,bar6,bar2", + "worker 0 (78 work units, 3 new bundles) => foo5,foo11,foo8,foo6,foo2,foo1,bar9,bar5,bar1", + "worker 1 (78 work units, 3 new bundles) => foo16,foo14,foo13,foo12,foo9,foo7,foo4,foo3,bar8,bar4,bar0", + "worker 2 (100 work units, 2 new bundles) => foo10,bar7,bar3", + "worker 3 (150 work units, 2 new bundles) => foo15,bar6,bar2", ] `); }); @@ -135,8 +135,8 @@ it('distributes 2 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (1 known modules) => foo1", - "worker 1 (2 known modules) => foo2", + "worker 0 (1 work units) => foo1", + "worker 1 (2 work units) => foo2", ] `); }); @@ -147,10 +147,10 @@ it('distributes 5 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (3 known modules) => foo2,foo1", - "worker 1 (3 known modules) => foo3", - "worker 2 (4 known modules) => foo4", - "worker 3 (50 known modules) => foo5", + "worker 0 (3 work units) => foo2,foo1", + "worker 1 (3 work units) => foo3", + "worker 2 (4 work units) => foo4", + "worker 3 (50 work units) => foo5", ] `); }); @@ -161,10 +161,10 @@ it('distributes 10 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (20 known modules) => foo9,foo6,foo4,foo1", - "worker 1 (20 known modules) => foo8,foo7,foo3,foo2", - "worker 2 (50 known modules) => foo5", - "worker 3 (100 known modules) => foo10", + "worker 0 (20 work units) => foo9,foo6,foo4,foo1", + "worker 1 (20 work units) => foo8,foo7,foo3,foo2", + "worker 2 (50 work units) => foo5", + "worker 3 (100 work units) => foo10", ] `); }); @@ -175,10 +175,10 @@ it('distributes 15 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (70 known modules) => foo14,foo13,foo12,foo11,foo9,foo6,foo4,foo1", - "worker 1 (70 known modules) => foo5,foo8,foo7,foo3,foo2", - "worker 2 (100 known modules) => foo10", - "worker 3 (150 known modules) => foo15", + "worker 0 (70 work units) => foo14,foo13,foo12,foo11,foo9,foo6,foo4,foo1", + "worker 1 (70 work units) => foo5,foo8,foo7,foo3,foo2", + "worker 2 (100 work units) => foo10", + "worker 3 (150 work units) => foo15", ] `); }); @@ -189,10 +189,10 @@ it('distributes 20 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (153 known modules) => foo15,foo3", - "worker 1 (153 known modules) => foo10,foo16,foo13,foo11,foo7,foo6", - "worker 2 (154 known modules) => foo5,foo19,foo18,foo17,foo14,foo12,foo9,foo8,foo4,foo2,foo1", - "worker 3 (200 known modules) => foo20", + "worker 0 (153 work units) => foo15,foo3", + "worker 1 (153 work units) => foo10,foo16,foo13,foo11,foo7,foo6", + "worker 2 (154 work units) => foo5,foo19,foo18,foo17,foo14,foo12,foo9,foo8,foo4,foo2,foo1", + "worker 3 (200 work units) => foo20", ] `); }); @@ -203,10 +203,10 @@ it('distributes 25 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (250 known modules) => foo20,foo17,foo13,foo9,foo8,foo2,foo1", - "worker 1 (250 known modules) => foo15,foo23,foo22,foo18,foo16,foo11,foo7,foo3", - "worker 2 (250 known modules) => foo10,foo5,foo24,foo21,foo19,foo14,foo12,foo6,foo4", - "worker 3 (250 known modules) => foo25", + "worker 0 (250 work units) => foo20,foo17,foo13,foo9,foo8,foo2,foo1", + "worker 1 (250 work units) => foo15,foo23,foo22,foo18,foo16,foo11,foo7,foo3", + "worker 2 (250 work units) => foo10,foo5,foo24,foo21,foo19,foo14,foo12,foo6,foo4", + "worker 3 (250 work units) => foo25", ] `); }); @@ -217,10 +217,10 @@ it('distributes 30 bundles to workers evenly', () => { assertReturnVal(workers); expect(readConfigs(workers)).toMatchInlineSnapshot(` Array [ - "worker 0 (352 known modules) => foo30,foo22,foo14,foo11,foo4,foo1", - "worker 1 (352 known modules) => foo15,foo10,foo28,foo24,foo19,foo16,foo9,foo6", - "worker 2 (353 known modules) => foo20,foo5,foo29,foo23,foo21,foo13,foo12,foo3,foo2", - "worker 3 (353 known modules) => foo25,foo27,foo26,foo18,foo17,foo8,foo7", + "worker 0 (352 work units) => foo30,foo22,foo14,foo11,foo4,foo1", + "worker 1 (352 work units) => foo15,foo10,foo28,foo24,foo19,foo16,foo9,foo6", + "worker 2 (353 work units) => foo20,foo5,foo29,foo23,foo21,foo13,foo12,foo3,foo2", + "worker 3 (353 work units) => foo25,foo27,foo26,foo18,foo17,foo8,foo7", ] `); }); diff --git a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.ts b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.ts index e1bcb22230bf9..44a3b21c5fd47 100644 --- a/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.ts +++ b/packages/kbn-optimizer/src/optimizer/assign_bundles_to_workers.ts @@ -20,19 +20,18 @@ import { Bundle, descending, ascending } from '../common'; // helper types used inside getWorkerConfigs so we don't have -// to calculate moduleCounts over and over - +// to calculate workUnits over and over export interface Assignments { - moduleCount: number; + workUnits: number; newBundles: number; bundles: Bundle[]; } /** assign a wrapped bundle to a worker */ const assignBundle = (worker: Assignments, bundle: Bundle) => { - const moduleCount = bundle.cache.getModuleCount(); - if (moduleCount !== undefined) { - worker.moduleCount += moduleCount; + const workUnits = bundle.cache.getWorkUnits(); + if (workUnits !== undefined) { + worker.workUnits += workUnits; } else { worker.newBundles += 1; } @@ -59,7 +58,7 @@ export function assignBundlesToWorkers(bundles: Bundle[], maxWorkerCount: number const workers: Assignments[] = []; for (let i = 0; i < workerCount; i++) { workers.push({ - moduleCount: 0, + workUnits: 0, newBundles: 0, bundles: [], }); @@ -67,18 +66,18 @@ export function assignBundlesToWorkers(bundles: Bundle[], maxWorkerCount: number /** * separate the bundles which do and don't have module - * counts and sort them by [moduleCount, id] + * counts and sort them by [workUnits, id] */ const bundlesWithCountsDesc = bundles - .filter((b) => b.cache.getModuleCount() !== undefined) + .filter((b) => b.cache.getWorkUnits() !== undefined) .sort( descending( - (b) => b.cache.getModuleCount(), + (b) => b.cache.getWorkUnits(), (b) => b.id ) ); const bundlesWithoutModuleCounts = bundles - .filter((b) => b.cache.getModuleCount() === undefined) + .filter((b) => b.cache.getWorkUnits() === undefined) .sort(descending((b) => b.id)); /** @@ -87,9 +86,9 @@ export function assignBundlesToWorkers(bundles: Bundle[], maxWorkerCount: number * with module counts are assigned */ while (bundlesWithCountsDesc.length) { - const [smallestWorker, nextSmallestWorker] = workers.sort(ascending((w) => w.moduleCount)); + const [smallestWorker, nextSmallestWorker] = workers.sort(ascending((w) => w.workUnits)); - while (!nextSmallestWorker || smallestWorker.moduleCount <= nextSmallestWorker.moduleCount) { + while (!nextSmallestWorker || smallestWorker.workUnits <= nextSmallestWorker.workUnits) { const bundle = bundlesWithCountsDesc.shift(); if (!bundle) { @@ -104,7 +103,7 @@ export function assignBundlesToWorkers(bundles: Bundle[], maxWorkerCount: number * assign bundles without module counts to workers round-robin * starting with the smallest workers */ - workers.sort(ascending((w) => w.moduleCount)); + workers.sort(ascending((w) => w.workUnits)); while (bundlesWithoutModuleCounts.length) { for (const worker of workers) { const bundle = bundlesWithoutModuleCounts.shift(); diff --git a/packages/kbn-optimizer/src/optimizer/filter_by_id.test.ts b/packages/kbn-optimizer/src/optimizer/filter_by_id.test.ts new file mode 100644 index 0000000000000..3e848fe616b49 --- /dev/null +++ b/packages/kbn-optimizer/src/optimizer/filter_by_id.test.ts @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { filterById, HasId } from './filter_by_id'; + +const bundles: HasId[] = [ + { id: 'foo' }, + { id: 'bar' }, + { id: 'abc' }, + { id: 'abcd' }, + { id: 'abcde' }, + { id: 'example_a' }, +]; + +const print = (result: HasId[]) => + result + .map((b) => b.id) + .sort((a, b) => a.localeCompare(b)) + .join(', '); + +it('[] matches everything', () => { + expect(print(filterById([], bundles))).toMatchInlineSnapshot( + `"abc, abcd, abcde, bar, example_a, foo"` + ); +}); + +it('* matches everything', () => { + expect(print(filterById(['*'], bundles))).toMatchInlineSnapshot( + `"abc, abcd, abcde, bar, example_a, foo"` + ); +}); + +it('combines mutliple filters to select any bundle which is matched', () => { + expect(print(filterById(['foo', 'bar'], bundles))).toMatchInlineSnapshot(`"bar, foo"`); + expect(print(filterById(['bar', 'abc*'], bundles))).toMatchInlineSnapshot( + `"abc, abcd, abcde, bar"` + ); +}); + +it('matches everything if any filter is *', () => { + expect(print(filterById(['*', '!abc*'], bundles))).toMatchInlineSnapshot( + `"abc, abcd, abcde, bar, example_a, foo"` + ); +}); + +it('only matches bundles which are matched by an entire single filter', () => { + expect(print(filterById(['*,!abc*'], bundles))).toMatchInlineSnapshot(`"bar, example_a, foo"`); +}); + +it('handles purely positive filters', () => { + expect(print(filterById(['abc*'], bundles))).toMatchInlineSnapshot(`"abc, abcd, abcde"`); +}); + +it('handles purely negative filters', () => { + expect(print(filterById(['!abc*'], bundles))).toMatchInlineSnapshot(`"bar, example_a, foo"`); +}); diff --git a/packages/kbn-optimizer/src/optimizer/filter_by_id.ts b/packages/kbn-optimizer/src/optimizer/filter_by_id.ts new file mode 100644 index 0000000000000..ccf61a9efc880 --- /dev/null +++ b/packages/kbn-optimizer/src/optimizer/filter_by_id.ts @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export interface HasId { + id: string; +} + +function parseFilter(filter: string) { + const positive: RegExp[] = []; + const negative: RegExp[] = []; + + for (const segment of filter.split(',')) { + let trimmed = segment.trim(); + let list = positive; + + if (trimmed.startsWith('!')) { + trimmed = trimmed.slice(1); + list = negative; + } + + list.push(new RegExp(`^${trimmed.split('*').join('.*')}$`)); + } + + return (bundle: HasId) => + (!positive.length || positive.some((p) => p.test(bundle.id))) && + (!negative.length || !negative.some((p) => p.test(bundle.id))); +} + +export function filterById(filterStrings: string[], bundles: T[]) { + const filters = filterStrings.map(parseFilter); + return bundles.filter((b) => !filters.length || filters.some((f) => f(b))); +} diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts index bbd3ddc11f448..a70cfc759dd55 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.test.ts @@ -32,18 +32,21 @@ it('returns a bundle for core and each plugin', () => { id: 'foo', isUiPlugin: true, extraPublicDirs: [], + manifestPath: '/repo/plugins/foo/kibana.json', }, { directory: '/repo/plugins/bar', id: 'bar', isUiPlugin: false, extraPublicDirs: [], + manifestPath: '/repo/plugins/bar/kibana.json', }, { directory: '/outside/of/repo/plugins/baz', id: 'baz', isUiPlugin: true, extraPublicDirs: [], + manifestPath: '/outside/of/repo/plugins/baz/kibana.json', }, ], '/repo' @@ -53,6 +56,7 @@ it('returns a bundle for core and each plugin', () => { Object { "contextDir": /plugins/foo, "id": "foo", + "manifestPath": /plugins/foo/kibana.json, "outputDir": /plugins/foo/target/public, "publicDirNames": Array [ "public", @@ -63,6 +67,7 @@ it('returns a bundle for core and each plugin', () => { Object { "contextDir": "/outside/of/repo/plugins/baz", "id": "baz", + "manifestPath": "/outside/of/repo/plugins/baz/kibana.json", "outputDir": "/outside/of/repo/plugins/baz/target/public", "publicDirNames": Array [ "public", diff --git a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts index 2635289088725..04ab992addeec 100644 --- a/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts +++ b/packages/kbn-optimizer/src/optimizer/get_plugin_bundles.ts @@ -35,6 +35,7 @@ export function getPluginBundles(plugins: KibanaPlatformPlugin[], repoRoot: stri sourceRoot: repoRoot, contextDir: p.directory, outputDir: Path.resolve(p.directory, 'target/public'), + manifestPath: p.manifestPath, }) ); } diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts index f7b457ca42c6d..06fffc953f58b 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.test.ts @@ -40,24 +40,28 @@ it('parses kibana.json files of plugins found in pluginDirs', () => { "extraPublicDirs": Array [], "id": "bar", "isUiPlugin": true, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/bar/kibana.json, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo, "extraPublicDirs": Array [], "id": "foo", "isUiPlugin": true, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/foo/kibana.json, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/nested/baz, "extraPublicDirs": Array [], "id": "baz", "isUiPlugin": false, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/mock_repo/plugins/nested/baz/kibana.json, }, Object { "directory": /packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz, "extraPublicDirs": Array [], "id": "test_baz", "isUiPlugin": false, + "manifestPath": /packages/kbn-optimizer/src/__fixtures__/mock_repo/test_plugins/test_baz/kibana.json, }, ] `); diff --git a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts index 83637691004f4..b489c53be47b9 100644 --- a/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts +++ b/packages/kbn-optimizer/src/optimizer/kibana_platform_plugins.ts @@ -24,6 +24,7 @@ import loadJsonFile from 'load-json-file'; export interface KibanaPlatformPlugin { readonly directory: string; + readonly manifestPath: string; readonly id: string; readonly isUiPlugin: boolean; readonly extraPublicDirs: string[]; @@ -92,6 +93,7 @@ function readKibanaPlatformPlugin(manifestPath: string): KibanaPlatformPlugin { return { directory: Path.dirname(manifestPath), + manifestPath, id: manifest.id, isUiPlugin: !!manifest.ui, extraPublicDirs: extraPublicDirs || [], diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts index 5b46d67479fd5..f97646e2bbbd3 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.test.ts @@ -21,6 +21,7 @@ jest.mock('./assign_bundles_to_workers.ts'); jest.mock('./kibana_platform_plugins.ts'); jest.mock('./get_plugin_bundles.ts'); jest.mock('../common/theme_tags.ts'); +jest.mock('./filter_by_id.ts'); import Path from 'path'; import Os from 'os'; @@ -113,6 +114,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, @@ -139,6 +141,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": false, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, @@ -165,6 +168,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, @@ -193,6 +197,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, @@ -218,6 +223,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 2, @@ -243,6 +249,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, @@ -265,6 +272,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": false, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, @@ -287,6 +295,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": false, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, @@ -310,6 +319,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": false, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, @@ -333,6 +343,7 @@ describe('OptimizerConfig::parseOptions()', () => { Object { "cache": true, "dist": false, + "filters": Array [], "includeCoreBundle": false, "inspectWorkers": false, "maxWorkerCount": 100, @@ -358,6 +369,7 @@ describe('OptimizerConfig::create()', () => { const findKibanaPlatformPlugins: jest.Mock = jest.requireMock('./kibana_platform_plugins.ts') .findKibanaPlatformPlugins; const getPluginBundles: jest.Mock = jest.requireMock('./get_plugin_bundles.ts').getPluginBundles; + const filterById: jest.Mock = jest.requireMock('./filter_by_id.ts').filterById; beforeEach(() => { if ('mock' in OptimizerConfig.parseOptions) { @@ -370,6 +382,7 @@ describe('OptimizerConfig::create()', () => { ]); findKibanaPlatformPlugins.mockReturnValue(Symbol('new platform plugins')); getPluginBundles.mockReturnValue([Symbol('bundle1'), Symbol('bundle2')]); + filterById.mockReturnValue(Symbol('filtered bundles')); jest.spyOn(OptimizerConfig, 'parseOptions').mockImplementation((): any => ({ cache: Symbol('parsed cache'), @@ -382,6 +395,7 @@ describe('OptimizerConfig::create()', () => { themeTags: Symbol('theme tags'), inspectWorkers: Symbol('parsed inspect workers'), profileWebpack: Symbol('parsed profile webpack'), + filters: [], })); }); @@ -392,10 +406,7 @@ describe('OptimizerConfig::create()', () => { expect(config).toMatchInlineSnapshot(` OptimizerConfig { - "bundles": Array [ - Symbol(bundle1), - Symbol(bundle2), - ], + "bundles": Symbol(filtered bundles), "cache": Symbol(parsed cache), "dist": Symbol(parsed dist), "inspectWorkers": Symbol(parsed inspect workers), @@ -431,6 +442,32 @@ describe('OptimizerConfig::create()', () => { } `); + expect(filterById.mock).toMatchInlineSnapshot(` + Object { + "calls": Array [ + Array [ + Array [], + Array [ + Symbol(bundle1), + Symbol(bundle2), + ], + ], + ], + "instances": Array [ + [Window], + ], + "invocationCallOrder": Array [ + 23, + ], + "results": Array [ + Object { + "type": "return", + "value": Symbol(filtered bundles), + }, + ], + } + `); + expect(getPluginBundles.mock).toMatchInlineSnapshot(` Object { "calls": Array [ diff --git a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts index 7757004139d0d..0e588ab36238b 100644 --- a/packages/kbn-optimizer/src/optimizer/optimizer_config.ts +++ b/packages/kbn-optimizer/src/optimizer/optimizer_config.ts @@ -31,6 +31,7 @@ import { import { findKibanaPlatformPlugins, KibanaPlatformPlugin } from './kibana_platform_plugins'; import { getPluginBundles } from './get_plugin_bundles'; +import { filterById } from './filter_by_id'; function pickMaxWorkerCount(dist: boolean) { // don't break if cpus() returns nothing, or an empty array @@ -77,6 +78,18 @@ interface Options { pluginScanDirs?: string[]; /** absolute paths that should be added to the default scan dirs */ extraPluginScanDirs?: string[]; + /** + * array of comma separated patterns that will be matched against bundle ids. + * bundles will only be built if they match one of the specified patterns. + * `*` can exist anywhere in each pattern and will match anything, `!` inverts the pattern + * + * examples: + * --filter foo --filter bar # [foo, bar], excludes [foobar] + * --filter foo,bar # [foo, bar], excludes [foobar] + * --filter foo* # [foo, foobar], excludes [bar] + * --filter f*r # [foobar], excludes [foo, bar] + */ + filter?: string[]; /** flag that causes the core bundle to be built along with plugins */ includeCoreBundle?: boolean; @@ -103,6 +116,7 @@ interface ParsedOptions { dist: boolean; pluginPaths: string[]; pluginScanDirs: string[]; + filters: string[]; inspectWorkers: boolean; includeCoreBundle: boolean; themeTags: ThemeTags; @@ -118,6 +132,7 @@ export class OptimizerConfig { const inspectWorkers = !!options.inspectWorkers; const cache = options.cache !== false && !process.env.KBN_OPTIMIZER_NO_CACHE; const includeCoreBundle = !!options.includeCoreBundle; + const filters = options.filter || []; const repoRoot = options.repoRoot; if (!Path.isAbsolute(repoRoot)) { @@ -172,6 +187,7 @@ export class OptimizerConfig { cache, pluginScanDirs, pluginPaths, + filters, inspectWorkers, includeCoreBundle, themeTags, @@ -198,7 +214,7 @@ export class OptimizerConfig { ]; return new OptimizerConfig( - bundles, + filterById(options.filters, bundles), options.cache, options.watch, options.inspectWorkers, diff --git a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts index cde25564cf528..563b4ecb4bc37 100644 --- a/packages/kbn-optimizer/src/worker/bundle_ref_module.ts +++ b/packages/kbn-optimizer/src/worker/bundle_ref_module.ts @@ -10,6 +10,7 @@ // @ts-ignore not typed by @types/webpack import Module from 'webpack/lib/Module'; +import { BundleRef } from '../common'; export class BundleRefModule extends Module { public built = false; @@ -17,12 +18,12 @@ export class BundleRefModule extends Module { public buildInfo?: any; public exportsArgument = '__webpack_exports__'; - constructor(public readonly exportId: string) { + constructor(public readonly ref: BundleRef) { super('kbn/bundleRef', null); } libIdent() { - return this.exportId; + return this.ref.exportId; } chunkCondition(chunk: any) { @@ -30,7 +31,7 @@ export class BundleRefModule extends Module { } identifier() { - return '@kbn/bundleRef ' + JSON.stringify(this.exportId); + return '@kbn/bundleRef ' + JSON.stringify(this.ref.exportId); } readableIdentifier() { @@ -51,7 +52,7 @@ export class BundleRefModule extends Module { source() { return ` __webpack_require__.r(__webpack_exports__); - var ns = __kbnBundles__.get('${this.exportId}'); + var ns = __kbnBundles__.get('${this.ref.exportId}'); Object.defineProperties(__webpack_exports__, Object.getOwnPropertyDescriptors(ns)) `; } diff --git a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts index 9c4d5ed7f8a98..5396d11726f7a 100644 --- a/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts +++ b/packages/kbn-optimizer/src/worker/bundle_refs_plugin.ts @@ -44,6 +44,7 @@ export class BundleRefsPlugin { private readonly resolvedRefEntryCache = new Map>(); private readonly resolvedRequestCache = new Map>(); private readonly ignorePrefix = Path.resolve(this.bundle.contextDir) + Path.sep; + private allowedBundleIds = new Set(); constructor(private readonly bundle: Bundle, private readonly bundleRefs: BundleRefs) {} @@ -81,6 +82,45 @@ export class BundleRefsPlugin { } ); }); + + compiler.hooks.compilation.tap('BundleRefsPlugin/getRequiredBundles', (compilation) => { + this.allowedBundleIds.clear(); + + const manifestPath = this.bundle.manifestPath; + if (!manifestPath) { + return; + } + + const deps = this.bundle.readBundleDeps(); + for (const ref of this.bundleRefs.forBundleIds([...deps.explicit, ...deps.implicit])) { + this.allowedBundleIds.add(ref.bundleId); + } + + compilation.hooks.additionalAssets.tap('BundleRefsPlugin/watchManifest', () => { + compilation.fileDependencies.add(manifestPath); + }); + + compilation.hooks.finishModules.tapPromise( + 'BundleRefsPlugin/finishModules', + async (modules) => { + const usedBundleIds = (modules as any[]) + .filter((m: any): m is BundleRefModule => m instanceof BundleRefModule) + .map((m) => m.ref.bundleId); + + const unusedBundleIds = deps.explicit + .filter((id) => !usedBundleIds.includes(id)) + .join(', '); + + if (unusedBundleIds) { + const error = new Error( + `Bundle for [${this.bundle.id}] lists [${unusedBundleIds}] as a required bundle, but does not use it. Please remove it.` + ); + (error as any).file = manifestPath; + compilation.errors.push(error); + } + } + ); + }); } private cachedResolveRefEntry(ref: BundleRef) { @@ -170,21 +210,29 @@ export class BundleRefsPlugin { return; } - const eligibleRefs = this.bundleRefs.filterByContextPrefix(this.bundle, resolved); - if (!eligibleRefs.length) { + const possibleRefs = this.bundleRefs.filterByContextPrefix(this.bundle, resolved); + if (!possibleRefs.length) { // import doesn't match a bundle context return; } - for (const ref of eligibleRefs) { + for (const ref of possibleRefs) { const resolvedEntry = await this.cachedResolveRefEntry(ref); - if (resolved === resolvedEntry) { - return new BundleRefModule(ref.exportId); + if (resolved !== resolvedEntry) { + continue; } + + if (!this.allowedBundleIds.has(ref.bundleId)) { + throw new Error( + `import [${request}] references a public export of the [${ref.bundleId}] bundle, but that bundle is not in the "requiredPlugins" or "requiredBundles" list in the plugin manifest [${this.bundle.manifestPath}]` + ); + } + + return new BundleRefModule(ref); } - const bundleId = Array.from(new Set(eligibleRefs.map((r) => r.bundleId))).join(', '); - const publicDir = eligibleRefs.map((r) => r.entry).join(', '); + const bundleId = Array.from(new Set(possibleRefs.map((r) => r.bundleId))).join(', '); + const publicDir = possibleRefs.map((r) => r.entry).join(', '); throw new Error( `import [${request}] references a non-public export of the [${bundleId}] bundle and must point to one of the public directories: [${publicDir}]` ); diff --git a/packages/kbn-optimizer/src/worker/run_compilers.ts b/packages/kbn-optimizer/src/worker/run_compilers.ts index ca7673748bde9..c7be943d65a48 100644 --- a/packages/kbn-optimizer/src/worker/run_compilers.ts +++ b/packages/kbn-optimizer/src/worker/run_compilers.ts @@ -50,6 +50,15 @@ import { const PLUGIN_NAME = '@kbn/optimizer'; +/** + * sass-loader creates about a 40% overhead on the overall optimizer runtime, and + * so this constant is used to indicate to assignBundlesToWorkers() that there is + * extra work done in a bundle that has a lot of scss imports. The value is + * arbitrary and just intended to weigh the bundles so that they are distributed + * across mulitple workers on machines with lots of cores. + */ +const EXTRA_SCSS_WORK_UNITS = 100; + /** * Create an Observable for a specific child compiler + bundle */ @@ -102,6 +111,11 @@ const observeCompiler = ( const bundleRefExportIds: string[] = []; const referencedFiles = new Set(); let normalModuleCount = 0; + let workUnits = stats.compilation.fileDependencies.size; + + if (bundle.manifestPath) { + referencedFiles.add(bundle.manifestPath); + } for (const module of stats.compilation.modules) { if (isNormalModule(module)) { @@ -111,6 +125,15 @@ const observeCompiler = ( if (!parsedPath.dirs.includes('node_modules')) { referencedFiles.add(path); + + if (path.endsWith('.scss')) { + workUnits += EXTRA_SCSS_WORK_UNITS; + + for (const depPath of module.buildInfo.fileDependencies) { + referencedFiles.add(depPath); + } + } + continue; } @@ -127,7 +150,7 @@ const observeCompiler = ( } if (module instanceof BundleRefModule) { - bundleRefExportIds.push(module.exportId); + bundleRefExportIds.push(module.ref.exportId); continue; } @@ -158,6 +181,7 @@ const observeCompiler = ( optimizerCacheKey: workerConfig.optimizerCacheKey, cacheKey: bundle.createCacheKey(files, mtimes), moduleCount: normalModuleCount, + workUnits, files, }); diff --git a/packages/kbn-ui-shared-deps/package.json b/packages/kbn-ui-shared-deps/package.json index 6ea4a621f92f6..8398d1c081da6 100644 --- a/packages/kbn-ui-shared-deps/package.json +++ b/packages/kbn-ui-shared-deps/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@elastic/charts": "19.8.1", - "@elastic/eui": "24.1.0", + "@elastic/eui": "26.3.1", "@elastic/numeral": "^2.5.0", "@kbn/i18n": "1.0.0", "@kbn/monaco": "1.0.0", diff --git a/src/cli/cluster/cluster_manager.ts b/src/cli/cluster/cluster_manager.ts index 6db6199b391e1..f193f33e6f47e 100644 --- a/src/cli/cluster/cluster_manager.ts +++ b/src/cli/cluster/cluster_manager.ts @@ -261,7 +261,7 @@ export class ClusterManager { /debug\.log$/, ...pluginInternalDirsIgnore, fromRoot('src/legacy/server/sass/__tmp__'), - fromRoot('x-pack/plugins/reporting/.chromium'), + fromRoot('x-pack/plugins/reporting/chromium'), fromRoot('x-pack/plugins/security_solution/cypress'), fromRoot('x-pack/plugins/apm/e2e'), fromRoot('x-pack/plugins/apm/scripts'), diff --git a/src/core/public/chrome/chrome_service.tsx b/src/core/public/chrome/chrome_service.tsx index 1b894bc400f08..d29120e6ee9ac 100644 --- a/src/core/public/chrome/chrome_service.tsx +++ b/src/core/public/chrome/chrome_service.tsx @@ -17,7 +17,7 @@ * under the License. */ -import { Breadcrumb as EuiBreadcrumb, IconType } from '@elastic/eui'; +import { EuiBreadcrumb, IconType } from '@elastic/eui'; import React from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { BehaviorSubject, combineLatest, merge, Observable, of, ReplaySubject } from 'rxjs'; diff --git a/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap b/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap index 1cfded4dc7b8f..9ecbc055e3320 100644 --- a/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap +++ b/src/core/public/chrome/ui/header/__snapshots__/collapsible_nav.test.tsx.snap @@ -372,12 +372,13 @@ exports[`CollapsibleNav renders links grouped by category 1`] = ` handler={[Function]} /> } /> @@ -3908,16 +3909,9 @@ exports[`CollapsibleNav renders the default nav 2`] = ` handler={[Function]} /> - - } - /> - + />