[kbn/pm] add caching to bootstrap

packages/kbn-dev-utils/package.json

@@ -15,6 +15,7 @@
     "execa": "^3.2.0",
     "exit-hook": "^2.2.0",
     "getopts": "^2.2.5",
+    "load-json-file": "^6.2.0",
     "moment": "^2.24.0",
     "rxjs": "^6.5.3",
     "tree-kill": "^1.2.1",

packages/kbn-dev-utils/src/index.ts

@@ -27,6 +27,6 @@ export {
 export { createAbsolutePathSerializer } from './serializers';
 export { CA_CERT_PATH, ES_KEY_PATH, ES_CERT_PATH } from './certs';
 export { run, createFailError, createFlagError, combineErrors, isFailError, Flags } from './run';
-export { REPO_ROOT } from './constants';
+export { REPO_ROOT } from './repo_root';
 export { KbnClient } from './kbn_client';
 export * from './axios';

packages/kbn-dev-utils/src/repo_root.ts

@@ -0,0 +1,59 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Path from 'path';
+import Fs from 'fs';
+
+import loadJsonFile from 'load-json-file';
+
+const isKibanaDir = (dir: string) => {
+  try {
+    const path = Path.resolve(dir, 'package.json');
+    const json = loadJsonFile.sync(path);
+    if (json && typeof json === 'object' && 'name' in json && json.name === 'kibana') {
+      return true;
+    }
+  } catch (error) {
+    if (error && error.code === 'ENOENT') {
+      return false;
+    }
+
+    throw error;
+  }
+};
+
+// search for the kibana directory, since this file is moved around it might
+// not be where we think but should always be a relatively close parent
+// of this directory
+const startDir = Fs.realpathSync(__dirname);
+const { root: rootDir } = Path.parse(startDir);
+let cursor = startDir;
+while (true) {
+  if (isKibanaDir(cursor)) {
+    break;
+  }
+
+  const parent = Path.dirname(cursor);
+  if (parent === rootDir) {
+    throw new Error(`unable to find kibana directory from ${startDir}`);
+  }
+  cursor = parent;
+}
+
+export const REPO_ROOT = cursor;

packages/kbn-pm/package.json

@@ -34,6 +34,8 @@
     "@types/tempy": "^0.2.0",
     "@types/wrap-ansi": "^2.0.15",
     "@types/write-pkg": "^3.1.0",
+    "@kbn/dev-utils": "1.0.0",
+    "@yarnpkg/lockfile": "^1.1.0",
     "babel-loader": "^8.0.6",
     "chalk": "^2.4.2",
     "cmd-shim": "^2.1.0",
@@ -48,6 +50,7 @@
     "indent-string": "^3.2.0",
     "lodash.clonedeepwith": "^4.5.0",
     "log-symbols": "^2.2.0",
+    "multimatch": "^4.0.0",
     "ncp": "^2.0.0",
     "ora": "^1.4.0",
     "prettier": "^1.19.1",

packages/kbn-pm/src/cli.ts

@@ -47,6 +47,7 @@ function help() {
        -i, --include          Include only specified projects. If left unspecified, it defaults to including all projects.
        --oss                  Do not include the x-pack when running command.
        --skip-kibana-plugins  Filter all plugins in ./plugins and ../kibana-extra when running command.
+       --no-cache             Disable the bootstrap cache
   `);
 }
 
@@ -65,7 +66,10 @@ export async function run(argv: string[]) {
       h: 'help',
       i: 'include',
     },
-    boolean: ['prefer-offline', 'frozen-lockfile'],
+    default: {
+      cache: true,
+    },
+    boolean: ['prefer-offline', 'frozen-lockfile', 'cache'],
   });
 
   const args = options._;

packages/kbn-pm/src/commands/bootstrap.test.ts

@@ -29,6 +29,7 @@ import { Project } from '../utils/project';
 import { buildProjectGraph } from '../utils/projects';
 import { installInDir, runScriptInPackageStreaming, yarnWorkspacesInfo } from '../utils/scripts';
 import { BootstrapCommand } from './bootstrap';
+import { Kibana } from '../utils/kibana';
 
 const mockInstallInDir = installInDir as jest.Mock;
 const mockRunScriptInPackageStreaming = runScriptInPackageStreaming as jest.Mock;
@@ -107,6 +108,7 @@ test('handles dependencies of dependencies', async () => {
     ['bar', bar],
     ['baz', baz],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -115,6 +117,7 @@ test('handles dependencies of dependencies', async () => {
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -142,6 +145,7 @@ test('does not run installer if no deps in package', async () => {
     ['kibana', kibana],
     ['bar', bar],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   const logMock = jest.spyOn(console, 'log').mockImplementation(noop);
@@ -150,6 +154,7 @@ test('does not run installer if no deps in package', async () => {
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -167,6 +172,7 @@ test('handles "frozen-lockfile"', async () => {
   });
 
   const projects = new Map([['kibana', kibana]]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   jest.spyOn(console, 'log').mockImplementation(noop);
@@ -177,6 +183,7 @@ test('handles "frozen-lockfile"', async () => {
       'frozen-lockfile': true,
     },
     rootPath: '',
+    kbn,
   });
 
   expect(mockInstallInDir.mock.calls).toMatchSnapshot('install in dir');
@@ -205,6 +212,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
     ['kibana', kibana],
     ['bar', bar],
   ]);
+  const kbn = new Kibana(projects);
   const projectGraph = buildProjectGraph(projects);
 
   jest.spyOn(console, 'log').mockImplementation(noop);
@@ -213,6 +221,7 @@ test('calls "kbn:bootstrap" scripts and links executables after installing deps'
     extraArgs: [],
     options: {},
     rootPath: '',
+    kbn,
   });
 
   expect(mockLinkProjectExecutables.mock.calls).toMatchSnapshot('link bins');

packages/kbn-pm/src/commands/bootstrap.ts

@@ -24,12 +24,14 @@ import { log } from '../utils/log';
 import { parallelizeBatches } from '../utils/parallelize';
 import { topologicallyBatchProjects } from '../utils/projects';
 import { ICommand } from './';
+import { getAllChecksums } from '../utils/project_checksums';
+import { BootstrapCacheFile } from '../utils/bootstrap_cache_file';
 
 export const BootstrapCommand: ICommand = {
   description: 'Install dependencies and crosslink projects',
   name: 'bootstrap',
 
-  async run(projects, projectGraph, { options }) {
+  async run(projects, projectGraph, { options, kbn }) {
     const batchedProjectsByWorkspace = topologicallyBatchProjects(projects, projectGraph, {
       batchByWorkspace: true,
     });
@@ -65,9 +67,18 @@ export const BootstrapCommand: ICommand = {
      * have to, as it will slow down the bootstrapping process.
      */
     log.write(chalk.bold('\nLinking executables completed, running `kbn:bootstrap` scripts\n'));
-    await parallelizeBatches(batchedProjects, async pkg => {
-      if (pkg.hasScript('kbn:bootstrap')) {
-        await pkg.runScriptStreaming('kbn:bootstrap');
+
+    const checksums = options.cache ? await getAllChecksums(kbn, log) : false;
+    await parallelizeBatches(batchedProjects, async project => {
+      if (project.hasScript('kbn:bootstrap')) {
+        const cacheFile = new BootstrapCacheFile(kbn, project, checksums);
+        if (cacheFile.isValid()) {
+          log.success(`[${project.name}] cache up to date`);
+        } else {
+          cacheFile.delete();
+          await project.runScriptStreaming('kbn:bootstrap');
+          cacheFile.write();
+        }
       }
     });
 

packages/kbn-pm/src/commands/index.ts

@@ -23,6 +23,7 @@ export interface ICommandConfig {
   extraArgs: string[];
   options: { [key: string]: any };
   rootPath: string;
+  kbn: Kibana;
 }
 
 export interface ICommand {
@@ -36,6 +37,7 @@ import { BootstrapCommand } from './bootstrap';
 import { CleanCommand } from './clean';
 import { RunCommand } from './run';
 import { WatchCommand } from './watch';
+import { Kibana } from '../utils/kibana';
 
 export const commands: { [key: string]: ICommand } = {
   bootstrap: BootstrapCommand,

packages/kbn-pm/src/config.ts

@@ -19,18 +19,16 @@
 
 import { resolve } from 'path';
 
-export interface IProjectPathOptions {
-  'skip-kibana-plugins'?: boolean;
-  oss?: boolean;
+interface Options {
+  rootPath: string;
+  skipKibanaPlugins?: boolean;
+  ossOnly?: boolean;
 }
 
 /**
  * Returns all the paths where plugins are located
  */
-export function getProjectPaths(rootPath: string, options: IProjectPathOptions = {}) {
-  const skipKibanaPlugins = Boolean(options['skip-kibana-plugins']);
-  const ossOnly = Boolean(options.oss);
-
+export function getProjectPaths({ rootPath, ossOnly, skipKibanaPlugins }: Options) {
   const projectPaths = [rootPath, resolve(rootPath, 'packages/*')];
 
   // This is needed in order to install the dependencies for the declared

packages/kbn-pm/src/production/build_production_projects.ts

@@ -66,7 +66,7 @@ export async function buildProductionProjects({
  * is supplied, we omit projects with build.oss in their package.json set to false.
  */
 async function getProductionProjects(rootPath: string, onlyOSS?: boolean) {
-  const projectPaths = getProjectPaths(rootPath, {});
+  const projectPaths = getProjectPaths({ rootPath });
   const projects = await getProjects(rootPath, projectPaths);
   const projectsSubset = [projects.get('kibana')!];
 

packages/kbn-pm/src/run.test.ts

@@ -38,7 +38,7 @@ function getExpectedProjectsAndGraph(runMock: any) {
 }
 
 let command: ICommand;
-let config: ICommandConfig;
+let config: Omit<ICommandConfig, 'kbn'>;
 beforeEach(() => {
   command = {
     description: 'test description',

packages/kbn-pm/src/run.ts

@@ -22,23 +22,24 @@ import indentString from 'indent-string';
 import wrapAnsi from 'wrap-ansi';
 
 import { ICommand, ICommandConfig } from './commands';
-import { getProjectPaths, IProjectPathOptions } from './config';
 import { CliError } from './utils/errors';
 import { log } from './utils/log';
-import { buildProjectGraph, getProjects } from './utils/projects';
+import { buildProjectGraph } from './utils/projects';
 import { renderProjectsTree } from './utils/projects_tree';
+import { Kibana } from './utils/kibana';
 
-export async function runCommand(command: ICommand, config: ICommandConfig) {
+export async function runCommand(command: ICommand, config: Omit<ICommandConfig, 'kbn'>) {
   try {
     log.write(
       chalk.bold(
         `Running [${chalk.green(command.name)}] command from [${chalk.yellow(config.rootPath)}]:\n`
       )
     );
 
-    const projectPaths = getProjectPaths(config.rootPath, config.options as IProjectPathOptions);
-
-    const projects = await getProjects(config.rootPath, projectPaths, {
+    const kbn = await Kibana.loadFrom(config.rootPath);
+    const projects = kbn.getFilteredProjects({
+      skipKibanaPlugins: Boolean(config.options['skip-kibana-plugins']),
+      ossOnly: Boolean(config.options.oss),
       exclude: toArray(config.options.exclude),
       include: toArray(config.options.include),
     });
@@ -57,7 +58,10 @@ export async function runCommand(command: ICommand, config: ICommandConfig) {
     log.write(chalk.bold(`Found [${chalk.green(projects.size.toString())}] projects:\n`));
     log.write(renderProjectsTree(config.rootPath, projects));
 
-    await command.run(projects, projectGraph, config);
+    await command.run(projects, projectGraph, {
+      ...config,
+      kbn,
+    });
   } catch (e) {
     log.write(chalk.bold.red(`\n[${command.name}] failed:\n`));