From 53d8118b8fa126399a41cab570ac890bf6627def Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Mon, 3 Jul 2017 10:30:04 -0400 Subject: [PATCH] chore: don't remove optional dependencies in clean-shrinkwrap.js (#1551) If we include a platform specific optional dependency in the shrinkwrap file, then npm will insist in installing it even if the platform doesn't match. As a solution, we figured out we can avoid putting this platform specific optional dependencies in the npm-shrinkwrap.json file. In order to do this, we currently have a script called `clean-shrinkwrap.js` that runs *before* any `npm shrinkwrap` file (its a `preshrinkwrap` npm script) that deletes all the platform specific modules we know about using `npm rm`. The problem with this approach is that `npm rm` will remove the module's code from `node_modules`, which means that if we run `npm shrinkwrap`, we will lose certain optional dependencies, that may be needed at a later stage. The solution is to modify the `clean-shrinkwrap.js` script to parse `npm-shrinkwrap.json`, and manually delete the entries that we want to omit. Also, the script needs to be run *after* `npm shrinkwrap`, so we change the npm script name to `postshrinkwrap`. Signed-off-by: Juan Cruz Viotti --- package.json | 14 +- scripts/ci/build-installers.sh | 4 +- scripts/clean-shrinkwrap.js | 263 ++++++++++++++++++++++++++++++--- 3 files changed, 253 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index e0ccf02c0c..7952bba6f6 100644 --- a/package.json +++ b/package.json @@ -19,21 +19,17 @@ "scripts": { "test": "make test", "start": "electron lib/start.js", - "preshrinkwrap": "node ./scripts/clean-shrinkwrap.js", + "postshrinkwrap": "node ./scripts/clean-shrinkwrap.js", "configure": "node-gyp configure", "build": "node-gyp build", "install": "node-gyp rebuild" }, "author": "Resin Inc. ", "license": "Apache-2.0", - "shrinkwrapIgnore": [ - "macos-alias", - "fs-xattr", - "ds-store", - "appdmg", - "7zip-bin-mac", - "7zip-bin-win", - "7zip-bin-linux" + "platformSpecificDependencies": [ + [ "7zip-bin-mac" ], + [ "7zip-bin-win" ], + [ "7zip-bin-linux" ] ], "dependencies": { "angular": "1.6.3", diff --git a/scripts/ci/build-installers.sh b/scripts/ci/build-installers.sh index fa00c2d338..f1d80e84eb 100755 --- a/scripts/ci/build-installers.sh +++ b/scripts/ci/build-installers.sh @@ -48,8 +48,8 @@ if [ "$ARGV_OPERATING_SYSTEM" == "linux" ]; then ./scripts/build/docker/run-command.sh \ -r "$TARGET_ARCH" \ -s "$(pwd)" \ - -c 'make electron-develop installers-all' + -c 'make installers-all' else ./scripts/build/check-dependency.sh make - make electron-develop installers-all + make installers-all fi diff --git a/scripts/clean-shrinkwrap.js b/scripts/clean-shrinkwrap.js index 8fe9fdb38a..473f6d1c72 100644 --- a/scripts/clean-shrinkwrap.js +++ b/scripts/clean-shrinkwrap.js @@ -15,29 +15,258 @@ 'use strict'; +const _ = require('lodash'); +const fs = require('fs'); const path = require('path'); -const os = require('os'); const packageJSON = require('../package.json'); -const spawn = require('child_process').spawn; -const shrinkwrapIgnore = packageJSON.shrinkwrapIgnore; +const NPM_SHRINKWRAP_FILE_PATH = path.join(__dirname, '..', 'npm-shrinkwrap.json'); +const shrinkwrapFile = require(NPM_SHRINKWRAP_FILE_PATH); +const platformSpecificDependencies = packageJSON.platformSpecificDependencies; +const JSON_INDENTATION_SPACES = 2; -console.log('Removing:', shrinkwrapIgnore.join(', ')); +console.log('Removing:', platformSpecificDependencies.join(', ')); /** - * Run an npm command - * @param {Array} command - list of arguments - * @returns {ChildProcess} + * @summary Get a shrinkwrap dependency object + * @function + * @private + * + * @param {Object} shrinkwrap - the shrinkwrap file contents + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object} shrinkwrap object + * + * @example + * const object = getShrinkwrapDependencyObject(require('./npm-shrinkwrap.json'), [ + * 'drivelist', + * 'lodash' + * ]); + * + * console.log(object.version); + * console.log(object.dependencies); */ -const npm = (command) => { - const npmBinary = os.platform() === 'win32' ? 'npm.cmd' : 'npm'; - return spawn(npmBinary, command, { - cwd: path.join(__dirname, '..'), - env: process.env, - stdio: [ process.stdin, process.stdout, process.stderr ] - }); +const getShrinkwrapDependencyObject = (shrinkwrap, shrinkwrapPath) => { + return _.reduce(shrinkwrapPath, (accumulator, dependency) => { + return _.get(accumulator, [ 'dependencies', dependency ], {}); + }, shrinkwrap); +}; + +/** + * @summary Get a cleaned shrinkwrap dependency object + * @function + * @private + * + * @description + * This function wraps `getShrinkwrapDependencyObject()` to + * omit unnecessary properties such as `from`, or `dependencies`. + * + * @param {Object} shrinkwrap - the shrinkwrap file contents + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object} pretty shrinkwrap object + * + * @example + * const object = getPrettyShrinkwrapDependencyObject(require('./npm-shrinkwrap.json'), [ + * 'drivelist', + * 'lodash' + * ]); + * + * console.log(object.name); + * console.log(object.path); + * console.log(object.version); + */ +const getPrettyShrinkwrapDependencyObject = (shrinkwrap, shrinkwrapPath) => { + const object = getShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath); + + if (_.isEmpty(object)) { + return null; + } + + return { + name: _.last(shrinkwrapPath), + path: shrinkwrapPath, + version: object.version, + development: Boolean(object.dev), + optional: Boolean(object.optional) + }; +}; + +/** + * @summary Get the manifest (package.json) of a shrinkwrap dependency + * @function + * @private + * + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object} dependency manifest + * + * @example + * const manifest = getShrinkwrapDependencyManifest([ 'bluebird' ]); + * console.log(manifest.devDependencies); + */ +const getShrinkwrapDependencyManifest = (shrinkwrapPath) => { + const manifestPath = _.chain(shrinkwrapPath) + .flatMap((dependency) => { + return [ 'node_modules', dependency ]; + }) + .concat([ 'package.json' ]) + .reduce((accumulator, file) => { + return path.join(accumulator, file); + }, '.') + .value(); + + try { + + // For example + // ./node_modules/drivelist/node_modules/lodash/package.json + return require(`.${path.sep}${manifestPath}`); + + } catch (error) { + if (error.code === 'MODULE_NOT_FOUND') { + return null; + } + + throw error; + } +}; + +/** + * @summary Get the top level dependencies of a shrinkwrap object + * @function + * @private + * + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object} top level dependencies + * + * @example + * const dependencies = getTopLevelDependenciesForShrinkwrapPath([ 'debug' ]); + * console.log(dependencies); + * > { + * > "lodash": "^4.0.0" + * > } + */ +const getTopLevelDependenciesForShrinkwrapPath = (shrinkwrapPath) => { + return _.get(getShrinkwrapDependencyManifest(shrinkwrapPath), [ 'dependencies' ], {}); }; -npm([ 'rm', '--ignore-scripts' ].concat(shrinkwrapIgnore)) - .once('close', () => { - console.log('Done.'); +/** + * @summary Get the dependency tree of a shrinkwrap dependency + * @function + * @private + * + * @param {Object} shrinkwrap - the shrinkwrap file contents + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object[]} dependency tree + * + * @example + * const dependencyTree = getDependencyTree(require('./npm-shrinkwrap.json'), [ 'drivelist' ]); + * + * _.each(dependencyTree, (dependency) => { + * console.log(dependency.name); + * console.log(dependency.path); + * console.log(dependency.version); + * }) + */ +const getDependencyTree = (shrinkwrap, shrinkwrapPath) => { + const dependencies = getTopLevelDependenciesForShrinkwrapPath(shrinkwrapPath); + + if (_.isEmpty(dependencies)) { + return []; + } + + const object = getShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath); + const result = _.map(dependencies, (version, name) => { + const dependencyPath = _.has(object.dependencies, name) ? _.concat(shrinkwrapPath, [ name ]) : [ name ]; + return getPrettyShrinkwrapDependencyObject(shrinkwrap, dependencyPath); }); + + return _.concat(result, _.flatMapDeep(result, (dependency) => { + return getDependencyTree(shrinkwrap, dependency.path); + })); +}; + +/** + * @summary Remove certain development optional dependencies from a shrinkwrap file + * @function + * @private + * + * @description + * A shrinkwrap object is a recursive data structure, that apart + * from some extra metadata, has the following structure: + * + * { + * ... + * "dependencies": { + * "": , + * "": , + * "": , + * ... + * } + * } + * + * The purpose of this function is to remove certain dependencies + * that match a blacklist. In order to do so, we start from the top + * level object, remove the blacklisted dependencies, and recurse + * if possible. + * + * @param {Object} shrinkwrap - the shrinkwrap object + * @param {Object[]} blacklist - dependency blacklist + * @returns {Object} filtered shrinkwrap object + * + * @example + * const shrinkwrapFile = require('./npm-shrinkwrap.json'); + * const dependencyTree = getDependencyTree(shrinkwrapFile, [ 'drivelist' ]); + * const filteredShrinkwrap = removeOptionalDevelopmentDependencies(shrinkwrapFile, dependencyTree); + */ +const removeOptionalDevelopmentDependencies = (shrinkwrap, blacklist) => { + if (!_.isEmpty(shrinkwrap.dependencies)) { + shrinkwrap.dependencies = _.chain(shrinkwrap.dependencies) + .omitBy((dependency, name) => { + return _.every([ + _.find(blacklist, { + name, + version: dependency.version + }), + dependency.dev, + dependency.optional + ]); + }) + .mapValues((dependency) => { + return removeOptionalDevelopmentDependencies(dependency, blacklist); + }) + .value(); + } + + return shrinkwrap; +}; + +/** + * @summary Get the dependency tree of a dependency plus the dependency itself + * @function + * @private + * + * @param {Object} shrinkwrap - the shrinkwrap file contents + * @param {String[]} shrinkwrapPath - path to shrinkwrap dependency + * @returns {Object[]} tree + * + * @example + * const tree = getTree(require('./npm-shrinkwrap.json'), [ 'drivelist' ]); + * + * _.each(tree, (dependency) => { + * console.log(dependency.name); + * console.log(dependency.path); + * console.log(dependency.version); + * }); + */ +const getTree = (shrinkwrap, shrinkwrapPath) => { + return _.compact(_.concat([ + getPrettyShrinkwrapDependencyObject(shrinkwrap, shrinkwrapPath) + ], getDependencyTree(shrinkwrap, shrinkwrapPath))); +}; + +const blacklist = _.reduce(platformSpecificDependencies, (accumulator, dependencyPath) => { + return _.concat(accumulator, getTree(shrinkwrapFile, dependencyPath)); +}, []); + +const filteredShrinkwrap = removeOptionalDevelopmentDependencies(shrinkwrapFile, blacklist); +const result = JSON.stringify(filteredShrinkwrap, null, JSON_INDENTATION_SPACES); + +fs.writeFileSync(NPM_SHRINKWRAP_FILE_PATH, `${result}\n`); +console.log('Done');