Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes for webpack 5 support (backward compatible) #472

Merged
merged 9 commits into from
Apr 24, 2019
+378 −363
Merged

Fixes for webpack 5 support (backward compatible) #472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0a067ff
Fixes for webpack 5 support
neilime Jan 9, 2019
309d089
Fix tests
Jan 9, 2019
d3fb131
Merge branch 'master' into patch-2
HyperBrain Apr 21, 2019
7983e44
Fix lint issues
neilime Apr 22, 2019
d3221ad
Merge branch 'master' into patch-2
neilime Apr 22, 2019
8e738b7
Add ne tests
Apr 23, 2019
1bb32c7
Add new tests
Apr 23, 2019
29f4fc4
Improve code coverage
Apr 23, 2019
a6d536c
Improve code of getExternalModules function
Apr 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
285 changes: 145 additions & 140 deletions lib/packExternalModules.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,20 +153,25 @@ function findExternalOrigin(issuer) {
}

function getExternalModules(stats) {
if (!stats.compilation.chunks) {
return [];
}
const externals = new Set();
for (const chunk of stats.compilation.chunks) {
if (!chunk.modulesIterable) {
continue;
}

_.forEach(stats.compilation.chunks, chunk => {
// Explore each module within the chunk (built inputs):
chunk.forEachModule(module => {
for (const module of chunk.modulesIterable) {
if (isExternalModule(module)) {
externals.add({
origin: _.get(findExternalOrigin(module.issuer), 'rawRequest'),
external: getExternalModuleName(module)
});
}
});
});

}
}
return Array.from(externals);
}

Expand Down Expand Up @@ -206,146 +211,146 @@ module.exports = {

// Determine and create packager
return BbPromise.try(() => Packagers.get.call(this, this.configuration.packager))
.then(packager => {
// Fetch needed original package.json sections
const sectionNames = packager.copyPackageSectionNames;
const packageJson = this.serverless.utils.readFileSync(packageJsonPath);
const packageSections = _.pick(packageJson, sectionNames);
if (!_.isEmpty(packageSections)) {
this.options.verbose && this.serverless.cli.log(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);
}
.then(packager => {
// Fetch needed original package.json sections
const sectionNames = packager.copyPackageSectionNames;
const packageJson = this.serverless.utils.readFileSync(packageJsonPath);
const packageSections = _.pick(packageJson, sectionNames);
if (!_.isEmpty(packageSections)) {
this.options.verbose && this.serverless.cli.log(`Using package.json sections ${_.join(_.keys(packageSections), ', ')}`);
}

// Get first level dependency graph
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
// Get first level dependency graph
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);

return packager.getProdDependencies(path.dirname(packageJsonPath), 1)
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph, packageForceExcludes);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = _.defaults({
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true,
scripts: packageScripts
}, packageSections);
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName);
let hasPackageLock = false;
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
let packageLockFile = this.serverless.utils.readFileSync(packageLockPath);
packageLockFile = packager.rebaseLockfile(relPath, packageLockFile);
if (_.isObject(packageLockFile)) {
packageLockFile = JSON.stringify(packageLockFile, null, 2);
}

this.serverless.utils.writeFileSync(path.join(compositeModulePath, packager.lockfileName), packageLockFile);
hasPackageLock = true;
} catch(err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
return packager.getProdDependencies(path.dirname(packageJsonPath), 1)
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}
}
return BbPromise.resolve();
})
.then(() => {
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return packager.install(compositeModulePath, this.configuration.packagerOptions)
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = _.defaults({
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true,
scripts: packageScripts,
dependencies: {}
}, packageSections);
const prodModules = getProdModules.call(this,
_.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph, packageForceExcludes);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.try(() => {
// Only copy dependency modules if demanded by packager
if (packager.mustCopyModules) {
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback));

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph, packageForceExcludes);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}
return BbPromise.resolve();
})
.then(() => hasPackageLock ?
BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, packager.lockfileName), path.join(modulePath, packager.lockfileName), callback)) :
BbPromise.resolve()
)
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return packager.prune(modulePath, this.configuration.packagerOptions)
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
})
.then(() => {
// Prune extraneous packages - removes not needed ones
const startRunScripts = _.now();
return packager.runScripts(modulePath, _.keys(packageScripts))
.tap(() => this.options.verbose && this.serverless.cli.log(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`));

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = _.defaults({
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true,
scripts: packageScripts
}, packageSections);
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName);
let hasPackageLock = false;
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
let packageLockFile = this.serverless.utils.readFileSync(packageLockPath);
packageLockFile = packager.rebaseLockfile(relPath, packageLockFile);
if (_.isObject(packageLockFile)) {
packageLockFile = JSON.stringify(packageLockFile, null, 2);
}

this.serverless.utils.writeFileSync(path.join(compositeModulePath, packager.lockfileName), packageLockFile);
hasPackageLock = true;
} catch (err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
}
}
return BbPromise.resolve();
})
.then(() => {
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return packager.install(compositeModulePath, this.configuration.packagerOptions)
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = _.defaults({
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true,
scripts: packageScripts,
dependencies: {}
}, packageSections);
const prodModules = getProdModules.call(this,
_.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph, packageForceExcludes);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.try(() => {
// Only copy dependency modules if demanded by packager
if (packager.mustCopyModules) {
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback));
}
return BbPromise.resolve();
})
.then(() => hasPackageLock ?
BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, packager.lockfileName), path.join(modulePath, packager.lockfileName), callback)) :
BbPromise.resolve()
)
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return packager.prune(modulePath, this.configuration.packagerOptions)
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
})
.then(() => {
// Prune extraneous packages - removes not needed ones
const startRunScripts = _.now();
return packager.runScripts(modulePath, _.keys(packageScripts))
.tap(() => this.options.verbose && this.serverless.cli.log(`Run scripts: ${modulePath} [${_.now() - startRunScripts} ms]`));
});
})
.return();
});
})
.return();
});
});
}
};
Loading