From 556a5043477feda3ca4ad8825ab6aa6ce8365eec Mon Sep 17 00:00:00 2001 From: Nicolas Lunet Date: Thu, 4 Jul 2019 16:00:47 +0200 Subject: [PATCH 01/15] [FEATURE] Sourcemap support in the ui5-builder preload generation - Always go for the predefine rewrite in the library build - Change the order of the steps to have debug file created early enough - Sourcemap support for the rewrite and for the preload generation - Uglifier can take input sourcemap as well (if you have a custom transpile step...) (cherry picked from commit 49812897b410238e50ead5aafef7a0c18deaab7a) --- .eslintrc.js | 2 +- lib/lbt/bundle/Builder.js | 92 +++++++++++++++----- lib/lbt/bundle/BundleWriter.js | 10 +++ lib/processors/bundlers/moduleBundler.js | 14 ++- lib/processors/uglifier.js | 13 ++- lib/tasks/bundlers/generateLibraryPreload.js | 12 ++- lib/types/library/LibraryBuilder.js | 10 +++ test/lib/tasks/uglify.js | 3 +- 8 files changed, 121 insertions(+), 35 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index b6b26aa9e..3b55d128c 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -4,7 +4,7 @@ module.exports = { "es6": true }, "parserOptions": { - "ecmaVersion": 8 + "ecmaVersion": 2018 }, "extends": ["eslint:recommended", "google"], "plugins": [ diff --git a/lib/lbt/bundle/Builder.js b/lib/lbt/bundle/Builder.js index 582023237..221e193ac 100644 --- a/lib/lbt/bundle/Builder.js +++ b/lib/lbt/bundle/Builder.js @@ -2,12 +2,13 @@ // for consistency of write calls, we generally allow template literals "use strict"; +const path = require("path"); const terser = require("terser"); const {pd} = require("pretty-data"); const esprima = require("esprima"); const escodegen = require("escodegen"); const {Syntax} = esprima; -// const MOZ_SourceMap = require("source-map"); +const sourceMap = require("source-map"); const {isMethodCall} = require("../utils/ASTUtils"); const ModuleName = require("../utils/ModuleName"); @@ -144,6 +145,9 @@ class BundleBuilder { this.jqglobalAvailable = !resolvedModule.containsGlobal; this.openModule(resolvedModule.name); + this.writeConfiguration(resolvedModule.configuration); // NODE-TODO configuration currently will be undefined + this._sourceMap = new sourceMap.SourceMapGenerator({file: module.name, sourceRoot: "."}); + // create all sections in sequence for ( const section of resolvedModule.sections ) { log.verbose(" adding section%s of type %s", @@ -159,6 +163,7 @@ class BundleBuilder { return { name: module.name, content: this.outW.toString(), + sourceMap: this._sourceMap.toString(), bundleInfo: bundleInfo }; } @@ -188,11 +193,7 @@ class BundleBuilder { this.outW.writeln(`if (oError.name != "Restart") { throw oError; }`); this.outW.writeln(`}`); } - /* NODE-TODO - if ( writeSourceMap && writeSourceMapAnnotation ) { - outW.ensureNewLine(); - outW.write("//# sourceMappingURL=" + moduleName.getBaseName().replaceFirst("\\.js$", ".js.map")); - }*/ + this.outW.writeln(`//# sourceMappingURL=${path.basename(resolvedModule.name)}.map`); } addSection(section) { @@ -271,7 +272,7 @@ class BundleBuilder { const sequence = section.modules.slice(); this.beforeWriteFunctionPreloadSection(sequence); - + this.outW.writeln(`//@ui5-bundle ${section.bundle.name}`); await this.rewriteAMDModules(sequence, avoidLazyParsing); if ( sequence.length > 0 ) { this.targetBundleFormat.beforePreloads(outW, section); @@ -285,6 +286,7 @@ class BundleBuilder { this.beforeWritePreloadModule(module, resource.info, resource); outW.write(`\t"${module.toString()}":`); outW.startSegment(module); + await this.writePreloadModule(module, resource.info, resource, avoidLazyParsing); const compressedSize = outW.endSegment(); log.verbose(" %s (%d,%d)", module, @@ -304,11 +306,15 @@ class BundleBuilder { // this.afterWriteFunctionPreloadSection(); } - async compressJS(fileContent, resource) { + async compressJS(fileContent, resource, fileMap) { if ( this.optimize ) { const result = await terser.minify({ [resource.name]: String(fileContent) }, { + sourceMap: { + content: fileMap ? fileMap.toString() : "inline" + }, + warnings: false, // TODO configure? compress: false, // TODO configure? output: { comments: copyrightCommentsPattern, @@ -317,13 +323,14 @@ class BundleBuilder { // , outFileName: resource.name // , outSourceMap: true }); - // console.log(result.map); - // const map = new MOZ_SourceMap.SourceMapConsumer(result.map); - // map.eachMapping(function (m) { console.log(m); }); // console.log(map); + if ( result.error ) { + throw result.error; + } fileContent = result.code; + fileMap = result.map; // throw new Error(); } - return fileContent; + return {compressedContent: fileContent, contentMap: fileMap}; } beforeWriteFunctionPreloadSection(sequence) { @@ -331,6 +338,27 @@ class BundleBuilder { sequence.sort(); } + addSourceMap(contentMap) { + const map = new sourceMap.SourceMapConsumer(contentMap); + map.eachMapping((mapping) => { + if (mapping.source) { + this._sourceMap.addMapping({ + generated: { + line: this.outW.lineOffset + mapping.generatedLine, + column: (mapping.generatedLine === 1 ? this.outW.columnOffset : 0) + mapping.generatedColumn + }, + original: mapping.originalLine == null ? null : { + line: mapping.originalLine, + column: mapping.originalColumn + }, + source: mapping.originalLine != null ? + mapping.source : null, + name: mapping.name + }); + } + }); + } + async rewriteAMDModules(sequence, avoidLazyParsing) { if ( this.options.usePredefineCalls ) { const outW = this.outW; @@ -340,14 +368,21 @@ class BundleBuilder { if ( /\.js$/.test(module) ) { // console.log("Processing " + module); const resource = await this.pool.findResourceWithInfo(module); + const sourceMapResource = await this.pool.findResourceWithInfo(`${module}.map`); const code = await resource.buffer(); - const ast = rewriteDefine(this.targetBundleFormat, code, module, avoidLazyParsing); + const sourceMapContent = (await sourceMapResource.buffer()).toString(); + let ast = rewriteDefine(this.targetBundleFormat, code, module, avoidLazyParsing, sourceMapContent); if ( ast ) { outW.startSegment(module); outW.ensureNewLine(); - const astAsCode = escodegen.generate(ast); - const fileContent = await this.compressJS(astAsCode, resource); - outW.write( fileContent ); + ast = escodegen.attachComments(ast, ast.comments, ast.tokens); + const astAsCode = escodegen.generate(ast, {comment: true, sourceMap: true, sourceMapWithCode: true}); + // Reapply sourcemap on top + astAsCode.map.applySourceMap(new sourceMap.SourceMapConsumer(sourceMapContent), path.basename(module, ".js")+"-dbg.js"); + const {compressedContent, contentMap} = {...await this.compressJS(astAsCode.code, resource, astAsCode.map.toString())}; + this.addSourceMap(contentMap); + + outW.write( compressedContent ); outW.ensureNewLine(); const compressedSize = outW.endSegment(); log.verbose(" %s (%d,%d)", module, @@ -384,11 +419,15 @@ class BundleBuilder { const outW = this.outW; if ( /\.js$/.test(module) && (info == null || !info.requiresTopLevelScope) ) { - const compressedContent = await this.compressJS( await resource.buffer(), resource ); if ( avoidLazyParsing ) { outW.write(`(`); } outW.write(`function(){`); + + const compressorOutput = await this.compressJSAsync( resource ); + const {compressedContent, contentMap} = {...compressorOutput}; + + this.addSourceMap(contentMap); outW.write( compressedContent ); this.exportGlobalNames(info); outW.ensureNewLine(); @@ -466,13 +505,15 @@ class BundleBuilder { const CALL_DEFINE = ["define"]; const CALL_SAP_UI_DEFINE = ["sap", "ui", "define"]; -function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) { - function _injectModuleNameIfNeeded(defineCall) { +function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing, sourceMapContent) { + function _injectModuleNameIfNeeded(defineCall, moduleName) { if ( defineCall.arguments.length == 0 || defineCall.arguments[0].type !== Syntax.Literal ) { defineCall.arguments.unshift({ type: Syntax.Literal, - value: ModuleName.toRequireJSName(moduleName) + value: ModuleName.toRequireJSName(moduleName), + loc: null, + range: [] }); } } @@ -504,7 +545,13 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) { let ast; try { - ast = esprima.parseScript(code.toString(), {loc: true}); + ast = esprima.parseScript(code.toString(), { + comment: true, + loc: true, + source: path.basename(moduleName, ".js")+"-dbg.js", + range: true, + tokens: true + }); } catch (e) { log.error("error while parsing %s: %s", moduleName, e.message); return; @@ -552,14 +599,12 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) { } // console.log("rewriting %s", module, defineCall); - return ast; } if ( isMethodCall(ast.body[0].expression, CALL_SAP_UI_DEFINE) ) { const defineCall = ast.body[0].expression; - // rewrite sap.ui.define to sap.ui.predefine if ( defineCall.callee.type === Syntax.MemberExpression && defineCall.callee.property.type === Syntax.Identifier && @@ -572,7 +617,6 @@ function rewriteDefine(targetBundleFormat, code, moduleName, avoidLazyParsing) { if ( avoidLazyParsing ) { enforceEagerParsing(defineCall); } - // console.log("rewriting %s", module, defineCall); return ast; diff --git a/lib/lbt/bundle/BundleWriter.js b/lib/lbt/bundle/BundleWriter.js index 2862fb35a..df5ef79f0 100644 --- a/lib/lbt/bundle/BundleWriter.js +++ b/lib/lbt/bundle/BundleWriter.js @@ -10,6 +10,8 @@ const ENDS_WITH_NEW_LINE = /(^|\r\n|\r|\n)[ \t]*$/; * * Most methods have been extracted from JSMergeWriter. * + * columnOffset and lineOffset are used for sourcemap merging as reference to where we are at a given point in time + * * @author Frank Weigel * @since 1.27.0 * @private @@ -17,6 +19,8 @@ const ENDS_WITH_NEW_LINE = /(^|\r\n|\r|\n)[ \t]*$/; class BundleWriter { constructor() { this.buf = ""; + this.lineOffset = 0; + this.columnOffset = 0; this.segments = []; this.currentSegment = null; this.currentSourceIndex = 0; @@ -25,6 +29,8 @@ class BundleWriter { write(...str) { for ( let i = 0; i < str.length; i++ ) { this.buf += str[i]; + this.lineOffset += str[i].split(NL).length - 1; + this.columnOffset += str[i].length; } } @@ -33,12 +39,16 @@ class BundleWriter { this.buf += str[i]; } this.buf += NL; + this.lineOffset += 1; + this.columnOffset = 0; } ensureNewLine() { // TODO this regexp might be quite expensive (use of $ anchor on long strings) if ( !ENDS_WITH_NEW_LINE.test(this.buf) ) { this.buf += NL; + this.lineOffset += 1; + this.columnOffset = 0; } } diff --git a/lib/processors/bundlers/moduleBundler.js b/lib/processors/bundlers/moduleBundler.js index 62e7f24c0..e6f4dfced 100644 --- a/lib/processors/bundlers/moduleBundler.js +++ b/lib/processors/bundlers/moduleBundler.js @@ -99,7 +99,7 @@ module.exports = function({resources, options: {bundleDefinition, bundleOptions} // console.log(JSON.stringify(options.bundleDefinition, null, 4)); // TODO 3.0: Fix defaulting behavior, align with JSDoc - bundleOptions = bundleOptions || {optimize: true}; + bundleOptions = bundleOptions || {optimize: true, usePredefineCalls: true}; const pool = new LocatorResourcePool({ ignoreMissingModules: bundleOptions.ignoreMissingModules @@ -116,13 +116,19 @@ module.exports = function({resources, options: {bundleDefinition, bundleOptions} bundles = [results]; } - return Promise.all(bundles.map(function({name, content, bundleInfo}) { + return Promise.all(bundles.map(function({name, sourceMap, content, bundleInfo}) { // console.log("creating bundle as '%s'", "/resources/" + name); const resource = new EvoResource({ path: "/resources/" + name, string: content }); - return resource; - })); + const sourceMapResource = new EvoResource({ + path: "/resources/" + name + ".map", + string: sourceMap + }); + + + return [resource, sourceMapResource]; + })).then((resources) => [].concat(...resources)); }); }; diff --git a/lib/processors/uglifier.js b/lib/processors/uglifier.js index 7b6e1ebc9..28b213a98 100644 --- a/lib/processors/uglifier.js +++ b/lib/processors/uglifier.js @@ -1,4 +1,5 @@ const terser = require("terser"); +const EvoResource = require("@ui5/fs").Resource; /** * Preserve comments which contain: *