Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Write mode #69

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
"main": "./dist/index.js",
"scripts": {
"build": "ncc build src/asset-relocator.js",
"watch": "ncc build src/asset-relocator.js --watch",
"codecov": "codecov",
"test": "npm run build && jest",
"test-coverage": "jest --coverage --globals \"{\\\"coverage\\\":true}\" && codecov"
Expand All @@ -14,7 +15,7 @@
"dist/*"
],
"devDependencies": {
"@zeit/ncc": "^0.17.0",
"@zeit/ncc": "^0.20.4",
"acorn": "^6.1.1",
"acorn-stage3": "^2.0.0",
"bindings": "^1.4.0",
Expand Down
2 changes: 2 additions & 0 deletions readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ Any `.node` files included will also support binary relocation.
// defined that should not be emitted
existingAssetNames: []
wrapperCompatibility: false, // optional, default
writeMode: false // optional, default
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

"writeMode" makes the option sound like you're switching from "read mode" to "write mode".

Let's rename this to be writeSupport or relocateWrites.

relativeToSource: false // optional, default
// build for process.env.NODE_ENV = 'production'
production: true, // optional, default is undefined
cwd: process.cwd(), // optional, default
Expand Down
66 changes: 56 additions & 10 deletions src/asset-relocator.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ const handleSpecialCases = require('./utils/special-cases');
const { getOptions } = require("loader-utils");
const resolve = require('resolve');
const stage3 = require('acorn-stage3');
const mergeSourceMaps = require('./utils/merge-source-maps');
acorn = acorn.Parser.extend(stage3);
const os = require('os');

Expand Down Expand Up @@ -152,6 +151,17 @@ const fsSymbols = {
stat: FS_FN,
statSync: FS_FN
};
const fsWriteSymbols = {
appendFile: FS_FN,
appendFileSync: FS_FN,
copyFile: FS_FN,
copyFileSync: FS_FN,
write: FS_FN,
mkdir: FS_FN,
mkdirSync: FS_FN,
writeFile: FS_FN,
writeFileSync: FS_FN
};
const staticModules = Object.assign(Object.create(null), {
bindings: {
default: BINDINGS
Expand Down Expand Up @@ -280,7 +290,7 @@ function generateWildcardRequire(dir, wildcardPath, wildcardParam, wildcardBlock
}

const hooked = new WeakSet();
function injectPathHook (compilation, outputAssetBase) {
function injectPathHook (compilation, outputAssetBase, relativeToSource) {
const { mainTemplate } = compilation;
if (!hooked.has(mainTemplate)) {
hooked.add(mainTemplate);
Expand All @@ -292,7 +302,25 @@ function injectPathHook (compilation, outputAssetBase) {
if (relBase.length)
relBase = '/' + relBase;
}
return `${source}\n${mainTemplate.requireFn}.ab = __dirname + ${JSON.stringify(relBase + '/' + assetBase(outputAssetBase))};`;

let relativeBase = ''
if(relativeToSource) {
for (let [_, value] of chunk._modules.entries()) {
if(value.rawRequest) {
// Gets relative path from source to output so it can be used for output operations
const outPath = mainTemplate.outputOptions.path === '/' ? value.context : mainTemplate.outputOptions.path
const outDir = path.dirname(path.join(outPath, chunk.name))
let relativePath
relativePath = path.relative(outDir, path.dirname(value.rawRequest))
if (relativePath.length)
relativePath = '/' + relativePath;

// Used for replacing __dirname
relativeBase = `\n__relativeToSource = __dirname${relativePath !== '' ? `+ ${JSON.stringify(relativePath)}` : ''};`
}
}
}
return `${source}\n${mainTemplate.requireFn}.ab = __dirname + ${JSON.stringify(relBase + '/' + assetBase(outputAssetBase))};${relativeBase}`;
});
}
}
Expand All @@ -307,7 +335,7 @@ module.exports = async function (content, map) {
// injection to set __webpack_require__.ab
const options = getOptions(this);

injectPathHook(this._compilation, options.outputAssetBase);
injectPathHook(this._compilation, options.outputAssetBase, options.relativeToSource);
Copy link
Member

@Timer Timer Sep 10, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure I understand the behavior for this option -- if you set relativeToSource: true it seems to rewrite both reads and writes to be relative to source? This almost seems like it'd never be what you want.

Wouldn't typical behavior be (for this option) to want to rewrite reads but leave writes in their original location? I guess this wouldn't work for an app that reads-its-own-writes. 🤔

Also, can you please point me to a library that needs write support? I've brought it up a couple times, but I'd rather land this in Next.js without write support first.


if (id.endsWith('.node')) {
const assetState = getAssetState(options, this._compilation);
Expand Down Expand Up @@ -578,7 +606,7 @@ module.exports = async function (content, map) {
const result = evaluate(expr, vars, computeBranches);
return result;
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change

// statically determinable leaves are tracked, and inlined when the
// greatest parent statically known leaf computation corresponds to an asset path
let staticChildNode, staticChildValue;
Expand Down Expand Up @@ -614,10 +642,15 @@ module.exports = async function (content, map) {
}

if (staticChildNode)
return backtrack(this, parent);

if (node.type === 'Identifier') {
if (isIdentifierRead(node, parent)) {
return backtrack(this, parent);
if (node.type === 'Identifier') {
// Replace in relativeToSource all __dirname with actual source __dirname
if(options.relativeToSource && node.name === '__dirname') {
if(node.start && node.end) {
magicString.overwrite(node.start, node.end, '__relativeToSource');
transformed = true;
}
} else if (isIdentifierRead(node, parent)) {
let binding;
// detect asset leaf expression triggers (if not already)
// __dirname, __filename
Expand Down Expand Up @@ -804,6 +837,11 @@ module.exports = async function (content, map) {
// if we have a direct pure static function,
// and that function has a [TRIGGER] symbol -> trigger asset emission from it
if (calleeValue && typeof calleeValue.value === 'function' && calleeValue.value[TRIGGER]) {
// Skip static analysis for write operations -> to replace for example __dirname
// It's not possible analyze not existing files
if(options.writeMode && parent && parent.callee && parent.callee.property && fsWriteSymbols[parent.callee.property.name]) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you absolutely sure this code path ever gets hit? I would have thought it would have caught typeof caleeValue.value === 'symbol' given the use of FS_FN?

Note that an easy way to do the write detection in that case is just to have a FS_WRITE_FN symbol detected in the switch statement a few lines down.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It hits this path, I added a test to cover this use case. It's function, I tested it on different cases and it was always function.

return
}
staticChildValue = computePureStaticValue(node, true).result;
// if it computes, then we start backtrackingelse
if (staticChildValue) {
Expand Down Expand Up @@ -920,7 +958,15 @@ module.exports = async function (content, map) {
if (computed && 'value' in computed) {
// var known = ...;
if (decl.id.type === 'Identifier') {
setKnownBinding(decl.id.name, computed.value);
// Replaces __dirname in relative to source mode
if(options.relativeToSource && decl.init.name === '__dirname') {
if(decl.init.start && decl.init.end) {
magicString.overwrite(decl.init.start, decl.init.end, '__relativeToSource');
transformed = true;
}
} else {
setKnownBinding(decl.id.name, computed.value);
}
}
// var { known } = ...;
else if (decl.id.type === 'ObjectPattern') {
Expand Down
4 changes: 2 additions & 2 deletions test/index.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ const MemoryFS = require("memory-fs");
jest.setTimeout(20000);

global._unit = true;

for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) {
it(`should generate correct output for ${unitTest}`, async () => {
// simple error test
Expand Down Expand Up @@ -51,7 +50,8 @@ for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) {
emitFilterAssetBaseAll: true,
wrapperCompatibility: true,
debugLog: true,
production: true
production: true,
writeMode: true
}
}]
}],
Expand Down
82 changes: 82 additions & 0 deletions test/relative-to-source.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
const fs = require('fs')
const path = require('path')
const webpack = require('webpack')
const MemoryFS = require('memory-fs')

jest.setTimeout(20000)

it(`should generate correct output for relative to source path`, async () => {
const dir = `${__dirname}/relative-to-source`
const entry = `${dir}/input.js`
const expected = fs
.readFileSync(`${dir}/output.js`)
.toString()
.trim()
// Windows support
.replace(/\r/g, '')

const mfs = new MemoryFS()
const compiler = webpack({
entry,
optimization: { nodeEnv: false, minimize: false },
mode: 'production',
target: 'node',
output: {
path: '/',
filename: 'index.js',
libraryTarget: 'commonjs2',
},
module: {
rules: [
{
test: /\.(js|mjs|node)$/,
parser: { amd: false },
use: [
{
loader:
__dirname + '/../src/asset-relocator',
options: {
relativeToSource: true,
writeMode: true
}
},
],
},
],
},
})

compiler.outputFileSystem = mfs

try {
var stats = await new Promise((resolve, reject) => {
compiler.run((err, stats) => {
if (err) return reject(err)
resolve(stats)
})
})
} catch (err) {
if (shouldError) return
throw err
}

let code
try {
code = mfs.readFileSync('/index.js', 'utf8')
} catch (e) {
throw new Error(stats.toString())
}

const actual = code
.trim()
// Windows support
.replace(/\r/g, '')

try {
expect(actual).toBe(expected)
} catch (e) {
// useful for updating fixtures
fs.writeFileSync(`${dir}/actual.js`, actual)
throw e
}
})
125 changes: 125 additions & 0 deletions test/relative-to-source/actual.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
module.exports =
/******/ (function(modules) { // webpackBootstrap
/******/ // The module cache
/******/ var installedModules = {};
/******/
/******/ // The require function
/******/ function __webpack_require__(moduleId) {
/******/
/******/ // Check if module is in cache
/******/ if(installedModules[moduleId]) {
/******/ return installedModules[moduleId].exports;
/******/ }
/******/ // Create a new module (and put it into the cache)
/******/ var module = installedModules[moduleId] = {
/******/ i: moduleId,
/******/ l: false,
/******/ exports: {}
/******/ };
/******/
/******/ // Execute the module function
/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__);
/******/
/******/ // Flag the module as loaded
/******/ module.l = true;
/******/
/******/ // Return the exports of the module
/******/ return module.exports;
/******/ }
/******/
/******/
/******/ // expose the modules object (__webpack_modules__)
/******/ __webpack_require__.m = modules;
/******/
/******/ // expose the module cache
/******/ __webpack_require__.c = installedModules;
/******/
/******/ // define getter function for harmony exports
/******/ __webpack_require__.d = function(exports, name, getter) {
/******/ if(!__webpack_require__.o(exports, name)) {
/******/ Object.defineProperty(exports, name, { enumerable: true, get: getter });
/******/ }
/******/ };
/******/
/******/ // define __esModule on exports
/******/ __webpack_require__.r = function(exports) {
/******/ if(typeof Symbol !== 'undefined' && Symbol.toStringTag) {
/******/ Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
/******/ }
/******/ Object.defineProperty(exports, '__esModule', { value: true });
/******/ };
/******/
/******/ // create a fake namespace object
/******/ // mode & 1: value is a module id, require it
/******/ // mode & 2: merge all properties of value into the ns
/******/ // mode & 4: return value when already ns object
/******/ // mode & 8|1: behave like require
/******/ __webpack_require__.t = function(value, mode) {
/******/ if(mode & 1) value = __webpack_require__(value);
/******/ if(mode & 8) return value;
/******/ if((mode & 4) && typeof value === 'object' && value && value.__esModule) return value;
/******/ var ns = Object.create(null);
/******/ __webpack_require__.r(ns);
/******/ Object.defineProperty(ns, 'default', { enumerable: true, value: value });
/******/ if(mode & 2 && typeof value != 'string') for(var key in value) __webpack_require__.d(ns, key, function(key) { return value[key]; }.bind(null, key));
/******/ return ns;
/******/ };
/******/
/******/ // getDefaultExport function for compatibility with non-harmony modules
/******/ __webpack_require__.n = function(module) {
/******/ var getter = module && module.__esModule ?
/******/ function getDefault() { return module['default']; } :
/******/ function getModuleExports() { return module; };
/******/ __webpack_require__.d(getter, 'a', getter);
/******/ return getter;
/******/ };
/******/
/******/ // Object.prototype.hasOwnProperty.call
/******/ __webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); };
/******/
/******/ // __webpack_public_path__
/******/ __webpack_require__.p = "";
/******/ __webpack_require__.ab = __dirname + "/";
/******/ __relativeToSource = __dirname;
/******/
/******/
/******/ // Load entry module and return exports
/******/ return __webpack_require__(__webpack_require__.s = 0);
/******/ })
/************************************************************************/
/******/ ([
/* 0 */
/***/ (function(module, exports, __webpack_require__) {

/* WEBPACK VAR INJECTION */(function(__dirname) {const fs = __webpack_require__(1)
const path = __webpack_require__(2)


fs.writeFileSync('./test.js', 'Test')

fs.writeFileSync('/tmp/test.js', 'Test')

fs.writeFileSync(__relativeToSource + 'test.js', 'Test')

fs.writeFileSync(path.resolve(__dirname, 'test.js'), 'Test')

const _basePath = __relativeToSource
const asset3 = 'asset3.txt';

fs.writeFileSync(_basePath + '/' + asset3, 'Test');
/* WEBPACK VAR INJECTION */}.call(this, "/"))

/***/ }),
/* 1 */
/***/ (function(module, exports) {

module.exports = require("fs");

/***/ }),
/* 2 */
/***/ (function(module, exports) {

module.exports = require("path");

/***/ })
/******/ ]);
Loading