emit ".node" binaries that are required

.circleci/config.yml

@@ -9,11 +9,14 @@ jobs:
       - run:
           name: Installing dependencies
           command: yarn install
+      - run:
+          name: Build unit test binary
+          command: yarn build-test-binary
       # - run:
       #     name: Linting
       #     command: yarn lint
       - run:
-          name: Integration tests
+          name: Unit & integration tests
           command: yarn test
       - run:
           name: Coverage

dist/ncc/loaders/readme.md

@@ -0,0 +1,10 @@
+# About this directory
+
+This directory will contain:
+
+- `node-loader.js` the ncc loader for supporting Node binary requires
+- `relocate-loader.js` the ncc loader for handling CommonJS asset and reference relocations
+
+These are generated by the `build` step defined in `../../../package.json`.
+
+These files are published to npm.

package.json

@@ -9,6 +9,7 @@
   },
   "scripts": {
     "build": "node scripts/build",
+    "build-test-binary": "cd test/binary && node-gyp rebuild && cp build/Release/hello.node ../integration/hello.node",
     "codecov": "codecov",
     "test": "npm run build && jest",
     "test-coverage": "jest --coverage --globals \"{\\\"coverage\\\":true}\" && codecov",
@@ -39,13 +40,16 @@
     "jest": "^23.6.0",
     "jimp": "^0.5.6",
     "koa": "^2.6.2",
+    "loader-utils": "^1.1.0",
     "magic-string": "^0.25.1",
     "mailgun": "^0.5.0",
     "mariadb": "^2.0.1-beta",
     "memcached": "^2.2.2",
     "mkdirp": "^0.5.1",
     "mongoose": "^5.3.12",
     "mysql": "^2.16.0",
+    "node-gyp": "^3.8.0",
+    "node-native-loader": "^1.1.1",
     "passport": "^0.4.0",
     "passport-google-oauth": "^1.0.0",
     "path-platform": "^0.11.15",

scripts/build.js

@@ -15,18 +15,21 @@ async function main() {
     // to bundle it. even if we did want watching and a bigger
     // bundle, webpack (and therefore ncc) cannot currently bundle
     // chokidar, which is quite convenient
-    externals: ["chokidar", "./asset-relocator.js"]
+    externals: ["chokidar", "./relocate-loader.js"]
   });
 
-  const { code: assetRelocator, assets: assetRelocatorAssets } = await ncc(__dirname + "/../src/asset-relocator");
+  const { code: relocateLoader, assets: relocateLoaderAssets } = await ncc(__dirname + "/../src/loaders/relocate-loader");
+  const { code: nodeLoader, assets: nodeLoaderAssets } = await ncc(__dirname + "/../src/loaders/node-loader");
 
-  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length || Object.keys(assetRelocatorAssets).length) {
+  if (Object.keys(cliAssets).length || Object.keys(indexAssets).length ||
+      Object.keys(relocateLoaderAssets).length || Object.keys(nodeLoaderAssets).length) {
     console.error('Assets emitted by core build, these need to be written into the dist directory');
   }
 
   writeFileSync(__dirname + "/../dist/ncc/cli.js", cli);
   writeFileSync(__dirname + "/../dist/ncc/index.js", index);
-  writeFileSync(__dirname + "/../dist/ncc/asset-relocator.js", assetRelocator);
+  writeFileSync(__dirname + "/../dist/ncc/loaders/relocate-loader.js", relocateLoader);
+  writeFileSync(__dirname + "/../dist/ncc/loaders/node-loader.js", nodeLoader);
 
   // copy webpack buildin
   await copy(

src/index.js

@@ -18,7 +18,7 @@ WebpackParser.parse = function (source, opts = {}) {
   });
 }
 
-const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json"];
+const SUPPORTED_EXTENSIONS = [".js", ".mjs", ".json", ".node"];
 
 function resolveModule(context, request, callback, forcedExternals = []) {
   const resolveOptions = {
@@ -46,6 +46,7 @@ function resolveModule(context, request, callback, forcedExternals = []) {
 
 module.exports = async (entry, { externals = [], minify = true, sourceMap = false } = {}) => {
   const mfs = new MemoryFS();
+  const assetNames = Object.create(null);
   const compiler = webpack({
     entry,
     optimization: {
@@ -75,8 +76,18 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals
           parser: { amd: false }
         },
         {
-          test: /\.(js|mjs)/,
-          use: [{ loader: __dirname + "/asset-relocator.js" }]
+          test: /\.(js|mjs)$/,
+          use: [{
+            loader: __dirname + "/loaders/relocate-loader.js",
+            options: { assetNames }
+          }]
+        },
+        {
+          test: /\.node$/,
+          use: [{
+            loader: __dirname + "/loaders/node-loader.js",
+            options: { assetNames }
+          }]
         }
       ]
     },

src/asset-relocator.js → src/loaders/relocate-loader.js

@@ -5,6 +5,8 @@ const MagicString = require('magic-string');
 const { attachScopes } = require('rollup-pluginutils');
 const evaluate = require('static-eval');
 const acorn = require('acorn');
+const getUniqueAssetName = require('../utils/dedupe-names');
+const { getOptions } = require('loader-utils');
 
 // Very basic first-pass fs.readFileSync inlining
 function isReference(node, parent) {
@@ -29,20 +31,16 @@ module.exports = function (code) {
   if (id.endsWith('.json') || !code.match(assetRegEx))
     return this.callback(null, code);
 
-  const assetNames = Object.create(null);
+  const options = getOptions(this);
   const emitAsset = (assetPath) => {
     // JS assets to support require(assetPath) and not fs-based handling
     // NB package.json is ambiguous here...
     if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs'))
       return;
 
+    const name = getUniqueAssetName(assetPath, options.assetNames);
+
     // console.log('Emitting ' + assetPath + ' for module ' + id);
-    const basename = path.basename(assetPath);
-    const ext = path.extname(basename);
-    let name = basename, i = 0;
-    while (name in assetNames && assetNames[name] !== assetPath)
-      name = basename.substr(0, basename.length - ext.length) + ++i + ext;
-    assetNames[name] = assetPath;
 
     this.emitFile(name, fs.readFileSync(assetPath));
     return "__dirname + '/" + JSON.stringify(name).slice(1, -1) + "'";
@@ -148,7 +146,7 @@ module.exports = function (code) {
       // detect asset leaf expression triggers (if not already)
       // __dirname and __filename only currently
       // Can add require.resolve, import.meta.url, even path-like environment variables
-      if (node.type === 'Identifier' && isReference(node, parent)) {
+      else if (node.type === 'Identifier' && isReference(node, parent)) {
         if (!shadowDepths[node.name] &&
             (node.name === '__dirname' || node.name === '__filename')) {
           curStaticValue = computeStaticValue(node, id);
@@ -164,7 +162,7 @@ module.exports = function (code) {
       // so "var { join } = require('path')" will only detect in the top scope.
       // Intermediate scope handling for these requires is straightforward, but
       // would need nested shadow depth handling of the pathIds.
-      if (parent === ast && node.type === 'VariableDeclaration') {
+      else if (parent === ast && node.type === 'VariableDeclaration') {
         for (const decl of node.declarations) {
           // var path = require('path')
           if (decl.id.type === 'Identifier' &&

src/utils/dedupe-names.js

@@ -0,0 +1,11 @@
+const path = require("path");
+
+module.exports = function (assetPath, assetNames) {
+  const basename = path.basename(assetPath);
+  const ext = path.extname(basename);
+  let name = basename, i = 0;
+  while (name in assetNames && assetNames[name] !== assetPath)
+    name = basename.substr(0, basename.length - ext.length) + ++i + ext;
+  assetNames[name] = assetPath;
+  return name;
+};

test/binary/binding.gyp

@@ -0,0 +1,11 @@
+{
+  "targets": [
+    {
+      "target_name": "hello",
+      "sources": [ "hello.cc" ],
+      "include_dirs": [
+        "<!(node -e \"require('nan')\")"
+      ]
+    }
+  ]
+}

test/binary/hello.cc

@@ -0,0 +1,26 @@
+// hello.cc
+#include <node.h>
+
+namespace demo {
+
+using v8::FunctionCallbackInfo;
+using v8::Isolate;
+using v8::Local;
+using v8::NewStringType;
+using v8::Object;
+using v8::String;
+using v8::Value;
+
+void Method(const FunctionCallbackInfo<Value>& args) {
+  Isolate* isolate = args.GetIsolate();
+  args.GetReturnValue().Set(String::NewFromUtf8(
+      isolate, "world", NewStringType::kNormal).ToLocalChecked());
+}
+
+void Initialize(Local<Object> exports) {
+  NODE_SET_METHOD(exports, "hello", Method);
+}
+
+NODE_MODULE(NODE_GYP_MODULE_NAME, Initialize)
+
+}  // namespace demo

test/index.test.js

@@ -73,7 +73,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
       mkdirp.sync(dirname(assetPath));
       fs.writeFileSync(assetPath, assets[asset]);
     }
-    (__dirname => {
+    ((__dirname, require) => {
       try {
         eval(`${code}\n//# sourceURL=${integrationTest}`);
       }
@@ -83,7 +83,7 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
         fs.writeFileSync(__dirname + "/index.js", code);
         throw e;
       }
-    })(__dirname + "/tmp");
+    })(__dirname + "/tmp", id => require(id.startsWith('./') ? './tmp/' + id.substr(2) : id));
     if ("function" !== typeof module.exports) {
       throw new Error(
         `Integration test "${integrationTest}" evaluation failed. It does not export a function`

test/integration/binary-require.js

@@ -0,0 +1,6 @@
+const binary = require('./hello.node');
+const assert = require('assert');
+
+module.exports = () => {
+  assert.equal(binary.hello(), 'world');
+};