nodeLoader is undefined - global.nodeLoader.setImportMapPromise(Promise.resolve(nodeImportMap))

[rgallego87]

Hello, I would like to address a couple of issues I'm encountering during the implementation of an important project. It's a microfrontends project aiming to use SSR (Server-Side Rendering), and for this purpose, I've drawn inspiration from the isomorphic-microfrontends example.

Describe the bug or question
One of the main issues I'm facing is that when using:
tsx watch --clear-screen=false --experimental-network-imports --experimental-loader @node-loader/core server/server.ts
global.nodeLoader.setImportMapPromise(Promise.resolve(nodeImportMap)); <-- nodeLoader is undefined and I believe it's crucial for the rest of the flow to work, namely the import of the apps within the renderApplication and other aspects.

To Reproduce
tsx watch --clear-screen=false --experimental-network-imports --experimental-loader @node-loader/core server/server.ts and when you invoke localhost:9000 it fails with:

TypeError: Cannot read properties of undefined (reading 'setImportMapPromise')

Main Context
Node.js Version: 20.3.1
App inspired by: isomorphic-microfrontends example

Root-app:

• package.json

{
  "type": "module",
  "scripts": {    
    "develop": "cross-env NODE_ENV='development' concurrently -n w: 'npm run develop:*'",
    "develop:node": "tsx watch --clear-screen=false --experimental-network-imports --experimental-loader @node-loader/core server/server.ts",
    "develop:webpack": "webpack-dev-server --mode=development --port 9876 --config webpack.config.cjs",
    "start:node": "node --experimental-network-imports --experimental-loader @node-loader/core --loader tsx server/server.ts",    
    "prettier": "prettier --write './**'",
    "start": "webpack serve --port 9000 --env isLocal",
    "lint": "eslint src --ext ts,js,ts,tsx",
    "test": "cross-env BABEL_ENV=test jest --passWithNoTests",
    "format": "prettier --write .",
    "check-format": "prettier --check .",
    "prepare": "husky install",
    "build": "concurrently npm:build:*",
    "build:webpack": "webpack --mode=production --config webpack.config.cjs",
    "build:types": "tsc"
  },
  "husky": {
    "hooks": {
      "pre-commit": "pretty-quick --staged && eslint browser server"
    }
  },
  "devDependencies": {    
    "clean-webpack-plugin": "^4.0.0",
    "@babel/core": "^7.23.3",
    "@babel/eslint-parser": "^7.23.3",
    "@babel/plugin-transform-runtime": "^7.23.4",
    "@babel/preset-env": "^7.23.3",
    "@babel/preset-typescript": "^7.23.3",
    "@babel/runtime": "^7.23.4",
    "concurrently": "^8.2.2",
    "cross-env": "^7.0.3",
    "eslint": "^8.54.0",
    "eslint-config-prettier": "^9.0.0",
    "eslint-config-ts-important-stuff": "^1.1.0",
    "eslint-plugin-prettier": "^5.0.1",
    "html-webpack-plugin": "^5.5.3",
    "husky": "^8.0.3",
    "jest": "^29.7.0",
    "jest-cli": "^29.7.0",
    "prettier": "^3.1.0",
    "pretty-quick": "^3.1.3",
    "serve": "^14.2.1",
    "ts-config-single-spa": "^3.0.0",
    "typescript": "^5.3.2",
    "webpack": "^5.89.0",
    "webpack-cli": "^5.1.4",
    "webpack-config-single-spa-ts": "^4.1.3",
    "webpack-dev-server": "^4.15.1",
    "webpack-merge": "^5.10.0",
    "@types/systemjs": "^6.13.5",
    "babel-loader": "^9.1.3",
    "eslint-config-important-stuff": "^1.1.0",
    "eslint-config-node-important-stuff": "^2.0.0",
    "nodemon": "^3.0.1",
    "tsx": "^4.6.0",
    "@types/node": "^20.10.0",    
    "ts-loader": "^9.5.1"
  },
  "dependencies": {    
    "@node-loader/core": "^2.0.0",
    "@node-loader/http": "^2.0.0",
    "@node-loader/import-maps": "^1.1.0",
    "source-map-loader": "^4.0.1",
    "ejs": "^3.1.9",
    "express": "^4.18.2",
    "import-map-overrides": "^3.1.1",
    "merge2": "^1.4.1",
    "morgan": "^1.10.0",
    "node-fetch": "^3.3.2",
    "parse5": "^7.1.2",
    "single-spa-web-server-utils": "^2.3.1",
    "@types/jest": "^29.5.10",    
    "@types/webpack-env": "^1.18.4",
    "single-spa": "^5.9.5",
    "single-spa-layout": "^2.2.0"
  },
  "types": "dist/root-config.d.ts"
}

• tsconfig.json

{
  "extends": "ts-config-single-spa",
  "files": ["browser/root-config.ts"],
  "compilerOptions": {
    "strict": false,
    "declaration": true,
    "declarationDir": "dist",
    "target": "es5",
    "module": "esnext",
    "lib": ["dom", "es2017"],
    "jsx": "react",
    "esModuleInterop": true,
    "allowSyntheticDefaultImports": true,
    "sourceMap": true,
    "noImplicitAny": false,
    "skipLibCheck": true,
    "forceConsistentCasingInFileNames": true,
    "allowJs": true,
    "checkJs": false,    
    "moduleResolution": "node",
    "outDir": "./dist",
    "baseUrl": ".",
    "paths": {
      "*": [
        "node_modules/*",
        "browser/*",
        "server/*"    
      ]
    }    
  },
  "include": ["browser/**/*", "server/**/*", "src/**/*", "browser/root-config.ts"],
  "exclude": ["src/**/*.test*"]
}

• webpack.config.cjs

const path = require("path");
const { CleanWebpackPlugin } = require("clean-webpack-plugin");

module.exports = {
  mode: "development",
  devtool: "source-map",
  target: "node",
  entry: path.resolve(__dirname, "browser/root-config.ts"),
  output: {
    filename: "root-config.js",
    libraryTarget: "system",
    path: path.resolve(__dirname, "dist"),
  },
  module: {    
    rules: [      
      {
        parser: { system: false },
        use: [
          {
            loader: "source-map-loader"
          },
          {            
            loader: "ts-loader",
            options: { 
              transpileOnly: true,
            },
          }    
        ],
        exclude: /node_modules/,
      },
    ],
  },
  resolve: {
    extensions: [".ts", ".tsx", ".js"],
    extensionAlias: {
      ".js": [".js", ".ts"],
      ".cjs": [".cjs", ".cts"],
      ".mjs": [".mjs", ".mts"]
    }
  },
  devServer: {
    historyApiFallback: true,
    headers: {
      "Access-Control-Allow-Origin": "*",
    },
    allowedHosts: 'all'
  },
  plugins: [new CleanWebpackPlugin()],  
  externals: ["single-spa", "mfe-auth-app"]
};

• node-loader.config.js

import * as importMapLoader from "@node-loader/import-maps";
import * as httpLoader from "@node-loader/http";

export default {
  loaders: [    
    importMapLoader, 
    httpLoader
  ],
};

• .babelrc

{
  "presets": [
    "@babel/preset-env",
    "@babel/preset-typescript"
  ],
  "plugins": [
    [
      "@babel/plugin-transform-runtime",
      {
        "useESModules": true,
        "regenerator": false
      }
    ]
  ]
}

• browser/root-config.ts

import { registerApplication, start } from "single-spa";
import {
  constructRoutes,
  constructApplications,
  constructLayoutEngine,
} from "single-spa-layout";

const data = {
  loaders: {},
  props: {
    gecoHeader: true,
    gecoFooter: true
  }
}

const routes = constructRoutes(document.querySelector("#single-spa-layout"), data);
const applications = constructApplications({
  routes,
  loadApp({ name }) {
    return System.import(name);
  },
});
const layoutEngine = constructLayoutEngine({ routes, applications }); // TODO Check

applications.forEach(registerApplication);

System.import('mfe-auth-app').then(() => {
  layoutEngine.activate(); // TODO Check
  start();
})

• server/views/index.html

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta charset="UTF-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
    <title></title>
    <meta
      name="importmap-type"
      content="systemjs-importmap"
      server-cookie
      server-only
    />
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/import-map-overrides.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/system.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/extras/amd.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/extras/named-exports.min.js"></script>
    <assets></assets>
  </head>
  <body>
    <template id="single-spa-layout">
      <single-spa-router>
        <application name="mfe-geco-app-header" props="gecoHeader"></application>      
        <route default>
          <div style="overflow-y: auto; overflow-x: none; height: 100%;">      
            <!-- <application name="mfe-main-app"></application>   -->  
            </div>
          </route>     
        <application name="mfe-geco-app-footer" props="gecoFooter"></application>
      </single-spa-router>
    </template>
    <fragment name="importmap"></fragment>
    <script>
      System.import("root-config");
    </script>
    <import-map-overrides-full
      show-when-local-storage="devtools"
      dev-libs
    ></import-map-overrides-full>
  </body>
</html>

• server/server.ts

import process from 'process';
import express from "express";
import path from "path";
import morgan from "morgan";
import { app } from "./app";
import "./static";
import "./index-html";

app.use(morgan("tiny"));
app.set("view engine", "ejs");
app.set("views", path.resolve(process.cwd(), "./server/views"));

app.use('/', express.static(path.join(process.cwd(), "./dist")))

const port = process.env.PORT || 9000;
app.listen(port);
console.log(`App is hosted at http://localhost/:${port}`);

• server/index-html.ts

import { app } from "./app";
import {
  constructServerLayout,
  sendLayoutHTTPResponse, // @ts-ignore
} from "single-spa-layout/server"; // @ts-ignore
import { applyOverrides, getOverridesFromCookies } from "import-map-overrides";

const serverLayout = constructServerLayout({
  filePath: "server/views/index.html",
});

app.use("*", (req, res, next) => {  
  const developmentMode = process.env.NODE_ENV === "development";
  // const importSuffix = developmentMode ? `?ts=${Date.now()}` : "";

  const fetchOriginalMap = {
    "imports": {
      "single-spa": "https://cdn.jsdelivr.net/npm/[email protected]/lib/system/single-spa.min.js",
      "mfe-auth-app": "//localhost:8500/base-mfe-auth-app.js",
      "mfe-main-app": "//localhost:4200/main.js",
      "mfe-geco-app-header": "//localhost:3000/index.js",        
      "mfe-geco-app-footer": "//localhost:3000/index.js",        
      "root-config": "//localhost:9000/root-config.js"
    }
  }
  const importMapsPromise = async function (originalMap) {
    const browserImportMap = applyOverrides(originalMap, getOverridesFromCookies(req));
    const nodeImportMap = JSON.parse(JSON.stringify(browserImportMap));
    return {
      browserImportMap,
      nodeImportMap,
    };
  }(fetchOriginalMap)  
  .then(({ nodeImportMap, browserImportMap }) => {
    console.log('[index-html.ts] >> importMapsPromise >> global >>', global);
    // global.nodeLoader.setImportMapPromise(Promise.resolve(nodeImportMap));       
    return { nodeImportMap, browserImportMap };
  });

  const fragments = {
    importmap: async () => {
      const { browserImportMap } = await importMapsPromise;
      return `<script type="systemjs-importmap">${JSON.stringify(
        browserImportMap,
        null,
        2
      )}</script>`;
    }
  };

  sendLayoutHTTPResponse({
    serverLayout,
    urlPath: req.originalUrl,
    res,
    async renderFragment(name) {      
      return await fragments[name]();
    },
    async renderApplication({ appName, propsPromise }) {
      await importMapsPromise;
      const [app, props] = await Promise.all([
        import(appName),
        propsPromise,
      ]);
      return app.serverRender(props);      

      // const htmlRaw = await fetch('');
      // const htmlDom = new JSDOM(await htmlRaw.text());      
      // return htmlDom.serialize();
      // return appName;
    },
    async retrieveApplicationHeaders({ appName, propsPromise }) {
      // await importMapsPromise;
      // const [app, props] = await Promise.all([
      //   import(appName),
      //   propsPromise,
      // ]);
      // console.log('[index-html.ts] >> retrieveApplicationHeaders >> app >>', app);
      // return app.getResponseHeaders(props);
      return {};
    },
    async retrieveProp(propName) {
      return {};
    },
    assembleFinalHeaders(allHeaders) {
      return Object.assign({}, Object.values(allHeaders));
    },
  })
    .then(next)
    .catch((err) => {
      console.error(err);
      res.status(500).send("A server error occurred");
    });
});

What could I do? Are there any examples like this with these updated versions? Any incompatibility I should be aware of? Any suggested changes or alterations?

Thank you in advance!

[MilanKovacic]

Hi, did you manage to solve the issue?

[J5]

While searching for a solution to this same issue I landed on this bug. A little more research and debugging and I've figured out the issue. Node 20 broke loaders that need to keep state. From my cursory investigation, Node 20 now loads loaders outside the main thread so global.nodeLoader is defined in one thread but not in the main thread. Apparently they are moving away from --experimentalLoader and --loader switched to using an --import switch and calling the init callback of the loader to get access to parameters to talk to the main thread. My solution for now is to back off to Node 18 but if I find some time I may want to tackle this bug and provide a more robust way to initialize the loaders. Nodejs issue for context - nodejs/loaders#147

[rgallego87]

While searching for a solution to this same issue I landed on this bug. A little more research and debugging and I've figured out the issue. Node 20 broke loaders that need to keep state. From my cursory investigation, Node 20 now loads loaders outside the main thread so global.nodeLoader is defined in one thread but not in the main thread. Apparently they are moving away from --experimentalLoader and --loader switched to using an --import switch and calling the init callback of the loader to get access to parameters to talk to the main thread. My solution for now is to back off to Node 18 but if I find some time I may want to tackle this bug and provide a more robust way to initialize the loaders. Nodejs issue for context - nodejs/loaders#147

That's right and for this I decided to change my strategy. I believe that Node.js does this for security reasons among others and seems there is no plan to change the address in the future. Now what I do is not import any module in that way, but instead process the HTML template directly from the backend with ExpressJS and a custom implementation. The template is served and used by the client, then the rest of mfes not need to be server side in my case.

[J5]

Even 18 doesn't work. I think the right approach here is to load the import maps in the main thread and build some communication channel to request the data in both userland (e.g. during the request) and in the loaders so we don't have to have duplicate pollers.

[J5]

@rgallego87 Waiting to be able to join the slack servers so I can discuss further. Though I have managed to fix a number of issues and have my child SPAs importing from the network, I have come to the same conclusion that arbitrary network imports aren't going to work for security reasons. My plan now is to move SSR down to the individual SPAs and have the layout simply inject the SSR rendered html.

At first they will all be rendered per usual through separate server instances but then I would like to make a single generic express server that can serve up the apps via s3 or similar so I don't have to deploy a new service every time I deploy a new SPA.

[joeldenning]

NodeJS has messed with the loaders API many times - it is no longer necessary to use @node-loader/core, since you can use loader chaining that is built into the latest versions of Node.

I recommend upgrading to the latest versions of Node and switching to --import flag rather than --experimental-loader or --loader