Add experimental remote MDX support to esbuild

Reviewed-by: Christian Murphy <[email protected]>

Closes GH-66.

lib/integration/esbuild.js

@@ -2,127 +2,206 @@
  * @typedef {import('esbuild').Plugin} Plugin
  * @typedef {import('esbuild').PluginBuild} PluginBuild
  * @typedef {import('esbuild').OnLoadArgs} OnLoadArgs
+ * @typedef {import('esbuild').OnLoadResult} OnLoadResult
+ * @typedef {import('esbuild').OnResolveArgs} OnResolveArgs
  * @typedef {import('esbuild').Message} Message
  * @typedef {import('vfile').VFileContents} VFileContents
  * @typedef {import('vfile-message').VFileMessage} VFileMessage
  * @typedef {import('unist').Point} Point
  * @typedef {import('../core.js').ProcessorOptions} ProcessorOptions
+ *
+ * @typedef {ProcessorOptions & {allowDangerousRemoteMdx?: boolean}} Options
  */
 
 import {promises as fs} from 'fs'
+import got from 'got'
 import vfile from 'vfile'
 import {createFormatAwareProcessors} from '../util/create-format-aware-processors.js'
 import {extnamesToRegex} from '../util/extnames-to-regex.js'
 
 const eol = /\r\n|\r|\n|\u2028|\u2029/g
 
+/** @type Map<string, string> */
+const cache = new Map()
+
+const p = process
+
 /**
  * Compile MDX w/ esbuild.
  *
- * @param {ProcessorOptions} [options]
+ * @param {Options} [options]
  * @return {Plugin}
  */
-export function esbuild(options) {
+export function esbuild(options = {}) {
+  const {allowDangerousRemoteMdx, ...rest} = options
   const name = 'esbuild-xdm'
-  const {extnames, process} = createFormatAwareProcessors(options)
+  const remoteNamespace = name + '-remote'
+  const {extnames, process} = createFormatAwareProcessors(rest)
 
   return {name, setup}
 
   /**
    * @param {PluginBuild} build
    */
   function setup(build) {
-    build.onLoad({filter: extnamesToRegex(extnames)}, onload)
-  }
+    const filter = extnamesToRegex(extnames)
+    /* eslint-disable-next-line security/detect-non-literal-regexp */
+    const filterHttp = new RegExp('^https?:\\/{2}.+' + filter.source)
+    const filterHttpOrRelative = /^(https?:\/{2}|.{1,2}\/).*/
 
-  /**
-   * @param {Omit.<OnLoadArgs, 'pluginData'> & {pluginData?: {contents?: string}}} data
-   */
-  async function onload(data) {
-    /** @type {string} */
-    const doc =
-      data.pluginData && data.pluginData.contents !== undefined
-        ? data.pluginData.contents
-        : String(await fs.readFile(data.path))
-
-    let file = vfile({contents: doc, path: data.path})
-    /** @type {VFileMessage[]} */
-    let messages = []
-    /** @type {Message[]} */
-    const errors = []
-    /** @type {Message[]} */
-    const warnings = []
-    /** @type {VFileContents} */
-    let contents
-    /** @type {VFileMessage} */
-    let message
-    /** @type {Point} */
-    let start
-    /** @type {Point} */
-    let end
-    /** @type {number} */
-    let length
-    /** @type {number} */
-    let lineStart
-    /** @type {number} */
-    let lineEnd
-    /** @type {RegExpExecArray} */
-    let match
-    /** @type {number} */
-    let line
-    /** @type {number} */
-    let column
-
-    try {
-      file = await process(file)
-      contents = file.contents
-      messages = file.messages
-    } catch (error) {
-      error.fatal = true
-      messages.push(error)
+    if (allowDangerousRemoteMdx) {
+      // Intercept import paths starting with "http:" and "https:" so
+      // esbuild doesn't attempt to map them to a file system location.
+      // Tag them with the "http-url" namespace to associate them with
+      // this plugin.
+      build.onResolve(
+        {filter: filterHttp, namespace: 'file'},
+        resolveRemoteInLocal
+      )
+
+      build.onResolve(
+        {filter: filterHttpOrRelative, namespace: remoteNamespace},
+        resolveInRemote
+      )
     }
 
-    for (message of messages) {
-      start = message.location.start
-      end = message.location.end
-      length = 0
-      lineStart = 0
-      line = undefined
-      column = undefined
-
-      if (start.line != null && start.column != null && start.offset != null) {
-        line = start.line
-        column = start.column - 1
-        lineStart = start.offset - column
-        length = 1
-
-        if (end.line != null && end.column != null && end.offset != null) {
-          length = end.offset - start.offset
-        }
+    build.onLoad({filter: /.*/, namespace: remoteNamespace}, onloadremote)
+    build.onLoad({filter}, onload)
+
+    /** @param {OnResolveArgs} args  */
+    function resolveRemoteInLocal(args) {
+      return {path: args.path, namespace: remoteNamespace}
+    }
+
+    // Intercept all import paths inside downloaded files and resolve them against
+    // the original URL. All of these
+    // files will be in the "http-url" namespace. Make sure to keep
+    // the newly resolved URL in the "http-url" namespace so imports
+    // inside it will also be resolved as URLs recursively.
+    /** @param {OnResolveArgs} args  */
+    function resolveInRemote(args) {
+      return {
+        path: String(new URL(args.path, args.importer)),
+        namespace: remoteNamespace
       }
+    }
 
-      eol.lastIndex = lineStart
-      match = eol.exec(doc)
-      lineEnd = match ? match.index : doc.length
-      ;(message.fatal ? errors : warnings).push({
-        pluginName: name,
-        text: message.reason,
-        notes: [],
-        location: {
-          namespace: 'file',
-          suggestion: '',
-          file: data.path,
-          line,
-          column,
-          length: Math.min(length, lineEnd),
-          lineText: doc.slice(lineStart, lineEnd)
-        },
-        detail: message
-      })
+    /**
+     * @param {OnLoadArgs} data
+     * @returns {Promise<OnLoadResult>}
+     */
+    async function onloadremote(data) {
+      const href = data.path
+      console.log('%s: downloading `%s`', remoteNamespace, href)
+      const contents = (await got(href, {cache})).body
+
+      return filter.test(href)
+        ? onload({
+            // Clean search and hash from URL.
+            path: Object.assign(new URL(href), {search: '', hash: ''}).href,
+            namespace: 'file',
+            pluginData: {contents}
+          })
+        : // V8 on Erbium.
+          /* c8 ignore next 2 */
+          {contents, loader: 'js', resolveDir: p.cwd()}
     }
 
-    // V8 on Erbium.
-    /* c8 ignore next 2 */
-    return {contents, errors, warnings}
+    /**
+     * @param {Omit.<OnLoadArgs, 'pluginData'> & {pluginData?: {contents?: string|Uint8Array}}} data
+     * @returns {Promise<OnLoadResult>}
+     */
+    async function onload(data) {
+      /** @type {string} */
+      const doc = String(
+        data.pluginData && data.pluginData.contents !== undefined
+          ? data.pluginData.contents
+          : await fs.readFile(data.path)
+      )
+
+      let file = vfile({contents: doc, path: data.path})
+      /** @type {VFileMessage[]} */
+      let messages = []
+      /** @type {Message[]} */
+      const errors = []
+      /** @type {Message[]} */
+      const warnings = []
+      /** @type {VFileContents} */
+      let contents
+      /** @type {VFileMessage} */
+      let message
+      /** @type {Point} */
+      let start
+      /** @type {Point} */
+      let end
+      /** @type {number} */
+      let length
+      /** @type {number} */
+      let lineStart
+      /** @type {number} */
+      let lineEnd
+      /** @type {RegExpExecArray} */
+      let match
+      /** @type {number} */
+      let line
+      /** @type {number} */
+      let column
+
+      try {
+        file = await process(file)
+        contents = file.contents
+        messages = file.messages
+      } catch (error) {
+        error.fatal = true
+        messages.push(error)
+      }
+
+      for (message of messages) {
+        start = message.location.start
+        end = message.location.end
+        length = 0
+        lineStart = 0
+        line = undefined
+        column = undefined
+
+        if (
+          start.line != null &&
+          start.column != null &&
+          start.offset != null
+        ) {
+          line = start.line
+          column = start.column - 1
+          lineStart = start.offset - column
+          length = 1
+
+          if (end.line != null && end.column != null && end.offset != null) {
+            length = end.offset - start.offset
+          }
+        }
+
+        eol.lastIndex = lineStart
+        match = eol.exec(doc)
+        lineEnd = match ? match.index : doc.length
+        ;(message.fatal ? errors : warnings).push({
+          pluginName: name,
+          text: message.reason,
+          notes: [],
+          location: {
+            namespace: 'file',
+            suggestion: '',
+            file: data.path,
+            line,
+            column,
+            length: Math.min(length, lineEnd),
+            lineText: doc.slice(lineStart, lineEnd)
+          },
+          detail: message
+        })
+      }
+
+      // V8 on Erbium.
+      /* c8 ignore next 2 */
+      return {contents, errors, warnings, resolveDir: p.cwd()}
+    }
   }
 }

lib/plugin/remark-mark-and-unravel.js

@@ -37,19 +37,28 @@ function onvisit(node, index, parent) {
   /** @type {Array.<Node>} */
   let children
 
-  if (parent && node.type === 'paragraph' && Array.isArray(node.children)) {
+  if (
+    parent &&
+    node.type === 'paragraph' &&
+    // @ts-expect-error: hush.
+    Array.isArray(node.children)
+  ) {
+    // @ts-expect-error: hush.
     // type-coverage:ignore-next-line
     children = node.children
 
     while (++offset < children.length) {
+      const child = children[offset]
+
       if (
-        children[offset].type === 'mdxJsxTextElement' ||
-        children[offset].type === 'mdxTextExpression'
+        child.type === 'mdxJsxTextElement' ||
+        child.type === 'mdxTextExpression'
       ) {
         oneOrMore = true
       } else if (
-        children[offset].type === 'text' &&
-        /^[\t\r\n ]+$/.test(String(children[offset].value))
+        child.type === 'text' &&
+        // @ts-expect-error: hush.
+        /^[\t\r\n ]+$/.test(String(child.value))
       ) {
         // Empty.
       } else {

lib/util/extnames-to-regex.js

@@ -6,5 +6,7 @@
  */
 export function extnamesToRegex(extnames) {
   // eslint-disable-next-line security/detect-non-literal-regexp
-  return new RegExp('\\.(' + extnames.map((d) => d.slice(1)).join('|') + ')$')
+  return new RegExp(
+    '\\.(' + extnames.map((d) => d.slice(1)).join('|') + ')([?#]|$)'
+  )
 }

package.json

@@ -43,6 +43,7 @@
     "estree-util-build-jsx": "^2.0.0",
     "estree-util-is-identifier-name": "^2.0.0",
     "estree-walker": "^3.0.0",
+    "got": "^11.0.0",
     "hast-util-to-estree": "^2.0.0",
     "loader-utils": "^2.0.0",
     "markdown-extensions": "^1.0.0",