[docs] Use custom webpack loader for markdown

.eslintrc.js

@@ -229,6 +229,9 @@ module.exports = {
     {
       files: ['docs/pages/**/*.js'],
       rules: {
+        // The code is already coupled to webpack.
+        'import/no-webpack-loader-syntax': 'off',
+
         'react/prop-types': 'off',
       },
     },

docs/next.config.js

@@ -191,7 +191,8 @@ module.exports = {
 
     // We want to speed-up the build of pull requests.
     // For crowdin PRs we want to build all locales for testing.
-    if (process.env.PULL_REQUEST === 'true' && !l10nPRInNetlify && !vercelDeploy) {
+    // FIXME: Revert before merging
+    if (process.env.PULL_REQUEST === 'false' && !l10nPRInNetlify && !vercelDeploy) {
       // eslint-disable-next-line no-console
       console.log('Considering only English for SSR');
       traverse(pages, 'en');

docs/package.json

@@ -85,13 +85,11 @@
     "lodash": "^4.17.15",
     "lz-string": "^1.4.4",
     "markdown-to-jsx": "^7.0.0",
-    "marked": "^2.0.0",
     "material-ui-popup-state": "^1.4.1",
     "next": "^10.0.0",
     "notistack": "^1.0.0",
     "nprogress": "^0.2.0",
     "postcss": "^8.0.6",
-    "prismjs": "^1.17.1",
     "prop-types": "^15.7.2",
     "raw-loader": "^1.0.0",
     "react": "^17.0.1",

docs/packages/markdown/loader.js

@@ -0,0 +1,127 @@
+const { promises: fs, readFileSync } = require('fs');
+const path = require('path');
+const { notEnglishMarkdownRegExp, prepareMarkdown } = require('./parseMarkdown');
+
+/**
+ * @param {string} string
+ */
+function upperCaseFirst(string) {
+  return `${string[0].toUpperCase()}${string.slice(1)}`;
+}
+
+/**
+ * @param {string} key
+ * @example keyToJSIdentifier('index.md') === 'IndexMd'
+ * @example keyToJSIdentifier('index-ja.md') === 'IndexJaMd'
+ */
+function keyToJSIdentifier(key) {
+  const delimiter = /(\.|-)/;
+  return key
+    .split(delimiter)
+    .filter((part) => !delimiter.test(part))
+    .map(upperCaseFirst)
+    .join('');
+}
+
+/**
+ * @example findTranslatedVersions('/a/b/index.md') === ['index-en.md', 'index-ja.md', 'index.md']
+ * @param {string} englishFilepath An absolute path to the english version written in markdown (.md)
+ */
+async function findTranslatedVersions(englishFilepath) {
+  const filename = path.basename(englishFilepath, '.md');
+  const files = await fs.readdir(path.dirname(englishFilepath));
+
+  // Given: index.md
+  // Match: index-en.md, index-ja.md
+  // Don't Match: otherindex-en.md, index-eng.md, index-e.md, index.tsx
+  const translatedVersionRegExp = new RegExp(`^${filename}${notEnglishMarkdownRegExp.source}`);
+
+  return files
+    .filter((filepath) => {
+      return translatedVersionRegExp.test(filepath);
+    })
+    .concat(path.basename(englishFilepath));
+}
+
+/**
+ * @type {import('webpack').loader.Loader}
+ */
+module.exports = async function demoLoader() {
+  const rawKeys = await findTranslatedVersions(this.resourcePath);
+
+  // TODO: Remove requireRaw mock (needs work in prepareMarkdown)
+  const requireRaw = (key) => {
+    const filepath = path.join(path.dirname(this.resourcePath), key);
+    this.addDependency(filepath);
+    return readFileSync(filepath, { encoding: 'utf-8' });
+  };
+  requireRaw.keys = () => rawKeys;
+  const pageFilename = this.context.replace(this.rootContext, '').replace(/^\/src\/pages\//, '');
+  const { docs } = prepareMarkdown({ pageFilename, requireRaw });
+
+  const demos = {};
+  const demoKeys = [];
+  new Set(
+    docs.en.rendered
+      .filter((markdownOrComponentConfig) => {
+        return typeof markdownOrComponentConfig !== 'string' && markdownOrComponentConfig.demo;
+      })
+      .map((demoConfig) => {
+        return path.basename(demoConfig.demo);
+      }),
+  ).forEach((filename) => {
+    const demoName = `pages/${pageFilename}/${filename
+      .replace(/\.\//g, '')
+      .replace(/\.tsx/g, '.js')}`;
+
+    demos[demoName] = {
+      module: filename,
+      raw: requireRaw(filename),
+    };
+    demoKeys.push(filename);
+
+    try {
+      const moduleTS = filename.replace(/\.js$/, '.tsx');
+      const rawTS = requireRaw(moduleTS);
+
+      demos[demoName].moduleTS = moduleTS;
+      demos[demoName].rawTS = rawTS;
+      demoKeys.push(moduleTS);
+    } catch (error) {
+      // TS version of the demo doesn't exist. This is fine.
+    }
+  });
+
+  /**
+   * @param {string} key
+   */
+  function getRequireDemoIdentifier(key) {
+    return keyToJSIdentifier(key);
+  }
+
+  const transformed = `
+    ${demoKeys
+      .map((key) => {
+        return `import ${getRequireDemoIdentifier(key)} from './${key}';`;
+      })
+      .join('\n')}
+
+    export const docs = ${JSON.stringify(docs, null, 2)};
+    export const demos = ${JSON.stringify(demos, null, 2)};
+    export function requireDemo(module) {
+      return {
+        ${demoKeys
+          .map((key) => {
+            // TODO: Remove ES module interop once all demos are loaded via loader
+            // i.e. replace `{ default: ... }`  with `...`
+            return `'${key}': { default: ${getRequireDemoIdentifier(key)} }`;
+          })
+          .join(',\n')}
+      }[module];
+    }
+    requireDemo.keys = () => {
+      return ${JSON.stringify(demoKeys, null, 2)}
+    }`;
+
+  return transformed;
+};

docs/packages/markdown/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@material-ui/markdown",
+  "version": "0.1.0",
+  "private": true,
+  "type": "commonjs",
+  "main": "./parseMarkdown.js",
+  "exports": {
+    ".": "./parseMarkdown.js",
+    "./loader": "./loader.js",
+    "./prism": "./prism.js"
+  },
+  "dependencies": {
+    "lodash": "^4.17.15",
+    "marked": "^2.0.0",
+    "prismjs": "^1.17.1"
+  }
+}

docs/src/modules/utils/parseMarkdown.js → docs/packages/markdown/parseMarkdown.js

@@ -1,8 +1,10 @@
-import marked from 'marked';
-import { LANGUAGES_IN_PROGRESS } from 'docs/src/modules/constants';
-import kebabCase from 'lodash/kebabCase';
-import textToHash from 'docs/src/modules/utils/textToHash';
-import prism from 'docs/src/modules/utils/prism';
+const marked = require('marked');
+const kebabCase = require('lodash/kebabCase');
+const textToHash = require('./textToHash');
+const prism = require('./prism');
+
+// TODO: pass as argument
+const LANGUAGES_IN_PROGRESS = ['en', 'zh', 'ru', 'pt', 'es', 'fr', 'de', 'ja'];
 
 const headerRegExp = /---[\r\n]([\s\S]*)[\r\n]---/;
 const titleRegExp = /# (.*)[\r\n]/;
@@ -25,7 +27,7 @@ const notEnglishMarkdownRegExp = /-([a-z]{2})\.md$/;
  * should output:
  * { title: 'Backdrop React Component', components: ['Backdrop'] }
  */
-export function getHeaders(markdown) {
+function getHeaders(markdown) {
   let header = markdown.match(headerRegExp);
 
   if (!header) {
@@ -64,14 +66,14 @@ export function getHeaders(markdown) {
   return headers;
 }
 
-export function getContents(markdown) {
+function getContents(markdown) {
   return markdown
     .replace(headerRegExp, '') // Remove header information
     .split(/^{{("(?:demo|component)":[^}]*)}}$/gm) // Split markdown into an array, separating demos
     .filter((content) => !emptyRegExp.test(content)); // Remove empty lines
 }
 
-export function getTitle(markdown) {
+function getTitle(markdown) {
   const matches = markdown.match(titleRegExp);
 
   if (!matches || !matches[1]) {
@@ -81,16 +83,19 @@ export function getTitle(markdown) {
   return matches[1];
 }
 
-export function getDescription(markdown) {
+function getDescription(markdown) {
   const matches = markdown.match(descriptionRegExp);
+  if (matches === null) {
+    return undefined;
+  }
 
-  return matches?.[1].trim();
+  return matches[1].trim();
 }
 
 /**
  * @param {string} markdown
  */
-export function renderInline(markdown) {
+function renderInline(markdown) {
   return marked.parseInline(markdown);
 }
 
@@ -120,7 +125,7 @@ const externs = [
  * @param {TableOfContentsEntry[]} context.toc - WILL BE MUTATED
  * @param {string} context.userLanguage
  */
-export function createRender(context) {
+function createRender(context) {
   const { headingHashes, toc, userLanguage } = context;
   const headingHashesFallbackTranslated = {};
   let headingIndex = -1;
@@ -238,10 +243,13 @@ export function createRender(context) {
  * @param {() => string} config.requireRaw - returnvalue of require.context
  * @param {string} config.pageFilename - filename relative to nextjs pages directory
  */
-export function prepareMarkdown(config) {
+function prepareMarkdown(config) {
   const { pageFilename, requireRaw } = config;
 
   const demos = {};
+  /**
+   * @type {Record<string, { rendered: Array<string | { component: string } | { demo:string }> }>}
+   */
   const docs = {};
   const headingHashes = {};
 
@@ -336,3 +344,14 @@ ${headers.components
 
   return { demos, docs };
 }
+
+module.exports = {
+  createRender,
+  notEnglishMarkdownRegExp,
+  getContents,
+  getDescription,
+  getHeaders,
+  getTitle,
+  prepareMarkdown,
+  renderInline,
+};

docs/src/modules/utils/prism.js → docs/packages/markdown/prism.js

@@ -1,13 +1,13 @@
-import prism from 'prismjs';
-import 'prismjs/components/prism-css';
-import 'prismjs/components/prism-diff';
-import 'prismjs/components/prism-javascript';
-import 'prismjs/components/prism-json';
-import 'prismjs/components/prism-jsx';
-import 'prismjs/components/prism-markup';
-import 'prismjs/components/prism-tsx';
+const prism = require('prismjs');
+require('prismjs/components/prism-css');
+require('prismjs/components/prism-diff');
+require('prismjs/components/prism-javascript');
+require('prismjs/components/prism-json');
+require('prismjs/components/prism-jsx');
+require('prismjs/components/prism-markup');
+require('prismjs/components/prism-tsx');
 
-export default function highlight(code, language) {
+function highlight(code, language) {
   let prismLanguage;
   switch (language) {
     case 'ts':
@@ -42,3 +42,5 @@ export default function highlight(code, language) {
 
   return prism.highlight(code, prismLanguage);
 }
+
+module.exports = highlight;

docs/src/modules/utils/textToHash.js → docs/packages/markdown/textToHash.js

@@ -14,7 +14,7 @@ function makeUnique(hash, unique, i = 1) {
  * @param {Record<string, boolean>} [unique] - Ensures that each output is unique in `unique`
  * @returns {string} that is safe to use in fragment links
  */
-export default function textToHash(text, unique = {}) {
+function textToHash(text, unique = {}) {
   return makeUnique(
     encodeURI(
       text
@@ -32,3 +32,5 @@ export default function textToHash(text, unique = {}) {
     unique,
   );
 }
+
+module.exports = textToHash;

docs/src/modules/utils/textToHash.test.js → docs/packages/markdown/textToHash.test.js

@@ -1,5 +1,5 @@
 import { expect } from 'chai';
-import { renderInline as renderInlineMarkdown } from './parseMarkdown';
+import { parseInline as renderInlineMarkdown } from 'marked';
 import textToHash from './textToHash';
 
 describe('textToHash', () => {

docs/pages/blog/2019-developer-survey-results.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2019-developer-survey-results.md';
 
-const pageFilename = 'blog/2019-developer-survey-results';
-const requireRaw = require.context('!raw-loader!./', false, /2019-developer-survey-results\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
   return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
-  const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
-  return { demos, docs };
-};

docs/pages/blog/2019.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2019.md';
 
-const pageFilename = 'blog/2019';
-const requireRaw = require.context('!raw-loader!./', false, /2019\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
   return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
-  const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
-  return { demos, docs };
-};

docs/pages/blog/2020-developer-survey-results.js

@@ -1,15 +1,7 @@
 import * as React from 'react';
 import TopLayoutBlog from 'docs/src/modules/components/TopLayoutBlog';
-import { prepareMarkdown } from 'docs/src/modules/utils/parseMarkdown';
+import { docs } from '!@material-ui/markdown/loader!./2020-developer-survey-results.md';
 
-const pageFilename = 'blog/2020-developer-survey-results';
-const requireRaw = require.context('!raw-loader!./', false, /2020-developer-survey-results\.md$/);
-
-export default function Page({ docs }) {
+export default function Page() {
   return <TopLayoutBlog docs={docs} />;
 }
-
-Page.getInitialProps = () => {
-  const { demos, docs } = prepareMarkdown({ pageFilename, requireRaw });
-  return { demos, docs };
-};