refactor(gatsby): use virtual modules for (a)sync-requires and match-paths.json

packages/gatsby/cache-dir/__tests__/static-entry.js

@@ -18,7 +18,7 @@ jest.mock(`gatsby/package.json`, () => {
 })
 
 jest.mock(
-  `../sync-requires`,
+  `$virtual/sync-requires`,
   () => {
     return {
       components: {

packages/gatsby/cache-dir/app.js

@@ -8,9 +8,9 @@ import emitter from "./emitter"
 import { apiRunner, apiRunnerAsync } from "./api-runner-browser"
 import { setLoader, publicLoader } from "./loader"
 import DevLoader from "./dev-loader"
-import syncRequires from "./sync-requires"
+import syncRequires from "$virtual/sync-requires"
 // Generated during bootstrap
-import matchPaths from "./match-paths.json"
+import matchPaths from "$virtual/match-paths.json"
 
 window.___emitter = emitter
 

packages/gatsby/cache-dir/production-app.js

@@ -11,7 +11,7 @@ import {
 } from "./navigation"
 import emitter from "./emitter"
 import PageRenderer from "./page-renderer"
-import asyncRequires from "./async-requires"
+import asyncRequires from "$virtual/async-requires"
 import {
   setLoader,
   ProdLoader,
@@ -22,7 +22,7 @@ import EnsureResources from "./ensure-resources"
 import stripPrefix from "./strip-prefix"
 
 // Generated during bootstrap
-import matchPaths from "./match-paths.json"
+import matchPaths from "$virtual/match-paths.json"
 
 const loader = new ProdLoader(asyncRequires, matchPaths)
 setLoader(loader)

packages/gatsby/cache-dir/static-entry.js

@@ -17,7 +17,7 @@ const {
 
 const { RouteAnnouncerProps } = require(`./route-announcer-props`)
 const apiRunner = require(`./api-runner-ssr`)
-const syncRequires = require(`./sync-requires`)
+const syncRequires = require(`$virtual/sync-requires`)
 const { version: gatsbyVersion } = require(`gatsby/package.json`)
 
 const stats = JSON.parse(

packages/gatsby/package.json

@@ -148,6 +148,7 @@
     "webpack-hot-middleware": "^2.25.0",
     "webpack-merge": "^4.2.2",
     "webpack-stats-plugin": "^0.3.1",
+    "webpack-virtual-modules": "^0.2.2",
     "xstate": "^4.11.0",
     "yaml-loader": "^0.6.0"
   },
@@ -162,6 +163,7 @@
     "@types/string-similarity": "^3.0.0",
     "@types/tmp": "^0.2.0",
     "@types/webpack-dev-middleware": "^3.7.1",
+    "@types/webpack-virtual-modules": "^0.1.0",
     "babel-preset-gatsby-package": "^0.5.1",
     "cross-env": "^5.2.1",
     "documentation": "^12.3.0",

packages/gatsby/src/bootstrap/requires-writer.ts

@@ -8,6 +8,7 @@ import { match } from "@reach/router/lib/utils"
 import { joinPath } from "gatsby-core-utils"
 import { store, emitter } from "../redux/"
 import { IGatsbyState, IGatsbyPage } from "../redux/types"
+import { writeModule } from "../utils/gatsby-webpack-virtual-modules"
 import { markWebpackStatusAsPending } from "../utils/webpack-status"
 
 interface IGatsbyPageComponent {
@@ -212,7 +213,7 @@ const preferDefault = m => m && m.default || m
     .map((c: IGatsbyPageComponent): string => {
       // we need a relative import path to keep contenthash the same if directory changes
       const relativeComponentPath = path.relative(
-        path.join(program.directory, `.cache`),
+        path.join(program.directory, `node_modules`, `$virtual`),
         c.component
       )
 
@@ -223,7 +224,16 @@ const preferDefault = m => m && m.default || m
     .join(`,\n`)}
 }\n\n`
 
-  const writeAndMove = (file: string, data: string): Promise<void> => {
+  const writeAndMove = (
+    virtualFilePath: string,
+    file: string,
+    data: string
+  ): Promise<void> => {
+    writeModule(virtualFilePath, data)
+
+    // files in .cache are not used anymore as part of webpack builds, but
+    // still can be used by other tools (for example `gatsby serve` reads
+    // `match-paths.json` to setup routing)
     const destination = joinPath(program.directory, `.cache`, file)
     const tmp = `${destination}.${Date.now()}`
     return fs
@@ -232,9 +242,17 @@ const preferDefault = m => m && m.default || m
   }
 
   await Promise.all([
-    writeAndMove(`sync-requires.js`, syncRequires),
-    writeAndMove(`async-requires.js`, asyncRequires),
-    writeAndMove(`match-paths.json`, JSON.stringify(matchPaths, null, 4)),
+    writeAndMove(`$virtual/sync-requires.js`, `sync-requires.js`, syncRequires),
+    writeAndMove(
+      `$virtual/async-requires.js`,
+      `async-requires.js`,
+      asyncRequires
+    ),
+    writeAndMove(
+      `$virtual/match-paths.json`,
+      `match-paths.json`,
+      JSON.stringify(matchPaths, null, 4)
+    ),
   ])
 
   return true

packages/gatsby/src/utils/gatsby-webpack-virtual-modules.ts

@@ -0,0 +1,50 @@
+import VirtualModulesPlugin from "webpack-virtual-modules"
+
+/*
+ * This module allows creating virtual (in memory only) modules / files
+ * that webpack compilation can access without the need to write module
+ * body to actual filesystem.
+ *
+ * It's useful for intermediate artifacts that are not part of final builds,
+ * but are used in some way to generate final ones (for example `async-requires.js`).
+ *
+ * Using virtual modules allow us to avoid unnecessary I/O to write/read those modules,
+ * but more importantly using virtual modules give us immediate invalidation events
+ * in webpack watching mode (as opposed to debounced/delayed events when filesystem is used).
+ * Instant invalidation events make it much easier to work with various state transitions
+ * in response to external events that are happening while `gatsby develop` is running.
+ */
+
+interface IGatsbyWebpackVirtualModulesContext {
+  writeModule: VirtualModulesPlugin["writeModule"]
+}
+
+const fileContentLookup: Record<string, string> = {}
+const instances: IGatsbyWebpackVirtualModulesContext[] = []
+
+export class GatsbyWebpackVirtualModules {
+  apply(compiler): void {
+    const virtualModules = new VirtualModulesPlugin(fileContentLookup)
+    virtualModules.apply(compiler)
+    instances.push({
+      writeModule: virtualModules.writeModule.bind(virtualModules),
+    })
+  }
+}
+
+export function writeModule(filePath: string, fileContents: string): void {
+  // "node_modules" added in front of filePath allow to allow importing
+  // those modules using same path
+  const adjustedFilePath = `node_modules/${filePath}`
+
+  if (fileContentLookup[adjustedFilePath] === fileContents) {
+    // we already have this, no need to cause invalidation
+    return
+  }
+
+  fileContentLookup[adjustedFilePath] = fileContents
+
+  instances.forEach(instance => {
+    instance.writeModule(adjustedFilePath, fileContents)
+  })
+}

packages/gatsby/src/utils/webpack-utils.ts

@@ -13,6 +13,7 @@ import { getBrowsersList } from "./browserslist"
 
 import { GatsbyWebpackStatsExtractor } from "./gatsby-webpack-stats-extractor"
 import { GatsbyWebpackEslintGraphqlSchemaReload } from "./gatsby-webpack-eslint-graphql-schema-reload-plugin"
+import { GatsbyWebpackVirtualModules } from "./gatsby-webpack-virtual-modules"
 
 import { builtinPlugins } from "./webpack-plugins"
 import { IProgram, Stage } from "../commands/types"
@@ -104,6 +105,7 @@ type PluginUtils = BuiltinPlugins & {
   minifyCss: PluginFactory
   fastRefresh: PluginFactory
   eslintGraphqlSchemaReload: PluginFactory
+  virtualModules: PluginFactory
 }
 
 /**
@@ -673,6 +675,9 @@ export const createWebpackUtils = (
   plugins.eslintGraphqlSchemaReload = (): GatsbyWebpackEslintGraphqlSchemaReload =>
     new GatsbyWebpackEslintGraphqlSchemaReload()
 
+  plugins.virtualModules = (): GatsbyWebpackVirtualModules =>
+    new GatsbyWebpackVirtualModules()
+
   return {
     loaders,
     rules,

packages/gatsby/src/utils/webpack.config.js

@@ -200,6 +200,8 @@ module.exports = async (
           program.prefixPaths ? assetPrefix : ``
         ),
       }),
+
+      plugins.virtualModules(),
     ]
 
     switch (stage) {

yarn.lock

@@ -3830,6 +3830,13 @@
     "@types/source-list-map" "*"
     source-map "^0.6.1"
 
+"@types/webpack-virtual-modules@^0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@types/webpack-virtual-modules/-/webpack-virtual-modules-0.1.0.tgz#6b43e8c71a2d24e4f75e33fe5305573117e0c0e3"
+  integrity sha512-uVJskUYzUUI1Yjcbu20RbzkWdrVTny0AMoWYpeeL9Y0ii3/bbitlG1/I2Z6vMSL2LB0QDSHf/EBbKtWkeCKJWw==
+  dependencies:
+    "@types/webpack" "*"
+
 "@types/webpack@*", "@types/webpack@^4.41.18":
   version "4.41.18"
   resolved "https://registry.yarnpkg.com/@types/webpack/-/webpack-4.41.18.tgz#2945202617866ecdffa582087f1b6de04a7eed55"
@@ -6212,10 +6219,6 @@ chardet@^0.7.0:
   version "0.7.0"
   resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e"
 
-charenc@~0.0.1:
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667"
-
 cheerio@^0.22.0:
   version "0.22.0"
   resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e"
@@ -7289,10 +7292,6 @@ cross-spawn@^7.0.0:
     shebang-command "^2.0.0"
     which "^2.0.1"
 
-crypt@~0.0.1:
-  version "0.0.2"
-  resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b"
-
 crypto-browserify@^3.11.0:
   version "3.12.0"
   resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec"
@@ -7731,7 +7730,7 @@ debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@~4.1.0:
   dependencies:
     ms "^2.1.1"
 
-debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6:
+debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6:
   version "3.2.6"
   resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b"
   dependencies:
@@ -10023,7 +10022,7 @@ flat-cache@^2.0.0, flat-cache@^2.0.1:
     rimraf "2.6.3"
     write "1.0.3"
 
-flat@^4.0.0, flat@^4.1.0:
+flat@^4.0.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/flat/-/flat-4.1.0.tgz#090bec8b05e39cba309747f1d588f04dbaf98db2"
   dependencies:
@@ -12432,15 +12431,7 @@ [email protected]:
     is-empty "0.0.1"
     is-whitespace "^0.3.0"
 
-is-blank@^2.1.0:
-  version "2.1.0"
-  resolved "https://registry.yarnpkg.com/is-blank/-/is-blank-2.1.0.tgz#69a73d3c0d4f417dfffb207a2795c0f0e576de04"
-  integrity sha1-aac9PA1PQX3/+yB6J5XA8OV23gQ=
-  dependencies:
-    is-empty latest
-    is-whitespace latest
-
-is-buffer@^1.1.4, is-buffer@^1.1.5, is-buffer@~1.1.1:
+is-buffer@^1.1.4, is-buffer@^1.1.5:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
 
@@ -12543,7 +12534,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/is-empty/-/is-empty-0.0.1.tgz#09fdc3d649dda5969156c0853a9b76bd781c5a33"
   integrity sha1-Cf3D1kndpZaRVsCFOpt2vXgcWjM=
 
-is-empty@^1.0.0, is-empty@latest:
+is-empty@^1.0.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/is-empty/-/is-empty-1.2.0.tgz#de9bb5b278738a05a0b09a57e1fb4d4a341a9f6b"
   integrity sha1-3pu1snhzigWgsJpX4ftNSjQan2s=
@@ -12957,7 +12948,7 @@ is-whitespace-character@^1.0.0:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-whitespace-character/-/is-whitespace-character-1.0.2.tgz#ede53b4c6f6fb3874533751ec9280d01928d03ed"
 
-is-whitespace@^0.3.0, is-whitespace@latest:
+is-whitespace@^0.3.0:
   version "0.3.0"
   resolved "https://registry.yarnpkg.com/is-whitespace/-/is-whitespace-0.3.0.tgz#1639ecb1be036aec69a54cbb401cfbed7114ab7f"
   integrity sha1-Fjnssb4DauxppUy7QBz77XEUq38=
@@ -15248,14 +15239,6 @@ md5.js@^1.3.4:
     hash-base "^3.0.0"
     inherits "^2.0.1"
 
-md5@^2.2.1:
-  version "2.2.1"
-  resolved "https://registry.yarnpkg.com/md5/-/md5-2.2.1.tgz#53ab38d5fe3c8891ba465329ea23fac0540126f9"
-  dependencies:
-    charenc "~0.0.1"
-    crypt "~0.0.1"
-    is-buffer "~1.1.1"
-
 mdast-comment-marker@^1.0.0:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/mdast-comment-marker/-/mdast-comment-marker-1.1.2.tgz#5ad2e42cfcc41b92a10c1421a98c288d7b447a6d"
@@ -21695,7 +21678,7 @@ stack-trace@^0.0.10:
   version "0.0.10"
   resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
 
-stack-utils@1.0.2, stack-utils@^1.0.1:
+stack-utils@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.2.tgz#33eba3897788558bebfc2db059dc158ec36cebb8"
 
@@ -24278,6 +24261,13 @@ webpack-stats-plugin@^0.3.1:
   resolved "https://registry.yarnpkg.com/webpack-stats-plugin/-/webpack-stats-plugin-0.3.1.tgz#1103c39a305a4e6ba15d5078db84bc0b35447417"
   integrity sha512-pxqzFE055NlNTlNyfDG3xlB2QwT1EWdm/CF5dCJI/e+rRHVxrWhWg1rf1lfsWhI1/EePv8gi/A36YxO/+u0FgQ==
 
+webpack-virtual-modules@^0.2.2:
+  version "0.2.2"
+  resolved "https://registry.yarnpkg.com/webpack-virtual-modules/-/webpack-virtual-modules-0.2.2.tgz#20863dc3cb6bb2104729fff951fbe14b18bd0299"
+  integrity sha512-kDUmfm3BZrei0y+1NTHJInejzxfhtU8eDj2M7OKb2IWrPFAeO1SOH2KuQ68MSZu9IGEHcxbkKKR1v18FrUSOmA==
+  dependencies:
+    debug "^3.0.0"
+
 webpack@^4.14.0, webpack@^4.43.0, webpack@~4.43.0:
   version "4.43.0"
   resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.43.0.tgz#c48547b11d563224c561dad1172c8aa0b8a678e6"