From 47a309d0dcf05b45165e1209afe84f92aa616255 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Mon, 29 Jun 2020 10:56:48 +0200 Subject: [PATCH 1/2] feat(webpack): add support for virtual modules --- packages/gatsby/package.json | 2 + .../utils/gatsby-webpack-virtual-modules.ts | 50 +++++++++++++++++++ packages/gatsby/src/utils/webpack-utils.ts | 5 ++ packages/gatsby/src/utils/webpack.config.js | 2 + yarn.lock | 50 ++++++++----------- 5 files changed, 79 insertions(+), 30 deletions(-) create mode 100644 packages/gatsby/src/utils/gatsby-webpack-virtual-modules.ts diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index b9c80e6a3581a..9a8ec765f99ec 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -148,6 +148,7 @@ "webpack-hot-middleware": "^2.25.0", "webpack-merge": "^4.2.2", "webpack-stats-plugin": "^0.3.1", + "webpack-virtual-modules": "^0.2.2", "xstate": "^4.11.0", "yaml-loader": "^0.6.0" }, @@ -162,6 +163,7 @@ "@types/string-similarity": "^3.0.0", "@types/tmp": "^0.2.0", "@types/webpack-dev-middleware": "^3.7.1", + "@types/webpack-virtual-modules": "^0.1.0", "babel-preset-gatsby-package": "^0.5.1", "cross-env": "^5.2.1", "documentation": "^12.3.0", diff --git a/packages/gatsby/src/utils/gatsby-webpack-virtual-modules.ts b/packages/gatsby/src/utils/gatsby-webpack-virtual-modules.ts new file mode 100644 index 0000000000000..41c6917714418 --- /dev/null +++ b/packages/gatsby/src/utils/gatsby-webpack-virtual-modules.ts @@ -0,0 +1,50 @@ +import VirtualModulesPlugin from "webpack-virtual-modules" + +/* + * This module allows creating virtual (in memory only) modules / files + * that webpack compilation can access without the need to write module + * body to actual filesystem. + * + * It's useful for intermediate artifacts that are not part of final builds, + * but are used in some way to generate final ones (for example `async-requires.js`). + * + * Using virtual modules allow us to avoid unnecessary I/O to write/read those modules, + * but more importantly using virtual modules give us immediate invalidation events + * in webpack watching mode (as opposed to debounced/delayed events when filesystem is used). + * Instant invalidation events make it much easier to work with various state transitions + * in response to external events that are happening while `gatsby develop` is running. + */ + +interface IGatsbyWebpackVirtualModulesContext { + writeModule: VirtualModulesPlugin["writeModule"] +} + +const fileContentLookup: Record = {} +const instances: IGatsbyWebpackVirtualModulesContext[] = [] + +export class GatsbyWebpackVirtualModules { + apply(compiler): void { + const virtualModules = new VirtualModulesPlugin(fileContentLookup) + virtualModules.apply(compiler) + instances.push({ + writeModule: virtualModules.writeModule.bind(virtualModules), + }) + } +} + +export function writeModule(filePath: string, fileContents: string): void { + // "node_modules" added in front of filePath allow to allow importing + // those modules using same path + const adjustedFilePath = `node_modules/${filePath}` + + if (fileContentLookup[adjustedFilePath] === fileContents) { + // we already have this, no need to cause invalidation + return + } + + fileContentLookup[adjustedFilePath] = fileContents + + instances.forEach(instance => { + instance.writeModule(adjustedFilePath, fileContents) + }) +} diff --git a/packages/gatsby/src/utils/webpack-utils.ts b/packages/gatsby/src/utils/webpack-utils.ts index 6391aac42a8d0..9a4a4bd9c541d 100644 --- a/packages/gatsby/src/utils/webpack-utils.ts +++ b/packages/gatsby/src/utils/webpack-utils.ts @@ -13,6 +13,7 @@ import { getBrowsersList } from "./browserslist" import { GatsbyWebpackStatsExtractor } from "./gatsby-webpack-stats-extractor" import { GatsbyWebpackEslintGraphqlSchemaReload } from "./gatsby-webpack-eslint-graphql-schema-reload-plugin" +import { GatsbyWebpackVirtualModules } from "./gatsby-webpack-virtual-modules" import { builtinPlugins } from "./webpack-plugins" import { IProgram, Stage } from "../commands/types" @@ -104,6 +105,7 @@ type PluginUtils = BuiltinPlugins & { minifyCss: PluginFactory fastRefresh: PluginFactory eslintGraphqlSchemaReload: PluginFactory + virtualModules: PluginFactory } /** @@ -673,6 +675,9 @@ export const createWebpackUtils = ( plugins.eslintGraphqlSchemaReload = (): GatsbyWebpackEslintGraphqlSchemaReload => new GatsbyWebpackEslintGraphqlSchemaReload() + plugins.virtualModules = (): GatsbyWebpackVirtualModules => + new GatsbyWebpackVirtualModules() + return { loaders, rules, diff --git a/packages/gatsby/src/utils/webpack.config.js b/packages/gatsby/src/utils/webpack.config.js index 95262fbe5c498..d3ee9647a0efa 100644 --- a/packages/gatsby/src/utils/webpack.config.js +++ b/packages/gatsby/src/utils/webpack.config.js @@ -200,6 +200,8 @@ module.exports = async ( program.prefixPaths ? assetPrefix : `` ), }), + + plugins.virtualModules(), ] switch (stage) { diff --git a/yarn.lock b/yarn.lock index 9e570518b8afd..2371e9edb99af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3830,6 +3830,13 @@ "@types/source-list-map" "*" source-map "^0.6.1" +"@types/webpack-virtual-modules@^0.1.0": + version "0.1.0" + resolved "https://registry.yarnpkg.com/@types/webpack-virtual-modules/-/webpack-virtual-modules-0.1.0.tgz#6b43e8c71a2d24e4f75e33fe5305573117e0c0e3" + integrity sha512-uVJskUYzUUI1Yjcbu20RbzkWdrVTny0AMoWYpeeL9Y0ii3/bbitlG1/I2Z6vMSL2LB0QDSHf/EBbKtWkeCKJWw== + dependencies: + "@types/webpack" "*" + "@types/webpack@*", "@types/webpack@^4.41.18": version "4.41.18" resolved "https://registry.yarnpkg.com/@types/webpack/-/webpack-4.41.18.tgz#2945202617866ecdffa582087f1b6de04a7eed55" @@ -6212,10 +6219,6 @@ chardet@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" -charenc@~0.0.1: - version "0.0.2" - resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" - cheerio@^0.22.0: version "0.22.0" resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-0.22.0.tgz#a9baa860a3f9b595a6b81b1a86873121ed3a269e" @@ -7289,10 +7292,6 @@ cross-spawn@^7.0.0: shebang-command "^2.0.0" which "^2.0.1" -crypt@~0.0.1: - version "0.0.2" - resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" - crypto-browserify@^3.11.0: version "3.12.0" resolved "https://registry.yarnpkg.com/crypto-browserify/-/crypto-browserify-3.12.0.tgz#396cf9f3137f03e4b8e532c58f698254e00f80ec" @@ -7731,7 +7730,7 @@ debug@4, debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@~4.1.0: dependencies: ms "^2.1.1" -debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6: +debug@^3.0.0, debug@^3.1.0, debug@^3.1.1, debug@^3.2.5, debug@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.6.tgz#e83d17de16d8a7efb7717edbe5fb10135eee629b" dependencies: @@ -10023,7 +10022,7 @@ flat-cache@^2.0.0, flat-cache@^2.0.1: rimraf "2.6.3" write "1.0.3" -flat@^4.0.0, flat@^4.1.0: +flat@^4.0.0: version "4.1.0" resolved "https://registry.yarnpkg.com/flat/-/flat-4.1.0.tgz#090bec8b05e39cba309747f1d588f04dbaf98db2" dependencies: @@ -12432,15 +12431,7 @@ is-blank@1.0.0: is-empty "0.0.1" is-whitespace "^0.3.0" -is-blank@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-blank/-/is-blank-2.1.0.tgz#69a73d3c0d4f417dfffb207a2795c0f0e576de04" - integrity sha1-aac9PA1PQX3/+yB6J5XA8OV23gQ= - dependencies: - is-empty latest - is-whitespace latest - -is-buffer@^1.1.4, is-buffer@^1.1.5, is-buffer@~1.1.1: +is-buffer@^1.1.4, is-buffer@^1.1.5: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" @@ -12543,7 +12534,7 @@ is-empty@0.0.1: resolved "https://registry.yarnpkg.com/is-empty/-/is-empty-0.0.1.tgz#09fdc3d649dda5969156c0853a9b76bd781c5a33" integrity sha1-Cf3D1kndpZaRVsCFOpt2vXgcWjM= -is-empty@^1.0.0, is-empty@latest: +is-empty@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/is-empty/-/is-empty-1.2.0.tgz#de9bb5b278738a05a0b09a57e1fb4d4a341a9f6b" integrity sha1-3pu1snhzigWgsJpX4ftNSjQan2s= @@ -12957,7 +12948,7 @@ is-whitespace-character@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/is-whitespace-character/-/is-whitespace-character-1.0.2.tgz#ede53b4c6f6fb3874533751ec9280d01928d03ed" -is-whitespace@^0.3.0, is-whitespace@latest: +is-whitespace@^0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/is-whitespace/-/is-whitespace-0.3.0.tgz#1639ecb1be036aec69a54cbb401cfbed7114ab7f" integrity sha1-Fjnssb4DauxppUy7QBz77XEUq38= @@ -15248,14 +15239,6 @@ md5.js@^1.3.4: hash-base "^3.0.0" inherits "^2.0.1" -md5@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/md5/-/md5-2.2.1.tgz#53ab38d5fe3c8891ba465329ea23fac0540126f9" - dependencies: - charenc "~0.0.1" - crypt "~0.0.1" - is-buffer "~1.1.1" - mdast-comment-marker@^1.0.0: version "1.1.2" resolved "https://registry.yarnpkg.com/mdast-comment-marker/-/mdast-comment-marker-1.1.2.tgz#5ad2e42cfcc41b92a10c1421a98c288d7b447a6d" @@ -21695,7 +21678,7 @@ stack-trace@^0.0.10: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" -stack-utils@1.0.2, stack-utils@^1.0.1: +stack-utils@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.2.tgz#33eba3897788558bebfc2db059dc158ec36cebb8" @@ -24278,6 +24261,13 @@ webpack-stats-plugin@^0.3.1: resolved "https://registry.yarnpkg.com/webpack-stats-plugin/-/webpack-stats-plugin-0.3.1.tgz#1103c39a305a4e6ba15d5078db84bc0b35447417" integrity sha512-pxqzFE055NlNTlNyfDG3xlB2QwT1EWdm/CF5dCJI/e+rRHVxrWhWg1rf1lfsWhI1/EePv8gi/A36YxO/+u0FgQ== +webpack-virtual-modules@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/webpack-virtual-modules/-/webpack-virtual-modules-0.2.2.tgz#20863dc3cb6bb2104729fff951fbe14b18bd0299" + integrity sha512-kDUmfm3BZrei0y+1NTHJInejzxfhtU8eDj2M7OKb2IWrPFAeO1SOH2KuQ68MSZu9IGEHcxbkKKR1v18FrUSOmA== + dependencies: + debug "^3.0.0" + webpack@^4.14.0, webpack@^4.43.0, webpack@~4.43.0: version "4.43.0" resolved "https://registry.yarnpkg.com/webpack/-/webpack-4.43.0.tgz#c48547b11d563224c561dad1172c8aa0b8a678e6" From 56d2938f5ad61cbe7522de0a818130c0a01b568e Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Sat, 20 Jun 2020 14:22:32 +0200 Subject: [PATCH 2/2] chore(requires-writer): use virtual modules instead of fs --- .../cache-dir/__tests__/static-entry.js | 2 +- packages/gatsby/cache-dir/app.js | 4 +-- packages/gatsby/cache-dir/production-app.js | 4 +-- packages/gatsby/cache-dir/static-entry.js | 2 +- .../gatsby/src/bootstrap/requires-writer.ts | 28 +++++++++++++++---- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/packages/gatsby/cache-dir/__tests__/static-entry.js b/packages/gatsby/cache-dir/__tests__/static-entry.js index f5876fef8c906..75f78be383a9e 100644 --- a/packages/gatsby/cache-dir/__tests__/static-entry.js +++ b/packages/gatsby/cache-dir/__tests__/static-entry.js @@ -18,7 +18,7 @@ jest.mock(`gatsby/package.json`, () => { }) jest.mock( - `../sync-requires`, + `$virtual/sync-requires`, () => { return { components: { diff --git a/packages/gatsby/cache-dir/app.js b/packages/gatsby/cache-dir/app.js index 393a7cb5d6a5e..7f272119e805a 100644 --- a/packages/gatsby/cache-dir/app.js +++ b/packages/gatsby/cache-dir/app.js @@ -8,9 +8,9 @@ import emitter from "./emitter" import { apiRunner, apiRunnerAsync } from "./api-runner-browser" import { setLoader, publicLoader } from "./loader" import DevLoader from "./dev-loader" -import syncRequires from "./sync-requires" +import syncRequires from "$virtual/sync-requires" // Generated during bootstrap -import matchPaths from "./match-paths.json" +import matchPaths from "$virtual/match-paths.json" window.___emitter = emitter diff --git a/packages/gatsby/cache-dir/production-app.js b/packages/gatsby/cache-dir/production-app.js index 23b319f5baff4..14bc5d7d64ed0 100644 --- a/packages/gatsby/cache-dir/production-app.js +++ b/packages/gatsby/cache-dir/production-app.js @@ -11,7 +11,7 @@ import { } from "./navigation" import emitter from "./emitter" import PageRenderer from "./page-renderer" -import asyncRequires from "./async-requires" +import asyncRequires from "$virtual/async-requires" import { setLoader, ProdLoader, @@ -22,7 +22,7 @@ import EnsureResources from "./ensure-resources" import stripPrefix from "./strip-prefix" // Generated during bootstrap -import matchPaths from "./match-paths.json" +import matchPaths from "$virtual/match-paths.json" const loader = new ProdLoader(asyncRequires, matchPaths) setLoader(loader) diff --git a/packages/gatsby/cache-dir/static-entry.js b/packages/gatsby/cache-dir/static-entry.js index 532389604ef70..1bb9c2e12a91e 100644 --- a/packages/gatsby/cache-dir/static-entry.js +++ b/packages/gatsby/cache-dir/static-entry.js @@ -17,7 +17,7 @@ const { const { RouteAnnouncerProps } = require(`./route-announcer-props`) const apiRunner = require(`./api-runner-ssr`) -const syncRequires = require(`./sync-requires`) +const syncRequires = require(`$virtual/sync-requires`) const { version: gatsbyVersion } = require(`gatsby/package.json`) const stats = JSON.parse( diff --git a/packages/gatsby/src/bootstrap/requires-writer.ts b/packages/gatsby/src/bootstrap/requires-writer.ts index b65be05a41fd2..0a289fa62508f 100644 --- a/packages/gatsby/src/bootstrap/requires-writer.ts +++ b/packages/gatsby/src/bootstrap/requires-writer.ts @@ -8,6 +8,7 @@ import { match } from "@reach/router/lib/utils" import { joinPath } from "gatsby-core-utils" import { store, emitter } from "../redux/" import { IGatsbyState, IGatsbyPage } from "../redux/types" +import { writeModule } from "../utils/gatsby-webpack-virtual-modules" import { markWebpackStatusAsPending } from "../utils/webpack-status" interface IGatsbyPageComponent { @@ -212,7 +213,7 @@ const preferDefault = m => m && m.default || m .map((c: IGatsbyPageComponent): string => { // we need a relative import path to keep contenthash the same if directory changes const relativeComponentPath = path.relative( - path.join(program.directory, `.cache`), + path.join(program.directory, `node_modules`, `$virtual`), c.component ) @@ -223,7 +224,16 @@ const preferDefault = m => m && m.default || m .join(`,\n`)} }\n\n` - const writeAndMove = (file: string, data: string): Promise => { + const writeAndMove = ( + virtualFilePath: string, + file: string, + data: string + ): Promise => { + writeModule(virtualFilePath, data) + + // files in .cache are not used anymore as part of webpack builds, but + // still can be used by other tools (for example `gatsby serve` reads + // `match-paths.json` to setup routing) const destination = joinPath(program.directory, `.cache`, file) const tmp = `${destination}.${Date.now()}` return fs @@ -232,9 +242,17 @@ const preferDefault = m => m && m.default || m } await Promise.all([ - writeAndMove(`sync-requires.js`, syncRequires), - writeAndMove(`async-requires.js`, asyncRequires), - writeAndMove(`match-paths.json`, JSON.stringify(matchPaths, null, 4)), + writeAndMove(`$virtual/sync-requires.js`, `sync-requires.js`, syncRequires), + writeAndMove( + `$virtual/async-requires.js`, + `async-requires.js`, + asyncRequires + ), + writeAndMove( + `$virtual/match-paths.json`, + `match-paths.json`, + JSON.stringify(matchPaths, null, 4) + ), ]) return true