From 4ea379e3bc9fff5573e3187c1870ebe3b6dfe048 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 26 Oct 2021 14:05:38 -0700 Subject: [PATCH 1/9] use lmdb for resultHash cache so shared across workers --- packages/gatsby/src/query/query-runner.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index b2ed9217bdcf9..b1ed38181ae6b 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -15,7 +15,11 @@ import { GraphQLRunner } from "./graphql-runner" import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" -const resultHashes = new Map() +const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default +const resultHashCache = new GatsbyCacheLmdbImpl({ + name: `query-result-hashes`, + encoding: `string`, +}).init() export interface IQueryJob { id: string @@ -172,11 +176,11 @@ export async function queryRunner( .digest(`base64`) if ( - resultHash !== resultHashes.get(queryJob.id) || + resultHash !== (await resultHashCache.get(queryJob.id)) || (queryJob.isPage && !pageDataExists(path.join(program.directory, `public`), queryJob.id)) ) { - resultHashes.set(queryJob.id, resultHash) + resultHashCache.set(queryJob.id, resultHash) if (queryJob.isPage) { // We need to save this temporarily in cache because From 553235a88f59d05ef084a0e05d3d633d97f7e6f9 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 24 Feb 2022 14:10:25 -0500 Subject: [PATCH 2/9] Clear lmdb before workers start processes queries --- packages/gatsby/src/query/query-compiler.js | 7 +++++++ packages/gatsby/src/utils/cache-lmdb.ts | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/query/query-compiler.js b/packages/gatsby/src/query/query-compiler.js index ebad3a61411c7..d00043ce8e952 100644 --- a/packages/gatsby/src/query/query-compiler.js +++ b/packages/gatsby/src/query/query-compiler.js @@ -45,6 +45,7 @@ const { default: errorParser, locInGraphQlToLocInFile, } = require(`./error-parser`) +const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default const overlayErrorID = `graphql-compiler` @@ -62,6 +63,12 @@ export default async function compile({ parentSpan } = {}): Promise< const errors = [] const addError = errors.push.bind(errors) + // clear our query-result-hashes cache before we start workers + new GatsbyCacheLmdbImpl({ + name: `query-result-hashes`, + encoding: `string`, + }).init({ reset: true }) + const parsedQueries = await parseQueries({ base: program.directory, additional: resolveThemes( diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index 98e94f45c1dc3..695f2ac57fdb9 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -13,6 +13,10 @@ const cacheDbFile = }` : `caches-lmdb` +interface IGatsbyCacheLmdbOptions { + reset?: boolean +} + export default class GatsbyCacheLmdb { private static store private db: Database | undefined @@ -38,8 +42,17 @@ export default class GatsbyCacheLmdb { this.cache = this } - init(): GatsbyCacheLmdb { + init({ reset }: IGatsbyCacheLmdbOptions = {}): GatsbyCacheLmdb { fs.ensureDirSync(this.directory) + + if (reset) { + const db = this.getDb() + if (db.getKeysCount({})) { + db.dropSync() + this.db = undefined + } + } + return this } From 2456be70c76b458b1edf904c241ee14457f9be77 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 24 Feb 2022 14:15:15 -0500 Subject: [PATCH 3/9] Pull db name into const --- packages/gatsby/src/query/query-compiler.js | 3 ++- packages/gatsby/src/query/query-runner.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/gatsby/src/query/query-compiler.js b/packages/gatsby/src/query/query-compiler.js index d00043ce8e952..40fe2efc67542 100644 --- a/packages/gatsby/src/query/query-compiler.js +++ b/packages/gatsby/src/query/query-compiler.js @@ -33,6 +33,7 @@ const { store } = require(`../redux`) import { actions } from "../redux/actions" import { websocketManager } from "../utils/websocket-manager" +import { RESULT_HASH_CACHE_NAME } from "./query-runner" const { default: FileParser } = require(`./file-parser`) const { graphqlError, @@ -65,7 +66,7 @@ export default async function compile({ parentSpan } = {}): Promise< // clear our query-result-hashes cache before we start workers new GatsbyCacheLmdbImpl({ - name: `query-result-hashes`, + name: RESULT_HASH_CACHE_NAME, encoding: `string`, }).init({ reset: true }) diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index b1ed38181ae6b..fd0c0639e9937 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -15,9 +15,10 @@ import { GraphQLRunner } from "./graphql-runner" import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" +export const RESULT_HASH_CACHE_NAME = `query-result-hashes` const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default const resultHashCache = new GatsbyCacheLmdbImpl({ - name: `query-result-hashes`, + name: RESULT_HASH_CACHE_NAME, encoding: `string`, }).init() From bcb0f3751af0268210d472862deeb7bccd12729c Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Mar 2022 08:58:57 -0500 Subject: [PATCH 4/9] Undo reset changes, as it's not actually needed --- e2e-tests/mdx/package.json | 8 ++++---- packages/gatsby/src/query/query-compiler.js | 8 -------- packages/gatsby/src/query/query-runner.ts | 3 +-- packages/gatsby/src/utils/cache-lmdb.ts | 15 +-------------- 4 files changed, 6 insertions(+), 28 deletions(-) diff --git a/e2e-tests/mdx/package.json b/e2e-tests/mdx/package.json index 5e103c197b723..7d2dc7153e907 100644 --- a/e2e-tests/mdx/package.json +++ b/e2e-tests/mdx/package.json @@ -7,9 +7,9 @@ "@mdx-js/react": "^1.6.6", "cypress": "^7.2.0", "fs-extra": "^8.1.0", - "gatsby": "^3.0.0", - "gatsby-plugin-mdx": "^2.0.0", - "gatsby-source-filesystem": "^3.0.0", + "gatsby": "4.10.0-next.2-dev-1646680688869", + "gatsby-plugin-mdx": "3.10.0-next.2-dev-1646680688869", + "gatsby-source-filesystem": "4.10.0-next.2-dev-1646680688869", "react": "^17.0.0", "react-dom": "^17.0.0", "theme-ui": "^0.3.1" @@ -37,7 +37,7 @@ }, "devDependencies": { "cross-env": "^5.2.0", - "gatsby-cypress": "^0.1.7", + "gatsby-cypress": "2.10.0-next.1-dev-1646680688869", "is-ci": "^2.0.0", "prettier": "2.0.4", "start-server-and-test": "^1.7.1" diff --git a/packages/gatsby/src/query/query-compiler.js b/packages/gatsby/src/query/query-compiler.js index 40fe2efc67542..ebad3a61411c7 100644 --- a/packages/gatsby/src/query/query-compiler.js +++ b/packages/gatsby/src/query/query-compiler.js @@ -33,7 +33,6 @@ const { store } = require(`../redux`) import { actions } from "../redux/actions" import { websocketManager } from "../utils/websocket-manager" -import { RESULT_HASH_CACHE_NAME } from "./query-runner" const { default: FileParser } = require(`./file-parser`) const { graphqlError, @@ -46,7 +45,6 @@ const { default: errorParser, locInGraphQlToLocInFile, } = require(`./error-parser`) -const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default const overlayErrorID = `graphql-compiler` @@ -64,12 +62,6 @@ export default async function compile({ parentSpan } = {}): Promise< const errors = [] const addError = errors.push.bind(errors) - // clear our query-result-hashes cache before we start workers - new GatsbyCacheLmdbImpl({ - name: RESULT_HASH_CACHE_NAME, - encoding: `string`, - }).init({ reset: true }) - const parsedQueries = await parseQueries({ base: program.directory, additional: resolveThemes( diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index fd0c0639e9937..b1ed38181ae6b 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -15,10 +15,9 @@ import { GraphQLRunner } from "./graphql-runner" import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" -export const RESULT_HASH_CACHE_NAME = `query-result-hashes` const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default const resultHashCache = new GatsbyCacheLmdbImpl({ - name: RESULT_HASH_CACHE_NAME, + name: `query-result-hashes`, encoding: `string`, }).init() diff --git a/packages/gatsby/src/utils/cache-lmdb.ts b/packages/gatsby/src/utils/cache-lmdb.ts index 695f2ac57fdb9..98e94f45c1dc3 100644 --- a/packages/gatsby/src/utils/cache-lmdb.ts +++ b/packages/gatsby/src/utils/cache-lmdb.ts @@ -13,10 +13,6 @@ const cacheDbFile = }` : `caches-lmdb` -interface IGatsbyCacheLmdbOptions { - reset?: boolean -} - export default class GatsbyCacheLmdb { private static store private db: Database | undefined @@ -42,17 +38,8 @@ export default class GatsbyCacheLmdb { this.cache = this } - init({ reset }: IGatsbyCacheLmdbOptions = {}): GatsbyCacheLmdb { + init(): GatsbyCacheLmdb { fs.ensureDirSync(this.directory) - - if (reset) { - const db = this.getDb() - if (db.getKeysCount({})) { - db.dropSync() - this.db = undefined - } - } - return this } From b107769b4ada7acd430d953024587a1ec56d1719 Mon Sep 17 00:00:00 2001 From: Josh Date: Tue, 8 Mar 2022 08:59:59 -0500 Subject: [PATCH 5/9] Undo accidental e2e change --- e2e-tests/mdx/package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-tests/mdx/package.json b/e2e-tests/mdx/package.json index 7d2dc7153e907..5e103c197b723 100644 --- a/e2e-tests/mdx/package.json +++ b/e2e-tests/mdx/package.json @@ -7,9 +7,9 @@ "@mdx-js/react": "^1.6.6", "cypress": "^7.2.0", "fs-extra": "^8.1.0", - "gatsby": "4.10.0-next.2-dev-1646680688869", - "gatsby-plugin-mdx": "3.10.0-next.2-dev-1646680688869", - "gatsby-source-filesystem": "4.10.0-next.2-dev-1646680688869", + "gatsby": "^3.0.0", + "gatsby-plugin-mdx": "^2.0.0", + "gatsby-source-filesystem": "^3.0.0", "react": "^17.0.0", "react-dom": "^17.0.0", "theme-ui": "^0.3.1" @@ -37,7 +37,7 @@ }, "devDependencies": { "cross-env": "^5.2.0", - "gatsby-cypress": "2.10.0-next.1-dev-1646680688869", + "gatsby-cypress": "^0.1.7", "is-ci": "^2.0.0", "prettier": "2.0.4", "start-server-and-test": "^1.7.1" From 4a9d9b5d1eda9f2ea06f97c0358913c1cd10e1a1 Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 9 Mar 2022 15:42:49 -0500 Subject: [PATCH 6/9] Add to pagepaths if component path/id was modified --- packages/gatsby/src/redux/actions/public.js | 3 +++ .../gatsby/src/redux/reducers/pending-page-data-writes.ts | 7 +++++++ packages/gatsby/src/redux/types.ts | 1 + 3 files changed, 11 insertions(+) diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index 7d234dab4d5ea..e4e1e19a6483c 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -429,6 +429,8 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} const oldPage: Page = store.getState().pages.get(internalPage.path) const contextModified = !!oldPage && !_.isEqual(oldPage.context, internalPage.context) + const componentModified = + !!oldPage && !_.isEqual(oldPage.component, internalPage.component) const alternateSlashPath = page.path.endsWith(`/`) ? page.path.slice(0, -1) @@ -496,6 +498,7 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} ...actionOptions, type: `CREATE_PAGE`, contextModified, + componentModified, plugin, payload: internalPage, }, diff --git a/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts b/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts index 84944b57ab360..27c96762afe72 100644 --- a/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts +++ b/packages/gatsby/src/redux/reducers/pending-page-data-writes.ts @@ -7,6 +7,12 @@ export const pendingPageDataWritesReducer = ( action: ActionsUnion ): IGatsbyState["pendingPageDataWrites"] => { switch (action.type) { + case `CREATE_PAGE`: + if (action.componentModified) { + state.pagePaths.add(action.payload.path) + } + + return state case `ADD_PENDING_PAGE_DATA_WRITE`: state.pagePaths.add(action.payload.path) return state @@ -15,6 +21,7 @@ export const pendingPageDataWritesReducer = ( for (const page of action.payload.pages) { state.pagePaths.add(page) } + return state } diff --git a/packages/gatsby/src/redux/types.ts b/packages/gatsby/src/redux/types.ts index 7aa9b417efa79..43b5e3c4ccc18 100644 --- a/packages/gatsby/src/redux/types.ts +++ b/packages/gatsby/src/redux/types.ts @@ -707,6 +707,7 @@ export interface ICreatePageAction { payload: IGatsbyPage plugin?: IGatsbyPlugin contextModified?: boolean + componentModified?: boolean } export interface ICreateRedirectAction { From 146ac203bf27bf6b9f3d08d2b35efad191d417aa Mon Sep 17 00:00:00 2001 From: Josh Date: Wed, 9 Mar 2022 16:19:35 -0500 Subject: [PATCH 7/9] Update snapshots --- .../gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap b/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap index 5f3d00b4f6315..b9a56bbce7c11 100644 --- a/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap +++ b/packages/gatsby/src/redux/__tests__/__snapshots__/pages.ts.snap @@ -80,6 +80,7 @@ Map { exports[`Add pages allows you to add pages 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", @@ -124,6 +125,7 @@ Map { exports[`Add pages allows you to add pages with context 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", @@ -172,6 +174,7 @@ Map { exports[`Add pages allows you to add pages with matchPath 1`] = ` Object { + "componentModified": false, "contextModified": false, "payload": Object { "component": "/whatever/index.js", From 1a65568cdbba95c17abcf68fe9a756d05f155a94 Mon Sep 17 00:00:00 2001 From: Josh Date: Thu, 10 Mar 2022 10:39:31 -0500 Subject: [PATCH 8/9] Move lmdb creation into function --- packages/gatsby/src/query/query-runner.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index b1ed38181ae6b..d002ca19505f8 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -16,10 +16,6 @@ import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default -const resultHashCache = new GatsbyCacheLmdbImpl({ - name: `query-result-hashes`, - encoding: `string`, -}).init() export interface IQueryJob { id: string @@ -175,6 +171,11 @@ export async function queryRunner( .update(resultJSON) .digest(`base64`) + const resultHashCache = new GatsbyCacheLmdbImpl({ + name: `query-result-hashes`, + encoding: `string`, + }).init() + if ( resultHash !== (await resultHashCache.get(queryJob.id)) || (queryJob.isPage && From cc30e3eddc8cf9399658e8c16605b28fd4cde341 Mon Sep 17 00:00:00 2001 From: Michal Piechowiak Date: Fri, 11 Mar 2022 11:49:09 +0100 Subject: [PATCH 9/9] require -> import --- .../src/query/__tests__/data-tracking.js | 1 + packages/gatsby/src/query/query-runner.ts | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/gatsby/src/query/__tests__/data-tracking.js b/packages/gatsby/src/query/__tests__/data-tracking.js index bbf8b203e116d..27e3b676f35ac 100644 --- a/packages/gatsby/src/query/__tests__/data-tracking.js +++ b/packages/gatsby/src/query/__tests__/data-tracking.js @@ -51,6 +51,7 @@ jest.mock(`../../utils/cache-lmdb`, () => { get = jest.fn(() => Promise.resolve()) set = jest.fn(() => Promise.resolve()) }, + __esModule: true, } }) diff --git a/packages/gatsby/src/query/query-runner.ts b/packages/gatsby/src/query/query-runner.ts index d002ca19505f8..6e52917f4b02e 100644 --- a/packages/gatsby/src/query/query-runner.ts +++ b/packages/gatsby/src/query/query-runner.ts @@ -14,8 +14,18 @@ import errorParser from "./error-parser" import { GraphQLRunner } from "./graphql-runner" import { IExecutionResult, PageContext } from "./types" import { pageDataExists, savePageQueryResult } from "../utils/page-data" - -const GatsbyCacheLmdbImpl = require(`../utils/cache-lmdb`).default +import GatsbyCacheLmdb from "../utils/cache-lmdb" + +let resultHashCache: GatsbyCacheLmdb | undefined +function getResultHashCache(): GatsbyCacheLmdb { + if (!resultHashCache) { + resultHashCache = new GatsbyCacheLmdb({ + name: `query-result-hashes`, + encoding: `string`, + }).init() + } + return resultHashCache +} export interface IQueryJob { id: string @@ -171,17 +181,13 @@ export async function queryRunner( .update(resultJSON) .digest(`base64`) - const resultHashCache = new GatsbyCacheLmdbImpl({ - name: `query-result-hashes`, - encoding: `string`, - }).init() - + const resultHashCache = getResultHashCache() if ( resultHash !== (await resultHashCache.get(queryJob.id)) || (queryJob.isPage && !pageDataExists(path.join(program.directory, `public`), queryJob.id)) ) { - resultHashCache.set(queryJob.id, resultHash) + await resultHashCache.set(queryJob.id, resultHash) if (queryJob.isPage) { // We need to save this temporarily in cache because