From f1426de6db3d87f6289ab947784d1ffacc58d3d2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 4 Nov 2021 14:47:55 -0700 Subject: [PATCH 1/3] fix(gatsby-source-drupal): Ensure all new nodes are created before creating relationships --- .../src/__tests__/fixtures/1593545806.json | 63 +++++++++++++++++++ .../src/__tests__/index.js | 4 ++ .../gatsby-source-drupal/src/gatsby-node.js | 33 ++++++++++ packages/gatsby-source-drupal/src/utils.js | 48 ++++++++++++++ 4 files changed, 148 insertions(+) diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json index 5ec0de662b78f..c2813c2b324ff 100644 --- a/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/1593545806.json @@ -98,6 +98,69 @@ } } } + }, + { + "jsonapi": { + "version": "1.0", + "meta": { + "links": { + "self": { + "href": "http://jsonapi.org/format/1.0/" + } + } + } + }, + "data": { + "type": "node--article", + "id": "article-10", + "attributes": { + "id": 100, + "uuid": "article-10", + "title": "Article #10", + "body": "Aliquam non varius libero, sit amet consequat ex. Aenean porta turpis quis vulputate blandit. Suspendisse in porta erat. Sed sit amet scelerisque turpis, at rutrum mauris. Sed tempor eleifend lobortis. Proin maximus, massa sed dignissim sollicitudin, quam risus mattis justo, sit amet aliquam odio ligula quis urna. Interdum et malesuada fames ac ante ipsum primis in faucibus. Ut mollis leo nisi, at interdum urna fermentum ut. Fusce id suscipit neque, eu fermentum lacus. Donec egestas laoreet felis ac luctus. Vestibulum molestie mattis ante, a vulputate nunc ullamcorper at. Ut hendrerit ipsum eget gravida ultricies." + }, + "relationships": { + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-10" + } + ] + } + } + } + }, + { + "jsonapi": { + "version": "1.0", + "meta": { + "links": { + "self": { + "href": "http://jsonapi.org/format/1.0/" + } + } + } + }, + "data": { + "type": "taxonomy_term--tags", + "id": "tag-10", + "attributes": { + "id": 110, + "uuid": "tag-10", + "langcode": "en", + "name": "Tag #10", + "description": null, + "weight": 0, + "changed": 1523031646, + "default_langcode": true, + "path": { + "alias": null, + "pid": null, + "langcode": "en" + } + } + } } ] } diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 2df04931d516b..2c240c1bbcac4 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -567,6 +567,10 @@ describe(`gatsby-source-drupal`, () => { nodes[createNodeId(`und.article-2`)].relationships .field_tertiary_image___NODE_image___NODE ).toBe(undefined) + expect( + nodes[createNodeId(`und.article-10`)].relationships.field_tags___NODE + .length + ).toBe(1) }) it(`Back references`, () => { diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index d3ae889019ab4..f3c6fc3696f4c 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -16,6 +16,7 @@ const { storeRefsLookups, handleReferences, handleWebhookUpdate, + createNodeIfItDoesNotExist, handleDeletedNode, } = require(`./utils`) @@ -234,6 +235,13 @@ ${JSON.stringify(webhookBody, null, 4)}` nodesToUpdate = [data] } + // First create all nodes that we haven't seen before. That + // way we can create relationships correctly next as the nodes + // will exist in Gatsby. + for (const nodeToUpdate of nodesToUpdate) { + createNodeIfItDoesNotExist(nodeToUpdate) + } + for (const nodeToUpdate of nodesToUpdate) { await handleWebhookUpdate( { @@ -345,6 +353,31 @@ ${JSON.stringify(webhookBody, null, 4)}` // Process sync data from Drupal. const nodesToSync = res.body.entities + + // First create all nodes that we haven't seen before. That + // way we can create relationships correctly next as the nodes + // will exist in Gatsby. + for (const nodeSyncData of nodesToSync) { + if (nodeSyncData.action === `delete`) { + continue + } + + let nodesToUpdate = nodeSyncData.data + if (!Array.isArray(nodeSyncData.data)) { + nodesToUpdate = [nodeSyncData.data] + } + for (const nodeToUpdate of nodesToUpdate) { + createNodeIfItDoesNotExist({ + nodeToUpdate, + actions, + createNodeId, + createContentDigest, + getNode, + reporter, + }) + } + } + for (const nodeSyncData of nodesToSync) { if (nodeSyncData.action === `delete`) { handleDeletedNode({ diff --git a/packages/gatsby-source-drupal/src/utils.js b/packages/gatsby-source-drupal/src/utils.js index ebd51f4a71478..9d45f7bdf02cb 100644 --- a/packages/gatsby-source-drupal/src/utils.js +++ b/packages/gatsby-source-drupal/src/utils.js @@ -240,6 +240,53 @@ const handleDeletedNode = async ({ return deletedNode } +function createNodeIfItDoesNotExist( + { + nodeToUpdate, + actions, + createNodeId, + createContentDigest, + getNode, + reporter, + }, + pluginOptions = {} +) { + if (!nodeToUpdate) { + reporter.warn( + `The updated node was empty. The fact you're seeing this warning means there's probably a bug in how we're creating and processing updates from Drupal. + +${JSON.stringify(nodeToUpdate, null, 4)} + ` + ) + + return + } + + const { createNode } = actions + const newNodeId = createNodeId( + createNodeIdWithVersion( + nodeToUpdate.id, + nodeToUpdate.type, + getOptions().languageConfig ? nodeToUpdate.langcode : `und`, + nodeToUpdate.meta?.target_version, + pluginOptions.entityReferenceRevisions + ) + ) + + const oldNode = getNode(newNodeId) + // Node doesn't yet exist so we'll create it now. + if (!oldNode) { + const newNode = nodeFromData( + nodeToUpdate, + createNodeId, + pluginOptions.entityReferenceRevisions + ) + + newNode.internal.contentDigest = createContentDigest(newNode) + createNode(newNode) + } +} + const handleWebhookUpdate = async ( { nodeToUpdate, @@ -371,3 +418,4 @@ ${JSON.stringify(nodeToUpdate, null, 4)} exports.handleWebhookUpdate = handleWebhookUpdate exports.handleDeletedNode = handleDeletedNode +exports.createNodeIfItDoesNotExist = createNodeIfItDoesNotExist From 8e65cafac46d8c8a1c68e1dbb17b4ff0cb4fec5a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Thu, 4 Nov 2021 22:43:17 -0700 Subject: [PATCH 2/3] fixes --- .../gatsby-source-drupal/src/gatsby-node.js | 7 ------ packages/gatsby-source-drupal/src/utils.js | 23 ++++++++----------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index f3c6fc3696f4c..03dbf7c315a2e 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -235,13 +235,6 @@ ${JSON.stringify(webhookBody, null, 4)}` nodesToUpdate = [data] } - // First create all nodes that we haven't seen before. That - // way we can create relationships correctly next as the nodes - // will exist in Gatsby. - for (const nodeToUpdate of nodesToUpdate) { - createNodeIfItDoesNotExist(nodeToUpdate) - } - for (const nodeToUpdate of nodesToUpdate) { await handleWebhookUpdate( { diff --git a/packages/gatsby-source-drupal/src/utils.js b/packages/gatsby-source-drupal/src/utils.js index 9d45f7bdf02cb..2a8fbca7b7f25 100644 --- a/packages/gatsby-source-drupal/src/utils.js +++ b/packages/gatsby-source-drupal/src/utils.js @@ -240,17 +240,14 @@ const handleDeletedNode = async ({ return deletedNode } -function createNodeIfItDoesNotExist( - { - nodeToUpdate, - actions, - createNodeId, - createContentDigest, - getNode, - reporter, - }, - pluginOptions = {} -) { +function createNodeIfItDoesNotExist({ + nodeToUpdate, + actions, + createNodeId, + createContentDigest, + getNode, + reporter, +}) { if (!nodeToUpdate) { reporter.warn( `The updated node was empty. The fact you're seeing this warning means there's probably a bug in how we're creating and processing updates from Drupal. @@ -269,7 +266,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)} nodeToUpdate.type, getOptions().languageConfig ? nodeToUpdate.langcode : `und`, nodeToUpdate.meta?.target_version, - pluginOptions.entityReferenceRevisions + getOptions().entityReferenceRevisions ) ) @@ -279,7 +276,7 @@ ${JSON.stringify(nodeToUpdate, null, 4)} const newNode = nodeFromData( nodeToUpdate, createNodeId, - pluginOptions.entityReferenceRevisions + getOptions().entityReferenceRevisions ) newNode.internal.contentDigest = createContentDigest(newNode) From 9f0b6691c4d9c5b57193900ea7371d9c1193373f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 5 Nov 2021 14:54:13 -0700 Subject: [PATCH 3/3] Add support for webhook bodies as well --- .../fixtures/webhook-body-multiple-nodes.json | 45 +++++++++++++++++++ .../src/__tests__/index.js | 20 +++++++++ .../gatsby-source-drupal/src/gatsby-node.js | 11 +++++ 3 files changed, 76 insertions(+) create mode 100644 packages/gatsby-source-drupal/src/__tests__/fixtures/webhook-body-multiple-nodes.json diff --git a/packages/gatsby-source-drupal/src/__tests__/fixtures/webhook-body-multiple-nodes.json b/packages/gatsby-source-drupal/src/__tests__/fixtures/webhook-body-multiple-nodes.json new file mode 100644 index 0000000000000..912445597024d --- /dev/null +++ b/packages/gatsby-source-drupal/src/__tests__/fixtures/webhook-body-multiple-nodes.json @@ -0,0 +1,45 @@ +{ + "action": "update", + "data": [ + { + "type": "node--article", + "id": "article-10", + "attributes": { + "id": 100, + "uuid": "article-10", + "title": "Article #10", + "body": "Aliquam non varius libero, sit amet consequat ex. Aenean porta turpis quis vulputate blandit. Suspendisse in porta erat. Sed sit amet scelerisque turpis, at rutrum mauris. Sed tempor eleifend lobortis. Proin maximus, massa sed dignissim sollicitudin, quam risus mattis justo, sit amet aliquam odio ligula quis urna. Interdum et malesuada fames ac ante ipsum primis in faucibus. Ut mollis leo nisi, at interdum urna fermentum ut. Fusce id suscipit neque, eu fermentum lacus. Donec egestas laoreet felis ac luctus. Vestibulum molestie mattis ante, a vulputate nunc ullamcorper at. Ut hendrerit ipsum eget gravida ultricies." + }, + "relationships": { + "field_tags": { + "data": [ + { + "type": "taxonomy_term--tags", + "id": "tag-10" + } + ] + } + } + }, + { + "type": "taxonomy_term--tags", + "id": "tag-10", + "attributes": { + "id": 110, + "uuid": "tag-10", + "langcode": "en", + "name": "Tag #10", + "description": null, + "weight": 0, + "changed": 1523031646, + "default_langcode": true, + "path": { + "alias": null, + "pid": null, + "langcode": "en" + } + } + } + ] +} + diff --git a/packages/gatsby-source-drupal/src/__tests__/index.js b/packages/gatsby-source-drupal/src/__tests__/index.js index 2c240c1bbcac4..d50fc53e4b343 100644 --- a/packages/gatsby-source-drupal/src/__tests__/index.js +++ b/packages/gatsby-source-drupal/src/__tests__/index.js @@ -355,6 +355,26 @@ describe(`gatsby-source-drupal`, () => { }) }) }) + describe(`multiple entities in webhook body`, () => { + let resp + beforeAll(async () => { + const webhookBody = require(`./fixtures/webhook-body-multiple-nodes.json`) + await sourceNodes( + { + ...args, + webhookBody, + }, + { baseUrl } + ) + }) + + it(`Relationships`, async () => { + expect( + nodes[createNodeId(`und.article-10`)].relationships.field_tags___NODE + .length + ).toBe(1) + }) + }) describe(`Insert content`, () => { it(`Node doesn't exist before webhook`, () => { diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js index 03dbf7c315a2e..8f0165507a473 100644 --- a/packages/gatsby-source-drupal/src/gatsby-node.js +++ b/packages/gatsby-source-drupal/src/gatsby-node.js @@ -235,6 +235,17 @@ ${JSON.stringify(webhookBody, null, 4)}` nodesToUpdate = [data] } + for (const nodeToUpdate of nodesToUpdate) { + await createNodeIfItDoesNotExist({ + nodeToUpdate, + actions, + createNodeId, + createContentDigest, + getNode, + reporter, + }) + } + for (const nodeToUpdate of nodesToUpdate) { await handleWebhookUpdate( {