From 6b4fc62a21b474a860fecbaa727d722dfb52cc08 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Thu, 23 Sep 2021 13:30:18 -0700 Subject: [PATCH 01/26] call createNodeManifest before creating nodes during previews --- .../gatsby-source-contentful/src/gatsby-node.js | 9 ++++++++- packages/gatsby-source-contentful/src/normalize.js | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 5f83bc7996a96..3454dff6b817d 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -177,7 +177,13 @@ exports.sourceNodes = async ( }, pluginOptions ) => { - const { createNode, deleteNode, touchNode, createTypes } = actions + const { + createNode, + deleteNode, + touchNode, + createTypes, + unstable_createNodeManifest, + } = actions let currentSyncData let contentTypeItems @@ -664,6 +670,7 @@ exports.sourceNodes = async ( entries: entryList[i], createNode, createNodeId, + unstable_createNodeManifest, getNode, resolvable, foreignReferenceMap, diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 82a31c9afadd7..9e0968b9e3082 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -232,6 +232,7 @@ exports.createNodesForContentType = ({ restrictedNodeFields, conflictFieldPrefix, entries, + unstable_createNodeManifest, createNode, createNodeId, getNode, @@ -426,6 +427,18 @@ exports.createNodesForContentType = ({ }, } + const isPreview = + pluginConfig.get(`environment`) === `preview.contentful.com` + + if (isPreview) { + // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content + + unstable_createNodeManifest({ + manifestId: `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}`, + node: entryNode, + }) + } + // Revision applies to entries, assets, and content types if (entryItem.sys.revision) { entryNode.sys.revision = entryItem.sys.revision From 0e2ec01247a7f16fe9f2c4abacbc64d1a810124d Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Thu, 23 Sep 2021 16:08:37 -0700 Subject: [PATCH 02/26] only call createNodeManifest if it's a fn --- packages/gatsby-source-contentful/src/normalize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 9e0968b9e3082..8d10823aa3483 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -430,7 +430,7 @@ exports.createNodesForContentType = ({ const isPreview = pluginConfig.get(`environment`) === `preview.contentful.com` - if (isPreview) { + if (isPreview && typeof unstable_createNodeManifest === `function`) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content unstable_createNodeManifest({ From fac3e047a87b09fd62b6a12c30357e1b6132e3de Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Mon, 27 Sep 2021 14:51:45 -0700 Subject: [PATCH 03/26] add warning about old versions of Gatsby core for unstable_createNodeManifest action --- packages/gatsby-source-contentful/src/normalize.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 8d10823aa3483..23f546ea37c26 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -430,13 +430,20 @@ exports.createNodesForContentType = ({ const isPreview = pluginConfig.get(`environment`) === `preview.contentful.com` - if (isPreview && typeof unstable_createNodeManifest === `function`) { + const createNodeManifestIsSupported = + typeof unstable_createNodeManifest === `function` + + if (isPreview && createNodeManifestIsSupported) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content unstable_createNodeManifest({ manifestId: `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}`, node: entryNode, }) + } else if (isPreview && !createNodeManifestIsSupported) { + console.warn( + `Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync.` + ) } // Revision applies to entries, assets, and content types From bc95598c1032c149c69a1a5cccf09071e6ab3f8f Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Mon, 27 Sep 2021 15:13:15 -0700 Subject: [PATCH 04/26] Update normalize.js --- packages/gatsby-source-contentful/src/normalize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 23f546ea37c26..fb58e59380f4f 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -442,7 +442,7 @@ exports.createNodesForContentType = ({ }) } else if (isPreview && !createNodeManifestIsSupported) { console.warn( - `Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync.` + `Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` ) } From 6968dc8d90a3113effff7a32bd532543748d1653 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Mon, 27 Sep 2021 15:23:51 -0700 Subject: [PATCH 05/26] canary log --- packages/gatsby-source-contentful/src/normalize.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index fb58e59380f4f..75188fc1238a0 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -433,6 +433,8 @@ exports.createNodesForContentType = ({ const createNodeManifestIsSupported = typeof unstable_createNodeManifest === `function` + console.log({ isPreview, createNodeManifestIsSupported }) + if (isPreview && createNodeManifestIsSupported) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content From 418eec671385559506ba035d77e8bf662fc854ca Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 13:34:25 -0700 Subject: [PATCH 06/26] check host plugin setting instead of environment --- packages/gatsby-source-contentful/src/normalize.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 75188fc1238a0..c0d5ef3a3da8a 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -427,8 +427,7 @@ exports.createNodesForContentType = ({ }, } - const isPreview = - pluginConfig.get(`environment`) === `preview.contentful.com` + const isPreview = pluginConfig.get(`host`) === `preview.contentful.com` const createNodeManifestIsSupported = typeof unstable_createNodeManifest === `function` From 1ff73f8e7b754e8be524079329fbb68a908bf331 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 14:12:08 -0700 Subject: [PATCH 07/26] log out manifest ID --- packages/gatsby-source-contentful/src/normalize.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index c0d5ef3a3da8a..2e62b5dcf9cf7 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -436,9 +436,14 @@ exports.createNodesForContentType = ({ if (isPreview && createNodeManifestIsSupported) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content + const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` + + console.log( + `[gatsby-source-contentful] Creating node manifest for id ${manifestId}` + ) unstable_createNodeManifest({ - manifestId: `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}`, + manifestId, node: entryNode, }) } else if (isPreview && !createNodeManifestIsSupported) { From 76f7e2fa6372c73fe80c2f0a3ec494735f627472 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 15:12:00 -0700 Subject: [PATCH 08/26] use webhook body temporarily --- packages/gatsby-source-contentful/src/normalize.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 2e62b5dcf9cf7..65fadbf2aa129 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -243,6 +243,7 @@ exports.createNodesForContentType = ({ space, useNameForId, pluginConfig, + webhookBody, }) => { // Establish identifier for content type // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, @@ -434,9 +435,13 @@ exports.createNodesForContentType = ({ console.log({ isPreview, createNodeManifestIsSupported }) - if (isPreview && createNodeManifestIsSupported) { + if ( + isPreview && + createNodeManifestIsSupported && + webhookBody.id === entryNode.contentful_id + ) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content - const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` + const { manifestId } = webhookBody console.log( `[gatsby-source-contentful] Creating node manifest for id ${manifestId}` From 575f7c151e5dbcb9edc63bddfa04de677d5797fd Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 15:16:48 -0700 Subject: [PATCH 09/26] webhooks don't always exist --- packages/gatsby-source-contentful/src/normalize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 65fadbf2aa129..8f3d9f2336a76 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -438,7 +438,7 @@ exports.createNodesForContentType = ({ if ( isPreview && createNodeManifestIsSupported && - webhookBody.id === entryNode.contentful_id + webhookBody?.id === entryNode.contentful_id ) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content const { manifestId } = webhookBody From 9c9e37a49f2447fcf0524d3742f43b8c94806b9e Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 15:36:31 -0700 Subject: [PATCH 10/26] create manifests if a webhook body exists --- packages/gatsby-source-contentful/src/normalize.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 8f3d9f2336a76..e7e1526e80b3c 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -435,11 +435,7 @@ exports.createNodesForContentType = ({ console.log({ isPreview, createNodeManifestIsSupported }) - if ( - isPreview && - createNodeManifestIsSupported && - webhookBody?.id === entryNode.contentful_id - ) { + if (isPreview && createNodeManifestIsSupported && webhookBody) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content const { manifestId } = webhookBody From af5b79321d6c10779e926f6662e6f55c6ec4f8cc Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 15:50:15 -0700 Subject: [PATCH 11/26] revert changes --- packages/gatsby-source-contentful/src/normalize.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index e7e1526e80b3c..5c93226f27c36 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -243,7 +243,6 @@ exports.createNodesForContentType = ({ space, useNameForId, pluginConfig, - webhookBody, }) => { // Establish identifier for content type // Use `name` if specified, otherwise, use internal id (usually a natural-language constant, @@ -435,7 +434,7 @@ exports.createNodesForContentType = ({ console.log({ isPreview, createNodeManifestIsSupported }) - if (isPreview && createNodeManifestIsSupported && webhookBody) { + if (isPreview && createNodeManifestIsSupported) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content const { manifestId } = webhookBody @@ -444,7 +443,7 @@ exports.createNodesForContentType = ({ ) unstable_createNodeManifest({ - manifestId, + manifestId: `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}`, node: entryNode, }) } else if (isPreview && !createNodeManifestIsSupported) { From 4ce2c3337f11eabe6bfa9f0f8ca17ea12fc1bf35 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 28 Sep 2021 15:51:11 -0700 Subject: [PATCH 12/26] more fixes --- packages/gatsby-source-contentful/src/normalize.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 5c93226f27c36..7c91cb35ee1cd 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -436,14 +436,13 @@ exports.createNodesForContentType = ({ if (isPreview && createNodeManifestIsSupported) { // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content - const { manifestId } = webhookBody - + const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` console.log( `[gatsby-source-contentful] Creating node manifest for id ${manifestId}` ) unstable_createNodeManifest({ - manifestId: `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}`, + manifestId, node: entryNode, }) } else if (isPreview && !createNodeManifestIsSupported) { From afc0fc3bb4df289e55fdc11aa83fac26b5617ab4 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 29 Sep 2021 15:43:30 -0700 Subject: [PATCH 13/26] only create manifests for contentful entries updated in the last 5 mins --- .../src/gatsby-node.js | 1 + .../gatsby-source-contentful/src/normalize.js | 30 ++++++++++++++----- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 3454dff6b817d..5d740f5491a2f 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -679,6 +679,7 @@ exports.sourceNodes = async ( space, useNameForId: pluginConfig.get(`useNameForId`), pluginConfig, + syncToken, }) ) } diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 7c91cb35ee1cd..9c3620696bc25 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -242,6 +242,7 @@ exports.createNodesForContentType = ({ locales, space, useNameForId, + syncToken, pluginConfig, }) => { // Establish identifier for content type @@ -432,13 +433,28 @@ exports.createNodesForContentType = ({ const createNodeManifestIsSupported = typeof unstable_createNodeManifest === `function` - console.log({ isPreview, createNodeManifestIsSupported }) - - if (isPreview && createNodeManifestIsSupported) { - // @todo figure out how to only create manifests for recent previews on cold builds. Probably on cold builds compare the updatedAt time vs the current time to find recently updated draft content + const cacheExists = !!syncToken + + const shouldCreateNodeManifest = + isPreview && + createNodeManifestIsSupported && + // and this is a delta update + (cacheExists || + // or this entry/node was updated in the last 5 mins + // we don't want older nodes because we only want to create + // node manifests for recently updated/created content. + (entryItem.sys.updatedAt && + Date.now() - new Date(entryItem.sys.updatedAt).getTime() <= + 60 * + 1000 * + (process.env + .CONTENT_SYNC_CONTENTFUL_MINUTES_SINCE_ENTRY_UPDATE || 5))) + + if (shouldCreateNodeManifest) { const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` - console.log( - `[gatsby-source-contentful] Creating node manifest for id ${manifestId}` + + console.info( + `Contentful: Creating node manifest with id ${manifestId}` ) unstable_createNodeManifest({ @@ -447,7 +463,7 @@ exports.createNodesForContentType = ({ }) } else if (isPreview && !createNodeManifestIsSupported) { console.warn( - `Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` + `Contentful: Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` ) } From 0677f711f32a166aec42c95cacedc9b35944c59d Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 5 Oct 2021 11:35:15 -0700 Subject: [PATCH 14/26] add debugging log via env var --- .../gatsby-source-contentful/src/normalize.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 9c3620696bc25..82f0b8d551b93 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -450,9 +450,20 @@ exports.createNodesForContentType = ({ (process.env .CONTENT_SYNC_CONTENTFUL_MINUTES_SINCE_ENTRY_UPDATE || 5))) - if (shouldCreateNodeManifest) { - const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` + const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` + + if (process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true`) { + console.info({ + isPreview, + cacheExists, + createNodeManifestIsSupported, + shouldCreateNodeManifest, + manifestId, + entryItemSysUpdatedAt: entryItem.sys.updatedAt, + }) + } + if (shouldCreateNodeManifest) { console.info( `Contentful: Creating node manifest with id ${manifestId}` ) From d9cbaf26c67a557d3b36dda2ca4480284cab5bdc Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 5 Oct 2021 11:43:16 -0700 Subject: [PATCH 15/26] Update normalize.js --- packages/gatsby-source-contentful/src/normalize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 82f0b8d551b93..5bf40530609e5 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -454,8 +454,8 @@ exports.createNodesForContentType = ({ if (process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true`) { console.info({ - isPreview, cacheExists, + isPreview, createNodeManifestIsSupported, shouldCreateNodeManifest, manifestId, From ede1b687592129d571f843f8fc23c85b92bd543a Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 5 Oct 2021 11:57:55 -0700 Subject: [PATCH 16/26] formatting --- .../gatsby-source-contentful/src/normalize.js | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 5bf40530609e5..d937c2c8880e0 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -440,27 +440,36 @@ exports.createNodesForContentType = ({ createNodeManifestIsSupported && // and this is a delta update (cacheExists || - // or this entry/node was updated in the last 5 mins + // or this entry/node was updated in the last 60 mins // we don't want older nodes because we only want to create // node manifests for recently updated/created content. (entryItem.sys.updatedAt && Date.now() - new Date(entryItem.sys.updatedAt).getTime() <= + // seconds 60 * + // milliseconds 1000 * + // minutes (process.env - .CONTENT_SYNC_CONTENTFUL_MINUTES_SINCE_ENTRY_UPDATE || 5))) + .CONTENT_SYNC_CONTENTFUL_MINUTES_SINCE_ENTRY_UPDATE || 60))) const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` if (process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true`) { - console.info({ - cacheExists, - isPreview, - createNodeManifestIsSupported, - shouldCreateNodeManifest, - manifestId, - entryItemSysUpdatedAt: entryItem.sys.updatedAt, - }) + console.info( + JSON.stringify( + { + cacheExists, + isPreview, + createNodeManifestIsSupported, + shouldCreateNodeManifest, + manifestId, + entryItemSysUpdatedAt: entryItem.sys.updatedAt, + }, + null, + 2 + ) + ) } if (shouldCreateNodeManifest) { From a6093acf28642e8dfd61ebc021522bdde6a46569 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Tue, 5 Oct 2021 13:06:16 -0700 Subject: [PATCH 17/26] move log to 1 line for Cloud --- .../gatsby-source-contentful/src/normalize.js | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index d937c2c8880e0..3d48041cab459 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -457,18 +457,14 @@ exports.createNodesForContentType = ({ if (process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true`) { console.info( - JSON.stringify( - { - cacheExists, - isPreview, - createNodeManifestIsSupported, - shouldCreateNodeManifest, - manifestId, - entryItemSysUpdatedAt: entryItem.sys.updatedAt, - }, - null, - 2 - ) + JSON.stringify({ + cacheExists, + isPreview, + createNodeManifestIsSupported, + shouldCreateNodeManifest, + manifestId, + entryItemSysUpdatedAt: entryItem.sys.updatedAt, + }) ) } From 1d1694858d8fdc60e10d96659654e52e030232fe Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Thu, 7 Oct 2021 14:51:01 -0700 Subject: [PATCH 18/26] arbitrary change --- packages/gatsby-source-contentful/src/normalize.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 3d48041cab459..1c1cde91d1bd7 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -440,7 +440,7 @@ exports.createNodesForContentType = ({ createNodeManifestIsSupported && // and this is a delta update (cacheExists || - // or this entry/node was updated in the last 60 mins + // or this entry/node was updated in the last 60 mins. // we don't want older nodes because we only want to create // node manifests for recently updated/created content. (entryItem.sys.updatedAt && From 7724a668e05692bcd4a87fec348acc04c8eff609 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 8 Oct 2021 10:16:02 -0700 Subject: [PATCH 19/26] create node manifests for updates in the last 2 days --- packages/gatsby-source-contentful/src/normalize.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 1c1cde91d1bd7..eaa8b62142dd6 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -440,7 +440,7 @@ exports.createNodesForContentType = ({ createNodeManifestIsSupported && // and this is a delta update (cacheExists || - // or this entry/node was updated in the last 60 mins. + // or this entry/node was updated in the last 2 days. // we don't want older nodes because we only want to create // node manifests for recently updated/created content. (entryItem.sys.updatedAt && @@ -450,8 +450,9 @@ exports.createNodesForContentType = ({ // milliseconds 1000 * // minutes + 60 * (process.env - .CONTENT_SYNC_CONTENTFUL_MINUTES_SINCE_ENTRY_UPDATE || 60))) + .CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE || 48))) const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` From 7d92aac71995fc1d54c35a652187025d56258650 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 8 Oct 2021 10:19:36 -0700 Subject: [PATCH 20/26] comment --- packages/gatsby-source-contentful/src/normalize.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index eaa8b62142dd6..fa378c5f1815e 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -445,12 +445,13 @@ exports.createNodesForContentType = ({ // node manifests for recently updated/created content. (entryItem.sys.updatedAt && Date.now() - new Date(entryItem.sys.updatedAt).getTime() <= - // seconds - 60 * - // milliseconds - 1000 * + // milliseconds + 1000 * + // seconds + 60 * // minutes 60 * + // hours (process.env .CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE || 48))) From 3973b2b298a912ea49159f9b32799ae408896a5b Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Fri, 8 Oct 2021 10:41:50 -0700 Subject: [PATCH 21/26] abstract out node manifest logic --- .../gatsby-source-contentful/src/normalize.js | 140 +++++++++++------- 1 file changed, 84 insertions(+), 56 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index fa378c5f1815e..96306f3fb6c50 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -227,6 +227,82 @@ function prepareJSONNode(id, node, key, content) { return JSONNode } +let numberOfContentSyncDebugLogs = 0 +const maxContentSyncDebugLogTimes = 50 + +/** + * This fn creates node manifests which are used for Gatsby Cloud Previews via the Content Sync API/feature. + * Content Sync routes a user from Contentful to a page created from the entry data they're interested in previewing. + */ +function contentfulCreateNodeManifest({ + pluginConfig, + syncToken, + entryItem, + entryNode, + space, + unstable_createNodeManifest, +}) { + const isPreview = pluginConfig.get(`host`) === `preview.contentful.com` + + const createNodeManifestIsSupported = + typeof unstable_createNodeManifest === `function` + + const cacheExists = !!syncToken + + const shouldCreateNodeManifest = + isPreview && + createNodeManifestIsSupported && + // and this is a delta update + (cacheExists || + // or this entry/node was updated in the last 2 days. + // we don't want older nodes because we only want to create + // node manifests for recently updated/created content. + (entryItem.sys.updatedAt && + Date.now() - new Date(entryItem.sys.updatedAt).getTime() <= + // milliseconds + 1000 * + // seconds + 60 * + // minutes + 60 * + // hours + (process.env.CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE || + 48))) + + const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` + + if ( + process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true` && + numberOfContentSyncDebugLogs <= maxContentSyncDebugLogTimes + ) { + numberOfContentSyncDebugLogs++ + + console.info( + JSON.stringify({ + cacheExists, + isPreview, + createNodeManifestIsSupported, + shouldCreateNodeManifest, + manifestId, + entryItemSysUpdatedAt: entryItem.sys.updatedAt, + }) + ) + } + + if (shouldCreateNodeManifest) { + console.info(`Contentful: Creating node manifest with id ${manifestId}`) + + unstable_createNodeManifest({ + manifestId, + node: entryNode, + }) + } else if (isPreview && !createNodeManifestIsSupported) { + console.warn( + `Contentful: Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` + ) + } +} + exports.createNodesForContentType = ({ contentTypeItem, restrictedNodeFields, @@ -428,62 +504,14 @@ exports.createNodesForContentType = ({ }, } - const isPreview = pluginConfig.get(`host`) === `preview.contentful.com` - - const createNodeManifestIsSupported = - typeof unstable_createNodeManifest === `function` - - const cacheExists = !!syncToken - - const shouldCreateNodeManifest = - isPreview && - createNodeManifestIsSupported && - // and this is a delta update - (cacheExists || - // or this entry/node was updated in the last 2 days. - // we don't want older nodes because we only want to create - // node manifests for recently updated/created content. - (entryItem.sys.updatedAt && - Date.now() - new Date(entryItem.sys.updatedAt).getTime() <= - // milliseconds - 1000 * - // seconds - 60 * - // minutes - 60 * - // hours - (process.env - .CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE || 48))) - - const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` - - if (process.env.CONTENTFUL_DEBUG_NODE_MANIFEST === `true`) { - console.info( - JSON.stringify({ - cacheExists, - isPreview, - createNodeManifestIsSupported, - shouldCreateNodeManifest, - manifestId, - entryItemSysUpdatedAt: entryItem.sys.updatedAt, - }) - ) - } - - if (shouldCreateNodeManifest) { - console.info( - `Contentful: Creating node manifest with id ${manifestId}` - ) - - unstable_createNodeManifest({ - manifestId, - node: entryNode, - }) - } else if (isPreview && !createNodeManifestIsSupported) { - console.warn( - `Contentful: Your version of Gatsby core doesn't support Content Sync (via the unstable_createNodeManifest action). Please upgrade to the latest version to use Content Sync in your site.` - ) - } + contentfulCreateNodeManifest({ + pluginConfig, + syncToken, + entryItem, + entryNode, + space, + unstable_createNodeManifest, + }) // Revision applies to entries, assets, and content types if (entryItem.sys.revision) { From ae771ee5a2274a2d394099ceb9e8c8c24dd25189 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 13 Oct 2021 10:38:53 -0700 Subject: [PATCH 22/26] cast env var to number --- packages/gatsby-source-contentful/src/normalize.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index a9a25ce4e9565..b77593dcf3551 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -259,8 +259,9 @@ function contentfulCreateNodeManifest({ // minutes 60 * // hours - (process.env.CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE || - 48))) + (Number( + process.env.CONTENT_SYNC_CONTENTFUL_HOURS_SINCE_ENTRY_UPDATE + ) || 48))) const manifestId = `${space.sys.id}-${entryItem.sys.id}-${entryItem.sys.updatedAt}` From 0132ac8f20ee2aa1c346e2e4923b00f09eb3a572 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 13 Oct 2021 11:58:40 -0700 Subject: [PATCH 23/26] format --- packages/gatsby-worker/src/index.ts | 19 +++++++++---------- .../src/bootstrap/load-plugins/index.ts | 8 +++----- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/packages/gatsby-worker/src/index.ts b/packages/gatsby-worker/src/index.ts index e650f741fba0a..1d43864b93a72 100644 --- a/packages/gatsby-worker/src/index.ts +++ b/packages/gatsby-worker/src/index.ts @@ -38,18 +38,17 @@ type WrapReturnInArray = MaybeFunction extends ( ? (...a: Parameters) => Array> : never -export type CreateWorkerPoolType = WorkerPool & - { - [FunctionName in keyof ExposedFunctions]: EnsureFunctionReturnsAPromise< - ExposedFunctions[FunctionName] +export type CreateWorkerPoolType = WorkerPool & { + [FunctionName in keyof ExposedFunctions]: EnsureFunctionReturnsAPromise< + ExposedFunctions[FunctionName] + > +} & { + all: { + [FunctionName in keyof ExposedFunctions]: WrapReturnInArray< + EnsureFunctionReturnsAPromise > - } & { - all: { - [FunctionName in keyof ExposedFunctions]: WrapReturnInArray< - EnsureFunctionReturnsAPromise - > - } } +} const childWrapperPath = require.resolve(`./child`) diff --git a/packages/gatsby/src/bootstrap/load-plugins/index.ts b/packages/gatsby/src/bootstrap/load-plugins/index.ts index 44316cc4bb877..50f064b7b023e 100644 --- a/packages/gatsby/src/bootstrap/load-plugins/index.ts +++ b/packages/gatsby/src/bootstrap/load-plugins/index.ts @@ -22,11 +22,9 @@ import { } from "./types" import { IPluginRefObject, PluginRef } from "gatsby-plugin-utils/dist/types" -const getAPI = ( - api: { - [exportType in ExportType]: { [api: string]: boolean } - } -): ICurrentAPIs => +const getAPI = (api: { + [exportType in ExportType]: { [api: string]: boolean } +}): ICurrentAPIs => _.keys(api).reduce>((merged, key) => { merged[key] = _.keys(api[key]) return merged From 373af2bc86272a46445792af659e94e99bb9abc2 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 13 Oct 2021 13:16:25 -0700 Subject: [PATCH 24/26] canary --- packages/gatsby-source-contentful/src/normalize.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index b77593dcf3551..0a80a37e7ea78 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -498,6 +498,7 @@ export const createNodesForContentType = ({ }, } + // contentfulCreateNodeManifest({ pluginConfig, syncToken, From fcc5a73fa1052222f32077894bbf9310f402cd84 Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 13 Oct 2021 13:32:12 -0700 Subject: [PATCH 25/26] pass missing args --- packages/gatsby-source-contentful/src/source-nodes.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/source-nodes.js b/packages/gatsby-source-contentful/src/source-nodes.js index 2228a018a2cfa..7cb6e7f79b1b8 100644 --- a/packages/gatsby-source-contentful/src/source-nodes.js +++ b/packages/gatsby-source-contentful/src/source-nodes.js @@ -54,7 +54,8 @@ export async function sourceNodes( }, pluginOptions ) { - const { createNode, touchNode, deleteNode } = actions + const { createNode, touchNode, deleteNode, unstable_createNodeManifest } = + actions const online = await isOnline() if ( @@ -427,6 +428,8 @@ export async function sourceNodes( space, useNameForId: pluginConfig.get(`useNameForId`), pluginConfig, + syncToken, + unstable_createNodeManifest, }) ) } From 2cf8f030e5543ecbfc3ffeb19789f02c7d558c3c Mon Sep 17 00:00:00 2001 From: Tyler Barnes Date: Wed, 13 Oct 2021 13:32:54 -0700 Subject: [PATCH 26/26] Revert "canary" This reverts commit 373af2bc86272a46445792af659e94e99bb9abc2. --- packages/gatsby-source-contentful/src/normalize.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/gatsby-source-contentful/src/normalize.js b/packages/gatsby-source-contentful/src/normalize.js index 0a80a37e7ea78..b77593dcf3551 100644 --- a/packages/gatsby-source-contentful/src/normalize.js +++ b/packages/gatsby-source-contentful/src/normalize.js @@ -498,7 +498,6 @@ export const createNodesForContentType = ({ }, } - // contentfulCreateNodeManifest({ pluginConfig, syncToken,