From dc0ce0c1152d486bc1e58ac7fcaac36924437de8 Mon Sep 17 00:00:00 2001 From: Matthew Miller Date: Wed, 21 Oct 2020 16:49:39 +1000 Subject: [PATCH] fix(gatsby-source-contentful): fixed contentful asset download stalling with high number of assets (#27563) * fix(gatsby-source-contentful): fixed contentful asset download stalling with high number of assets * Add new option to README * Fix test * Switch to pop and fix a typo made when cleaning up code --- packages/gatsby-source-contentful/README.md | 4 +++ .../__tests__/download-contentful-assets.js | 1 + .../src/download-contentful-assets.js | 30 +++++++++++++++---- .../src/gatsby-node.js | 7 +++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/packages/gatsby-source-contentful/README.md b/packages/gatsby-source-contentful/README.md index 640ac7592879c..970346f2a869f 100644 --- a/packages/gatsby-source-contentful/README.md +++ b/packages/gatsby-source-contentful/README.md @@ -177,6 +177,10 @@ If you are confident your Content Types will have natural-language IDs (e.g. `bl Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g `50`. +**`assetDownloadWorkers`** [number][optional] [default: `50`] + +Number of workers to use when downloading contentful assets. Due to technical limitations, opening too many concurrent requests can cause stalled downloads. If you encounter this issue you can set this param to a lower number than 50, e.g 25. + **`richText.resolveFieldLocales`** [boolean][optional] [default: `false`] If you want to resolve the locales in fields of assets and entries that are referenced by rich text (e.g., via embedded entries or entry hyperlinks), set this to `true`. Otherwise, fields of referenced assets or entries will be objects keyed by locale. diff --git a/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js b/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js index 34be21b4611d6..eaa325c26cd13 100644 --- a/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js +++ b/packages/gatsby-source-contentful/src/__tests__/download-contentful-assets.js @@ -65,6 +65,7 @@ describe.only(`downloadContentfulAssets`, () => { actions: { touchNode: jest.fn() }, getNodesByType: () => fixtures, cache, + assetDownloadWorkers: 50, }) fixtures.forEach(n => { diff --git a/packages/gatsby-source-contentful/src/download-contentful-assets.js b/packages/gatsby-source-contentful/src/download-contentful-assets.js index 83aee3f0a408f..a3f6a5f6d8c65 100644 --- a/packages/gatsby-source-contentful/src/download-contentful-assets.js +++ b/packages/gatsby-source-contentful/src/download-contentful-assets.js @@ -9,7 +9,23 @@ const bar = new ProgressBar( } ) -let totalJobs = 0 +/** + * @name distributeWorkload + * @param workers A list of async functions to complete + * @param {number} count The number of task runners to use + */ + +async function distributeWorkload(workers, count) { + const methods = workers.slice() + + async function task() { + while (methods.length > 0) { + await methods.pop()() + } + } + + await Promise.all(new Array(count).fill(undefined).map(() => task())) +} /** * @name downloadContentfulAssets @@ -27,16 +43,17 @@ const downloadContentfulAssets = async gatsbyFunctions => { getCache, getNodesByType, reporter, + assetDownloadWorkers, } = gatsbyFunctions // Any ContentfulAsset nodes will be downloaded, cached and copied to public/static // regardless of if you use `localFile` to link an asset or not. - await Promise.all( - getNodesByType(`ContentfulAsset`).map(async node => { - totalJobs += 1 - bar.total = totalJobs + const assetNodes = getNodesByType(`ContentfulAsset`) + bar.total = assetNodes.length + await distributeWorkload( + assetNodes.map(node => async () => { let fileNodeID const { contentful_id: id, node_locale: locale } = node const remoteDataCacheKey = `contentful-asset-${id}-${locale}` @@ -90,7 +107,8 @@ const downloadContentfulAssets = async gatsbyFunctions => { } return node - }) + }), + assetDownloadWorkers ) } exports.downloadContentfulAssets = downloadContentfulAssets diff --git a/packages/gatsby-source-contentful/src/gatsby-node.js b/packages/gatsby-source-contentful/src/gatsby-node.js index 478172b58a0d5..b3e2406170899 100644 --- a/packages/gatsby-source-contentful/src/gatsby-node.js +++ b/packages/gatsby-source-contentful/src/gatsby-node.js @@ -122,6 +122,12 @@ List of locales and their codes can be found in Contentful app -> Settings -> Lo `Number of entries to retrieve from Contentful at a time. Due to some technical limitations, the response payload should not be greater than 7MB when pulling content from Contentful. If you encounter this issue you can set this param to a lower number than 100, e.g 50.` ) .default(100), + assetDownloadWorkers: Joi.number() + .integer() + .description( + `Number of workers to use when downloading contentful assets. Due to technical limitations, opening too many concurrent requests can cause stalled downloads. If you encounter this issue you can set this param to a lower number than 50, e.g 25.` + ) + .default(50), proxy: Joi.object() .keys({ host: Joi.string().required(), @@ -562,6 +568,7 @@ exports.sourceNodes = async ( getCache, getNodesByType, reporter, + assetDownloadWorkers: pluginConfig.get(`assetDownloadWorkers`), }) }