feat(gatsby-plugin-offline): Allow precaching custom pages (#16877)

* Allow precaching custom pages rather than just the offline shell
* Document this and add tests

packages/gatsby-plugin-offline/.gitignore

@@ -1,3 +1,4 @@
 /*.js
 !index.js
+!src/__tests__/fixtures/public
 yarn.lock

packages/gatsby-plugin-offline/README.md

@@ -21,7 +21,22 @@ plugins: [`gatsby-plugin-offline`]
 
 ## Available options
 
-As of `gatsby-plugin-offline` 3.0.0, the following options are available:
+In `gatsby-plugin-offline` 3.x, the following options are available:
+
+- `precachePages` lets you specify pages whose resources should be precached by the service worker, using an array of globs. For example:
+
+  ```javascript:title=gatsby-config.js
+  plugins: [
+    {
+      resolve: `gatsby-plugin-offline`,
+      options: {
+        precachePages: [`/about-us/`, `/projects/*`],
+      },
+    },
+  ]
+  ```
+
+  Note: while essential resources of specified pages will be precached, such as JavaScript and CSS, non-essential resources such as fonts and images will not be included. Instead, these will be cached at runtime when a user visits a given page that includes these resources.
 
 - `appendScript` lets you specify a file to be appended at the end of the generated service worker (`sw.js`). For example:
 

packages/gatsby-plugin-offline/package.json

@@ -9,6 +9,7 @@
   "dependencies": {
     "@babel/runtime": "^7.5.5",
     "cheerio": "^1.0.0-rc.3",
+    "glob": "^7.1.4",
     "idb-keyval": "^3.2.0",
     "lodash": "^4.17.15",
     "slash": "^3.0.0",
@@ -19,7 +20,8 @@
     "@babel/core": "^7.5.5",
     "babel-preset-gatsby-package": "^0.2.3",
     "cpx": "^1.5.0",
-    "cross-env": "^5.2.1"
+    "cross-env": "^5.2.1",
+    "rewire": "^4.0.1"
   },
   "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-plugin-offline#readme",
   "keywords": [

packages/gatsby-plugin-offline/src/__tests__/__snapshots__/gatsby-node.js.snap

@@ -0,0 +1,18 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getPrecachePages correctly matches pages 1`] = `
+Array [
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/index.html",
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/index.html",
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/index.html",
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/test.html",
+]
+`;
+
+exports[`getPrecachePages correctly matches pages 2`] = `
+Array [
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/index.html",
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/page1.html",
+  "<PROJECT_ROOT>/packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/page2.html",
+]
+`;

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/index.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/page1.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/page2.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/script.js

@@ -0,0 +1,2 @@
+// This is a script. How exciting!
+// Content of this file is irrelevant; it is used for testing `getPrecachePages`

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir1/style.css

@@ -0,0 +1,2 @@
+/* This is a stylesheet. How exciting!
+   Content of this file is irrelevant; it is used for testing `getPrecachePages` */

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/index.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/page1.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/page2.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/script.js

@@ -0,0 +1,2 @@
+// This is a script. How exciting!
+// Content of this file is irrelevant; it is used for testing `getPrecachePages`

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/dir2/style.css

@@ -0,0 +1,2 @@
+/* This is a stylesheet. How exciting!
+   Content of this file is irrelevant; it is used for testing `getPrecachePages` */

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/no-index-html/style.css

@@ -0,0 +1,2 @@
+/* This is a stylesheet. How exciting!
+   Content of this file is irrelevant; it is used for testing `getPrecachePages` */

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/script.js

@@ -0,0 +1,2 @@
+// This is a script. How exciting!
+// Content of this file is irrelevant; it is used for testing `getPrecachePages`

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/style.css

@@ -0,0 +1,2 @@
+/* This is a stylesheet. How exciting!
+   Content of this file is irrelevant; it is used for testing `getPrecachePages` */

packages/gatsby-plugin-offline/src/__tests__/fixtures/public/test.html

@@ -0,0 +1,2 @@
+<!-- This is an HTML file. How exciting!
+     Content of this file is irrelevant; it is used for testing `getPrecachePages` -->

packages/gatsby-plugin-offline/src/__tests__/gatsby-node.js

@@ -1,20 +1,26 @@
-const { onPostBuild } = require(`../gatsby-node`)
+const rewire = require(`rewire`)
 const fs = require(`fs`)
+const path = require(`path`)
 
-jest.mock(`fs`)
-jest.mock(`../get-resources-from-html`, () => () => [])
+describe(`getPrecachePages`, () => {
+  const gatsbyNode = rewire(`../gatsby-node`)
+  const getPrecachePages = gatsbyNode.__get__(`getPrecachePages`)
 
-let swText = ``
+  it(`correctly matches pages`, () => {
+    const base = `${__dirname}/fixtures/public`
 
-jest.mock(`workbox-build`, () => {
-  return {
-    generateSW() {
-      return Promise.resolve({ count: 1, size: 1, warnings: [] })
-    },
-  }
+    const allPages = getPrecachePages([`**/*`], base)
+    expect(allPages.map(page => path.normalize(page))).toMatchSnapshot()
+
+    const dir1Pages = getPrecachePages([`/dir1/*`], base)
+    expect(dir1Pages.map(page => path.normalize(page))).toMatchSnapshot()
+  })
 })
 
 describe(`onPostBuild`, () => {
+  let swText = ``
+  const gatsbyNode = rewire(`../gatsby-node`)
+
   const componentChunkName = `chunkName`
   const chunks = [`chunk1`, `chunk2`]
 
@@ -26,31 +32,53 @@ describe(`onPostBuild`, () => {
   }
 
   // Mock out filesystem functions
-  fs.readFileSync.mockImplementation(file => {
-    if (file === `${process.cwd()}/public/webpack.stats.json`) {
-      return JSON.stringify(stats)
-    } else if (file === `public/sw.js`) {
-      return swText
-    } else if (file.match(/\/sw-append\.js/)) {
-      return ``
-    } else {
-      return jest.requireActual(`fs`).readFileSync(file)
-    }
-  })
+  const mockFs = {
+    ...fs,
 
-  fs.appendFileSync.mockImplementation((file, text) => {
-    swText += text
-  })
+    readFileSync(file) {
+      if (file === `${process.cwd()}/public/webpack.stats.json`) {
+        return JSON.stringify(stats)
+      } else if (file === `public/sw.js`) {
+        return swText
+      } else if (file.match(/\/sw-append\.js/)) {
+        return ``
+      } else {
+        return fs.readFileSync(file)
+      }
+    },
 
-  fs.createReadStream.mockImplementation(() => {
-    return { pipe() {} }
-  })
+    appendFileSync(file, text) {
+      swText += text
+    },
+
+    createReadStream() {
+      return { pipe() {} }
+    },
+
+    createWriteStream() {},
+  }
+
+  const mockWorkboxBuild = {
+    generateSW() {
+      return Promise.resolve({ count: 1, size: 1, warnings: [] })
+    },
+  }
 
-  fs.createWriteStream.mockImplementation(() => {})
+  gatsbyNode.__set__(`fs`, mockFs)
+  gatsbyNode.__set__(`getResourcesFromHTML`, () => [])
+  gatsbyNode.__set__(`workboxBuild`, mockWorkboxBuild)
+  gatsbyNode.__set__(`console`, { log() {} })
 
   it(`appends to sw.js`, async () => {
-    await onPostBuild(
-      { pathPrefix: `` },
+    await gatsbyNode.onPostBuild(
+      {
+        pathPrefix: ``,
+        reporter: {
+          info(message) {
+            console.log(message)
+          },
+        },
+      },
       { appendScript: `${__dirname}/fixtures/custom-sw-code.js` }
     )
 

packages/gatsby-plugin-offline/src/__tests__/get-resources-from-html.js

@@ -1,9 +1,9 @@
 const path = require(`path`)
 const getResourcesFromHTML = require(`../get-resources-from-html`)
 
-const htmlPath = path.resolve(`${__dirname}/index.html`)
+const htmlPath = path.resolve(`${__dirname}/fixtures/public/index.html`)
 
 it(`it extracts resources correctly`, () => {
-  const resources = getResourcesFromHTML(htmlPath)
+  const resources = getResourcesFromHTML(htmlPath, ``)
   expect(resources).toMatchSnapshot()
 })

packages/gatsby-plugin-offline/src/gatsby-node.js

@@ -1,10 +1,12 @@
-const fs = require(`fs`)
-const workboxBuild = require(`workbox-build`)
+// use `let` to workaround https://github.com/jhnns/rewire/issues/144
+let fs = require(`fs`)
+let workboxBuild = require(`workbox-build`)
 const path = require(`path`)
 const slash = require(`slash`)
+const glob = require(`glob`)
 const _ = require(`lodash`)
 
-const getResourcesFromHTML = require(`./get-resources-from-html`)
+let getResourcesFromHTML = require(`./get-resources-from-html`)
 
 exports.createPages = ({ actions }) => {
   if (process.env.NODE_ENV === `production`) {
@@ -28,15 +30,44 @@ const readStats = () => {
   }
 }
 
-const getAssetsForChunks = chunks => {
+function getAssetsForChunks(chunks) {
   const files = _.flatten(
     chunks.map(chunk => readStats().assetsByChunkName[chunk])
   )
   return _.compact(files)
 }
 
-exports.onPostBuild = (args, pluginOptions) => {
-  const { pathPrefix } = args
+function getPrecachePages(globs, base) {
+  const precachePages = []
+
+  globs.forEach(page => {
+    const matches = glob.sync(base + page)
+    matches.forEach(path => {
+      const isDirectory = fs.lstatSync(path).isDirectory()
+      let precachePath
+
+      if (isDirectory && fs.existsSync(`${path}/index.html`)) {
+        precachePath = `${path}/index.html`
+      } else if (path.endsWith(`.html`)) {
+        precachePath = path
+      } else {
+        return
+      }
+
+      if (precachePages.indexOf(precachePath) === -1) {
+        precachePages.push(precachePath)
+      }
+    })
+  })
+
+  return precachePages
+}
+
+exports.onPostBuild = (
+  args,
+  { precachePages: precachePagesGlobs = [], appendScript = null, workboxConfig }
+) => {
+  const { pathPrefix, reporter } = args
   const rootDir = `public`
 
   // Get exact asset filenames for app and offline app shell chunks
@@ -47,16 +78,25 @@ exports.onPostBuild = (args, pluginOptions) => {
   ])
   const appFile = files.find(file => file.startsWith(`app-`))
 
-  // Remove the custom prefix (if any) so Workbox can find the files.
-  // This is added back at runtime (see modifyUrlPrefix) in order to serve
-  // from the correct location.
-  const omitPrefix = path => path.slice(pathPrefix.length)
+  function flat(arr) {
+    return Array.prototype.flat ? arr.flat() : [].concat(...arr)
+  }
 
-  const criticalFilePaths = getResourcesFromHTML(
-    `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html`
-  ).map(omitPrefix)
+  const offlineShellPath = `${process.cwd()}/${rootDir}/offline-plugin-app-shell-fallback/index.html`
+  const precachePages = [
+    offlineShellPath,
+    ...getPrecachePages(
+      precachePagesGlobs,
+      `${process.cwd()}/${rootDir}`
+    ).filter(page => page !== offlineShellPath),
+  ]
+
+  const criticalFilePaths = _.uniq(
+    flat(precachePages.map(page => getResourcesFromHTML(page, pathPrefix)))
+  )
 
   const globPatterns = files.concat([
+    // criticalFilePaths doesn't include HTML pages (we only need this one)
     `offline-plugin-app-shell-fallback/index.html`,
     ...criticalFilePaths,
   ])
@@ -108,7 +148,7 @@ exports.onPostBuild = (args, pluginOptions) => {
 
   const combinedOptions = {
     ...options,
-    ...pluginOptions.workboxConfig,
+    ...workboxConfig,
   }
 
   const idbKeyvalFile = `idb-keyval-iife.min.js`
@@ -129,18 +169,22 @@ exports.onPostBuild = (args, pluginOptions) => {
 
       fs.appendFileSync(`public/sw.js`, `\n` + swAppend)
 
-      if (pluginOptions.appendScript) {
+      if (appendScript !== null) {
         let userAppend
         try {
-          userAppend = fs.readFileSync(pluginOptions.appendScript, `utf8`)
+          userAppend = fs.readFileSync(appendScript, `utf8`)
         } catch (e) {
           throw new Error(`Couldn't find the specified offline inject script`)
         }
         fs.appendFileSync(`public/sw.js`, `\n` + userAppend)
       }
 
-      console.log(
-        `Generated ${swDest}, which will precache ${count} files, totaling ${size} bytes.`
+      reporter.info(
+        `Generated ${swDest}, which will precache ${count} files, totaling ${size} bytes.\n` +
+          `The following pages will be precached:\n` +
+          precachePages
+            .map(path => path.replace(`${process.cwd()}/public`, ``))
+            .join(`\n`)
       )
     })
 }

packages/gatsby-plugin-offline/src/get-resources-from-html.js

@@ -3,7 +3,7 @@ const path = require(`path`)
 const fs = require(`fs`)
 const _ = require(`lodash`)
 
-module.exports = htmlPath => {
+module.exports = (htmlPath, pathPrefix) => {
   // load index.html to pull scripts/links necessary for proper offline reload
   let html
   try {
@@ -41,10 +41,15 @@ module.exports = htmlPath => {
     // Don't cache XML files, or external resources (beginning with // or http)
     const blackListRegex = /(\.xml$|^\/\/|^http)/
 
-    if (!blackListRegex.test(url)) {
+    // check resource URLs from header tags start with the correct prefix
+    // (these are not page URLs)
+    if (!blackListRegex.test(url) && url.startsWith(pathPrefix)) {
       criticalFilePaths.push(url.replace(/^\//, ``))
     }
   })
 
-  return _.uniq(criticalFilePaths)
+  // Remove the custom prefix (if any) so Workbox can find the files.
+  // This is added back at runtime (see modifyUrlPrefix in gatsby-node.js) in
+  // order to serve from the correct location.
+  return _.uniq(criticalFilePaths).map(url => url.slice(pathPrefix.length))
 }