From 96c44f8edf8605c9b474e89a012b462bf1d9de47 Mon Sep 17 00:00:00 2001 From: Adam Bradley Date: Tue, 27 Oct 2020 14:43:23 -0500 Subject: [PATCH] fix(prerender): cache writing hashed assets --- src/compiler/prerender/prerender-optimize.ts | 118 +++++++++++++----- .../prerender/prerender-worker-ctx.ts | 31 +++++ src/compiler/prerender/prerender-worker.ts | 24 ++-- 3 files changed, 128 insertions(+), 45 deletions(-) create mode 100644 src/compiler/prerender/prerender-worker-ctx.ts diff --git a/src/compiler/prerender/prerender-optimize.ts b/src/compiler/prerender/prerender-optimize.ts index b29de9d5322..dc22844f94e 100644 --- a/src/compiler/prerender/prerender-optimize.ts +++ b/src/compiler/prerender/prerender-optimize.ts @@ -6,6 +6,7 @@ import { optimizeCss } from '../optimize/optimize-css'; import { optimizeJs } from '../optimize/optimize-js'; import { join } from 'path'; import { minifyCss } from '../optimize/minify-css'; +import { PrerenderContext } from './prerender-worker-ctx'; export const inlineExternalStyleSheets = async (sys: d.CompilerSystem, appDir: string, doc: Document) => { const documentLinks = Array.from(doc.querySelectorAll('link[rel=stylesheet]')) as HTMLLinkElement[]; @@ -95,7 +96,13 @@ export const minifyScriptElements = async (doc: Document, addMinifiedAttr: boole ); }; -export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string, doc: Document, currentUrl: URL, addMinifiedAttr: boolean) => { +export const minifyStyleElements = async ( + sys: d.CompilerSystem, + appDir: string, + doc: Document, + currentUrl: URL, + addMinifiedAttr: boolean, +) => { const styleElms = Array.from(doc.querySelectorAll('style')).filter(styleElm => { if (styleElm.hasAttribute(dataMinifiedAttr)) { return false; @@ -115,7 +122,7 @@ export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string, const hash = await getAssetFileHash(sys, appDir, assetUrl); assetUrl.searchParams.append('v', hash); return assetUrl.pathname + assetUrl.search; - } + }, }); if (optimizeResults.diagnostics.length === 0) { styleElm.innerHTML = optimizeResults.output; @@ -128,7 +135,11 @@ export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string, ); }; -export const excludeStaticComponents = (doc: Document, hydrateOpts: d.PrerenderHydrateOptions, hydrateResults: d.HydrateResults) => { +export const excludeStaticComponents = ( + doc: Document, + hydrateOpts: d.PrerenderHydrateOptions, + hydrateResults: d.HydrateResults, +) => { const staticComponents = hydrateOpts.staticComponents.filter(tag => { return hydrateResults.components.some(cmp => cmp.tag === tag); }); @@ -158,7 +169,12 @@ s&&((s['data-opts']=s['data-opts']||{}).exclude=__EXCLUDE__); .replace(/\n/g, '') .trim(); -export const addModulePreloads = (doc: Document, hydrateOpts: d.PrerenderHydrateOptions, hydrateResults: d.HydrateResults, componentGraph: Map) => { +export const addModulePreloads = ( + doc: Document, + hydrateOpts: d.PrerenderHydrateOptions, + hydrateResults: d.HydrateResults, + componentGraph: Map, +) => { if (!componentGraph) { return false; } @@ -167,7 +183,9 @@ export const addModulePreloads = (doc: Document, hydrateOpts: d.PrerenderHydrate const cmpTags = hydrateResults.components.filter(cmp => !staticComponents.includes(cmp.tag)); - const modulePreloads = unique(flatOne(cmpTags.map(cmp => getScopeId(cmp.tag, cmp.mode)).map(scopeId => componentGraph.get(scopeId) || []))); + const modulePreloads = unique( + flatOne(cmpTags.map(cmp => getScopeId(cmp.tag, cmp.mode)).map(scopeId => componentGraph.get(scopeId) || [])), + ); injectModulePreloads(doc, modulePreloads); return true; @@ -194,7 +212,15 @@ export const hasStencilScript = (doc: Document) => { return !!doc.querySelector('script[data-stencil]'); }; -export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnostic[], hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL) => { +export const hashAssets = async ( + sys: d.CompilerSystem, + prerenderCtx: PrerenderContext, + diagnostics: d.Diagnostic[], + hydrateOpts: d.PrerenderHydrateOptions, + appDir: string, + doc: Document, + currentUrl: URL, +) => { // do one at a time to prevent too many opened files and memory usage issues // hash id is cached in each worker, so shouldn't have to do this for every page @@ -208,8 +234,13 @@ export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnosti if (currentUrl.host === stylesheetUrl.host) { try { const filePath = join(appDir, stylesheetUrl.pathname); + if (prerenderCtx.hashedFile.has(filePath)) { + continue; + } + prerenderCtx.hashedFile.add(filePath); + let css = await sys.readFile(filePath); - if (isString(css)) { + if (isString(css) && css.length > 0) { css = await minifyCss({ css, async resolveUrl(urlProp) { @@ -217,9 +248,9 @@ export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnosti const hash = await getAssetFileHash(sys, appDir, assetUrl); assetUrl.searchParams.append('v', hash); return assetUrl.pathname + assetUrl.search; - } + }, }); - await sys.writeFile(filePath, css); + sys.writeFileSync(filePath, css); } } catch (e) { catchError(diagnostics, e); @@ -238,24 +269,36 @@ export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnosti await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'script', ['src']); await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'img', ['src', 'srcset']); await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'picture > source', ['srcset']); - - const pageStates = Array.from(doc.querySelectorAll('script[data-stencil-static="page.state"][type="application/json"]')) as HTMLScriptElement[]; + + const pageStates = Array.from( + doc.querySelectorAll('script[data-stencil-static="page.state"][type="application/json"]'), + ) as HTMLScriptElement[]; if (pageStates.length > 0) { - await Promise.all(pageStates.map(async pageStateScript => { - const pageState = JSON.parse(pageStateScript.textContent); - if (pageState && Array.isArray(pageState.ast)) { - for (const node of pageState.ast) { - if (Array.isArray(node)) { - await hashPageStateAstAssets(sys, hydrateOpts, appDir, currentUrl, pageStateScript, node); + await Promise.all( + pageStates.map(async pageStateScript => { + const pageState = JSON.parse(pageStateScript.textContent); + if (pageState && Array.isArray(pageState.ast)) { + for (const node of pageState.ast) { + if (Array.isArray(node)) { + await hashPageStateAstAssets(sys, hydrateOpts, appDir, currentUrl, pageStateScript, node); + } } + pageStateScript.textContent = JSON.stringify(pageState); } - pageStateScript.textContent = JSON.stringify(pageState); - } - })); + }), + ); } -} +}; -const hashAsset = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL, selector: string, srcAttrs: string[]) => { +const hashAsset = async ( + sys: d.CompilerSystem, + hydrateOpts: d.PrerenderHydrateOptions, + appDir: string, + doc: Document, + currentUrl: URL, + selector: string, + srcAttrs: string[], +) => { const elms = Array.from(doc.querySelectorAll(selector)); // do one at a time to prevent too many opened files and memory usage issues @@ -281,7 +324,14 @@ const hashAsset = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateO } }; -const hashPageStateAstAssets = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateOptions, appDir: string, currentUrl: URL, pageStateScript: HTMLScriptElement, node: any[]) => { +const hashPageStateAstAssets = async ( + sys: d.CompilerSystem, + hydrateOpts: d.PrerenderHydrateOptions, + appDir: string, + currentUrl: URL, + pageStateScript: HTMLScriptElement, + node: any[], +) => { const tagName = node[0]; const attrs = node[1]; @@ -318,21 +368,25 @@ const hashPageStateAstAssets = async (sys: d.CompilerSystem, hydrateOpts: d.Prer }; export const getAttrUrls = (attrName: string, attrValue: string) => { - const srcValues: { src: string, descriptor?: string }[] = []; + const srcValues: { src: string; descriptor?: string }[] = []; if (isString(attrValue)) { if (attrName.toLowerCase() === 'srcset') { - attrValue.split(',').map(a => a.trim()).filter(a => a.length > 0).forEach(src => { - const spaceSplt = src.split(' '); - if (spaceSplt[0].length > 0) { - srcValues.push({ src: spaceSplt[0], descriptor: spaceSplt[1] }); - } - }); + attrValue + .split(',') + .map(a => a.trim()) + .filter(a => a.length > 0) + .forEach(src => { + const spaceSplt = src.split(' '); + if (spaceSplt[0].length > 0) { + srcValues.push({ src: spaceSplt[0], descriptor: spaceSplt[1] }); + } + }); } else { srcValues.push({ src: attrValue }); } } return srcValues; -} +}; export const setAttrUrls = (url: URL, descriptor: string) => { let src = url.pathname + url.search; @@ -352,6 +406,6 @@ const getAssetFileHash = async (sys: d.CompilerSystem, appDir: string, assetUrl: hashedAssets.set(assetUrl.pathname, p); } return p; -} +}; const dataMinifiedAttr = 'data-m'; diff --git a/src/compiler/prerender/prerender-worker-ctx.ts b/src/compiler/prerender/prerender-worker-ctx.ts new file mode 100644 index 00000000000..64790cb6407 --- /dev/null +++ b/src/compiler/prerender/prerender-worker-ctx.ts @@ -0,0 +1,31 @@ +import type * as d from '../../declarations'; + +export interface PrerenderContext { + buildId: string; + componentGraph: Map; + prerenderConfig: d.PrerenderConfig; + ensuredDirs: Set; + templateHtml: string; + hashedFile: Set; +} + +const prerenderCtx: PrerenderContext = { + buildId: null, + componentGraph: null, + prerenderConfig: null, + ensuredDirs: null, + templateHtml: null, + hashedFile: null, +}; + +export const getPrerenderCtx = (prerenderRequest: d.PrerenderUrlRequest) => { + if (prerenderRequest.buildId !== prerenderCtx.buildId) { + prerenderCtx.buildId = prerenderRequest.buildId; + prerenderCtx.componentGraph = null; + prerenderCtx.prerenderConfig = null; + prerenderCtx.ensuredDirs = new Set(); + prerenderCtx.templateHtml = null; + prerenderCtx.hashedFile = new Set(); + } + return prerenderCtx; +}; diff --git a/src/compiler/prerender/prerender-worker.ts b/src/compiler/prerender/prerender-worker.ts index e422b10b81a..720cfb38308 100644 --- a/src/compiler/prerender/prerender-worker.ts +++ b/src/compiler/prerender/prerender-worker.ts @@ -10,18 +10,12 @@ import { } from './prerender-optimize'; import { catchError, isPromise, isRootPath, normalizePath, isFunction } from '@utils'; import { crawlAnchorsForNextUrls } from './crawl-urls'; +import { getPrerenderCtx, PrerenderContext } from './prerender-worker-ctx'; import { getHydrateOptions } from './prerender-hydrate-options'; import { getPrerenderConfig } from './prerender-config'; import { requireFunc } from '../sys/environment'; import { dirname, join } from 'path'; -const prerenderCtx = { - componentGraph: null as Map, - prerenderConfig: null as d.PrerenderConfig, - ensuredDirs: new Set(), - templateHtml: null as string, -}; - export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d.PrerenderUrlRequest) => { // worker thread! const results: d.PrerenderUrlResults = { @@ -31,9 +25,11 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d }; try { + const prerenderCtx = getPrerenderCtx(prerenderRequest); + const url = new URL(prerenderRequest.url, prerenderRequest.devServerHostUrl); const baseUrl = new URL(prerenderRequest.baseUrl); - const componentGraph = getComponentGraph(sys, prerenderRequest.componentGraphPath); + const componentGraph = getComponentGraph(sys, prerenderCtx, prerenderRequest.componentGraphPath); // webpack work-around/hack const hydrateApp = requireFunc(prerenderRequest.hydrateAppFilePath); @@ -128,12 +124,14 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d if (hydrateOpts.hashAssets && !prerenderRequest.isDebug) { try { - docPromises.push(hashAssets(sys, results.diagnostics, hydrateOpts, prerenderRequest.appDir, doc, url)); + docPromises.push( + hashAssets(sys, prerenderCtx, results.diagnostics, hydrateOpts, prerenderRequest.appDir, doc, url), + ); } catch (e) { catchError(results.diagnostics, e); } } - + if (docPromises.length > 0) { await Promise.all(docPromises); } @@ -168,7 +166,7 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d const html = hydrateApp.serializeDocumentToString(doc, hydrateOpts); - prerenderEnsureDir(sys, results.filePath); + prerenderEnsureDir(sys, prerenderCtx, results.filePath); const writePromise = sys.writeFile(results.filePath, html); @@ -207,7 +205,7 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d return results; }; -const getComponentGraph = (sys: d.CompilerSystem, componentGraphPath: string) => { +const getComponentGraph = (sys: d.CompilerSystem, prerenderCtx: PrerenderContext, componentGraphPath: string) => { if (componentGraphPath == null) { return undefined; } @@ -218,7 +216,7 @@ const getComponentGraph = (sys: d.CompilerSystem, componentGraphPath: string) => return prerenderCtx.componentGraph; }; -const prerenderEnsureDir = (sys: d.CompilerSystem, p: string) => { +const prerenderEnsureDir = (sys: d.CompilerSystem, prerenderCtx: PrerenderContext, p: string) => { const allDirs: string[] = []; while (true) {