From 7ea970913379e4b0262f984624450243b394879b Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Wed, 18 Nov 2020 18:15:30 -0800 Subject: [PATCH] add builtin optimize support (#1615) --- esinstall/package.json | 2 +- .../rollup-plugin-wrap-install-targets.ts | 2 +- snowpack/package.json | 6 +- snowpack/src/build/import-resolver.ts | 7 +- snowpack/src/build/optimize.ts | 608 ++++++++++++++++++ snowpack/src/commands/build.ts | 73 ++- snowpack/src/commands/dev.ts | 3 +- snowpack/src/config.ts | 10 + snowpack/src/types/snowpack.ts | 17 +- snowpack/src/util.ts | 7 +- yarn.lock | 102 ++- 11 files changed, 790 insertions(+), 47 deletions(-) create mode 100644 snowpack/src/build/optimize.ts diff --git a/esinstall/package.json b/esinstall/package.json index 681923e892..fa8d09ac2e 100644 --- a/esinstall/package.json +++ b/esinstall/package.json @@ -45,7 +45,7 @@ "@rollup/plugin-json": "^4.0.0", "@rollup/plugin-node-resolve": "^10.0.0", "@rollup/plugin-replace": "^2.3.3", - "cjs-module-lexer": "^0.5.0", + "cjs-module-lexer": "^1.0.0", "es-module-lexer": "^0.3.24", "is-builtin-module": "^3.0.0", "kleur": "^4.1.1", diff --git a/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts b/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts index 94f5e2e589..d4c088d267 100644 --- a/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts +++ b/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts @@ -4,7 +4,7 @@ import fs from 'fs'; import {Plugin} from 'rollup'; import {InstallTarget, AbstractLogger} from '../types'; import {getWebDependencyName} from '../util.js'; -import parse from 'cjs-module-lexer'; +const {parse} = require('cjs-module-lexer'); /** * rollup-plugin-wrap-install-targets diff --git a/snowpack/package.json b/snowpack/package.json index 25089c2e6c..21f3eb7dc9 100644 --- a/snowpack/package.json +++ b/snowpack/package.json @@ -50,8 +50,10 @@ "dependencies": { "@snowpack/plugin-build-script": "^2.0.11", "@snowpack/plugin-run-script": "^2.2.0", + "@types/cheerio": "^0.22.22", "cacache": "^15.0.0", "cachedir": "^2.3.0", + "cheerio": "^1.0.0-rc.3", "chokidar": "^3.4.0", "compressible": "^2.0.18", "cosmiconfig": "^7.0.0", @@ -59,7 +61,7 @@ "deepmerge": "^4.2.2", "detect-port": "^1.3.0", "es-module-lexer": "^0.3.24", - "esbuild": "^0.8.0", + "esbuild": "^0.8.7", "esinstall": "^0.3.7", "etag": "^1.8.1", "execa": "^4.0.3", @@ -80,9 +82,9 @@ "resolve-from": "^5.0.0", "rimraf": "^3.0.0", "signal-exit": "^3.0.3", + "skypack": "^0.0.1", "source-map": "^0.7.3", "strip-ansi": "^6.0.0", - "skypack": "^0.0.1", "strip-comments": "^2.0.1", "validate-npm-package-name": "^3.0.0", "ws": "^7.3.0", diff --git a/snowpack/src/build/import-resolver.ts b/snowpack/src/build/import-resolver.ts index ad8f1f6909..0d8f4a4a7e 100644 --- a/snowpack/src/build/import-resolver.ts +++ b/snowpack/src/build/import-resolver.ts @@ -1,8 +1,7 @@ import fs from 'fs'; import path from 'path'; -import url from 'url'; import {ImportMap, SnowpackConfig} from '../types/snowpack'; -import {findMatchingAliasEntry, getExt, replaceExt} from '../util'; +import {findMatchingAliasEntry, getExt, isRemoteSpecifier, replaceExt} from '../util'; import {getUrlForFile} from './file-urls'; const cwd = process.cwd(); @@ -57,7 +56,7 @@ export function createImportResolver({ }: ImportResolverOptions) { return function importResolver(spec: string): string | false { // Ignore "http://*" imports - if (url.parse(spec).protocol) { + if (isRemoteSpecifier(spec)) { return spec; } // Ignore packages marked as external @@ -67,7 +66,7 @@ export function createImportResolver({ // Support snowpack.lock.json entry if (lockfile && lockfile.imports[spec]) { const mappedImport = lockfile.imports[spec]; - if (url.parse(mappedImport).protocol) { + if (isRemoteSpecifier(mappedImport)) { return mappedImport; } throw new Error( diff --git a/snowpack/src/build/optimize.ts b/snowpack/src/build/optimize.ts new file mode 100644 index 0000000000..915e4b9841 --- /dev/null +++ b/snowpack/src/build/optimize.ts @@ -0,0 +1,608 @@ +import cheerio from 'cheerio'; +import * as esbuild from 'esbuild'; +import {promises as fs, readFileSync, unlinkSync, writeFileSync} from 'fs'; +import {glob} from 'glob'; +import mkdirp from 'mkdirp'; +import path from 'path'; +import rimraf from 'rimraf'; +import {logger} from '../logger'; +import {OptimizeOptions, SnowpackConfig} from '../types/snowpack'; +import { + addLeadingSlash, + addTrailingSlash, + isRemoteSpecifier, + isTruthy, + PROJECT_CACHE_DIR, + removeLeadingSlash, + removeTrailingSlash, +} from '../util'; +import {getUrlForFile} from './file-urls'; + +interface ESBuildMetaInput { + bytes: number; + imports: {path: string}[]; +} +interface ESBuildMetaOutput { + imports: []; + exports?: []; + // unclear what this exact interface is, and we don't use it + inputs: any; + bytes: number; +} +interface ESBuildMetaManifest { + inputs: Record; + outputs?: Record; +} + +interface ScannedHtmlEntrypoint { + file: string; + root: cheerio.Root; + getScripts: () => cheerio.Cheerio; + getStyles: () => cheerio.Cheerio; + getLinks: (rel: 'stylesheet') => cheerio.Cheerio; +} + +// We want to output our bundled build directly into our build directory, but esbuild +// has a bug where it complains about overwriting source files even when write: false. +// We create a fake bundle directory for now. Nothing ever actually gets written here. +const FAKE_BUILD_DIRECTORY = path.join(PROJECT_CACHE_DIR, '~~bundle~~'); +const FAKE_BUILD_DIRECTORY_REGEX = /.*\~\~bundle\~\~\//; +/** + * Scan a directory and remove any empty folders, recursively. + */ +async function removeEmptyFolders(directoryLoc: string): Promise { + if ((await fs.stat(directoryLoc)).isDirectory()) { + return false; + } + // If folder is empty, clear it + const files = await fs.readdir(directoryLoc); + if (files.length === 0) { + await fs.rmdir(directoryLoc); + return false; + } + // Otherwise, step in and clean each contained item + await Promise.all(files.map((file) => removeEmptyFolders(path.join(directoryLoc, file)))); + // After, check again if folder is now empty + const afterFiles = await fs.readdir(directoryLoc); + if (afterFiles.length == 0) { + await fs.rmdir(directoryLoc); + } + return true; +} + +/** Collect deep imports in the given set, recursively. */ +function collectDeepImports(url: string, manifest: ESBuildMetaManifest, set: Set): void { + if (set.has(url)) { + return; + } + set.add(url); + const manifestEntry = manifest.inputs[url]; + if (!manifestEntry) { + throw new Error('Not Found in manifest: ' + url); + } + manifestEntry.imports.forEach(({path}) => collectDeepImports(path, manifest, set)); + return; +} + +/** + * Scan a collection of HTML files for entrypoints. A file is deemed an "html entrypoint" + * if it contains an element. This prevents partials from being scanned. + */ +async function scanHtmlEntrypoints(htmlFiles: string[]): Promise<(ScannedHtmlEntrypoint | null)[]> { + return Promise.all( + htmlFiles.map(async (htmlFile) => { + const code = await fs.readFile(htmlFile, 'utf-8'); + const root = cheerio.load(code); + const isHtmlFragment = root.html().startsWith(''); + if (isHtmlFragment) { + return null; + } + return { + file: htmlFile, + root, + getScripts: () => root('script[type="module"]'), + getStyles: () => root('style'), + getLinks: (rel: 'stylesheet') => root(`link[rel="${rel}"]`), + }; + }), + ); +} + +async function extractBaseUrl(htmlData: ScannedHtmlEntrypoint, baseUrl: string): Promise { + const {root, getScripts, getLinks} = htmlData; + if (!baseUrl || baseUrl === '/') { + return; + } + getScripts().each((_, elem) => { + const scriptRoot = root(elem); + const scriptSrc = scriptRoot.attr('src'); + if (!scriptSrc || !scriptSrc.startsWith(baseUrl)) { + return; + } + scriptRoot.attr('src', addLeadingSlash(scriptSrc.replace(baseUrl, ''))); + scriptRoot.attr('snowpack-baseurl', 'true'); + }); + getLinks('stylesheet').each((_, elem) => { + const linkRoot = root(elem); + const styleHref = linkRoot.attr('href'); + if (!styleHref || !styleHref.startsWith(baseUrl)) { + return; + } + linkRoot.attr('href', addLeadingSlash(styleHref.replace(baseUrl, ''))); + linkRoot.attr('snowpack-baseurl', 'true'); + }); +} + +async function restitchBaseUrl(htmlData: ScannedHtmlEntrypoint, baseUrl: string): Promise { + const {root, getScripts, getLinks} = htmlData; + getScripts() + .filter('[snowpack-baseurl]') + .each((_, elem) => { + const scriptRoot = root(elem); + const scriptSrc = scriptRoot.attr('src')!; + scriptRoot.attr('src', removeTrailingSlash(baseUrl) + addLeadingSlash(scriptSrc)); + scriptRoot.removeAttr('snowpack-baseurl'); + }); + getLinks('stylesheet') + .filter('[snowpack-baseurl]') + .each((_, elem) => { + const linkRoot = root(elem); + const styleHref = linkRoot.attr('href')!; + linkRoot.attr('href', removeTrailingSlash(baseUrl) + addLeadingSlash(styleHref)); + linkRoot.removeAttr('snowpack-baseurl'); + }); +} + +async function extractInlineScripts(htmlData: ScannedHtmlEntrypoint): Promise { + const {file, root, getScripts, getStyles} = htmlData; + getScripts().each((i, elem) => { + const scriptRoot = root(elem); + const scriptContent = scriptRoot.contents().text(); + if (!scriptContent) { + return; + } + scriptRoot.empty(); + writeFileSync(file + `.inline.${i}.js`, scriptContent); + scriptRoot.attr('src', `./${path.basename(file)}.inline.${i}.js`); + scriptRoot.attr('snowpack-inline', `true`); + }); + getStyles().each((i, elem) => { + const styleRoot = root(elem); + const styleContent = styleRoot.contents().text(); + if (!styleContent) { + return; + } + styleRoot.after( + ``, + ); + styleRoot.remove(); + writeFileSync(file + `.inline.${i}.css`, styleContent); + }); +} + +async function restitchInlineScripts(htmlData: ScannedHtmlEntrypoint): Promise { + const {file, root, getScripts, getLinks} = htmlData; + getScripts() + .filter('[snowpack-inline]') + .each((_, elem) => { + const scriptRoot = root(elem); + const scriptFile = path.resolve(file, '..', scriptRoot.attr('src')!); + const scriptContent = readFileSync(scriptFile, 'utf-8'); + scriptRoot.text(scriptContent); + scriptRoot.removeAttr('src'); + scriptRoot.removeAttr('snowpack-inline'); + unlinkSync(scriptFile); + }); + getLinks('stylesheet') + .filter('[snowpack-inline]') + .each((_, elem) => { + const linkRoot = root(elem); + const styleFile = path.resolve(file, '..', linkRoot.attr('href')!); + const styleContent = readFileSync(styleFile, 'utf-8'); + const newStyleEl = root(''); + newStyleEl.text(styleContent); + linkRoot.after(newStyleEl); + linkRoot.remove(); + unlinkSync(styleFile); + }); +} + +/** Add new bundled CSS files to the HTML entrypoint file, if not already there. */ +function addNewBundledCss( + htmlData: ScannedHtmlEntrypoint, + manifest: ESBuildMetaManifest, + baseUrl: string, +): void { + if (!manifest.outputs) { + return; + } + for (const key of Object.keys(manifest.outputs)) { + if (!key.endsWith('.css')) { + continue; + } + const scriptKey = key.replace('.css', '.js'); + if (!manifest.outputs[scriptKey]) { + continue; + } + const hasCssImportAlready = htmlData + .getLinks('stylesheet') + .toArray() + .some((v) => v.attribs.href.includes(removeLeadingSlash(key))); + const hasScriptImportAlready = htmlData + .getScripts() + .toArray() + .some((v) => v.attribs.src.includes(removeLeadingSlash(scriptKey))); + + if (hasCssImportAlready || !hasScriptImportAlready) { + continue; + } + const linkHref = removeTrailingSlash(baseUrl) + addLeadingSlash(key); + htmlData.root('head').append(``); + } +} + +/** + * Traverse the entrypoint for JS scripts, and add preload links to the HTML entrypoint. + */ +function preloadEntrypoint( + htmlData: ScannedHtmlEntrypoint, + manifest: ESBuildMetaManifest, + config: SnowpackConfig, +): void { + const {root, getScripts} = htmlData; + const preloadScripts = getScripts() + .map((_, elem) => elem.attribs.src) + .get() + .filter(isTruthy); + const collectedDeepImports = new Set(); + for (const preloadScript of preloadScripts) { + const normalizedPreloadScript = removeLeadingSlash( + path.posix.resolve('/', removeLeadingSlash(preloadScript)), + ); + collectDeepImports(normalizedPreloadScript, manifest, collectedDeepImports); + } + const baseUrl = config.buildOptions.baseUrl; + for (const imp of collectedDeepImports) { + const preloadUrl = (baseUrl ? removeTrailingSlash(baseUrl) : '') + addLeadingSlash(imp); + root('head').append(``); + } +} + +/** + * Handle the many different user input formats to return an array of strings. + * resolve "auto" mode here. + */ +async function getEntrypoints( + entrypoints: OptimizeOptions['entrypoints'], + allBuildFiles: string[], +) { + if (entrypoints === 'auto') { + // TODO: Filter allBuildFiles by HTML with head & body + return allBuildFiles.filter((f) => f.endsWith('.html')); + } + if (Array.isArray(entrypoints)) { + return entrypoints; + } + if (typeof entrypoints === 'function') { + return entrypoints({files: allBuildFiles}); + } + throw new Error('UNEXPECTED ENTRYPOINTS: ' + entrypoints); +} + +/** + * Resolve an array of string entrypoints to absolute file paths. Handle + * source vs. build directory relative entrypoints here as well. + */ +async function resolveEntrypoints( + entrypoints: string[], + cwd: string, + buildDirectoryLoc: string, + config: SnowpackConfig, +) { + return Promise.all( + entrypoints.map(async (entrypoint) => { + if (path.isAbsolute(entrypoint)) { + return entrypoint; + } + const buildEntrypoint = path.resolve(buildDirectoryLoc, entrypoint); + if (await fs.stat(buildEntrypoint).catch(() => null)) { + return buildEntrypoint; + } + const resolvedSourceFile = path.resolve(cwd, entrypoint); + let resolvedSourceEntrypoint: string | undefined; + if (await fs.stat(resolvedSourceFile).catch(() => null)) { + const resolvedSourceUrl = getUrlForFile(resolvedSourceFile, config); + if (resolvedSourceUrl) { + resolvedSourceEntrypoint = path.resolve( + buildDirectoryLoc, + removeLeadingSlash(resolvedSourceUrl), + ); + if (await fs.stat(resolvedSourceEntrypoint).catch(() => null)) { + return resolvedSourceEntrypoint; + } + } + } + logger.error(`Error: entrypoint "${entrypoint}" not found in either build or source:`, { + name: 'optimize', + }); + logger.error(` ✘ Build Entrypoint: ${buildEntrypoint}`, {name: 'optimize'}); + logger.error( + ` ✘ Source Entrypoint: ${resolvedSourceFile} ${ + resolvedSourceEntrypoint ? `-> ${resolvedSourceEntrypoint}` : '' + }`, + {name: 'optimize'}, + ); + throw new Error(`Optimize entrypoint "${entrypoint}" does not exist.`); + }), + ); +} + +/** + * Process your entrypoints as either all JS or all HTML. If HTML, + * scan those HTML files and add a Cheerio-powered root document + * so that we can modify the HTML files as we go. + */ +async function processEntrypoints( + originalEntrypointValue: OptimizeOptions['entrypoints'], + entrypointFiles: string[], + buildDirectoryLoc: string, + baseUrl: string, +): Promise<{htmlEntrypoints: null | ScannedHtmlEntrypoint[]; bundleEntrypoints: string[]}> { + // If entrypoints are JS: + if (entrypointFiles.every((f) => f.endsWith('.js'))) { + return {htmlEntrypoints: null, bundleEntrypoints: entrypointFiles}; + } + // If entrypoints are HTML: + if (entrypointFiles.every((f) => f.endsWith('.html'))) { + const rawHtmlEntrypoints = await scanHtmlEntrypoints(entrypointFiles); + const htmlEntrypoints = rawHtmlEntrypoints.filter(isTruthy); + if ( + originalEntrypointValue !== 'auto' && + rawHtmlEntrypoints.length !== htmlEntrypoints.length + ) { + throw new Error('INVALID HTML ENTRYPOINTS: ' + originalEntrypointValue); + } + htmlEntrypoints.forEach((val) => extractBaseUrl(val, baseUrl)); + htmlEntrypoints.forEach(extractInlineScripts); + const bundleEntrypoints = Array.from( + htmlEntrypoints.reduce((all, val) => { + val.getLinks('stylesheet').each((_, elem) => { + if (!elem.attribs.href || isRemoteSpecifier(elem.attribs.href)) { + return; + } + const resolvedCSS = + elem.attribs.href[0] === '/' + ? path.resolve(buildDirectoryLoc, removeLeadingSlash(elem.attribs.href)) + : path.resolve(val.file, '..', elem.attribs.href); + all.add(resolvedCSS); + }); + val.getScripts().each((_, elem) => { + if (!elem.attribs.src || isRemoteSpecifier(elem.attribs.src)) { + return; + } + const resolvedJS = + elem.attribs.src[0] === '/' + ? path.join(buildDirectoryLoc, removeLeadingSlash(elem.attribs.src)) + : path.join(val.file, '..', elem.attribs.src); + all.add(resolvedJS); + }); + return all; + }, new Set()), + ); + return {htmlEntrypoints, bundleEntrypoints}; + } + // If entrypoints are mixed or neither, throw an error. + throw new Error('MIXED ENTRYPOINTS: ' + entrypointFiles); +} + +/** + * Run esbuild on the build directory. This is run regardless of bundle=true or false, + * since we use the generated manifest in either case. + */ +async function runEsbuildOnBuildDirectory( + bundleEntrypoints: string[], + config: SnowpackConfig, + esbuildService: esbuild.Service, +): Promise<{manifest: ESBuildMetaManifest; outputFiles: esbuild.OutputFile[]}> { + const {outputFiles, warnings} = await esbuildService.build({ + entryPoints: bundleEntrypoints, + outdir: FAKE_BUILD_DIRECTORY, + outbase: config.buildOptions.out, + write: false, + bundle: true, + splitting: true, + format: 'esm', + platform: 'browser', + metafile: path.join(config.buildOptions.out, 'build-manifest.json'), + publicPath: config.buildOptions.baseUrl, + minify: config.experiments.optimize!.minify, + target: config.experiments.optimize!.target, + }); + const manifestFile = outputFiles!.find((f) => f.path.endsWith('build-manifest.json'))!; + const manifestContents = manifestFile.text; + const manifest = JSON.parse(manifestContents); + if (!outputFiles) { + throw new Error('EMPTY BUILD'); + } + if (warnings.length > 0) { + console.warn(warnings); + } + outputFiles.forEach( + (f) => + (f.path = f.path.replace( + FAKE_BUILD_DIRECTORY_REGEX, + addTrailingSlash(config.buildOptions.out), + )), + ); + if (!config.experiments.optimize?.bundle) { + delete manifest.outputs; + } else { + Object.entries(manifest.outputs).forEach(([f, val]) => { + const newKey = f.replace(FAKE_BUILD_DIRECTORY_REGEX, '/'); + manifest.outputs[newKey] = val; + delete manifest.outputs[f]; + }); + } + logger.debug(`outputFiles: ${JSON.stringify(outputFiles)}`); + logger.debug(`manifest: ${JSON.stringify(manifest)}`); + + return {outputFiles, manifest}; +} + +/** The main optimize function: runs optimization on a build directory. */ +export async function runBuiltInOptimize(config: SnowpackConfig) { + const originalCwd = process.cwd(); + const buildDirectoryLoc = config.buildOptions.out; + const options = config.experiments.optimize; + if (!options) { + return; + } + + logger.warn( + '(early preview: experiments.optimize is experimental, and still subject to change.)', + { + name: 'optimize', + }, + ); + + // * Scan to collect all build files: We'll need this throughout. + const allBuildFiles = glob.sync('**/*', { + cwd: buildDirectoryLoc, + nodir: true, + absolute: true, + }); + + // * Resolve and validate your entrypoints: they may be JS or HTML + const userEntrypoints = await getEntrypoints(options.entrypoints, allBuildFiles); + logger.debug(JSON.stringify(userEntrypoints), {name: 'optimize.entrypoints'}); + const resolvedEntrypoints = await resolveEntrypoints( + userEntrypoints, + originalCwd, + buildDirectoryLoc, + config, + ); + logger.debug('(resolved) ' + JSON.stringify(resolvedEntrypoints), {name: 'optimize.entrypoints'}); + const {htmlEntrypoints, bundleEntrypoints} = await processEntrypoints( + options.entrypoints, + resolvedEntrypoints, + buildDirectoryLoc, + config.buildOptions.baseUrl, + ); + logger.debug(`htmlEntrypoints: ${JSON.stringify(htmlEntrypoints?.map((f) => f.file))}`); + logger.debug(`bundleEntrypoints: ${JSON.stringify(bundleEntrypoints)}`); + + if ((!htmlEntrypoints || htmlEntrypoints.length === 0) && bundleEntrypoints.length === 0) { + throw new Error( + '[optimize] No HTML entrypoints detected. Set "entrypoints" manually if your site HTML is generated outside of Snowpack (SSR, Rails, PHP, etc.).', + ); + } + + // NOTE: esbuild has no `cwd` support, and assumes that you're always bundling the + // current working directory. To get around this, we change the current working directory + // for this run only, and then reset it on exit. + process.chdir(buildDirectoryLoc); + + // * Run esbuild on the entire build directory. Even if you are not writing the result + // to disk (bundle: false), we still use the bundle manifest as an in-memory representation + // of our import graph, saved to disk. + const esbuildService = await esbuild.startService(); + const {manifest, outputFiles} = await runEsbuildOnBuildDirectory( + bundleEntrypoints, + config, + esbuildService, + ); + + // * BUNDLE: TRUE - Save the bundle result to the build directory, and clean up to remove all original + // build files that now live in the bundles. + if (options.bundle) { + for (const bundledInput of Object.keys(manifest.inputs)) { + if (!manifest.outputs![bundledInput]) { + logger.debug( + `Removing bundled source file: ${path.resolve(buildDirectoryLoc, bundledInput)}`, + ); + await fs.unlink(path.resolve(buildDirectoryLoc, bundledInput)); + } + } + rimraf.sync( + path.resolve(buildDirectoryLoc, removeLeadingSlash(config.buildOptions.webModulesUrl)), + ); + await removeEmptyFolders(buildDirectoryLoc); + for (const outputFile of outputFiles!) { + mkdirp.sync(path.dirname(outputFile.path)); + await fs.writeFile(outputFile.path, outputFile.contents); + } + if (htmlEntrypoints) { + for (const htmlEntrypoint of htmlEntrypoints) { + addNewBundledCss(htmlEntrypoint, manifest, config.buildOptions.baseUrl); + } + } + } + // * BUNDLE: FALSE - Just minifying & transform the CSS & JS files in place. + else if (options.minify || options.target) { + for (const f of allBuildFiles) { + if (['.js', '.css'].includes(path.extname(f))) { + let code = await fs.readFile(f, 'utf-8'); + const minified = await esbuildService.transform(code, { + sourcefile: path.basename(f), + loader: path.extname(f).slice(1) as 'js' | 'css', + minify: options.minify, + target: options.target, + }); + code = minified.code; + await fs.writeFile(f, code); + } + } + } + + // * Restitch any inline scripts into HTML entrypoints that had been split out + // for the sake of bundling/manifest. + if (htmlEntrypoints) { + for (const htmlEntrypoint of htmlEntrypoints) { + restitchInlineScripts(htmlEntrypoint); + } + } + + // * PRELOAD: TRUE - Add preload link elements for each HTML entrypoint, to flatten + // and optimize any deep import waterfalls. + if (options.preload) { + if (options.bundle) { + throw new Error('preload is not needed when bundle=true, and cannot be used in combination.'); + } + if (!htmlEntrypoints || htmlEntrypoints.length === 0) { + throw new Error('preload only works with HTML entrypoints.'); + } + for (const htmlEntrypoint of htmlEntrypoints) { + preloadEntrypoint(htmlEntrypoint, manifest, config); + } + } + + // * Restitch any inline scripts into HTML entrypoints that had been split out + // for the sake of bundling/manifest. + if (htmlEntrypoints) { + for (const htmlEntrypoint of htmlEntrypoints) { + restitchBaseUrl(htmlEntrypoint, config.buildOptions.baseUrl); + } + } + + // Write the final HTML entrypoints to disk (if they exist). + if (htmlEntrypoints) { + for (const htmlEntrypoint of htmlEntrypoints) { + await fs.writeFile(htmlEntrypoint.file, htmlEntrypoint.root.html()); + } + } + + // Write the final build manifest to disk. + if (options.manifest) { + await fs.writeFile( + path.join(config.buildOptions.out, 'build-manifest.json'), + JSON.stringify(manifest), + ); + } + + // Cleanup and exit. + esbuildService.stop(); + process.chdir(originalCwd); + return; +} diff --git a/snowpack/src/commands/build.ts b/snowpack/src/commands/build.ts index a2e7a02f13..8e94285868 100644 --- a/snowpack/src/commands/build.ts +++ b/snowpack/src/commands/build.ts @@ -31,6 +31,7 @@ import { cssSourceMappingURL, HMR_CLIENT_CODE, HMR_OVERLAY_CODE, + isRemoteSpecifier, jsSourceMappingURL, readFile, relativeURL, @@ -38,6 +39,7 @@ import { replaceExt, } from '../util'; import {getInstallTargets, run as installRunner} from './install'; +import {runBuiltInOptimize} from '../build/optimize'; const CONCURRENT_WORKERS = require('os').cpus().length; @@ -231,7 +233,7 @@ class FileBuilder { return spec; } // Ignore "http://*" imports - if (url.parse(resolvedImportUrl).protocol) { + if (isRemoteSpecifier(resolvedImportUrl)) { return spec; } // Ignore packages marked as external @@ -240,10 +242,14 @@ class FileBuilder { } // Handle normal "./" & "../" import specifiers const importExtName = path.extname(resolvedImportUrl); + const isBundling = !!this.config.experiments.optimize?.bundle; const isProxyImport = importExtName && (file.baseExt === '.js' || file.baseExt === '.html') && - importExtName !== '.js'; + importExtName !== '.js' && + // If using our built-in bundler, treat CSS as a first class citizen (no proxy file needed). + // TODO: Remove special `.module.css` handling by building css modules to native JS + CSS. + (!isBundling || !/(? p.optimize)) { + const optimizeStart = performance.now(); + logger.info(colors.yellow('! optimizing build...')); + await runBuiltInOptimize(config); + await runPipelineOptimizeStep(buildDirectoryLoc, { + plugins: config.plugins, + isDev: false, + isSSR: config.experiments.ssr, + isHmrEnabled: false, + sourceMaps: config.buildOptions.sourceMaps, + }); + const optimizeEnd = performance.now(); + logger.info( + `${colors.green('✔')} optimize complete ${colors.dim( + `[${((optimizeEnd - optimizeStart) / 1000).toFixed(2)}s]`, + )}`, + ); + } await runPipelineCleanupStep(config); - await runPipelineOptimizeStep(buildDirectoryLoc, { - plugins: config.plugins, - isDev: false, - isSSR: config.experiments.ssr, - isHmrEnabled: false, - sourceMaps: config.buildOptions.sourceMaps, - }); - logger.info(`${colors.underline(colors.green(colors.bold('▶ Build Complete!')))}`); + logger.info(`${colors.underline(colors.green(colors.bold('▶ Build Complete!')))}\n\n`); return; } diff --git a/snowpack/src/commands/dev.ts b/snowpack/src/commands/dev.ts index 768b401f9c..3e9e32db8c 100644 --- a/snowpack/src/commands/dev.ts +++ b/snowpack/src/commands/dev.ts @@ -78,6 +78,7 @@ import { getExt, HMR_CLIENT_CODE, HMR_OVERLAY_CODE, + isRemoteSpecifier, jsSourceMappingURL, openInBrowser, parsePackageImportSpecifier, @@ -765,7 +766,7 @@ export async function startDevServer(commandOptions: CommandOptions): Promise string[]); + preload: boolean; + bundle: boolean; + manifest: boolean; + minify: boolean; + target: 'es2020' | 'es2019' | 'es2018' | 'es2017'; +} + // interface this library uses internally export interface SnowpackConfig { install: string[]; @@ -226,6 +235,8 @@ export interface SnowpackConfig { res: http.ServerResponse, next: (err?: Error) => void, ) => unknown; + /** (EXPERIMENTAL) Optimize your site for production. */ + optimize?: OptimizeOptions; }; _extensionMap: Record; } @@ -244,7 +255,11 @@ export type SnowpackUserConfig = { installOptions?: Partial; buildOptions?: Partial; testOptions?: Partial; - experiments?: Partial; + experiments?: { + ssr?: SnowpackConfig['experiments']['ssr']; + app?: SnowpackConfig['experiments']['app']; + optimize?: Partial; + }; }; export interface CLIFlags extends Omit { diff --git a/snowpack/src/util.ts b/snowpack/src/util.ts index 9ba2a78213..a9538416e0 100644 --- a/snowpack/src/util.ts +++ b/snowpack/src/util.ts @@ -1,8 +1,8 @@ import cacache from 'cacache'; import globalCacheDir from 'cachedir'; +import crypto from 'crypto'; import etag from 'etag'; import execa from 'execa'; -import crypto from 'crypto'; import projectCacheDir from 'find-cache-dir'; import findUp from 'find-up'; import fs from 'fs'; @@ -11,6 +11,7 @@ import mkdirp from 'mkdirp'; import open from 'open'; import path from 'path'; import rimraf from 'rimraf'; +import url from 'url'; import {ImportMap, SnowpackConfig} from './types/snowpack'; export const GLOBAL_CACHE_DIR = globalCacheDir('snowpack'); @@ -301,6 +302,10 @@ export function replaceExt(fileName: string, oldExt: string, newExt: string): st return fileName.replace(extToReplace, newExt); } +/** determine if remote package or not */ +export function isRemoteSpecifier(specifier) { + return specifier.startsWith('//') || url.parse(specifier).protocol; +} /** * Sanitizes npm packages that end in .js (e.g `tippy.js` -> `tippyjs`). * This is necessary because Snowpack can’t create both a file and directory diff --git a/yarn.lock b/yarn.lock index f4cb7fe1bd..a6e07c3fb0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2649,6 +2649,13 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.14.tgz#44d2dd0b5de6185089375d976b4ec5caf6861193" integrity sha512-G+ITQPXkwTrslfG5L/BksmbLUA0M1iybEsmCWPqzSxsRRhJZimBKJkoMi8fr/CPygPTj4zO5pJH7I2/cm9M7SQ== +"@types/cheerio@^0.22.22": + version "0.22.22" + resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.22.tgz#ae71cf4ca59b8bbaf34c99af7a5d6c8894988f5f" + integrity sha512-05DYX4zU96IBfZFY+t3Mh88nlwSMtmmzSYaQkKN48T495VV1dkHSah6qYyDTN5ngaS0i0VonH37m+RuzSM0YiA== + dependencies: + "@types/node" "*" + "@types/compressible@^2.0.0": version "2.0.0" resolved "https://registry.yarnpkg.com/@types/compressible/-/compressible-2.0.0.tgz#5a3f431179860a5d7f60c447fb9a69f846367ea6" @@ -4838,6 +4845,18 @@ check-error@^1.0.2: resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" integrity sha1-V00xLt2Iu13YkS6Sht1sCu1KrII= +cheerio@^1.0.0-rc.3: + version "1.0.0-rc.3" + resolved "https://registry.yarnpkg.com/cheerio/-/cheerio-1.0.0-rc.3.tgz#094636d425b2e9c0f4eb91a46c05630c9a1a8bf6" + integrity sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA== + dependencies: + css-select "~1.2.0" + dom-serializer "~0.1.1" + entities "~1.1.1" + htmlparser2 "^3.9.1" + lodash "^4.15.0" + parse5 "^3.0.1" + chokidar@2.1.8, chokidar@^2.1.8: version "2.1.8" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-2.1.8.tgz#804b3a7b6a99358c3c5c61e71d8728f041cff917" @@ -4919,10 +4938,10 @@ cjs-module-lexer@^0.4.2: resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-0.4.3.tgz#9e31f7fe701f5fcee5793f77ab4e58fa8dcde8bc" integrity sha512-5RLK0Qfs0PNDpEyBXIr3bIT1Muw3ojSlvpw6dAmkUcO0+uTrsBn7GuEIgx40u+OzbCBLDta7nvmud85P4EmTsQ== -cjs-module-lexer@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-0.5.0.tgz#25ca2a6799f5ead21d006293c69a198789b488c6" - integrity sha512-L0yxs8h0rEzHxkqqYbHmCHmS43hO/gqD93vVBC/qKaIpl/xVLovKsmM4RjGnP3YmeQ8HTsO73oDcUxBj/1WmWg== +cjs-module-lexer@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.0.0.tgz#c125ff0f4ab2c898dda909352f254d55e2213261" + integrity sha512-bLSEjEwg4knnuXt7LIWegvgTOClk6ZonZY6g4CFGBly1EjRqVjTjI8Dwnb/dsu1PwJjYBKxnguE5bRTdk+bFOA== "cjs-named-export-pkg-02@file:./test/esinstall/auto-named-exports/packages/cjs-named-export-pkg-02": version "1.2.3" @@ -5655,6 +5674,16 @@ css-select@^2.0.0: domutils "^1.7.0" nth-check "^1.0.2" +css-select@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/css-select/-/css-select-1.2.0.tgz#2b3a110539c5355f1cd8d314623e870b121ec858" + integrity sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg= + dependencies: + boolbase "~1.0.0" + css-what "2.1" + domutils "1.5.1" + nth-check "~1.0.1" + css-selector-tokenizer@^0.7.0: version "0.7.3" resolved "https://registry.yarnpkg.com/css-selector-tokenizer/-/css-selector-tokenizer-0.7.3.tgz#735f26186e67c749aaf275783405cf0661fae8f1" @@ -5687,6 +5716,11 @@ css-tree@^1.0.0: mdn-data "2.0.12" source-map "^0.6.1" +css-what@2.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/css-what/-/css-what-2.1.3.tgz#a6d7604573365fe74686c3f311c56513d88285f2" + integrity sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg== + css-what@^3.2.1: version "3.4.2" resolved "https://registry.yarnpkg.com/css-what/-/css-what-3.4.2.tgz#ea7026fcb01777edbde52124e21f327e7ae950e4" @@ -6233,12 +6267,20 @@ dom-serializer@0: domelementtype "^2.0.1" entities "^2.0.0" +dom-serializer@~0.1.1: + version "0.1.1" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-0.1.1.tgz#1ec4059e284babed36eec2941d4a970a189ce7c0" + integrity sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA== + dependencies: + domelementtype "^1.3.0" + entities "^1.1.1" + domain-browser@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.2.0.tgz#3d31f50191a6749dd1375a7f522e823d42e54eda" integrity sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA== -domelementtype@1: +domelementtype@1, domelementtype@^1.3.0, domelementtype@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.1.tgz#d048c44b37b0d10a7f2a3d5fee3f4333d790481f" integrity sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w== @@ -6255,7 +6297,22 @@ domexception@^2.0.1: dependencies: webidl-conversions "^5.0.0" -domutils@^1.7.0: +domhandler@^2.3.0: + version "2.4.2" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.4.2.tgz#8805097e933d65e85546f726d60f5eb88b44f803" + integrity sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA== + dependencies: + domelementtype "1" + +domutils@1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.5.1.tgz#dcd8488a26f563d61079e48c9f7b7e32373682cf" + integrity sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8= + dependencies: + dom-serializer "0" + domelementtype "1" + +domutils@^1.5.1, domutils@^1.7.0: version "1.7.0" resolved "https://registry.yarnpkg.com/domutils/-/domutils-1.7.0.tgz#56ea341e834e06e6748af7a1cb25da67ea9f8c2a" integrity sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg== @@ -6514,6 +6571,11 @@ enhanced-resolve@^4.1.1, enhanced-resolve@^4.3.0: memory-fs "^0.5.0" tapable "^1.0.0" +entities@^1.1.1, entities@~1.1.1: + version "1.1.2" + resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56" + integrity sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w== + entities@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" @@ -6640,6 +6702,11 @@ esbuild@^0.8.0: resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.8.0.tgz#3173e851303dce0682ebbcbaf6072b991dd6f60e" integrity sha512-xCHJpLRlU0NIANQHNsiMDNC/HlrKoye7iH5YOcoZNurauUZgMhjmm9PCal+Oo9ARYZrWxN15mykbfCX//UEvng== +esbuild@^0.8.7: + version "0.8.11" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.8.11.tgz#45b0066433463bb16e1bfef1cb7b2f8822de32fc" + integrity sha512-PTBeZ71qh7/Dm/57Sd10VG9TMKQAUbM9W6WD59ZYV62dDA/2a1xybzWqR3X7zbtxyqtFfY1PhYtg85QV0mrXNg== + escalade@^3.1.0, escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -7854,6 +7921,18 @@ html-minifier@^4.0.0: relateurl "^0.2.7" uglify-js "^3.5.1" +htmlparser2@^3.9.1: + version "3.10.1" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-3.10.1.tgz#bd679dc3f59897b6a34bb10749c855bb53a9392f" + integrity sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ== + dependencies: + domelementtype "^1.3.1" + domhandler "^2.3.0" + domutils "^1.5.1" + entities "^1.1.1" + inherits "^2.0.1" + readable-stream "^3.1.1" + http-assert@^1.3.0: version "1.4.1" resolved "https://registry.yarnpkg.com/http-assert/-/http-assert-1.4.1.tgz#c5f725d677aa7e873ef736199b89686cceb37878" @@ -9710,7 +9789,7 @@ lodash.uniq@^4.5.0: resolved "https://registry.yarnpkg.com/lodash.uniq/-/lodash.uniq-4.5.0.tgz#d0225373aeb652adc1bc82e4945339a842754773" integrity sha1-0CJTc662Uq3BvILklFM5qEJ1R3M= -"lodash@>=3.5 <5", lodash@>=4.17.19, lodash@^4.0.0, lodash@^4.17.10, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.1, lodash@~4.17.10: +"lodash@>=3.5 <5", lodash@>=4.17.19, lodash@^4.0.0, lodash@^4.15.0, lodash@^4.17.10, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.17.19, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.1, lodash@~4.17.10: version "4.17.20" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== @@ -10683,7 +10762,7 @@ npmlog@^4.0.0, npmlog@^4.1.2: gauge "~2.7.3" set-blocking "~2.0.0" -nth-check@^1.0.2: +nth-check@^1.0.2, nth-check@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/nth-check/-/nth-check-1.0.2.tgz#b2bd295c37e3dd58a3bf0700376663ba4d9cf05c" integrity sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg== @@ -11204,6 +11283,13 @@ parse5@5.1.1: resolved "https://registry.yarnpkg.com/parse5/-/parse5-5.1.1.tgz#f68e4e5ba1852ac2cadc00f4555fff6c2abb6178" integrity sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug== +parse5@^3.0.1: + version "3.0.3" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-3.0.3.tgz#042f792ffdd36851551cf4e9e066b3874ab45b5c" + integrity sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA== + dependencies: + "@types/node" "*" + parse5@^6.0.0, parse5@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b"