From 70ad6a8a3e079579bb89073b24b40e20366896a8 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Wed, 9 Dec 2020 22:21:57 -0800 Subject: [PATCH] update source:skypack support --- .../app-template-react/snowpack.config.js | 6 +-- .../rollup-plugin-wrap-install-targets.ts | 4 ++ skypack/src/rollup-plugin-remote-cdn.ts | 4 +- snowpack/index.esm.mjs | 1 + snowpack/src/build/import-resolver.ts | 31 ++++---------- snowpack/src/commands/add-rm.ts | 17 ++++++-- snowpack/src/commands/build.ts | 7 ++-- snowpack/src/commands/dev.ts | 1 - snowpack/src/commands/install.ts | 16 ------- snowpack/src/index.ts | 1 + snowpack/src/sources/skypack.ts | 23 +++++++--- snowpack/src/types/snowpack.ts | 16 +++++-- snowpack/src/util.ts | 42 +++++++++++-------- 13 files changed, 91 insertions(+), 78 deletions(-) diff --git a/create-snowpack-app/app-template-react/snowpack.config.js b/create-snowpack-app/app-template-react/snowpack.config.js index 426315b638..da7bfdea56 100644 --- a/create-snowpack-app/app-template-react/snowpack.config.js +++ b/create-snowpack-app/app-template-react/snowpack.config.js @@ -20,7 +20,7 @@ module.exports = { proxy: { /* ... */ }, - alias: { - /* ... */ - }, + experiments: { + source: 'skypack' + } }; diff --git a/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts b/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts index 8b8cd2c6d1..cf323f7d69 100644 --- a/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts +++ b/esinstall/src/rollup-plugins/rollup-plugin-wrap-install-targets.ts @@ -1,5 +1,6 @@ import * as colors from 'kleur/colors'; import path from 'path'; +import url from 'url'; import fs from 'fs'; import {VM as VM2} from 'vm2'; import {Plugin} from 'rollup'; @@ -115,6 +116,9 @@ export function rollupPluginWrapInstallTargets( buildStart(inputOptions) { const input = inputOptions.input as {[entryAlias: string]: string}; for (const [key, val] of Object.entries(input)) { + if (url.parse(val).protocol) { + continue; + } const allInstallTargets = installTargets.filter( (imp) => getWebDependencyName(imp.specifier) === key, ); diff --git a/skypack/src/rollup-plugin-remote-cdn.ts b/skypack/src/rollup-plugin-remote-cdn.ts index 2524477458..007b2b5b5a 100644 --- a/skypack/src/rollup-plugin-remote-cdn.ts +++ b/skypack/src/rollup-plugin-remote-cdn.ts @@ -1,7 +1,7 @@ import cacache from 'cacache'; import {OutputOptions, Plugin, ResolvedId} from 'rollup'; import {fetchCDN} from './index'; -import {SKYPACK_ORIGIN, HAS_CDN_HASH_REGEX, RESOURCE_CACHE} from './util'; +import {SKYPACK_ORIGIN, HAS_CDN_HASH_REGEX, RESOURCE_CACHE, ImportMap} from './util'; const CACHED_FILE_ID_PREFIX = 'snowpack-pkg-cache:'; const PIKA_CDN_TRIM_LENGTH = SKYPACK_ORIGIN.length; @@ -15,8 +15,10 @@ const PIKA_CDN_TRIM_LENGTH = SKYPACK_ORIGIN.length; * rollup that it's safe to load from the cache in the `load()` hook. */ export function rollupPluginSkypack({ + lockfile, installTypes, }: { + lockfile?: ImportMap | null; installTypes: boolean; }) { // const allTypesToInstall = new Set(); diff --git a/snowpack/index.esm.mjs b/snowpack/index.esm.mjs index 5c482a6fe2..6cf6912272 100644 --- a/snowpack/index.esm.mjs +++ b/snowpack/index.esm.mjs @@ -4,4 +4,5 @@ export const startDevServer = Pkg.startDevServer; export const buildProject = Pkg.buildProject; export const loadAndValidateConfig = Pkg.loadAndValidateConfig; export const createConfiguration = Pkg.createConfiguration; +export const loadLockfile = Pkg.loadLockfile; export const getUrlForFile = Pkg.getUrlForFile; diff --git a/snowpack/src/build/import-resolver.ts b/snowpack/src/build/import-resolver.ts index 5501cd6d0b..e97a776b44 100644 --- a/snowpack/src/build/import-resolver.ts +++ b/snowpack/src/build/import-resolver.ts @@ -1,17 +1,11 @@ import fs from 'fs'; import path from 'path'; -import { ImportMap, SnowpackConfig } from '../types/snowpack'; -import { findMatchingAliasEntry, getExt, isRemoteSpecifier, replaceExt } from '../util'; -import { getUrlForFile } from './file-urls'; +import {ImportMap, SnowpackConfig} from '../types/snowpack'; +import {findMatchingAliasEntry, getExt, isRemoteSpecifier, replaceExt} from '../util'; +import {getUrlForFile} from './file-urls'; const cwd = process.cwd(); -interface ImportResolverOptions { - fileLoc: string; - lockfile?: ImportMap | null; - config: SnowpackConfig; -} - /** Perform a file disk lookup for the requested import specifier. */ export function getImportStats(importedFileOnDisk: string): fs.Stats | false { try { @@ -47,11 +41,7 @@ function resolveSourceSpecifier(spec: string, stats: fs.Stats | false, config: S * to a proper URL. Returns false if no matching import was found, which usually indicates a package * not found in the import map. */ -export function createImportResolver({ - fileLoc, - lockfile, - config, -}: ImportResolverOptions) { +export function createImportResolver({fileLoc, config}: {fileLoc: string; config: SnowpackConfig}) { return function importResolver(spec: string): string | false { // Ignore "http://*" imports if (isRemoteSpecifier(spec)) { @@ -61,14 +51,6 @@ export function createImportResolver({ if (config.installOptions.externalPackage?.includes(spec)) { return spec; } - // Support snowpack.lock.json entry - if (lockfile && lockfile.imports[spec]) { - const mappedImport = lockfile.imports[spec]; - if (isRemoteSpecifier(mappedImport)) { - return mappedImport; - } - throw new Error(`Not supported: "${spec}" lockfile entry must be a full URL (https://...).`); - } if (spec.startsWith('/')) { const importStats = getImportStats(path.resolve(cwd, spec.substr(1))); return resolveSourceSpecifier(spec, importStats, config); @@ -80,9 +62,12 @@ export function createImportResolver({ return resolveSourceSpecifier(newSpec, importStats, config); } const aliasEntry = findMatchingAliasEntry(config, spec); - if (aliasEntry && aliasEntry.type === 'path') { + if (aliasEntry && (aliasEntry.type === 'path' || aliasEntry.type === 'url')) { const {from, to} = aliasEntry; let result = spec.replace(from, to); + if (aliasEntry.type === 'url') { + return result; + } const importedFileLoc = path.resolve(cwd, result); const importStats = getImportStats(importedFileLoc); const newSpec = getUrlForFile(importedFileLoc, config) || spec; diff --git a/snowpack/src/commands/add-rm.ts b/snowpack/src/commands/add-rm.ts index 5fb6e882ab..ebf1df17e9 100644 --- a/snowpack/src/commands/add-rm.ts +++ b/snowpack/src/commands/add-rm.ts @@ -3,7 +3,7 @@ import {cyan, dim, underline} from 'kleur/colors'; import path from 'path'; import {generateImportMap} from 'skypack'; import {logger} from '../logger'; -import {CommandOptions} from '../types/snowpack'; +import {CommandOptions, LockfileManifest} from '../types/snowpack'; import {writeLockfile} from '../util'; export async function addCommand(addValue: string, commandOptions: CommandOptions) { @@ -20,7 +20,14 @@ export async function addCommand(addValue: string, commandOptions: CommandOption underline(`https://cdn.skypack.dev/${pkgName}@${pkgSemver}`), )} to your project lockfile. ${dim('(snowpack.lock.json)')}`, ); - const newLockfile = await generateImportMap({[pkgName]: pkgSemver}, lockfile || undefined); + const addedDependency = {[pkgName]: pkgSemver}; + const newLockfile: LockfileManifest = { + ...(await generateImportMap(addedDependency, lockfile || undefined)), + dependencies: { + ...lockfile?.dependencies, + ...addedDependency, + }, + }; await writeLockfile(path.join(cwd, 'snowpack.lock.json'), newLockfile); } @@ -28,6 +35,10 @@ export async function rmCommand(addValue: string, commandOptions: CommandOptions const {cwd, lockfile} = commandOptions; let [pkgName] = addValue.split('@'); logger.info(`removing ${cyan(pkgName)} from project lockfile...`); - const newLockfile = await generateImportMap({[pkgName]: null}, lockfile || undefined); + const newLockfile: LockfileManifest = { + ...(await generateImportMap({[pkgName]: null}, lockfile || undefined)), + dependencies: lockfile?.dependencies ?? {}, + }; + delete newLockfile.dependencies[pkgName]; await writeLockfile(path.join(cwd, 'snowpack.lock.json'), newLockfile); } diff --git a/snowpack/src/commands/build.ts b/snowpack/src/commands/build.ts index 3d53c7413e..65055f83d9 100644 --- a/snowpack/src/commands/build.ts +++ b/snowpack/src/commands/build.ts @@ -87,7 +87,7 @@ async function installOptimizedDependencies( }); const pkgSource = getPackageSource(commandOptions.config.experiments.source); - pkgSource.modifyBuildInstallConfig(installConfig); + pkgSource.modifyBuildInstallConfig({config: installConfig, lockfile: commandOptions.lockfile}); // Unlike dev (where we scan from source code) the built output guarantees that we // will can scan all used entrypoints. Set to `[]` to improve tree-shaking performance. @@ -239,7 +239,6 @@ class FileBuilder { const file = rawFile as SnowpackSourceFile; const resolveImportSpecifier = createImportResolver({ fileLoc: file.locOnDisk!, // we’re confident these are reading from disk because we just read them - lockfile: this.lockfile, config: this.config, }); const resolvedCode = await transformFileImports(file, (spec) => { @@ -247,7 +246,7 @@ class FileBuilder { let resolvedImportUrl = resolveImportSpecifier(spec); // If not resolved, then this is a package. During build, dependencies are always // installed locally via esinstall, so use localPackageSource here. - if (!resolvedImportUrl) { + if (importMap.imports[spec]) { resolvedImportUrl = localPackageSource.resolvePackageImport(spec, importMap, this.config); } // If still not resolved, then this imported package somehow evaded detection @@ -259,7 +258,7 @@ class FileBuilder { } // Ignore "http://*" imports if (isRemoteSpecifier(resolvedImportUrl)) { - return spec; + return resolvedImportUrl; } // Ignore packages marked as external if (this.config.installOptions.externalPackage?.includes(resolvedImportUrl)) { diff --git a/snowpack/src/commands/dev.ts b/snowpack/src/commands/dev.ts index 9a9cfeb775..9e0d505e72 100644 --- a/snowpack/src/commands/dev.ts +++ b/snowpack/src/commands/dev.ts @@ -712,7 +712,6 @@ export async function startDevServer(commandOptions: CommandOptions): Promise ent.endsWith('/')); - installTargets = installTargets.filter((t) => { - if (lockfile.imports[t.specifier]) { - return false; - } - if ( - t.specifier.includes('/') && - importMapSubPaths.some((ent) => t.specifier.startsWith(ent)) - ) { - return false; - } - return true; - }); - } return installTargets; } @@ -110,7 +95,6 @@ export async function run({ } let newLockfile: ImportMap | null = null; - const finalResult = await install(installTargets, { cwd, lockfile: newLockfile || undefined, diff --git a/snowpack/src/index.ts b/snowpack/src/index.ts index d7a71e448f..076e15ed43 100644 --- a/snowpack/src/index.ts +++ b/snowpack/src/index.ts @@ -17,6 +17,7 @@ export * from './types/snowpack'; export {startDevServer} from './commands/dev'; export {buildProject} from './commands/build'; export {loadConfigurationForCLI as loadAndValidateConfig, createConfiguration} from './config.js'; +export {readLockfile as loadLockfile} from './util.js'; export {getUrlForFile} from './build/file-urls'; const cwd = process.cwd(); diff --git a/snowpack/src/sources/skypack.ts b/snowpack/src/sources/skypack.ts index dd6503339e..7733ecb033 100644 --- a/snowpack/src/sources/skypack.ts +++ b/snowpack/src/sources/skypack.ts @@ -8,7 +8,7 @@ import { SKYPACK_ORIGIN, } from 'skypack'; import {logger} from '../logger'; -import {ImportMap, PackageSource, SnowpackConfig} from '../types/snowpack'; +import {ImportMap, LockfileManifest, PackageSource, SnowpackConfig} from '../types/snowpack'; const fetchedPackages = new Set(); function logFetching(packageName: string) { @@ -45,15 +45,24 @@ export default { return {imports: {}}; }, - async modifyBuildInstallConfig(config: SnowpackConfig) { - config.installOptions.rollup?.plugins?.push( - rollupPluginSkypack({installTypes: false}) as Plugin, + async modifyBuildInstallConfig({ + config, + lockfile, + }: { + config: SnowpackConfig; + lockfile: LockfileManifest | null; + }) { + config.installOptions.lockfile = lockfile || undefined; + config.installOptions.rollup = config.installOptions.rollup || {}; + config.installOptions.rollup.plugins = config.installOptions.rollup.plugins || []; + config.installOptions.rollup.plugins.push( + rollupPluginSkypack({installTypes: false, lockfile: lockfile || undefined}) as Plugin, ); }, async load( spec: string, - {config, lockfile}: {config: SnowpackConfig; lockfile: ImportMap | null}, + {config, lockfile}: {config: SnowpackConfig; lockfile: LockfileManifest | null}, ): Promise { let body: string; if ( @@ -70,7 +79,9 @@ export default { } else if (lockfile && lockfile.imports[packageName + '/']) { body = (await fetchCDN(lockfile.imports[packageName + '/'] + packagePath)).body; } else { - const _packageSemver = config.webDependencies && config.webDependencies[packageName]; + // TODO: When config.root is added, look up package.json "dependencies" & "devDependencies" + // from the root and fallback to those if lockfile.dependencies[packageName] doesn't exist. + const _packageSemver = lockfile?.dependencies && lockfile.dependencies[packageName]; if (!_packageSemver) { logFetching(packageName); } diff --git a/snowpack/src/types/snowpack.ts b/snowpack/src/types/snowpack.ts index 3f3cf15496..7dd4d7d26d 100644 --- a/snowpack/src/types/snowpack.ts +++ b/snowpack/src/types/snowpack.ts @@ -212,7 +212,6 @@ export interface SnowpackConfig { extends?: string; exclude: string[]; knownEntrypoints: string[]; - webDependencies: Record; mount: Record; alias: Record; scripts: Record; @@ -246,6 +245,8 @@ export interface SnowpackConfig { testOptions: { files: string[]; }; + // @deprecated - now found at lockfile.dependencies + webDependencies: Record; // @deprecated - Use experiments.routes instead proxy: Proxy[]; /** EXPERIMENTAL - This section is experimental and not yet finalized. May change across minor versions. */ @@ -306,14 +307,18 @@ export interface CLIFlags extends Omit { } export interface ImportMap { - imports: {[packageName: string]: string}; + imports: {[specifier: string]: string}; +} + +export interface LockfileManifest extends ImportMap { + dependencies: {[packageName: string]: string}; } export interface CommandOptions { // TODO(fks): remove `cwd`, replace with a new `config.root` property on SnowpackConfig. cwd: string; config: SnowpackConfig; - lockfile: ImportMap | null; + lockfile: LockfileManifest | null; } export type LoggerLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent'; // same as Pino @@ -346,5 +351,8 @@ export interface PackageSource { /** Handle 1+ missing package imports before failing, if possible. */ recoverMissingPackageImport(missingPackages: string[]): Promise; /** Modify the build install config for optimized build install. */ - modifyBuildInstallConfig(config: SnowpackConfig): Promise; + modifyBuildInstallConfig(options: { + config: SnowpackConfig; + lockfile: ImportMap | null; + }): Promise; } diff --git a/snowpack/src/util.ts b/snowpack/src/util.ts index 050146d4d1..a95323d9c9 100644 --- a/snowpack/src/util.ts +++ b/snowpack/src/util.ts @@ -15,7 +15,7 @@ import {clearCache as clearSkypackCache} from 'skypack'; import url from 'url'; import localPackageSource from './sources/local'; import skypackPackageSource from './sources/skypack'; -import {ImportMap, PackageSource, SnowpackConfig} from './types/snowpack'; +import {ImportMap, LockfileManifest, PackageSource, SnowpackConfig} from './types/snowpack'; export const GLOBAL_CACHE_DIR = globalCacheDir('snowpack'); @@ -51,7 +51,7 @@ export async function readFile(filepath: URL): Promise { return isBinary ? data : data.toString('utf-8'); } -export async function readLockfile(cwd: string): Promise { +export async function readLockfile(cwd: string): Promise { try { var lockfileContents = fs.readFileSync(path.join(cwd, 'snowpack.lock.json'), { encoding: 'utf-8', @@ -64,12 +64,18 @@ export async function readLockfile(cwd: string): Promise { return JSON.parse(lockfileContents); } -export async function writeLockfile(loc: string, importMap: ImportMap): Promise { - const sortedImportMap: ImportMap = {imports: {}}; - for (const key of Object.keys(importMap.imports).sort()) { - sortedImportMap.imports[key] = importMap.imports[key]; +function sortObject(originalObject: Record): Record { + const newObject = {}; + for (const key of Object.keys(originalObject).sort()) { + newObject[key] = originalObject[key]; } - fs.writeFileSync(loc, JSON.stringify(sortedImportMap, undefined, 2), {encoding: 'utf-8'}); + return newObject; +} + +export async function writeLockfile(loc: string, importMap: LockfileManifest): Promise { + importMap.dependencies = sortObject(importMap.dependencies); + importMap.imports = sortObject(importMap.imports); + fs.writeFileSync(loc, JSON.stringify(importMap, undefined, 2), {encoding: 'utf-8'}); } export function isTruthy(item: T | false | null | undefined): item is T { @@ -256,13 +262,23 @@ export async function clearCache() { ]); } +/** + * For the given import specifier, return an alias entry if one is matched. + */ +export function getAliasType(val: string): 'package' | 'path' | 'url' { + if (url.parse(val).protocol) { + return 'url'; + } + return !path.isAbsolute(val) ? 'package' : 'path'; +} + /** * For the given import specifier, return an alias entry if one is matched. */ export function findMatchingAliasEntry( config: SnowpackConfig, spec: string, -): {from: string; to: string; type: 'package' | 'path'} | undefined { +): {from: string; to: string; type: 'package' | 'path' | 'url'} | undefined { // Only match bare module specifiers. relative and absolute imports should not match if ( spec === '.' || @@ -277,26 +293,18 @@ export function findMatchingAliasEntry( } for (const [from, to] of Object.entries(config.alias)) { - let foundType: 'package' | 'path' = isPackageAliasEntry(to) ? 'package' : 'path'; const isExactMatch = spec === from; const isDeepMatch = spec.startsWith(addTrailingSlash(from)); if (isExactMatch || isDeepMatch) { return { from, to, - type: foundType, + type: getAliasType(to), }; } } } -/** - * For the given import specifier, return an alias entry if one is matched. - */ -export function isPackageAliasEntry(val: string): boolean { - return !path.isAbsolute(val); -} - /** Get full extensions of files */ export function getExt(fileName: string) { return {