add import map support (#1545)

* add true snowpack.lock.json support

* update snapshots

skypack/package.json

@@ -8,7 +8,6 @@
     "cdn",
     "sdk"
   ],
-  "private": true,
   "author": "Fred K. Schott <[email protected]>",
   "license": "MIT",
   "publishConfig": {

skypack/src/index.ts

@@ -17,15 +17,15 @@ function parseRawPackageImport(spec: string): [string, string | null] {
 }
 
 export async function generateImportMap(
-  webDependencies: Record<string, string>,
+  webDependencies: Record<string, string | null>,
   inheritFromImportMap?: ImportMap,
 ): Promise<ImportMap> {
-  const newLockfile: ImportMap = {imports: {}};
+  const newLockfile: ImportMap = inheritFromImportMap ? {imports: {...inheritFromImportMap.imports}} : {imports: {}};
   await Promise.all(
     Object.entries(webDependencies).map(async ([packageName, packageSemver]) => {
-      if (inheritFromImportMap && inheritFromImportMap.imports[packageName]) {
-        newLockfile.imports[packageName] = inheritFromImportMap.imports[packageName];
-        newLockfile.imports[packageName + '/'] = inheritFromImportMap.imports[packageName + '/'];
+      if (packageSemver === null) {
+        delete newLockfile.imports[packageName];
+        delete newLockfile.imports[packageName + '/'];
         return;
       }
       const lookupResponse = await lookupBySpecifier(packageName, packageSemver);
@@ -46,8 +46,8 @@ export async function generateImportMap(
             deepPinnedUrlParts.push(investigate);
           }
         }
-        newLockfile.imports[packageName] = '/' + deepPinnedUrlParts.join('/');
-        newLockfile.imports[packageName + '/'] = '/' + deepPinnedUrlParts.join('/') + '/';
+        newLockfile.imports[packageName] = SKYPACK_ORIGIN + '/' + deepPinnedUrlParts.join('/');
+        newLockfile.imports[packageName + '/'] = SKYPACK_ORIGIN + '/' + deepPinnedUrlParts.join('/') + '/';
       }
     }),
   );
@@ -102,7 +102,7 @@ export async function fetchCDN(
   let freshResult: Response<string>;
   try {
     freshResult = await got(resourceUrl, {
-      headers: {'user-agent': userAgent || `snowpack/v3.0 (https://www.snowpack.dev)`},
+      headers: {'user-agent': userAgent || `skypack/v0.0.1`},
       throwHttpErrors: false,
     });
   } catch (err) {
@@ -196,7 +196,10 @@ export async function lookupBySpecifier(
     (semverString ? `@${semverString}` : ``) +
     (packagePath ? `/${packagePath}` : ``);
   try {
-    const {body, headers, isCached, isStale} = await fetchCDN(lookupUrl, userAgent);
+    const {body, statusCode, headers, isCached, isStale} = await fetchCDN(lookupUrl, userAgent);
+    if (statusCode !== 200) {
+      return {error: new Error(body)};
+    }
     return {
       error: null,
       body,

snowpack/package.json

@@ -82,6 +82,7 @@
     "signal-exit": "^3.0.3",
     "source-map": "^0.7.3",
     "strip-ansi": "^6.0.0",
+    "skypack": "^0.0.1",
     "strip-comments": "^2.0.1",
     "validate-npm-package-name": "^3.0.0",
     "ws": "^7.3.0",

snowpack/src/build/import-resolver.ts

@@ -9,7 +9,8 @@ const cwd = process.cwd();
 
 interface ImportResolverOptions {
   fileLoc: string;
-  dependencyImportMap: ImportMap | null | undefined;
+  lockfile?: ImportMap | null;
+  installImportMap?: ImportMap | null;
   config: SnowpackConfig;
 }
 
@@ -50,20 +51,29 @@ function resolveSourceSpecifier(spec: string, stats: fs.Stats | false, config: S
  */
 export function createImportResolver({
   fileLoc,
-  dependencyImportMap,
+  lockfile,
+  installImportMap,
   config,
 }: ImportResolverOptions) {
   return function importResolver(spec: string): string | false {
     // Ignore "http://*" imports
     if (url.parse(spec).protocol) {
       return spec;
     }
-
     // Ignore packages marked as external
     if (config.installOptions.externalPackage?.includes(spec)) {
       return spec;
     }
-
+    // Support snowpack.lock.json entry
+    if (lockfile && lockfile.imports[spec]) {
+      const mappedImport = lockfile.imports[spec];
+      if (url.parse(mappedImport).protocol) {
+        return mappedImport;
+      }
+      throw new Error(
+        `Not yet supported: "${spec}" lockfile entry must be a full URL (https://...).`,
+      );
+    }
     if (spec.startsWith('/')) {
       const importStats = getImportStats(path.resolve(cwd, spec.substr(1)));
       return resolveSourceSpecifier(spec, importStats, config);
@@ -83,13 +93,11 @@ export function createImportResolver({
       const newSpec = getUrlForFile(importedFileLoc, config) || spec;
       return resolveSourceSpecifier(newSpec, importStats, config);
     }
-    if (dependencyImportMap) {
+    if (installImportMap && installImportMap.imports[spec]) {
       // NOTE: We don't need special handling for an alias here, since the aliased "from"
       // is already the key in the import map. The aliased "to" value is also an entry.
-      const importMapEntry = dependencyImportMap.imports[spec];
-      if (importMapEntry) {
-        return path.posix.resolve(config.buildOptions.webModulesUrl, importMapEntry);
-      }
+      const importMapEntry = installImportMap.imports[spec];
+      return path.posix.resolve(config.buildOptions.webModulesUrl, importMapEntry);
     }
     return false;
   };

snowpack/src/commands/add-rm.ts

@@ -1,31 +1,33 @@
-import {promises as fs} from 'fs';
 import {send} from 'httpie';
+import {cyan, dim, underline} from 'kleur/colors';
 import path from 'path';
+import {generateImportMap} from 'skypack';
+import {logger} from '../logger';
 import {CommandOptions} from '../types/snowpack';
-import {command as installCommand} from './install';
+import {writeLockfile} from '../util';
 
 export async function addCommand(addValue: string, commandOptions: CommandOptions) {
-  const {cwd, config, pkgManifest} = commandOptions;
+  const {cwd, lockfile} = commandOptions;
   let [pkgName, pkgSemver] = addValue.split('@');
-  if (!pkgSemver) {
+  const installMessage = pkgSemver ? `${pkgName}@${pkgSemver}` : pkgName;
+  logger.info(`fetching ${cyan(installMessage)} from Skypack CDN...`);
+  if (!pkgSemver || pkgSemver === 'latest') {
     const {data} = await send('GET', `http://registry.npmjs.org/${pkgName}/latest`);
     pkgSemver = `^${data.version}`;
   }
-  pkgManifest.webDependencies = pkgManifest.webDependencies || {};
-  pkgManifest.webDependencies[pkgName] = pkgSemver;
-  config.webDependencies = config.webDependencies || {};
-  config.webDependencies[pkgName] = pkgSemver;
-  await fs.writeFile(path.join(cwd, 'package.json'), JSON.stringify(pkgManifest, null, 2));
-  await installCommand(commandOptions);
+  logger.info(
+    `adding ${cyan(
+      underline(`https://cdn.skypack.dev/${pkgName}@${pkgSemver}`),
+    )} to your project lockfile. ${dim('(snowpack.lock.json)')}`,
+  );
+  const newLockfile = await generateImportMap({[pkgName]: pkgSemver}, lockfile || undefined);
+  await writeLockfile(path.join(cwd, 'snowpack.lock.json'), newLockfile);
 }
 
 export async function rmCommand(addValue: string, commandOptions: CommandOptions) {
-  const {cwd, config, pkgManifest} = commandOptions;
+  const {cwd, lockfile} = commandOptions;
   let [pkgName] = addValue.split('@');
-  pkgManifest.webDependencies = pkgManifest.webDependencies || {};
-  delete pkgManifest.webDependencies[pkgName];
-  config.webDependencies = config.webDependencies || {};
-  delete config.webDependencies[pkgName];
-  await fs.writeFile(path.join(cwd, 'package.json'), JSON.stringify(pkgManifest, null, 2));
-  await installCommand(commandOptions);
+  logger.info(`removing ${cyan(pkgName)} from project lockfile...`);
+  const newLockfile = await generateImportMap({[pkgName]: null}, lockfile || undefined);
+  await writeLockfile(path.join(cwd, 'snowpack.lock.json'), newLockfile);
 }

snowpack/src/commands/build.ts

@@ -69,13 +69,17 @@ async function installOptimizedDependencies(
   // will can scan all used entrypoints. Set to `[]` to improve tree-shaking performance.
   installConfig.knownEntrypoints = [];
   // 1. Scan imports from your final built JS files.
-  const installTargets = await getInstallTargets(installConfig, scannedFiles);
+  const installTargets = await getInstallTargets(
+    installConfig,
+    commandOptions.lockfile,
+    scannedFiles,
+  );
   // 2. Install dependencies, based on the scan of your final build.
   const installResult = await installRunner({
     ...commandOptions,
     installTargets,
     config: installConfig,
-    shouldPrintStats: true,
+    shouldPrintStats: false,
     shouldWriteLockfile: false,
   });
   return installResult;
@@ -95,22 +99,26 @@ class FileBuilder {
   readonly mountEntry: MountEntry;
   readonly outDir: string;
   readonly config: SnowpackConfig;
+  readonly lockfile: ImportMap | null;
 
   constructor({
     filepath,
     mountEntry,
     outDir,
     config,
+    lockfile,
   }: {
     filepath: string;
     mountEntry: MountEntry;
     outDir: string;
     config: SnowpackConfig;
+    lockfile: ImportMap | null;
   }) {
     this.filepath = filepath;
     this.mountEntry = mountEntry;
     this.outDir = outDir;
     this.config = config;
+    this.lockfile = lockfile;
   }
 
   async buildFile() {
@@ -203,7 +211,8 @@ class FileBuilder {
       const file = rawFile as SnowpackSourceFile<string>;
       const resolveImportSpecifier = createImportResolver({
         fileLoc: file.locOnDisk!, // we’re confident these are reading from disk because we just read them
-        dependencyImportMap: importMap,
+        lockfile: this.lockfile,
+        installImportMap: importMap,
         config: this.config,
       });
       const resolvedCode = await transformFileImports(file, (spec) => {
@@ -295,7 +304,7 @@ class FileBuilder {
 }
 
 export async function command(commandOptions: CommandOptions) {
-  const {config} = commandOptions;
+  const {config, lockfile} = commandOptions;
   const isDev = !!config.buildOptions.watch;
   const isSSR = !!config.experiments.ssr;
 
@@ -359,7 +368,7 @@ export async function command(commandOptions: CommandOptions) {
     hmrEngine = new EsmHmrEngine({port: config.devOptions.hmrPort});
   }
 
-  logger.info(colors.yellow('! building source…'));
+  logger.info(colors.yellow('! building source files...'));
   const buildStart = performance.now();
   const buildPipelineFiles: Record<string, FileBuilder> = {};
 
@@ -412,7 +421,13 @@ export async function command(commandOptions: CommandOptions) {
       const finalUrl = getUrlForFileMount({fileLoc, mountKey: mountedDir, mountEntry, config})!;
       const finalDestLoc = path.join(buildDirectoryLoc, finalUrl);
       const outDir = path.dirname(finalDestLoc);
-      const buildPipelineFile = new FileBuilder({filepath: fileLoc, mountEntry, outDir, config});
+      const buildPipelineFile = new FileBuilder({
+        filepath: fileLoc,
+        mountEntry,
+        outDir,
+        config,
+        lockfile,
+      });
       buildPipelineFiles[fileLoc] = buildPipelineFile;
     }
   }
@@ -440,6 +455,7 @@ export async function command(commandOptions: CommandOptions) {
   logger.info(colors.yellow('! verifying build...'));
 
   // 3. Resolve all built file imports.
+  const verifyStart = performance.now();
   for (const buildPipelineFile of allBuildPipelineFiles) {
     parallelWorkQueue.add(() =>
       buildPipelineFile
@@ -448,7 +464,12 @@ export async function command(commandOptions: CommandOptions) {
     );
   }
   await parallelWorkQueue.onIdle();
-  logger.info(`${colors.green('✔')} verification complete`);
+  const verifyEnd = performance.now();
+  logger.info(
+    `${colors.green('✔')} verification complete ${colors.dim(
+      `[${((verifyEnd - verifyStart) / 1000).toFixed(2)}s]`,
+    )}`,
+  );
 
   // 4. Write files to disk.
   logger.info(colors.yellow('! writing build to disk...'));
@@ -469,14 +490,9 @@ export async function command(commandOptions: CommandOptions) {
   }
   await parallelWorkQueue.onIdle();
 
-  logger.info(
-    `${colors.green('✔')} build complete ${colors.dim(
-      `[${((buildEnd - buildStart) / 1000).toFixed(2)}s]`,
-    )}`,
-  );
-
   // 5. Optimize the build.
   if (!config.buildOptions.watch) {
+    logger.info(colors.yellow('! optimizing build...'));
     await runPipelineCleanupStep(config);
     await runPipelineOptimizeStep(buildDirectoryLoc, {
       plugins: config.plugins,
@@ -485,7 +501,7 @@ export async function command(commandOptions: CommandOptions) {
       isHmrEnabled: false,
       sourceMaps: config.buildOptions.sourceMaps,
     });
-    logger.info(`${colors.underline(colors.green(colors.bold('▶ Build Complete!')))}\n\n`);
+    logger.info(`${colors.underline(colors.green(colors.bold('▶ Build Complete!')))}`);
     return;
   }
 
@@ -515,7 +531,13 @@ export async function command(commandOptions: CommandOptions) {
     const finalDest = path.join(buildDirectoryLoc, finalUrl);
     const outDir = path.dirname(finalDest);
 
-    const changedPipelineFile = new FileBuilder({filepath: fileLoc, mountEntry, outDir, config});
+    const changedPipelineFile = new FileBuilder({
+      filepath: fileLoc,
+      mountEntry,
+      outDir,
+      config,
+      lockfile,
+    });
     buildPipelineFiles[fileLoc] = changedPipelineFile;
     // 1. Build the file.
     await changedPipelineFile.buildFile().catch((err) => {

snowpack/src/commands/dev.ts

@@ -156,7 +156,7 @@ class NotFoundError extends Error {
  */
 async function installDependencies(commandOptions: CommandOptions) {
   const {config} = commandOptions;
-  const installTargets = await getInstallTargets(config);
+  const installTargets = await getInstallTargets(config, commandOptions.lockfile);
   if (installTargets.length === 0) {
     logger.info('Nothing to install.');
     return;
@@ -286,6 +286,7 @@ function handleResponseError(req, res, err: Error | NotFoundError) {
 }
 
 export async function startDevServer(commandOptions: CommandOptions): Promise<SnowpackDevServer> {
+  const {lockfile} = commandOptions;
   // Start the startup timer!
   let serverStart = performance.now();
 
@@ -744,7 +745,8 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
       let missingPackages: string[] = [];
       const resolveImportSpecifier = createImportResolver({
         fileLoc,
-        dependencyImportMap,
+        lockfile: lockfile,
+        installImportMap: dependencyImportMap,
         config,
       });
       wrappedResponse = await transformFileImports(
@@ -764,7 +766,7 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
           }
           // Ignore "http://*" imports
           if (url.parse(resolvedImportUrl).protocol) {
-            return spec;
+            return resolvedImportUrl;
           }
           // Ignore packages marked as external
           if (config.installOptions.externalPackage?.includes(resolvedImportUrl)) {