add experiments.source (local, skypack)

snowpack/package.json

@@ -86,6 +86,7 @@
     "skypack": "^0.0.1",
     "source-map": "^0.7.3",
     "strip-ansi": "^6.0.0",
+    "skypack": "^0.0.1",
     "strip-comments": "^2.0.1",
     "validate-npm-package-name": "^3.0.0",
     "ws": "^7.3.0",

snowpack/src/build/import-resolver.ts

@@ -1,15 +1,14 @@
 import fs from 'fs';
 import path from 'path';
-import {ImportMap, SnowpackConfig} from '../types/snowpack';
-import {findMatchingAliasEntry, getExt, isRemoteSpecifier, replaceExt} from '../util';
-import {getUrlForFile} from './file-urls';
+import { ImportMap, SnowpackConfig } from '../types/snowpack';
+import { findMatchingAliasEntry, getExt, isRemoteSpecifier, replaceExt } from '../util';
+import { getUrlForFile } from './file-urls';
 
 const cwd = process.cwd();
 
 interface ImportResolverOptions {
   fileLoc: string;
   lockfile?: ImportMap | null;
-  installImportMap?: ImportMap | null;
   config: SnowpackConfig;
 }
 
@@ -51,7 +50,6 @@ function resolveSourceSpecifier(spec: string, stats: fs.Stats | false, config: S
 export function createImportResolver({
   fileLoc,
   lockfile,
-  installImportMap,
   config,
 }: ImportResolverOptions) {
   return function importResolver(spec: string): string | false {
@@ -69,9 +67,7 @@ export function createImportResolver({
       if (isRemoteSpecifier(mappedImport)) {
         return mappedImport;
       }
-      throw new Error(
-        `Not yet supported: "${spec}" lockfile entry must be a full URL (https://...).`,
-      );
+      throw new Error(`Not supported: "${spec}" lockfile entry must be a full URL (https://...).`);
     }
     if (spec.startsWith('/')) {
       const importStats = getImportStats(path.resolve(cwd, spec.substr(1)));
@@ -92,12 +88,6 @@ export function createImportResolver({
       const newSpec = getUrlForFile(importedFileLoc, config) || spec;
       return resolveSourceSpecifier(newSpec, importStats, config);
     }
-    if (installImportMap && installImportMap.imports[spec]) {
-      // NOTE: We don't need special handling for an alias here, since the aliased "from"
-      // is already the key in the import map. The aliased "to" value is also an entry.
-      const importMapEntry = installImportMap.imports[spec];
-      return path.posix.resolve(config.buildOptions.webModulesUrl, importMapEntry);
-    }
     return false;
   };
 }

snowpack/src/commands/build.ts

@@ -15,11 +15,13 @@ import {
   wrapImportProxy,
 } from '../build/build-import-proxy';
 import {buildFile, runPipelineCleanupStep, runPipelineOptimizeStep} from '../build/build-pipeline';
+import {getMountEntryForFile, getUrlForFileMount} from '../build/file-urls';
 import {createImportResolver} from '../build/import-resolver';
-import {getUrlForFileMount, getMountEntryForFile} from '../build/file-urls';
+import {runBuiltInOptimize} from '../build/optimize';
 import {EsmHmrEngine} from '../hmr-server-engine';
 import {logger} from '../logger';
 import {transformFileImports} from '../rewrite-imports';
+import localPackageSource from '../sources/local';
 import {
   CommandOptions,
   ImportMap,
@@ -29,6 +31,7 @@ import {
 } from '../types/snowpack';
 import {
   cssSourceMappingURL,
+  getPackageSource,
   HMR_CLIENT_CODE,
   HMR_OVERLAY_CODE,
   isRemoteSpecifier,
@@ -39,7 +42,6 @@ import {
   replaceExt,
 } from '../util';
 import {getInstallTargets, run as installRunner} from './install';
-import {runBuiltInOptimize} from '../build/optimize';
 
 const CONCURRENT_WORKERS = require('os').cpus().length;
 
@@ -67,6 +69,10 @@ async function installOptimizedDependencies(
         : commandOptions.config.installOptions.treeshake ?? true,
     },
   });
+
+  const pkgSource = getPackageSource(commandOptions.config.experiments.source);
+  pkgSource.modifyBuildInstallConfig(installConfig);
+
   // Unlike dev (where we scan from source code) the built output guarantees that we
   // will can scan all used entrypoints. Set to `[]` to improve tree-shaking performance.
   installConfig.knownEntrypoints = [];
@@ -218,15 +224,18 @@ class FileBuilder {
       const resolveImportSpecifier = createImportResolver({
         fileLoc: file.locOnDisk!, // we’re confident these are reading from disk because we just read them
         lockfile: this.lockfile,
-        installImportMap: importMap,
         config: this.config,
       });
       const resolvedCode = await transformFileImports(file, (spec) => {
         // Try to resolve the specifier to a known URL in the project
         let resolvedImportUrl = resolveImportSpecifier(spec);
-        // NOTE: If the import cannot be resolved, we'll need to re-install
-        // your dependencies. We don't support this yet, but we will.
-        // Until supported, just exit here.
+        // If not resolved, then this is a package. During build, dependencies are always
+        // installed locally via esinstall, so use localPackageSource here.
+        if (!resolvedImportUrl) {
+          resolvedImportUrl = localPackageSource.resolvePackageImport(spec, importMap, this.config);
+        }
+        // If still not resolved, then this imported package somehow evaded detection
+        // when we scanned it in the previous step. If you find a bug here, report it!
         if (!resolvedImportUrl) {
           isSuccess = false;
           logger.error(`${file.locOnDisk} - Could not resolve unknown import "${spec}".`);
@@ -386,9 +395,11 @@ export async function command(commandOptions: CommandOptions) {
       .map((f) => Object.values(f.filesToResolve))
       .reduce((flat, item) => flat.concat(item), []);
     const installDest = path.join(buildDirectoryLoc, config.buildOptions.webModulesUrl);
-    const installResult = await installOptimizedDependencies(scannedFiles, installDest, {
-      ...commandOptions,
-    });
+    const installResult = await installOptimizedDependencies(
+      scannedFiles,
+      installDest,
+      commandOptions,
+    );
     const allFiles = glob.sync(`**/*`, {
       cwd: installDest,
       absolute: true,

snowpack/src/commands/dev.ts

@@ -26,10 +26,9 @@
 
 import cacache from 'cacache';
 import isCompressible from 'compressible';
-import merge from 'deepmerge';
 import etag from 'etag';
 import {EventEmitter} from 'events';
-import {createReadStream, existsSync, promises as fs, statSync} from 'fs';
+import {createReadStream, promises as fs, statSync} from 'fs';
 import http from 'http';
 import HttpProxy from 'http-proxy';
 import http2 from 'http2';
@@ -65,7 +64,6 @@ import {
 import {matchDynamicImportValue} from '../scan-imports';
 import {
   CommandOptions,
-  ImportMap,
   LoadResult,
   OnFileChangeCallback,
   RouteConfigObject,
@@ -74,10 +72,9 @@ import {
 } from '../types/snowpack';
 import {
   BUILD_CACHE,
-  checkLockfileHash,
   cssSourceMappingURL,
-  DEV_DEPENDENCIES_DIR,
   getExt,
+  getPackageSource,
   HMR_CLIENT_CODE,
   HMR_OVERLAY_CODE,
   isRemoteSpecifier,
@@ -88,9 +85,7 @@ import {
   relativeURL,
   replaceExt,
   resolveDependencyManifest,
-  updateLockfileHash,
 } from '../util';
-import {getInstallTargets, run as installRunner} from './install';
 import {getPort, getServerInfoMessage, paintDashboard, paintEvent} from './paint';
 
 interface FoundFile {
@@ -151,30 +146,6 @@ class NotFoundError extends Error {
   }
 }
 
-/**
- * Install dependencies needed in "dev" mode. Generally speaking, this scans
- * your entire source app for dependency install targets, installs them,
- * and then updates the "hash" file used to check node_modules freshness.
- */
-async function installDependencies(commandOptions: CommandOptions) {
-  const {config} = commandOptions;
-  const installTargets = await getInstallTargets(config, commandOptions.lockfile);
-  if (installTargets.length === 0) {
-    logger.info('Nothing to install.');
-    return;
-  }
-  // 2. Install dependencies, based on the scan of your final build.
-  const installResult = await installRunner({
-    ...commandOptions,
-    installTargets,
-    config,
-    shouldPrintStats: true,
-    shouldWriteLockfile: false,
-  });
-  await updateLockfileHash(DEV_DEPENDENCIES_DIR);
-  return installResult;
-}
-
 function shouldProxy(pathPrefix: string, reqUrl: string) {
   const reqPath = decodeURI(url.parse(reqUrl).pathname!);
   return reqPath.startsWith(pathPrefix);
@@ -288,14 +259,14 @@ function handleResponseError(req, res, err: Error | NotFoundError) {
 }
 
 export async function startDevServer(commandOptions: CommandOptions): Promise<SnowpackDevServer> {
-  const {lockfile} = commandOptions;
+  const {cwd, config, lockfile} = commandOptions;
   // Start the startup timer!
   let serverStart = performance.now();
 
-  const {cwd, config} = commandOptions;
   const {port: defaultPort, hostname, open} = config.devOptions;
   const messageBus = new EventEmitter();
   const port = await getPort(defaultPort);
+  const pkgSource = getPackageSource(config.experiments.source);
 
   // Reset the clock if we had to wait for the user prompt to select a new port.
   if (port !== defaultPort) {
@@ -350,36 +321,7 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
     },
   });
 
-  // Set the proper install options, in case an install is needed.
-  const dependencyImportMapLoc = path.join(DEV_DEPENDENCIES_DIR, 'import-map.json');
-  logger.debug(`Using cache folder: ${path.relative(cwd, DEV_DEPENDENCIES_DIR)}`);
-  const installCommandOptions = merge(commandOptions, {
-    config: {
-      installOptions: {
-        dest: DEV_DEPENDENCIES_DIR,
-        env: {NODE_ENV: process.env.NODE_ENV || 'development'},
-        treeshake: false,
-      },
-    },
-  });
-
-  // Start with a fresh install of your dependencies, if needed.
-  let dependencyImportMap: ImportMap = {imports: {}};
-  try {
-    dependencyImportMap = JSON.parse(
-      await fs.readFile(dependencyImportMapLoc, {encoding: 'utf-8'}),
-    );
-  } catch (err) {
-    // no import-map found, safe to ignore
-  }
-
-  if (!(await checkLockfileHash(DEV_DEPENDENCIES_DIR)) || !existsSync(dependencyImportMapLoc)) {
-    logger.debug('Cache out of date or missing. Updating...');
-    const installResult = await installDependencies(installCommandOptions);
-    dependencyImportMap = installResult?.importMap || dependencyImportMap;
-  } else {
-    logger.debug(`Cache up-to-date. Using existing cache`);
-  }
+  let sourceImportMap = await pkgSource.prepare(commandOptions);
 
   const devProxies = {};
   config.proxy.forEach(([pathPrefix, proxyOptions]) => {
@@ -535,6 +477,31 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
       };
     }
 
+    if (reqPath.startsWith(config.buildOptions.webModulesUrl)) {
+      try {
+        const webModuleUrl = reqPath.substr(config.buildOptions.webModulesUrl.length + 1);
+        const response = await pkgSource.load(webModuleUrl, commandOptions);
+        return {
+          contents: encodeResponse(response, encoding),
+          originalFileLoc: null,
+          contentType: path.extname(reqPath)
+            ? mime.lookup(path.extname(reqPath))
+            : 'application/javascript',
+        };
+      } catch (err) {
+        const errorTitle = `Dependency Load Error`;
+        const errorMessage = err.message;
+        logger.error(`${errorTitle}: ${errorMessage}`);
+        hmrEngine.broadcastMessage({
+          type: 'error',
+          title: errorTitle,
+          errorMessage,
+          fileLoc: reqPath,
+        });
+        throw err;
+      }
+    }
+
     const attemptedFileLoads: string[] = [];
     function attemptLoadFile(requestedFile): Promise<null | string> {
       if (attemptedFileLoads.includes(requestedFile)) {
@@ -553,15 +520,6 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
     let isRoute = !requestedFileExt || requestedFileExt === '.html';
 
     async function getFileFromUrl(reqPath: string): Promise<FoundFile | null> {
-      if (reqPath.startsWith(config.buildOptions.webModulesUrl)) {
-        const dependencyFileLoc =
-          reqPath.replace(config.buildOptions.webModulesUrl, DEV_DEPENDENCIES_DIR) +
-          (isSourceMap ? '.map' : '');
-        const foundFile = await attemptLoadFile(dependencyFileLoc);
-        if (foundFile) {
-          return {fileLoc: foundFile, isStatic: true, isResolve: false};
-        }
-      }
       for (const [mountKey, mountEntry] of Object.entries(config.mount)) {
         let requestedFile: string;
         if (mountEntry.url === '/') {
@@ -751,7 +709,6 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
       const resolveImportSpecifier = createImportResolver({
         fileLoc,
         lockfile: lockfile,
-        installImportMap: dependencyImportMap,
         config,
       });
       wrappedResponse = await transformFileImports(
@@ -764,7 +721,11 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
         (spec) => {
           // Try to resolve the specifier to a known URL in the project
           let resolvedImportUrl = resolveImportSpecifier(spec);
-          // Handle an import that couldn't be resolved
+          // Handle a package import
+          if (!resolvedImportUrl) {
+            resolvedImportUrl = pkgSource.resolvePackageImport(spec, sourceImportMap, config);
+          }
+          // Handle a package import that couldn't be resolved
           if (!resolvedImportUrl) {
             missingPackages.push(spec);
             return spec;
@@ -807,9 +768,7 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
         // Only retry once, to prevent an infinite loop when a package doesn't actually exist.
         if (retryMissing) {
           try {
-            logger.info(colors.yellow('Dependency cache out of date. Updating...'));
-            const installResult = await installDependencies(installCommandOptions);
-            dependencyImportMap = installResult?.importMap || dependencyImportMap;
+            sourceImportMap = await pkgSource.recoverMissingPackageImport(missingPackages);
             return resolveResponseImports(fileLoc, responseExt, wrappedResponse, false);
           } catch (err) {
             const errorTitle = `Dependency Install Error`;
@@ -1418,7 +1377,7 @@ export async function startDevServer(commandOptions: CommandOptions): Promise<Sn
 
   // Watch node_modules & rerun snowpack install if symlinked dep updates
   const symlinkedFileLocs = new Set(
-    Object.keys(dependencyImportMap.imports)
+    Object.keys(sourceImportMap.imports)
       .map((specifier) => {
         const [packageName] = parsePackageImportSpecifier(specifier);
         return resolveDependencyManifest(packageName, cwd);

snowpack/src/commands/install.ts

@@ -1,8 +1,8 @@
 import {DependencyStatsOutput, install, InstallTarget, printStats} from 'esinstall';
 import * as colors from 'kleur/colors';
-import util from 'util';
 import path from 'path';
 import {performance} from 'perf_hooks';
+import util from 'util';
 import {logger} from '../logger';
 import {scanDepList, scanImports, scanImportsFromFiles} from '../scan-imports.js';
 import {CommandOptions, ImportMap, SnowpackConfig, SnowpackSourceFile} from '../types/snowpack';
@@ -92,7 +92,14 @@ export async function run({
 }: InstallRunOptions): Promise<InstallRunResult> {
   // start
   const installStart = performance.now();
-  logger.info(colors.yellow('! installing dependencies...'));
+  logger.info(
+    colors.yellow(
+      '! installing dependencies...' +
+        colors.cyan(
+          config.experiments.source === 'local' ? '' : ` (source: ${config.experiments.source})`,
+        ),
+    ),
+  );
 
   if (installTargets.length === 0) {
     return {