More compact storage micro-optimization.

jestjs · Apr 19, 2016 · 0607cf7 · 0607cf7
1 parent fbea74b
commit 0607cf7
Show file tree

Hide file tree

Showing 9 changed files with 138 additions and 98 deletions.
diff --git a/packages/jest-haste-map/src/constants.js b/packages/jest-haste-map/src/constants.js
@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+/*
+ * This file exports a set of constants that are used for Jest's haste map
+ * serialization. On very large repositories, the haste map cache becomes very
+ * large to the point where it is the largest overhead in starting up Jest.
+ *
+ * This constant key map allows to keep the map smaller without having to build
+ * a custom serialization library.
+ */
+module.exports = {
+  /* shared in file map and module map */
+  ID: 0,
+
+  /* file map attributes */
+  MTIME: 1,
+  VISITED: 2,
+  DEPENDENCIES: 3,
+
+  /* module map attributes */
+  PATH: 1,
+  TYPE: 2,
+
+  /* module types */
+  MODULE: 0,
+  PACKAGE: 1,
+
+  /* platforms */
+  GENERIC_PLATFORM: 'g',
+};
diff --git a/packages/jest-haste-map/src/crawlers/node.js b/packages/jest-haste-map/src/crawlers/node.js
@@ -9,12 +9,14 @@
 
 'use strict';
 
+const H = require('../constants');
+
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const spawn = require('child_process').spawn;
 
-function find(roots, extensions, ignorePattern, callback) {
+function find(roots, extensions, ignore, callback) {
   const result = [];
   let activeCalls = 0;
 
@@ -28,7 +30,7 @@ function find(roots, extensions, ignorePattern, callback) {
       }
 
       names.forEach(file => {
-        if (ignorePattern.test(file)) {
+        if (ignore(file)) {
           return;
         }
         activeCalls++;
@@ -61,7 +63,7 @@ function find(roots, extensions, ignorePattern, callback) {
   roots.forEach(search);
 }
 
-function findNative(roots, extensions, ignorePattern, callback) {
+function findNative(roots, extensions, ignore, callback) {
   const args = [].concat(roots);
   args.push('-type', 'f');
   extensions.forEach((ext, index) => {
@@ -80,7 +82,7 @@ function findNative(roots, extensions, ignorePattern, callback) {
   child.stdout.on('close', code => {
     const lines = stdout.trim()
       .split('\n')
-      .filter(x => !ignorePattern.test(x));
+      .filter(x => !ignore(x));
     const result = [];
     let count = lines.length;
     lines.forEach(path => {
@@ -96,33 +98,28 @@ function findNative(roots, extensions, ignorePattern, callback) {
   });
 }
 
-module.exports = function nodeCrawl(roots, extensions, ignorePattern, data) {
+module.exports = function nodeCrawl(roots, extensions, ignore, data) {
   return new Promise(resolve => {
     const callback = list => {
       const files = Object.create(null);
       list.forEach(fileData => {
         const name = fileData[0];
         const mtime = fileData[1];
         const existingFile = data.files[name];
-        if (existingFile && existingFile.mtime === mtime) {
-          //console.log('exists', name);
+        if (existingFile && existingFile[H.MTIME] === mtime) {
           files[name] = existingFile;
         } else {
-          //console.log('add', name);
-          files[name] = {
-            mtime,
-            visited: false,
-          };
+          files[name] = [0, mtime, 0, []];
         }
       });
       data.files = files;
       resolve(data);
     };
 
     if (os.platform() == 'win32') {
-      find(roots, extensions, ignorePattern, callback);
+      find(roots, extensions, ignore, callback);
     } else {
-      findNative(roots, extensions, ignorePattern, callback);
+      findNative(roots, extensions, ignore, callback);
     }
   });
 };
diff --git a/packages/jest-haste-map/src/crawlers/watchman.js b/packages/jest-haste-map/src/crawlers/watchman.js
@@ -9,6 +9,8 @@
 
 'use strict';
 
+const H = require('../constants');
+
 const denodeify = require('denodeify');
 const path = require('../fastpath');
 const watchman = require('fb-watchman');
@@ -29,7 +31,7 @@ function WatchmanError(error) {
 module.exports = function watchmanCrawl(
   roots,
   extensions,
-  ignorePattern,
+  ignore,
   data
 ) {
   const files = data.files;
@@ -78,18 +80,12 @@ module.exports = function watchmanCrawl(
           response.files.forEach(fileData => {
             const name = root + path.sep + fileData.name;
             if (!fileData.exists) {
-              //console.log('remove', name);
               delete files[name];
-            } else if (!ignorePattern.test(name)) {
-              //console.log('add', name);
+            } else if (!ignore(name)) {
               const mtime = fileData.mtime_ms.toNumber();
-              const isNew = !files[name] || files[name].mtime !== mtime;
+              const isNew = !files[name] || files[name][H.MTIME] !== mtime;
               if (isNew) {
-                files[name] = {
-                  id: null,
-                  mtime,
-                  visited: false,
-                };
+                files[name] = [0, mtime, 0, []];
               }
             }
           });

diff --git a/packages/jest-haste-map/src/index.js b/packages/jest-haste-map/src/index.js
@@ -8,6 +8,8 @@
  */
 'use strict';
 
+const H = require('./constants');
+
 const crypto = require('crypto');
 const denodeify = require('denodeify');
 const execSync = require('child_process').execSync;
@@ -20,7 +22,6 @@ const watchmanCrawl = require('./crawlers/watchman');
 const worker = require('./worker');
 const workerFarm = require('worker-farm');
 
-const GENERIC_PLATFORM = 'g';
 const NODE_MODULES = path.sep + 'node_modules' + path.sep;
 const VERSION = require('../package.json').version;
 
@@ -42,6 +43,7 @@ class HasteMap {
       maxWorkers: options.maxWorkers,
       mocksPattern:
         options.mocksPattern ? new RegExp(options.mocksPattern) : null,
+      name: options.name,
       platforms: options.platforms,
       resetCache: options.resetCache,
       roots: options.roots,
@@ -57,6 +59,7 @@ class HasteMap {
     this._cachePath = HasteMap.getCacheFilePath(
       this._options.cacheDirectory,
       VERSION,
+      this._options.name,
       this._options.roots.join(':'),
       this._options.extensions.join(':'),
       this._options.platforms.join(':'),
@@ -121,17 +124,19 @@ class HasteMap {
     const mocksPattern = this._options.mocksPattern;
     const promises = [];
     const setModule = module => {
-      if (!map[module.id]) {
-        map[module.id] = Object.create(null);
+      if (!map[module[H.ID]]) {
+        map[module[H.ID]] = Object.create(null);
       }
-      const moduleMap = map[module.id];
-      const platform = getPlatformExtension(module.path) || GENERIC_PLATFORM;
+      const moduleMap = map[module[H.ID]];
+      const platform =
+        getPlatformExtension(module[H.PATH]) || H.GENERIC_PLATFORM;
       const existingModule = moduleMap[platform];
-      if (existingModule && existingModule.path !== module.path) {
+      if (existingModule && existingModule[H.PATH] !== module[H.PATH]) {
         console.warn(
           `@providesModule naming collision:\n` +
           `  Duplicate module name: ${module.id}\n` +
-          `  Paths: ${module.path} collides with ${existingModule.path}\n\n` +
+          `  Paths: ${module[H.PATH]} collides with ` +
+          `${existingModule[H.PATH]}\n\n` +
           `This warning is caused by a @providesModule declaration ` +
           `with the same name accross two different files.`
         );
@@ -145,31 +150,29 @@ class HasteMap {
         mocks[path.basename(filePath, path.extname(filePath))] = filePath;
       }
 
-      if (!this._isNodeModulesDir(filePath)) {
-        const fileData = data.files[filePath];
-        const moduleData = data.map[fileData.id];
-        if (fileData.visited) {
-          if (!fileData.id) {
-            continue;
-          } else if (fileData.id && moduleData) {
-            map[fileData.id] = moduleData;
-            continue;
-          }
+      const fileData = data.files[filePath];
+      const moduleData = data.map[fileData[H.ID]];
+      if (fileData[H.VISITED]) {
+        if (!fileData[H.ID]) {
+          continue;
+        } else if (fileData[H.ID] && moduleData) {
+          map[fileData[H.ID]] = moduleData;
+          continue;
         }
-
-        promises.push(
-          this._getWorker()({filePath}).then(data => {
-            fileData.visited = true;
-            if (data.module) {
-              fileData.id = data.module.id;
-              setModule(data.module);
-            }
-            if (data.dependencies) {
-              fileData.dependencies = data.dependencies;
-            }
-          })
-        );
       }
+
+      promises.push(
+        this._getWorker()({filePath}).then(data => {
+          fileData[H.VISITED] = 1;
+          if (data.module) {
+            fileData[H.ID] = data.module[H.ID];
+            setModule(data.module);
+          }
+          if (data.dependencies) {
+            fileData[H.DEPENDENCIES] = data.dependencies;
+          }
+        })
+      );
     }
 
     return Promise.all(promises)
@@ -228,11 +231,18 @@ class HasteMap {
     return crawl(
       this._options.roots,
       this._options.extensions,
-      this._options.ignorePattern,
+      this._ignore.bind(this),
       data
     );
   }
 
+  _ignore(filePath) {
+    return (
+      this._options.ignorePattern.test(filePath) ||
+      this._isNodeModulesDir(filePath)
+    );
+  }
+
   _isNodeModulesDir(filePath) {
     if (!filePath.includes(NODE_MODULES)) {
       return false;
@@ -257,6 +267,4 @@ class HasteMap {
 
 }
 
-HasteMap.GENERIC_PLATFORM = GENERIC_PLATFORM;
-
 module.exports = HasteMap;
diff --git a/packages/jest-haste-map/src/worker.js b/packages/jest-haste-map/src/worker.js
@@ -7,6 +7,8 @@
  */
 'use strict';
 
+const H = require('./constants');
+
 const docblock = require('./lib/docblock');
 const extractRequires = require('./lib/extractRequires');
 const fs = require('graceful-fs');
@@ -40,22 +42,14 @@ module.exports = (data, callback) => {
     if (filePath.endsWith(PACKAGE_JSON)) {
       const fileData = JSON.parse(content);
       if (fileData.name) {
-        module = {
-          id: fileData.name,
-          path: filePath,
-          type: 'package',
-        };
+        module = [fileData.name, filePath, H.PACKAGE];
       }
     } else {
       const doc = docblock.parse(docblock.extract(content));
       const id = doc.providesModule || doc.provides;
       dependencies = extractRequires(content);
       if (id) {
-        module = {
-          id,
-          path: filePath,
-          type: 'module',
-        };
+        module = [id, filePath, H.MODULE];
       }
     }
     callback(null, {module, dependencies});

diff --git a/src/Runtime/Runtime.js b/src/Runtime/Runtime.js
@@ -8,7 +8,7 @@
 
 'use strict';
 
-const HasteMap = require('jest-haste-map');
+const H = require('jest-haste-map/src/constants');
 
 const constants = require('../constants');
 const fs = require('graceful-fs');
@@ -456,23 +456,22 @@ class Runtime {
 
   _getModule(name, type) {
     if (!type) {
-      type = 'module';
+      type = H.MODULE;
     }
 
     const map = this._modules[name];
     if (map) {
-      const module =
-        map[this._defaultPlatform] || map[HasteMap.GENERIC_PLATFORM];
-      if (module && module.type == type) {
-        return module.path;
+      const module = map[this._defaultPlatform] || map[H.GENERIC_PLATFORM];
+      if (module && module[H.TYPE] == type) {
+        return module[H.PATH];
       }
     }
 
     return null;
   }
 
   _getPackage(name) {
-    return this._getModule(name, 'package');
+    return this._getModule(name, H.PACKAGE);
   }
 
   _getMockModule(name) {