Asset directory emission, test refactoring

package.json

@@ -58,9 +58,9 @@
     "mkdirp": "^0.5.1",
     "mongoose": "^5.3.12",
     "mysql": "^2.16.0",
-    "node-pre-gyp": "^0.12.0",
     "node-gyp": "^3.8.0",
     "node-native-loader": "^1.1.1",
+    "node-pre-gyp": "^0.12.0",
     "passport": "^0.4.0",
     "passport-google-oauth": "^1.0.0",
     "path-platform": "^0.11.15",
@@ -76,6 +76,7 @@
     "saslprep": "^1.0.2",
     "source-map-support": "^0.5.9",
     "stripe": "^6.15.0",
+    "strong-globalize": "^4.1.2",
     "the-answer": "^1.0.0",
     "twilio": "^3.23.2",
     "vue": "^2.5.17",

src/index.js

@@ -47,6 +47,7 @@ function resolveModule(context, request, callback, forcedExternals = []) {
 module.exports = async (entry, { externals = [], minify = true, sourceMap = false, filename = "index.js" } = {}) => {
   const mfs = new MemoryFS();
   const assetNames = Object.create(null);
+  const assets = Object.create(null);
   const compiler = webpack({
     entry,
     optimization: {
@@ -79,14 +80,14 @@ module.exports = async (entry, { externals = [], minify = true, sourceMap = fals
           test: /\.(js|mjs)$/,
           use: [{
             loader: __dirname + "/loaders/relocate-loader.js",
-            options: { assetNames }
+            options: { assetNames, assets }
           }]
         },
         {
           test: /\.node$/,
           use: [{
             loader: __dirname + "/loaders/node-loader.js",
-            options: { assetNames }
+            options: { assetNames, assets }
           }]
         }
       ]

src/loaders/relocate-loader.js

@@ -125,6 +125,7 @@ function unwrapIfAMD (ast, scope, magicString, len) {
 }
 
 const relocateRegEx = /_\_dirname|_\_filename|require\.main|node-pre-gyp|bindings|define/;
+const pkgNameRegEx = /(@[^\\\/]+[\\\/])?[^\\\/]+/;
 
 module.exports = function (code) {
   const id = this.resourcePath;
@@ -139,13 +140,51 @@ module.exports = function (code) {
     if (assetPath.endsWith('.js') || assetPath.endsWith('.mjs'))
       return;
 
+    if (options.assets[assetPath])
+      return "__dirname + '/" + JSON.stringify(options.assets[assetPath]).slice(1, -1) + "'";
+
     const name = getUniqueAssetName(assetPath, options.assetNames);
+    options.assets[assetPath] = name;
 
     // console.log('Emitting ' + assetPath + ' for module ' + id);
 
     this.emitFile(name, fs.readFileSync(assetPath));
     return "__dirname + '/" + JSON.stringify(name).slice(1, -1) + "'";
   };
+  const emitAssetDirectory = (assetDirPath) => {
+    if (options.assets[assetDirPath])
+      return "__dirname + '/" + JSON.stringify(options.assets[assetDirPath]).slice(1, -1) + "'";
+
+    const dirName = path.basename(assetDirPath);
+    const name = getUniqueAssetName(dirName, options.assetNames);
+    options.assets[assetDirPath] = name;
+
+    console.log('Emitting directory ' + assetDirPath + " as " + name);
+    for (const file of fs.readdirSync(assetDirPath)) {
+      let source;
+      try {
+        source = fs.readFileSync(assetDirPath + '/' + file);
+      }
+      catch (e) {
+        // nested directories not yet supported
+        continue;
+      }
+      this.emitFile(name + '/' + file, source);
+    }
+
+    return "__dirname + '/" + JSON.stringify(options.assets[assetDirPath]).slice(1, -1) + "'";
+  };
+
+  // determined the node_modules package folder as pkgBase
+  let pkgBase = '';
+  const pkgIndex = id.lastIndexOf('node_modules');
+  if (pkgIndex !== -1 &&
+      (id[pkgIndex - 1] === '/' || id[pkgIndex - 1] === '\\') &&
+      (id[pkgIndex + 12] === '/' || id[pkgIndex + 12] === '\\')) {
+    const pkgNameMatch = id.substr(pkgIndex + 13).match(pkgNameRegEx);
+    if (pkgNameMatch)
+      pkgBase = id.substr(0, pkgIndex + 13 + pkgNameMatch[0].length);
+  }
 
   const magicString = new MagicString(code);
 
@@ -414,14 +453,14 @@ module.exports = function (code) {
         }
         // no static value -> see if we should emit the asset if it exists
         // Currently we only handle files. In theory whole directories could also be emitted if necessary.
-        let isFile = false;
+        let stats;
         if (typeof staticChildValue === 'string') {
           try {
-            isFile = fs.statSync(staticChildValue).isFile();
+            stats = fs.statSync(staticChildValue);
           }
           catch (e) {}
         }
-        if (isFile) {
+        if (stats && stats.isFile()) {
           let replacement = emitAsset(path.resolve(staticChildValue));
           // require('bindings')(...)
           // -> require(require('bindings')(...))
@@ -434,6 +473,17 @@ module.exports = function (code) {
           }
           staticChildNode = staticChildValue = undefined;
         }
+        else if (stats && stats.isDirectory() &&
+            // dont emit __dirname or package base
+            staticChildValue !== path.dirname(id) &&
+            staticChildValue !== pkgBase) {
+          let replacement = emitAssetDirectory(path.resolve(staticChildValue));
+          if (replacement) {
+            transformed = true;
+            magicString.overwrite(staticChildNode.start, staticChildNode.end, replacement);
+          }
+          staticChildNode = staticChildValue = undefined;
+        }
       }
     }
   });

test/index.test.js

@@ -4,21 +4,6 @@ const mkdirp = require("mkdirp");
 const rimraf = require("rimraf");
 const { dirname } = require("path");
 
-const sourceMapSources = {};
-if (!global.coverage) {
-  require('source-map-support').install({
-    retrieveSourceMap (source) {
-      if (!sourceMapSources[source])
-        return null;
-
-      return {
-        url: source,
-        map: sourceMapSources[source]
-      };
-    }
-  });
-}
-
 for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) {
   it(`should generate correct output for ${unitTest}`, async () => {
     const expected = fs.readFileSync(`${__dirname}/unit/${unitTest}/output.js`)
@@ -49,9 +34,9 @@ for (const unitTest of fs.readdirSync(`${__dirname}/unit`)) {
 // the twilio test can take a while (large codebase)
 jest.setTimeout(30000);
 
-function clearTmp () {
+function clearDir (dir) {
   try {
-    rimraf.sync(__dirname + "/tmp");
+    rimraf.sync(dir);
   }
   catch (e) {
     if (e.code !== "ENOENT")
@@ -64,32 +49,28 @@ for (const integrationTest of fs.readdirSync(__dirname + "/integration")) {
   if (!integrationTest.endsWith(".js")) continue;
   it(`should evaluate ${integrationTest} without errors`, async () => {
     const { code, map, assets } = await ncc(__dirname + "/integration/" + integrationTest, { sourceMap: true });
-    module.exports = null;
-    sourceMapSources[integrationTest] = map;
-    // integration tests will load assets relative to __dirname
-    clearTmp();
+    const tmpDir = `${__dirname}/tmp/${integrationTest}/`;
+    clearDir(tmpDir);
+    mkdirp.sync(tmpDir);
     for (const asset of Object.keys(assets)) {
-      const assetPath = __dirname + "/tmp/" + asset;
+      const assetPath = tmpDir + asset;
       mkdirp.sync(dirname(assetPath));
       fs.writeFileSync(assetPath, assets[asset]);
     }
-    ((__dirname, require) => {
-      try {
-        eval(`${code}\n//# sourceURL=${integrationTest}`);
-      }
-      catch (e) {
-        // useful for debugging
-        mkdirp.sync(__dirname);
-        fs.writeFileSync(__dirname + "/index.js", code);
-        throw e;
-      }
-    })(__dirname + "/tmp", id => require(id.startsWith('./') ? './tmp/' + id.substr(2) : id));
-    if ("function" !== typeof module.exports) {
-      throw new Error(
-        `Integration test "${integrationTest}" evaluation failed. It does not export a function`
-      );
-    }
-    await module.exports();
+    fs.writeFileSync(tmpDir + "index.js", code);
+    fs.writeFileSync(tmpDir + "index.js.map", map);
+    await new Promise((resolve, reject) => {
+      const ps = require("child_process").fork(tmpDir + "index.js", {
+        execArgv: ["-r", "source-map-support/register.js"]
+      });
+      ps.on("close", (code) => {
+        if (code === 0)
+          resolve();
+        else
+          reject(new Error(`Test failed.`));
+      });
+    });
+    clearDir(tmpDir);
   });
 }
 

test/integration/analytics-node.js

@@ -1,5 +1,3 @@
 const Analytics = require("analytics-node");
 
-module.exports = () => {
-  new Analytics("YOUR_WRITE_KEY");
-};
+new Analytics("YOUR_WRITE_KEY");

test/integration/apollo.js

@@ -15,8 +15,7 @@ const resolvers = {
   }
 };
 
-module.exports = () => {
-  const server = new ApolloServer({ typeDefs, resolvers });
-  const app = express();
-  server.applyMiddleware({ app });
-};
+
+const server = new ApolloServer({ typeDefs, resolvers });
+const app = express();
+server.applyMiddleware({ app });

test/integration/auth0.js

@@ -1,10 +1,10 @@
 const ManagementClient = require("auth0").ManagementClient;
-module.exports = () => {
-  try {
-    new ManagementClient();
-  } catch (err) {
-    if (!/Management API SDK options must be an object/.test(err.message)) {
-      throw err;
-    }
+
+try {
+  new ManagementClient();
+} catch (err) {
+  if (!/Management API SDK options must be an object/.test(err.message)) {
+    throw err;
   }
-};
+}
+

test/integration/aws-sdk.js

@@ -1,5 +1,3 @@
 const aws = require('aws-sdk');
 
-module.exports = () => {
-  new aws.S3();
-}
+new aws.S3();

test/integration/axios.js

@@ -1,9 +1,10 @@
 const axios = require("axios");
-module.exports = async () => {
+
+(async () => {
   const { data } = await axios({
     url: "https://dog.ceo/api/breeds/image/random"
   });
   if (data.status !== "success") {
     throw new Error("Unexpected response: " + JSON.stringify(data));
-  }
-};
+  }  
+})();

test/integration/azure-cosmos.js

@@ -1,2 +1,2 @@
 require("@azure/cosmos");
-module.exports = () => {};
+

test/integration/azure-storage.js

@@ -1,2 +1 @@
 require("azure-storage");
-module.exports = () => {};

test/integration/binary-require.js

@@ -1,6 +1,5 @@
 const binary = require('./hello.node');
 const assert = require('assert');
 
-module.exports = () => {
-  assert.equal(binary.hello(), 'world');
-};
+
+assert.equal(binary.hello(), 'world');

test/integration/core-js.js

@@ -1,5 +1,2 @@
-require("core-js");
 
-module.exports = () => {
-  require("core-js").Array.map([], () => {});
-};
+require("core-js").Array.map([], () => {});

test/integration/cowsay.js

@@ -1,7 +1,7 @@
 const { say } = require("cowsay/build/cowsay.umd.js");
-module.exports = () => {
-  const nate = say({ text: "nate" });
-  if (!(nate.indexOf("nate") > 0)) {
-    throw new Error('cowsay did not work. String "nate" not found in: ' + nate);
-  }
-};
+
+const nate = say({ text: "nate" });
+if (!(nate.indexOf("nate") > 0)) {
+  throw new Error('cowsay did not work. String "nate" not found in: ' + nate);
+}
+

test/integration/env-var.js

@@ -1,5 +1,3 @@
 const env = process.env.NODE_ENV = 'development'
 
-module.exports = () => {
-  return env
-}
+module.exports = env;

test/integration/express.js

@@ -1,2 +1,2 @@
 const app = require('express');
-module.exports = () => {}
+

test/integration/fetch-h2.js

@@ -1,2 +1 @@
 const fetch = require("fetch-h2");
-module.exports = () => {};

test/integration/firebase.js

@@ -1,2 +1 @@
 const firebase = require('firebase/app');
-module.exports = () => {};

test/integration/firestore.js

@@ -1,2 +1 @@
 require("@google-cloud/firestore");
-module.exports = () => {};

test/integration/google-bigquery.js

@@ -1,7 +1,5 @@
 const {BigQuery} = require('@google-cloud/bigquery');
 
-module.exports = () => {
-  const bigquery = new BigQuery({
-    projectId: 'PROJECT_ID',
-  });
-}
+const bigquery = new BigQuery({
+  projectId: 'PROJECT_ID',
+});

test/integration/got.js

@@ -1,4 +1 @@
 require('got');
-module.exports = () => {
-
-}

test/integration/grpc.js

@@ -1,2 +1 @@
 const grpc = require("grpc");
-module.exports = () => {};

test/integration/hot-shots.js

@@ -1,2 +1 @@
 require("hot-shots");
-module.exports = () => {};

test/integration/ioredis.js

@@ -1,3 +1 @@
 const ioredis = require("ioredis");
-
-module.exports = () => {};

test/integration/isomorphic-unfetch.js

@@ -1,8 +1,9 @@
 const fetch = require("isomorphic-unfetch");
-module.exports = async () => {
+
+(async () => {
   const res = await fetch("https://dog.ceo/api/breeds/image/random");
   const data = await res.json()
   if (data.status !== "success") {
     throw new Error("Unexpected response: " + JSON.stringify(data));
-  }
-};
+  }  
+})();

test/integration/jimp.js

@@ -1,4 +1 @@
 const jimp = require('jimp')
-module.exports = () => {
-
-}

test/integration/json-without-ext.js

@@ -1,2 +1,2 @@
 const { a } = require('./json-without-ext-sample')
-module.exports = () => {}
+