Implement fast incremental compiler

.travis.yml

@@ -3,7 +3,7 @@ language: node_js
 node_js:
   - "6"
 
-sudo: false
+sudo: required
 dist: trusty
 
 addons:

blueprints/ember-cli-typescript/files/tsconfig.json

@@ -20,10 +20,8 @@ inRepoAddons.forEach(function(path) { %>,
       ]
     }
   },
-  "exclude": [
-    "tmp",
-    "dist",
-    "node_modules",
-    "bower_components"
+  "include": [
+    "app",
+    "tests"
   ]
 }

blueprints/ember-cli-typescript/index.js

@@ -1,5 +1,6 @@
 /* eslint-env node */
 
+const { existsSync } = require('fs');
 const path = require('path');
 
 module.exports = {

index.js

@@ -1,13 +1,124 @@
+// @ts-check
 /* eslint-env node */
 
 const fs = require('fs');
+const os = require('os');
 const path = require('path');
 const SilentError = require('silent-error');
 const TsPreprocessor = require('./lib/typescript-preprocessor');
+const buildServeCommand = require('./lib/serve-ts');
+const funnel = require('broccoli-funnel');
+const mergeTrees = require('broccoli-merge-trees');
+const mkdirp = require('mkdirp');
 
 module.exports = {
   name: 'ember-cli-typescript',
 
+  _isRunningServeTS() {
+    return this.project._isRunningServeTS;
+  },
+
+  _tempDir() {
+    if (!this.project._tsTempDir) {
+      const tempDir = path.join(os.tmpdir(), `e-c-ts-${process.pid}`);
+      this.project._tsTempDir = tempDir;
+      mkdirp.sync(tempDir);
+    }
+
+    return this.project._tsTempDir;
+  },
+
+  _inRepoAddons() {
+    const pkg = this.project.pkg;
+    if (!pkg || !pkg['ember-addon'] || !pkg['ember-addon'].paths) {
+      return [];
+    }
+
+    return pkg['ember-addon'].paths;
+  },
+
+  includedCommands() {
+    return {
+      'serve-ts': buildServeCommand(this.project, this._tempDir()),
+    };
+  },
+
+  // Stolen from ember-cli-mirage.
+  included() {
+    let app;
+
+    // If the addon has the _findHost() method (in ember-cli >= 2.7.0), we'll just
+    // use that.
+    if (typeof this._findHost === 'function') {
+      app = this._findHost();
+    } else {
+      // Otherwise, we'll use this implementation borrowed from the _findHost()
+      // method in ember-cli.
+      let current = this;
+      do {
+        app = current.app || app;
+      } while (current.parent.parent && (current = current.parent));
+    }
+
+    this.app = app;
+
+    this._super.included.apply(this, arguments);
+  },
+
+  treeForApp(tree) {
+    const { include } = JSON.parse(
+      fs.readFileSync(path.resolve(this.app.project.root, 'tsconfig.json'), { encoding: 'utf8' })
+    );
+
+    const includes = ['types']
+      .concat(include ? include : [])
+      .reduce((unique, entry) => (unique.indexOf(entry) === -1 ? unique.concat(entry) : unique), [])
+      .map(p => path.resolve(this.app.project.root, p))
+      .filter(fs.existsSync);
+
+    const additionalTrees = includes.map(p => funnel(p, { destDir: p }));
+
+    if (!this._isRunningServeTS()) {
+      return mergeTrees([tree, ...additionalTrees]);
+    }
+
+    const roots = ['.', ...includes, ...this._inRepoAddons()].map(root => path.join(root, 'app/'));
+
+    // funnel will fail if the directory doesn't exist
+    roots.forEach(root => {
+      mkdirp.sync(path.join(this._tempDir(), root));
+    });
+
+    const ts = funnel(this._tempDir(), {
+      exclude: ['tests'],
+      getDestinationPath(relativePath) {
+        const prefix = roots.find(root => relativePath.startsWith(root));
+        if (prefix) {
+          // strip any app/ or lib/in-repo-addon/app/ prefix
+          return relativePath.substr(prefix.length);
+        }
+
+        return relativePath;
+      },
+    });
+
+    return mergeTrees([tree, ts]);
+  },
+
+  treeForTestSupport(tree) {
+    if (!this._isRunningServeTS()) {
+      return tree;
+    }
+
+    const tests = path.join(this._tempDir(), 'tests');
+
+    // funnel will fail if the directory doesn't exist
+    mkdirp.sync(tests);
+
+    const ts = funnel(tests);
+    return tree ? mergeTrees([tree, ts]) : ts;
+  },
+
   setupPreprocessorRegistry(type, registry) {
     if (!fs.existsSync(path.join(this.project.root, 'tsconfig.json'))) {
       // Do nothing; we just won't have the plugin available. This means that if you
@@ -23,6 +134,11 @@ module.exports = {
       return;
     }
 
+    if (type === 'self' || this._isRunningServeTS()) {
+      // TODO: still need to compile TS addons
+      return;
+    }
+
     try {
       registry.add(
         'js',
@@ -32,9 +148,7 @@ module.exports = {
       );
     } catch (ex) {
       throw new SilentError(
-        `Failed to instantiate TypeScript preprocessor, probably due to an invalid tsconfig.json. Please fix or run \`ember generate ember-cli-typescript\`.\n${
-          ex
-        }`
+        `Failed to instantiate TypeScript preprocessor, probably due to an invalid tsconfig.json. Please fix or run \`ember generate ember-cli-typescript\`.\n${ex}`
       );
     }
   },

lib/serve-ts.js

@@ -0,0 +1,126 @@
+// @ts-check
+/* eslint-env node */
+
+const child_process = require('child_process');
+const fs = require('fs');
+
+const rimraf = require('rimraf');
+
+module.exports = (project, outDir) => {
+  const Serve = project.require('ember-cli/lib/commands/serve');
+  const Builder = project.require('ember-cli/lib/models/builder');
+  const Watcher = project.require('ember-cli/lib/models/watcher');
+
+  /**
+   * Exclude .ts files from being watched
+   */
+  function filterTS(name) {
+    if (name.startsWith('.')) {
+      // these files are filtered by default
+      return false;
+    }
+
+    // typescript sources are watched by `tsc --watch` instead
+    return !name.endsWith('.ts');
+  }
+
+  class WatcherNonTS extends Watcher {
+    buildOptions() {
+      let options = super.buildOptions();
+      options.filter = filterTS;
+      return options;
+    }
+  }
+
+  return Serve.extend({
+    name: 'serve-ts',
+    aliases: ['st'],
+    works: 'insideProject',
+    description:
+      'Serve the app/addon with the TypeScript compiler in incremental mode. (Much faster!)',
+
+    run(options) {
+      const config = this.project.config(options.environment);
+
+      this.project._isRunningServeTS = true;
+
+      return new Promise(resolve => {
+        let started = false;
+
+        this.ui.startProgress('Starting TypeScript compilation...');
+
+        // TODO: typescript might be installed globally?
+        // argument sequence here is meaningful; don't apply prettier.
+        // prettier-ignore
+        this.tsc = child_process.fork(
+          'node_modules/typescript/bin/tsc',
+          [
+            '--watch',
+            '--outDir', outDir,
+            '--allowJs', 'false',
+            '--noEmit', 'false',
+            '--sourceMap', 'false', // TODO: enable if sourcemaps=true
+          ],
+          {
+            silent: true,
+            execArgv: [],
+          }
+        );
+
+        this.tsc.stderr.on('data', data => {
+          this.ui.writeError(data);
+        });
+
+        this.tsc.stdout.on('data', data => {
+          this.ui.write(data);
+
+          // Wait for the initial compilation to complete before continuing to
+          // minimize thrashing during startup.
+          if (data.indexOf('Compilation complete') !== -1 && !started) {
+            started = true;
+            this.ui.stopProgress();
+            resolve();
+          }
+        });
+      }).then(() => {
+        const builder = new Builder({
+          ui: this.ui,
+          outputPath: options.outputPath,
+          project: this.project,
+          environment: options.environment,
+        });
+
+        // This will be populated later by the superclass, but is needed by the Watcher now
+        options.rootURL = config.rootURL || '/';
+
+        // We're ignoring this because TS doesn't have types for `Watcher`; this is
+        // fine, though.
+        // @ts-ignore
+        const watcher = new WatcherNonTS({
+          ui: this.ui,
+          builder,
+          analytics: this.analytics,
+          options,
+          serving: true,
+        });
+
+        options._builder = builder;
+        options._watcher = watcher;
+
+        return Serve.prototype.run.call(this, options);
+      });
+    },
+
+    onInterrupt() {
+      return Serve.prototype.onInterrupt.apply(this, arguments).then(() => {
+        if (this.tsc) {
+          this.tsc.kill();
+        }
+
+        if (fs.existsSync(outDir)) {
+          rimraf.sync(outDir);
+        }
+      });
+    },
+  });
+};

lib/typescript-preprocessor.js

@@ -1,10 +1,12 @@
+// @ts-check
 /* eslint-env node */
+
 const fs = require('fs');
 const path = require('path');
 
 const funnel = require('broccoli-funnel');
 const mergeTrees = require('broccoli-merge-trees');
-const TypeScriptPlugin = require('broccoli-typescript-compiler').TypeScriptPlugin;
+const { typescript } = require('broccoli-typescript-compiler');
 
 const BroccoliDebug = require('broccoli-debug');
 
@@ -13,50 +15,56 @@ let tag = 0;
 class TypeScriptPreprocessor {
   constructor(options) {
     this.name = 'ember-cli-typescript';
-    this._tag = tag++;
+    this._tag = tag;
     this.ext = 'ts';
     this.ui = options.ui;
 
     // Update the config for how Broccoli handles the file system: no need for
     // includes, always emit, and let Broccoli manage any outDir.
-    this.config = JSON.parse(fs.readFileSync(path.join(process.cwd(), 'tsconfig.json')));
+    const config = fs.readFileSync(path.join(process.cwd(), 'tsconfig.json'), { encoding: 'utf8' });
+    this.config = JSON.parse(config);
     this.config.compilerOptions.noEmit = false;
     this.config.compilerOptions.allowJs = false;
     delete this.config.compilerOptions.outDir;
     delete this.config.include;
   }
 
-  toTree(inputNode, inputPath, outputPath) {
+  toTree(inputNode /*, inputPath, outputPath*/) {
+    // increment every time toTree is called so we have some idea what's going on here.
+    this._tag = tag++;
+
+    const debugTree = BroccoliDebug.buildDebugCallback('ember-cli-typescript');
+
     const js = funnel(inputNode, {
       exclude: ['**/*.ts'],
       annotation: 'JS files',
     });
 
-    const debugTree = BroccoliDebug.buildDebugCallback('ember-cli-typescript');
-
     const uncompiledTs = debugTree(
       funnel(inputNode, {
         include: ['**/*.ts'],
         annotation: 'uncompiled TS files',
       }),
-      `${this._tag}`
+      `${this._tag}:uncompiled-ts`
     );
 
-    const tsc = new TypeScriptPlugin(uncompiledTs, {
+    const tsc = typescript(uncompiledTs, {
       throwOnError: this.config.compilerOptions.noEmitOnError,
       annotation: 'Compiled TS files',
-      include: ['**/*'],
       tsconfig: this.config,
     });
     tsc.setDiagnosticWriter(this.ui.writeWarnLine.bind(this.ui));
 
-    const ts = debugTree(tsc, `${this._tag}`);
+    const ts = debugTree(tsc, `${this._tag}:compiled-ts`);
 
     // Put everything together.
-    return mergeTrees([js, ts], {
-      overwrite: true,
-      annotation: 'merged JS & compiled TS',
-    });
+    return debugTree(
+      mergeTrees([js, ts], {
+        overwrite: true,
+        annotation: 'merged JS & compiled TS',
+      }),
+      `${this._tag}:final`
+    );
   }
 }