From 81d504f93bb3b52f69e016828034d5713a5e76dd Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 1 Oct 2013 21:03:16 -0300 Subject: [PATCH 01/31] Refactor to use a disk-persistent cache. This should speed up future rebuilds more than the old in-memory cache, as it will also cache the preprocessed AST and only re-read/parse files that were modified. The speed gain should be even greater on compile-to-js languages as recompilation of unmodified files will be completely skipped. Fixes #83. --- package.json | 1 - src/command.coffee | 36 +++++++++++++++++++++++++----- src/traverse-dependencies.coffee | 38 +++++++++++++++++++++++--------- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 3b3c3bd..5d7f9f5 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "esmangle": "~0.0.10", "esprima": "~1.0.2", "estraverse": "1.3.x", - "MD5": "1.x.x", "mktemp": "~0.3.0", "nopt": "~2.1.2", "resolve": "0.5.x", diff --git a/src/command.coffee b/src/command.coffee index a59023e..091f875 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -45,6 +45,7 @@ delete options.argv # default values options.node ?= on options['inline-sources'] ?= on +options['cache-path'] ?= '.commonjs-everywhere-cache.json' options.alias ?= [] options.handler ?= [] @@ -52,6 +53,7 @@ options.ignoreMissing = options['ignore-missing'] options.sourceMap = options['source-map'] options.inlineSources = options['inline-sources'] options.inlineSourceMap = options['inline-source-map'] +options.cachePath = options['cache-path'] if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] @@ -74,6 +76,9 @@ if options.help --inline-source-map include the source map as a data URI in the generated bundle --inline-sources include source content in generated source maps; default: on --node include process object; emulate node environment; default: on + --cache-path file where to read/write a json-encoded cache that will be + used to speed up future rebuilds. default: + '.commonjs-everywhere-cache.json' in the current directory --version display the version number and exit " process.exit 0 @@ -117,12 +122,12 @@ if options.watch and not options.output console.error '--watch requires --ouput' process.exit 1 -build = (entryPoint, processed = {}) -> +build = (entryPoint) -> + processed = options.processed try newDeps = CJSEverywhere.traverseDependencies entryPoint, root, options if options.watch - console.error "built #{dep.canonicalName} (#{options.cache[filename]})" for own filename, dep of newDeps - processed[file] = newDeps[file] for own file of newDeps + console.error "built #{dep.canonicalName}" for own filename, dep of newDeps catch e if options.watch then console.error "ERROR: #{e.message}" else throw e bundled = CJSEverywhere.bundle processed, originalEntryPoint, root, options @@ -165,14 +170,33 @@ build = (entryPoint, processed = {}) -> processed + startBuild = -> + process.on 'exit', -> + fs.writeFileSync options.cachePath, JSON.stringify options.processed + + process.on 'uncaughtException', (e) -> + # An exception may be thrown due to corrupt cache or incompatibilities + # between versions, remove it to be safe + try fs.unlinkSync options.cachePath + options.processed = {} + throw e + + if fs.existsSync options.cachePath + processed = options.processed = JSON.parse fs.readFileSync options.cachePath, 'utf8' + else + processed = options.processed = {} + if options.watch - options.cache = {} console.error "BUNDLING starting at #{originalEntryPoint}" - processed = build originalEntryPoint + build originalEntryPoint if options.watch + # Flush the cache when the user presses CTRL+C or the process is + # terminated from outside + process.on 'SIGINT', process.exit + process.on 'SIGTERM', process.exit watching = [] do startWatching = (processed) -> for own file, {canonicalName} of processed when file not in watching then do (file, canonicalName) -> @@ -183,7 +207,7 @@ startBuild = -> console.error "WARNING: watched file #{file} has disappeared" return console.error "REBUNDLING starting at #{canonicalName}" - processed = build file, processed + build file startWatching processed return diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index a1643aa..5b4d59d 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -1,10 +1,10 @@ fs = require 'fs' path = require 'path' +util = require 'util' CoffeeScript = require 'coffee-script-redux' esprima = require 'esprima' estraverse = require 'estraverse' -md5 = require 'MD5' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' @@ -31,7 +31,8 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> extensions = ['.js', (ext for own ext of handlers)...] worklist = [relativeResolve {extensions, aliases, root, path: entryPoint}] - processed = {} + processed = options.processed or {} + checked = {} while worklist.length {filename, canonicalName} = worklist.pop() @@ -40,16 +41,18 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> continue unless filename # filter duplicates - continue if {}.hasOwnProperty.call processed, filename + continue if {}.hasOwnProperty.call checked, filename + checked[filename] = true extname = path.extname filename - fileContents = (fs.readFileSync filename).toString() + mtime = (fs.statSync filename).mtime.getTime() + + if processed[filename]?.mtime == mtime + # ignore files that have not changed, but also check its dependencies + worklist = worklist.concat processed[filename].deps + continue - # ignore files that have not changed - if options.cache - digest = md5 fileContents.toString() - continue if options.cache[filename] is digest - options.cache[filename] = digest + fileContents = (fs.readFileSync filename).toString() astOrJs = # handle compile-to-JS languages and other non-JS files @@ -66,13 +69,27 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> else astOrJs - processed[filename] = {canonicalName, ast, fileContents} + deps = [] + processed[filename] = {canonicalName, ast, fileContents, mtime, deps} # add source file information to the AST root node ast.loc ?= {} estraverse.replace ast, enter: (node, parents) -> + if node.type is 'Literal' and util.isRegExp node.value + # RegExps can't be serialized to json(which is done when caching + # processed ASTs), but escodegen never uses any 'instanceof' checks, + # instead it considers all nodes of type 'Literal' with a + # non-primitive value to be RegExp literals. It extracts regexp + # data(source and flags) by calling its 'toString' method and parsing + # the result. + # + # Luckly wrapping a stringified regexp into an array will create an + # object whose toString method produces the same output as calling + # toString in the original RegExp object. + node.value = [node.value.toString()] + # add source file information to each node with source position information if node.loc? then node.loc.source = canonicalName # ignore anything that's not a `require` call @@ -89,6 +106,7 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> try resolved = relativeResolve {extensions, aliases, root, cwd, path: node.arguments[0].value} worklist.push resolved + deps.push resolved catch e if options.ignoreMissing return { type: 'Literal', value: null } From 000a9e5ec92527dc14c0d06dc760f555df5c69fb Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Wed, 2 Oct 2013 17:03:58 -0300 Subject: [PATCH 02/31] Add --module-uids option. Enabling this option will make commonjs-everywhere replace all module names by a unique integer id instead of the root-relative path. This will improve minification a little but will break code that uses __dirname or __filename (which should not be very common in browser-compatible libraries) --- src/bundle.coffee | 11 +++++++---- src/command.coffee | 31 +++++++++++++++++++++++++++---- src/traverse-dependencies.coffee | 6 ++++-- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/src/bundle.coffee b/src/bundle.coffee index be0095c..fd35102 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -53,23 +53,26 @@ require.resolve = function(file){ require.define = function(file, fn){ require.modules[file] = fn; }; """ -wrapFile = (name, program) -> +wrapFile = (name, program, uidFor) -> wrapperProgram = esprima.parse 'require.define(0, function(module, exports, __dirname, __filename){});' wrapper = wrapperProgram.body[0] - wrapper.expression.arguments[0] = { type: 'Literal', value: name } + wrapper.expression.arguments[0] = { type: 'Literal', value: if uidFor then uidFor(name) else name } wrapper.expression.arguments[1].body.body = program.body wrapper module.exports = (processed, entryPoint, root, options) -> prelude = if options.node ? yes then "#{PRELUDE}\n#{PRELUDE_NODE}" else PRELUDE program = esprima.parse prelude + uidFor = options.uidFor for own filename, {ast} of processed - program.body.push wrapFile ast.loc.source, ast + program.body.push wrapFile ast.loc.source, ast, uidFor + + canonicalEntryPoint = canonicalise root, entryPoint requireEntryPoint = type: 'CallExpression' callee: { type: 'Identifier', name: 'require' } - arguments: [{ type: 'Literal', value: canonicalise root, entryPoint }] + arguments: [{ type: 'Literal', value: if uidFor then uidFor(canonicalEntryPoint) else canonicalEntryPoint }] # require/expose the entry point if options.export? diff --git a/src/command.coffee b/src/command.coffee index 091f875..26ccdf2 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -54,6 +54,7 @@ options.sourceMap = options['source-map'] options.inlineSources = options['inline-sources'] options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] +options.moduleUids = options['module-uids'] if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] @@ -79,6 +80,9 @@ if options.help --cache-path file where to read/write a json-encoded cache that will be used to speed up future rebuilds. default: '.commonjs-everywhere-cache.json' in the current directory + --module-uids Instead of replacing module names by their full path, + use unique ids for better minification + (breaks __dirname/__filename) --version display the version number and exit " process.exit 0 @@ -173,7 +177,11 @@ build = (entryPoint) -> startBuild = -> process.on 'exit', -> - fs.writeFileSync options.cachePath, JSON.stringify options.processed + cache = + processed: options.processed + uids: options.uids + moduleUids: options.moduleUids + fs.writeFileSync options.cachePath, JSON.stringify cache process.on 'uncaughtException', (e) -> # An exception may be thrown due to corrupt cache or incompatibilities @@ -183,9 +191,24 @@ startBuild = -> throw e if fs.existsSync options.cachePath - processed = options.processed = JSON.parse fs.readFileSync options.cachePath, 'utf8' - else - processed = options.processed = {} + cache = JSON.parse fs.readFileSync options.cachePath, 'utf8' + {processed, uids, moduleUids} = cache + + if not processed or moduleUids != options.moduleUids + # Either the cache doesn't exist or the cache was saved with a different + # 'moduleUids' value. In either case we must reset it. + processed = {} + uids = {next: 1, names: {}} + + options.processed = processed + options.uids = uids + options.uidFor = (name) -> + if not options.moduleUids + return name + if not {}.hasOwnProperty.call(uids.names, name) + uid = uids.next++ + uids.names[name] = uid + uids.names[name] if options.watch console.error "BUNDLING starting at #{originalEntryPoint}" diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 5b4d59d..dfd7ea0 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -20,6 +20,7 @@ badRequireError = (filename, node, msg) -> module.exports = (entryPoint, root = process.cwd(), options = {}) -> aliases = options.aliases ? {} + uidFor = options.uidFor handlers = '.coffee': (coffee, canonicalName) -> @@ -112,13 +113,14 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> return { type: 'Literal', value: null } else throw e - # rewrite the require to use the root-relative path + # rewrite the require to use the root-relative path or the uid if + # enabled { type: 'CallExpression' callee: node.callee arguments: [{ type: 'Literal' - value: resolved.canonicalName + value: if uidFor then uidFor(resolved.canonicalName) else resolved.canonicalName }, { type: 'Identifier' name: 'module' From 6d6fd8895c520320d78717257da55b8c10e208ba Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Fri, 4 Oct 2013 11:25:34 -0300 Subject: [PATCH 03/31] Refactored, now using string concatenation and manual source map computation instead escodegen to generate the final result --- lib/bundle.js | 161 ++++++++++++++---------- lib/command.js | 208 +++++++++++++++---------------- lib/traverse-dependencies.js | 65 ++++++---- package.json | 4 +- src/bundle.coffee | 202 +++++++++++++++++------------- src/command.coffee | 73 ++++------- src/sourcemap-to-ast.coffee | 19 +++ src/traverse-dependencies.coffee | 45 ++++--- 8 files changed, 424 insertions(+), 353 deletions(-) create mode 100644 src/sourcemap-to-ast.coffee diff --git a/lib/bundle.js b/lib/bundle.js index a57be2e..3e14067 100644 --- a/lib/bundle.js +++ b/lib/bundle.js @@ -1,78 +1,109 @@ // Generated by CoffeeScript 2.0.0-beta7 void function () { - var canonicalise, esprima, PRELUDE, PRELUDE_NODE, wrapFile; + var btoa, bundle, cache$, canonicalise, escodegen, esprima, path, PRELUDE, PRELUDE_NODE, SourceMapConsumer, SourceMapGenerator, sourceMapToAst, wrap, wrapNode; esprima = require('esprima'); + path = require('path'); + cache$ = require('source-map'); + SourceMapConsumer = cache$.SourceMapConsumer; + SourceMapGenerator = cache$.SourceMapGenerator; + btoa = require('Base64').btoa; + escodegen = require('escodegen'); + sourceMapToAst = require('./sourcemap-to-ast'); canonicalise = require('./canonicalise'); - PRELUDE_NODE = "\nvar process = function(){\n var cwd = '/';\n return {\n title: 'browser',\n version: '" + process.version + "',\n browser: true,\n env: {},\n argv: [],\n nextTick: global.setImmediate || function(fn){ setTimeout(fn, 0); },\n cwd: function(){ return cwd; },\n chdir: function(dir){ cwd = dir; }\n };\n}();"; - PRELUDE = "\nfunction require(file, parentModule){\n if({}.hasOwnProperty.call(require.cache, file))\n return require.cache[file];\n\n var resolved = require.resolve(file);\n if(!resolved) throw new Error('Failed to resolve module ' + file);\n\n var module$ = {\n id: file,\n require: require,\n filename: file,\n exports: {},\n loaded: false,\n parent: parentModule,\n children: []\n };\n if(parentModule) parentModule.children.push(module$);\n var dirname = file.slice(0, file.lastIndexOf('/') + 1);\n\n require.cache[file] = module$.exports;\n resolved.call(module$.exports, module$, module$.exports, dirname, file);\n module$.loaded = true;\n return require.cache[file] = module$.exports;\n}\n\nrequire.modules = {};\nrequire.cache = {};\n\nrequire.resolve = function(file){\n return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0;\n};\nrequire.define = function(file, fn){ require.modules[file] = fn; };"; - wrapFile = function (name, program) { - var wrapper, wrapperProgram; - wrapperProgram = esprima.parse('require.define(0, function(module, exports, __dirname, __filename){});'); - wrapper = wrapperProgram.body[0]; - wrapper.expression['arguments'][0] = { - type: 'Literal', - value: name - }; - wrapper.expression['arguments'][1].body.body = program.body; - return wrapper; + PRELUDE_NODE = "\n(function(){\n var cwd = '/';\n return {\n title: 'browser',\n version: '" + process.version + "',\n browser: true,\n env: {},\n argv: [],\n nextTick: global.setImmediate || function(fn){ setTimeout(fn, 0); },\n cwd: function(){ return cwd; },\n chdir: function(dir){ cwd = dir; }\n };\n})()"; + PRELUDE = "\n(function() {\n function require(file, parentModule){\n if({}.hasOwnProperty.call(require.cache, file))\n return require.cache[file];\n\n var resolved = require.resolve(file);\n if(!resolved) throw new Error('Failed to resolve module ' + file);\n\n var module$ = {\n id: file,\n require: require,\n filename: file,\n exports: {},\n loaded: false,\n parent: parentModule,\n children: []\n };\n if(parentModule) parentModule.children.push(module$);\n var dirname = file.slice(0, file.lastIndexOf('/') + 1);\n\n require.cache[file] = module$.exports;\n resolved.call(module$.exports, module$, module$.exports, dirname, file);\n module$.loaded = true;\n return require.cache[file] = module$.exports;\n }\n\n require.modules = {};\n require.cache = {};\n\n require.resolve = function(file){\n return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0;\n };\n require.define = function(file, fn){ require.modules[file] = fn; };\n\n return require;\n)()"; + wrap = function (modules) { + return '(function(global, require, undefined) {\n' + modules + '\n})(this, ' + PRELUDE + ');'; + }; + wrapNode = function (modules) { + return '(function(global, require, process, undefined) {\n' + modules + '\n})(this, ' + PRELUDE + ', ' + PRELUDE_NODE + ');'; }; - module.exports = function (processed, entryPoint, root, options) { - var ast, exportExpression, filename, iife, lhsExpression, prelude, program, requireEntryPoint; - prelude = (null != options.node ? options.node : true) ? '' + PRELUDE + '\n' + PRELUDE_NODE : PRELUDE; - program = esprima.parse(prelude); - for (filename in processed) { - if (!isOwn$(processed, filename)) + bundle = function (entryPoint, options) { + var cache$1, code, filename, lineCount, lineOffset, map, name, orig, src, srcMap; + code = ''; + map = new SourceMapGenerator({ + file: path.basename(options.outFile), + sourceRoot: path.relative(path.dirname(options.outFile), options.root) + }); + lineOffset = 1; + for (filename in options.processed) { + if (!isOwn$(options.processed, filename)) continue; - ast = processed[filename].ast; - program.body.push(wrapFile(ast.loc.source, ast)); - } - requireEntryPoint = { - type: 'CallExpression', - callee: { - type: 'Identifier', - name: 'require' - }, - 'arguments': [{ - type: 'Literal', - value: canonicalise(root, entryPoint) - }] - }; - if (null != options['export']) { - exportExpression = esprima.parse(options['export']).body[0].expression; - lhsExpression = exportExpression.type === 'Identifier' ? { - type: 'MemberExpression', - computed: false, - object: { - type: 'Identifier', - name: 'global' - }, - property: { - type: 'Identifier', - name: exportExpression.name - } - } : exportExpression; - program.body.push({ - type: 'ExpressionStatement', - expression: { - type: 'AssignmentExpression', - operator: '=', - left: lhsExpression, - right: requireEntryPoint - } + { + cache$1 = options.processed[filename]; + name = cache$1.name; + src = cache$1.src; + srcMap = cache$1.srcMap; + lineCount = cache$1.lineCount; + } + if (typeof name !== 'number') + name = "'" + name + "'"; + code += '\nrequire.define(' + name + ', function(module, exports, __dirname, __filename){\n' + src + '\n});'; + lineOffset++; + orig = new SourceMapConsumer(srcMap); + orig.eachMapping(function (m) { + return map.addMapping({ + generated: { + line: m.generatedLine + lineOffset, + column: m.generatedColumn + }, + original: { + line: m.originalLine || m.generatedLine, + column: m.originalColumn || m.generatedColumn + }, + source: filename + }); }); + lineOffset += lineCount + 1; + } + if (typeof entryPoint !== 'number') + entryPoint = "'" + entryPoint + "'"; + code += '\nrequire(' + entryPoint + ');'; + if (options.node) { + code = wrapNode(code); } else { - program.body.push({ - type: 'ExpressionStatement', - expression: requireEntryPoint + code = wrap(code); + } + return { + code: code, + map: map + }; + }; + module.exports = function (entryPoint, options) { + var ast, cache$1, cache$2, code, datauri, esmangle, filename, map, src; + cache$1 = bundle(entryPoint, options); + code = cache$1.code; + map = cache$1.map; + if (options.minify) { + esmangle = require('esmangle'); + ast = esprima.parse(bundled, { loc: true }); + sourceMapToAst(ast, srcMap); + ast = esmangle.mangle(esmangle.optimize(ast), { destructive: true }); + cache$2 = escodegen.generate(ast, { + sourceMap: true, + format: escodegen.FORMAT_MINIFY, + sourceMapWithCode: true, + sourceMapRoot: null != options.sourceMap ? path.relative(path.dirname(options.sourceMap), options.root) || '.' : void 0 }); + code = cache$2.code; + map = cache$2.map; + cache$2; } - iife = esprima.parse('(function(global){}).call(this, this);'); - iife.body[0].expression.callee.object.body.body = program.body; - iife.leadingComments = [{ - type: 'Line', - value: ' Generated by CommonJS Everywhere ' + require('../package.json').version - }]; - return iife; + if ((options.sourceMap || options.inlineSourceMap) && options.inlineSources) + for (filename in processed) { + if (!isOwn$(processed, filename)) + continue; + src = processed[filename].src; + map.setSourceContent(filename, src); + } + if (options.inlineSourceMap) { + datauri = 'data:application/json;charset=utf-8;base64,' + btoa('' + map); + code = '' + code + '\n//# sourceMappingURL=' + datauri; + } + return { + code: code, + map: map + }; }; function isOwn$(o, p) { return {}.hasOwnProperty.call(o, p); diff --git a/lib/command.js b/lib/command.js index fbe5d4e..f7917fb 100644 --- a/lib/command.js +++ b/lib/command.js @@ -1,12 +1,11 @@ // Generated by CoffeeScript 2.0.0-beta7 void function () { - var $0, _, aliasPair, btoa, build, cache$3, cache$4, CJSEverywhere, dep, deps, escodegen, escodegenFormat, fs, handlerPair, knownOpts, match, nopt, opt, optAliases, options, originalEntryPoint, path, positionalArgs, root, startBuild, stdinput; + var $0, _, aliasPair, buildBundle, bundle, cache$3, cache$4, dep, deps, escodegenFormat, fs, handlerPair, knownOpts, match, nopt, opt, optAliases, options, originalEntryPoint, path, positionalArgs, startBuild, stdinput, traverseDependencies; fs = require('fs'); path = require('path'); - escodegen = require('escodegen'); nopt = require('nopt'); - btoa = require('Base64').btoa; - CJSEverywhere = require('./module'); + bundle = require('./bundle'); + traverseDependencies = require('./traverse-dependencies'); escodegenFormat = { indent: { style: ' ', @@ -73,6 +72,14 @@ void function () { options['inline-sources']; else options['inline-sources'] = true; + if (null != options['cache-path']) + options['cache-path']; + else + options['cache-path'] = '.commonjs-everywhere-cache.json'; + if (null != options.root) + options.root; + else + options.root = process.cwd(); if (null != options.alias) options.alias; else @@ -85,10 +92,12 @@ void function () { options.sourceMap = options['source-map']; options.inlineSources = options['inline-sources']; options.inlineSourceMap = options['inline-source-map']; + options.cachePath = options['cache-path']; + options.moduleUids = options['module-uids']; if (options.help) { $0 = process.argv[0] === 'node' ? process.argv[1] : process.argv[0]; $0 = path.basename($0); - console.log('\n Usage: ' + $0 + ' OPT* path/to/entry-file.ext OPT*\n\n -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO\n -h, --handler EXT:MODULE handle files with extension EXT with module MODULE\n -m, --minify minify output\n -o, --output FILE output to FILE instead of stdout\n -r, --root DIR unqualified requires are relative to DIR; default: cwd\n -s, --source-map FILE output a source map to FILE\n -v, --verbose verbose output sent to stderr\n -w, --watch watch input files/dependencies for changes and rebuild bundle\n -x, --export NAME export the given entry module as NAME\n --deps do not bundle; just list the files that would be bundled\n --help display this help message and exit\n --ignore-missing continue without error when dependency resolution fails\n --inline-source-map include the source map as a data URI in the generated bundle\n --inline-sources include source content in generated source maps; default: on\n --node include process object; emulate node environment; default: on\n --version display the version number and exit\n'); + console.log('\n Usage: ' + $0 + " OPT* path/to/entry-file.ext OPT*\n\n -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO\n -h, --handler EXT:MODULE handle files with extension EXT with module MODULE\n -m, --minify minify output\n -o, --output FILE output to FILE instead of stdout\n -r, --root DIR unqualified requires are relative to DIR; default: cwd\n -s, --source-map FILE output a source map to FILE\n -v, --verbose verbose output sent to stderr\n -w, --watch watch input files/dependencies for changes and rebuild bundle\n -x, --export NAME export the given entry module as NAME\n --deps do not bundle; just list the files that would be bundled\n --help display this help message and exit\n --ignore-missing continue without error when dependency resolution fails\n --inline-source-map include the source map as a data URI in the generated bundle\n --inline-sources include source content in generated source maps; default: on\n --node include process object; emulate node environment; default: on\n --cache-path file where to read/write a json-encoded cache that will be\n used to speed up future rebuilds. default:\n '.commonjs-everywhere-cache.json' in the current directory\n --module-uids Instead of replacing module names by their full path,\n use unique ids for better minification\n (breaks __dirname/__filename)\n --version display the version number and exit\n"); process.exit(0); } if (options.version) { @@ -125,10 +134,9 @@ void function () { } } delete options.handler; - root = options.root ? path.resolve(options.root) : process.cwd(); originalEntryPoint = positionalArgs[0]; if (options.deps) { - deps = CJSEverywhere.traverseDependencies(originalEntryPoint, root, options); + deps = traverseDependencies(originalEntryPoint, options.root, options); for (_ in deps) { if (!isOwn$(deps, _)) continue; @@ -138,60 +146,15 @@ void function () { process.exit(0); } if (options.watch && !options.output) { - console.error('--watch requires --ouput'); + console.error('--watch requires --output'); process.exit(1); } - build = function (entryPoint, processed) { - var bundled, cache$5, cache$6, canonicalName, code, datauri, e, esmangle, file, fileContents, filename, map, newDeps, sourceMappingUrl; - if (null == processed) - processed = {}; - try { - newDeps = CJSEverywhere.traverseDependencies(entryPoint, root, options); - if (options.watch) - for (filename in newDeps) { - if (!isOwn$(newDeps, filename)) - continue; - dep = newDeps[filename]; - console.error('built ' + dep.canonicalName + ' (' + options.cache[filename] + ')'); - } - for (file in newDeps) { - if (!isOwn$(newDeps, file)) - continue; - processed[file] = newDeps[file]; - } - } catch (e$) { - e = e$; - if (options.watch) { - console.error('ERROR: ' + e.message); - } else { - throw e; - } - } - bundled = CJSEverywhere.bundle(processed, originalEntryPoint, root, options); - if (options.minify) { - esmangle = require('esmangle'); - bundled = esmangle.mangle(esmangle.optimize(bundled), { destructive: true }); - } - cache$5 = escodegen.generate(bundled, { - comment: !options.minify, - sourceMap: true, - sourceMapWithCode: true, - sourceMapRoot: null != options.sourceMap ? path.relative(path.dirname(options.sourceMap), root) || '.' : void 0, - format: options.minify ? escodegen.FORMAT_MINIFY : escodegenFormat - }); + buildBundle = function () { + var cache$5, code, datauri, map, sourceMappingUrl; + traverseDependencies(originalEntryPoint, options); + cache$5 = bundle(originalEntryPoint, options); code = cache$5.code; map = cache$5.map; - if ((options.sourceMap || options.inlineSourceMap) && options.inlineSources) - for (filename in processed) { - if (!isOwn$(processed, filename)) - continue; - { - cache$6 = processed[filename]; - canonicalName = cache$6.canonicalName; - fileContents = cache$6.fileContents; - } - map.setSourceContent(canonicalName, fileContents); - } if (options.sourceMap) { fs.writeFileSync(options.sourceMap, '' + map); sourceMappingUrl = options.output ? path.relative(path.dirname(options.output), options.sourceMap) : options.sourceMap; @@ -203,53 +166,96 @@ void function () { code = '' + code + '\n//# sourceMappingURL=' + datauri; } if (options.output) { - fs.writeFileSync(options.output, code); + return fs.writeFileSync(options.output, code); } else { - process.stdout.write('' + code + '\n'); + return process.stdout.write('' + code + '\n'); } - if (options.watch || options.verbose) - console.error('BUNDLE COMPLETE'); - return processed; }; startBuild = function () { - var processed, startWatching, watching; - if (options.watch) { - options.cache = {}; - console.error('BUNDLING starting at ' + originalEntryPoint); + var building, cache, cache$5, moduleUids, processed, uids, watching; + process.on('exit', function () { + var cache; + cache = { + processed: options.processed, + uids: options.uids, + moduleUids: options.moduleUids + }; + return fs.writeFileSync(options.cachePath, JSON.stringify(cache)); + }); + process.on('uncaughtException', function (e) { + try { + fs.unlinkSync(options.cachePath); + } catch (e$) { + } + options.processed = {}; + throw e; + }); + if (fs.existsSync(options.cachePath)) { + cache = JSON.parse(fs.readFileSync(options.cachePath, 'utf8')); + cache$5 = cache; + processed = cache$5.processed; + uids = cache$5.uids; + moduleUids = cache$5.moduleUids; + cache$5; + } + if (!processed || moduleUids !== options.moduleUids) { + processed = {}; + uids = { + next: 1, + names: {} + }; } - processed = build(originalEntryPoint); + options.processed = processed; + options.uids = uids; + options.uidFor = function (name) { + var uid; + if (!options.moduleUids) + return name; + if (!{}.hasOwnProperty.call(uids.names, name)) { + uid = uids.next++; + uids.names[name] = uid; + } + return uids.names[name]; + }; + if (options.watch) + console.error('BUNDLING starting at ' + originalEntryPoint); + buildBundle(originalEntryPoint); if (options.watch) { - watching = []; - return (startWatching = function (processed) { - return function (accum$) { - var canonicalName, file; - for (file in processed) { - if (!isOwn$(processed, file)) - continue; - canonicalName = processed[file].canonicalName; - if (!!in$(file, watching)) - continue; - accum$.push(function (file, canonicalName) { - watching.push(file); - return fs.watchFile(file, { - persistent: true, - interval: 500 - }, function (curr, prev) { - var ino; - ino = process.platform === 'win32' ? null != curr.ino : curr.ino; - if (!ino) { - console.error('WARNING: watched file ' + file + ' has disappeared'); - return; - } - console.error('REBUNDLING starting at ' + canonicalName); - processed = build(file, processed); - startWatching(processed); - }); - }(file, canonicalName)); - } - return accum$; - }.call(this, []); - })(processed); + process.on('SIGINT', process.exit); + process.on('SIGTERM', process.exit); + watching = {}; + building = false; + return function (accum$) { + var file; + for (file in processed) { + if (!isOwn$(processed, file)) + continue; + if (!!(file in watching)) + continue; + accum$.push(function (file) { + watching[file] = true; + return fs.watchFile(file, { + persistent: true, + interval: 500 + }, function (curr, prev) { + var ino; + if (building) + return; + building = true; + ino = process.platform === 'win32' ? null != curr.ino : curr.ino; + if (!ino) { + console.error('WARNING: watched file ' + file + ' has disappeared'); + return; + } + console.error('' + file + ' changed, rebuilding'); + buildBundle(originalEntryPoint); + console.error('done'); + building = false; + }); + }(file)); + } + return accum$; + }.call(this, []); } }; if (originalEntryPoint === '-') { @@ -273,10 +279,4 @@ void function () { function isOwn$(o, p) { return {}.hasOwnProperty.call(o, p); } - function in$(member, list) { - for (var i = 0, length = list.length; i < length; ++i) - if (i in list && list[i] === member) - return true; - return false; - } }.call(this); diff --git a/lib/traverse-dependencies.js b/lib/traverse-dependencies.js index 63e8214..7c65fcc 100644 --- a/lib/traverse-dependencies.js +++ b/lib/traverse-dependencies.js @@ -1,12 +1,12 @@ // Generated by CoffeeScript 2.0.0-beta7 void function () { - var badRequireError, canonicalise, CoffeeScript, esprima, estraverse, fs, md5, path, relativeResolve; + var badRequireError, canonicalise, escodegen, esprima, estraverse, fs, path, relativeResolve, util; fs = require('fs'); path = require('path'); - CoffeeScript = require('coffee-script-redux'); + util = require('util'); esprima = require('esprima'); estraverse = require('estraverse'); - md5 = require('MD5'); + escodegen = require('escodegen'); canonicalise = require('./canonicalise'); relativeResolve = require('./relative-resolve'); badRequireError = function (filename, node, msg) { @@ -14,13 +14,11 @@ void function () { filename = '' + filename + ':' + node.loc.start.line + ':' + node.loc.start.column; throw 'illegal require: ' + msg + '\n `' + require('escodegen').generate(node) + '`\n in ' + filename + ''; }; - module.exports = function (entryPoint, root, options) { - var aliases, ast, astOrJs, cache$, cache$1, canonicalName, digest, ext, extensions, extname, fileContents, filename, handler, handlers, processed, worklist; - if (null == root) - root = process.cwd(); - if (null == options) - options = {}; + module.exports = function (entryPoint, options) { + var aliases, ast, astOrJs, cache$, cache$1, cache$2, canonicalName, checked, deps, ext, extensions, extname, filename, handler, handlers, lineCount, mtime, name, processed, root, src, srcMap, uidFor, worklist; aliases = null != options.aliases ? options.aliases : {}; + uidFor = options.uidFor; + root = options.root; handlers = { '.coffee': function (coffee, canonicalName) { return CoffeeScript.compile(CoffeeScript.parse(coffee, { raw: true }), { bare: true }); @@ -52,24 +50,25 @@ void function () { root: root, path: entryPoint })]; - processed = {}; + processed = options.processed || {}; + checked = {}; while (worklist.length) { cache$1 = worklist.pop(); filename = cache$1.filename; canonicalName = cache$1.canonicalName; if (!filename) continue; - if ({}.hasOwnProperty.call(processed, filename)) + if ({}.hasOwnProperty.call(checked, filename)) continue; + checked[filename] = true; extname = path.extname(filename); - fileContents = fs.readFileSync(filename).toString(); - if (options.cache) { - digest = md5(fileContents.toString()); - if (options.cache[filename] === digest) - continue; - options.cache[filename] = digest; + mtime = fs.statSync(filename).mtime.getTime(); + if ((null != processed[filename] ? processed[filename].mtime : void 0) === mtime) { + worklist = worklist.concat(processed[filename].deps); + continue; } - astOrJs = {}.hasOwnProperty.call(handlers, extname) ? handlers[extname](fileContents, canonicalName) : fileContents; + src = fs.readFileSync(filename).toString(); + astOrJs = {}.hasOwnProperty.call(handlers, extname) ? handlers[extname](src, canonicalName) : src; ast = typeof astOrJs === 'string' ? function () { var e; try { @@ -82,15 +81,12 @@ void function () { throw new Error('Syntax error in ' + filename + ' at line ' + e.lineNumber + ', column ' + e.column + e.message.slice(e.message.indexOf(':'))); } }.call(this) : astOrJs; - processed[filename] = { - canonicalName: canonicalName, - ast: ast, - fileContents: fileContents - }; if (null != ast.loc) ast.loc; else ast.loc = {}; + deps = []; + name = uidFor(canonicalName); estraverse.replace(ast, { enter: function (node, parents) { var cwd, e, resolved; @@ -109,11 +105,12 @@ void function () { resolved = relativeResolve({ extensions: extensions, aliases: aliases, - root: root, + root: options.root, cwd: cwd, path: node['arguments'][0].value }); worklist.push(resolved); + deps.push(resolved); } catch (e$) { e = e$; if (options.ignoreMissing) { @@ -131,7 +128,7 @@ void function () { 'arguments': [ { type: 'Literal', - value: resolved.canonicalName + value: uidFor(resolved.canonicalName) }, { type: 'Identifier', @@ -141,6 +138,24 @@ void function () { }; } }); + cache$2 = escodegen.generate(ast, { + sourceMap: true, + format: escodegen.FORMAT_DEFAULTS, + sourceMapWithCode: true, + sourceMapRoot: null != options.sourceMap ? path.relative(path.dirname(options.sourceMap), options.root) || '.' : void 0 + }); + src = cache$2.code; + srcMap = cache$2.map; + srcMap = srcMap.toString(); + lineCount = src.split('\n').length - 1; + processed[filename] = { + name: name, + src: src, + srcMap: srcMap, + lineCount: lineCount, + mtime: mtime, + deps: deps + }; } return processed; }; diff --git a/package.json b/package.json index 5d7f9f5..0f74233 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,9 @@ "punycode": "*", "querystring": "*", "vm-browserify": "*", - "zlib-browserify": "*" + "zlib-browserify": "*", + "source-map": "~0.1.30", + "handlebars": "~1.0.12" }, "devDependencies": { "mocha": "~1.8.1", diff --git a/src/bundle.coffee b/src/bundle.coffee index fd35102..0904d76 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -1,9 +1,14 @@ esprima = require 'esprima' +path = require 'path' +{SourceMapConsumer, SourceMapGenerator} = require 'source-map' +{btoa} = require 'Base64' +escodegen = require 'escodegen' +sourceMapToAst = require './sourcemap-to-ast' canonicalise = require './canonicalise' PRELUDE_NODE = """ -var process = function(){ +(function(){ var cwd = '/'; return { title: 'browser', @@ -15,94 +20,121 @@ var process = function(){ cwd: function(){ return cwd; }, chdir: function(dir){ cwd = dir; } }; -}(); +})() """ PRELUDE = """ -function require(file, parentModule){ - if({}.hasOwnProperty.call(require.cache, file)) - return require.cache[file]; - - var resolved = require.resolve(file); - if(!resolved) throw new Error('Failed to resolve module ' + file); - - var module$ = { - id: file, - require: require, - filename: file, - exports: {}, - loaded: false, - parent: parentModule, - children: [] +(function() { + function require(file, parentModule){ + if({}.hasOwnProperty.call(require.cache, file)) + return require.cache[file]; + + var resolved = require.resolve(file); + if(!resolved) throw new Error('Failed to resolve module ' + file); + + var module$ = { + id: file, + require: require, + filename: file, + exports: {}, + loaded: false, + parent: parentModule, + children: [] + }; + if(parentModule) parentModule.children.push(module$); + var dirname = file.slice(0, file.lastIndexOf('/') + 1); + + require.cache[file] = module$.exports; + resolved.call(module$.exports, module$, module$.exports, dirname, file); + module$.loaded = true; + return require.cache[file] = module$.exports; + } + + require.modules = {}; + require.cache = {}; + + require.resolve = function(file){ + return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0; }; - if(parentModule) parentModule.children.push(module$); - var dirname = file.slice(0, file.lastIndexOf('/') + 1); - - require.cache[file] = module$.exports; - resolved.call(module$.exports, module$, module$.exports, dirname, file); - module$.loaded = true; - return require.cache[file] = module$.exports; -} - -require.modules = {}; -require.cache = {}; - -require.resolve = function(file){ - return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0; -}; -require.define = function(file, fn){ require.modules[file] = fn; }; + require.define = function(file, fn){ require.modules[file] = fn; }; + + return require; +)() """ -wrapFile = (name, program, uidFor) -> - wrapperProgram = esprima.parse 'require.define(0, function(module, exports, __dirname, __filename){});' - wrapper = wrapperProgram.body[0] - wrapper.expression.arguments[0] = { type: 'Literal', value: if uidFor then uidFor(name) else name } - wrapper.expression.arguments[1].body.body = program.body - wrapper - -module.exports = (processed, entryPoint, root, options) -> - prelude = if options.node ? yes then "#{PRELUDE}\n#{PRELUDE_NODE}" else PRELUDE - program = esprima.parse prelude - uidFor = options.uidFor - for own filename, {ast} of processed - program.body.push wrapFile ast.loc.source, ast, uidFor - - canonicalEntryPoint = canonicalise root, entryPoint - - requireEntryPoint = - type: 'CallExpression' - callee: { type: 'Identifier', name: 'require' } - arguments: [{ type: 'Literal', value: if uidFor then uidFor(canonicalEntryPoint) else canonicalEntryPoint }] - - # require/expose the entry point - if options.export? - exportExpression = (esprima.parse options.export).body[0].expression - lhsExpression = - if exportExpression.type is 'Identifier' - type: 'MemberExpression' - computed: false - object: { type: 'Identifier', name: 'global' } - property: { type: 'Identifier', name: exportExpression.name } - else - exportExpression - program.body.push - type: 'ExpressionStatement' - expression: - type: 'AssignmentExpression' - operator: '=' - left: lhsExpression - right: requireEntryPoint +wrap = (modules) -> """ + (function(global, require, undefined) { + #{modules} + })(this, #{PRELUDE}); + """ + +wrapNode = (modules) -> """ + (function(global, require, process, undefined) { + #{modules} + })(this, #{PRELUDE}, #{PRELUDE_NODE}); + """ + +bundle = (entryPoint, options) -> + code = '' + map = new SourceMapGenerator + file: path.basename(options.outFile) + sourceRoot: path.relative(path.dirname(options.outFile), options.root) + lineOffset = 1 + + for own filename, {name, src, srcMap, lineCount} of options.processed + if typeof name != 'number' + name = "'#{name}'" + code += """ + \nrequire.define(#{name}, function(module, exports, __dirname, __filename){ + #{src} + }); + """ + lineOffset++ # skip the 'require.define' line + orig = new SourceMapConsumer srcMap + orig.eachMapping (m) -> + map.addMapping + generated: + line: m.generatedLine + lineOffset + column: m.generatedColumn + original: + line: m.originalLine or m.generatedLine + column: m.originalColumn or m.generatedColumn + source: filename + lineOffset += lineCount + 1 # skip all the source lines plus the '})' line + + if typeof entryPoint != 'number' + entryPoint = "'#{entryPoint}'" + + code += "\nrequire(#{entryPoint});" + + if options.node + code = wrapNode(code) else - program.body.push - type: 'ExpressionStatement' - expression: requireEntryPoint - - # wrap everything in IIFE for safety; define global var - iife = esprima.parse '(function(global){}).call(this, this);' - iife.body[0].expression.callee.object.body.body = program.body - iife.leadingComments = [ - type: 'Line' - value: " Generated by CommonJS Everywhere #{(require '../package.json').version}" - ] - - iife + code = wrap(code) + + return {code, map} + + +module.exports = (entryPoint, options) -> + {code, map} = bundle entryPoint, options + + if options.minify + esmangle = require 'esmangle' + ast = esprima.parse bundled, loc: yes + sourceMapToAst ast, srcMap + ast = esmangle.mangle (esmangle.optimize ast), destructive: yes + {code, map} = escodegen.generate ast, + sourceMap: yes + format: escodegen.FORMAT_MINIFY + sourceMapWithCode: yes + sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' + + if (options.sourceMap or options.inlineSourceMap) and options.inlineSources + for own filename, {src} of processed + map.setSourceContent filename, src + + if options.inlineSourceMap + datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" + code = "#{code}\n//# sourceMappingURL=#{datauri}" + + return {code, map} diff --git a/src/command.coffee b/src/command.coffee index 26ccdf2..23ad068 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -1,11 +1,9 @@ fs = require 'fs' path = require 'path' - -escodegen = require 'escodegen' nopt = require 'nopt' -{btoa} = require 'Base64' -CJSEverywhere = require './module' +bundle = require './bundle' +traverseDependencies = require './traverse-dependencies' escodegenFormat = indent: @@ -46,6 +44,7 @@ delete options.argv options.node ?= on options['inline-sources'] ?= on options['cache-path'] ?= '.commonjs-everywhere-cache.json' +options['root'] ?= process.cwd() options.alias ?= [] options.handler ?= [] @@ -114,42 +113,20 @@ for handlerPair in options.handler process.exit 1 delete options.handler -root = if options.root then path.resolve options.root else process.cwd() originalEntryPoint = positionalArgs[0] if options.deps - deps = CJSEverywhere.traverseDependencies originalEntryPoint, root, options + deps = traverseDependencies originalEntryPoint, options.root, options console.log dep.canonicalName for own _, dep of deps process.exit 0 if options.watch and not options.output - console.error '--watch requires --ouput' + console.error '--watch requires --output' process.exit 1 -build = (entryPoint) -> - processed = options.processed - try - newDeps = CJSEverywhere.traverseDependencies entryPoint, root, options - if options.watch - console.error "built #{dep.canonicalName}" for own filename, dep of newDeps - catch e - if options.watch then console.error "ERROR: #{e.message}" else throw e - bundled = CJSEverywhere.bundle processed, originalEntryPoint, root, options - - if options.minify - esmangle = require 'esmangle' - bundled = esmangle.mangle (esmangle.optimize bundled), destructive: yes - - {code, map} = escodegen.generate bundled, - comment: not options.minify - sourceMap: yes - sourceMapWithCode: yes - sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), root) or '.' - format: if options.minify then escodegen.FORMAT_MINIFY else escodegenFormat - - if (options.sourceMap or options.inlineSourceMap) and options.inlineSources - for own filename, {canonicalName, fileContents} of processed - map.setSourceContent canonicalName, fileContents +buildBundle = -> + traverseDependencies originalEntryPoint, options + {code, map} = bundle originalEntryPoint, options if options.sourceMap fs.writeFileSync options.sourceMap, "#{map}" @@ -169,11 +146,6 @@ build = (entryPoint) -> else process.stdout.write "#{code}\n" - if options.watch or options.verbose - console.error 'BUNDLE COMPLETE' - - processed - startBuild = -> process.on 'exit', -> @@ -213,26 +185,29 @@ startBuild = -> if options.watch console.error "BUNDLING starting at #{originalEntryPoint}" - build originalEntryPoint + buildBundle originalEntryPoint if options.watch # Flush the cache when the user presses CTRL+C or the process is # terminated from outside process.on 'SIGINT', process.exit process.on 'SIGTERM', process.exit - watching = [] - do startWatching = (processed) -> - for own file, {canonicalName} of processed when file not in watching then do (file, canonicalName) -> - watching.push file - fs.watchFile file, {persistent: yes, interval: 500}, (curr, prev) -> - ino = if process.platform is 'win32' then curr.ino? else curr.ino - unless ino - console.error "WARNING: watched file #{file} has disappeared" - return - console.error "REBUNDLING starting at #{canonicalName}" - build file - startWatching processed + watching = {} + building = false + for own file of processed when file not of watching then do (file) -> + watching[file] = true + fs.watchFile file, {persistent: yes, interval: 500}, (curr, prev) -> + if building then return + building = true + ino = if process.platform is 'win32' then curr.ino? else curr.ino + unless ino + console.error "WARNING: watched file #{file} has disappeared" return + console.error "#{file} changed, rebuilding" + buildBundle originalEntryPoint + console.error "done" + building = false + return if originalEntryPoint is '-' # support reading input from stdin diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee new file mode 100644 index 0000000..afcb710 --- /dev/null +++ b/src/sourcemap-to-ast.coffee @@ -0,0 +1,19 @@ +{SourceMapConsumer} = require 'source-map' +{traverse} = require 'estraverse' + +module.exports = (ast, srcMap) -> + map = new SourceMapConsumer srcMap + + traverse ast, + enter: (node) -> + origStart = map.originalPositionFor node.loc.start + origEnd = map.originalPositionFor node.loc.end + assert origStart.source == origEnd.source, 'Invalid source map' + node.loc = + start: + line: origStart.line + column: origStart.column + end: + line: origEnd.line + column: origEnd.column + source: origStart.source diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index dfd7ea0..1c3d8ec 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -2,9 +2,9 @@ fs = require 'fs' path = require 'path' util = require 'util' -CoffeeScript = require 'coffee-script-redux' esprima = require 'esprima' estraverse = require 'estraverse' +escodegen = require 'escodegen' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' @@ -18,9 +18,10 @@ badRequireError = (filename, node, msg) -> in #{filename} """ -module.exports = (entryPoint, root = process.cwd(), options = {}) -> +module.exports = (entryPoint, options) -> aliases = options.aliases ? {} uidFor = options.uidFor + root = options.root handlers = '.coffee': (coffee, canonicalName) -> @@ -53,14 +54,14 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> worklist = worklist.concat processed[filename].deps continue - fileContents = (fs.readFileSync filename).toString() + src = (fs.readFileSync filename).toString() astOrJs = # handle compile-to-JS languages and other non-JS files if {}.hasOwnProperty.call handlers, extname - handlers[extname] fileContents, canonicalName + handlers[extname] src, canonicalName else # assume JS - fileContents + src ast = if typeof astOrJs is 'string' @@ -70,28 +71,13 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> else astOrJs - deps = [] - processed[filename] = {canonicalName, ast, fileContents, mtime, deps} - # add source file information to the AST root node ast.loc ?= {} + deps = [] + name = uidFor(canonicalName) estraverse.replace ast, enter: (node, parents) -> - if node.type is 'Literal' and util.isRegExp node.value - # RegExps can't be serialized to json(which is done when caching - # processed ASTs), but escodegen never uses any 'instanceof' checks, - # instead it considers all nodes of type 'Literal' with a - # non-primitive value to be RegExp literals. It extracts regexp - # data(source and flags) by calling its 'toString' method and parsing - # the result. - # - # Luckly wrapping a stringified regexp into an array will create an - # object whose toString method produces the same output as calling - # toString in the original RegExp object. - node.value = [node.value.toString()] - - # add source file information to each node with source position information if node.loc? then node.loc.source = canonicalName # ignore anything that's not a `require` call return unless node.type is 'CallExpression' and node.callee.type is 'Identifier' and node.callee.name is 'require' @@ -105,7 +91,7 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> console.error "required \"#{node.arguments[0].value}\" from \"#{canonicalName}\"" # if we are including this file, its requires need to be processed as well try - resolved = relativeResolve {extensions, aliases, root, cwd, path: node.arguments[0].value} + resolved = relativeResolve {extensions, aliases, root: options.root, cwd, path: node.arguments[0].value} worklist.push resolved deps.push resolved catch e @@ -120,11 +106,22 @@ module.exports = (entryPoint, root = process.cwd(), options = {}) -> callee: node.callee arguments: [{ type: 'Literal' - value: if uidFor then uidFor(resolved.canonicalName) else resolved.canonicalName + value: uidFor(resolved.canonicalName) }, { type: 'Identifier' name: 'module' }] } + {code: src, map: srcMap} = escodegen.generate ast, + sourceMap: yes + format: escodegen.FORMAT_DEFAULTS + sourceMapWithCode: yes + sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' + + srcMap = srcMap.toString() + + lineCount = src.split('\n').length - 1 + processed[filename] = {name, src, srcMap, lineCount, mtime, deps} + processed From 47a2782ace1c474a13ae10dbec999c677b7bae53 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Fri, 4 Oct 2013 11:37:34 -0300 Subject: [PATCH 04/31] Forked and updated package.json --- package.json | 19 +++++++++++-------- src/bundle.coffee | 5 ++--- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 0f74233..53893b4 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { - "name": "commonjs-everywhere", - "version": "0.9.4", - "description": "CommonJS browser bundler with aliasing, extensibility, and source maps from the minified JS bundle", - "homepage": "https://github.com/michaelficarra/commonjs-everywhere", + "name": "powerbuild", + "version": "0.0.1", + "description": "CommonJS browser bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere, speed improvements, persistent disk cache for incremental builds and bundled grunt task", + "homepage": "https://github.com/tarruda/powerbuild", "keywords": [ "CommonJS", "CommonJS Everywhere", @@ -17,15 +17,19 @@ "name": "Michael Ficarra", "email": "git@michael.ficarra.me" }, + "contributors": [ + "name": "Thiago de arruda", + "email": "tpadilha84@gmail.com" + ], "repository": { "type": "git", - "url": "git://github.com/michaelficarra/commonjs-everywhere.git" + "url": "git://github.com/tarruda/powerbuild.git" }, - "bugs": "https://github.com/michaelficarra/commonjs-everywhere/issues", + "bugs": "https://github.com/tarruda/powerbuild/issues", "engines": { "node": "0.8.x || 0.9.x || 0.10.x" }, - "main": "lib/module", + "main": "lib/index", "directories": { "bin": "bin", "lib": "lib", @@ -33,7 +37,6 @@ }, "dependencies": { "Base64": "~0.1.2", - "coffee-script-redux": "2.0.0-beta7", "escodegen": "~0.0.24", "esmangle": "~0.0.10", "esprima": "~1.0.2", diff --git a/src/bundle.coffee b/src/bundle.coffee index 0904d76..d47a625 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -79,7 +79,7 @@ bundle = (entryPoint, options) -> map = new SourceMapGenerator file: path.basename(options.outFile) sourceRoot: path.relative(path.dirname(options.outFile), options.root) - lineOffset = 1 + lineOffset = 1 # global wrapper for own filename, {name, src, srcMap, lineCount} of options.processed if typeof name != 'number' @@ -89,7 +89,7 @@ bundle = (entryPoint, options) -> #{src} }); """ - lineOffset++ # skip the 'require.define' line + lineOffset += 2# skip linefeed plus the 'require.define' line orig = new SourceMapConsumer srcMap orig.eachMapping (m) -> map.addMapping @@ -100,7 +100,6 @@ bundle = (entryPoint, options) -> line: m.originalLine or m.generatedLine column: m.originalColumn or m.generatedColumn source: filename - lineOffset += lineCount + 1 # skip all the source lines plus the '})' line if typeof entryPoint != 'number' entryPoint = "'#{entryPoint}'" From a086a449ea41a2ea09b53e13e5d54f9d66393e67 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Fri, 4 Oct 2013 13:14:00 -0300 Subject: [PATCH 05/31] Removed coffeescript redux dependency, improved handler extensibility to enable usage of input source maps --- .gitignore | 3 ++- Makefile | 6 +++--- package.json | 10 ++++++--- src/bundle.coffee | 28 ++++++++++++------------ src/command.coffee | 4 ++-- src/sourcemap-to-ast.coffee | 3 ++- src/traverse-dependencies.coffee | 37 +++++++++++++++++++++----------- 7 files changed, 55 insertions(+), 36 deletions(-) diff --git a/.gitignore b/.gitignore index 8b394c1..361440b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ commonjs-everywhere-*.tgz -test/fixtures/* \ No newline at end of file +test/fixtures/* +/lib/ diff --git a/Makefile b/Makefile index 9f08ed2..3100627 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,15 @@ CHANGELOG=CHANGELOG SRC = $(shell find src -name "*.coffee" -type f | sort) LIB = $(SRC:src/%.coffee=lib/%.js) -COFFEE=node_modules/.bin/coffee --js -MOCHA=node_modules/.bin/mocha --compilers coffee:coffee-script-redux/register -r coffee-script-redux/register -r test-setup.coffee -u tdd +COFFEE=node_modules/.bin/coffee +MOCHA=node_modules/.bin/mocha --compilers coffee:coffee-script -r coffee-script -r test-setup.coffee -u tdd all: build test build: $(LIB) lib/%.js: src/%.coffee @dirname "$@" | xargs mkdir -p - $(COFFEE) <"$<" >"$@" + $(COFFEE) -c -m -o lib "$<" .PHONY: release test loc clean diff --git a/package.json b/package.json index 53893b4..da5639a 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,10 @@ "email": "git@michael.ficarra.me" }, "contributors": [ - "name": "Thiago de arruda", - "email": "tpadilha84@gmail.com" + { + "name": "Thiago de arruda", + "email": "tpadilha84@gmail.com" + } ], "repository": { "type": "git", @@ -54,7 +56,9 @@ "vm-browserify": "*", "zlib-browserify": "*", "source-map": "~0.1.30", - "handlebars": "~1.0.12" + "handlebars": "~1.0.12", + "coffee-script": "~1.6.3", + "source-map-support": "~0.2.3" }, "devDependencies": { "mocha": "~1.8.1", diff --git a/src/bundle.coffee b/src/bundle.coffee index d47a625..4568941 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -75,24 +75,24 @@ wrapNode = (modules) -> """ """ bundle = (entryPoint, options) -> - code = '' - map = new SourceMapGenerator + result = '' + resultMap = new SourceMapGenerator file: path.basename(options.outFile) sourceRoot: path.relative(path.dirname(options.outFile), options.root) lineOffset = 1 # global wrapper - for own filename, {name, src, srcMap, lineCount} of options.processed + for own filename, {name, code, map, lineCount} of options.processed if typeof name != 'number' name = "'#{name}'" - code += """ + result += """ \nrequire.define(#{name}, function(module, exports, __dirname, __filename){ - #{src} + #{code} }); """ lineOffset += 2# skip linefeed plus the 'require.define' line - orig = new SourceMapConsumer srcMap + orig = new SourceMapConsumer map orig.eachMapping (m) -> - map.addMapping + resultMap.addMapping generated: line: m.generatedLine + lineOffset column: m.generatedColumn @@ -104,14 +104,14 @@ bundle = (entryPoint, options) -> if typeof entryPoint != 'number' entryPoint = "'#{entryPoint}'" - code += "\nrequire(#{entryPoint});" + result += "\nrequire(#{entryPoint});" if options.node - code = wrapNode(code) + result = wrapNode(result) else - code = wrap(code) + result = wrap(result) - return {code, map} + return {code: result, map: resultMap} module.exports = (entryPoint, options) -> @@ -120,7 +120,7 @@ module.exports = (entryPoint, options) -> if options.minify esmangle = require 'esmangle' ast = esprima.parse bundled, loc: yes - sourceMapToAst ast, srcMap + sourceMapToAst ast, map ast = esmangle.mangle (esmangle.optimize ast), destructive: yes {code, map} = escodegen.generate ast, sourceMap: yes @@ -129,8 +129,8 @@ module.exports = (entryPoint, options) -> sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' if (options.sourceMap or options.inlineSourceMap) and options.inlineSources - for own filename, {src} of processed - map.setSourceContent filename, src + for own filename, {code} of processed + map.setSourceContent filename, code if options.inlineSourceMap datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" diff --git a/src/command.coffee b/src/command.coffee index 23ad068..d90be4e 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -43,7 +43,7 @@ delete options.argv # default values options.node ?= on options['inline-sources'] ?= on -options['cache-path'] ?= '.commonjs-everywhere-cache.json' +options['cache-path'] ?= '.powerbuild-cache~' options['root'] ?= process.cwd() options.alias ?= [] options.handler ?= [] @@ -78,7 +78,7 @@ if options.help --node include process object; emulate node environment; default: on --cache-path file where to read/write a json-encoded cache that will be used to speed up future rebuilds. default: - '.commonjs-everywhere-cache.json' in the current directory + '.powerbuild-cache~' in the current directory --module-uids Instead of replacing module names by their full path, use unique ids for better minification (breaks __dirname/__filename) diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee index afcb710..517a2f0 100644 --- a/src/sourcemap-to-ast.coffee +++ b/src/sourcemap-to-ast.coffee @@ -1,5 +1,6 @@ {SourceMapConsumer} = require 'source-map' {traverse} = require 'estraverse' +assert = require 'assert' module.exports = (ast, srcMap) -> map = new SourceMapConsumer srcMap @@ -16,4 +17,4 @@ module.exports = (ast, srcMap) -> end: line: origEnd.line column: origEnd.column - source: origStart.source + source: origStart.source || node.loc.source diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 1c3d8ec..0585cd1 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -2,12 +2,14 @@ fs = require 'fs' path = require 'path' util = require 'util' +coffee = require 'coffee-script' esprima = require 'esprima' estraverse = require 'estraverse' escodegen = require 'escodegen' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' +sourceMapToAst = require './sourcemap-to-ast' badRequireError = (filename, node, msg) -> if node.loc? and node.loc?.start? @@ -24,8 +26,9 @@ module.exports = (entryPoint, options) -> root = options.root handlers = - '.coffee': (coffee, canonicalName) -> - CoffeeScript.compile (CoffeeScript.parse coffee, raw: yes), bare: yes + '.coffee': (src, canonicalName) -> + {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true + return {code: js, map: v3SourceMap} '.json': (json, canonicalName) -> esprima.parse "module.exports = #{json}", loc: yes, source: canonicalName for own ext, handler of options.handlers ? {} @@ -63,13 +66,21 @@ module.exports = (entryPoint, options) -> else # assume JS src - ast = - if typeof astOrJs is 'string' - try esprima.parse astOrJs, loc: yes, source: canonicalName - catch e + if typeof astOrJs == 'string' + astOrJs = {code: astOrJs} + + if astOrJs.code + try + ast = esprima.parse astOrJs.code, loc: yes, source: canonicalName + if astOrJs.map + sourceMapToAst ast, astOrJs.map + catch e + if e.lineNumber throw new Error "Syntax error in #{filename} at line #{e.lineNumber}, column #{e.column}#{e.message[(e.message.indexOf ':')..]}" - else - astOrJs + else + throw e + else + ast = astOrJs # add source file information to the AST root node ast.loc ?= {} @@ -113,15 +124,17 @@ module.exports = (entryPoint, options) -> }] } - {code: src, map: srcMap} = escodegen.generate ast, + {code, map} = escodegen.generate ast, sourceMap: yes format: escodegen.FORMAT_DEFAULTS sourceMapWithCode: yes sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' - srcMap = srcMap.toString() + map = map.toString() - lineCount = src.split('\n').length - 1 - processed[filename] = {name, src, srcMap, lineCount, mtime, deps} + # cache linecount for a little more efficiency when calculating offsets + # later + lineCount = code.split('\n').length - 1 + processed[filename] = {name, code, map, lineCount, mtime, deps} processed From 105e10e052e42353dc1c9f396585e85815c6bbea Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Fri, 4 Oct 2013 14:20:56 -0300 Subject: [PATCH 06/31] Enable usage of multiple entry points --- lib/bundle.js | 111 ------------ lib/canonicalise.js | 8 - lib/cjsify.js | 17 -- lib/command.js | 282 ------------------------------- lib/core-modules.js | 41 ----- lib/is-core.js | 9 - lib/module.js | 4 - lib/relative-resolve.js | 81 --------- lib/traverse-dependencies.js | 165 ------------------ src/bundle.coffee | 15 +- src/command.coffee | 36 ++-- src/traverse-dependencies.coffee | 13 +- 12 files changed, 35 insertions(+), 747 deletions(-) delete mode 100644 lib/bundle.js delete mode 100644 lib/canonicalise.js delete mode 100644 lib/cjsify.js delete mode 100644 lib/command.js delete mode 100644 lib/core-modules.js delete mode 100644 lib/is-core.js delete mode 100644 lib/module.js delete mode 100644 lib/relative-resolve.js delete mode 100644 lib/traverse-dependencies.js diff --git a/lib/bundle.js b/lib/bundle.js deleted file mode 100644 index 3e14067..0000000 --- a/lib/bundle.js +++ /dev/null @@ -1,111 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var btoa, bundle, cache$, canonicalise, escodegen, esprima, path, PRELUDE, PRELUDE_NODE, SourceMapConsumer, SourceMapGenerator, sourceMapToAst, wrap, wrapNode; - esprima = require('esprima'); - path = require('path'); - cache$ = require('source-map'); - SourceMapConsumer = cache$.SourceMapConsumer; - SourceMapGenerator = cache$.SourceMapGenerator; - btoa = require('Base64').btoa; - escodegen = require('escodegen'); - sourceMapToAst = require('./sourcemap-to-ast'); - canonicalise = require('./canonicalise'); - PRELUDE_NODE = "\n(function(){\n var cwd = '/';\n return {\n title: 'browser',\n version: '" + process.version + "',\n browser: true,\n env: {},\n argv: [],\n nextTick: global.setImmediate || function(fn){ setTimeout(fn, 0); },\n cwd: function(){ return cwd; },\n chdir: function(dir){ cwd = dir; }\n };\n})()"; - PRELUDE = "\n(function() {\n function require(file, parentModule){\n if({}.hasOwnProperty.call(require.cache, file))\n return require.cache[file];\n\n var resolved = require.resolve(file);\n if(!resolved) throw new Error('Failed to resolve module ' + file);\n\n var module$ = {\n id: file,\n require: require,\n filename: file,\n exports: {},\n loaded: false,\n parent: parentModule,\n children: []\n };\n if(parentModule) parentModule.children.push(module$);\n var dirname = file.slice(0, file.lastIndexOf('/') + 1);\n\n require.cache[file] = module$.exports;\n resolved.call(module$.exports, module$, module$.exports, dirname, file);\n module$.loaded = true;\n return require.cache[file] = module$.exports;\n }\n\n require.modules = {};\n require.cache = {};\n\n require.resolve = function(file){\n return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0;\n };\n require.define = function(file, fn){ require.modules[file] = fn; };\n\n return require;\n)()"; - wrap = function (modules) { - return '(function(global, require, undefined) {\n' + modules + '\n})(this, ' + PRELUDE + ');'; - }; - wrapNode = function (modules) { - return '(function(global, require, process, undefined) {\n' + modules + '\n})(this, ' + PRELUDE + ', ' + PRELUDE_NODE + ');'; - }; - bundle = function (entryPoint, options) { - var cache$1, code, filename, lineCount, lineOffset, map, name, orig, src, srcMap; - code = ''; - map = new SourceMapGenerator({ - file: path.basename(options.outFile), - sourceRoot: path.relative(path.dirname(options.outFile), options.root) - }); - lineOffset = 1; - for (filename in options.processed) { - if (!isOwn$(options.processed, filename)) - continue; - { - cache$1 = options.processed[filename]; - name = cache$1.name; - src = cache$1.src; - srcMap = cache$1.srcMap; - lineCount = cache$1.lineCount; - } - if (typeof name !== 'number') - name = "'" + name + "'"; - code += '\nrequire.define(' + name + ', function(module, exports, __dirname, __filename){\n' + src + '\n});'; - lineOffset++; - orig = new SourceMapConsumer(srcMap); - orig.eachMapping(function (m) { - return map.addMapping({ - generated: { - line: m.generatedLine + lineOffset, - column: m.generatedColumn - }, - original: { - line: m.originalLine || m.generatedLine, - column: m.originalColumn || m.generatedColumn - }, - source: filename - }); - }); - lineOffset += lineCount + 1; - } - if (typeof entryPoint !== 'number') - entryPoint = "'" + entryPoint + "'"; - code += '\nrequire(' + entryPoint + ');'; - if (options.node) { - code = wrapNode(code); - } else { - code = wrap(code); - } - return { - code: code, - map: map - }; - }; - module.exports = function (entryPoint, options) { - var ast, cache$1, cache$2, code, datauri, esmangle, filename, map, src; - cache$1 = bundle(entryPoint, options); - code = cache$1.code; - map = cache$1.map; - if (options.minify) { - esmangle = require('esmangle'); - ast = esprima.parse(bundled, { loc: true }); - sourceMapToAst(ast, srcMap); - ast = esmangle.mangle(esmangle.optimize(ast), { destructive: true }); - cache$2 = escodegen.generate(ast, { - sourceMap: true, - format: escodegen.FORMAT_MINIFY, - sourceMapWithCode: true, - sourceMapRoot: null != options.sourceMap ? path.relative(path.dirname(options.sourceMap), options.root) || '.' : void 0 - }); - code = cache$2.code; - map = cache$2.map; - cache$2; - } - if ((options.sourceMap || options.inlineSourceMap) && options.inlineSources) - for (filename in processed) { - if (!isOwn$(processed, filename)) - continue; - src = processed[filename].src; - map.setSourceContent(filename, src); - } - if (options.inlineSourceMap) { - datauri = 'data:application/json;charset=utf-8;base64,' + btoa('' + map); - code = '' + code + '\n//# sourceMappingURL=' + datauri; - } - return { - code: code, - map: map - }; - }; - function isOwn$(o, p) { - return {}.hasOwnProperty.call(o, p); - } -}.call(this); diff --git a/lib/canonicalise.js b/lib/canonicalise.js deleted file mode 100644 index 3e44c24..0000000 --- a/lib/canonicalise.js +++ /dev/null @@ -1,8 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var path; - path = require('path'); - module.exports = function (root, file) { - return ('/' + path.relative(root, path.resolve(root, file))).replace(/\\/g, '/'); - }; -}.call(this); diff --git a/lib/cjsify.js b/lib/cjsify.js deleted file mode 100644 index 2b710a9..0000000 --- a/lib/cjsify.js +++ /dev/null @@ -1,17 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var bundle, traverseDependencies; - bundle = require('./bundle'); - traverseDependencies = require('./traverse-dependencies'); - module.exports = function (entryPoint, root, options) { - var processed; - if (null == root) - root = process.cwd(); - if (null == options) - options = {}; - processed = traverseDependencies(entryPoint, root, options); - if (options.verbose) - console.error('\nIncluded modules:\n ' + Object.keys(processed).sort().join('\n ')); - return bundle(processed, entryPoint, root, options); - }; -}.call(this); diff --git a/lib/command.js b/lib/command.js deleted file mode 100644 index f7917fb..0000000 --- a/lib/command.js +++ /dev/null @@ -1,282 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var $0, _, aliasPair, buildBundle, bundle, cache$3, cache$4, dep, deps, escodegenFormat, fs, handlerPair, knownOpts, match, nopt, opt, optAliases, options, originalEntryPoint, path, positionalArgs, startBuild, stdinput, traverseDependencies; - fs = require('fs'); - path = require('path'); - nopt = require('nopt'); - bundle = require('./bundle'); - traverseDependencies = require('./traverse-dependencies'); - escodegenFormat = { - indent: { - style: ' ', - base: 0 - }, - renumber: true, - hexadecimal: true, - quotes: 'auto', - parentheses: false - }; - knownOpts = {}; - for (var cache$ = [ - 'deps', - 'help', - 'ignore-missing', - 'inline-source-map', - 'inline-sources', - 'minify', - 'node', - 'verbose', - 'watch' - ], i$ = 0, length$ = cache$.length; i$ < length$; ++i$) { - opt = cache$[i$]; - knownOpts[opt] = Boolean; - } - for (var cache$1 = [ - 'export', - 'output', - 'root', - 'source-map' - ], i$1 = 0, length$1 = cache$1.length; i$1 < length$1; ++i$1) { - opt = cache$1[i$1]; - knownOpts[opt] = String; - } - for (var cache$2 = [ - 'alias', - 'handler' - ], i$2 = 0, length$2 = cache$2.length; i$2 < length$2; ++i$2) { - opt = cache$2[i$2]; - knownOpts[opt] = [ - String, - Array - ]; - } - optAliases = { - a: '--alias', - h: '--handler', - m: '--minify', - o: '--output', - r: '--root', - s: '--source-map', - v: '--verbose', - w: '--watch', - x: '--export' - }; - options = nopt(knownOpts, optAliases, process.argv, 2); - positionalArgs = options.argv.remain; - delete options.argv; - if (null != options.node) - options.node; - else - options.node = true; - if (null != options['inline-sources']) - options['inline-sources']; - else - options['inline-sources'] = true; - if (null != options['cache-path']) - options['cache-path']; - else - options['cache-path'] = '.commonjs-everywhere-cache.json'; - if (null != options.root) - options.root; - else - options.root = process.cwd(); - if (null != options.alias) - options.alias; - else - options.alias = []; - if (null != options.handler) - options.handler; - else - options.handler = []; - options.ignoreMissing = options['ignore-missing']; - options.sourceMap = options['source-map']; - options.inlineSources = options['inline-sources']; - options.inlineSourceMap = options['inline-source-map']; - options.cachePath = options['cache-path']; - options.moduleUids = options['module-uids']; - if (options.help) { - $0 = process.argv[0] === 'node' ? process.argv[1] : process.argv[0]; - $0 = path.basename($0); - console.log('\n Usage: ' + $0 + " OPT* path/to/entry-file.ext OPT*\n\n -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO\n -h, --handler EXT:MODULE handle files with extension EXT with module MODULE\n -m, --minify minify output\n -o, --output FILE output to FILE instead of stdout\n -r, --root DIR unqualified requires are relative to DIR; default: cwd\n -s, --source-map FILE output a source map to FILE\n -v, --verbose verbose output sent to stderr\n -w, --watch watch input files/dependencies for changes and rebuild bundle\n -x, --export NAME export the given entry module as NAME\n --deps do not bundle; just list the files that would be bundled\n --help display this help message and exit\n --ignore-missing continue without error when dependency resolution fails\n --inline-source-map include the source map as a data URI in the generated bundle\n --inline-sources include source content in generated source maps; default: on\n --node include process object; emulate node environment; default: on\n --cache-path file where to read/write a json-encoded cache that will be\n used to speed up future rebuilds. default:\n '.commonjs-everywhere-cache.json' in the current directory\n --module-uids Instead of replacing module names by their full path,\n use unique ids for better minification\n (breaks __dirname/__filename)\n --version display the version number and exit\n"); - process.exit(0); - } - if (options.version) { - console.log(require('../package.json').version); - process.exit(0); - } - if (!(positionalArgs.length === 1)) { - console.error('wrong number of entry points given; expected 1'); - process.exit(1); - } - options.aliases = {}; - for (var i$3 = 0, length$3 = options.alias.length; i$3 < length$3; ++i$3) { - aliasPair = options.alias[i$3]; - match = aliasPair.match((cache$3 = /([^:]+):(.*)/, null != cache$3 ? cache$3 : [])); - if (null != match) { - options.aliases[match[1]] = match[2]; - } else { - console.error('invalid alias: ' + aliasPair); - process.exit(1); - } - } - delete options.alias; - options.handlers = {}; - for (var i$4 = 0, length$4 = options.handler.length; i$4 < length$4; ++i$4) { - handlerPair = options.handler[i$4]; - match = handlerPair.match((cache$4 = /([^:]+):(.*)/, null != cache$4 ? cache$4 : [])); - if (null != match) { - (function (ext, mod) { - return options.handlers[ext] = require(mod); - }('.' + match[1], match[2])); - } else { - console.error('invalid handler: ' + handlerPair); - process.exit(1); - } - } - delete options.handler; - originalEntryPoint = positionalArgs[0]; - if (options.deps) { - deps = traverseDependencies(originalEntryPoint, options.root, options); - for (_ in deps) { - if (!isOwn$(deps, _)) - continue; - dep = deps[_]; - console.log(dep.canonicalName); - } - process.exit(0); - } - if (options.watch && !options.output) { - console.error('--watch requires --output'); - process.exit(1); - } - buildBundle = function () { - var cache$5, code, datauri, map, sourceMappingUrl; - traverseDependencies(originalEntryPoint, options); - cache$5 = bundle(originalEntryPoint, options); - code = cache$5.code; - map = cache$5.map; - if (options.sourceMap) { - fs.writeFileSync(options.sourceMap, '' + map); - sourceMappingUrl = options.output ? path.relative(path.dirname(options.output), options.sourceMap) : options.sourceMap; - if (!options.inlineSourceMap) - code = '' + code + '\n//# sourceMappingURL=' + sourceMappingUrl; - } - if (options.inlineSourceMap) { - datauri = 'data:application/json;charset=utf-8;base64,' + btoa('' + map); - code = '' + code + '\n//# sourceMappingURL=' + datauri; - } - if (options.output) { - return fs.writeFileSync(options.output, code); - } else { - return process.stdout.write('' + code + '\n'); - } - }; - startBuild = function () { - var building, cache, cache$5, moduleUids, processed, uids, watching; - process.on('exit', function () { - var cache; - cache = { - processed: options.processed, - uids: options.uids, - moduleUids: options.moduleUids - }; - return fs.writeFileSync(options.cachePath, JSON.stringify(cache)); - }); - process.on('uncaughtException', function (e) { - try { - fs.unlinkSync(options.cachePath); - } catch (e$) { - } - options.processed = {}; - throw e; - }); - if (fs.existsSync(options.cachePath)) { - cache = JSON.parse(fs.readFileSync(options.cachePath, 'utf8')); - cache$5 = cache; - processed = cache$5.processed; - uids = cache$5.uids; - moduleUids = cache$5.moduleUids; - cache$5; - } - if (!processed || moduleUids !== options.moduleUids) { - processed = {}; - uids = { - next: 1, - names: {} - }; - } - options.processed = processed; - options.uids = uids; - options.uidFor = function (name) { - var uid; - if (!options.moduleUids) - return name; - if (!{}.hasOwnProperty.call(uids.names, name)) { - uid = uids.next++; - uids.names[name] = uid; - } - return uids.names[name]; - }; - if (options.watch) - console.error('BUNDLING starting at ' + originalEntryPoint); - buildBundle(originalEntryPoint); - if (options.watch) { - process.on('SIGINT', process.exit); - process.on('SIGTERM', process.exit); - watching = {}; - building = false; - return function (accum$) { - var file; - for (file in processed) { - if (!isOwn$(processed, file)) - continue; - if (!!(file in watching)) - continue; - accum$.push(function (file) { - watching[file] = true; - return fs.watchFile(file, { - persistent: true, - interval: 500 - }, function (curr, prev) { - var ino; - if (building) - return; - building = true; - ino = process.platform === 'win32' ? null != curr.ino : curr.ino; - if (!ino) { - console.error('WARNING: watched file ' + file + ' has disappeared'); - return; - } - console.error('' + file + ' changed, rebuilding'); - buildBundle(originalEntryPoint); - console.error('done'); - building = false; - }); - }(file)); - } - return accum$; - }.call(this, []); - } - }; - if (originalEntryPoint === '-') { - stdinput = ''; - process.stdin.on('data', function (data) { - return stdinput += data; - }); - process.stdin.on('end', function () { - originalEntryPoint = require('mktemp').createFileSync('temp-XXXXXXXXX.js'); - fs.writeFileSync(originalEntryPoint, stdinput); - process.on('exit', function () { - return fs.unlinkSync(originalEntryPoint); - }); - return startBuild(); - }); - process.stdin.setEncoding('utf8'); - process.stdin.resume(); - } else { - startBuild(); - } - function isOwn$(o, p) { - return {}.hasOwnProperty.call(o, p); - } -}.call(this); diff --git a/lib/core-modules.js b/lib/core-modules.js deleted file mode 100644 index f748bcf..0000000 --- a/lib/core-modules.js +++ /dev/null @@ -1,41 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var CJS_DIR, CORE_MODULES, mod, NODE_CORE_MODULES, path, resolve; - path = require('path'); - resolve = require('resolve').sync; - CJS_DIR = path.join(__dirname, '..'); - CORE_MODULES = { - buffer: resolve('buffer-browserify'), - constants: resolve('constants-browserify'), - crypto: resolve('crypto-browserify'), - events: resolve('events-browserify'), - http: resolve('http-browserify'), - punycode: resolve('./node_modules/punycode', { basedir: CJS_DIR }), - querystring: resolve('./node_modules/querystring', { basedir: CJS_DIR }), - vm: resolve('vm-browserify'), - zlib: resolve('zlib-browserify') - }; - NODE_CORE_MODULES = [ - '_stream_duplex', - '_stream_passthrough', - '_stream_readable', - '_stream_transform', - '_stream_writable', - 'assert', - 'console', - 'domain', - 'freelist', - 'path', - 'readline', - 'stream', - 'string_decoder', - 'sys', - 'url', - 'util' - ]; - for (var i$ = 0, length$ = NODE_CORE_MODULES.length; i$ < length$; ++i$) { - mod = NODE_CORE_MODULES[i$]; - CORE_MODULES[mod] = path.join(CJS_DIR, 'node', 'lib', '' + mod + '.js'); - } - module.exports = CORE_MODULES; -}.call(this); diff --git a/lib/is-core.js b/lib/is-core.js deleted file mode 100644 index 4c179ba..0000000 --- a/lib/is-core.js +++ /dev/null @@ -1,9 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var CORE_MODULES, resolve; - resolve = require('resolve'); - CORE_MODULES = require('./core-modules'); - module.exports = function (x) { - return resolve.isCore(x) || [].hasOwnProperty.call(CORE_MODULES, x); - }; -}.call(this); diff --git a/lib/module.js b/lib/module.js deleted file mode 100644 index 607a813..0000000 --- a/lib/module.js +++ /dev/null @@ -1,4 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -exports.bundle = require('./bundle'); -exports.cjsify = require('./cjsify'); -exports.traverseDependencies = require('./traverse-dependencies'); diff --git a/lib/relative-resolve.js b/lib/relative-resolve.js deleted file mode 100644 index da92cbb..0000000 --- a/lib/relative-resolve.js +++ /dev/null @@ -1,81 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var canonicalise, CORE_MODULES, fs, isCore, path, resolve, resolvePath; - fs = require('fs'); - path = require('path'); - resolve = require('resolve').sync; - CORE_MODULES = require('./core-modules'); - isCore = require('./is-core'); - canonicalise = require('./canonicalise'); - resolvePath = function (param$) { - var aliases, cache$, corePath, cwd, e, extensions, givenPath, root; - { - cache$ = param$; - extensions = cache$.extensions; - aliases = cache$.aliases; - root = cache$.root; - cwd = cache$.cwd; - givenPath = cache$.path; - } - if (null != aliases) - aliases; - else - aliases = {}; - if (isCore(givenPath)) { - if ({}.hasOwnProperty.call(aliases, givenPath)) - return; - corePath = CORE_MODULES[givenPath]; - if (!fs.existsSync(corePath)) - throw new Error('Core module "' + givenPath + '" has not yet been ported to the browser'); - givenPath = corePath; - } - try { - return resolve(givenPath, { - extensions: extensions, - basedir: cwd || root - }); - } catch (e$) { - e = e$; - try { - return resolve(path.join(root, givenPath), { extensions: extensions }); - } catch (e$1) { - e = e$1; - throw new Error('Cannot find module "' + givenPath + '" in "' + root + '"'); - } - } - }; - module.exports = function (param$) { - var aliases, cache$, canonicalName, cwd, extensions, givenPath, resolved, root; - { - cache$ = param$; - extensions = cache$.extensions; - aliases = cache$.aliases; - root = cache$.root; - cwd = cache$.cwd; - givenPath = cache$.path; - } - if (null != aliases) - aliases; - else - aliases = {}; - resolved = resolvePath({ - extensions: extensions, - aliases: aliases, - root: root, - cwd: cwd, - path: givenPath - }); - canonicalName = isCore(givenPath) ? givenPath : canonicalise(root, resolved); - if ({}.hasOwnProperty.call(aliases, canonicalName)) - resolved = aliases[canonicalName] && resolvePath({ - extensions: extensions, - aliases: aliases, - root: root, - path: aliases[canonicalName] - }); - return { - filename: resolved, - canonicalName: canonicalName - }; - }; -}.call(this); diff --git a/lib/traverse-dependencies.js b/lib/traverse-dependencies.js deleted file mode 100644 index 7c65fcc..0000000 --- a/lib/traverse-dependencies.js +++ /dev/null @@ -1,165 +0,0 @@ -// Generated by CoffeeScript 2.0.0-beta7 -void function () { - var badRequireError, canonicalise, escodegen, esprima, estraverse, fs, path, relativeResolve, util; - fs = require('fs'); - path = require('path'); - util = require('util'); - esprima = require('esprima'); - estraverse = require('estraverse'); - escodegen = require('escodegen'); - canonicalise = require('./canonicalise'); - relativeResolve = require('./relative-resolve'); - badRequireError = function (filename, node, msg) { - if (null != node.loc && null != (null != node.loc ? node.loc.start : void 0)) - filename = '' + filename + ':' + node.loc.start.line + ':' + node.loc.start.column; - throw 'illegal require: ' + msg + '\n `' + require('escodegen').generate(node) + '`\n in ' + filename + ''; - }; - module.exports = function (entryPoint, options) { - var aliases, ast, astOrJs, cache$, cache$1, cache$2, canonicalName, checked, deps, ext, extensions, extname, filename, handler, handlers, lineCount, mtime, name, processed, root, src, srcMap, uidFor, worklist; - aliases = null != options.aliases ? options.aliases : {}; - uidFor = options.uidFor; - root = options.root; - handlers = { - '.coffee': function (coffee, canonicalName) { - return CoffeeScript.compile(CoffeeScript.parse(coffee, { raw: true }), { bare: true }); - }, - '.json': function (json, canonicalName) { - return esprima.parse('module.exports = ' + json, { - loc: true, - source: canonicalName - }); - } - }; - for (ext in cache$ = null != options.handlers ? options.handlers : {}) { - if (!isOwn$(cache$, ext)) - continue; - handler = cache$[ext]; - handlers[ext] = handler; - } - extensions = ['.js'].concat([].slice.call(function (accum$) { - for (ext in handlers) { - if (!isOwn$(handlers, ext)) - continue; - accum$.push(ext); - } - return accum$; - }.call(this, []))); - worklist = [relativeResolve({ - extensions: extensions, - aliases: aliases, - root: root, - path: entryPoint - })]; - processed = options.processed || {}; - checked = {}; - while (worklist.length) { - cache$1 = worklist.pop(); - filename = cache$1.filename; - canonicalName = cache$1.canonicalName; - if (!filename) - continue; - if ({}.hasOwnProperty.call(checked, filename)) - continue; - checked[filename] = true; - extname = path.extname(filename); - mtime = fs.statSync(filename).mtime.getTime(); - if ((null != processed[filename] ? processed[filename].mtime : void 0) === mtime) { - worklist = worklist.concat(processed[filename].deps); - continue; - } - src = fs.readFileSync(filename).toString(); - astOrJs = {}.hasOwnProperty.call(handlers, extname) ? handlers[extname](src, canonicalName) : src; - ast = typeof astOrJs === 'string' ? function () { - var e; - try { - return esprima.parse(astOrJs, { - loc: true, - source: canonicalName - }); - } catch (e$) { - e = e$; - throw new Error('Syntax error in ' + filename + ' at line ' + e.lineNumber + ', column ' + e.column + e.message.slice(e.message.indexOf(':'))); - } - }.call(this) : astOrJs; - if (null != ast.loc) - ast.loc; - else - ast.loc = {}; - deps = []; - name = uidFor(canonicalName); - estraverse.replace(ast, { - enter: function (node, parents) { - var cwd, e, resolved; - if (null != node.loc) - node.loc.source = canonicalName; - if (!(node.type === 'CallExpression' && node.callee.type === 'Identifier' && node.callee.name === 'require')) - return; - if (!(node['arguments'].length === 1)) - badRequireError(filename, node, 'require must be given exactly one argument'); - if (!(node['arguments'][0].type === 'Literal' && typeof node['arguments'][0].value === 'string')) - badRequireError(filename, node, 'argument of require must be a constant string'); - cwd = path.dirname(fs.realpathSync(filename)); - if (options.verbose) - console.error('required "' + node['arguments'][0].value + '" from "' + canonicalName + '"'); - try { - resolved = relativeResolve({ - extensions: extensions, - aliases: aliases, - root: options.root, - cwd: cwd, - path: node['arguments'][0].value - }); - worklist.push(resolved); - deps.push(resolved); - } catch (e$) { - e = e$; - if (options.ignoreMissing) { - return { - type: 'Literal', - value: null - }; - } else { - throw e; - } - } - return { - type: 'CallExpression', - callee: node.callee, - 'arguments': [ - { - type: 'Literal', - value: uidFor(resolved.canonicalName) - }, - { - type: 'Identifier', - name: 'module' - } - ] - }; - } - }); - cache$2 = escodegen.generate(ast, { - sourceMap: true, - format: escodegen.FORMAT_DEFAULTS, - sourceMapWithCode: true, - sourceMapRoot: null != options.sourceMap ? path.relative(path.dirname(options.sourceMap), options.root) || '.' : void 0 - }); - src = cache$2.code; - srcMap = cache$2.map; - srcMap = srcMap.toString(); - lineCount = src.split('\n').length - 1; - processed[filename] = { - name: name, - src: src, - srcMap: srcMap, - lineCount: lineCount, - mtime: mtime, - deps: deps - }; - } - return processed; - }; - function isOwn$(o, p) { - return {}.hasOwnProperty.call(o, p); - } -}.call(this); diff --git a/src/bundle.coffee b/src/bundle.coffee index 4568941..5344686 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -74,7 +74,7 @@ wrapNode = (modules) -> """ })(this, #{PRELUDE}, #{PRELUDE_NODE}); """ -bundle = (entryPoint, options) -> +bundle = (options) -> result = '' resultMap = new SourceMapGenerator file: path.basename(options.outFile) @@ -101,10 +101,11 @@ bundle = (entryPoint, options) -> column: m.originalColumn or m.generatedColumn source: filename - if typeof entryPoint != 'number' - entryPoint = "'#{entryPoint}'" - - result += "\nrequire(#{entryPoint});" + for entryPoint in options.entryPoints + name = options.processed[entryPoint].name + if typeof name != 'number' + name = "'#{name}'" + result += "\nrequire(#{name});" if options.node result = wrapNode(result) @@ -114,8 +115,8 @@ bundle = (entryPoint, options) -> return {code: result, map: resultMap} -module.exports = (entryPoint, options) -> - {code, map} = bundle entryPoint, options +module.exports = (options) -> + {code, map} = bundle options if options.minify esmangle = require 'esmangle' diff --git a/src/command.coffee b/src/command.coffee index d90be4e..10f0b1b 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -18,7 +18,7 @@ knownOpts = {} # options knownOpts[opt] = Boolean for opt in [ 'deps', 'help', 'ignore-missing', 'inline-source-map', 'inline-sources', - 'minify', 'node', 'verbose', 'watch' + 'minify', 'node', 'verbose', 'watch', 'module-uids', 'cache-path' ] # parameters knownOpts[opt] = String for opt in ['export', 'output', 'root', 'source-map'] @@ -35,9 +35,10 @@ optAliases = v: '--verbose' w: '--watch' x: '--export' + e: '--main' options = nopt knownOpts, optAliases, process.argv, 2 -positionalArgs = options.argv.remain +options.entryPoints = entryPoints = options.argv.remain delete options.argv # default values @@ -54,6 +55,7 @@ options.inlineSources = options['inline-sources'] options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] options.moduleUids = options['module-uids'] +options.entryPoint = options['entry-point'] if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] @@ -61,6 +63,8 @@ if options.help console.log " Usage: #{$0} OPT* path/to/entry-file.ext OPT* + -e, --main main module to export/initialize when multiple + files are specified -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO -h, --handler EXT:MODULE handle files with extension EXT with module MODULE -m, --minify minify output @@ -90,10 +94,6 @@ if options.version console.log (require '../package.json').version process.exit 0 -unless positionalArgs.length is 1 - console.error 'wrong number of entry points given; expected 1' - process.exit 1 - options.aliases = {} for aliasPair in options.alias match = aliasPair.match /([^:]+):(.*)/ ? [] @@ -113,10 +113,9 @@ for handlerPair in options.handler process.exit 1 delete options.handler -originalEntryPoint = positionalArgs[0] if options.deps - deps = traverseDependencies originalEntryPoint, options.root, options + deps = traverseDependencies options console.log dep.canonicalName for own _, dep of deps process.exit 0 @@ -125,8 +124,8 @@ if options.watch and not options.output process.exit 1 buildBundle = -> - traverseDependencies originalEntryPoint, options - {code, map} = bundle originalEntryPoint, options + traverseDependencies options + {code, map} = bundle options if options.sourceMap fs.writeFileSync options.sourceMap, "#{map}" @@ -174,7 +173,7 @@ startBuild = -> options.processed = processed options.uids = uids - options.uidFor = (name) -> + uidFor = options.uidFor = (name) -> if not options.moduleUids return name if not {}.hasOwnProperty.call(uids.names, name) @@ -182,10 +181,7 @@ startBuild = -> uids.names[name] = uid uids.names[name] - if options.watch - console.error "BUNDLING starting at #{originalEntryPoint}" - - buildBundle originalEntryPoint + buildBundle() if options.watch # Flush the cache when the user presses CTRL+C or the process is @@ -204,19 +200,19 @@ startBuild = -> console.error "WARNING: watched file #{file} has disappeared" return console.error "#{file} changed, rebuilding" - buildBundle originalEntryPoint + buildBundle() console.error "done" building = false return -if originalEntryPoint is '-' +if entryPoints.length == 1 and entryPoints[0] is '-' # support reading input from stdin stdinput = '' process.stdin.on 'data', (data) -> stdinput += data process.stdin.on 'end', -> - originalEntryPoint = (require 'mktemp').createFileSync 'temp-XXXXXXXXX.js' - fs.writeFileSync originalEntryPoint, stdinput - process.on 'exit', -> fs.unlinkSync originalEntryPoint + entryPoints[0] = (require 'mktemp').createFileSync 'temp-XXXXXXXXX.js' + fs.writeFileSync entryPoints[0], stdinput + process.on 'exit', -> fs.unlinkSync entryPoints[0] do startBuild process.stdin.setEncoding 'utf8' do process.stdin.resume diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 0585cd1..c64a505 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -20,7 +20,7 @@ badRequireError = (filename, node, msg) -> in #{filename} """ -module.exports = (entryPoint, options) -> +module.exports = (options) -> aliases = options.aliases ? {} uidFor = options.uidFor root = options.root @@ -35,7 +35,16 @@ module.exports = (entryPoint, options) -> handlers[ext] = handler extensions = ['.js', (ext for own ext of handlers)...] - worklist = [relativeResolve {extensions, aliases, root, path: entryPoint}] + worklist = [] + resolvedEntryPoints = [] + + for ep in options.entryPoints + resolved = relativeResolve {extensions, aliases, root, path: ep} + worklist.push(resolved) + resolvedEntryPoints.push(resolved.filename) + + options.entryPoints = resolvedEntryPoints + processed = options.processed or {} checked = {} From 6a88429676a8d591ad3e4dddb7947cde36241376 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Fri, 4 Oct 2013 23:52:40 -0300 Subject: [PATCH 07/31] More refactoring, switch from esmangle to uglify since its like 10x faster. Also fixed concatenated source map generation --- package.json | 7 ++-- src/bundle.coffee | 71 ++++++++++++++++++-------------- src/canonicalise.coffee | 2 +- src/command.coffee | 7 +++- src/sourcemap-to-ast.coffee | 16 +++++-- src/traverse-dependencies.coffee | 17 +++++--- 6 files changed, 72 insertions(+), 48 deletions(-) diff --git a/package.json b/package.json index da5639a..b9e9aa5 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,6 @@ "dependencies": { "Base64": "~0.1.2", "escodegen": "~0.0.24", - "esmangle": "~0.0.10", - "esprima": "~1.0.2", "estraverse": "1.3.x", "mktemp": "~0.3.0", "nopt": "~2.1.2", @@ -58,7 +56,10 @@ "source-map": "~0.1.30", "handlebars": "~1.0.12", "coffee-script": "~1.6.3", - "source-map-support": "~0.2.3" + "source-map-support": "~0.2.3", + "lodash": "~2.2.1", + "acorn": "~0.3.1", + "uglify-js": "~2.4.0" }, "devDependencies": { "mocha": "~1.8.1", diff --git a/src/bundle.coffee b/src/bundle.coffee index 5344686..bbe3180 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -1,20 +1,22 @@ -esprima = require 'esprima' +acorn = require 'acorn' path = require 'path' {SourceMapConsumer, SourceMapGenerator} = require 'source-map' {btoa} = require 'Base64' escodegen = require 'escodegen' +UglifyJS = require 'uglify-js' sourceMapToAst = require './sourcemap-to-ast' canonicalise = require './canonicalise' PRELUDE_NODE = """ -(function(){ +(function() { var cwd = '/'; return { title: 'browser', version: '#{process.version}', browser: true, env: {}, + on: function() {}, argv: [], nextTick: global.setImmediate || function(fn){ setTimeout(fn, 0); }, cwd: function(){ return cwd; }, @@ -25,7 +27,7 @@ PRELUDE_NODE = """ PRELUDE = """ (function() { - function require(file, parentModule){ + function require(file, parentModule) { if({}.hasOwnProperty.call(require.cache, file)) return require.cache[file]; @@ -45,7 +47,7 @@ PRELUDE = """ var dirname = file.slice(0, file.lastIndexOf('/') + 1); require.cache[file] = module$.exports; - resolved.call(module$.exports, module$, module$.exports, dirname, file); + resolved.call(this, module$, module$.exports, dirname, file); module$.loaded = true; return require.cache[file] = module$.exports; } @@ -59,37 +61,37 @@ PRELUDE = """ require.define = function(file, fn){ require.modules[file] = fn; }; return require; -)() +})() """ wrap = (modules) -> """ - (function(global, require, undefined) { + (function(require, undefined) { var global = this; #{modules} - })(this, #{PRELUDE}); + })(#{PRELUDE}); """ wrapNode = (modules) -> """ - (function(global, require, process, undefined) { + (function(require, process, undefined) { var global = this; #{modules} - })(this, #{PRELUDE}, #{PRELUDE_NODE}); + })(#{PRELUDE}, #{PRELUDE_NODE}); """ bundle = (options) -> result = '' resultMap = new SourceMapGenerator file: path.basename(options.outFile) - sourceRoot: path.relative(path.dirname(options.outFile), options.root) + sourceRoot: options.sourceMapRoot lineOffset = 1 # global wrapper - for own filename, {name, code, map, lineCount} of options.processed - if typeof name != 'number' - name = "'#{name}'" + for own filename, {id, canonicalName, code, map, lineCount} of options.processed + if typeof id != 'number' + id = "'#{id}'" result += """ - \nrequire.define(#{name}, function(module, exports, __dirname, __filename){ + \nrequire.define(#{id}, function(module, exports, __dirname, __filename){ #{code} }); """ - lineOffset += 2# skip linefeed plus the 'require.define' line + lineOffset += 2 # skip linefeed plus the 'require.define' line orig = new SourceMapConsumer map orig.eachMapping (m) -> resultMap.addMapping @@ -99,39 +101,44 @@ bundle = (options) -> original: line: m.originalLine or m.generatedLine column: m.originalColumn or m.generatedColumn - source: filename + source: canonicalName + lineOffset += lineCount for entryPoint in options.entryPoints - name = options.processed[entryPoint].name - if typeof name != 'number' - name = "'#{name}'" - result += "\nrequire(#{name});" + {id} = options.processed[entryPoint] + if typeof id != 'number' + id = "'#{id}'" + result += "\nrequire(#{id});" if options.node result = wrapNode(result) else result = wrap(result) - return {code: result, map: resultMap} + return {code: result, map: resultMap.toString()} module.exports = (options) -> {code, map} = bundle options if options.minify - esmangle = require 'esmangle' - ast = esprima.parse bundled, loc: yes - sourceMapToAst ast, map - ast = esmangle.mangle (esmangle.optimize ast), destructive: yes - {code, map} = escodegen.generate ast, - sourceMap: yes - format: escodegen.FORMAT_MINIFY - sourceMapWithCode: yes - sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' + uglifyAst = UglifyJS.parse code + uglifyAst.figure_out_scope() + uglifyAst = uglifyAst.transform UglifyJS.Compressor() + uglifyAst.figure_out_scope() + uglifyAst.compute_char_frequency() + uglifyAst.mangle_names() + sm = UglifyJS.SourceMap { + file: options.output + root: options.sourceMapRoot + orig: map + } + code = uglifyAst.print_to_string source_map: sm + map = sm.toString() if (options.sourceMap or options.inlineSourceMap) and options.inlineSources - for own filename, {code} of processed - map.setSourceContent filename, code + for own filename, {code: src, canonicalName} of options.processed + map.setSourceContent canonicalName, src if options.inlineSourceMap datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" diff --git a/src/canonicalise.coffee b/src/canonicalise.coffee index 32d9fc0..5b810f8 100644 --- a/src/canonicalise.coffee +++ b/src/canonicalise.coffee @@ -1,4 +1,4 @@ path = require 'path' module.exports = (root, file) -> - "/#{path.relative root, path.resolve root, file}".replace /\\/g, '/' + "#{path.relative root, path.resolve root, file}".replace /\\/g, '/' diff --git a/src/command.coffee b/src/command.coffee index 10f0b1b..37569d2 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -1,6 +1,7 @@ fs = require 'fs' path = require 'path' nopt = require 'nopt' +_ = require 'lodash' bundle = require './bundle' traverseDependencies = require './traverse-dependencies' @@ -38,12 +39,12 @@ optAliases = e: '--main' options = nopt knownOpts, optAliases, process.argv, 2 -options.entryPoints = entryPoints = options.argv.remain +options.entryPoints = entryPoints = _.uniq options.argv.remain delete options.argv # default values options.node ?= on -options['inline-sources'] ?= on +options['inline-sources'] ?= false options['cache-path'] ?= '.powerbuild-cache~' options['root'] ?= process.cwd() options.alias ?= [] @@ -56,6 +57,8 @@ options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] options.moduleUids = options['module-uids'] options.entryPoint = options['entry-point'] +options.sourceMapRoot = + path.relative(path.dirname(options.output), options.root) if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee index 517a2f0..19862c2 100644 --- a/src/sourcemap-to-ast.coffee +++ b/src/sourcemap-to-ast.coffee @@ -7,14 +7,22 @@ module.exports = (ast, srcMap) -> traverse ast, enter: (node) -> + if not node.type + return + if node.type == 'TryStatement' and not node.guardedHandlers + node.guardedHandlers = [] origStart = map.originalPositionFor node.loc.start origEnd = map.originalPositionFor node.loc.end - assert origStart.source == origEnd.source, 'Invalid source map' + if origStart.source != origEnd.source + delete node.loc + # This is a top-level node like program or a top-level wrapper + # function, dont care about the source file in these cases + return node.loc = start: line: origStart.line - column: origStart.column + column: origStart.column + 1 end: line: origEnd.line - column: origEnd.column - source: origStart.source || node.loc.source + column: origEnd.column + 1 + source: origStart.source diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index c64a505..61748e8 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -3,7 +3,7 @@ path = require 'path' util = require 'util' coffee = require 'coffee-script' -esprima = require 'esprima' +acorn = require 'acorn' estraverse = require 'estraverse' escodegen = require 'escodegen' @@ -30,7 +30,7 @@ module.exports = (options) -> {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true return {code: js, map: v3SourceMap} '.json': (json, canonicalName) -> - esprima.parse "module.exports = #{json}", loc: yes, source: canonicalName + acorn.parse "module.exports = #{json}", locations: yes for own ext, handler of options.handlers ? {} handlers[ext] = handler extensions = ['.js', (ext for own ext of handlers)...] @@ -80,7 +80,8 @@ module.exports = (options) -> if astOrJs.code try - ast = esprima.parse astOrJs.code, loc: yes, source: canonicalName + ast = acorn.parse astOrJs.code, locations: yes + ast.loc ?= {} if astOrJs.map sourceMapToAst ast, astOrJs.map catch e @@ -94,11 +95,15 @@ module.exports = (options) -> # add source file information to the AST root node ast.loc ?= {} deps = [] - name = uidFor(canonicalName) + id = uidFor(canonicalName) estraverse.replace ast, enter: (node, parents) -> if node.loc? then node.loc.source = canonicalName + if node.type == 'TryStatement' and not node.guardedHandlers + # escodegen will break when generating from acorn's ast unless + # we add this + node.guardedHandlers = [] # ignore anything that's not a `require` call return unless node.type is 'CallExpression' and node.callee.type is 'Identifier' and node.callee.name is 'require' # illegal requires @@ -143,7 +148,7 @@ module.exports = (options) -> # cache linecount for a little more efficiency when calculating offsets # later - lineCount = code.split('\n').length - 1 - processed[filename] = {name, code, map, lineCount, mtime, deps} + lineCount = code.split('\n').length + processed[filename] = {id, canonicalName, code, map, lineCount, mtime, deps} processed From 46fa7e2d71f182d25f1e37eef6fb6a793a02a848 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 00:33:24 -0300 Subject: [PATCH 08/31] Update escodegen version --- package.json | 6 +++--- src/bundle.coffee | 4 ++-- src/sourcemap-to-ast.coffee | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index b9e9aa5..33bfc8c 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,6 @@ }, "dependencies": { "Base64": "~0.1.2", - "escodegen": "~0.0.24", - "estraverse": "1.3.x", "mktemp": "~0.3.0", "nopt": "~2.1.2", "resolve": "0.5.x", @@ -59,7 +57,9 @@ "source-map-support": "~0.2.3", "lodash": "~2.2.1", "acorn": "~0.3.1", - "uglify-js": "~2.4.0" + "uglify-js": "~2.4.0", + "estraverse": "~1.3.1", + "escodegen": "0.0.27" }, "devDependencies": { "mocha": "~1.8.1", diff --git a/src/bundle.coffee b/src/bundle.coffee index bbe3180..e410ba3 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -123,8 +123,8 @@ module.exports = (options) -> if options.minify uglifyAst = UglifyJS.parse code - uglifyAst.figure_out_scope() - uglifyAst = uglifyAst.transform UglifyJS.Compressor() + # uglifyAst.figure_out_scope() + # uglifyAst = uglifyAst.transform UglifyJS.Compressor warnings: false uglifyAst.figure_out_scope() uglifyAst.compute_char_frequency() uglifyAst.mangle_names() diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee index 19862c2..8528c52 100644 --- a/src/sourcemap-to-ast.coffee +++ b/src/sourcemap-to-ast.coffee @@ -18,6 +18,7 @@ module.exports = (ast, srcMap) -> # This is a top-level node like program or a top-level wrapper # function, dont care about the source file in these cases return + # console.error(origStart.name, origEnd.name) node.loc = start: line: origStart.line From 182882bd7321be694215c05c5eb5a6e5469a57a0 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 12:37:33 -0300 Subject: [PATCH 09/31] Fixed all tests --- changelog.sh | 16 ------ src/bundle.coffee | 17 +++++-- src/cjsify.coffee | 8 --- src/index.coffee | 42 ++++++++++++++++ src/module.coffee | 3 -- src/sourcemap-to-ast.coffee | 2 +- src/traverse-dependencies.coffee | 4 +- test-setup.coffee | 11 ++-- test/dependency-resolution.coffee | 83 +++++++++++++++++-------------- test/module-resolution.coffee | 22 ++++---- test/module-spec.coffee | 2 +- 11 files changed, 123 insertions(+), 87 deletions(-) delete mode 100755 changelog.sh delete mode 100644 src/cjsify.coffee create mode 100644 src/index.coffee delete mode 100644 src/module.coffee diff --git a/changelog.sh b/changelog.sh deleted file mode 100755 index 89fc71d..0000000 --- a/changelog.sh +++ /dev/null @@ -1,16 +0,0 @@ -if [ $# -gt 0 ]; then next_version=$1; else next_version=master; fi -initial_commit=$(git log --pretty=%H | tail -1) - -printf "$next_version" -date '+ (%Y-%m-%d)' - -tags=($(echo $initial_commit; git tag; echo master)) -for ((i = ${#tags[@]}-1; i > 0; i--)); do - if [ "${tags[i]}" '!=' master ]; then - printf "${tags[i]}" - git --no-pager log -1 --date=short --pretty=' (%ad)' "${tags[i]}" - fi - git --no-pager log --date=short --pretty='- %h: %s' "${tags[i-1]}..${tags[i]}" - if [ $i -gt 1 ]; then echo; fi -done -git --no-pager log -1 --date=short --pretty='- %h: %s' $initial_commit diff --git a/src/bundle.coffee b/src/bundle.coffee index e410ba3..00b5074 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -9,7 +9,7 @@ sourceMapToAst = require './sourcemap-to-ast' canonicalise = require './canonicalise' PRELUDE_NODE = """ -(function() { +(function() { var global = this; var cwd = '/'; return { title: 'browser', @@ -102,13 +102,20 @@ bundle = (options) -> line: m.originalLine or m.generatedLine column: m.originalColumn or m.generatedColumn source: canonicalName + name: m.name lineOffset += lineCount - for entryPoint in options.entryPoints - {id} = options.processed[entryPoint] + if options.export + {id} = options.processed[options.entryPoints[0]] if typeof id != 'number' id = "'#{id}'" - result += "\nrequire(#{id});" + result += "\n#{options.export} = require(#{id});" + else + for entryPoint in options.entryPoints + {id} = options.processed[entryPoint] + if typeof id != 'number' + id = "'#{id}'" + result += "\nrequire(#{id});" if options.node result = wrapNode(result) @@ -123,6 +130,8 @@ module.exports = (options) -> if options.minify uglifyAst = UglifyJS.parse code + # Enabling the compressor seems to break the source map, leave commented + # until a solution is found # uglifyAst.figure_out_scope() # uglifyAst = uglifyAst.transform UglifyJS.Compressor warnings: false uglifyAst.figure_out_scope() diff --git a/src/cjsify.coffee b/src/cjsify.coffee deleted file mode 100644 index b2bd579..0000000 --- a/src/cjsify.coffee +++ /dev/null @@ -1,8 +0,0 @@ -bundle = require './bundle' -traverseDependencies = require './traverse-dependencies' - -module.exports = (entryPoint, root = process.cwd(), options = {}) -> - processed = traverseDependencies entryPoint, root, options - if options.verbose - console.error "\nIncluded modules:\n #{(Object.keys processed).sort().join "\n "}" - bundle processed, entryPoint, root, options diff --git a/src/index.coffee b/src/index.coffee new file mode 100644 index 0000000..3d0a6f4 --- /dev/null +++ b/src/index.coffee @@ -0,0 +1,42 @@ +bundle = require './bundle' +traverseDependencies = require './traverse-dependencies' + +exports.bundle = bundle +exports.traverseDependencies = traverseDependencies +exports.cjsify = (options = {}) -> + if options.export and options.entryPoints.length != 1 + throw new Error('Can only set the export option with one entry point') + options.processed or= {} + if options.output + options.sourceMapRoot = + path.relative(path.dirname(options.output), options.root) + options.alias ?= [] + options.aliases = {} + for aliasPair in options.alias + match = aliasPair.match /([^:]+):(.*)/ ? [] + if match? then options.aliases[match[1]] = match[2] + else + console.error "invalid alias: #{aliasPair}" + process.exit 1 + delete options.alias + options.handler ?= [] + options.handlers = {} + for handlerPair in options.handler + match = handlerPair.match /([^:]+):(.*)/ ? [] + if match? then do (ext = ".#{match[1]}", mod = match[2]) -> + options.handlers[ext] = require mod + else + console.error "invalid handler: #{handlerPair}" + process.exit 1 + delete options.handler + options.inlineSources ?= false + options.cachePath ?= '.powerbuild-cache~' + options.log or= -> + options.root or= process.cwd() + options.uidFor or= (name) -> name + options.node ?= true + options.processed = {} + traverseDependencies options + if options.verbose + options.log "\nIncluded modules:\n #{(Object.keys options.processed).sort().join "\n "}" + bundle options diff --git a/src/module.coffee b/src/module.coffee deleted file mode 100644 index bd4f42e..0000000 --- a/src/module.coffee +++ /dev/null @@ -1,3 +0,0 @@ -exports.bundle = require './bundle' -exports.cjsify = require './cjsify' -exports.traverseDependencies = require './traverse-dependencies' diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee index 8528c52..1c9b5e4 100644 --- a/src/sourcemap-to-ast.coffee +++ b/src/sourcemap-to-ast.coffee @@ -18,7 +18,6 @@ module.exports = (ast, srcMap) -> # This is a top-level node like program or a top-level wrapper # function, dont care about the source file in these cases return - # console.error(origStart.name, origEnd.name) node.loc = start: line: origStart.line @@ -27,3 +26,4 @@ module.exports = (ast, srcMap) -> line: origEnd.line column: origEnd.column + 1 source: origStart.source + name: origStart.name diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 61748e8..870e9cb 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -45,7 +45,7 @@ module.exports = (options) -> options.entryPoints = resolvedEntryPoints - processed = options.processed or {} + processed = options.processed checked = {} while worklist.length @@ -78,7 +78,7 @@ module.exports = (options) -> if typeof astOrJs == 'string' astOrJs = {code: astOrJs} - if astOrJs.code + if astOrJs.code? try ast = acorn.parse astOrJs.code, locations: yes ast.loc ?= {} diff --git a/test-setup.coffee b/test-setup.coffee index bcc9b55..faa9850 100644 --- a/test-setup.coffee +++ b/test-setup.coffee @@ -31,7 +31,7 @@ sfs.reset = -> fs.mkdirpSync FIXTURES_DIR do sfs.reset -global[k] = v for own k, v of require './src/module' +global[k] = v for own k, v of require './src' global.FIXTURES_DIR = FIXTURES_DIR global.path = path global.escodegen = escodegen @@ -41,14 +41,17 @@ global.fixtures = (opts) -> sfs.applySync opts global.bundle = bundle = (entryPoint, opts) -> - root = path.resolve FIXTURES_DIR, (opts.root ? '') - escodegen.generate cjsify entryPoint, root, opts + opts.root = path.resolve FIXTURES_DIR, (opts.root ? '') + opts.entryPoints = [entryPoint] + {code} = cjsify opts + return code global.bundleEval = (entryPoint, opts = {}, env = {}) -> global$ = Object.create null global$.module$ = module$ = {} global$[key] = val for own key, val of env opts.export = 'module$.exports' - vm.runInNewContext (bundle entryPoint, opts), global$, '' + code = bundle entryPoint, opts + vm.runInNewContext code, global$, '' module$.exports extensions = ['.js', '.coffee'] diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index fddf991..22936e7 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -1,74 +1,83 @@ +path = require 'path' + + suite 'Dependency Resolution', -> deps = (entryFile, opts) -> entryFile = path.resolve path.join FIXTURES_DIR, entryFile - for filename in (Object.keys traverseDependencies entryFile, FIXTURES_DIR, opts).sort() + opts or= {} + opts.processed or= {} + opts.uidFor or= (name) -> name + opts.root = path.join process.cwd(), 'fixtures' + opts.entryPoints = [entryFile] + rv = [] + for filename in (Object.keys traverseDependencies opts).sort() if filename[...FIXTURES_DIR.length] is FIXTURES_DIR - "/#{path.relative FIXTURES_DIR, filename}" + "#{path.relative FIXTURES_DIR, filename}" else path.relative __dirname, filename test 'no dependencies', -> - fixtures '/a.js': '' - arrayEq ['/a.js'], deps '/a.js' + fixtures 'a.js': '' + arrayEq ['a.js'], deps 'a.js' test 'a single dependency', -> fixtures - '/a.js': 'require("./b")' - '/b.js': '' - arrayEq ['/a.js', '/b.js'], deps '/a.js' + 'a.js': 'require("./b")' + 'b.js': '' + arrayEq ['a.js', 'b.js'], deps 'a.js' test 'more than one dependency', -> fixtures - '/a.js': 'require("./b"); require("./c")' - '/b.js': '' - '/c.js': '' - arrayEq ['/a.js', '/b.js', '/c.js'], deps '/a.js' + 'a.js': 'require("./b"); require("./c")' + 'b.js': '' + 'c.js': '' + arrayEq ['a.js', 'b.js', 'c.js'], deps 'a.js' test 'transitive dependencies', -> fixtures - '/a.js': 'require("./b");' - '/b.js': 'require("./c")' - '/c.js': '' - arrayEq ['/a.js', '/b.js', '/c.js'], deps '/a.js' + 'a.js': 'require("./b");' + 'b.js': 'require("./c")' + 'c.js': '' + arrayEq ['a.js', 'b.js', 'c.js'], deps 'a.js' test 'circular dependencies', -> fixtures - '/a.js': 'require("./b");' - '/b.js': 'require("./a")' - arrayEq ['/a.js', '/b.js'], deps '/a.js' + 'a.js': 'require("./b");' + 'b.js': 'require("./a")' + arrayEq ['a.js', 'b.js'], deps 'a.js' test 'core dependencies', -> - fixtures '/a.js': 'require("freelist")' - arrayEq ['/a.js', '../node/lib/freelist.js'], deps '/a.js' + fixtures 'a.js': 'require("freelist")' + arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js' test 'missing dependencies', -> - fixtures '/a.js': 'require("./b")' - throws -> deps '/a.js' + fixtures 'a.js': 'require("./b")' + throws -> deps 'a.js' test 'ignoreMissing option ignores missing dependencies', -> - fixtures '/a.js': 'require("./b")' - arrayEq ['/a.js'], deps '/a.js', ignoreMissing: yes + fixtures 'a.js': 'require("./b")' + arrayEq ['a.js'], deps 'a.js', ignoreMissing: yes suite 'Aliasing', -> test 'basic alias', -> fixtures - '/a.js': 'require("./b")' - '/b.js': '' # /b.js still needs to exist - '/c.js': '' - arrayEq ['/a.js', '/c.js'], deps '/a.js', aliases: {'/b.js': '/c.js'} + 'a.js': 'require("./b")' + 'b.js': '' # /b.js still needs to exist + 'c.js': '' + arrayEq ['a.js', 'c.js'], deps 'a.js', aliases: {'b.js': 'c.js'} test 'alias to falsey value to omit', -> fixtures - '/a.js': 'require("./b")' - '/b.js': '' - arrayEq ['/a.js'], deps '/a.js', aliases: {'/b.js': ''} - arrayEq ['/a.js'], deps '/a.js', aliases: {'/b.js': null} - arrayEq ['/a.js'], deps '/a.js', aliases: {'/b.js': false} + 'a.js': 'require("./b")' + 'b.js': '' + arrayEq ['a.js'], deps 'a.js', aliases: {'b.js': ''} + arrayEq ['a.js'], deps 'a.js', aliases: {'b.js': null} + arrayEq ['a.js'], deps 'a.js', aliases: {'b.js': false} test 'alias a core module', -> - fixtures '/a.js': 'require("fs")' - arrayEq ['/a.js', '../node/lib/freelist.js'], deps '/a.js', aliases: {fs: 'freelist'} - fixtures '/a.js': 'require("path")' - arrayEq ['/a.js', '../node/lib/path.js', '../node/lib/util.js'], deps '/a.js', aliases: {child_process: null, fs: null} + fixtures 'a.js': 'require("fs")' + arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js', aliases: {fs: 'freelist'} + fixtures 'a.js': 'require("path")' + arrayEq ['a.js', '../node/lib/path.js', '../node/lib/util.js'], deps 'a.js', aliases: {child_process: null, fs: null} diff --git a/test/module-resolution.coffee b/test/module-resolution.coffee index 3a51c9b..e001057 100644 --- a/test/module-resolution.coffee +++ b/test/module-resolution.coffee @@ -4,37 +4,37 @@ suite 'Module Resolution', -> test 'node modules', -> fixtures '/node_modules/node-module-name/index.js': '' - eq '/node_modules/node-module-name/index.js', resolve 'node-module-name' - eq '/node_modules/node-module-name/index.js', resolve 'node-module-name/index' + eq 'node_modules/node-module-name/index.js', resolve 'node-module-name' + eq 'node_modules/node-module-name/index.js', resolve 'node-module-name/index' test 'relative requires', -> fixtures '/file.js': '' - eq '/file.js', resolve './file' + eq 'file.js', resolve './file' fixtures '/dir/file.js': '' - eq '/dir/file.js', resolve './file', 'dir' + eq 'dir/file.js', resolve './file', 'dir' fixtures '/dir/': yes '/file.js': '' - eq '/file.js', resolve '../file', 'dir' + eq 'file.js', resolve '../file', 'dir' test 'CoffeeScript files', -> fixtures '/coffee-file.coffee': '' - eq '/coffee-file.coffee', resolve './coffee-file' + eq 'coffee-file.coffee', resolve './coffee-file' test '"absolute" paths', -> fixtures '/dir/file.js': '' - eq '/dir/file.js', resolve 'dir/file' - eq '/dir/file.js', resolve 'dir/file', 'dir' + eq 'dir/file.js', resolve 'dir/file' + eq 'dir/file.js', resolve 'dir/file', 'dir' test 'directories', -> fixtures '/dir/index.js': '' '/dir/dir/index.js': '' - eq '/dir/index.js', resolve 'dir' - eq '/dir/index.js', resolve 'dir', 'dir' - eq '/dir/dir/index.js', resolve 'dir/dir' + eq 'dir/index.js', resolve 'dir' + eq 'dir/index.js', resolve 'dir', 'dir' + eq 'dir/dir/index.js', resolve 'dir/dir' test 'core module', -> doesNotThrow => resolve 'freelist' diff --git a/test/module-spec.coffee b/test/module-spec.coffee index 18cbb3f..e2abcdb 100644 --- a/test/module-spec.coffee +++ b/test/module-spec.coffee @@ -21,4 +21,4 @@ suite 'Module Spec', -> fixtures '/a.js': 'require("./b"); module.exports = module.children[0].exports' '/b.js': 'module.exports = module.filename' - eq '/b.js', bundleEval 'a.js' + eq 'b.js', bundleEval 'a.js' From e6777ef16a8315f5077187b0cb9189eaef10640f Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 14:00:01 -0300 Subject: [PATCH 10/31] Switched from make to grunt --- Gruntfile.coffee | 41 +++++++++++++++++++++++++++++++++++++++++ Makefile | 48 ------------------------------------------------ bin/cjsify | 3 ++- package.json | 6 ++++-- 4 files changed, 47 insertions(+), 51 deletions(-) create mode 100644 Gruntfile.coffee delete mode 100644 Makefile diff --git a/Gruntfile.coffee b/Gruntfile.coffee new file mode 100644 index 0000000..c1c09c2 --- /dev/null +++ b/Gruntfile.coffee @@ -0,0 +1,41 @@ +module.exports = (grunt) -> + + grunt.initConfig + clean: + all: ['build'] + + powerbuild: + all: + 'lib': 'src/*.coffee' + + mocha_debug: + options: + ui: 'tdd' + reporter: 'dot' + check: ['src/*.coffee', 'test/*.coffee'] + nodejs: + options: + src: ['test-setup.coffee', 'test/*.coffee'] + + watch: + options: + nospawn: true + all: + files: [ + 'Gruntfile.coffee' + 'src/*.coffee' + 'test/*.coffee' + ] + tasks: [ + 'test' + ] + + + # grunt.loadTasks('tasks') + + grunt.loadNpmTasks('grunt-release') + grunt.loadNpmTasks('grunt-mocha-debug') + grunt.loadNpmTasks('grunt-contrib-watch') + + grunt.registerTask('test', ['mocha_debug']) + grunt.registerTask('default', ['test', 'watch']) diff --git a/Makefile b/Makefile deleted file mode 100644 index 3100627..0000000 --- a/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -default: build - -CHANGELOG=CHANGELOG - -SRC = $(shell find src -name "*.coffee" -type f | sort) -LIB = $(SRC:src/%.coffee=lib/%.js) - -COFFEE=node_modules/.bin/coffee -MOCHA=node_modules/.bin/mocha --compilers coffee:coffee-script -r coffee-script -r test-setup.coffee -u tdd - -all: build test -build: $(LIB) - -lib/%.js: src/%.coffee - @dirname "$@" | xargs mkdir -p - $(COFFEE) -c -m -o lib "$<" - -.PHONY: release test loc clean - -VERSION = $(shell node -pe 'require("./package.json").version') -release-patch: NEXT_VERSION = $(shell node -pe 'require("semver").inc("$(VERSION)", "patch")') -release-minor: NEXT_VERSION = $(shell node -pe 'require("semver").inc("$(VERSION)", "minor")') -release-major: NEXT_VERSION = $(shell node -pe 'require("semver").inc("$(VERSION)", "major")') -release-patch: release -release-minor: release -release-major: release - -release: build test - @printf "Current version is $(VERSION). This will publish version $(NEXT_VERSION). Press [enter] to continue." >&2 - @read - ./changelog.sh "v$(NEXT_VERSION)" >"$(CHANGELOG)" - node -e '\ - var j = require("./package.json");\ - j.version = "$(NEXT_VERSION)";\ - var s = JSON.stringify(j, null, 2) + "\n";\ - require("fs").writeFileSync("./package.json", s);' - git commit package.json "$(CHANGELOG)" -m 'Version $(NEXT_VERSION)' - git tag -a "v$(NEXT_VERSION)" -m "Version $(NEXT_VERSION)" - git push --tags origin HEAD:master - npm publish - -test: - $(MOCHA) -R dot test/*.coffee - -loc: - @wc -l src/* -clean: - @rm -rf lib diff --git a/bin/cjsify b/bin/cjsify index 6e81aa6..2c88d11 100755 --- a/bin/cjsify +++ b/bin/cjsify @@ -1,2 +1,3 @@ #!/usr/bin/env node -require(require('path').join(__dirname, '..', 'lib', 'command')); +require('coffee-script'); +require(require('path').join(__dirname, '..', 'src', 'command')); diff --git a/package.json b/package.json index 33bfc8c..d2c0256 100644 --- a/package.json +++ b/package.json @@ -62,9 +62,11 @@ "escodegen": "0.0.27" }, "devDependencies": { - "mocha": "~1.8.1", "scopedfs": "~0.1.0", - "semver": "~1.1.4" + "grunt": "~0.4.1", + "grunt-release": "~0.6.0", + "grunt-mocha-debug": "0.0.6", + "grunt-contrib-watch": "~0.5.3" }, "scripts": { "test": "make test" From b12cf4020bfce7622e6cac9bd3f80630bf5d1754 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 18:27:39 -0300 Subject: [PATCH 11/31] Refactored options handling into a separate class for reuse with the grunt task --- Gruntfile.coffee | 3 +- src/bundle.coffee | 36 +++++----- src/command.coffee | 97 +++++++-------------------- src/index.coffee | 105 +++++++++++++++++++----------- src/traverse-dependencies.coffee | 24 +++---- test-setup.coffee | 7 +- test/dependency-resolution.coffee | 10 +-- 7 files changed, 133 insertions(+), 149 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index c1c09c2..7608cf8 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -12,7 +12,7 @@ module.exports = (grunt) -> options: ui: 'tdd' reporter: 'dot' - check: ['src/*.coffee', 'test/*.coffee'] + check: ['test-setup.coffee', 'src/*.coffee', 'test/*.coffee'] nodejs: options: src: ['test-setup.coffee', 'test/*.coffee'] @@ -22,6 +22,7 @@ module.exports = (grunt) -> nospawn: true all: files: [ + 'test-setup.coffee' 'Gruntfile.coffee' 'src/*.coffee' 'test/*.coffee' diff --git a/src/bundle.coffee b/src/bundle.coffee index 00b5074..98fcce3 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -76,14 +76,14 @@ wrapNode = (modules) -> """ })(#{PRELUDE}, #{PRELUDE_NODE}); """ -bundle = (options) -> +bundle = (build) -> result = '' resultMap = new SourceMapGenerator - file: path.basename(options.outFile) - sourceRoot: options.sourceMapRoot + file: path.basename(build.output) + sourceRoot: build.sourceMapRoot lineOffset = 1 # global wrapper - for own filename, {id, canonicalName, code, map, lineCount} of options.processed + for own filename, {id, canonicalName, code, map, lineCount} of build.processed if typeof id != 'number' id = "'#{id}'" result += """ @@ -105,19 +105,19 @@ bundle = (options) -> name: m.name lineOffset += lineCount - if options.export - {id} = options.processed[options.entryPoints[0]] + if build.export + {id} = build.processed[build.entryPoints[0]] if typeof id != 'number' id = "'#{id}'" - result += "\n#{options.export} = require(#{id});" + result += "\n#{build.export} = require(#{id});" else - for entryPoint in options.entryPoints - {id} = options.processed[entryPoint] + for entryPoint in build.entryPoints + {id} = build.processed[entryPoint] if typeof id != 'number' id = "'#{id}'" result += "\nrequire(#{id});" - if options.node + if build.node result = wrapNode(result) else result = wrap(result) @@ -125,10 +125,10 @@ bundle = (options) -> return {code: result, map: resultMap.toString()} -module.exports = (options) -> - {code, map} = bundle options +module.exports = (build) -> + {code, map} = bundle build - if options.minify + if build.minify uglifyAst = UglifyJS.parse code # Enabling the compressor seems to break the source map, leave commented # until a solution is found @@ -138,18 +138,18 @@ module.exports = (options) -> uglifyAst.compute_char_frequency() uglifyAst.mangle_names() sm = UglifyJS.SourceMap { - file: options.output - root: options.sourceMapRoot + file: build.output + root: build.sourceMapRoot orig: map } code = uglifyAst.print_to_string source_map: sm map = sm.toString() - if (options.sourceMap or options.inlineSourceMap) and options.inlineSources - for own filename, {code: src, canonicalName} of options.processed + if (build.sourceMap or build.inlineSourceMap) and build.inlineSources + for own filename, {code: src, canonicalName} of build.processed map.setSourceContent canonicalName, src - if options.inlineSourceMap + if build.inlineSourceMap datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" code = "#{code}\n//# sourceMappingURL=#{datauri}" diff --git a/src/command.coffee b/src/command.coffee index 37569d2..a2df466 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -3,17 +3,9 @@ path = require 'path' nopt = require 'nopt' _ = require 'lodash' -bundle = require './bundle' +Powerbuild = require './index' traverseDependencies = require './traverse-dependencies' -escodegenFormat = - indent: - style: ' ' - base: 0 - renumber: yes - hexadecimal: yes - quotes: 'auto' - parentheses: no knownOpts = {} # options @@ -36,19 +28,13 @@ optAliases = v: '--verbose' w: '--watch' x: '--export' - e: '--main' options = nopt knownOpts, optAliases, process.argv, 2 options.entryPoints = entryPoints = _.uniq options.argv.remain delete options.argv # default values -options.node ?= on -options['inline-sources'] ?= false options['cache-path'] ?= '.powerbuild-cache~' -options['root'] ?= process.cwd() -options.alias ?= [] -options.handler ?= [] options.ignoreMissing = options['ignore-missing'] options.sourceMap = options['source-map'] @@ -97,38 +83,38 @@ if options.version console.log (require '../package.json').version process.exit 0 +if options.deps + options.processed = {} + traverseDependencies options + console.log dep.canonicalName for own _, dep of options.processed + process.exit 0 + +if options.watch and not options.output + console.error '--watch requires --output' + process.exit 1 + +options.alias ?= [] options.aliases = {} + for aliasPair in options.alias - match = aliasPair.match /([^:]+):(.*)/ ? [] - if match? then options.aliases[match[1]] = match[2] + if match = aliasPair.match /([^:]+):(.*)/ ? [] + options.aliases[match[1]] = match[2] else - console.error "invalid alias: #{aliasPair}" - process.exit 1 -delete options.alias + throw new Error "invalid alias: #{aliasPair}" + +options.handler ?= [] options.handlers = {} + for handlerPair in options.handler - match = handlerPair.match /([^:]+):(.*)/ ? [] - if match? then do (ext = ".#{match[1]}", mod = match[2]) -> - options.handlers[ext] = require mod + if match = handlerPair.match /([^:]+):(.*)/ ? [] + options.handlers[match[1]] = require match[2] else - console.error "invalid handler: #{handlerPair}" - process.exit 1 -delete options.handler + throw new Error "invalid handler: #{handlerPair}" -if options.deps - deps = traverseDependencies options - console.log dep.canonicalName for own _, dep of deps - process.exit 0 - -if options.watch and not options.output - console.error '--watch requires --output' - process.exit 1 - buildBundle = -> - traverseDependencies options - {code, map} = bundle options + {code, map} = powerbuild.bundle() if options.sourceMap fs.writeFileSync options.sourceMap, "#{map}" @@ -148,42 +134,7 @@ buildBundle = -> else process.stdout.write "#{code}\n" - startBuild = -> - process.on 'exit', -> - cache = - processed: options.processed - uids: options.uids - moduleUids: options.moduleUids - fs.writeFileSync options.cachePath, JSON.stringify cache - - process.on 'uncaughtException', (e) -> - # An exception may be thrown due to corrupt cache or incompatibilities - # between versions, remove it to be safe - try fs.unlinkSync options.cachePath - options.processed = {} - throw e - - if fs.existsSync options.cachePath - cache = JSON.parse fs.readFileSync options.cachePath, 'utf8' - {processed, uids, moduleUids} = cache - - if not processed or moduleUids != options.moduleUids - # Either the cache doesn't exist or the cache was saved with a different - # 'moduleUids' value. In either case we must reset it. - processed = {} - uids = {next: 1, names: {}} - - options.processed = processed - options.uids = uids - uidFor = options.uidFor = (name) -> - if not options.moduleUids - return name - if not {}.hasOwnProperty.call(uids.names, name) - uid = uids.next++ - uids.names[name] = uid - uids.names[name] - buildBundle() if options.watch @@ -208,6 +159,8 @@ startBuild = -> building = false return +powerbuild = new Powerbuild options + if entryPoints.length == 1 and entryPoints[0] is '-' # support reading input from stdin stdinput = '' diff --git a/src/index.coffee b/src/index.coffee index 3d0a6f4..2fba1c2 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -1,42 +1,69 @@ +_ = require 'lodash' +fs = require 'fs' bundle = require './bundle' traverseDependencies = require './traverse-dependencies' -exports.bundle = bundle -exports.traverseDependencies = traverseDependencies -exports.cjsify = (options = {}) -> - if options.export and options.entryPoints.length != 1 - throw new Error('Can only set the export option with one entry point') - options.processed or= {} - if options.output - options.sourceMapRoot = - path.relative(path.dirname(options.output), options.root) - options.alias ?= [] - options.aliases = {} - for aliasPair in options.alias - match = aliasPair.match /([^:]+):(.*)/ ? [] - if match? then options.aliases[match[1]] = match[2] - else - console.error "invalid alias: #{aliasPair}" - process.exit 1 - delete options.alias - options.handler ?= [] - options.handlers = {} - for handlerPair in options.handler - match = handlerPair.match /([^:]+):(.*)/ ? [] - if match? then do (ext = ".#{match[1]}", mod = match[2]) -> - options.handlers[ext] = require mod - else - console.error "invalid handler: #{handlerPair}" - process.exit 1 - delete options.handler - options.inlineSources ?= false - options.cachePath ?= '.powerbuild-cache~' - options.log or= -> - options.root or= process.cwd() - options.uidFor or= (name) -> name - options.node ?= true - options.processed = {} - traverseDependencies options - if options.verbose - options.log "\nIncluded modules:\n #{(Object.keys options.processed).sort().join "\n "}" - bundle options + +class Powerbuild + constructor: (options) -> + if options.export and options.entryPoints.length != 1 + throw new Error('Can only set the export option with one entry point') + + options.inlineSources ?= false + options.log or= -> + options.root or= process.cwd() + options.node ?= true + {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, + @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @moduleUids, + @mainModule, @minify, @aliases, @handlers} = options + + if @output + @sourceMapRoot = path.relative(path.dirname(@output), @root) + + if cachePath = options.cachePath + process.on 'exit', => + cache = + processed: @processed + uids: @uids + moduleUids: @moduleUids + fs.writeFileSync cachePath, JSON.stringify cache + + process.on 'uncaughtException', (e) -> + # An exception may be thrown due to corrupt cache or incompatibilities + # between versions, remove it to be safe + try fs.unlinkSync cachePath + @processed = {} + throw e + + if fs.existsSync cachePath + cache = JSON.parse fs.readFileSync cachePath, 'utf8' + {@processed, @uids, @moduleUids} = cache + + if not @processed or @moduleUids != options.moduleUids + # Either the cache doesn't exist or the cache was saved with a different + # 'moduleUids' value. In either case we must reset it. + @processed = {} + @uids = {next: 1, names: {}} + + + bundle: -> + @traverseDependencies() + bundle this + + + traverseDependencies: -> + traverseDependencies this + if @verbose + @log "Included modules: #{(Object.keys @processed).sort()}" + + + uidFor: (name) -> + if not @moduleUids + return name + if not {}.hasOwnProperty.call(@uids.names, name) + uid = @uids.next++ + @uids.names[name] = uid + @uids.names[name] + + +module.exports = Powerbuild diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 870e9cb..7a84584 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -20,10 +20,10 @@ badRequireError = (filename, node, msg) -> in #{filename} """ -module.exports = (options) -> - aliases = options.aliases ? {} - uidFor = options.uidFor - root = options.root +module.exports = (build) -> + aliases = build.aliases ? {} + uidFor = build.uidFor + root = build.root handlers = '.coffee': (src, canonicalName) -> @@ -31,21 +31,21 @@ module.exports = (options) -> return {code: js, map: v3SourceMap} '.json': (json, canonicalName) -> acorn.parse "module.exports = #{json}", locations: yes - for own ext, handler of options.handlers ? {} + for own ext, handler of build.handlers ? {} handlers[ext] = handler extensions = ['.js', (ext for own ext of handlers)...] worklist = [] resolvedEntryPoints = [] - for ep in options.entryPoints + for ep in build.entryPoints resolved = relativeResolve {extensions, aliases, root, path: ep} worklist.push(resolved) resolvedEntryPoints.push(resolved.filename) - options.entryPoints = resolvedEntryPoints + build.entryPoints = resolvedEntryPoints - processed = options.processed + processed = build.processed checked = {} while worklist.length @@ -112,15 +112,15 @@ module.exports = (options) -> unless node.arguments[0].type is 'Literal' and typeof node.arguments[0].value is 'string' badRequireError filename, node, 'argument of require must be a constant string' cwd = path.dirname fs.realpathSync filename - if options.verbose + if build.verbose console.error "required \"#{node.arguments[0].value}\" from \"#{canonicalName}\"" # if we are including this file, its requires need to be processed as well try - resolved = relativeResolve {extensions, aliases, root: options.root, cwd, path: node.arguments[0].value} + resolved = relativeResolve {extensions, aliases, root: build.root, cwd, path: node.arguments[0].value} worklist.push resolved deps.push resolved catch e - if options.ignoreMissing + if build.ignoreMissing return { type: 'Literal', value: null } else throw e @@ -142,7 +142,7 @@ module.exports = (options) -> sourceMap: yes format: escodegen.FORMAT_DEFAULTS sourceMapWithCode: yes - sourceMapRoot: if options.sourceMap? then (path.relative (path.dirname options.sourceMap), options.root) or '.' + sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' map = map.toString() diff --git a/test-setup.coffee b/test-setup.coffee index faa9850..8ff6ed0 100644 --- a/test-setup.coffee +++ b/test-setup.coffee @@ -31,7 +31,8 @@ sfs.reset = -> fs.mkdirpSync FIXTURES_DIR do sfs.reset -global[k] = v for own k, v of require './src' +# global[k] = v for own k, v of require './src' +global.Powerbuild = require './src' global.FIXTURES_DIR = FIXTURES_DIR global.path = path global.escodegen = escodegen @@ -43,8 +44,10 @@ global.fixtures = (opts) -> global.bundle = bundle = (entryPoint, opts) -> opts.root = path.resolve FIXTURES_DIR, (opts.root ? '') opts.entryPoints = [entryPoint] - {code} = cjsify opts + powerbuild = new Powerbuild opts + {code} = powerbuild.bundle() return code + global.bundleEval = (entryPoint, opts = {}, env = {}) -> global$ = Object.create null global$.module$ = module$ = {} diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index 22936e7..b5486ac 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -1,17 +1,17 @@ path = require 'path' - +# traverseDependencies = require '../src/traverse-dependencies' suite 'Dependency Resolution', -> deps = (entryFile, opts) -> entryFile = path.resolve path.join FIXTURES_DIR, entryFile opts or= {} - opts.processed or= {} - opts.uidFor or= (name) -> name - opts.root = path.join process.cwd(), 'fixtures' + opts.root = FIXTURES_DIR opts.entryPoints = [entryFile] + powerbuild = new Powerbuild opts + powerbuild.traverseDependencies() rv = [] - for filename in (Object.keys traverseDependencies opts).sort() + for filename in (Object.keys powerbuild.processed).sort() if filename[...FIXTURES_DIR.length] is FIXTURES_DIR "#{path.relative FIXTURES_DIR, filename}" else From 066833d8dadb9991c07e01cc76f2f8ce4d5ae7cf Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 21:20:46 -0300 Subject: [PATCH 12/31] Implement grunt task --- Gruntfile.coffee | 11 ++++-- package.json | 4 +-- src/build-cache.coffee | 24 +++++++++++++ src/bundle.coffee | 16 ++++++--- src/command.coffee | 30 +++++----------- src/index.coffee | 44 ++++++++++------------- src/traverse-dependencies.coffee | 62 ++++++++++++++++++++------------ tasks/powerbuild.coffee | 30 ++++++++++++++++ 8 files changed, 141 insertions(+), 80 deletions(-) create mode 100644 src/build-cache.coffee create mode 100644 tasks/powerbuild.coffee diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 7608cf8..1a7d4d3 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -5,8 +5,13 @@ module.exports = (grunt) -> all: ['build'] powerbuild: + options: + sourceMap: true + ignoreMissing: true all: - 'lib': 'src/*.coffee' + files: [ + {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'bundle.js'} + ] mocha_debug: options: @@ -32,11 +37,11 @@ module.exports = (grunt) -> ] - # grunt.loadTasks('tasks') + grunt.loadTasks('tasks') grunt.loadNpmTasks('grunt-release') grunt.loadNpmTasks('grunt-mocha-debug') grunt.loadNpmTasks('grunt-contrib-watch') - grunt.registerTask('test', ['mocha_debug']) + grunt.registerTask('test', ['powerbuild', 'mocha_debug']) grunt.registerTask('default', ['test', 'watch']) diff --git a/package.json b/package.json index d2c0256..6d8c3f3 100644 --- a/package.json +++ b/package.json @@ -56,10 +56,10 @@ "coffee-script": "~1.6.3", "source-map-support": "~0.2.3", "lodash": "~2.2.1", - "acorn": "~0.3.1", "uglify-js": "~2.4.0", "estraverse": "~1.3.1", - "escodegen": "0.0.27" + "escodegen": "0.0.27", + "esprima": "~1.0.4" }, "devDependencies": { "scopedfs": "~0.1.0", diff --git a/src/build-cache.coffee b/src/build-cache.coffee new file mode 100644 index 0000000..06523c8 --- /dev/null +++ b/src/build-cache.coffee @@ -0,0 +1,24 @@ +fs = require 'fs' +path = require 'path' + + +module.exports = (cachePath = path.join(process.cwd(), '.powerbuild~')) -> + process.on 'exit', -> + fs.writeFileSync cachePath, JSON.stringify cache + + process.on 'uncaughtException', (e) -> + # An exception may be thrown due to corrupt cache or incompatibilities + # between versions, remove it to be safe + try fs.unlinkSync cachePath + cache.processed = {} + throw e + + if fs.existsSync cachePath + cache = JSON.parse fs.readFileSync cachePath, 'utf8' + + if not cache + cache = + processed: {} + uids: {next: 1, names: []} + + return cache diff --git a/src/bundle.coffee b/src/bundle.coffee index 98fcce3..0aebfef 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -1,4 +1,3 @@ -acorn = require 'acorn' path = require 'path' {SourceMapConsumer, SourceMapGenerator} = require 'source-map' {btoa} = require 'Base64' @@ -96,11 +95,11 @@ bundle = (build) -> orig.eachMapping (m) -> resultMap.addMapping generated: - line: m.generatedLine + lineOffset - column: m.generatedColumn + line: m.generatedLine + lineOffset + column: m.generatedColumn original: - line: m.originalLine or m.generatedLine - column: m.originalColumn or m.generatedColumn + line: m.originalLine or m.generatedLine + column: m.originalColumn or m.generatedColumn source: canonicalName name: m.name lineOffset += lineCount @@ -149,8 +148,15 @@ module.exports = (build) -> for own filename, {code: src, canonicalName} of build.processed map.setSourceContent canonicalName, src + sourceMappingUrl = + if build.output + path.relative (path.dirname build.output), build.sourceMap + else build.sourceMap + if build.inlineSourceMap datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" code = "#{code}\n//# sourceMappingURL=#{datauri}" + else + code = "#{code}\n//# sourceMappingURL=#{sourceMappingUrl}" return {code, map} diff --git a/src/command.coffee b/src/command.coffee index a2df466..58f3a37 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -6,7 +6,6 @@ _ = require 'lodash' Powerbuild = require './index' traverseDependencies = require './traverse-dependencies' - knownOpts = {} # options knownOpts[opt] = Boolean for opt in [ @@ -43,8 +42,6 @@ options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] options.moduleUids = options['module-uids'] options.entryPoint = options['entry-point'] -options.sourceMapRoot = - path.relative(path.dirname(options.output), options.root) if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] @@ -102,7 +99,6 @@ for aliasPair in options.alias else throw new Error "invalid alias: #{aliasPair}" - options.handler ?= [] options.handlers = {} @@ -114,23 +110,13 @@ for handlerPair in options.handler buildBundle = -> - {code, map} = powerbuild.bundle() - - if options.sourceMap - fs.writeFileSync options.sourceMap, "#{map}" - sourceMappingUrl = - if options.output - path.relative (path.dirname options.output), options.sourceMap - else options.sourceMap - unless options.inlineSourceMap - code = "#{code}\n//# sourceMappingURL=#{sourceMappingUrl}" - - if options.inlineSourceMap - datauri = "data:application/json;charset=utf-8;base64,#{btoa "#{map}"}" - code = "#{code}\n//# sourceMappingURL=#{datauri}" - - if options.output - fs.writeFileSync options.output, code + {code, map} = build.bundle() + + if build.sourceMap + fs.writeFileSync build.sourceMap, "#{map}" + + if build.output + fs.writeFileSync build.output, code else process.stdout.write "#{code}\n" @@ -159,7 +145,7 @@ startBuild = -> building = false return -powerbuild = new Powerbuild options +build = new Powerbuild options if entryPoints.length == 1 and entryPoints[0] is '-' # support reading input from stdin diff --git a/src/index.coffee b/src/index.coffee index 2fba1c2..054814b 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -1,5 +1,7 @@ _ = require 'lodash' -fs = require 'fs' +acorn = require 'acorn' +coffee = require 'coffee-script' +path = require 'path' bundle = require './bundle' traverseDependencies = require './traverse-dependencies' @@ -11,44 +13,36 @@ class Powerbuild options.inlineSources ?= false options.log or= -> + options.processed or= {} + options.moduleUids ?= false options.root or= process.cwd() options.node ?= true {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @moduleUids, - @mainModule, @minify, @aliases, @handlers} = options + @mainModule, @minify, @aliases, @handlers, @processed, @uids, + @moduleUids} = options if @output @sourceMapRoot = path.relative(path.dirname(@output), @root) + if @sourceMap == true + @sourceMap = "#{@output}.map" - if cachePath = options.cachePath - process.on 'exit', => - cache = - processed: @processed - uids: @uids - moduleUids: @moduleUids - fs.writeFileSync cachePath, JSON.stringify cache + @handlers = + '.coffee': (src, canonicalName) -> + {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true + return {code: js, map: v3SourceMap} + '.json': (json, canonicalName) -> + acorn.parse "module.exports = #{json}", locations: yes - process.on 'uncaughtException', (e) -> - # An exception may be thrown due to corrupt cache or incompatibilities - # between versions, remove it to be safe - try fs.unlinkSync cachePath - @processed = {} - throw e + for own ext, handler of options.handlers ? {} + @handlers[ext] = handler - if fs.existsSync cachePath - cache = JSON.parse fs.readFileSync cachePath, 'utf8' - {@processed, @uids, @moduleUids} = cache - - if not @processed or @moduleUids != options.moduleUids - # Either the cache doesn't exist or the cache was saved with a different - # 'moduleUids' value. In either case we must reset it. - @processed = {} - @uids = {next: 1, names: {}} + @extensions = ['.js', (ext for own ext of @handlers)...] bundle: -> @traverseDependencies() - bundle this + return bundle this traverseDependencies: -> diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 7a84584..ae90193 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -2,8 +2,7 @@ fs = require 'fs' path = require 'path' util = require 'util' -coffee = require 'coffee-script' -acorn = require 'acorn' +esprima = require 'esprima' estraverse = require 'estraverse' escodegen = require 'escodegen' @@ -25,21 +24,11 @@ module.exports = (build) -> uidFor = build.uidFor root = build.root - handlers = - '.coffee': (src, canonicalName) -> - {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true - return {code: js, map: v3SourceMap} - '.json': (json, canonicalName) -> - acorn.parse "module.exports = #{json}", locations: yes - for own ext, handler of build.handlers ? {} - handlers[ext] = handler - extensions = ['.js', (ext for own ext of handlers)...] - worklist = [] resolvedEntryPoints = [] for ep in build.entryPoints - resolved = relativeResolve {extensions, aliases, root, path: ep} + resolved = relativeResolve {extensions: build.extensions, aliases, root, path: ep} worklist.push(resolved) resolvedEntryPoints.push(resolved.filename) @@ -70,18 +59,40 @@ module.exports = (build) -> astOrJs = # handle compile-to-JS languages and other non-JS files - if {}.hasOwnProperty.call handlers, extname - handlers[extname] src, canonicalName + if {}.hasOwnProperty.call build.handlers, extname + build.handlers[extname] src, canonicalName else # assume JS src if typeof astOrJs == 'string' astOrJs = {code: astOrJs} + adjustWrapperLocation = false + if astOrJs.code? try - ast = acorn.parse astOrJs.code, locations: yes - ast.loc ?= {} + # wrap into a function so top-level 'return' statements wont break + # when parsing + astOrJs.code = "(function(){#{astOrJs.code}})()" + ast = esprima.parse astOrJs.code, + loc: yes, comment: true, range: true, tokens: true + # unwrap the function + ast.body = ast.body[0].expression.callee.body.body + # adjust the range/column offsets to ignore the wrapped function + adjustWrapperLocation = true + # Remove the extra tokens + ast.tokens = ast.tokens.slice(5, ast.tokens.length - 4) + # Fix comments/token position info + for t in ast.comments.concat(ast.tokens) + t.range[0] -= 12 + t.range[1] -= 12 + if t.loc.start.line == 1 + t.loc.start.column -= 12 + if t.loc.end.line == 1 + t.loc.end.column -= 12 + # Also adjust top node end range/column + ast.range[1] -= 4 + ast.loc.end.column -= 4 if astOrJs.map sourceMapToAst ast, astOrJs.map catch e @@ -99,11 +110,16 @@ module.exports = (build) -> estraverse.replace ast, enter: (node, parents) -> - if node.loc? then node.loc.source = canonicalName - if node.type == 'TryStatement' and not node.guardedHandlers - # escodegen will break when generating from acorn's ast unless - # we add this - node.guardedHandlers = [] + if node.loc? + node.loc.source = canonicalName + if node.type != 'Program' and adjustWrapperLocation + # Adjust the location info to reflect the removed function wrapper + if node.loc.start.line == 1 and node.loc.start.column >= 12 + node.loc.start.column -= 12 + if node.loc.end.line == 1 and node.loc.end.column >= 12 + node.loc.end.column -= 12 + node.range[0] -= 12 + node.range[1] -= 12 # ignore anything that's not a `require` call return unless node.type is 'CallExpression' and node.callee.type is 'Identifier' and node.callee.name is 'require' # illegal requires @@ -116,7 +132,7 @@ module.exports = (build) -> console.error "required \"#{node.arguments[0].value}\" from \"#{canonicalName}\"" # if we are including this file, its requires need to be processed as well try - resolved = relativeResolve {extensions, aliases, root: build.root, cwd, path: node.arguments[0].value} + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: node.arguments[0].value} worklist.push resolved deps.push resolved catch e diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee new file mode 100644 index 0000000..36f3929 --- /dev/null +++ b/tasks/powerbuild.coffee @@ -0,0 +1,30 @@ +_ = require 'lodash' +path = require 'path' +Powerbuild = require '../src' +buildCache = require '../src/build-cache' + + +NAME = 'powerbuild' +DESC = 'Builds commonjs projects to run anywhere, transpiling to javascript if +necessary, besides generating concatenated source maps.' + + +cache = buildCache() + + +module.exports = (grunt) -> + grunt.registerMultiTask NAME, DESC, -> + options = @options() + options.processed = cache.processed + options.uids = cache.uids + for f in @files + opts = _.clone(options) + opts.entryPoints = grunt.file.expand(f.orig.src) + opts.output = f.dest + build = new Powerbuild(opts) + {code, map} = build.bundle() + grunt.file.write build.output, code + grunt.log.ok("Created #{build.output}") + if build.sourceMap + grunt.file.write build.sourceMap, map + grunt.log.ok("Created #{build.sourceMap}") From 3ce875da8e630e0154e8827a5a97b23c28b5d4ba Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sat, 5 Oct 2013 21:27:13 -0300 Subject: [PATCH 13/31] Re-add disk cache to command --- src/command.coffee | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/command.coffee b/src/command.coffee index 58f3a37..fc5e25c 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -4,6 +4,7 @@ nopt = require 'nopt' _ = require 'lodash' Powerbuild = require './index' +buildCache = require '../src/build-cache' traverseDependencies = require './traverse-dependencies' knownOpts = {} @@ -145,6 +146,9 @@ startBuild = -> building = false return +cache = buildCache() +options.processed = cache.processed +options.uids = cache.uids build = new Powerbuild options if entryPoints.length == 1 and entryPoints[0] is '-' From 58d54a448c791cc117ee82043ac6413dd481e27e Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sun, 6 Oct 2013 12:13:47 -0300 Subject: [PATCH 14/31] By default ignore source maps for npm modules. Also by default, dont check if npm modules were changed(very unlikely that npm modules will change) --- Gruntfile.coffee | 1 + src/build-cache.coffee | 3 +++ src/bundle.coffee | 23 ++++++++++--------- src/command.coffee | 14 +++++------- src/index.coffee | 4 +++- src/traverse-dependencies.coffee | 39 +++++++++++++++++++++----------- tasks/powerbuild.coffee | 2 ++ 7 files changed, 53 insertions(+), 33 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 1a7d4d3..c3c1cf6 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -29,6 +29,7 @@ module.exports = (grunt) -> files: [ 'test-setup.coffee' 'Gruntfile.coffee' + 'tasks/*.coffee' 'src/*.coffee' 'test/*.coffee' ] diff --git a/src/build-cache.coffee b/src/build-cache.coffee index 06523c8..538701e 100644 --- a/src/build-cache.coffee +++ b/src/build-cache.coffee @@ -6,6 +6,9 @@ module.exports = (cachePath = path.join(process.cwd(), '.powerbuild~')) -> process.on 'exit', -> fs.writeFileSync cachePath, JSON.stringify cache + process.on 'SIGINT', process.exit + process.on 'SIGTERM', process.exit + process.on 'uncaughtException', (e) -> # An exception may be thrown due to corrupt cache or incompatibilities # between versions, remove it to be safe diff --git a/src/bundle.coffee b/src/bundle.coffee index 0aebfef..82e7257 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -91,17 +91,18 @@ bundle = (build) -> }); """ lineOffset += 2 # skip linefeed plus the 'require.define' line - orig = new SourceMapConsumer map - orig.eachMapping (m) -> - resultMap.addMapping - generated: - line: m.generatedLine + lineOffset - column: m.generatedColumn - original: - line: m.originalLine or m.generatedLine - column: m.originalColumn or m.generatedColumn - source: canonicalName - name: m.name + if map + orig = new SourceMapConsumer map + orig.eachMapping (m) -> + resultMap.addMapping + generated: + line: m.generatedLine + lineOffset + column: m.generatedColumn + original: + line: m.originalLine or m.generatedLine + column: m.originalColumn or m.generatedColumn + source: canonicalName + name: m.name lineOffset += lineCount if build.export diff --git a/src/command.coffee b/src/command.coffee index fc5e25c..d160ff4 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -121,17 +121,20 @@ buildBundle = -> else process.stdout.write "#{code}\n" +cache = buildCache() +options.processed = cache.processed +options.uids = cache.uids +build = new Powerbuild options + startBuild = -> buildBundle() if options.watch # Flush the cache when the user presses CTRL+C or the process is # terminated from outside - process.on 'SIGINT', process.exit - process.on 'SIGTERM', process.exit watching = {} building = false - for own file of processed when file not of watching then do (file) -> + for own file of build.processed when file not of watching then do (file) -> watching[file] = true fs.watchFile file, {persistent: yes, interval: 500}, (curr, prev) -> if building then return @@ -146,11 +149,6 @@ startBuild = -> building = false return -cache = buildCache() -options.processed = cache.processed -options.uids = cache.uids -build = new Powerbuild options - if entryPoints.length == 1 and entryPoints[0] is '-' # support reading input from stdin stdinput = '' diff --git a/src/index.coffee b/src/index.coffee index 054814b..96a0a8e 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -14,13 +14,15 @@ class Powerbuild options.inlineSources ?= false options.log or= -> options.processed or= {} + options.checkNpmModules ?= false + options.npmSourceMaps ?= false options.moduleUids ?= false options.root or= process.cwd() options.node ?= true {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @moduleUids, @mainModule, @minify, @aliases, @handlers, @processed, @uids, - @moduleUids} = options + @moduleUids, @checkNpmModules, @npmSourceMaps} = options if @output @sourceMapRoot = path.relative(path.dirname(@output), @root) diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index ae90193..a06bc56 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -2,6 +2,7 @@ fs = require 'fs' path = require 'path' util = require 'util' +_ = require 'lodash' esprima = require 'esprima' estraverse = require 'estraverse' escodegen = require 'escodegen' @@ -29,7 +30,7 @@ module.exports = (build) -> for ep in build.entryPoints resolved = relativeResolve {extensions: build.extensions, aliases, root, path: ep} - worklist.push(resolved) + worklist.push(_.assign(resolved, {isNpmModule: false})) resolvedEntryPoints.push(resolved.filename) build.entryPoints = resolvedEntryPoints @@ -38,7 +39,7 @@ module.exports = (build) -> checked = {} while worklist.length - {filename, canonicalName} = worklist.pop() + {filename, canonicalName, isNpmModule} = worklist.pop() # support aliasing to falsey values to omit files continue unless filename @@ -52,7 +53,9 @@ module.exports = (build) -> if processed[filename]?.mtime == mtime # ignore files that have not changed, but also check its dependencies - worklist = worklist.concat processed[filename].deps + for dep in processed[filename].deps + if dep.isNpmModule and build.checkNpmModules + worklist.push dep continue src = (fs.readFileSync filename).toString() @@ -132,9 +135,14 @@ module.exports = (build) -> console.error "required \"#{node.arguments[0].value}\" from \"#{canonicalName}\"" # if we are including this file, its requires need to be processed as well try - resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: node.arguments[0].value} - worklist.push resolved - deps.push resolved + moduleName = node.arguments[0].value + isNpmDep = /^[^/.]/.test(moduleName) + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: moduleName} + dep = _.assign(resolved, {isNpmModule: isNpmDep}) + if dep.filename not of build.processed or + isNpmDep and bundle.checkNpmModules + worklist.push dep + deps.push dep catch e if build.ignoreMissing return { type: 'Literal', value: null } @@ -154,13 +162,18 @@ module.exports = (build) -> }] } - {code, map} = escodegen.generate ast, - sourceMap: yes - format: escodegen.FORMAT_DEFAULTS - sourceMapWithCode: yes - sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' - - map = map.toString() + map = null + if isNpmModule or build.npmSourceMaps + {code, map} = escodegen.generate ast, + sourceMap: true + format: escodegen.FORMAT_DEFAULTS + sourceMapWithCode: true + sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' + map = map.toString() + else + code = escodegen.generate ast, + sourceMap: false + format: escodegen.FORMAT_DEFAULTS # cache linecount for a little more efficiency when calculating offsets # later diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index 36f3929..f0e3176 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -22,7 +22,9 @@ module.exports = (grunt) -> opts.entryPoints = grunt.file.expand(f.orig.src) opts.output = f.dest build = new Powerbuild(opts) + start = new Date().getTime() {code, map} = build.bundle() + console.error("took #{new Date().valueOf() - start} ms") grunt.file.write build.output, code grunt.log.ok("Created #{build.output}") if build.sourceMap From 457c6dbc849d455c32cb6b0ed89e49d4e20110f0 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sun, 6 Oct 2013 19:14:02 -0300 Subject: [PATCH 15/31] Wrap bundle into umd --- src/bundle.coffee | 49 +++++++++++++++++++++-------- src/command.coffee | 13 +++++--- src/index.coffee | 1 + src/traverse-dependencies.coffee | 53 +++++++++++++++++++------------- tasks/powerbuild.coffee | 3 +- 5 files changed, 78 insertions(+), 41 deletions(-) diff --git a/src/bundle.coffee b/src/bundle.coffee index 82e7257..95f2424 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -66,21 +66,38 @@ PRELUDE = """ wrap = (modules) -> """ (function(require, undefined) { var global = this; #{modules} - })(#{PRELUDE}); + })(#{PRELUDE}) """ wrapNode = (modules) -> """ (function(require, process, undefined) { var global = this; #{modules} - })(#{PRELUDE}, #{PRELUDE_NODE}); + })(#{PRELUDE}, #{PRELUDE_NODE}) """ +wrapUmd = (exports, commonjs) -> """ + (function(exported) { + if (typeof exports === 'object') { + module.exports = exported; + } else if (typeof define === 'function' && define.amd) { + define(function() { + return exported; + }); + } else { + #{exports} + } + })(#{commonjs}); + """ + +umdOffset = wrapUmd('', '').split('\n').length + + bundle = (build) -> result = '' resultMap = new SourceMapGenerator file: path.basename(build.output) sourceRoot: build.sourceMapRoot - lineOffset = 1 # global wrapper + lineOffset = umdOffset for own filename, {id, canonicalName, code, map, lineCount} of build.processed if typeof id != 'number' @@ -105,22 +122,28 @@ bundle = (build) -> name: m.name lineOffset += lineCount - if build.export - {id} = build.processed[build.entryPoints[0]] + for i in [0...build.entryPoints.length] + entryPoint = build.entryPoints[i] + {id} = build.processed[entryPoint] if typeof id != 'number' id = "'#{id}'" - result += "\n#{build.export} = require(#{id});" - else - for entryPoint in build.entryPoints - {id} = build.processed[entryPoint] - if typeof id != 'number' - id = "'#{id}'" + if i == build.entryPoints.length - 1 + # export the last entry point + result += "\nreturn require(#{id});" + else result += "\nrequire(#{id});" + if build.export + exports = "#{build.export} = exported;" + else + exports = '' + if build.node - result = wrapNode(result) + commonjs = wrapNode(result) else - result = wrap(result) + commonjs = wrap(result) + + result = wrapUmd(exports, commonjs) return {code: result, map: resultMap.toString()} diff --git a/src/command.coffee b/src/command.coffee index d160ff4..e260dcb 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -111,13 +111,16 @@ for handlerPair in options.handler buildBundle = -> + start = new Date().getTime() {code, map} = build.bundle() - if build.sourceMap - fs.writeFileSync build.sourceMap, "#{map}" - if build.output fs.writeFileSync build.output, code + console.error("Created #{build.output}") + if build.sourceMap + console.error("Created #{build.sourceMap}") + fs.writeFileSync build.sourceMap, "#{map}" + console.error("Completed in #{new Date().getTime() - start} ms") else process.stdout.write "#{code}\n" @@ -130,6 +133,7 @@ startBuild = -> buildBundle() if options.watch + console.error("Watching for changes...") # Flush the cache when the user presses CTRL+C or the process is # terminated from outside watching = {} @@ -138,14 +142,13 @@ startBuild = -> watching[file] = true fs.watchFile file, {persistent: yes, interval: 500}, (curr, prev) -> if building then return + console.error("File '#{file}' as changed, starting rebuild") building = true ino = if process.platform is 'win32' then curr.ino? else curr.ino unless ino console.error "WARNING: watched file #{file} has disappeared" return - console.error "#{file} changed, rebuilding" buildBundle() - console.error "done" building = false return diff --git a/src/index.coffee b/src/index.coffee index 96a0a8e..605adcb 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -16,6 +16,7 @@ class Powerbuild options.processed or= {} options.checkNpmModules ?= false options.npmSourceMaps ?= false + options.bundleNpmModules ?= true options.moduleUids ?= false options.root or= process.cwd() options.node ?= true diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index a06bc56..ca8dccf 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -10,6 +10,7 @@ escodegen = require 'escodegen' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' sourceMapToAst = require './sourcemap-to-ast' +isCore = require './is-core' badRequireError = (filename, node, msg) -> if node.loc? and node.loc?.start? @@ -22,7 +23,6 @@ badRequireError = (filename, node, msg) -> module.exports = (build) -> aliases = build.aliases ? {} - uidFor = build.uidFor root = build.root worklist = [] @@ -109,7 +109,7 @@ module.exports = (build) -> # add source file information to the AST root node ast.loc ?= {} deps = [] - id = uidFor(canonicalName) + id = build.uidFor(canonicalName) estraverse.replace ast, enter: (node, parents) -> @@ -133,16 +133,22 @@ module.exports = (build) -> cwd = path.dirname fs.realpathSync filename if build.verbose console.error "required \"#{node.arguments[0].value}\" from \"#{canonicalName}\"" - # if we are including this file, its requires need to be processed as well + # if we are including this file, its requires need to be processed as + # well try moduleName = node.arguments[0].value - isNpmDep = /^[^/.]/.test(moduleName) - resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: moduleName} - dep = _.assign(resolved, {isNpmModule: isNpmDep}) - if dep.filename not of build.processed or - isNpmDep and bundle.checkNpmModules - worklist.push dep - deps.push dep + rewriteRequire = false + if not (isCore(moduleName)) or build.node + rewriteRequire = true + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: moduleName} + # Only include an external dep if its not a core module or + # we are emulating a node.js environment + isNpmDep = isNpmModule or /^[^/.]/.test(moduleName) + dep = _.assign(resolved, {isNpmModule: isNpmDep}) + if dep.filename not of build.processed or + isNpmDep and build.checkNpmModules + worklist.push dep + deps.push dep catch e if build.ignoreMissing return { type: 'Literal', value: null } @@ -150,20 +156,22 @@ module.exports = (build) -> throw e # rewrite the require to use the root-relative path or the uid if # enabled - { - type: 'CallExpression' - callee: node.callee - arguments: [{ - type: 'Literal' - value: uidFor(resolved.canonicalName) - }, { - type: 'Identifier' - name: 'module' - }] - } + if rewriteRequire + return { + type: 'CallExpression' + callee: node.callee + arguments: [{ + type: 'Literal' + value: build.uidFor(dep.canonicalName) + }, { + type: 'Identifier' + name: 'module' + }] + } + return map = null - if isNpmModule or build.npmSourceMaps + if not isNpmModule or build.npmSourceMaps {code, map} = escodegen.generate ast, sourceMap: true format: escodegen.FORMAT_DEFAULTS @@ -174,6 +182,7 @@ module.exports = (build) -> code = escodegen.generate ast, sourceMap: false format: escodegen.FORMAT_DEFAULTS + comment: true # cache linecount for a little more efficiency when calculating offsets # later diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index f0e3176..a2db051 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -23,8 +23,9 @@ module.exports = (grunt) -> opts.output = f.dest build = new Powerbuild(opts) start = new Date().getTime() + grunt.log.ok("Build started...") {code, map} = build.bundle() - console.error("took #{new Date().valueOf() - start} ms") + console.error("Completed in #{new Date().getTime() - start} ms") grunt.file.write build.output, code grunt.log.ok("Created #{build.output}") if build.sourceMap From a0f4d20e26e79e0795518467d2ea050566e2e8c2 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sun, 6 Oct 2013 20:32:16 -0300 Subject: [PATCH 16/31] Add support for loading in node.js environments --- src/bundle.coffee | 37 ++++++++++++++++++-------------- src/traverse-dependencies.coffee | 3 ++- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/bundle.coffee b/src/bundle.coffee index 95f2424..be69886 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -1,11 +1,9 @@ path = require 'path' {SourceMapConsumer, SourceMapGenerator} = require 'source-map' {btoa} = require 'Base64' -escodegen = require 'escodegen' UglifyJS = require 'uglify-js' sourceMapToAst = require './sourcemap-to-ast' -canonicalise = require './canonicalise' PRELUDE_NODE = """ (function() { var global = this; @@ -26,16 +24,23 @@ PRELUDE_NODE = """ PRELUDE = """ (function() { - function require(file, parentModule) { - if({}.hasOwnProperty.call(require.cache, file)) - return require.cache[file]; + var outer; + if (typeof require === 'function') { + outer = require; + } + function inner(file, parentModule) { + if({}.hasOwnProperty.call(inner.cache, file)) + return inner.cache[file]; - var resolved = require.resolve(file); - if(!resolved) throw new Error('Failed to resolve module ' + file); + var resolved = inner.resolve(file); + if(!resolved && outer) { + return inner.cache[file] = outer(file); + } + if(!resolved) throw new Error("Failed to resolve module '" + file + "'"); var module$ = { id: file, - require: require, + require: inner, filename: file, exports: {}, loaded: false, @@ -45,21 +50,21 @@ PRELUDE = """ if(parentModule) parentModule.children.push(module$); var dirname = file.slice(0, file.lastIndexOf('/') + 1); - require.cache[file] = module$.exports; + inner.cache[file] = module$.exports; resolved.call(this, module$, module$.exports, dirname, file); module$.loaded = true; - return require.cache[file] = module$.exports; + return inner.cache[file] = module$.exports; } - require.modules = {}; - require.cache = {}; + inner.modules = {}; + inner.cache = {}; - require.resolve = function(file){ - return {}.hasOwnProperty.call(require.modules, file) ? require.modules[file] : void 0; + inner.resolve = function(file){ + return {}.hasOwnProperty.call(inner.modules, file) ? inner.modules[file] : void 0; }; - require.define = function(file, fn){ require.modules[file] = fn; }; + inner.define = function(file, fn){ inner.modules[file] = fn; }; - return require; + return inner; })() """ diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index ca8dccf..7f40d4b 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -173,6 +173,7 @@ module.exports = (build) -> map = null if not isNpmModule or build.npmSourceMaps {code, map} = escodegen.generate ast, + comment: true sourceMap: true format: escodegen.FORMAT_DEFAULTS sourceMapWithCode: true @@ -180,9 +181,9 @@ module.exports = (build) -> map = map.toString() else code = escodegen.generate ast, + comment: true sourceMap: false format: escodegen.FORMAT_DEFAULTS - comment: true # cache linecount for a little more efficiency when calculating offsets # later From 8be8895ad9ef75f7e88e2b5a8d6157387c74c4a4 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Sun, 6 Oct 2013 22:47:23 -0300 Subject: [PATCH 17/31] Add support for global Buffer/process/setImmediate with on-demand inclusion --- package.json | 4 ++- src/bundle.coffee | 46 +++++++++++++++++++------------ src/relative-resolve.coffee | 15 +++++++--- src/traverse-dependencies.coffee | 42 +++++++++++++++++++++++++++- test/dependency-resolution.coffee | 3 +- test/process-spec.coffee | 10 ------- 6 files changed, 86 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index 6d8c3f3..f1c35c9 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,9 @@ "uglify-js": "~2.4.0", "estraverse": "~1.3.1", "escodegen": "0.0.27", - "esprima": "~1.0.4" + "esprima": "~1.0.4", + "escope": "~1.0.0", + "setimmediate": "~1.0.1" }, "devDependencies": { "scopedfs": "~0.1.0", diff --git a/src/bundle.coffee b/src/bundle.coffee index be69886..9cffd7e 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -5,7 +5,7 @@ UglifyJS = require 'uglify-js' sourceMapToAst = require './sourcemap-to-ast' -PRELUDE_NODE = """ +PROCESS = """ (function() { var global = this; var cwd = '/'; return { @@ -15,14 +15,14 @@ PRELUDE_NODE = """ env: {}, on: function() {}, argv: [], - nextTick: global.setImmediate || function(fn){ setTimeout(fn, 0); }, + nextTick: setImmediate, cwd: function(){ return cwd; }, chdir: function(dir){ cwd = dir; } }; })() """ -PRELUDE = """ +REQUIRE = """ (function() { var outer; if (typeof require === 'function') { @@ -71,13 +71,7 @@ PRELUDE = """ wrap = (modules) -> """ (function(require, undefined) { var global = this; #{modules} - })(#{PRELUDE}) - """ - -wrapNode = (modules) -> """ - (function(require, process, undefined) { var global = this; - #{modules} - })(#{PRELUDE}, #{PRELUDE_NODE}) + })(#{REQUIRE}) """ wrapUmd = (exports, commonjs) -> """ @@ -103,12 +97,18 @@ bundle = (build) -> file: path.basename(build.output) sourceRoot: build.sourceMapRoot lineOffset = umdOffset - - for own filename, {id, canonicalName, code, map, lineCount} of build.processed + useProcess = false + bufferPath = false + setImmediatePath = false + + for own filename, {id, canonicalName, code, map, lineCount, nodeFeatures} of build.processed + useProcess = useProcess or nodeFeatures.process + setImmediatePath = setImmediatePath or nodeFeatures.setImmediate + bufferPath = bufferPath or nodeFeatures.Buffer if typeof id != 'number' id = "'#{id}'" result += """ - \nrequire.define(#{id}, function(module, exports, __dirname, __filename){ + \nrequire.define(#{id}, function(module, exports, __dirname, __filename, undefined){ #{code} }); """ @@ -127,6 +127,21 @@ bundle = (build) -> name: m.name lineOffset += lineCount + if bufferPath + {id} = build.processed[bufferPath] + if typeof id != 'number' + id = "'#{id}'" + result += "\nvar Buffer = require(#{id});" + + if setImmediatePath + {id} = build.processed[setImmediatePath] + if typeof id != 'number' + id = "'#{id}'" + result += "\nrequire(#{id});" + + if useProcess and build.node + result += "\nvar process = #{PROCESS};" + for i in [0...build.entryPoints.length] entryPoint = build.entryPoints[i] {id} = build.processed[entryPoint] @@ -143,10 +158,7 @@ bundle = (build) -> else exports = '' - if build.node - commonjs = wrapNode(result) - else - commonjs = wrap(result) + commonjs = wrap(result) result = wrapUmd(exports, commonjs) diff --git a/src/relative-resolve.coffee b/src/relative-resolve.coffee index 134984d..cfe1a19 100644 --- a/src/relative-resolve.coffee +++ b/src/relative-resolve.coffee @@ -7,7 +7,11 @@ CORE_MODULES = require './core-modules' isCore = require './is-core' canonicalise = require './canonicalise' -resolvePath = ({extensions, aliases, root, cwd, path: givenPath}) -> +resolvePath = ({extensions, aliases, root, cwd, path: givenPath}, pkgMainField = 'browser') -> + packageFilter = (pkg) -> + if pkg[pkgMainField] + pkg.main = pkg[pkgMainField] + return pkg aliases ?= {} if isCore givenPath return if {}.hasOwnProperty.call aliases, givenPath @@ -16,11 +20,14 @@ resolvePath = ({extensions, aliases, root, cwd, path: givenPath}) -> throw new Error "Core module \"#{givenPath}\" has not yet been ported to the browser" givenPath = corePath # try regular CommonJS requires - try resolve givenPath, {extensions, basedir: cwd or root} + try + resolve givenPath, {extensions, basedir: cwd or root, packageFilter} catch e # support non-standard root-relative requires - try resolve (path.join root, givenPath), {extensions} - catch e then throw new Error "Cannot find module \"#{givenPath}\" in \"#{root}\"" + try + resolve (path.join root, givenPath), {extensions, packageFilter} + catch e + throw new Error "Cannot find module \"#{givenPath}\" in \"#{root}\"" module.exports = ({extensions, aliases, root, cwd, path: givenPath}) -> aliases ?= {} diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 7f40d4b..8a1dfcb 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -6,12 +6,20 @@ _ = require 'lodash' esprima = require 'esprima' estraverse = require 'estraverse' escodegen = require 'escodegen' +escope = require 'escope' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' sourceMapToAst = require './sourcemap-to-ast' isCore = require './is-core' + +isImplicit = (name, scope) -> + _.any scope.scopes, (scope) -> + _.any scope.references, (reference) -> + reference.identifier.name == name && not reference.resolved + + badRequireError = (filename, node, msg) -> if node.loc? and node.loc?.start? filename = "#{filename}:#{node.loc.start.line}:#{node.loc.start.column}" @@ -24,6 +32,11 @@ badRequireError = (filename, node, msg) -> module.exports = (build) -> aliases = build.aliases ? {} root = build.root + globalFeatures = { + setImmediate: false + process: false + Buffer: false + } worklist = [] resolvedEntryPoints = [] @@ -108,6 +121,7 @@ module.exports = (build) -> # add source file information to the AST root node ast.loc ?= {} + scope = escope.analyze ast deps = [] id = build.uidFor(canonicalName) @@ -170,6 +184,31 @@ module.exports = (build) -> } return + nodeFeatures = { + setImmediate: isImplicit 'setImmediate', scope + process: isImplicit 'process', scope + Buffer: isImplicit 'Buffer', scope + __filename: isImplicit '__filename', scope + __dirname: isImplicit '__dirname', scope + } + + baseDir = path.dirname path.resolve __dirname + if nodeFeatures.process + globalFeatures.process = true + + if not globalFeatures.setImmediate and nodeFeatures.setImmediate or + nodeFeatures.process + globalFeatures.setImmediate = true + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'setimmediate'} + nodeFeatures.setImmediate = resolved.filename + worklist.unshift(resolved) + + if not globalFeatures.Buffer and nodeFeatures.Buffer + globalFeatures.Buffer = true + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'buffer-browserify'} + nodeFeatures.Buffer = resolved.filename + worklist.unshift(resolved) + map = null if not isNpmModule or build.npmSourceMaps {code, map} = escodegen.generate ast, @@ -188,6 +227,7 @@ module.exports = (build) -> # cache linecount for a little more efficiency when calculating offsets # later lineCount = code.split('\n').length - processed[filename] = {id, canonicalName, code, map, lineCount, mtime, deps} + processed[filename] = {id, canonicalName, code, map, lineCount, mtime, + deps, nodeFeatures} processed diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index b5486ac..c8808e6 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -80,4 +80,5 @@ suite 'Dependency Resolution', -> fixtures 'a.js': 'require("fs")' arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js', aliases: {fs: 'freelist'} fixtures 'a.js': 'require("path")' - arrayEq ['a.js', '../node/lib/path.js', '../node/lib/util.js'], deps 'a.js', aliases: {child_process: null, fs: null} + arrayEq ['a.js', '../node/lib/path.js', '../node/lib/util.js' + '../node_modules/setimmediate/setImmediate.js'], deps 'a.js', aliases: {child_process: null, fs: null} diff --git a/test/process-spec.coffee b/test/process-spec.coffee index b7fd853..35fce37 100644 --- a/test/process-spec.coffee +++ b/test/process-spec.coffee @@ -39,13 +39,3 @@ suite 'Process Spec', -> fixtures '/a.js': 'module.exports = process.nextTick' indicator = -> eq indicator, bundleEval 'a.js', null, setImmediate: indicator - - test 'process.nextTick should use setTimeout if setImmediate is not available', -> - fixtures '/a.js': 'process.nextTick(setTimeout)' - called = no - indicator = (fn, delay) -> - called = yes - eq indicator, fn - eq 0, delay - bundleEval 'a.js', null, setImmediate: null, setTimeout: indicator - ok called From cb3f627b9d3281438e38195ebf60b79092fefd07 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Mon, 7 Oct 2013 09:54:17 -0300 Subject: [PATCH 18/31] Replace all module canonical paths by uids. When a module uses __filename or __dirname a map will be added for that module --- src/bundle.coffee | 119 ++++++++++++++++--------------- src/command.coffee | 6 +- src/index.coffee | 10 ++- src/traverse-dependencies.coffee | 4 +- test/module-spec.coffee | 2 +- 5 files changed, 70 insertions(+), 71 deletions(-) diff --git a/src/bundle.coffee b/src/bundle.coffee index 9cffd7e..fcb5b8f 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -22,56 +22,61 @@ PROCESS = """ })() """ -REQUIRE = """ -(function() { - var outer; - if (typeof require === 'function') { - outer = require; - } - function inner(file, parentModule) { - if({}.hasOwnProperty.call(inner.cache, file)) - return inner.cache[file]; - - var resolved = inner.resolve(file); - if(!resolved && outer) { - return inner.cache[file] = outer(file); +commonjs = (filenameMap) -> """ + (function() { + var files = #{JSON.stringify(filenameMap)}; + var outer; + if (typeof require === 'function') { + outer = require; + } + function inner(id, parentModule) { + if({}.hasOwnProperty.call(inner.cache, id)) + return inner.cache[id]; + + var resolved = inner.resolve(id); + if(!resolved && outer) { + return inner.cache[id] = outer(id); + } + if(!resolved) throw new Error("Failed to resolve module '" + id + "'"); + + var dirname; + var filename = files[id] || ''; + if (filename) + dirname = filename.slice(0, filename.lastIndexOf('/') + 1); + else + dirname = ''; + var module$ = { + id: id, + require: inner, + exports: {}, + loaded: false, + parent: parentModule, + children: [] + }; + if(parentModule) parentModule.children.push(module$); + + inner.cache[id] = module$.exports; + resolved.call(this, module$, module$.exports, dirname, filename); + module$.loaded = true; + return inner.cache[id] = module$.exports; } - if(!resolved) throw new Error("Failed to resolve module '" + file + "'"); - - var module$ = { - id: file, - require: inner, - filename: file, - exports: {}, - loaded: false, - parent: parentModule, - children: [] - }; - if(parentModule) parentModule.children.push(module$); - var dirname = file.slice(0, file.lastIndexOf('/') + 1); - - inner.cache[file] = module$.exports; - resolved.call(this, module$, module$.exports, dirname, file); - module$.loaded = true; - return inner.cache[file] = module$.exports; - } - inner.modules = {}; - inner.cache = {}; + inner.modules = {}; + inner.cache = {}; - inner.resolve = function(file){ - return {}.hasOwnProperty.call(inner.modules, file) ? inner.modules[file] : void 0; - }; - inner.define = function(file, fn){ inner.modules[file] = fn; }; + inner.resolve = function(id){ + return {}.hasOwnProperty.call(inner.modules, id) ? inner.modules[id] : void 0; + }; + inner.define = function(id, fn){ inner.modules[id] = fn; }; - return inner; -})() -""" + return inner; + })() + """ -wrap = (modules) -> """ +wrap = (modules, commonjs) -> """ (function(require, undefined) { var global = this; #{modules} - })(#{REQUIRE}) + })(#{commonjs}) """ wrapUmd = (exports, commonjs) -> """ @@ -101,14 +106,16 @@ bundle = (build) -> bufferPath = false setImmediatePath = false + files = {} + for own filename, {id, canonicalName, code, map, lineCount, nodeFeatures} of build.processed + if nodeFeatures.__filename or nodeFeatures.__dirname + files[id] = canonicalName useProcess = useProcess or nodeFeatures.process setImmediatePath = setImmediatePath or nodeFeatures.setImmediate bufferPath = bufferPath or nodeFeatures.Buffer - if typeof id != 'number' - id = "'#{id}'" result += """ - \nrequire.define(#{id}, function(module, exports, __dirname, __filename, undefined){ + \nrequire.define('#{id}', function(module, exports, __dirname, __filename, undefined){ #{code} }); """ @@ -129,15 +136,11 @@ bundle = (build) -> if bufferPath {id} = build.processed[bufferPath] - if typeof id != 'number' - id = "'#{id}'" - result += "\nvar Buffer = require(#{id});" + result += "\nvar Buffer = require('#{id}');" if setImmediatePath {id} = build.processed[setImmediatePath] - if typeof id != 'number' - id = "'#{id}'" - result += "\nrequire(#{id});" + result += "\nrequire('#{id}');" if useProcess and build.node result += "\nvar process = #{PROCESS};" @@ -145,22 +148,22 @@ bundle = (build) -> for i in [0...build.entryPoints.length] entryPoint = build.entryPoints[i] {id} = build.processed[entryPoint] - if typeof id != 'number' - id = "'#{id}'" if i == build.entryPoints.length - 1 # export the last entry point - result += "\nreturn require(#{id});" + result += "\nreturn require('#{id}');" else - result += "\nrequire(#{id});" + result += "\nrequire('#{id}');" if build.export exports = "#{build.export} = exported;" else exports = '' - commonjs = wrap(result) + req = commonjs(files) + + cjs = wrap(result, req) - result = wrapUmd(exports, commonjs) + result = wrapUmd(exports, cjs) return {code: result, map: resultMap.toString()} diff --git a/src/command.coffee b/src/command.coffee index e260dcb..7185892 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -11,7 +11,7 @@ knownOpts = {} # options knownOpts[opt] = Boolean for opt in [ 'deps', 'help', 'ignore-missing', 'inline-source-map', 'inline-sources', - 'minify', 'node', 'verbose', 'watch', 'module-uids', 'cache-path' + 'minify', 'node', 'verbose', 'watch', 'cache-path' ] # parameters knownOpts[opt] = String for opt in ['export', 'output', 'root', 'source-map'] @@ -41,7 +41,6 @@ options.sourceMap = options['source-map'] options.inlineSources = options['inline-sources'] options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] -options.moduleUids = options['module-uids'] options.entryPoint = options['entry-point'] if options.help @@ -70,9 +69,6 @@ if options.help --cache-path file where to read/write a json-encoded cache that will be used to speed up future rebuilds. default: '.powerbuild-cache~' in the current directory - --module-uids Instead of replacing module names by their full path, - use unique ids for better minification - (breaks __dirname/__filename) --version display the version number and exit " process.exit 0 diff --git a/src/index.coffee b/src/index.coffee index 605adcb..969f034 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -14,16 +14,16 @@ class Powerbuild options.inlineSources ?= false options.log or= -> options.processed or= {} + options.uids or= {next: 1, names: []} options.checkNpmModules ?= false options.npmSourceMaps ?= false options.bundleNpmModules ?= true - options.moduleUids ?= false options.root or= process.cwd() options.node ?= true {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, - @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @moduleUids, + @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @mainModule, @minify, @aliases, @handlers, @processed, @uids, - @moduleUids, @checkNpmModules, @npmSourceMaps} = options + @checkNpmModules, @npmSourceMaps} = options if @output @sourceMapRoot = path.relative(path.dirname(@output), @root) @@ -55,12 +55,10 @@ class Powerbuild uidFor: (name) -> - if not @moduleUids - return name if not {}.hasOwnProperty.call(@uids.names, name) uid = @uids.next++ @uids.names[name] = uid - @uids.names[name] + return @uids.names[name] module.exports = Powerbuild diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 8a1dfcb..9fa1466 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -176,11 +176,13 @@ module.exports = (build) -> callee: node.callee arguments: [{ type: 'Literal' - value: build.uidFor(dep.canonicalName) + value: build.uidFor(dep.canonicalName).toString() }, { type: 'Identifier' name: 'module' }] + loc: node.loc + range: node.range } return diff --git a/test/module-spec.coffee b/test/module-spec.coffee index e2abcdb..42d280e 100644 --- a/test/module-spec.coffee +++ b/test/module-spec.coffee @@ -20,5 +20,5 @@ suite 'Module Spec', -> test 'module.children contains required modules', -> fixtures '/a.js': 'require("./b"); module.exports = module.children[0].exports' - '/b.js': 'module.exports = module.filename' + '/b.js': 'module.exports = __filename' eq 'b.js', bundleEval 'a.js' From c97453ca4c225338c0ae704430bd1d7ac3ef8f62 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Mon, 7 Oct 2013 11:11:05 -0300 Subject: [PATCH 19/31] Add more options to the cli --- src/build-cache.coffee | 20 ++++++++------ src/bundle.coffee | 11 ++++---- src/command.coffee | 47 +++++++++++++++++++++----------- src/index.coffee | 6 ++-- src/traverse-dependencies.coffee | 44 ++++++++++++++---------------- tasks/powerbuild.coffee | 19 +++++++++---- 6 files changed, 85 insertions(+), 62 deletions(-) diff --git a/src/build-cache.coffee b/src/build-cache.coffee index 538701e..3591b5e 100644 --- a/src/build-cache.coffee +++ b/src/build-cache.coffee @@ -2,26 +2,28 @@ fs = require 'fs' path = require 'path' +caches = {} + module.exports = (cachePath = path.join(process.cwd(), '.powerbuild~')) -> - process.on 'exit', -> - fs.writeFileSync cachePath, JSON.stringify cache + if {}.hasOwnProperty.call caches, cachePath + return caches[cachePath] - process.on 'SIGINT', process.exit - process.on 'SIGTERM', process.exit + process.on 'exit', -> + fs.writeFileSync cachePath, JSON.stringify caches[cachePath] process.on 'uncaughtException', (e) -> # An exception may be thrown due to corrupt cache or incompatibilities # between versions, remove it to be safe try fs.unlinkSync cachePath - cache.processed = {} + caches[cachePath].processed = {} throw e if fs.existsSync cachePath - cache = JSON.parse fs.readFileSync cachePath, 'utf8' + caches[cachePath] = JSON.parse fs.readFileSync cachePath, 'utf8' - if not cache - cache = + if not caches[cachePath] + caches[cachePath] = processed: {} uids: {next: 1, names: []} - return cache + return caches[cachePath] diff --git a/src/bundle.coffee b/src/bundle.coffee index fcb5b8f..bbe59ef 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -108,7 +108,7 @@ bundle = (build) -> files = {} - for own filename, {id, canonicalName, code, map, lineCount, nodeFeatures} of build.processed + for own filename, {id, canonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of build.processed if nodeFeatures.__filename or nodeFeatures.__dirname files[id] = canonicalName useProcess = useProcess or nodeFeatures.process @@ -120,7 +120,7 @@ bundle = (build) -> }); """ lineOffset += 2 # skip linefeed plus the 'require.define' line - if map + if build.npmSourceMaps or not isNpmModule orig = new SourceMapConsumer map orig.eachMapping (m) -> resultMap.addMapping @@ -173,10 +173,9 @@ module.exports = (build) -> if build.minify uglifyAst = UglifyJS.parse code - # Enabling the compressor seems to break the source map, leave commented - # until a solution is found - # uglifyAst.figure_out_scope() - # uglifyAst = uglifyAst.transform UglifyJS.Compressor warnings: false + if build.compress + uglifyAst.figure_out_scope() + uglifyAst = uglifyAst.transform UglifyJS.Compressor warnings: false uglifyAst.figure_out_scope() uglifyAst.compute_char_frequency() uglifyAst.mangle_names() diff --git a/src/command.coffee b/src/command.coffee index 7185892..66ce304 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -11,7 +11,8 @@ knownOpts = {} # options knownOpts[opt] = Boolean for opt in [ 'deps', 'help', 'ignore-missing', 'inline-source-map', 'inline-sources', - 'minify', 'node', 'verbose', 'watch', 'cache-path' + 'minify', 'compress', 'node', 'verbose', 'watch', 'cache-path', + '--disable-disk-cache', '--npm-source-maps' ] # parameters knownOpts[opt] = String for opt in ['export', 'output', 'root', 'source-map'] @@ -22,6 +23,7 @@ optAliases = a: '--alias' h: '--handler' m: '--minify' + c: '--compress' o: '--output' r: '--root' s: '--source-map' @@ -34,41 +36,49 @@ options.entryPoints = entryPoints = _.uniq options.argv.remain delete options.argv # default values -options['cache-path'] ?= '.powerbuild-cache~' - options.ignoreMissing = options['ignore-missing'] options.sourceMap = options['source-map'] options.inlineSources = options['inline-sources'] options.inlineSourceMap = options['inline-source-map'] options.cachePath = options['cache-path'] -options.entryPoint = options['entry-point'] +options.disableDiskCache = options['cache-path'] +options.npmSourceMaps = options['npm-source-maps'] if options.help $0 = if process.argv[0] is 'node' then process.argv[1] else process.argv[0] $0 = path.basename $0 console.log " - Usage: #{$0} OPT* path/to/entry-file.ext OPT* + Usage: #{$0} OPT* ENTRY_FILE+ OPT* - -e, --main main module to export/initialize when multiple - files are specified -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO -h, --handler EXT:MODULE handle files with extension EXT with module MODULE - -m, --minify minify output + -m, --minify minify output using uglify.js + -c, --compress Compress/optimize code when minifying + (automatically enabled by this option). Enabling + will break the generated source map. -o, --output FILE output to FILE instead of stdout -r, --root DIR unqualified requires are relative to DIR; default: cwd -s, --source-map FILE output a source map to FILE -v, --verbose verbose output sent to stderr -w, --watch watch input files/dependencies for changes and rebuild bundle - -x, --export NAME export the given entry module as NAME + -x, --export NAME export the last given entry module as NAME --deps do not bundle; just list the files that would be bundled --help display this help message and exit --ignore-missing continue without error when dependency resolution fails --inline-source-map include the source map as a data URI in the generated bundle --inline-sources include source content in generated source maps; default: on - --node include process object; emulate node environment; default: on - --cache-path file where to read/write a json-encoded cache that will be - used to speed up future rebuilds. default: - '.powerbuild-cache~' in the current directory + --node if needed by any module, emulate a node.js + environment by including globals such as Buffer, + process and setImmediate; default: on + --cache-path file where to read/write a json-encoded cache that + is used for fast, incremental builds. + default: '.powerbuild-cache~' in the current + directory + --disable-disk-cache disables persistence of incremental build cache + to disk. Incremental build will only work with the + --watch option + --npm-source-maps add mappings for npm modules in the resulting + source map(significantly increases the build time) --version display the version number and exit " process.exit 0 @@ -120,15 +130,20 @@ buildBundle = -> else process.stdout.write "#{code}\n" -cache = buildCache() -options.processed = cache.processed -options.uids = cache.uids +if not options.disableDiskCache + cache = buildCache(options.cachePath) + options.processed = cache.processed + options.uids = cache.uids + build = new Powerbuild options startBuild = -> buildBundle() if options.watch + process.on 'SIGINT', process.exit + process.on 'SIGTERM', process.exit + console.error("Watching for changes...") # Flush the cache when the user presses CTRL+C or the process is # terminated from outside diff --git a/src/index.coffee b/src/index.coffee index 969f034..fe42d2f 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -14,16 +14,16 @@ class Powerbuild options.inlineSources ?= false options.log or= -> options.processed or= {} + if options.compress + options.minify = true options.uids or= {next: 1, names: []} - options.checkNpmModules ?= false options.npmSourceMaps ?= false - options.bundleNpmModules ?= true options.root or= process.cwd() options.node ?= true {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @mainModule, @minify, @aliases, @handlers, @processed, @uids, - @checkNpmModules, @npmSourceMaps} = options + @npmSourceMaps, @compress} = options if @output @sourceMapRoot = path.relative(path.dirname(@output), @root) diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 9fa1466..675aab1 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -52,7 +52,7 @@ module.exports = (build) -> checked = {} while worklist.length - {filename, canonicalName, isNpmModule} = worklist.pop() + {filename, canonicalName, isNpmModule, isCoreModule} = worklist.pop() # support aliasing to falsey values to omit files continue unless filename @@ -67,8 +67,7 @@ module.exports = (build) -> if processed[filename]?.mtime == mtime # ignore files that have not changed, but also check its dependencies for dep in processed[filename].deps - if dep.isNpmModule and build.checkNpmModules - worklist.push dep + worklist.push dep continue src = (fs.readFileSync filename).toString() @@ -152,16 +151,14 @@ module.exports = (build) -> try moduleName = node.arguments[0].value rewriteRequire = false - if not (isCore(moduleName)) or build.node + if not (isCoreDep = isCoreModule or isCore(moduleName)) or build.node rewriteRequire = true resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd, path: moduleName} # Only include an external dep if its not a core module or # we are emulating a node.js environment isNpmDep = isNpmModule or /^[^/.]/.test(moduleName) - dep = _.assign(resolved, {isNpmModule: isNpmDep}) - if dep.filename not of build.processed or - isNpmDep and build.checkNpmModules - worklist.push dep + dep = _.assign(resolved, {isNpmModule: isNpmDep, isCoreModule: isCoreDep}) + worklist.push dep deps.push dep catch e if build.ignoreMissing @@ -202,34 +199,35 @@ module.exports = (build) -> nodeFeatures.process globalFeatures.setImmediate = true resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'setimmediate'} + resolved = _.extend resolved, isCoreModule: true, isNpmModule: true nodeFeatures.setImmediate = resolved.filename worklist.unshift(resolved) if not globalFeatures.Buffer and nodeFeatures.Buffer globalFeatures.Buffer = true resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'buffer-browserify'} + resolved = _.extend resolved, isCoreModule: true, isNpmModule: true nodeFeatures.Buffer = resolved.filename worklist.unshift(resolved) - map = null - if not isNpmModule or build.npmSourceMaps - {code, map} = escodegen.generate ast, - comment: true - sourceMap: true - format: escodegen.FORMAT_DEFAULTS - sourceMapWithCode: true - sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' - map = map.toString() - else - code = escodegen.generate ast, - comment: true - sourceMap: false - format: escodegen.FORMAT_DEFAULTS + {code, map} = escodegen.generate ast, + comment: true + sourceMap: true + format: escodegen.FORMAT_DEFAULTS + sourceMapWithCode: true + sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' + map = map.toString() # cache linecount for a little more efficiency when calculating offsets # later lineCount = code.split('\n').length processed[filename] = {id, canonicalName, code, map, lineCount, mtime, - deps, nodeFeatures} + deps, nodeFeatures, isNpmModule, isCoreModule} + + # remove old dependencies + for own k, {isCoreModule} of processed + if not (isCoreModule or k of checked) + console.error("deleting #{k}") + delete processed[k] processed diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index a2db051..2e34aa2 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -5,18 +5,27 @@ buildCache = require '../src/build-cache' NAME = 'powerbuild' -DESC = 'Builds commonjs projects to run anywhere, transpiling to javascript if -necessary, besides generating concatenated source maps.' +DESC = 'Wraps commonjs projects into single umd function that will run + anywhere, generating concatenated source maps to debug files individually.' -cache = buildCache() +initialized = false + +initSignalHandlers = -> + if initialized then return + initialized = true + process.on 'SIGINT', process.exit + process.on 'SIGTERM', process.exit module.exports = (grunt) -> grunt.registerMultiTask NAME, DESC, -> options = @options() - options.processed = cache.processed - options.uids = cache.uids + if not options.disableDiskCache + initSignalHandlers() + cache = buildCache(options.cachePath) + options.processed = cache.processed + options.uids = cache.uids for f in @files opts = _.clone(options) opts.entryPoints = grunt.file.expand(f.orig.src) From 20b99107be87faabfac51e29fe0e005c514872c8 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Mon, 7 Oct 2013 12:35:32 -0300 Subject: [PATCH 20/31] Refactor gruntfile to run tests from self-generated node.js bundle --- .gitignore | 2 ++ Gruntfile.coffee | 7 ++++--- src/bundle.coffee | 2 ++ src/traverse-dependencies.coffee | 9 +++++---- test/dependency-resolution.coffee | 8 -------- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 361440b..e670244 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ commonjs-everywhere-*.tgz test/fixtures/* /lib/ +tests.js +tests.js.map diff --git a/Gruntfile.coffee b/Gruntfile.coffee index c3c1cf6..7cf5f22 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -7,10 +7,11 @@ module.exports = (grunt) -> powerbuild: options: sourceMap: true - ignoreMissing: true + node: false all: files: [ - {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'bundle.js'} + {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'tests.js'} + # {src: 'src/index', dest: 'lib/main.js'} ] mocha_debug: @@ -20,7 +21,7 @@ module.exports = (grunt) -> check: ['test-setup.coffee', 'src/*.coffee', 'test/*.coffee'] nodejs: options: - src: ['test-setup.coffee', 'test/*.coffee'] + src: ['tests.js'] watch: options: diff --git a/src/bundle.coffee b/src/bundle.coffee index bbe59ef..26964a6 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -41,6 +41,8 @@ commonjs = (filenameMap) -> """ var dirname; var filename = files[id] || ''; + if (filename && typeof __dirname === 'string') + filename = __dirname + '/' + filename; if (filename) dirname = filename.slice(0, filename.lastIndexOf('/') + 1); else diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 675aab1..771f72a 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -161,10 +161,12 @@ module.exports = (build) -> worklist.push dep deps.push dep catch e + rv = undefined if build.ignoreMissing - return { type: 'Literal', value: null } - else - throw e + rv = { type: 'Literal', value: null } + rv.loc = node.loc + rv.range = node.range + return rv # rewrite the require to use the root-relative path or the uid if # enabled if rewriteRequire @@ -227,7 +229,6 @@ module.exports = (build) -> # remove old dependencies for own k, {isCoreModule} of processed if not (isCoreModule or k of checked) - console.error("deleting #{k}") delete processed[k] processed diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index c8808e6..27b1ade 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -51,14 +51,6 @@ suite 'Dependency Resolution', -> fixtures 'a.js': 'require("freelist")' arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js' - test 'missing dependencies', -> - fixtures 'a.js': 'require("./b")' - throws -> deps 'a.js' - - test 'ignoreMissing option ignores missing dependencies', -> - fixtures 'a.js': 'require("./b")' - arrayEq ['a.js'], deps 'a.js', ignoreMissing: yes - suite 'Aliasing', -> test 'basic alias', -> From 9dca92d83e5c88d8fb0a62f67c7a4be052dacf37 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Mon, 7 Oct 2013 14:38:18 -0300 Subject: [PATCH 21/31] Refactored so that the disk cache may be reused with many configurations --- Gruntfile.coffee | 4 ++-- package.json | 4 ++-- src/build-cache.coffee | 7 +++++++ src/bundle.coffee | 16 ++++++++-------- src/command.coffee | 3 --- src/index.coffee | 12 +++++------- src/traverse-dependencies.coffee | 28 +++++++++++++++------------- tasks/powerbuild.coffee | 20 +++++--------------- test-setup.coffee | 1 + test/dependency-resolution.coffee | 5 ++--- 10 files changed, 47 insertions(+), 53 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 7cf5f22..45c9616 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -11,7 +11,7 @@ module.exports = (grunt) -> all: files: [ {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'tests.js'} - # {src: 'src/index', dest: 'lib/main.js'} + {src: 'src/index.coffee', dest: 'lib/main.js'} ] mocha_debug: @@ -19,7 +19,7 @@ module.exports = (grunt) -> ui: 'tdd' reporter: 'dot' check: ['test-setup.coffee', 'src/*.coffee', 'test/*.coffee'] - nodejs: + all: options: src: ['tests.js'] diff --git a/package.json b/package.json index f1c35c9..00ec2c6 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,6 @@ "source-map": "~0.1.30", "handlebars": "~1.0.12", "coffee-script": "~1.6.3", - "source-map-support": "~0.2.3", "lodash": "~2.2.1", "uglify-js": "~2.4.0", "estraverse": "~1.3.1", @@ -68,7 +67,8 @@ "grunt": "~0.4.1", "grunt-release": "~0.6.0", "grunt-mocha-debug": "0.0.6", - "grunt-contrib-watch": "~0.5.3" + "grunt-contrib-watch": "~0.5.3", + "source-map-support": "~0.2.3" }, "scripts": { "test": "make test" diff --git a/src/build-cache.coffee b/src/build-cache.coffee index 3591b5e..9ebf846 100644 --- a/src/build-cache.coffee +++ b/src/build-cache.coffee @@ -1,6 +1,13 @@ fs = require 'fs' path = require 'path' +initialized = false + +initSignalHandlers = -> + if initialized then return + initialized = true + process.on 'SIGINT', process.exit + process.on 'SIGTERM', process.exit caches = {} diff --git a/src/bundle.coffee b/src/bundle.coffee index 26964a6..f82d2b0 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -98,7 +98,7 @@ wrapUmd = (exports, commonjs) -> """ umdOffset = wrapUmd('', '').split('\n').length -bundle = (build) -> +bundle = (build, processed) -> result = '' resultMap = new SourceMapGenerator file: path.basename(build.output) @@ -110,7 +110,7 @@ bundle = (build) -> files = {} - for own filename, {id, canonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of build.processed + for own filename, {id, canonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of processed if nodeFeatures.__filename or nodeFeatures.__dirname files[id] = canonicalName useProcess = useProcess or nodeFeatures.process @@ -137,11 +137,11 @@ bundle = (build) -> lineOffset += lineCount if bufferPath - {id} = build.processed[bufferPath] + {id} = processed[bufferPath] result += "\nvar Buffer = require('#{id}');" if setImmediatePath - {id} = build.processed[setImmediatePath] + {id} = processed[setImmediatePath] result += "\nrequire('#{id}');" if useProcess and build.node @@ -149,7 +149,7 @@ bundle = (build) -> for i in [0...build.entryPoints.length] entryPoint = build.entryPoints[i] - {id} = build.processed[entryPoint] + {id} = processed[entryPoint] if i == build.entryPoints.length - 1 # export the last entry point result += "\nreturn require('#{id}');" @@ -170,8 +170,8 @@ bundle = (build) -> return {code: result, map: resultMap.toString()} -module.exports = (build) -> - {code, map} = bundle build +module.exports = (build, processed) -> + {code, map} = bundle build, processed if build.minify uglifyAst = UglifyJS.parse code @@ -190,7 +190,7 @@ module.exports = (build) -> map = sm.toString() if (build.sourceMap or build.inlineSourceMap) and build.inlineSources - for own filename, {code: src, canonicalName} of build.processed + for own filename, {code: src, canonicalName} of processed map.setSourceContent canonicalName, src sourceMappingUrl = diff --git a/src/command.coffee b/src/command.coffee index 66ce304..75e1e16 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -141,9 +141,6 @@ startBuild = -> buildBundle() if options.watch - process.on 'SIGINT', process.exit - process.on 'SIGTERM', process.exit - console.error("Watching for changes...") # Flush the cache when the user presses CTRL+C or the process is # terminated from outside diff --git a/src/index.coffee b/src/index.coffee index fe42d2f..08136b0 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -12,15 +12,13 @@ class Powerbuild throw new Error('Can only set the export option with one entry point') options.inlineSources ?= false - options.log or= -> - options.processed or= {} if options.compress options.minify = true options.uids or= {next: 1, names: []} options.npmSourceMaps ?= false options.root or= process.cwd() options.node ?= true - {@output, @export, @entryPoints, @root, @node, @log, @inlineSources, + {@output, @export, @entryPoints, @root, @node, @inlineSources, @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @mainModule, @minify, @aliases, @handlers, @processed, @uids, @npmSourceMaps, @compress} = options @@ -44,14 +42,14 @@ class Powerbuild bundle: -> - @traverseDependencies() - return bundle this + return bundle this, @traverseDependencies() traverseDependencies: -> - traverseDependencies this + processed = traverseDependencies this, @processed if @verbose - @log "Included modules: #{(Object.keys @processed).sort()}" + console.error "Included modules: #{(Object.keys processed).sort()}" + return processed uidFor: (name) -> diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 771f72a..e1f33de 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -29,7 +29,7 @@ badRequireError = (filename, node, msg) -> in #{filename} """ -module.exports = (build) -> +module.exports = (build, processedCache) -> aliases = build.aliases ? {} root = build.root globalFeatures = { @@ -48,7 +48,11 @@ module.exports = (build) -> build.entryPoints = resolvedEntryPoints - processed = build.processed + if processedCache + processed = _.clone processedCache + else + processed = {} + checked = {} while worklist.length @@ -186,26 +190,22 @@ module.exports = (build) -> return nodeFeatures = { - setImmediate: isImplicit 'setImmediate', scope - process: isImplicit 'process', scope - Buffer: isImplicit 'Buffer', scope __filename: isImplicit '__filename', scope __dirname: isImplicit '__dirname', scope } baseDir = path.dirname path.resolve __dirname - if nodeFeatures.process - globalFeatures.process = true + if isImplicit 'process', scope + nodeFeatures.process = globalFeatures.process = true - if not globalFeatures.setImmediate and nodeFeatures.setImmediate or - nodeFeatures.process + if not globalFeatures.setImmediate and (isImplicit 'setImmediate', scope) or nodeFeatures.process globalFeatures.setImmediate = true resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'setimmediate'} resolved = _.extend resolved, isCoreModule: true, isNpmModule: true nodeFeatures.setImmediate = resolved.filename worklist.unshift(resolved) - if not globalFeatures.Buffer and nodeFeatures.Buffer + if not globalFeatures.Buffer and isImplicit 'Buffer', scope globalFeatures.Buffer = true resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'buffer-browserify'} resolved = _.extend resolved, isCoreModule: true, isNpmModule: true @@ -225,10 +225,12 @@ module.exports = (build) -> lineCount = code.split('\n').length processed[filename] = {id, canonicalName, code, map, lineCount, mtime, deps, nodeFeatures, isNpmModule, isCoreModule} + if processedCache + processedCache[filename] = processed[filename] - # remove old dependencies - for own k, {isCoreModule} of processed + # remove old dependencies and update the cache + for own k, {isCoreModule, nodeFeatures} of processed if not (isCoreModule or k of checked) delete processed[k] - processed + return processed diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index 2e34aa2..9d101bc 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -5,29 +5,19 @@ buildCache = require '../src/build-cache' NAME = 'powerbuild' -DESC = 'Wraps commonjs projects into single umd function that will run +DESC = 'Wraps node.js/commonjs projects into single umd function that will run anywhere, generating concatenated source maps to debug files individually.' -initialized = false - -initSignalHandlers = -> - if initialized then return - initialized = true - process.on 'SIGINT', process.exit - process.on 'SIGTERM', process.exit - - module.exports = (grunt) -> grunt.registerMultiTask NAME, DESC, -> options = @options() - if not options.disableDiskCache - initSignalHandlers() - cache = buildCache(options.cachePath) - options.processed = cache.processed - options.uids = cache.uids for f in @files opts = _.clone(options) + if not opts.disableDiskCache + cache = buildCache(opts.cachePath) + opts.processed = cache.processed + opts.uids = cache.uids opts.entryPoints = grunt.file.expand(f.orig.src) opts.output = f.dest build = new Powerbuild(opts) diff --git a/test-setup.coffee b/test-setup.coffee index 8ff6ed0..76468c5 100644 --- a/test-setup.coffee +++ b/test-setup.coffee @@ -1,3 +1,4 @@ +require('source-map-support').install() escodegen = require 'escodegen' fs = require 'scopedfs' path = require 'path' diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index 27b1ade..391c7f6 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -1,5 +1,4 @@ path = require 'path' -# traverseDependencies = require '../src/traverse-dependencies' suite 'Dependency Resolution', -> @@ -9,9 +8,9 @@ suite 'Dependency Resolution', -> opts.root = FIXTURES_DIR opts.entryPoints = [entryFile] powerbuild = new Powerbuild opts - powerbuild.traverseDependencies() + processed = powerbuild.traverseDependencies() rv = [] - for filename in (Object.keys powerbuild.processed).sort() + for filename in (Object.keys processed).sort() if filename[...FIXTURES_DIR.length] is FIXTURES_DIR "#{path.relative FIXTURES_DIR, filename}" else From 938b9965ecdbcf4657b2493a45a83be55b91498a Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 01:26:49 -0300 Subject: [PATCH 22/31] Now considering 'sourceMappingURL' comments to set the actual positions in the result source map --- Gruntfile.coffee | 16 +++++++++++-- package.json | 4 +++- src/build-cache.coffee | 8 +++++++ src/bundle.coffee | 6 ++--- src/index.coffee | 2 ++ src/sourcemap-to-ast.coffee | 2 -- src/traverse-dependencies.coffee | 41 ++++++++++++++------------------ tasks/powerbuild.coffee | 8 +++---- test-setup.coffee | 3 +-- 9 files changed, 53 insertions(+), 37 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index 45c9616..bb8c2c8 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -4,6 +4,17 @@ module.exports = (grunt) -> clean: all: ['build'] + coffee: + options: + sourceMap: true + all: + src: '*.coffee' + dest: 'lib' + cwd: 'src' + flatten: true + expand: true + ext: '.js' + powerbuild: options: sourceMap: true @@ -11,7 +22,6 @@ module.exports = (grunt) -> all: files: [ {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'tests.js'} - {src: 'src/index.coffee', dest: 'lib/main.js'} ] mocha_debug: @@ -44,6 +54,8 @@ module.exports = (grunt) -> grunt.loadNpmTasks('grunt-release') grunt.loadNpmTasks('grunt-mocha-debug') grunt.loadNpmTasks('grunt-contrib-watch') + grunt.loadNpmTasks('grunt-contrib-coffee') + grunt.loadNpmTasks('grunt-newer') - grunt.registerTask('test', ['powerbuild', 'mocha_debug']) + grunt.registerTask('test', ['newer:coffee', 'powerbuild', 'mocha_debug']) grunt.registerTask('default', ['test', 'watch']) diff --git a/package.json b/package.json index 00ec2c6..88bbbe7 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,9 @@ "grunt-release": "~0.6.0", "grunt-mocha-debug": "0.0.6", "grunt-contrib-watch": "~0.5.3", - "source-map-support": "~0.2.3" + "source-map-support": "~0.2.3", + "grunt-newer": "~0.5.4", + "grunt-contrib-coffee": "~0.7.0" }, "scripts": { "test": "make test" diff --git a/src/build-cache.coffee b/src/build-cache.coffee index 9ebf846..3509fde 100644 --- a/src/build-cache.coffee +++ b/src/build-cache.coffee @@ -8,6 +8,12 @@ initSignalHandlers = -> initialized = true process.on 'SIGINT', process.exit process.on 'SIGTERM', process.exit + process.on 'uncaughtException', (e) -> + # to be safe, remove all cache files + for own k of caches + if fs.existsSync(k) + fs.unlinkSync(k) + throw e caches = {} @@ -15,6 +21,8 @@ module.exports = (cachePath = path.join(process.cwd(), '.powerbuild~')) -> if {}.hasOwnProperty.call caches, cachePath return caches[cachePath] + initSignalHandlers() + process.on 'exit', -> fs.writeFileSync cachePath, JSON.stringify caches[cachePath] diff --git a/src/bundle.coffee b/src/bundle.coffee index f82d2b0..1fe0da2 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -110,9 +110,9 @@ bundle = (build, processed) -> files = {} - for own filename, {id, canonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of processed + for own filename, {id, canonicalName, realCanonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of processed if nodeFeatures.__filename or nodeFeatures.__dirname - files[id] = canonicalName + files[id] = realCanonicalName or canonicalName useProcess = useProcess or nodeFeatures.process setImmediatePath = setImmediatePath or nodeFeatures.setImmediate bufferPath = bufferPath or nodeFeatures.Buffer @@ -132,7 +132,7 @@ bundle = (build, processed) -> original: line: m.originalLine or m.generatedLine column: m.originalColumn or m.generatedColumn - source: canonicalName + source: realCanonicalName or canonicalName name: m.name lineOffset += lineCount diff --git a/src/index.coffee b/src/index.coffee index 08136b0..3fd5075 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -23,6 +23,8 @@ class Powerbuild @mainModule, @minify, @aliases, @handlers, @processed, @uids, @npmSourceMaps, @compress} = options + @sourceMapRoot = if @sourceMap? then (path.relative (path.dirname @sourceMap), @root) or '.' + if @output @sourceMapRoot = path.relative(path.dirname(@output), @root) if @sourceMap == true diff --git a/src/sourcemap-to-ast.coffee b/src/sourcemap-to-ast.coffee index 1c9b5e4..cfd3a3f 100644 --- a/src/sourcemap-to-ast.coffee +++ b/src/sourcemap-to-ast.coffee @@ -9,8 +9,6 @@ module.exports = (ast, srcMap) -> enter: (node) -> if not node.type return - if node.type == 'TryStatement' and not node.guardedHandlers - node.guardedHandlers = [] origStart = map.originalPositionFor node.loc.start origEnd = map.originalPositionFor node.loc.end if origStart.source != origEnd.source diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index e1f33de..57088df 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -7,6 +7,7 @@ esprima = require 'esprima' estraverse = require 'estraverse' escodegen = require 'escodegen' escope = require 'escope' +{SourceMapConsumer} = require 'source-map' canonicalise = require './canonicalise' relativeResolve = require './relative-resolve' @@ -87,31 +88,31 @@ module.exports = (build, processedCache) -> astOrJs = {code: astOrJs} adjustWrapperLocation = false + realCanonicalName = null if astOrJs.code? try # wrap into a function so top-level 'return' statements wont break # when parsing astOrJs.code = "(function(){#{astOrJs.code}})()" - ast = esprima.parse astOrJs.code, - loc: yes, comment: true, range: true, tokens: true + ast = esprima.parse astOrJs.code, loc: yes, comment: yes # unwrap the function ast.body = ast.body[0].expression.callee.body.body - # adjust the range/column offsets to ignore the wrapped function + # adjust the column offsets to ignore the wrapped function adjustWrapperLocation = true - # Remove the extra tokens - ast.tokens = ast.tokens.slice(5, ast.tokens.length - 4) # Fix comments/token position info - for t in ast.comments.concat(ast.tokens) - t.range[0] -= 12 - t.range[1] -= 12 - if t.loc.start.line == 1 - t.loc.start.column -= 12 - if t.loc.end.line == 1 - t.loc.end.column -= 12 # Also adjust top node end range/column - ast.range[1] -= 4 ast.loc.end.column -= 4 + lastComment = ast.comments[ast.comments.length - 1] + if lastComment and match = /[#@] sourceMappingURL=(.+)/.exec(lastComment.value) + dn = path.dirname(filename) + mapPath = path.join(dn, match[1]) + m = fs.readFileSync(mapPath, 'utf8') + consumer = new SourceMapConsumer m + sources = consumer.sources + sources[0] = path.resolve(path.join(dn, sources[0])) + realCanonicalName = path.relative(build.sourceMapRoot, sources[0]) + astOrJs.map = m if astOrJs.map sourceMapToAst ast, astOrJs.map catch e @@ -131,15 +132,12 @@ module.exports = (build, processedCache) -> estraverse.replace ast, enter: (node, parents) -> if node.loc? - node.loc.source = canonicalName if node.type != 'Program' and adjustWrapperLocation # Adjust the location info to reflect the removed function wrapper if node.loc.start.line == 1 and node.loc.start.column >= 12 node.loc.start.column -= 12 if node.loc.end.line == 1 and node.loc.end.column >= 12 node.loc.end.column -= 12 - node.range[0] -= 12 - node.range[1] -= 12 # ignore anything that's not a `require` call return unless node.type is 'CallExpression' and node.callee.type is 'Identifier' and node.callee.name is 'require' # illegal requires @@ -169,7 +167,6 @@ module.exports = (build, processedCache) -> if build.ignoreMissing rv = { type: 'Literal', value: null } rv.loc = node.loc - rv.range = node.range return rv # rewrite the require to use the root-relative path or the uid if # enabled @@ -185,7 +182,6 @@ module.exports = (build, processedCache) -> name: 'module' }] loc: node.loc - range: node.range } return @@ -213,7 +209,6 @@ module.exports = (build, processedCache) -> worklist.unshift(resolved) {code, map} = escodegen.generate ast, - comment: true sourceMap: true format: escodegen.FORMAT_DEFAULTS sourceMapWithCode: true @@ -224,13 +219,13 @@ module.exports = (build, processedCache) -> # later lineCount = code.split('\n').length processed[filename] = {id, canonicalName, code, map, lineCount, mtime, - deps, nodeFeatures, isNpmModule, isCoreModule} + deps, nodeFeatures, isNpmModule, isCoreModule, realCanonicalName} if processedCache processedCache[filename] = processed[filename] # remove old dependencies and update the cache - for own k, {isCoreModule, nodeFeatures} of processed - if not (isCoreModule or k of checked) - delete processed[k] + # for own k, {isCoreModule} of processed + # if not (isCoreModule or k of checked) + # delete processed[k] return processed diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index 9d101bc..389c97b 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -22,11 +22,11 @@ module.exports = (grunt) -> opts.output = f.dest build = new Powerbuild(opts) start = new Date().getTime() - grunt.log.ok("Build started...") + grunt.log.writeln("Build started...") {code, map} = build.bundle() - console.error("Completed in #{new Date().getTime() - start} ms") grunt.file.write build.output, code - grunt.log.ok("Created #{build.output}") + grunt.log.writeln("Created #{build.output}") if build.sourceMap grunt.file.write build.sourceMap, map - grunt.log.ok("Created #{build.sourceMap}") + grunt.log.writeln("Created #{build.sourceMap}") + grunt.log.ok("Completed in #{new Date().getTime() - start} ms") diff --git a/test-setup.coffee b/test-setup.coffee index 76468c5..b7305ba 100644 --- a/test-setup.coffee +++ b/test-setup.coffee @@ -32,8 +32,7 @@ sfs.reset = -> fs.mkdirpSync FIXTURES_DIR do sfs.reset -# global[k] = v for own k, v of require './src' -global.Powerbuild = require './src' +global.Powerbuild = require './lib' global.FIXTURES_DIR = FIXTURES_DIR global.path = path global.escodegen = escodegen From bbb7028e21e5d30a669ba223549a3b4d8ad8255f Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 09:08:31 -0300 Subject: [PATCH 23/31] Fixed cache bug due to incurrect uid persistence --- .gitignore | 1 + .npmignore | 2 +- src/build-cache.coffee | 2 +- src/index.coffee | 2 +- src/traverse-dependencies.coffee | 10 ++++++---- tasks/powerbuild.coffee | 5 ++++- test-setup.coffee | 2 +- 7 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index e670244..c44e92d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ test/fixtures/* /lib/ tests.js tests.js.map +powerbuild-cache diff --git a/.npmignore b/.npmignore index 3546f08..00443a9 100644 --- a/.npmignore +++ b/.npmignore @@ -1,4 +1,3 @@ -commonjs-everywhere-*.tgz node node/* !node/lib @@ -6,3 +5,4 @@ src test .gitmodules Makefile +powerbuild-cache diff --git a/src/build-cache.coffee b/src/build-cache.coffee index 3509fde..3ac3879 100644 --- a/src/build-cache.coffee +++ b/src/build-cache.coffee @@ -39,6 +39,6 @@ module.exports = (cachePath = path.join(process.cwd(), '.powerbuild~')) -> if not caches[cachePath] caches[cachePath] = processed: {} - uids: {next: 1, names: []} + uids: {next: 1, names: {}} return caches[cachePath] diff --git a/src/index.coffee b/src/index.coffee index 3fd5075..34ff81a 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -14,7 +14,7 @@ class Powerbuild options.inlineSources ?= false if options.compress options.minify = true - options.uids or= {next: 1, names: []} + options.uids or= {next: 1, names: {}} options.npmSourceMaps ?= false options.root or= process.cwd() options.node ?= true diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 57088df..3e7ba5c 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -221,11 +221,13 @@ module.exports = (build, processedCache) -> processed[filename] = {id, canonicalName, code, map, lineCount, mtime, deps, nodeFeatures, isNpmModule, isCoreModule, realCanonicalName} if processedCache + # Cache entries are only updated, never deleted, this enables multiple + # build configurations to share it processedCache[filename] = processed[filename] - # remove old dependencies and update the cache - # for own k, {isCoreModule} of processed - # if not (isCoreModule or k of checked) - # delete processed[k] + # remove old dependencies + for own k, {isCoreModule} of processed + if not (isCoreModule or k of checked) + delete processed[k] return processed diff --git a/tasks/powerbuild.coffee b/tasks/powerbuild.coffee index 389c97b..21d1896 100644 --- a/tasks/powerbuild.coffee +++ b/tasks/powerbuild.coffee @@ -9,9 +9,12 @@ DESC = 'Wraps node.js/commonjs projects into single umd function that will run anywhere, generating concatenated source maps to debug files individually.' +defaultCachePath = path.join(__dirname, '..', 'powerbuild-cache') + + module.exports = (grunt) -> grunt.registerMultiTask NAME, DESC, -> - options = @options() + options = @options(cachePath: defaultCachePath) for f in @files opts = _.clone(options) if not opts.disableDiskCache diff --git a/test-setup.coffee b/test-setup.coffee index b7305ba..fa6598d 100644 --- a/test-setup.coffee +++ b/test-setup.coffee @@ -58,7 +58,7 @@ global.bundleEval = (entryPoint, opts = {}, env = {}) -> module$.exports extensions = ['.js', '.coffee'] -relativeResolve = require './src/relative-resolve' +relativeResolve = require './lib/relative-resolve' global.resolve = (givenPath, cwd = '') -> realCwd = path.resolve path.join FIXTURES_DIR, cwd resolved = relativeResolve {extensions, root: FIXTURES_DIR, cwd: realCwd, path: givenPath} From d8f5ea2243629569328e937bd6b195f849045170 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 10:49:35 -0300 Subject: [PATCH 24/31] Updated README and changed package name --- README.md | 181 +++++++++++++++++++++++-------- bin/cjsify | 3 - bin/powerbuild | 2 + package.json | 5 +- src/bundle.coffee | 1 + src/command.coffee | 2 +- src/index.coffee | 7 +- src/traverse-dependencies.coffee | 6 +- 8 files changed, 145 insertions(+), 62 deletions(-) delete mode 100755 bin/cjsify create mode 100755 bin/powerbuild diff --git a/README.md b/README.md index b95d203..6dc80f5 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,75 @@ -# CommonJS Everywhere +# Powerbuild + +> CommonJS bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere adding speed improvements, persistent disk cache for incremental builds, support for reading '// [#@] sourceMappingURL' from input files and bundled grunt task + +## Main changes from commonjs-everywhere + + - Escodegen is only used when generating partial source maps, the final + result is computed manually. + - Esmangle/escodegen is replaced by UglifyJS for two reasons: + * It was breaking in some of my tests + * It is 10x slower than UglifyJS(literally 10x slower in my tests. For + a bundle with 50k lines of code UglifyJS took about 4 seconds versus + 35 seconds from esmangle/escodegen) + - Dependency on coffee-script-redux was removed. While its still possible + to use the 'handlers' option to build compile-to-js languages directly, + this tool now reads '// @sourceMappingURL' tags from the end of the file + in order to map correctly to the original files. This means any + compile-to-js language is supported out-of-box. + - By default, source maps for npm dependencies are not included. + - Module paths are replaced by unique identifiers, which leads to a small + improvement in the resulting size. When the __filename or __dirname + variables are used, a mapping for that module uid to the filename will + be used. + - Multiple entry points can be specified, with the last being exported + if the 'export' option is used. This can be used to create test bundles. + - The result is wrapped into [UMD](https://github.com/umdjs/umd). + - If the 'node' option is unset, it will disable node.js emulation and + inclusion of core modules. The result can be loaded as a normal node.js + module. -CommonJS (node module) browser bundler with source maps from the minified JS bundle to the original source, aliasing for browser overrides, and extensibility for arbitrary compile-to-JS language support. ## Install - npm install -g commonjs-everywhere + npm install -g powerbuild ## Usage ### CLI - $ bin/cjsify --help + $ bin/powerbuild --help - Usage: cjsify OPT* path/to/entry-file.ext OPT* + Usage: powerbuild OPT* ENTRY_FILE+ OPT* -a, --alias ALIAS:TO replace requires of file identified by ALIAS with TO -h, --handler EXT:MODULE handle files with extension EXT with module MODULE - -m, --minify minify output + -m, --minify minify output using uglify.js + -c, --compress Compress/optimize code when minifying + (automatically enabled by this option). Enabling + will break the generated source map. -o, --output FILE output to FILE instead of stdout -r, --root DIR unqualified requires are relative to DIR; default: cwd -s, --source-map FILE output a source map to FILE -v, --verbose verbose output sent to stderr -w, --watch watch input files/dependencies for changes and rebuild bundle - -x, --export NAME export the given entry module as NAME + -x, --export NAME export the last given entry module as NAME --deps do not bundle; just list the files that would be bundled --help display this help message and exit --ignore-missing continue without error when dependency resolution fails --inline-source-map include the source map as a data URI in the generated bundle - --inline-sources include source content in generated source maps; default: on - --node include process object; emulate node environment; default: on + --inline-sources include source content in generated source maps + --node if needed by any module, emulate a node.js + environment by including globals such as Buffer, + process and setImmediate; default: on + --cache-path file where to read/write a json-encoded cache that + is used for fast, incremental builds. + default: '.powerbuild-cache~' in the current + directory + --disable-disk-cache disables persistence of incremental build cache + to disk. Incremental build will only work with the + --watch option + --npm-source-maps add mappings for npm modules in the resulting + source map(significantly increases the build time) --version display the version number and exit *Note:* use `-` as an entry file to accept JavaScript over stdin @@ -37,38 +78,96 @@ CommonJS (node module) browser bundler with source maps from the minified JS bun #### Example: -Common usage +Common usage, a single entry point which will be used to build the entire +dependency graph. Whatever is exported by 'entry-file.js' will go to the +global property 'MyLibrary': ```bash -cjsify src/entry-file.js --export MyLibrary --source-map my-library.js.map >my-library.js +powerbuild src/entry-file.js --export MyLibrary --source-map my-library.js.map >my-library.js ``` -Watch entry file, its dependencies, and even newly added dependencies. Notice that only the files that need to be rebuilt are accessed when one of the watched dependencies are touched. This is a much more efficient approach than simply rebuilding everything. +Specify multiple entry points which will be "required" at startup. Only +the last entry point will be exported when used in conjunction with the +'--export' option. This is mostly useful for building test bundles which +can be referenced from a single 'script' tag ```bash -cjsify -wo my-library.js -x MyLibrary src/entry-file.js +powerbuild test/*.js --source-map tests.js.map -o tests.js ``` -Use a browser-specific version of `/lib/node-compatible.js` (remember to use `root`-relative paths for aliasing). An empty alias target is used to delay errors to runtime when requiring the source module (`fs` in this case). +Watch every file in the dependency graph and rebuild when a file changes. +Unlike commonjs-everywhere, this tool caches partial builds to disk, so this +is not necessary for incremental builds. ```bash -cjsify -a /lib/node-compatible.js:/lib/browser-compatible.js -a fs: -x MyLibrary lib/entry-file.js +powerbuild -wo my-library.js -x MyLibrary src/entry-file.js ``` -### Module Interface +Use a browser-specific version of `/lib/node-compatible.js` (remember to use +`root`-relative paths for aliasing). An empty alias target is used to delay +errors to runtime when requiring the source module (`fs` in this case). The +'browser' field in package.json will also be used if available when building +bundles with node.js emulation(which is the default). -#### `cjsify(entryPoint, root, options)` → Spidermonkey AST -Bundles the given file and its dependencies; returns a Spidermonkey AST representation of the bundle. Run the AST through `escodegen` to generate JS code. +```bash +powerbuild -a /lib/node-compatible.js:/lib/browser-compatible.js -a fs: -x MyLibrary lib/entry-file.js +``` -* `entryPoint` is a file relative to `process.cwd()` that will be the initial module marked for inclusion in the bundle as well as the exported module -* `root` is the directory to which unqualified requires are relative; defaults to `process.cwd()` -* `options` is an optional object (defaulting to `{}`) with zero or more of the following properties - * `export`: a variable name to add to the global scope; assigned the exported object from the `entryPoint` module. Any valid [Left-Hand-Side Expression](http://es5.github.com/#x11.2) may be given instead. - * `aliases`: an object whose keys and values are `root`-rooted paths (`/src/file.js`), representing values that will replace requires that resolve to the associated keys - * `handlers`: an object whose keys are file extensions (`'.roy'`) and whose values are functions from the file contents to either a Spidermonkey-format JS AST like the one esprima produces or a string of JS. Handlers for CoffeeScript and JSON are included by default. If no handler is defined for a file extension, it is assumed to be JavaScript. - * `node`: a falsey value causes the bundling phase to omit the `process` stub that emulates a node environment - * `verbose`: log additional operational information to stderr - * `ignoreMissing`: continue without error when dependency resolution fails +### Module Interface + +#### `new Powerbuild(options)` +Constructor for an object that can keeps track of build options and is used to +trigger incremental rebuilds. + +* `options` is an object that can contain the following properties: + * `entryPoints` is an array of filenames relative to `process.cwd()` that + will be used to initialize the bundle. The last item in this array will + also be used when the 'export' option is specified + * `root`: Same as cli. + * `export`: Same as cli. + * `aliases`: an object whose keys and values are `root`-rooted paths + (`/src/file.js`), representing values that will replace requires that + resolve to the associated keys + * `handlers`: an object whose keys are file extensions (`'.coffee'`) and + whose values are functions that receives the file contents as arguments + and returns one of the following: + - Spidermonkey-format JS AST like the one esprima produces + - A string of javascript + - An object with the keys 'code' and 'map' containing strings with + javascript and sourcemaps respectively. + A handler for JSON is included by default. If no handler is defined for + a file extension, it is assumed to be JavaScript. (The default + coffeescript-redux handler was removed because this tool now reads + '// @sourceMappingURL' comment tags, so it can be used in conjunction + with the default coffeescript compiler) + * `node`: Same as cli. When true(default) the bundling phase will emit + globals for 'process', 'Buffer' or 'setImmediate' if any of those are + used by any of the bundled modules. Setting this to false will completely + disable node.js emulation, excluding core node.js modules(path, util...) + from the bundle. This may be used to create bundles targeted at node.js. + (While this will not be a very common case, it can be used for example + to distribute node.js apps as a single javascript file containing all + dependencies). + * `verbose`: Same as cli. + * `ignoreMissing`: Same as cli. + * `minify`: Same as cli. + * `compress`: Same as cli. + * `output`: Name of the output file. The file will not be written, this + is used when building the source map. + * `sourceMap`: Same as cli. This may be true to make the source map have + the same name as 'output' with '.map' appended. + * `inlineSourceMap`: Same as cli. + * `inlineSources`: Same as cli. + * `npmSourceMaps`: Same as cli. This is disabled by default because + it greatly increases build efficiency and normally you wont care about + debugging external modules. + + +### Grunt task + +This package includes a grunt task that takes any of the API or cli options( +with dashes removed and converted to camelCase). For an example see this +package's [test bundle configuration]() ## Examples @@ -89,29 +188,19 @@ Say we have the following directory tree: Running the following command will export `index.coffee` and its dependencies as `App.Todos`. ``` -cjsify -o public/javascripts/app.js -x App.Todos -r components components/todos/index.coffee +powerbuild -o public/javascripts/app.js -x App.Todos -r components components/todos/index.coffee ``` -Since the above command specifies `components` as the root directory for unqualified requires, we are able to require `components/users/model.coffee` with `require 'users/model'`. The output file will be `public/javascripts/app.js`. +Since the above command specifies `components` as the root directory for +unqualified requires, we are able to require `components/users/model.coffee` +with `require 'users/model'`. The output file will be +`public/javascripts/app.js`. ### Node Module Example ```coffee -jsAst = (require 'commonjs-everywhere').cjsify 'src/entry-file.coffee', __dirname, - export: 'MyLibrary' - aliases: - '/src/module-that-only-works-in-node.coffee': '/src/module-that-does-the-same-thing-in-the-browser.coffee' - handlers: - '.roy': (roySource, filename) -> - # the Roy compiler outputs JS code right now, so we parse it with esprima - (require 'esprima').parse (require 'roy').compile roySource, {filename} - -{map, code} = (require 'escodegen').generate jsAst, - sourceMapRoot: __dirname - sourceMapWithCode: true - sourceMap: true +opts.root = 'components' +opts.entryPoints = ['index.coffee'] +powerbuild = new Powerbuild opts +{code, map} = powerbuild.bundle() ``` - -### Sample Output - -![](http://i.imgur.com/oDcQh8H.png) diff --git a/bin/cjsify b/bin/cjsify deleted file mode 100755 index 2c88d11..0000000 --- a/bin/cjsify +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env node -require('coffee-script'); -require(require('path').join(__dirname, '..', 'src', 'command')); diff --git a/bin/powerbuild b/bin/powerbuild new file mode 100755 index 0000000..6e81aa6 --- /dev/null +++ b/bin/powerbuild @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require(require('path').join(__dirname, '..', 'lib', 'command')); diff --git a/package.json b/package.json index 88bbbe7..dcdabdf 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "powerbuild", "version": "0.0.1", - "description": "CommonJS browser bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere, speed improvements, persistent disk cache for incremental builds and bundled grunt task", + "description": "CommonJS bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere adding speed improvements, persistent disk cache for incremental builds, support for reading '// [#@] sourceMappingURL' from input files and bundled grunt task", "homepage": "https://github.com/tarruda/powerbuild", "keywords": [ "CommonJS", @@ -53,7 +53,6 @@ "zlib-browserify": "*", "source-map": "~0.1.30", "handlebars": "~1.0.12", - "coffee-script": "~1.6.3", "lodash": "~2.2.1", "uglify-js": "~2.4.0", "estraverse": "~1.3.1", @@ -78,7 +77,7 @@ "licenses": [ { "type": "BSD", - "url": "https://github.com/michaelficarra/commonjs-everywhere/blob/master/LICENSE" + "url": "https://github.com/tarruda/powerbuild/blob/master/LICENSE" } ] } diff --git a/src/bundle.coffee b/src/bundle.coffee index 1fe0da2..5138766 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -3,6 +3,7 @@ path = require 'path' {btoa} = require 'Base64' UglifyJS = require 'uglify-js' sourceMapToAst = require './sourcemap-to-ast' +UglifyJS.warn_function = -> PROCESS = """ diff --git a/src/command.coffee b/src/command.coffee index 75e1e16..56a5166 100644 --- a/src/command.coffee +++ b/src/command.coffee @@ -4,7 +4,7 @@ nopt = require 'nopt' _ = require 'lodash' Powerbuild = require './index' -buildCache = require '../src/build-cache' +buildCache = require './build-cache' traverseDependencies = require './traverse-dependencies' knownOpts = {} diff --git a/src/index.coffee b/src/index.coffee index 34ff81a..b12908c 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -1,5 +1,5 @@ _ = require 'lodash' -acorn = require 'acorn' +esprima = require 'esprima' coffee = require 'coffee-script' path = require 'path' bundle = require './bundle' @@ -31,11 +31,8 @@ class Powerbuild @sourceMap = "#{@output}.map" @handlers = - '.coffee': (src, canonicalName) -> - {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true - return {code: js, map: v3SourceMap} '.json': (json, canonicalName) -> - acorn.parse "module.exports = #{json}", locations: yes + esprima.parse "module.exports = #{json}", loc: yes for own ext, handler of options.handlers ? {} @handlers[ext] = handler diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 3e7ba5c..261f8db 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -163,11 +163,9 @@ module.exports = (build, processedCache) -> worklist.push dep deps.push dep catch e - rv = undefined if build.ignoreMissing - rv = { type: 'Literal', value: null } - rv.loc = node.loc - return rv + return + throw e # rewrite the require to use the root-relative path or the uid if # enabled if rewriteRequire From 274fe1e414121f375122b02246b2c2b7b13a0758 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 12:31:36 -0300 Subject: [PATCH 25/31] Fixed source maps of files generated with handlers --- Gruntfile.coffee | 7 +++++++ package.json | 3 ++- src/core-modules.coffee | 4 ++-- src/index.coffee | 1 - src/traverse-dependencies.coffee | 3 ++- test/bundling.coffee | 6 +++--- 6 files changed, 16 insertions(+), 8 deletions(-) diff --git a/Gruntfile.coffee b/Gruntfile.coffee index bb8c2c8..773494b 100644 --- a/Gruntfile.coffee +++ b/Gruntfile.coffee @@ -1,3 +1,5 @@ +coffee = require 'coffee-script' + module.exports = (grunt) -> grunt.initConfig @@ -19,6 +21,11 @@ module.exports = (grunt) -> options: sourceMap: true node: false + handlers: + '.coffee': (src, canonicalName) -> + {js, v3SourceMap} = coffee.compile src, sourceMap: true, bare: true + return {code: js, map: v3SourceMap} + all: files: [ {src: ['test-setup.coffee', 'test/*.coffee'], dest: 'tests.js'} diff --git a/package.json b/package.json index dcdabdf..319915c 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,8 @@ "grunt-contrib-watch": "~0.5.3", "source-map-support": "~0.2.3", "grunt-newer": "~0.5.4", - "grunt-contrib-coffee": "~0.7.0" + "grunt-contrib-coffee": "~0.7.0", + "coffee-script": "~1.6.3" }, "scripts": { "test": "make test" diff --git a/src/core-modules.coffee b/src/core-modules.coffee index 47f9536..dd7b1e0 100644 --- a/src/core-modules.coffee +++ b/src/core-modules.coffee @@ -9,8 +9,8 @@ CORE_MODULES = crypto: resolve 'crypto-browserify' events: resolve 'events-browserify' http: resolve 'http-browserify' - punycode: resolve './node_modules/punycode', basedir: CJS_DIR - querystring: resolve './node_modules/querystring', basedir: CJS_DIR + punycode: resolve('./node_modules/punycode', basedir: CJS_DIR) + querystring: resolve('./node_modules/querystring', basedir: CJS_DIR) vm: resolve 'vm-browserify' zlib: resolve 'zlib-browserify' diff --git a/src/index.coffee b/src/index.coffee index b12908c..31bbab2 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -1,6 +1,5 @@ _ = require 'lodash' esprima = require 'esprima' -coffee = require 'coffee-script' path = require 'path' bundle = require './bundle' traverseDependencies = require './traverse-dependencies' diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index 261f8db..ca0564a 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -132,6 +132,7 @@ module.exports = (build, processedCache) -> estraverse.replace ast, enter: (node, parents) -> if node.loc? + node.loc.source = canonicalName if node.type != 'Program' and adjustWrapperLocation # Adjust the location info to reflect the removed function wrapper if node.loc.start.line == 1 and node.loc.start.column >= 12 @@ -210,7 +211,7 @@ module.exports = (build, processedCache) -> sourceMap: true format: escodegen.FORMAT_DEFAULTS sourceMapWithCode: true - sourceMapRoot: if build.sourceMap? then (path.relative (path.dirname build.sourceMap), build.root) or '.' + sourceMapRoot: build.sourceMapRoot map = map.toString() # cache linecount for a little more efficiency when calculating offsets diff --git a/test/bundling.coffee b/test/bundling.coffee index 4601e58..8750b38 100644 --- a/test/bundling.coffee +++ b/test/bundling.coffee @@ -33,10 +33,10 @@ suite 'Bundling', -> eq 5, obj.a eq 3, obj.b - test 'ignoreMissing option produces null values for missing dependencies', -> + test 'ignoreMissing option throws resolve exception at runtime', -> fixtures '/a.js': 'module.exports = require("./b")' - throws -> bundleEval 'a.js' - eq null, bundleEval 'a.js', ignoreMissing: yes + throws((-> bundleEval('a.js')), /Cannot find module/) + throws((-> bundleEval('a.js', ignoreMissing: true)), /Failed to resolve/) test '#78: fix canonicalisation of paths', -> fixtures '/src/main.coffee': 'module.exports = 1' From 70ac3f5f97c6bd06a611966e5a31623b28e65b7c Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 14:39:54 -0300 Subject: [PATCH 26/31] Switched to 'browser-builtins' module for core modules --- .gitmodules | 3 --- node | 1 - src/core-modules.coffee | 39 +------------------------------- src/is-core.coffee | 2 +- src/traverse-dependencies.coffee | 2 +- test/bundling.coffee | 2 +- 6 files changed, 4 insertions(+), 45 deletions(-) delete mode 100644 .gitmodules delete mode 160000 node diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 42d63da..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "node"] - path = node - url = git://github.com/joyent/node.git diff --git a/node b/node deleted file mode 160000 index e32660a..0000000 --- a/node +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e32660a984427d46af6a144983cf7b8045b7299c diff --git a/src/core-modules.coffee b/src/core-modules.coffee index dd7b1e0..b40d8d9 100644 --- a/src/core-modules.coffee +++ b/src/core-modules.coffee @@ -1,38 +1 @@ -path = require 'path' -{sync: resolve} = require 'resolve' - -CJS_DIR = path.join __dirname, '..' - -CORE_MODULES = - buffer: resolve 'buffer-browserify' - constants: resolve 'constants-browserify' - crypto: resolve 'crypto-browserify' - events: resolve 'events-browserify' - http: resolve 'http-browserify' - punycode: resolve('./node_modules/punycode', basedir: CJS_DIR) - querystring: resolve('./node_modules/querystring', basedir: CJS_DIR) - vm: resolve 'vm-browserify' - zlib: resolve 'zlib-browserify' - -NODE_CORE_MODULES = [ - '_stream_duplex' - '_stream_passthrough' - '_stream_readable' - '_stream_transform' - '_stream_writable' - 'assert' - 'console' - 'domain' - 'freelist' - 'path' - 'readline' - 'stream' - 'string_decoder' - 'sys' - 'url' - 'util' -] -for mod in NODE_CORE_MODULES - CORE_MODULES[mod] = path.join CJS_DIR, 'node', 'lib', "#{mod}.js" - -module.exports = CORE_MODULES +module.exports = require 'browser-builtins' diff --git a/src/is-core.coffee b/src/is-core.coffee index 2453c87..8c23bfe 100644 --- a/src/is-core.coffee +++ b/src/is-core.coffee @@ -2,4 +2,4 @@ resolve = require 'resolve' CORE_MODULES = require './core-modules' -module.exports = (x) -> (resolve.isCore x) or [].hasOwnProperty.call CORE_MODULES, x +module.exports = (x) -> x == 'browser-builtins' or (resolve.isCore x) or [].hasOwnProperty.call CORE_MODULES, x diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index ca0564a..d694313 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -165,7 +165,7 @@ module.exports = (build, processedCache) -> deps.push dep catch e if build.ignoreMissing - return + return { type: 'Literal', value: null } throw e # rewrite the require to use the root-relative path or the uid if # enabled diff --git a/test/bundling.coffee b/test/bundling.coffee index 8750b38..2a6b513 100644 --- a/test/bundling.coffee +++ b/test/bundling.coffee @@ -36,7 +36,7 @@ suite 'Bundling', -> test 'ignoreMissing option throws resolve exception at runtime', -> fixtures '/a.js': 'module.exports = require("./b")' throws((-> bundleEval('a.js')), /Cannot find module/) - throws((-> bundleEval('a.js', ignoreMissing: true)), /Failed to resolve/) + eq bundleEval('a.js', ignoreMissing: true), null test '#78: fix canonicalisation of paths', -> fixtures '/src/main.coffee': 'module.exports = 1' From 3e781cd21807b1e82783a4039a1e63fa33841384 Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 14:49:55 -0300 Subject: [PATCH 27/31] Add test shim for 'freelist' and fixed tests --- src/core-modules.coffee | 22 +++++++++++++++++++++- test/dependency-resolution.coffee | 9 ++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/core-modules.coffee b/src/core-modules.coffee index b40d8d9..589ad1f 100644 --- a/src/core-modules.coffee +++ b/src/core-modules.coffee @@ -1 +1,21 @@ -module.exports = require 'browser-builtins' +path = require 'path' + +_ = require 'lodash' +browserBuiltins = require 'browser-builtins' + +modules = _.clone browserBuiltins +modules.freelist = path.join __dirname, '..', 'core', 'freelist.js' +delete modules.fs +delete modules.domain +delete modules.readline +delete modules.repl +delete modules.tls +delete modules.net +delete modules.domain +delete modules.dns +delete modules.dgram +delete modules.cluster +delete modules.child_process + + +module.exports = modules diff --git a/test/dependency-resolution.coffee b/test/dependency-resolution.coffee index 391c7f6..388d806 100644 --- a/test/dependency-resolution.coffee +++ b/test/dependency-resolution.coffee @@ -48,7 +48,7 @@ suite 'Dependency Resolution', -> test 'core dependencies', -> fixtures 'a.js': 'require("freelist")' - arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js' + arrayEq ['../core/freelist.js', 'a.js'], deps 'a.js' suite 'Aliasing', -> @@ -69,7 +69,6 @@ suite 'Dependency Resolution', -> test 'alias a core module', -> fixtures 'a.js': 'require("fs")' - arrayEq ['a.js', '../node/lib/freelist.js'], deps 'a.js', aliases: {fs: 'freelist'} - fixtures 'a.js': 'require("path")' - arrayEq ['a.js', '../node/lib/path.js', '../node/lib/util.js' - '../node_modules/setimmediate/setImmediate.js'], deps 'a.js', aliases: {child_process: null, fs: null} + arrayEq(['../core/freelist.js', 'a.js'], deps 'a.js', aliases: {fs: 'freelist'}) + fixtures 'a.js': 'require("freelist")' + arrayEq ['../core/freelist.js', 'a.js'], deps 'a.js', aliases: {child_process: null, fs: null} From 77acfc94866fa91d50bbe8457615fc45e220379a Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 15:18:52 -0300 Subject: [PATCH 28/31] Add detection for console --- package.json | 13 +------------ src/bundle.coffee | 20 +++++++++++--------- src/core-modules.coffee | 22 +++++++++++----------- src/index.coffee | 4 +++- src/traverse-dependencies.coffee | 18 ++++++++++++------ test/process-spec.coffee | 4 ---- 6 files changed, 38 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index 319915c..768c34e 100644 --- a/package.json +++ b/package.json @@ -42,24 +42,13 @@ "mktemp": "~0.3.0", "nopt": "~2.1.2", "resolve": "0.5.x", - "buffer-browserify": "*", - "constants-browserify": ">= 0.0.1", - "crypto-browserify": "*", - "events-browserify": "*", - "http-browserify": "*", - "punycode": "*", - "querystring": "*", - "vm-browserify": "*", - "zlib-browserify": "*", "source-map": "~0.1.30", - "handlebars": "~1.0.12", "lodash": "~2.2.1", "uglify-js": "~2.4.0", "estraverse": "~1.3.1", "escodegen": "0.0.27", "esprima": "~1.0.4", - "escope": "~1.0.0", - "setimmediate": "~1.0.1" + "escope": "~1.0.0" }, "devDependencies": { "scopedfs": "~0.1.0", diff --git a/src/bundle.coffee b/src/bundle.coffee index 5138766..64c3e3c 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -105,17 +105,17 @@ bundle = (build, processed) -> file: path.basename(build.output) sourceRoot: build.sourceMapRoot lineOffset = umdOffset - useProcess = false + setImmediate = false bufferPath = false - setImmediatePath = false + consolePath = false files = {} for own filename, {id, canonicalName, realCanonicalName, code, map, lineCount, isNpmModule, nodeFeatures} of processed if nodeFeatures.__filename or nodeFeatures.__dirname files[id] = realCanonicalName or canonicalName - useProcess = useProcess or nodeFeatures.process - setImmediatePath = setImmediatePath or nodeFeatures.setImmediate + setImmediate = setImmediate or nodeFeatures.setImmediate + consolePath = consolePath or nodeFeatures.console bufferPath = bufferPath or nodeFeatures.Buffer result += """ \nrequire.define('#{id}', function(module, exports, __dirname, __filename, undefined){ @@ -137,15 +137,17 @@ bundle = (build, processed) -> name: m.name lineOffset += lineCount - if bufferPath + if bufferPath and build.node {id} = processed[bufferPath] result += "\nvar Buffer = require('#{id}');" - if setImmediatePath - {id} = processed[setImmediatePath] - result += "\nrequire('#{id}');" + if consolePath and build.node + {id} = processed[consolePath] + result += "\nvar console = require('#{id}');" - if useProcess and build.node + if setImmediate and build.node + {id} = processed[setImmediate] + result += "\nvar setImmediate = require('#{id}').setImmediate;" result += "\nvar process = #{PROCESS};" for i in [0...build.entryPoints.length] diff --git a/src/core-modules.coffee b/src/core-modules.coffee index 589ad1f..432f59d 100644 --- a/src/core-modules.coffee +++ b/src/core-modules.coffee @@ -5,17 +5,17 @@ browserBuiltins = require 'browser-builtins' modules = _.clone browserBuiltins modules.freelist = path.join __dirname, '..', 'core', 'freelist.js' -delete modules.fs -delete modules.domain -delete modules.readline -delete modules.repl -delete modules.tls -delete modules.net -delete modules.domain -delete modules.dns -delete modules.dgram -delete modules.cluster -delete modules.child_process +# delete modules.fs +# delete modules.domain +# delete modules.readline +# delete modules.repl +# delete modules.tls +# delete modules.net +# delete modules.domain +# delete modules.dns +# delete modules.dgram +# delete modules.cluster +# delete modules.child_process module.exports = modules diff --git a/src/index.coffee b/src/index.coffee index 31bbab2..48d948d 100644 --- a/src/index.coffee +++ b/src/index.coffee @@ -20,7 +20,7 @@ class Powerbuild {@output, @export, @entryPoints, @root, @node, @inlineSources, @verbose, @ignoreMissing, @sourceMap, @inlineSourceMap, @mainModule, @minify, @aliases, @handlers, @processed, @uids, - @npmSourceMaps, @compress} = options + @npmSourceMaps, @compress, @debug} = options @sourceMapRoot = if @sourceMap? then (path.relative (path.dirname @sourceMap), @root) or '.' @@ -51,6 +51,8 @@ class Powerbuild uidFor: (name) -> + if @debug + return name if not {}.hasOwnProperty.call(@uids.names, name) uid = @uids.next++ @uids.names[name] = uid diff --git a/src/traverse-dependencies.coffee b/src/traverse-dependencies.coffee index d694313..4e6455d 100644 --- a/src/traverse-dependencies.coffee +++ b/src/traverse-dependencies.coffee @@ -34,8 +34,8 @@ module.exports = (build, processedCache) -> aliases = build.aliases ? {} root = build.root globalFeatures = { + console: false setImmediate: false - process: false Buffer: false } @@ -190,19 +190,25 @@ module.exports = (build, processedCache) -> } baseDir = path.dirname path.resolve __dirname - if isImplicit 'process', scope - nodeFeatures.process = globalFeatures.process = true - if not globalFeatures.setImmediate and (isImplicit 'setImmediate', scope) or nodeFeatures.process + if not globalFeatures.setImmediate and + (isImplicit('setImmediate', scope) or isImplicit('process', scope)) globalFeatures.setImmediate = true - resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'setimmediate'} + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'timers'} resolved = _.extend resolved, isCoreModule: true, isNpmModule: true nodeFeatures.setImmediate = resolved.filename worklist.unshift(resolved) + if not globalFeatures.console and isImplicit 'console', scope + globalFeatures.console = true + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'console'} + resolved = _.extend resolved, isCoreModule: true, isNpmModule: true + nodeFeatures.console = resolved.filename + worklist.unshift(resolved) + if not globalFeatures.Buffer and isImplicit 'Buffer', scope globalFeatures.Buffer = true - resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'buffer-browserify'} + resolved = relativeResolve {extensions: build.extensions, aliases, root: build.root, cwd: baseDir, path: 'buffer'} resolved = _.extend resolved, isCoreModule: true, isNpmModule: true nodeFeatures.Buffer = resolved.filename worklist.unshift(resolved) diff --git a/test/process-spec.coffee b/test/process-spec.coffee index 35fce37..14439ff 100644 --- a/test/process-spec.coffee +++ b/test/process-spec.coffee @@ -35,7 +35,3 @@ suite 'Process Spec', -> fixtures '/a.js': 'module.exports = Object.keys(process.env)' arrayEq [], bundleEval 'a.js' - test 'process.nextTick should use setImmediate if available', -> - fixtures '/a.js': 'module.exports = process.nextTick' - indicator = -> - eq indicator, bundleEval 'a.js', null, setImmediate: indicator From 5553a045e7b50d76305b1f056c23074759dcf35e Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 15:48:24 -0300 Subject: [PATCH 29/31] Disable UglifyJS messages about mappings --- README.md | 10 +++++----- src/bundle.coffee | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6dc80f5..5d3f7a0 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,16 @@ - Escodegen is only used when generating partial source maps, the final result is computed manually. - - Esmangle/escodegen is replaced by UglifyJS for two reasons: + - For minification esmangle/escodegen is replaced by UglifyJS for two + reasons: * It was breaking in some of my tests - * It is 10x slower than UglifyJS(literally 10x slower in my tests. For - a bundle with 50k lines of code UglifyJS took about 4 seconds versus - 35 seconds from esmangle/escodegen) + * It is 10x slower than UglifyJS. For a bundle with 50k lines of code + UglifyJS took about 4 seconds versus 35 seconds from esmangle/escodegen) - Dependency on coffee-script-redux was removed. While its still possible to use the 'handlers' option to build compile-to-js languages directly, this tool now reads '// @sourceMappingURL' tags from the end of the file in order to map correctly to the original files. This means any - compile-to-js language is supported out-of-box. + compile-to-js language that produces source maps is supported out-of-box. - By default, source maps for npm dependencies are not included. - Module paths are replaced by unique identifiers, which leads to a small improvement in the resulting size. When the __filename or __dirname diff --git a/src/bundle.coffee b/src/bundle.coffee index 64c3e3c..ec8bb14 100644 --- a/src/bundle.coffee +++ b/src/bundle.coffee @@ -3,7 +3,7 @@ path = require 'path' {btoa} = require 'Base64' UglifyJS = require 'uglify-js' sourceMapToAst = require './sourcemap-to-ast' -UglifyJS.warn_function = -> +UglifyJS.AST_Node.warn_function = -> PROCESS = """ From 4eff2dcbf513b192f61b9e3775e3dd5bf73ac80c Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 15:49:22 -0300 Subject: [PATCH 30/31] Reset package version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 768c34e..edd7a91 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "powerbuild", - "version": "0.0.1", + "version": "0.0.0", "description": "CommonJS bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere adding speed improvements, persistent disk cache for incremental builds, support for reading '// [#@] sourceMappingURL' from input files and bundled grunt task", "homepage": "https://github.com/tarruda/powerbuild", "keywords": [ From 0ef2c978670cb51b7a794d8af648d8ce3a37dbac Mon Sep 17 00:00:00 2001 From: Thiago de Arruda Date: Tue, 8 Oct 2013 15:50:51 -0300 Subject: [PATCH 31/31] release 0.0.1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index edd7a91..768c34e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "powerbuild", - "version": "0.0.0", + "version": "0.0.1", "description": "CommonJS bundler with aliasing, extensibility, and source maps from the minified JS bundle. Forked from commonjs-everywhere adding speed improvements, persistent disk cache for incremental builds, support for reading '// [#@] sourceMappingURL' from input files and bundled grunt task", "homepage": "https://github.com/tarruda/powerbuild", "keywords": [