diff --git a/lib/pack.js b/lib/pack.js index 938ece8e..81bc10f3 100644 --- a/lib/pack.js +++ b/lib/pack.js @@ -91,6 +91,15 @@ const Pack = warner(class Pack extends Minipass { this.zip.on('end', _ => super.end()) this.zip.on('drain', _ => this[ONDRAIN]()) this.on('resume', _ => this.zip.resume()) + } else if (opt.brotli) { + if (typeof opt.brotli !== 'object') { + opt.brotli = {} + } + this.zip = new zlib.BrotliCompress(opt.brotli) + this.zip.on('data', chunk => super.write(chunk)) + this.zip.on('end', _ => super.end()) + this.zip.on('drain', _ => this[ONDRAIN]()) + this.on('resume', _ => this.zip.resume()) } else { this.on('drain', this[ONDRAIN]) } diff --git a/lib/parse.js b/lib/parse.js index 4b85915c..8b8c8cee 100644 --- a/lib/parse.js +++ b/lib/parse.js @@ -97,6 +97,9 @@ module.exports = warner(class Parser extends EE { this.strict = !!opt.strict this.maxMetaEntrySize = opt.maxMetaEntrySize || maxMetaEntrySize this.filter = typeof opt.filter === 'function' ? opt.filter : noop + // Unlike gzip, brotli doesn't have any magic bytes to identify it + // Users need to explicitly tell us they're extracting a brotli file + this.brotli = opt.brotli // have to set this so that streams are ok piping into it this.writable = true @@ -356,6 +359,21 @@ module.exports = warner(class Parser extends EE { this[BUFFER] = chunk return true } + if (this[UNZIP] === null && this.brotli) { + const ended = this[ENDED] + this[ENDED] = false + this[UNZIP] = new zlib.BrotliDecompress() + this[UNZIP].on('data', chunk => this[CONSUMECHUNK](chunk)) + this[UNZIP].on('error', er => this.abort(er)) + this[UNZIP].on('end', _ => { + this[ENDED] = true + this[CONSUMECHUNK]() + }) + this[WRITING] = true + const ret = this[UNZIP][ended ? 'end' : 'write'](chunk) + this[WRITING] = false + return ret + } for (let i = 0; this[UNZIP] === null && i < gzipHeader.length; i++) { if (chunk[i] !== gzipHeader[i]) { this[UNZIP] = false diff --git a/lib/replace.js b/lib/replace.js index c6e619be..8db6800b 100644 --- a/lib/replace.js +++ b/lib/replace.js @@ -23,7 +23,7 @@ module.exports = (opt_, files, cb) => { throw new TypeError('file is required') } - if (opt.gzip) { + if (opt.gzip || opt.brotli || opt.file.endsWith('.br') || opt.file.endsWith('.tbr')) { throw new TypeError('cannot append to compressed archives') } diff --git a/lib/update.js b/lib/update.js index ded977dc..4d328543 100644 --- a/lib/update.js +++ b/lib/update.js @@ -13,7 +13,7 @@ module.exports = (opt_, files, cb) => { throw new TypeError('file is required') } - if (opt.gzip) { + if (opt.gzip || opt.brotli || opt.file.endsWith('.br') || opt.file.endsWith('.tbr')) { throw new TypeError('cannot append to compressed archives') } diff --git a/test/extract.js b/test/extract.js index e6c21d81..b79b91cc 100644 --- a/test/extract.js +++ b/test/extract.js @@ -310,3 +310,30 @@ t.test('sync gzip error edge case test', async t => { t.end() }) + +t.test('brotli', async t => { + const file = path.resolve(__dirname, 'fixtures/example.tbr') + const dir = path.resolve(__dirname, 'brotli') + + t.beforeEach(async () => { + await mkdirp(dir) + }) + + t.afterEach(async () => { + await rimraf(dir) + }) + + t.test('fails if brotli', async t => { + const expect = new Error("TAR_BAD_ARCHIVE: Unrecognized archive format") + t.throws(_ => x({ sync: true, file: file }), expect) + }) + + t.test('succeeds', t => { + x({ sync: true, file: file, C: dir, brotli: true }) + + t.same(fs.readdirSync(dir + '/x').sort(), + ['1', '10', '2', '3', '4', '5', '6', '7', '8', '9']) + t.end() + }) +}) + diff --git a/test/fixtures/example.tbr b/test/fixtures/example.tbr new file mode 100644 index 00000000..01fbefa1 Binary files /dev/null and b/test/fixtures/example.tbr differ diff --git a/test/pack.js b/test/pack.js index 46246ab1..74158d7f 100644 --- a/test/pack.js +++ b/test/pack.js @@ -375,6 +375,13 @@ t.test('if gzip is truthy, make it an object', t => { t.end() }) +t.test('if brotli is truthy, make it an object', t => { + const opt = { brotli: true } + new Pack(opt) + t.type(opt.brotli, 'object') + t.end() +}) + t.test('gzip, also a very deep path', t => { const out = [] @@ -454,6 +461,85 @@ t.test('gzip, also a very deep path', t => { }) }) +t.test('brotli, also a very deep path', t => { + const out = [] + + new Pack({ + cwd: files, + brotli: { flush: 1 }, + }) + .add('dir') + .add('long-path') + .on('data', c => out.push(c)) + .end() + .on('end', _ => { + const zipped = Buffer.concat(out) + const data = zlib.brotliDecompressSync(zipped) + const entries = [] + for (var i = 0; i < data.length; i += 512) { + const slice = data.slice(i, i + 512) + const h = new Header(slice) + if (h.nullBlock) { + entries.push('null block') + } else if (h.cksumValid) { + entries.push([h.type, h.path]) + } else if (entries[entries.length - 1][0] === 'File') { + entries[entries.length - 1].push(slice.toString().replace(/\0.*$/, '')) + } + } + + const expect = [ + ['Directory', 'dir/'], + ['Directory', 'long-path/'], + ['File', 'dir/x'], + ['Directory', 'long-path/r/'], + ['Directory', 'long-path/r/e/'], + ['Directory', 'long-path/r/e/a/'], + ['Directory', 'long-path/r/e/a/l/'], + ['Directory', 'long-path/r/e/a/l/l/'], + ['Directory', 'long-path/r/e/a/l/l/y/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/a.txt', 'short\n'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'], + ['ExtendedHeader', 'PaxHeader/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222'], + ['ExtendedHeader', 'PaxHeader/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccc'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccccccccccccc', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'], + ['ExtendedHeader', 'PaxHeader/Ω.txt'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/Ω.txt', 'Ω'], + 'null block', + 'null block', + ] + + let ok = true + entries.forEach((entry, i) => { + ok = ok && + t.equal(entry[0], expect[i][0]) && + t.equal(entry[1], expect[i][1]) && + (!entry[2] || t.equal(entry[2], expect[i][2])) + }) + + // t.match(entries, expect) + t.end() + }) +}) + t.test('very deep gzip path, sync', t => { const pack = new PackSync({ cwd: files, @@ -533,6 +619,85 @@ t.test('very deep gzip path, sync', t => { t.end() }) +t.test('very deep brotli path, sync', t => { + const pack = new PackSync({ + cwd: files, + brotli: true, + }).add('dir') + .add('long-path') + .end() + + // these do nothing! + pack.pause() + pack.resume() + + const zipped = pack.read() + t.type(zipped, Buffer) + const data = zlib.brotliDecompressSync(zipped) + const entries = [] + for (var i = 0; i < data.length; i += 512) { + const slice = data.slice(i, i + 512) + const h = new Header(slice) + if (h.nullBlock) { + entries.push('null block') + } else if (h.cksumValid) { + entries.push([h.type, h.path]) + } else if (entries[entries.length - 1][0] === 'File') { + entries[entries.length - 1].push(slice.toString().replace(/\0.*$/, '')) + } + } + + const expect = [ + ['Directory', 'dir/'], + ['File', 'dir/x'], + ['Directory', 'long-path/'], + ['Directory', 'long-path/r/'], + ['Directory', 'long-path/r/e/'], + ['Directory', 'long-path/r/e/a/'], + ['Directory', 'long-path/r/e/a/l/'], + ['Directory', 'long-path/r/e/a/l/l/'], + ['Directory', 'long-path/r/e/a/l/l/y/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/'], + ['Directory', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/a.txt', 'short\n'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111'], + ['ExtendedHeader', 'PaxHeader/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', '2222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222'], + ['ExtendedHeader', 'PaxHeader/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccc'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxccccccccccccccccccccccccccccccccccccccccccccccccc', 'cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc'], + ['ExtendedHeader', 'PaxHeader/Ω.txt'], + ['File', 'long-path/r/e/a/l/l/y/-/d/e/e/p/-/f/o/l/d/e/r/-/p/a/t/h/Ω.txt', 'Ω'], + 'null block', + 'null block', + ] + + let ok = true + entries.forEach((entry, i) => { + ok = ok && + t.equal(entry[0], expect[i][0]) && + t.equal(entry[1], expect[i][1]) && + (!entry[2] || t.equal(entry[2], expect[i][2])) + }) + + // t.match(entries, expect) + t.end() +}) + t.test('write after end', t => { const p = new Pack() p.end() diff --git a/test/parse.js b/test/parse.js index 3066f7e8..0113e7b4 100644 --- a/test/parse.js +++ b/test/parse.js @@ -125,6 +125,30 @@ t.test('fixture tests', t => { bs.end(zlib.gzipSync(tardata)) }) + t.test('compress with brotli all at once', t => { + const p = new Parse({ + maxMetaEntrySize: maxMeta, + filter: filter ? (path, entry) => entry.size % 2 !== 0 : null, + strict: strict, + brotli: {} + }) + trackEvents(t, expect, p) + p.end(zlib.brotliCompressSync(tardata)) + }) + + t.test('compress with brotli byte at a time', t => { + const bs = new ByteStream() + const bp = new Parse({ + maxMetaEntrySize: maxMeta, + filter: filter ? (path, entry) => entry.size % 2 !== 0 : null, + strict: strict, + brotli: {}, + }) + trackEvents(t, expect, bp) + bs.pipe(bp) + bs.end(zlib.brotliCompressSync(tardata)) + }) + t.test('async chunks', t => { const p = new Parse({ maxMetaEntrySize: maxMeta, diff --git a/test/replace.js b/test/replace.js index 62c41eb8..75c97027 100644 --- a/test/replace.js +++ b/test/replace.js @@ -23,6 +23,7 @@ const fixtureDef = { 'zero.tar': Buffer.from(''), 'empty.tar': Buffer.alloc(512), 'compressed.tgz': zlib.gzipSync(data), + 'compressed.tbr': zlib.brotliCompressSync(data), } t.test('basic file add to archive (good or truncated)', t => { @@ -211,6 +212,30 @@ t.test('cannot append to gzipped archives', async t => { }, [path.basename(__filename)], er => t.match(er, expect)) }) +t.test('cannot append to brotli compressed archives', async t => { + const dir = t.testdir({ + 'compressed.tbr': fixtureDef['compressed.tbr'], + }) + const file = resolve(dir, 'compressed.tbr') + + const expect = new Error('cannot append to compressed archives') + const expectT = new TypeError('cannot append to compressed archives') + + t.throws(_ => r({ + file, + cwd: __dirname, + brotli: true, + }, [path.basename(__filename)]), expectT) + + t.throws(_ => r({ + file, + cwd: __dirname, + sync: true, + }, [path.basename(__filename)]), expect) + + t.end() +}) + t.test('other throws', t => { t.throws(_ => r({}, ['asdf']), new TypeError('file is required')) t.throws(_ => r({ file: 'asdf' }, []), diff --git a/test/update.js b/test/update.js index 0c8675f3..ff74c6e5 100644 --- a/test/update.js +++ b/test/update.js @@ -9,6 +9,7 @@ const { resolve } = require('path') const fixtures = path.resolve(__dirname, 'fixtures') const tars = path.resolve(fixtures, 'tars') const zlib = require('zlib') +const r = require("../lib/replace"); const spawn = require('child_process').spawn @@ -22,6 +23,7 @@ const fixtureDef = { 'zero.tar': Buffer.from(''), 'empty.tar': Buffer.alloc(512), 'compressed.tgz': zlib.gzipSync(data), + 'compressed.tbr': zlib.brotliCompressSync(data), } t.test('basic file add to archive (good or truncated)', t => { @@ -213,6 +215,30 @@ t.test('cannot append to gzipped archives', t => { }) }) +t.test('cannot append to brotli archives', t => { + const dir = t.testdir({ + 'compressed.tbr': fixtureDef['compressed.tbr'], + }) + const file = resolve(dir, 'compressed.tbr') + + const expect = new Error('cannot append to compressed archives') + const expectT = new TypeError('cannot append to compressed archives') + + t.throws(_ => u({ + file, + cwd: __dirname, + brotli: true, + }, [path.basename(__filename)]), expectT) + + t.throws(_ => u({ + file, + cwd: __dirname, + sync: true, + }, [path.basename(__filename)]), expect) + + t.end() +}) + t.test('other throws', t => { t.throws(_ => u({}, ['asdf']), new TypeError('file is required')) t.throws(_ => u({ file: 'asdf' }, []),