From dcf0d5be08cf0b49d2aa1d6f02a268de6da4d4c9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Fri, 24 Sep 2021 16:22:56 -0500 Subject: [PATCH] feat: Pull from GitHub instead of npm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fetch from ICU’s GitHub release instead of npm (ICU v50+) - set env FULL_ICU_PREFER_NPM to prefer npm instead - add .eslint Fixes: #36 --- .eslintrc.js | 17 +++++++++++ .gitignore | 2 ++ .npmignore | 3 +- .npmrc | 1 + .travis.yml | 14 ++++++++++ install-gh.js | 62 +++++++++++++++++++++++++++++++++++++++++ myFetch.js | 58 ++++++++++++++++++++++++++++++++++++++ package.json | 17 +++++++++-- postinstall.js | 12 ++++++-- test/data/haystack.zip | Bin 0 -> 1725 bytes test/test-unzipOne.js | 32 +++++++++++++++++++++ test/tmp/.keep | 0 unzipOne.js | 40 ++++++++++++++++++++++++++ 13 files changed, 252 insertions(+), 6 deletions(-) create mode 100644 .eslintrc.js create mode 100644 .npmrc create mode 100644 .travis.yml create mode 100644 install-gh.js create mode 100644 myFetch.js create mode 100644 test/data/haystack.zip create mode 100644 test/test-unzipOne.js create mode 100644 test/tmp/.keep create mode 100644 unzipOne.js diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000..5527d44 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,17 @@ +module.exports = { + "env": { + "commonjs": true, + "es6": true, + "node": true + }, + "extends": "standard", + "globals": { + "Atomics": "readonly", + "SharedArrayBuffer": "readonly" + }, + "parserOptions": { + "ecmaVersion": 2018 + }, + "rules": { + } +}; \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1f28d02..49c9602 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ node_modules *.dat npm-debug.log /yarn.lock +package-lock.json +/.nyc_output diff --git a/.npmignore b/.npmignore index c597d8d..ee3326c 100644 --- a/.npmignore +++ b/.npmignore @@ -2,4 +2,5 @@ node_modules .svn .git -npm-debug.log \ No newline at end of file +npm-debug.log +/test diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..43c97e7 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..3097174 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: node_js +node_js: +- '8' +- '10' +- '11' +- '12' +script: +- npm install +- npm t +cache: + directories: + - node_modules + - ".nvm" +# this is a comment diff --git a/install-gh.js b/install-gh.js new file mode 100644 index 0000000..78e13b3 --- /dev/null +++ b/install-gh.js @@ -0,0 +1,62 @@ +// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved. + +// Install by using spawn + +// var fs = require('fs'); +const { URL } = require('url') +const process = require('process') +const myFetch = require('./myFetch') +// const yauzl = require('yauzl'); + +// var isglobal = process.env.npm_config_global === 'true'; + +module.exports = async function installFromGithub (fullIcu, advice) { + const icupkg = fullIcu.icupkg + const icudat = fullIcu.icudat + + // var cmdPath = nodePath = process.env.npm_node_execpath; + + // var npmPath = process.env.npm_execpath; + + // var args; + // https://github.com/unicode-org/icu/releases/download/release-51-3/icu4c-51_3-src.zip + const _baseUrl = process.env.FULL_ICU_BASEURL || 'https://github.com/unicode-org/icu/releases/' + const baseUrl = new URL(_baseUrl) + const versionsAsHyphen = fullIcu.icuver.replace(/\./g, '-') + const versionsAsUnderscore = fullIcu.icuver.replace(/\./g, '_') + const tag = `release-${versionsAsHyphen}` + const file = `icu4c-${versionsAsUnderscore}-src.zip` + const fullUrl = new URL(`./download/${tag}/${file}`, baseUrl) + console.log(fullUrl.toString()) + const [srcZip, tmpd] = await myFetch(fullUrl) + + console.log(srcZip, tmpd) + // now, unpack it + +/* + if(spawned.error) { + throw(spawned.error); + } else if(spawned.status !== 0) { + throw(Error(cmdPath + ' ' + args.join(' ') + ' --> status ' + spawned.status)); + } else { + var datPath; + if(fs.existsSync(icudat)) { + console.log(' √ ' + icudat + " (existing link?)"); + } else if(!fs.existsSync(datPath)) { + console.log(' • ' + ' (no ' + icudat + ' at ‘' + datPath+'’)'); + } else { + try { + fs.linkSync(datPath, icudat); + console.log(' √ ' + icudat + " (link)"); + } catch(e) { + fs.symlinkSync(datPath, icudat); + console.log(' √ ' + icudat + " (symlink)"); + } + } + if(!fullIcu.haveDat()) { + throw Error('Somehow failed to install ' + icudat); + } else { + advice(); + } + } */ +} diff --git a/myFetch.js b/myFetch.js new file mode 100644 index 0000000..1ada1a1 --- /dev/null +++ b/myFetch.js @@ -0,0 +1,58 @@ +// Copyright (C) 2015-2016 IBM Corporation and Others. All Rights Reserved. + +const os = require('os') +const path = require('path') +const fs = require('fs') + +function getFetcher (u) { + if (u.protocol === 'https:') return require('https') + if (u.protocol === 'http:') return require('http') + return null +} + +/** + * @param {URL} fullUrl url to fetch + * @returns {Promse} filename, tmpdir + */ +function myFetch (fullUrl) { + return new Promise((resolve, reject) => { + const fetcher = getFetcher(fullUrl) + console.log('Fetch:', fullUrl.toString()) + if (!fetcher) { + return reject(Error(`Unknown URL protocol ${fullUrl.protocol} in ${fullUrl.toString()}`)) + } + + fetcher.get(fullUrl, res => { + const length = res.headers['content-length'] + if (res.statusCode === 302 && res.headers.location) { + return resolve(myFetch(new URL(res.headers.location))) + } else if (res.statusCode !== 200) { + return reject(Error(`Bad status code ${res.statusCode}`)) + } + const tmpd = fs.mkdtempSync(os.tmpdir()) + const tmpf = path.join(tmpd, 'icu-download.zip') + let gotSoFar = 0 + console.dir(tmpd) + + res.on('data', data => { + gotSoFar += data.length + fs.appendFileSync(tmpf, data) + // console.dir(res.headers); + process.stdout.write(`${gotSoFar}/${length}\r`) + // console.log(`chunk: ${data.length}`); + }) + res.on('end', () => { + resolve([tmpf, tmpd]) + console.log(`${gotSoFar}/${length}\n`) + }) + res.on('error', error => { + fs.unlinkSync(tmpf) + fs.rmdirSync(tmpd) + console.error(error) + return reject(error) + }) + }) + }) +} + +module.exports = myFetch diff --git a/package.json b/package.json index 000b6e7..14b1ac1 100644 --- a/package.json +++ b/package.json @@ -3,8 +3,9 @@ "version": "1.3.5-0", "description": "install 'full-icu' data for your current node", "scripts": { - "lint": "standard", - "postinstall": "node postinstall.js" + "lint": "standard && eslint *.js test/*.js", + "postinstall": "node postinstall.js", + "test": "tap test/*.js" }, "keywords": [ "icu4c" @@ -22,7 +23,17 @@ "bugs": { "url": "https://github.com/unicode-org/full-icu-npm/issues" }, + "dependencies": { + "yauzl": "^2.10.0" + }, "devDependencies": { - "standard": "^16.0.3" + "eslint": "^7.7.0", + "eslint-config-standard": "^16.0.3", + "eslint-plugin-header": "^3.0.0", + "eslint-plugin-import": "^2.24.2", + "eslint-plugin-node": "^11.1.0", + "eslint-plugin-promise": "^4.2.1", + "standard": "^16.0.3", + "tap": "^15.0.10" } } diff --git a/postinstall.js b/postinstall.js index c4241b4..3be3522 100644 --- a/postinstall.js +++ b/postinstall.js @@ -60,8 +60,16 @@ function advice () { console.log("... will show “enero”. If it shows “January” you don't have full data.") } -// install by using spawn -const npmInstall = require('./install-spawn') +// Choose install method +let npmInstall + +// GitHub has v50+ as releases +// Experimentally, pull from GitHub for little endian +if (fullIcu.icuend === 'l' && !process.env.FULL_ICU_PREFER_NPM) { + npmInstall = require('./install-gh') +} else { + npmInstall = require('./install-spawn') +} if (fs.existsSync(fullIcu.icudat)) { console.log('√ ' + fullIcu.icudat + ' Already there (for Node ' + fullIcu.nodever + ' and small-icu ' + fullIcu.icuver + ')') diff --git a/test/data/haystack.zip b/test/data/haystack.zip new file mode 100644 index 0000000000000000000000000000000000000000..c30ccab406611dac681ed798f84c3030ad09bd93 GIT binary patch literal 1725 zcma)+Jxjwt7{{OHrC=>>2L}gnaW}XNzAOr&6x6j)P5J^Nw5D1Fhl05I0UQf{3c=NH z&dbIc_XYvf+^COxd#e4T3E zUyaDy;D2?ESx;ndDyGl1o$E4vo8%X~GnR38J>OROhj`k5+KuP2xD6|)tb(xOP3T!v z5C&1j^hrqtNDq?yqpbqukNusc&yNMGBX=JJ_fZtyMXCdk=EXAO=`3ob&+||_4rXDP z9A(JiQ4wIyqKIKIs+hjicE&6qJx=oPyfc<@7W8~e<-_qbRy7zoHD}v3`LHR?=+N&G zE@6kC*ikr-nd4EGvoAO>#ZJ=ayq+UHOA@SMR;e-RsdQFbv4}nwUy7A6$PU#Va1<1k zx%}RMP4*krIr+*C1P5l>Nky-+<-bT0EMpejSn(i>jwLLHaAn_$l{L;zEvlA1VbdWY z9&;exiqq+wd}pVE0}1UUU7`G#y-5;;WLBw+>6q!PmYN83_M=#-oF=AU=bEX>-w&6l H#f#`4YC!@{ literal 0 HcmV?d00001 diff --git a/test/test-unzipOne.js b/test/test-unzipOne.js new file mode 100644 index 0000000..eca01a3 --- /dev/null +++ b/test/test-unzipOne.js @@ -0,0 +1,32 @@ +const tap = require('tap') +const fs = require('fs') +const unzipOne = require('../unzipOne') + +tap.test('unzipOne', async t => { + t.test('setup', t => { + try { + fs.unlinkSync('test/tmp/needle.txt') + } catch (e) { /* ignore */ } + t.end() + }) + t.test('no easteregg in haystack.zip', async t => { + const ee = await unzipOne('./test/data/haystack.zip', 'easteregg.txt', './test/tmp/') + t.notOk(ee, 'Did not expect to find easteregg in haystack: ' + ee) + t.end() + }) + t.test('get needle.txt in haystack.zip', async t => { + const ee = await unzipOne('./test/data/haystack.zip', 'needle.txt', './test/tmp/') + t.ok(ee, 'Did expect to find needle.txt in haystack: ' + ee) + const truism = fs.readFileSync('./test/tmp/needle.txt', 'utf-8') + t.ok(truism) + t.equal(truism.trim(), 'true') + t.end() + }) + t.test('cleanup', t => { + try { + fs.unlinkSync('test/tmp/needle.txt') + } catch (e) { /* ignore */ } + t.end() + }) + t.end() +}) diff --git a/test/tmp/.keep b/test/tmp/.keep new file mode 100644 index 0000000..e69de29 diff --git a/unzipOne.js b/unzipOne.js new file mode 100644 index 0000000..68ab271 --- /dev/null +++ b/unzipOne.js @@ -0,0 +1,40 @@ +const yauzl = require('yauzl') +const { basename, join } = require('path') +const fs = require('fs') + +/** + * unzip and write file 'fn' to 'dstDir' + * @param {String} srcZip source zipfile + * @param {String} fn to unzip + * @param {String} dstDir destination dir + * @returns {Promise} to output filename if successful, or falsy if the file was not found. + */ +function unzipOne (srcZip, fn, dstDir) { + const outFile = join(dstDir, fn) + return new Promise((resolve, reject) => { + yauzl.open(srcZip, { lazyEntries: true }, + (err, zipfile) => { + if (err) return reject(err) + zipfile.readEntry() + zipfile.on('entry', entry => { + if (basename(entry.fileName) === fn) { + zipfile.openReadStream(entry, (err, readStream) => { + if (err) return reject(err) + readStream.on('end', () => { + zipfile.close() + resolve(entry.fileName) + }) + readStream.pipe(fs.createWriteStream(outFile)) + }) + } else { + zipfile.readEntry() + } + }) + zipfile.on('end', () => { + resolve() // not found + }) + }) + }) +} + +module.exports = unzipOne