From fb560cf796c0b9ff6ffabf80411207e1309605ef Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 15:35:45 -0700 Subject: [PATCH 01/17] [plugin/install] plugins need to have their own deps (support no internet installs) --- src/cli/plugin/__tests__/npmInstall.js | 69 -------------------------- src/cli/plugin/npmInstall.js | 35 ------------- src/cli/plugin/pluginDownloader.js | 17 +++---- src/cli/plugin/pluginInstaller.js | 4 -- 4 files changed, 7 insertions(+), 118 deletions(-) delete mode 100644 src/cli/plugin/__tests__/npmInstall.js delete mode 100644 src/cli/plugin/npmInstall.js diff --git a/src/cli/plugin/__tests__/npmInstall.js b/src/cli/plugin/__tests__/npmInstall.js deleted file mode 100644 index 7a8ee7596b31b..0000000000000 --- a/src/cli/plugin/__tests__/npmInstall.js +++ /dev/null @@ -1,69 +0,0 @@ -var expect = require('expect.js'); -var nock = require('nock'); -var glob = require('glob'); -var rimraf = require('rimraf'); -var fs = require('fs'); -var { join } = require('path'); -var sinon = require('sinon'); - -var pluginLogger = require('../pluginLogger'); -var npmInstall = require('../npmInstall'); - -describe('kibana cli', function () { - - describe('plugin installer', function () { - - describe('npmInstall', function () { - - var testWorkingPath = join(__dirname, '.test.data'); - var logger; - var statSyncStub; - - beforeEach(function () { - statSyncStub = undefined; - logger = pluginLogger(false); - rimraf.sync(testWorkingPath); - sinon.stub(logger, 'log'); - sinon.stub(logger, 'error'); - }); - - afterEach(function () { - logger.log.restore(); - logger.error.restore(); - rimraf.sync(testWorkingPath); - if (statSyncStub) statSyncStub.restore(); - }); - - it('should throw an error if there is no package.json file in the archive', function () { - fs.mkdirSync(testWorkingPath); - - var errorStub = sinon.stub(); - return npmInstall(testWorkingPath, logger) - .catch(errorStub) - .then(function (data) { - expect(errorStub.called).to.be(true); - expect(errorStub.lastCall.args[0].message).to.match(/package.json/); - }); - }); - - it('should rethrow any errors other than "ENOENT" from fs.statSync', function () { - fs.mkdirSync(testWorkingPath); - - statSyncStub = sinon.stub(fs, 'statSync', function () { - throw new Error('This is unexpected.'); - }); - - var errorStub = sinon.stub(); - return npmInstall(testWorkingPath, logger) - .catch(errorStub) - .then(function (data) { - expect(errorStub.called).to.be(true); - expect(errorStub.lastCall.args[0].message).to.match(/This is unexpected./); - }); - }); - - }); - - }); - -}); diff --git a/src/cli/plugin/npmInstall.js b/src/cli/plugin/npmInstall.js deleted file mode 100644 index a8e1ee34c5de4..0000000000000 --- a/src/cli/plugin/npmInstall.js +++ /dev/null @@ -1,35 +0,0 @@ -var Promise = require('bluebird'); -var fs = require('fs'); -var path = require('path'); -var exec = require('child_process').exec; - -module.exports = function (dest, logger) { - return new Promise(function (resolve, reject) { - //throw an exception if package.json does not exist - try { - var packageFile = path.join(dest, 'package.json'); - fs.statSync(packageFile); - } catch (e) { - if (e.code !== 'ENOENT') throw e; - - return reject(new Error('Plugin does not contain package.json file')); - } - - var cmd = '"' + path.resolve(path.dirname(process.execPath), 'npm').replace(/\\/g, '/') + '" install --production'; - - var child = exec(cmd, { cwd: dest }); - child.on('error', function (err) { - reject(err); - }); - child.on('exit', function (code, signal) { - if (code === 0) { - resolve(); - } else { - reject(new Error('npm install failed with code ' + code)); - } - }); - - logger.error(child.stderr); - logger.log(child.stdout); - }); -}; diff --git a/src/cli/plugin/pluginDownloader.js b/src/cli/plugin/pluginDownloader.js index 8cbe5581ec408..15884a604a5a7 100644 --- a/src/cli/plugin/pluginDownloader.js +++ b/src/cli/plugin/pluginDownloader.js @@ -1,7 +1,7 @@ var _ = require('lodash'); var zlib = require('zlib'); var Promise = require('bluebird'); -var request = require('request'); +var wreck = require('wreck'); var tar = require('tar'); var progressReporter = require('./progressReporter'); @@ -48,12 +48,7 @@ module.exports = function (settings, logger) { var gunzip = zlib.createGunzip(); var tarExtract = new tar.Extract({ path: dest, strip: 1 }); - var requestOptions = { url: source }; - if (timeout !== 0) { - requestOptions.timeout = timeout; - } - - return wrappedRequest(requestOptions) + return wrappedRequest(source, timeout) .then(function (req) { var reporter = progressReporter(logger, req); @@ -71,9 +66,11 @@ module.exports = function (settings, logger) { }); } - function wrappedRequest(requestOptions) { - return Promise.try(function () { - return request.get(requestOptions); + function wrappedRequest(url, timeout) { + if (timeout) timeout = false; + + return Promise.fromNode(cb => { + return wreck.defaults({ timeout }).request('GET', url, null, cb); }) .catch(function (err) { if (err.message.match(/invalid uri/i)) { diff --git a/src/cli/plugin/pluginInstaller.js b/src/cli/plugin/pluginInstaller.js index aedfc50dfb750..fae2d150d45ee 100644 --- a/src/cli/plugin/pluginInstaller.js +++ b/src/cli/plugin/pluginInstaller.js @@ -1,6 +1,5 @@ var pluginDownloader = require('./pluginDownloader'); var pluginCleaner = require('./pluginCleaner'); -var npmInstall = require('./npmInstall'); var fs = require('fs'); module.exports = { @@ -26,9 +25,6 @@ function install(settings, logger) { .then(function () { return downloader.download(); }) - .then(function () { - return npmInstall(settings.workingPath, logger); - }) .then(function (curious) { fs.renameSync(settings.workingPath, settings.pluginPath); logger.log('Plugin installation complete'); From ee83c4103cca55e2ad31125fb751b6312f4db796 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 15:43:58 -0700 Subject: [PATCH 02/17] [build/downloadNode] async/await --- Gruntfile.js | 9 +++- tasks/downloadNodeBins.js | 80 +++++++++++++++++++++++++++++++++ tasks/download_node_binaries.js | 65 --------------------------- 3 files changed, 87 insertions(+), 67 deletions(-) create mode 100644 tasks/downloadNodeBins.js delete mode 100644 tasks/download_node_binaries.js diff --git a/Gruntfile.js b/Gruntfile.js index 9e837033d02c6..fee7d99969287 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -26,8 +26,13 @@ module.exports = function (grunt) { } }()), - nodeVersion: '0.10.35', - platforms: ['darwin-x64', 'linux-x64', 'linux-x86', 'windows'], + nodeVersion: '2.5.0', + platforms: [ + 'darwin-x64', + 'linux-x64', + 'linux-x86', + 'windows' + ], services: [ ['launchd', '10.9'], ['upstart', '1.5'], diff --git a/tasks/downloadNodeBins.js b/tasks/downloadNodeBins.js new file mode 100644 index 0000000000000..988252dc75ffa --- /dev/null +++ b/tasks/downloadNodeBins.js @@ -0,0 +1,80 @@ +module.exports = function (grunt) { + grunt.registerTask('downloadNodeBins', function () { + let { map, fromNode, promisify } = require('bluebird'); + let { resolve } = require('path'); + let { createWriteStream } = require('fs'); + let { createGunzip } = require('zlib'); + let { Extract } = require('tar'); + + let mkdirp = promisify(require('mkdirp')); + let rename = promisify(require('fs').rename); + let get = (uri) => fromNode(cb => require('wreck').request('GET', uri, null, cb)); + + let exists = (path) => fromNode(cb => { + require('fs').stat(path, err => cb(null, !err)); + }); + + let platforms = grunt.config.get('platforms'); + let nodeVersion = grunt.config.get('nodeVersion'); + + let nodeDir = require('requirefrom')('src/utils')('fromRoot')(`.node_binaries/${nodeVersion}`); + let baseUri = `https://iojs.org/dist/v${nodeVersion}`; + + let writeTar = async (to, from) => { + await fromNode(cb => { + from + .pipe(createGunzip()) + .on('error', cb) + .pipe(new Extract({ path: to, strip: 1 })) + .on('error', cb) + .on('end', cb); + }); + }; + + let writeExe = async (to, from) => { + let winBinDir = resolve(to, 'bin'); + await mkdirp(winBinDir); + await fromNode(cb => { + from + .pipe(createWriteStream(resolve(winBinDir, 'node.exe'))) + .on('error', cb) + .on('finish', cb); + }); + }; + + mkdirp(nodeDir) + .then(function () { + grunt.log.ok(`downloading node binaries for ${platforms.join(', ')}`); + return platforms; + }) + .map(async function (platform) { + let finalDir = resolve(nodeDir, `${platform}`); + let downloadDir = `${finalDir}.temp`; + + if (await exists(finalDir)) { + grunt.log.ok(`${platform} download exists`); + return; + } + + let uri; + if (platform === 'windows') { + uri = `${baseUri}/win-x64/iojs.exe`; + } else { + uri = `${baseUri}/iojs-v${nodeVersion}-${platform}.tar.gz`; + } + + let resp = await get('GET', uri); + if (resp.statusCode !== 200) { + throw new Error(uri + ' failed with a ' + resp.statusCode); + } + + let write = platform === 'windows' ? writeExe : writeTar; + await write(downloadDir, resp); + await rename(downloadDir, finalDir); + + grunt.log.ok(`${platform} download complete`); + }) + .nodeify(this.async()); + }); +}; + diff --git a/tasks/download_node_binaries.js b/tasks/download_node_binaries.js deleted file mode 100644 index 758803c01d31c..0000000000000 --- a/tasks/download_node_binaries.js +++ /dev/null @@ -1,65 +0,0 @@ -var _ = require('lodash'); -var zlib = require('zlib'); -var tar = require('tar'); -var request = require('request'); -var mkdirp = require('mkdirp'); -var fs = require('fs'); -var { join } = require('path'); -var filesPatern = _.template('node-v<%- version %>-<%- platform %>.tar.gz'); -var urlPattern = _.template('http://nodejs.org/dist/v<%- version %>/<%- file %>'); -var Promise = require('bluebird'); - -module.exports = function (grunt) { - grunt.registerTask('download_node_binaries', 'Download the node.js binaries', function () { - var platforms = _.without(grunt.config.get('platforms'), 'windows'); - var rootPath = grunt.config.get('root'); - var version = grunt.config.get('nodeVersion'); - - var handle404 = function (response) { - if (response.statusCode !== 200) { - throw new Error(response.request.href + ' failed with a ' + response.statusCode); - } - }; - - var downloadWindows = function (cb) { - return new Promise(function (resolve, reject) { - var dest = join(rootPath, '.node_binaries', 'windows'); - fs.stat(dest, function (err) { - if (!err) return resolve(); // skip downloading if we already have them - var url = urlPattern({ version: version, file: 'node.exe'}); - mkdirp(dest, function (err) { - if (err) return reject(err); - var out = fs.createWriteStream(join(dest, 'node.exe')); - out.on('close', resolve).on('error', reject); - var req = request.get(url); - req.on('response', handle404); - req.pipe(out); - }); - }); - }); - }; - - var download = function (platform) { - return new Promise(function (resolve, reject) { - var dest = join(rootPath, '.node_binaries', platform); - fs.stat(dest, function (err) { - if (!err) return resolve(); // skip downloading if we already have them - var file = filesPatern({ version: version, platform: platform }); - var url = urlPattern({ version: version, file: file }); - mkdirp(dest, function (err) { - if (err) return reject(err); - var unzip = zlib.createGunzip(); - var out = new tar.Extract({ path: dest, strip: 1 }); - out.on('close', resolve).on('error', reject); - var req = request.get(url); - req.on('response', handle404); - req.pipe(unzip).pipe(out); - }); - }); - }); - }; - - return Promise.map(platforms, download).then(downloadWindows).nodeify(this.async()); - }); -}; - From 33df0deee1bcb5deea60f7e3145e569f6e3925cc Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 15:45:52 -0700 Subject: [PATCH 03/17] [cli/serve] disable --dev options if serve/cluster is removed --- src/cli/serve/serve.js | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/cli/serve/serve.js b/src/cli/serve/serve.js index 0af3b370a8a6b..6515aab6a07fe 100644 --- a/src/cli/serve/serve.js +++ b/src/cli/serve/serve.js @@ -6,6 +6,14 @@ let cwd = process.cwd(); let src = require('requirefrom')('src'); let fromRoot = src('utils/fromRoot'); +let canCluster; +try { + require.resolve('../cluster/ClusterManager'); + canCluster = true; +} catch (e) { + canCluster = false; +} + let pathCollector = function () { let paths = []; return function (path) { @@ -18,8 +26,9 @@ let pluginDirCollector = pathCollector(); let pluginPathCollector = pathCollector(); module.exports = function (program) { - program - .command('serve') + let command = program.command('serve'); + + command .description('Run the kibana server') .collectUnknownOptions() .option('-e, --elasticsearch ', 'Elasticsearch instance') @@ -47,11 +56,17 @@ module.exports = function (program) { pluginPathCollector, [] ) - .option('--plugins ', 'an alias for --plugin-dir', pluginDirCollector) - .option('--dev', 'Run the server with development mode defaults') - .option('--no-watch', 'Prevents automatic restarts of the server in --dev mode') + .option('--plugins ', 'an alias for --plugin-dir', pluginDirCollector); + + if (canCluster) { + command + .option('--dev', 'Run the server with development mode defaults') + .option('--no-watch', 'Prevents automatic restarts of the server in --dev mode'); + } + + command .action(async function (opts) { - if (opts.dev && !isWorker) { + if (canCluster && opts.dev && !isWorker) { // stop processing the action and handoff to cluster manager let ClusterManager = require('../cluster/ClusterManager'); new ClusterManager(opts); From ce366a06f46c728d443a788bbd760121b3577079 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 21:16:54 -0700 Subject: [PATCH 04/17] [build] initial fix --- .gitignore | 1 + Gruntfile.js | 24 ++-- package.json | 4 +- .../styles/{.editor.less => _editor.less} | 0 .../kibana/public/visualize/styles/main.less | 2 +- tasks/build.js | 26 ---- tasks/build/build.js | 29 ++++ tasks/build/cliIndex.js | 18 +++ tasks/build/downloadNodes.js | 87 ++++++++++++ .../{get_build_props.js => build/getProps.js} | 4 +- tasks/build/installNpmDeps.js | 16 +++ tasks/build/installedPlugins.js | 5 + tasks/build/linkNodes.js | 31 ++++ tasks/build/osPackages.js | 52 +++++++ tasks/build/packageJson.js | 21 +++ tasks/build/packages.js | 42 ++++++ tasks/build/pleaseManageUser.js | 10 ++ tasks/build/pleaserun.js | 25 ++++ tasks/build/readme.js | 37 +++++ tasks/{create_shasums.js => build/shasums.js} | 13 +- tasks/config/babel.js | 19 +++ tasks/config/clean.js | 5 +- tasks/config/compress.js | 49 ------- tasks/config/copy.js | 133 +++--------------- tasks/config/packages.js | 22 +++ tasks/config/run.js | 15 ++ tasks/config/s3.js | 36 +++-- tasks/config/services.js | 14 ++ tasks/create_packages.js | 76 ---------- tasks/create_services.js | 51 ------- tasks/dist_package_json.js | 19 --- tasks/dist_readme.js | 61 -------- tasks/downloadNodeBins.js | 80 ----------- tasks/licenses.js | 2 +- tasks/npm_install_kibana.js | 21 --- tasks/plugins.js | 27 ---- 36 files changed, 501 insertions(+), 576 deletions(-) rename src/plugins/kibana/public/visualize/editor/styles/{.editor.less => _editor.less} (100%) delete mode 100644 tasks/build.js create mode 100644 tasks/build/build.js create mode 100644 tasks/build/cliIndex.js create mode 100644 tasks/build/downloadNodes.js rename tasks/{get_build_props.js => build/getProps.js} (52%) create mode 100644 tasks/build/installNpmDeps.js create mode 100644 tasks/build/installedPlugins.js create mode 100644 tasks/build/linkNodes.js create mode 100644 tasks/build/osPackages.js create mode 100644 tasks/build/packageJson.js create mode 100644 tasks/build/packages.js create mode 100644 tasks/build/pleaseManageUser.js create mode 100644 tasks/build/pleaserun.js create mode 100644 tasks/build/readme.js rename tasks/{create_shasums.js => build/shasums.js} (57%) create mode 100644 tasks/config/babel.js delete mode 100644 tasks/config/compress.js create mode 100644 tasks/config/packages.js create mode 100644 tasks/config/services.js delete mode 100644 tasks/create_packages.js delete mode 100644 tasks/create_services.js delete mode 100644 tasks/dist_package_json.js delete mode 100644 tasks/dist_readme.js delete mode 100644 tasks/downloadNodeBins.js delete mode 100644 tasks/npm_install_kibana.js delete mode 100644 tasks/plugins.js diff --git a/.gitignore b/.gitignore index 114956a07d99b..296c6fa548de2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules trash bundles target +/build .jruby .idea *.iml diff --git a/Gruntfile.js b/Gruntfile.js index fee7d99969287..9a6d44afd0711 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -27,20 +27,6 @@ module.exports = function (grunt) { }()), nodeVersion: '2.5.0', - platforms: [ - 'darwin-x64', - 'linux-x64', - 'linux-x86', - 'windows' - ], - services: [ - ['launchd', '10.9'], - ['upstart', '1.5'], - ['systemd', 'default'], - ['sysv', 'lsb-3.1'] - ], - - devPlugins: 'devMode', meta: { banner: '/*! <%= package.name %> - v<%= package.version %> - ' + @@ -49,6 +35,7 @@ module.exports = function (grunt) { ' * Copyright (c) <%= grunt.template.today("yyyy") %> <%= package.author.company %>;' + ' Licensed <%= package.license %> */\n' }, + lintThese: [ 'Gruntfile.js', '<%= root %>/tasks/**/*.js', @@ -59,6 +46,12 @@ module.exports = function (grunt) { grunt.config.merge(config); + config.userScriptsDir = __dirname + '/build/userScripts'; + config.services = require('./tasks/config/services')(grunt); + config.platforms = require('./tasks/config/packages')(grunt); + + grunt.config.merge(config); + // load plugins require('load-grunt-config')(grunt, { configPath: __dirname + '/tasks/config', @@ -69,6 +62,9 @@ module.exports = function (grunt) { } }); + + // load task definitions grunt.task.loadTasks('tasks'); + grunt.task.loadTasks('tasks/build'); }; diff --git a/package.json b/package.json index 7e2e8217d1a5f..d71a084c5e652 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "babel": "^5.8.21", "babel-core": "^5.8.21", "babel-loader": "^5.3.2", + "babel-runtime": "^5.8.20", "bluebird": "^2.9.27", "boom": "^2.8.0", "bootstrap": "^3.3.5", @@ -117,6 +118,7 @@ "webpack": "^1.10.0", "webpack-directory-name-as-main": "^1.0.0", "whatwg-fetch": "^0.9.0", + "wreck": "^6.1.0", "zeroclipboard": "^2.2.0" }, "devDependencies": { @@ -126,9 +128,9 @@ "expect.js": "^0.3.1", "faker": "^1.1.0", "grunt": "^0.4.5", + "grunt-babel": "^5.0.1", "grunt-cli": "0.1.13", "grunt-contrib-clean": "^0.6.0", - "grunt-contrib-compress": "^0.13.0", "grunt-contrib-copy": "^0.8.0", "grunt-esvm": "^1.1.3", "grunt-karma": "^0.12.0", diff --git a/src/plugins/kibana/public/visualize/editor/styles/.editor.less b/src/plugins/kibana/public/visualize/editor/styles/_editor.less similarity index 100% rename from src/plugins/kibana/public/visualize/editor/styles/.editor.less rename to src/plugins/kibana/public/visualize/editor/styles/_editor.less diff --git a/src/plugins/kibana/public/visualize/styles/main.less b/src/plugins/kibana/public/visualize/styles/main.less index 4dc46a0b3c75a..19bd8b643bcaa 100644 --- a/src/plugins/kibana/public/visualize/styles/main.less +++ b/src/plugins/kibana/public/visualize/styles/main.less @@ -56,4 +56,4 @@ } -@import "../editor/styles/.editor.less"; +@import "../editor/styles/_editor.less"; diff --git a/tasks/build.js b/tasks/build.js deleted file mode 100644 index d0e796011ac48..0000000000000 --- a/tasks/build.js +++ /dev/null @@ -1,26 +0,0 @@ -module.exports = function (grunt) { - grunt.registerTask('build', [ - 'get_build_props', - 'clean:target', - 'clean:build', - 'less:build', - 'copy:kibana_src', - 'clean:noDistPlugins', - - 'webpack:build', - 'clean:unneeded_source_in_build', - - 'copy:dist', - 'dist_readme', - 'dist_package_json', - 'chmod_kibana', - 'make_plugin_dir', - 'copy:plugin_readme', - 'clean:test_from_node_modules', - 'download_node_binaries', - 'copy:versioned_dist', - 'create_services', - 'create_packages', - 'create_shasums' - ]); -}; diff --git a/tasks/build/build.js b/tasks/build/build.js new file mode 100644 index 0000000000000..397901f54ce39 --- /dev/null +++ b/tasks/build/build.js @@ -0,0 +1,29 @@ +module.exports = function (grunt) { + let { flatten } = require('lodash'); + + grunt.registerTask('build', flatten([ + 'build-getProps', + 'clean:build', + 'clean:target', + 'downloadNodes:start', + 'copy:devSource', + 'babel:build', + 'build-cliIndex', + 'build-installedPlugins', + 'build-packageJson', + 'build-readme', + 'build-installNpmDeps', + 'run:optimizeBuild', + 'stop:optimizeBuild', + 'downloadNodes:finish', + 'clean:testsFromModules', + 'build-versionedLinks', + 'build-archives', + !grunt.option('os-packages') ? [] : [ + 'build-pleaseRun', + 'build-pleaseManageUser', + 'build-osPackages', + ], + 'build-shasums' + ])); +}; diff --git a/tasks/build/cliIndex.js b/tasks/build/cliIndex.js new file mode 100644 index 0000000000000..308cc82cae80c --- /dev/null +++ b/tasks/build/cliIndex.js @@ -0,0 +1,18 @@ +module.exports = function (grunt) { + + var file = 'build/kibana/src/cli/index.js'; + var blurb = `require('babel/register')(require('../optimize/babelOptions'));\n`; + + grunt.registerTask('build-cliIndex', function () { + var before = grunt.file.read(file); + var after = before.replace(blurb, ''); + + if (before === after) { + grunt.log.error(`unable to remove "${blurb}" from ${file}`); + return; + } + + grunt.file.write(file, after); + }); + +}; diff --git a/tasks/build/downloadNodes.js b/tasks/build/downloadNodes.js new file mode 100644 index 0000000000000..a437d7bf23dc6 --- /dev/null +++ b/tasks/build/downloadNodes.js @@ -0,0 +1,87 @@ +module.exports = function (grunt) { + let { map, fromNode, promisify } = require('bluebird'); + let { resolve } = require('path'); + let { createWriteStream } = require('fs'); + let { createGunzip } = require('zlib'); + let { Extract } = require('tar'); + + let mkdirp = promisify(require('mkdirp')); + let rename = promisify(require('fs').rename); + let get = (uri) => fromNode(cb => require('wreck').request('GET', uri, null, cb)); + + let exists = (path) => fromNode(cb => { + require('fs').stat(path, err => cb(null, !err)); + }); + + let platforms = grunt.config.get('platforms'); + let nodeVersion = grunt.config.get('nodeVersion'); + let baseUri = `https://iojs.org/dist/v${nodeVersion}`; + + let writeTar = async (to, from) => { + await fromNode(cb => { + from + .pipe(createGunzip()) + .on('error', cb) + .pipe(new Extract({ path: to, strip: 1 })) + .on('error', cb) + .on('end', cb); + }); + }; + + let writeExe = async (to, from) => { + let winBinDir = resolve(to, 'bin'); + await mkdirp(winBinDir); + await fromNode(cb => { + from + .pipe(createWriteStream(resolve(winBinDir, 'node.exe'))) + .on('error', cb) + .on('finish', cb); + }); + }; + + let run = () => { + grunt.log.ok(`downloading node binaries`); + + return map(platforms, async function (platform) { + grunt.file.mkdir(platform.nodeDir); + + let finalDir = resolve(grunt.config.get('root'), platform.nodeDir); + let downloadDir = `${finalDir}.temp`; + + if (await exists(finalDir)) { + grunt.log.debug(`${platform} download exists`); + return; + } + + let uri; + if (platform === 'windows') { + uri = `${baseUri}/win-x64/iojs.exe`; + } else { + uri = `${baseUri}/iojs-v${nodeVersion}-${platform}.tar.gz`; + } + + let resp = await get('GET', uri); + if (resp.statusCode !== 200) { + throw new Error(uri + ' failed with a ' + resp.statusCode); + } + + let write = platform === 'windows' ? writeExe : writeTar; + await write(downloadDir, resp); + await rename(downloadDir, finalDir); + + grunt.log.debug(`${platform} download complete`); + }); + }; + + let current = null; + let start = () => current || (current = run()); + + grunt.registerTask('downloadNodes:start', start); + grunt.registerTask('downloadNodes:finish', function () { + start().then(() => { + grunt.log.ok(`node binaries for ${platforms.join(', ')} downloaded`); + }) + .nodeify(this.async()); + }); +}; + diff --git a/tasks/get_build_props.js b/tasks/build/getProps.js similarity index 52% rename from tasks/get_build_props.js rename to tasks/build/getProps.js index 9c2dc15476195..2d031eec93d84 100644 --- a/tasks/get_build_props.js +++ b/tasks/build/getProps.js @@ -1,7 +1,7 @@ module.exports = function (grunt) { var exec = require('child_process').execSync; - grunt.registerTask('get_build_props', function () { + grunt.registerTask('build-getProps', function () { grunt.config.set('buildSha', String(exec('git rev-parse HEAD')).trim()); - grunt.config.set('buildNum', parseFloat(exec('git log --format="%h" | wc -l')).trim()); + grunt.config.set('buildNum', parseFloat(String(exec('git log --format="%h" | wc -l')).trim())); }); }; diff --git a/tasks/build/installNpmDeps.js b/tasks/build/installNpmDeps.js new file mode 100644 index 0000000000000..91c1acab6d68e --- /dev/null +++ b/tasks/build/installNpmDeps.js @@ -0,0 +1,16 @@ +module.exports = function (grunt) { + var { execSync } = require('child_process'); + var { resolve } = require('path'); + + grunt.registerTask('build-installNpmDeps', function () { + grunt.file.mkdir('build/kibana/node_modules'); + + execSync('npm install --production --no-optional', { + cwd: grunt.config.process('<%= root %>/build/kibana') + }); + + grunt.log.ok('done'); + }); +}; + + diff --git a/tasks/build/installedPlugins.js b/tasks/build/installedPlugins.js new file mode 100644 index 0000000000000..d66f0b8c5610c --- /dev/null +++ b/tasks/build/installedPlugins.js @@ -0,0 +1,5 @@ +module.exports = function (grunt) { + grunt.registerTask('build-installedPlugins', function () { + grunt.file.mkdir('build/kibana/installedPlugins'); + }); +}; diff --git a/tasks/build/linkNodes.js b/tasks/build/linkNodes.js new file mode 100644 index 0000000000000..fcf108681b402 --- /dev/null +++ b/tasks/build/linkNodes.js @@ -0,0 +1,31 @@ +module.exports = function (grunt) { + let { execFileSync } = require('child_process'); + let { basename, resolve } = require('path'); + let { forOwn } = require('lodash'); + + grunt.registerTask('build-versionedLinks', function () { + let buildFiles = grunt.file.expand('build/kibana/{*,.*}'); + let rootDir = grunt.config.get('root'); + + let buildMap = buildFiles.reduce(function (map, file) { + map[file] = basename(file); + return map; + }, {}); + + let ln = (source, link) => { + execFileSync('ln', [ + '-s', + resolve(rootDir, source), + resolve(rootDir, link) + ]); + }; + + grunt.config.get('platforms').forEach(function (platform) { + grunt.file.mkdir(platform.buildDir); + forOwn(buildMap, function (link, source) { + ln(source, resolve(platform.buildDir, link)); + }); + ln(platform.nodeDir, resolve(platform.buildDir, 'node')); + }); + }); +}; diff --git a/tasks/build/osPackages.js b/tasks/build/osPackages.js new file mode 100644 index 0000000000000..755e07765e1dc --- /dev/null +++ b/tasks/build/osPackages.js @@ -0,0 +1,52 @@ +module.exports = function (grunt) { + let { resolve } = require('path'); + let { indexBy } = require('lodash'); + let { execFileSync } = require('child_process'); + + let { config } = grunt; + let targetDir = config.get('target'); + let version = config.get('pkg.version'); + let userScriptsDir = config.get('userScriptsDir'); + let servicesByName = indexBy(config.get('services'), 'id'); + + grunt.registerTask('build-osPackages', function () { + grunt.config.get('platforms').forEach(({ name, buildDir }) => { + + let arch = /x64$/.test(name) ? 'x86_64' : 'i686'; + let files = buildDir + '/=/opt/kibana'; + let fpm = args => execFileSync('fpm', args); + + let args = [ + '-f', + '-p', targetDir, + '-s', 'dir', + '-n', 'kibana', + '-v', version, + '--after-install', resolve(userScriptsDir, 'installer.sh'), + '--after-remove', resolve(userScriptsDir, 'remover.sh'), + '--config-files', '/opt/kibana/config/kibana.yml' + ]; + + grunt.file.mkdir(targetDir); + + // TODO(sissel): Check if `fpm` is available + if (/linux-x(86|64)$/.test(name)) { + // kibana.rpm and kibana.deb + let sysv = servicesByName.sysv; + let sysvInit = sysv.outputDir + '/etc/=/etc/'; + fpm(args.concat('-t', 'rpm', '-a', arch, '--rpm-os', 'linux', files, sysvInit)); + fpm(args.concat('-t', 'deb', '-a', arch, files, sysvInit)); + return; + } + + if (/darwin-x(86|64)$/.test(name)) { + // kibana.pkg + let launchd = servicesByName.launchd; + let launchdInit = launchd.outputDir + '/=/'; + fpm(args.concat('-t', 'osxpkg', '-a', arch, files, launchdInit)); + return; + } + + }); + }); +}; diff --git a/tasks/build/packageJson.js b/tasks/build/packageJson.js new file mode 100644 index 0000000000000..32da2284b5d32 --- /dev/null +++ b/tasks/build/packageJson.js @@ -0,0 +1,21 @@ +module.exports = function (grunt) { + grunt.registerTask('build-packageJson', function () { + var pkg = grunt.config.get('pkg'); + + grunt.file.write( + 'build/kibana/package.json', + JSON.stringify({ + name: pkg.name, + description: pkg.description, + keywords: pkg.keywords, + version: pkg.version, + build: { + number: grunt.config.get('buildNum'), + sha: grunt.config.get('buildSha') + }, + repository: pkg.repository, + dependencies: pkg.dependencies + }, null, ' ') + ); + }); +}; diff --git a/tasks/build/packages.js b/tasks/build/packages.js new file mode 100644 index 0000000000000..537a2f9ec455f --- /dev/null +++ b/tasks/build/packages.js @@ -0,0 +1,42 @@ +module.exports = function createPackages(grunt) { + let { config } = grunt; + let { resolve } = require('path'); + let { execFile } = require('child_process'); + let { all, fromNode } = require('bluebird'); + + let version = config.get('pkg.version'); + let rootDir = config.get('root'); + let targetDir = resolve(rootDir, 'target'); + let buildDir = resolve(rootDir, 'build/kibana'); + + let exec = async (cmd, args) => { + grunt.log.writeln(` > ${cmd} ${args.join(' ')}`); + await fromNode(cb => execFile(cmd, args, { cwd: rootDir }, cb)); + }; + + let archives = async (platform) => { + // kibana.tar.gz + await exec('tar', ['-zchf', platform.tarPath, platform.buildDir]); + + // kibana.zip + if (/windows/.test(platform.name)) { + await exec('zip', ['-rq', '-ll', platform.zipPath, platform.buildDir]); + } else { + await exec('zip', ['-rq', platform.zipPath, platform.buildDir]); + } + }; + + grunt.registerTask('build-archives', function () { + + all( + grunt.config.get('platforms') + .map(async platform => { + + grunt.file.mkdir('target'); + await archives(platform); + }) + ) + .nodeify(this.async()); + + }); +}; diff --git a/tasks/build/pleaseManageUser.js b/tasks/build/pleaseManageUser.js new file mode 100644 index 0000000000000..30e007285f37b --- /dev/null +++ b/tasks/build/pleaseManageUser.js @@ -0,0 +1,10 @@ +module.exports = function (grunt) { + let { execFileSync } = require('child_process'); + let { resolve } = require('path'); + let userScriptsDir = grunt.config.get('userScriptsDir'); + + grunt.registerTask('build-pleaseManageUser', function () { + grunt.file.mkdir(userScriptsDir); + execFileSync('please-manage-user', ['--output', userScriptsDir, 'kibana']); + }); +}; diff --git a/tasks/build/pleaserun.js b/tasks/build/pleaserun.js new file mode 100644 index 0000000000000..a3620ef1655e3 --- /dev/null +++ b/tasks/build/pleaserun.js @@ -0,0 +1,25 @@ +var { resolve } = require('path'); +var { execFileSync } = require('child_process'); + +module.exports = function createServices(grunt) { + grunt.registerTask('build-pleaserun', function () { + // TODO(sissel): Detect if 'pleaserun' is found, and provide a useful error + // to the user if it is missing. + + grunt.config.get('services').forEach(function (service) { + grunt.file.mkdir(service.outDir); + execFileSync('pleaserun', [ + '--install', + '--no-install-actions', + '--install-prefix', service.outDir, + '--overwrite', + '--user', 'kibana', + '--sysv-log-path', '/var/log/kibana/', + '-p', service.name, + '-v', service.version, + '/opt/kibana/bin/kibana' + ]); + }); + + }); +}; diff --git a/tasks/build/readme.js b/tasks/build/readme.js new file mode 100644 index 0000000000000..d6884aaf54c3d --- /dev/null +++ b/tasks/build/readme.js @@ -0,0 +1,37 @@ +let marked = require('marked'); +let Promise = require('bluebird'); +let { join } = require('path'); +let TextRenderer = require('marked-text-renderer'); +let _ = require('lodash'); +let fs = require('fs'); +let { AllHtmlEntities } = require('html-entities'); +let entities = new AllHtmlEntities(); + +TextRenderer.prototype.heading = function (text, level, raw) { + return '\n\n' + text + '\n' + _.map(text, function () { return '='; }).join('') + '\n'; +}; + +module.exports = function (grunt) { + + grunt.registerTask('build-readme', function () { + let transform = function (input) { + let output = input.replace(/<\!\-\- [^\-]+ \-\->/g, '\n'); + output = marked(output); + return entities.decode(output); + }; + + marked.setOptions({ + renderer: new TextRenderer(), + tables: true, + breaks: false, + pedantic: false, + sanitize: false, + smartLists: true, + smartypants: false + }); + + grunt.file.write('build/kibana/README.txt', transform(grunt.file.read('README.md'))); + grunt.file.write('build/kibana/LICENSE.txt', transform(grunt.file.read('LICENSE.md'))); + }); + +}; diff --git a/tasks/create_shasums.js b/tasks/build/shasums.js similarity index 57% rename from tasks/create_shasums.js rename to tasks/build/shasums.js index 3a32c53948d92..e4d9c9fb15e4f 100644 --- a/tasks/create_shasums.js +++ b/tasks/build/shasums.js @@ -1,12 +1,9 @@ -var childProcess = require('child_process'); -var Promise = require('bluebird'); -var fs = require('fs'); -var readdir = Promise.promisify(fs.readdir); -var exec = Promise.promisify(childProcess.exec); -var _ = require('lodash'); -module.exports = function (grunt) { +var { promisify } = require('bluebird'); +var readdir = promisify(require('fs').readdir); +var exec = promisify(require('child_process').exec); - grunt.registerTask('create_shasums', function () { +module.exports = function (grunt) { + grunt.registerTask('build-shasums', function () { var targetDir = grunt.config.get('target'); readdir(targetDir) diff --git a/tasks/config/babel.js b/tasks/config/babel.js new file mode 100644 index 0000000000000..08a2e3a059a0b --- /dev/null +++ b/tasks/config/babel.js @@ -0,0 +1,19 @@ +let { defaults } = require('lodash'); +let babelOptions = require('requirefrom')('src')('optimize/babelOptions'); + +module.exports = { + build: { + options: defaults({ + optional: ['runtime'] + }, babelOptions), + src: [ + 'build/kibana/**/*.js', + '!**/public/**', + '!**/node_modules/**', + '!**/bower_components/**', + '!**/__tests__/**' + ], + dest: '.', + expand: true + } +}; diff --git a/tasks/config/clean.js b/tasks/config/clean.js index 291ba4e5f4bdf..5be3ca3e2846f 100644 --- a/tasks/config/clean.js +++ b/tasks/config/clean.js @@ -1,6 +1,7 @@ module.exports = function (grunt) { return { - build: '<%= build %>', - target: '<%= target %>' + build: 'build', + target: 'target', + testsFromModules: 'build/kibana/node_modules/**/*test*' }; }; diff --git a/tasks/config/compress.js b/tasks/config/compress.js deleted file mode 100644 index d38be88dc7629..0000000000000 --- a/tasks/config/compress.js +++ /dev/null @@ -1,49 +0,0 @@ -module.exports = function (grunt) { - var _ = require('lodash'); - var archiveName = function (plugin) { - return '<%= target %>/<%= pkg.name %>-' + (plugin ? 'plugin-' : '') + '<%= pkg.version %>'; - }; - - return _.mapValues({ - build_zip: archiveName() + '.zip', - build_tarball: archiveName() + '.tar.gz', - plugin: archiveName(true) + '.tar.gz' - }, function (filename, task) { - return { - options: { - archive: filename - }, - files: [ - { - flatten: true, - src: '<%= build %>/dist/bin/kibana', - dest: '<%= pkg.name %>/bin/kibana', - mode: 755 - }, - { - flatten: true, - src: '<%= build %>/dist/bin/kibana.bat', - dest: '<%= pkg.name %>/bin/kibana.bat' - }, - { - expand: true, - cwd: '<%= build %>/dist/config', - src: ['**/*'], - dest: '<%= pkg.name %>/config' - }, - { - expand: true, - cwd: '<%= build %>/dist/lib', - src: ['**/*'], - dest: '<%= pkg.name %>/lib' - }, - { - expand: true, - cwd: '<%= build %>/dist', - src: ['*.txt'], - dest: '<%= pkg.name %>' - } - ] - }; - }); -}; diff --git a/tasks/config/copy.js b/tasks/config/copy.js index a1d3c629983fc..33466449981e6 100644 --- a/tasks/config/copy.js +++ b/tasks/config/copy.js @@ -1,122 +1,21 @@ module.exports = function (grunt) { - var version = grunt.config.get('pkg.version'); - var platforms = grunt.config.get('platforms'); - var config = { - - kibana_src: { - expand: true, - cwd: '<%= app %>', - src: '**', - dest: '<%= build %>/src/' - }, - - server_src: { - files: [ - { - src: '<%= root %>/package.json', - dest: '<%= build %>/kibana/package.json' - }, - { - src: '<%= server %>/app.js', - dest: '<%= build %>/kibana/app.js' - }, - { - src: '<%= server %>/index.js', - dest: '<%= build %>/kibana/index.js' - }, - { - expand: true, - cwd: '<%= server %>/bin/', - src: '**', - dest: '<%= build %>/kibana/bin' - }, - { - expand: true, - cwd: '<%= server %>/config/', - src: '*.yml', - dest: '<%= build %>/kibana/config' - }, - { - expand: true, - cwd: '<%= server %>/lib/', - src: '**', - dest: '<%= build %>/kibana/lib' - }, - { - expand: true, - cwd: '<%= server %>/plugins/', - src: '**', - dest: '<%= build %>/kibana/plugins' - }, - { - expand: true, - cwd: '<%= server %>/routes/', - src: '**', - dest: '<%= build %>/kibana/routes' - }, - { - expand: true, - cwd: '<%= server %>/views/', - src: '**', - dest: '<%= build %>/kibana/views' - } - ] - }, - - dist: { - options: { mode: true }, - files: [ - { - expand: true, - cwd: '<%= build %>/kibana', - src: '**', - dest: '<%= build %>/dist/kibana/src' - }, - { - expand: true, - cwd: '<%= server %>/config/', - src: 'kibana.yml', - dest: '<%= build %>/dist/kibana/config/' - }, - { - expand: true, - cwd: '<%= bowerComponentsDir %>/ace-builds/src-noconflict/', - src: 'worker-json.js', - dest: '<%= build %>/dist/kibana/src/public/' - } - ] - }, - - versioned_dist: { + return { + devSource: { options: { mode: true }, - files: [] + src: [ + 'src/**', + 'bin/**', + 'config/kibana.yml', + '.node-version', + '!src/**/__tests__/**', + '!src/testUtils/**', + '!src/fixtures/**', + '!src/plugins/devMode/**', + '!src/plugins/testsBundle/**', + '!src/cli/cluster/**', + ], + dest: 'build/kibana', + expand: true }, - - plugin_readme: { - files: [ - { - src: '<%= build %>/kibana/public/plugins/README.txt', - dest: '<%= build %>/dist/kibana/plugins/README.txt' - } - ] - } - }; - - platforms.forEach(function (platform) { - config.versioned_dist.files.push({ - expand: true, - cwd: '<%= build %>/dist/kibana', - src: '**', - dest: '<%= build %>/dist/kibana-' + version + '-' + platform - }); - config.versioned_dist.files.push({ - expand: true, - cwd: '<%= root %>/.node_binaries/' + platform, - src: '**', - dest: '<%= build %>/dist/kibana-' + version + '-' + platform + '/node' - }); - }); - - return config; }; diff --git a/tasks/config/packages.js b/tasks/config/packages.js new file mode 100644 index 0000000000000..e3c0b25ae709e --- /dev/null +++ b/tasks/config/packages.js @@ -0,0 +1,22 @@ +module.exports = function (grunt) { + let version = grunt.config.get('pkg.version'); + let nodeVersion = grunt.config.get('nodeVersion'); + + return [ + 'darwin-x64', + 'linux-x64', + 'linux-x86', + 'windows' + ].map(function (name) { + var filename = `kibana-${version}-${name}`; + + return { + name, + filename, + tarPath: `target/${filename}.tar.gz`, + zipPath: `target/${filename}.zip`, + buildDir: `build/${filename}`, + nodeDir: `.node_binaries/${nodeVersion}/${name}`, + }; + }); +}; diff --git a/tasks/config/run.js b/tasks/config/run.js index 56ae1a7da48c5..a2727f3502580 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -17,6 +17,21 @@ module.exports = function (grunt) { '--optimize.tests=true', '--optimize.lazy=false' ] + }, + + optimizeBuild: { + options: { + wait: false, + ready: /Server running/, + quiet: false + }, + cmd: './build/kibana/bin/kibana', + args: [ + '--env.name=production', + '--logging.json=false', + '--plugins.initialize=false', + '--server.autoListen=false' + ] } }; diff --git a/tasks/config/s3.js b/tasks/config/s3.js index 21cb599c0f018..5202b0d895714 100644 --- a/tasks/config/s3.js +++ b/tasks/config/s3.js @@ -1,31 +1,27 @@ -var createPackages = require('../create_packages'); var _ = require('lodash'); -var getBaseNames = createPackages.getBaseNames; module.exports = function (grunt) { - var upload = _(getBaseNames(grunt)) - .map(function (basename) { - return [ - basename + '.tar.gz', - basename + '.tar.gz.sha1.txt', - basename + '.zip', - basename + '.zip.sha1.txt' - ]; - }) - .flattenDeep() - .map(function (filename) { - return { - src: 'target/' + filename, - dest: 'kibana/kibana/' + filename - }; - }) - .value(); + var { config } = grunt; return { release: { bucket: 'download.elasticsearch.org', access: 'private', // debug: true, // uncommment to prevent actual upload - upload: upload + upload: config.get('platforms') + .reduce(function (files, platform) { + return files.concat( + platform.tarPath, + platform.tarPath + '.sha1.txt', + platform.zipPath, + platform.zipPath + '.sha1.txt' + ); + }, []) + .map(function (filename) { + return { + src: 'target/' + filename, + dest: 'kibana/kibana/' + filename + }; + }) } }; }; diff --git a/tasks/config/services.js b/tasks/config/services.js new file mode 100644 index 0000000000000..37fa6cb59d0ec --- /dev/null +++ b/tasks/config/services.js @@ -0,0 +1,14 @@ +module.exports = function (grunt) { + [ + ['launchd', '10.9'], + ['upstart', '1.5'], + ['systemd', 'default'], + ['sysv', 'lsb-3.1'] + ] + .map(function ([ name, version ]) { + return { + name, + version, + outputDir: `build/services/${name}` }; + }); +}; diff --git a/tasks/create_packages.js b/tasks/create_packages.js deleted file mode 100644 index b706c02684f4a..0000000000000 --- a/tasks/create_packages.js +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint camelcase:0 */ - -var child_process = require('child_process'); -var Promise = require('bluebird'); -var { join } = require('path'); -var mkdirp = Promise.promisifyAll(require('mkdirp')); - -var execFile = Promise.promisify(child_process.execFile); - -var getBaseNames = function (grunt) { - var packageName = grunt.config.get('pkg.name'); - var version = grunt.config.get('pkg.version'); - var platforms = grunt.config.get('platforms'); - return platforms.map(function (platform) { - return packageName + '-' + version + '-' + platform; - }); -}; - -function createPackages(grunt) { - grunt.registerTask('create_packages', function () { - var done = this.async(); - var target = grunt.config.get('target'); - var distPath = join(grunt.config.get('build'), 'dist'); - var version = grunt.config.get('pkg.version'); - - var createPackage = function (name) { - var options = { cwd: distPath }; - var archiveName = join(target, name); - var commands = []; - var arch = /x64$/.test(name) ? 'x86_64' : 'i686'; - - var fpm_options = [ 'fpm', '-f', '-p', target, '-s', 'dir', '-n', 'kibana', '-v', version, - '--after-install', join(distPath, 'user', 'installer.sh'), - '--after-remove', join(distPath, 'user', 'remover.sh'), - '--config-files', '/opt/kibana/config/kibana.yml' ]; - var fpm_files = join(distPath, name) + '/=/opt/kibana'; - - // kibana.tar.gz - commands.push([ 'tar', '-zcf', archiveName + '.tar.gz', name ]); - - // kibana.zip - if (/windows/.test(name)) { - commands.push([ 'zip', '-rq', '-ll', archiveName + '.zip', name ]); - } else { - commands.push([ 'zip', '-rq', archiveName + '.zip', name ]); - } - - if (grunt.option('os-packages')) { - // TODO(sissel): Add before-install scripts to create kibana user - // TODO(sissel): Check if `fpm` is available - if (/linux-x(86|64)$/.test(name)) { - // kibana.rpm and kibana.deb - var sysv_init = join(distPath, 'services', 'sysv') + '/etc/=/etc/'; - commands.push(fpm_options.concat(['-t', 'rpm', '-a', arch, '--rpm-os', 'linux', fpm_files, sysv_init])); - commands.push(fpm_options.concat(['-t', 'deb', '-a', arch, fpm_files, sysv_init])); - } else if (/darwin-x(86|64)$/.test(name)) { - // kibana.pkg - var launchd = join(distPath, 'services', 'launchd') + '/=/'; - commands.push(fpm_options.concat(['-t', 'osxpkg', '-a', arch, fpm_files, launchd])); - } - } - - return mkdirp.mkdirpAsync(target) - .then(function (arg) { - return Promise.map(commands, function (cmd) { - return execFile(cmd.shift(), cmd, options); - }); - }, function (err) { console.log('Failure on ' + name + ': ' + err); }); - }; - - Promise.map(getBaseNames(grunt), createPackage).finally(done); - }); -} - -module.exports = createPackages; -createPackages.getBaseNames = getBaseNames; diff --git a/tasks/create_services.js b/tasks/create_services.js deleted file mode 100644 index 52655a56cccf0..0000000000000 --- a/tasks/create_services.js +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint camelcase:0 */ - -var childProcess = require('child_process'); -var Promise = require('bluebird'); -var { join } = require('path'); -var mkdirp = Promise.promisifyAll(require('mkdirp')); -var execFile = Promise.promisify(childProcess.execFile); - -function createServices(grunt) { - grunt.registerTask('create_services', function () { - var done = this.async(); - var target = grunt.config.get('target'); - var distPath = join(grunt.config.get('build'), 'dist'); - var services = grunt.config.get('services'); - - var createService = function (arg) { - var service = arg[0]; - var service_version = arg[1]; - var options = { cwd: distPath }; - var output = join(distPath, 'services', service); - var pleaserun_args = ['--install', '--no-install-actions', - '--install-prefix', output, '--overwrite', - '--user', 'kibana', - '--sysv-log-path', '/var/log/kibana/', - '-p', service, '-v', service_version, - '/opt/kibana/bin/kibana']; - - return mkdirp.mkdirpAsync(target) - .then(function (arg) { - return execFile('pleaserun', pleaserun_args, options); - }, function (err) { console.log('pleaserun failed: ' + err + '. Args: ' + pleaserun_args.join(' ')); }); - }; - - // TODO(sissel): Detect if 'pleaserun' is found, and provide a useful error - // to the user if it is missing. - mkdirp.mkdirpAsync(distPath) - .then(function () { - return Promise.map(services, createService); - }) - .then(function (arg) { - // Create the user-management scripts - var output = join(distPath, 'user'); - return mkdirp.mkdirpAsync(output).then(function () { - return execFile('please-manage-user', ['--output', output, 'kibana'], { cwd: distPath }); - }); - }, function (err) { console.log('please-manage-user failed: ' + err + '.'); }) - .finally(done); - }); -} - -module.exports = createServices; diff --git a/tasks/dist_package_json.js b/tasks/dist_package_json.js deleted file mode 100644 index fca775f089e6a..0000000000000 --- a/tasks/dist_package_json.js +++ /dev/null @@ -1,19 +0,0 @@ -module.exports = function (grunt) { - grunt.registerTask('dist_package_json', function () { - var path = grunt.config.process('<%= build %>/kibana/package.json'); - var pkg = grunt.config.get('pkg'); - - grunt.file.write(path, JSON.stringify({ - name: pkg.name, - description: pkg.description, - keywords: pkg.keywords, - version: pkg.version, - build: { - number: grunt.config.get('buildNum'), - sha: grunt.config.get('buildSha') - }, - repository: pkg.repository, - dependencies: pkg.dependencies - }, null, ' ')); - }); -}; diff --git a/tasks/dist_readme.js b/tasks/dist_readme.js deleted file mode 100644 index 23445a485bdfd..0000000000000 --- a/tasks/dist_readme.js +++ /dev/null @@ -1,61 +0,0 @@ -var marked = require('marked'); -var Promise = require('bluebird'); -var { join } = require('path'); -var TextRenderer = require('marked-text-renderer'); -var _ = require('lodash'); -var fs = require('fs'); -var { AllHtmlEntities } = require('html-entities'); -var entities = new AllHtmlEntities(); - -var readFile = Promise.promisify(fs.readFile); -var writeFile = Promise.promisify(fs.writeFile); - -TextRenderer.prototype.heading = function (text, level, raw) { - return '\n\n' + text + '\n' + _.map(text, function () { return '='; }).join('') + '\n'; -}; - -var process = function (input) { - var output = input.replace(/<\!\-\- [^\-]+ \-\->/g, '\n'); - output = marked(output); - return entities.decode(output); -}; - -module.exports = function (grunt) { - - grunt.registerTask('dist_readme', function () { - var done = this.async(); - var root = grunt.config.get('root'); - var build = grunt.config.get('build'); - - var srcReadme = join(root, 'README.md'); - var distReadme = join(build, 'dist', 'kibana', 'README.txt'); - - var srcLicense = join(root, 'LICENSE.md'); - var distLicense = join(build, 'dist', 'kibana', 'LICENSE.txt'); - - marked.setOptions({ - renderer: new TextRenderer(), - tables: true, - breaks: false, - pedantic: false, - sanitize: false, - smartLists: true, - smartypants: false - }); - - readFile(srcReadme, 'utf-8') - .then(function (data) { - return writeFile(distReadme, process(data.toString())); - }) - .then(function () { - return readFile(srcLicense, 'utf-8'); - }) - .then(function (data) { - return writeFile(distLicense, process(data.toString())); - }) - .then(done) - .catch(done); - - }); - -}; diff --git a/tasks/downloadNodeBins.js b/tasks/downloadNodeBins.js deleted file mode 100644 index 988252dc75ffa..0000000000000 --- a/tasks/downloadNodeBins.js +++ /dev/null @@ -1,80 +0,0 @@ -module.exports = function (grunt) { - grunt.registerTask('downloadNodeBins', function () { - let { map, fromNode, promisify } = require('bluebird'); - let { resolve } = require('path'); - let { createWriteStream } = require('fs'); - let { createGunzip } = require('zlib'); - let { Extract } = require('tar'); - - let mkdirp = promisify(require('mkdirp')); - let rename = promisify(require('fs').rename); - let get = (uri) => fromNode(cb => require('wreck').request('GET', uri, null, cb)); - - let exists = (path) => fromNode(cb => { - require('fs').stat(path, err => cb(null, !err)); - }); - - let platforms = grunt.config.get('platforms'); - let nodeVersion = grunt.config.get('nodeVersion'); - - let nodeDir = require('requirefrom')('src/utils')('fromRoot')(`.node_binaries/${nodeVersion}`); - let baseUri = `https://iojs.org/dist/v${nodeVersion}`; - - let writeTar = async (to, from) => { - await fromNode(cb => { - from - .pipe(createGunzip()) - .on('error', cb) - .pipe(new Extract({ path: to, strip: 1 })) - .on('error', cb) - .on('end', cb); - }); - }; - - let writeExe = async (to, from) => { - let winBinDir = resolve(to, 'bin'); - await mkdirp(winBinDir); - await fromNode(cb => { - from - .pipe(createWriteStream(resolve(winBinDir, 'node.exe'))) - .on('error', cb) - .on('finish', cb); - }); - }; - - mkdirp(nodeDir) - .then(function () { - grunt.log.ok(`downloading node binaries for ${platforms.join(', ')}`); - return platforms; - }) - .map(async function (platform) { - let finalDir = resolve(nodeDir, `${platform}`); - let downloadDir = `${finalDir}.temp`; - - if (await exists(finalDir)) { - grunt.log.ok(`${platform} download exists`); - return; - } - - let uri; - if (platform === 'windows') { - uri = `${baseUri}/win-x64/iojs.exe`; - } else { - uri = `${baseUri}/iojs-v${nodeVersion}-${platform}.tar.gz`; - } - - let resp = await get('GET', uri); - if (resp.statusCode !== 200) { - throw new Error(uri + ' failed with a ' + resp.statusCode); - } - - let write = platform === 'windows' ? writeExe : writeTar; - await write(downloadDir, resp); - await rename(downloadDir, finalDir); - - grunt.log.ok(`${platform} download complete`); - }) - .nodeify(this.async()); - }); -}; - diff --git a/tasks/licenses.js b/tasks/licenses.js index d0dabe0adf26f..b7977b418630b 100644 --- a/tasks/licenses.js +++ b/tasks/licenses.js @@ -10,7 +10,7 @@ module.exports = function (grunt) { var done = this.async(); var result = {}; - var options = {start: process.cwd(), json: true }; + var options = { start: process.cwd(), json: true }; var checkQueueLength = 2; function processPackage(info, dependency) { diff --git a/tasks/npm_install_kibana.js b/tasks/npm_install_kibana.js deleted file mode 100644 index b70a878503512..0000000000000 --- a/tasks/npm_install_kibana.js +++ /dev/null @@ -1,21 +0,0 @@ -module.exports = function (grunt) { - var childProcess = require('child_process'); - var { join } = require('path'); - - grunt.registerTask('npm_install_kibana', 'NPM install kibana server into dist', function () { - var done = this.async(); - var cwd = join(grunt.config.get('build'), 'dist', 'kibana', 'src'); - var command = 'npm install --production --no-optional'; - var options = { cwd: cwd }; - childProcess.exec(command, options, function (err, stdout, stderr) { - if (err) { - grunt.log.error(stderr); - return done(err); - } - grunt.log.writeln(stdout); - return done(); - }); - }); -}; - - diff --git a/tasks/plugins.js b/tasks/plugins.js deleted file mode 100644 index 43d06f04be103..0000000000000 --- a/tasks/plugins.js +++ /dev/null @@ -1,27 +0,0 @@ -module.exports = function (grunt) { - grunt.registerTask('make_plugin_dir', function () { - var dir = grunt.config.process('<%= build %>/dist/kibana/plugins'); - grunt.file.mkdir(dir); - }); - - grunt.registerTask('describe_bundled_plugins', function () { - var configKey = 'bundled_plugin_ids'; - var file = grunt.config.process('<%= build %>/dist/kibana/config/kibana.yml'); - var idList = grunt.config.get('bundled_plugin_module_ids').map(function (id) { - return ' - ' + id; - }).join('\n'); - - var contents = grunt.file.read(file); - if (contents.indexOf(configKey) !== -1) { - grunt.log.error('bundled plugin ids already written to config/kibana.yml'); - return; - } - - contents += - '\n# Plugins that are included in the build, and no longer found in the plugins/ folder' + - '\n' + configKey + ':' + - '\n' + idList; - - grunt.file.write(file, contents); - }); -}; From af30b5825bb114926acd73f0ab83cfacf488ab09 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 21:21:03 -0700 Subject: [PATCH 05/17] [deps] remove request dep in favor of wreck --- package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package.json b/package.json index d71a084c5e652..bdf417bf7aa2a 100644 --- a/package.json +++ b/package.json @@ -107,7 +107,6 @@ "numeral": "^1.5.3", "nvd3": "panda01/nvd3#kibana", "raw-loader": "^0.5.1", - "request": "^2.40.0", "requirefrom": "^0.2.0", "rimraf": "^2.4.1", "script-loader": "^0.6.1", From afb283d1b51b74044ed1d763f0d5b2ce1aa0e9d6 Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 21:25:40 -0700 Subject: [PATCH 06/17] [build] stick with grunt:task naming convention --- tasks/build/build.js | 28 ++++++++++++++-------------- tasks/build/cliIndex.js | 2 +- tasks/build/downloadNodes.js | 4 ++-- tasks/build/getProps.js | 2 +- tasks/build/installNpmDeps.js | 2 +- tasks/build/installedPlugins.js | 2 +- tasks/build/linkNodes.js | 2 +- tasks/build/osPackages.js | 2 +- tasks/build/packageJson.js | 2 +- tasks/build/packages.js | 2 +- tasks/build/pleaseManageUser.js | 2 +- tasks/build/pleaserun.js | 2 +- tasks/build/readme.js | 2 +- tasks/build/shasums.js | 2 +- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tasks/build/build.js b/tasks/build/build.js index 397901f54ce39..6baadf55a5fed 100644 --- a/tasks/build/build.js +++ b/tasks/build/build.js @@ -2,28 +2,28 @@ module.exports = function (grunt) { let { flatten } = require('lodash'); grunt.registerTask('build', flatten([ - 'build-getProps', + 'build:getProps', 'clean:build', 'clean:target', - 'downloadNodes:start', + 'build:downloadNodes:start', 'copy:devSource', 'babel:build', - 'build-cliIndex', - 'build-installedPlugins', - 'build-packageJson', - 'build-readme', - 'build-installNpmDeps', + 'build:cliIndex', + 'build:installedPlugins', + 'build:packageJson', + 'build:readme', + 'build:installNpmDeps', 'run:optimizeBuild', 'stop:optimizeBuild', - 'downloadNodes:finish', + 'build:downloadNodes:finish', 'clean:testsFromModules', - 'build-versionedLinks', - 'build-archives', + 'build:versionedLinks', + 'build:archives', !grunt.option('os-packages') ? [] : [ - 'build-pleaseRun', - 'build-pleaseManageUser', - 'build-osPackages', + 'build:pleaseRun', + 'build:pleaseManageUser', + 'build:osPackages', ], - 'build-shasums' + 'build:shasums' ])); }; diff --git a/tasks/build/cliIndex.js b/tasks/build/cliIndex.js index 308cc82cae80c..30299372c087e 100644 --- a/tasks/build/cliIndex.js +++ b/tasks/build/cliIndex.js @@ -3,7 +3,7 @@ module.exports = function (grunt) { var file = 'build/kibana/src/cli/index.js'; var blurb = `require('babel/register')(require('../optimize/babelOptions'));\n`; - grunt.registerTask('build-cliIndex', function () { + grunt.registerTask('build:cliIndex', function () { var before = grunt.file.read(file); var after = before.replace(blurb, ''); diff --git a/tasks/build/downloadNodes.js b/tasks/build/downloadNodes.js index a437d7bf23dc6..6de387ff122e4 100644 --- a/tasks/build/downloadNodes.js +++ b/tasks/build/downloadNodes.js @@ -76,8 +76,8 @@ module.exports = function (grunt) { let current = null; let start = () => current || (current = run()); - grunt.registerTask('downloadNodes:start', start); - grunt.registerTask('downloadNodes:finish', function () { + grunt.registerTask('build:downloadNodes:start', start); + grunt.registerTask('build:downloadNodes:finish', function () { start().then(() => { grunt.log.ok(`node binaries for ${platforms.join(', ')} downloaded`); }) diff --git a/tasks/build/getProps.js b/tasks/build/getProps.js index 2d031eec93d84..17fcc3b0f56e9 100644 --- a/tasks/build/getProps.js +++ b/tasks/build/getProps.js @@ -1,6 +1,6 @@ module.exports = function (grunt) { var exec = require('child_process').execSync; - grunt.registerTask('build-getProps', function () { + grunt.registerTask('build:getProps', function () { grunt.config.set('buildSha', String(exec('git rev-parse HEAD')).trim()); grunt.config.set('buildNum', parseFloat(String(exec('git log --format="%h" | wc -l')).trim())); }); diff --git a/tasks/build/installNpmDeps.js b/tasks/build/installNpmDeps.js index 91c1acab6d68e..c8390e29b412e 100644 --- a/tasks/build/installNpmDeps.js +++ b/tasks/build/installNpmDeps.js @@ -2,7 +2,7 @@ module.exports = function (grunt) { var { execSync } = require('child_process'); var { resolve } = require('path'); - grunt.registerTask('build-installNpmDeps', function () { + grunt.registerTask('build:installNpmDeps', function () { grunt.file.mkdir('build/kibana/node_modules'); execSync('npm install --production --no-optional', { diff --git a/tasks/build/installedPlugins.js b/tasks/build/installedPlugins.js index d66f0b8c5610c..342571b1f4904 100644 --- a/tasks/build/installedPlugins.js +++ b/tasks/build/installedPlugins.js @@ -1,5 +1,5 @@ module.exports = function (grunt) { - grunt.registerTask('build-installedPlugins', function () { + grunt.registerTask('build:installedPlugins', function () { grunt.file.mkdir('build/kibana/installedPlugins'); }); }; diff --git a/tasks/build/linkNodes.js b/tasks/build/linkNodes.js index fcf108681b402..4a7a4e3cc32c3 100644 --- a/tasks/build/linkNodes.js +++ b/tasks/build/linkNodes.js @@ -3,7 +3,7 @@ module.exports = function (grunt) { let { basename, resolve } = require('path'); let { forOwn } = require('lodash'); - grunt.registerTask('build-versionedLinks', function () { + grunt.registerTask('build:versionedLinks', function () { let buildFiles = grunt.file.expand('build/kibana/{*,.*}'); let rootDir = grunt.config.get('root'); diff --git a/tasks/build/osPackages.js b/tasks/build/osPackages.js index 755e07765e1dc..649e0707819b9 100644 --- a/tasks/build/osPackages.js +++ b/tasks/build/osPackages.js @@ -9,7 +9,7 @@ module.exports = function (grunt) { let userScriptsDir = config.get('userScriptsDir'); let servicesByName = indexBy(config.get('services'), 'id'); - grunt.registerTask('build-osPackages', function () { + grunt.registerTask('build:osPackages', function () { grunt.config.get('platforms').forEach(({ name, buildDir }) => { let arch = /x64$/.test(name) ? 'x86_64' : 'i686'; diff --git a/tasks/build/packageJson.js b/tasks/build/packageJson.js index 32da2284b5d32..e368cd3369e46 100644 --- a/tasks/build/packageJson.js +++ b/tasks/build/packageJson.js @@ -1,5 +1,5 @@ module.exports = function (grunt) { - grunt.registerTask('build-packageJson', function () { + grunt.registerTask('build:packageJson', function () { var pkg = grunt.config.get('pkg'); grunt.file.write( diff --git a/tasks/build/packages.js b/tasks/build/packages.js index 537a2f9ec455f..b751a4cb9037a 100644 --- a/tasks/build/packages.js +++ b/tasks/build/packages.js @@ -26,7 +26,7 @@ module.exports = function createPackages(grunt) { } }; - grunt.registerTask('build-archives', function () { + grunt.registerTask('build:archives', function () { all( grunt.config.get('platforms') diff --git a/tasks/build/pleaseManageUser.js b/tasks/build/pleaseManageUser.js index 30e007285f37b..c61663ed166ac 100644 --- a/tasks/build/pleaseManageUser.js +++ b/tasks/build/pleaseManageUser.js @@ -3,7 +3,7 @@ module.exports = function (grunt) { let { resolve } = require('path'); let userScriptsDir = grunt.config.get('userScriptsDir'); - grunt.registerTask('build-pleaseManageUser', function () { + grunt.registerTask('build:pleaseManageUser', function () { grunt.file.mkdir(userScriptsDir); execFileSync('please-manage-user', ['--output', userScriptsDir, 'kibana']); }); diff --git a/tasks/build/pleaserun.js b/tasks/build/pleaserun.js index a3620ef1655e3..26a3ecb30cce1 100644 --- a/tasks/build/pleaserun.js +++ b/tasks/build/pleaserun.js @@ -2,7 +2,7 @@ var { resolve } = require('path'); var { execFileSync } = require('child_process'); module.exports = function createServices(grunt) { - grunt.registerTask('build-pleaserun', function () { + grunt.registerTask('build:pleaserun', function () { // TODO(sissel): Detect if 'pleaserun' is found, and provide a useful error // to the user if it is missing. diff --git a/tasks/build/readme.js b/tasks/build/readme.js index d6884aaf54c3d..540ba98fbcce1 100644 --- a/tasks/build/readme.js +++ b/tasks/build/readme.js @@ -13,7 +13,7 @@ TextRenderer.prototype.heading = function (text, level, raw) { module.exports = function (grunt) { - grunt.registerTask('build-readme', function () { + grunt.registerTask('build:readme', function () { let transform = function (input) { let output = input.replace(/<\!\-\- [^\-]+ \-\->/g, '\n'); output = marked(output); diff --git a/tasks/build/shasums.js b/tasks/build/shasums.js index e4d9c9fb15e4f..ace8eb0ce9633 100644 --- a/tasks/build/shasums.js +++ b/tasks/build/shasums.js @@ -3,7 +3,7 @@ var readdir = promisify(require('fs').readdir); var exec = promisify(require('child_process').exec); module.exports = function (grunt) { - grunt.registerTask('build-shasums', function () { + grunt.registerTask('build:shasums', function () { var targetDir = grunt.config.get('target'); readdir(targetDir) From b0711eb7022910500c7f7c4d81dc47e2d77e11df Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 21:47:22 -0700 Subject: [PATCH 07/17] [plugin/download] expose the request's abort method on the response --- src/cli/plugin/pluginDownloader.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cli/plugin/pluginDownloader.js b/src/cli/plugin/pluginDownloader.js index 15884a604a5a7..e7fdd3d29f7a3 100644 --- a/src/cli/plugin/pluginDownloader.js +++ b/src/cli/plugin/pluginDownloader.js @@ -70,7 +70,15 @@ module.exports = function (settings, logger) { if (timeout) timeout = false; return Promise.fromNode(cb => { - return wreck.defaults({ timeout }).request('GET', url, null, cb); + let request = wreck + .defaults({ timeout }) + .request('GET', url, null, (err, response) => { + if (err || !response) cb(err); + + response.request = request; + response.abort = () => request.abort(); + cb(null, response); + }); }) .catch(function (err) { if (err.message.match(/invalid uri/i)) { From 4f8cad5a0d20e349ae0ce554587d67d8868f434e Mon Sep 17 00:00:00 2001 From: spalger Date: Thu, 13 Aug 2015 23:14:22 -0700 Subject: [PATCH 08/17] [webpack] switch to rjs-repack-loader for shorter id and smaller dep tree --- package.json | 2 +- src/optimize/BaseOptimizer.js | 2 +- tasks/build/downloadNodes.js | 3 ++- tasks/config/run.js | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index bdf417bf7aa2a..322ec0636a86f 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,6 @@ "angular-nvd3": "panda01/angular-nvd3#kibana", "angular-route": "1.2.28", "ansicolors": "^0.3.2", - "auto-preload-rjscommon-deps-loader": "^1.0.4", "autoprefixer": "^5.2.0", "autoprefixer-loader": "^2.0.0", "babel": "^5.8.21", @@ -109,6 +108,7 @@ "raw-loader": "^0.5.1", "requirefrom": "^0.2.0", "rimraf": "^2.4.1", + "rjs-repack-loader": "^1.0.6", "script-loader": "^0.6.1", "semver": "^4.3.6", "style-loader": "^0.12.3", diff --git a/src/optimize/BaseOptimizer.js b/src/optimize/BaseOptimizer.js index 170a892f59b3f..f30f7b863cad7 100644 --- a/src/optimize/BaseOptimizer.js +++ b/src/optimize/BaseOptimizer.js @@ -108,7 +108,7 @@ class BaseOptimizer { { test: /\.(html|tmpl)$/, loader: 'raw' }, { test: /\.png$/, loader: 'url?limit=10000&name=[path][name].[ext]' }, { test: /\.(woff|woff2|ttf|eot|svg|ico)(\?|$)/, loader: 'file?name=[path][name].[ext]' }, - { test: /[\/\\]src[\/\\](plugins|ui)[\/\\].+\.js$/, loader: `auto-preload-rjscommon-deps${mapQ}` }, + { test: /[\/\\]src[\/\\](plugins|ui)[\/\\].+\.js$/, loader: `rjs-repack${mapQ}` }, { test: /\.js$/, exclude: /[\/\\](node_modules|bower_components)[\/\\]/, diff --git a/tasks/build/downloadNodes.js b/tasks/build/downloadNodes.js index 6de387ff122e4..cc829d4a14392 100644 --- a/tasks/build/downloadNodes.js +++ b/tasks/build/downloadNodes.js @@ -1,6 +1,7 @@ module.exports = function (grunt) { let { map, fromNode, promisify } = require('bluebird'); let { resolve } = require('path'); + let { pluck } = require('lodash'); let { createWriteStream } = require('fs'); let { createGunzip } = require('zlib'); let { Extract } = require('tar'); @@ -79,7 +80,7 @@ module.exports = function (grunt) { grunt.registerTask('build:downloadNodes:start', start); grunt.registerTask('build:downloadNodes:finish', function () { start().then(() => { - grunt.log.ok(`node binaries for ${platforms.join(', ')} downloaded`); + grunt.log.ok(`node binaries for ${pluck(platforms, 'name').join(', ')} downloaded`); }) .nodeify(this.async()); }); diff --git a/tasks/config/run.js b/tasks/config/run.js index a2727f3502580..3927a1532463e 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -22,7 +22,7 @@ module.exports = function (grunt) { optimizeBuild: { options: { wait: false, - ready: /Server running/, + ready: /Optimization .+ complete/, quiet: false }, cmd: './build/kibana/bin/kibana', From e4c4bcc51305237a41afc601d2dbd9cba1ac2e7a Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 11:51:23 -0700 Subject: [PATCH 09/17] [build] move some select node modules up in the tree to prevent paths over windows limit --- Gruntfile.js | 20 +++++++++++++++++--- tasks/build/packageJson.js | 8 ++++++-- tasks/config/clean.js | 4 +++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 9a6d44afd0711..86c2446cfc0a5 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -41,7 +41,23 @@ module.exports = function (grunt) { '<%= root %>/tasks/**/*.js', '<%= src %>/**/*.js', '!<%= src %>/fixtures/**/*.js' - ] + ], + + deepModules: { + 'caniuse-db': '1.0.30000265', + 'chalk': '1.1.0', + 'glob': '4.5.3', + 'har-validator': '1.8.0', + 'json5': '0.4.0', + 'loader-utils': '0.2.11', + 'micromatch': '2.2.0', + 'postcss-normalize-url': '2.1.1', + 'postcss-reduce-idents': '1.0.2', + 'postcss-unique-selectors': '1.0.0', + 'postcss-minify-selectors': '1.4.6', + 'postcss-single-charset': '0.3.0', + 'regenerator': '0.8.36' + } }; grunt.config.merge(config); @@ -62,8 +78,6 @@ module.exports = function (grunt) { } }); - - // load task definitions grunt.task.loadTasks('tasks'); grunt.task.loadTasks('tasks/build'); diff --git a/tasks/build/packageJson.js b/tasks/build/packageJson.js index e368cd3369e46..6b219f8577adc 100644 --- a/tasks/build/packageJson.js +++ b/tasks/build/packageJson.js @@ -1,6 +1,10 @@ module.exports = function (grunt) { + let { defaults } = require('lodash'); + + let pkg = grunt.config.get('pkg'); + let deepModules = grunt.config.get('deepModules'); + grunt.registerTask('build:packageJson', function () { - var pkg = grunt.config.get('pkg'); grunt.file.write( 'build/kibana/package.json', @@ -14,7 +18,7 @@ module.exports = function (grunt) { sha: grunt.config.get('buildSha') }, repository: pkg.repository, - dependencies: pkg.dependencies + dependencies: defaults({}, pkg.dependencies, deepModules) }, null, ' ') ); }); diff --git a/tasks/config/clean.js b/tasks/config/clean.js index 5be3ca3e2846f..db5f004f6a35d 100644 --- a/tasks/config/clean.js +++ b/tasks/config/clean.js @@ -1,7 +1,9 @@ module.exports = function (grunt) { + let modules = Object.keys(grunt.config.get('deepModules')); return { build: 'build', target: 'target', - testsFromModules: 'build/kibana/node_modules/**/*test*' + testsFromModules: 'build/kibana/node_modules/**/*test*/**', + deepModules: 'build/kibana/node_modules/*/node_modules/**/{' + modules.join(',') + '}/**' }; }; From 552b824711cdf9bc8011aa4782515bcacfbd2246 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 11:53:26 -0700 Subject: [PATCH 10/17] [build] finish updates --- Gruntfile.js | 3 +- tasks/build/{packages.js => archives.js} | 16 ++- tasks/build/downloadNodes.js | 130 +++++++++--------- tasks/build/{build.js => index.js} | 3 +- tasks/build/installNpmDeps.js | 8 +- .../build/{linkNodes.js => versionedLinks.js} | 0 tasks/config/packages.js | 22 --- tasks/config/platforms.js | 26 ++++ tasks/config/run.js | 2 +- tasks/config/s3.js | 10 +- tasks/jenkins.js | 3 +- tasks/release.js | 16 +++ 12 files changed, 132 insertions(+), 107 deletions(-) rename tasks/build/{packages.js => archives.js} (63%) rename tasks/build/{build.js => index.js} (96%) rename tasks/build/{linkNodes.js => versionedLinks.js} (100%) delete mode 100644 tasks/config/packages.js create mode 100644 tasks/config/platforms.js diff --git a/Gruntfile.js b/Gruntfile.js index 86c2446cfc0a5..3f3dbeca4ab9b 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -63,8 +63,9 @@ module.exports = function (grunt) { grunt.config.merge(config); config.userScriptsDir = __dirname + '/build/userScripts'; + // ensure that these run first, other configs need them config.services = require('./tasks/config/services')(grunt); - config.platforms = require('./tasks/config/packages')(grunt); + config.platforms = require('./tasks/config/platforms')(grunt); grunt.config.merge(config); diff --git a/tasks/build/packages.js b/tasks/build/archives.js similarity index 63% rename from tasks/build/packages.js rename to tasks/build/archives.js index b751a4cb9037a..1f3dd34d8b029 100644 --- a/tasks/build/packages.js +++ b/tasks/build/archives.js @@ -4,25 +4,27 @@ module.exports = function createPackages(grunt) { let { execFile } = require('child_process'); let { all, fromNode } = require('bluebird'); - let version = config.get('pkg.version'); let rootDir = config.get('root'); - let targetDir = resolve(rootDir, 'target'); - let buildDir = resolve(rootDir, 'build/kibana'); + let buildDir = resolve(rootDir, 'build'); let exec = async (cmd, args) => { grunt.log.writeln(` > ${cmd} ${args.join(' ')}`); - await fromNode(cb => execFile(cmd, args, { cwd: rootDir }, cb)); + await fromNode(cb => execFile(cmd, args, { cwd: buildDir }, cb)); }; + let archives = async (platform) => { + let tarPath = resolve(rootDir, platform.tarPath); + let zipPath = resolve(rootDir, platform.zipPath); + // kibana.tar.gz - await exec('tar', ['-zchf', platform.tarPath, platform.buildDir]); + await exec('tar', ['-zchf', tarPath, platform.buildName]); // kibana.zip if (/windows/.test(platform.name)) { - await exec('zip', ['-rq', '-ll', platform.zipPath, platform.buildDir]); + await exec('zip', ['-rq', '-ll', zipPath, platform.buildName]); } else { - await exec('zip', ['-rq', platform.zipPath, platform.buildDir]); + await exec('zip', ['-rq', zipPath, platform.buildName]); } }; diff --git a/tasks/build/downloadNodes.js b/tasks/build/downloadNodes.js index cc829d4a14392..32de0f6820961 100644 --- a/tasks/build/downloadNodes.js +++ b/tasks/build/downloadNodes.js @@ -1,86 +1,88 @@ module.exports = function (grunt) { - let { map, fromNode, promisify } = require('bluebird'); + let { map, fromNode } = require('bluebird'); let { resolve } = require('path'); let { pluck } = require('lodash'); let { createWriteStream } = require('fs'); let { createGunzip } = require('zlib'); let { Extract } = require('tar'); - - let mkdirp = promisify(require('mkdirp')); - let rename = promisify(require('fs').rename); - let get = (uri) => fromNode(cb => require('wreck').request('GET', uri, null, cb)); - - let exists = (path) => fromNode(cb => { - require('fs').stat(path, err => cb(null, !err)); - }); + let { rename } = require('fs'); + let wreck = require('wreck'); let platforms = grunt.config.get('platforms'); - let nodeVersion = grunt.config.get('nodeVersion'); - let baseUri = `https://iojs.org/dist/v${nodeVersion}`; - - let writeTar = async (to, from) => { - await fromNode(cb => { - from - .pipe(createGunzip()) - .on('error', cb) - .pipe(new Extract({ path: to, strip: 1 })) - .on('error', cb) - .on('end', cb); + let activeDownloads = []; + + let start = async (platform) => { + let finalDir = platform.nodeDir; + if (!grunt.file.isPathAbsolute(finalDir)) { + // since we are using fs module function we need absolute paths + finalDir = resolve(grunt.config.get('root'), finalDir); + } + + let downloadDir = `${finalDir}.temp`; + + if (grunt.file.isDir(platform.nodeDir)) { + grunt.log.ok(`${platform.name} exists`); + return; + } + + let resp = await fromNode(cb => { + let req = wreck.request('GET', platform.nodeUrl, null, function (err, resp) { + if (err) { + return cb(err); + } + + if (resp.statusCode !== 200) { + return cb(new Error(`${platform.nodeUrl} failed with a ${resp.statusCode} response`)); + } + + return cb(null, resp); + }); }); - }; - - let writeExe = async (to, from) => { - let winBinDir = resolve(to, 'bin'); - await mkdirp(winBinDir); - await fromNode(cb => { - from - .pipe(createWriteStream(resolve(winBinDir, 'node.exe'))) - .on('error', cb) - .on('finish', cb); - }); - }; - let run = () => { - grunt.log.ok(`downloading node binaries`); - - return map(platforms, async function (platform) { - grunt.file.mkdir(platform.nodeDir); - - let finalDir = resolve(grunt.config.get('root'), platform.nodeDir); - let downloadDir = `${finalDir}.temp`; - - if (await exists(finalDir)) { - grunt.log.debug(`${platform} download exists`); - return; - } - - let uri; - if (platform === 'windows') { - uri = `${baseUri}/win-x64/iojs.exe`; + // use an async iife to store promise for download + // then store platform in active downloads list + // which we will read from in the finish task + platform.downloadPromise = (async () => { + grunt.file.mkdir(downloadDir); + + if (platform.name === 'windows') { + await fromNode(cb => { + resp + .pipe(createWriteStream(resolve(downloadDir, 'node.exe'))) + .on('error', cb) + .on('finish', cb); + }); } else { - uri = `${baseUri}/iojs-v${nodeVersion}-${platform}.tar.gz`; + await fromNode(cb => { + resp + .pipe(createGunzip()) + .on('error', cb) + .pipe(new Extract({ path: downloadDir, strip: 1 })) + .on('error', cb) + .on('end', cb); + }); } - let resp = await get('GET', uri); - if (resp.statusCode !== 200) { - throw new Error(uri + ' failed with a ' + resp.statusCode); - } + await fromNode(cb => { + rename(downloadDir, finalDir, cb); + }); + }()); - let write = platform === 'windows' ? writeExe : writeTar; - await write(downloadDir, resp); - await rename(downloadDir, finalDir); + activeDownloads.push(platform); - grunt.log.debug(`${platform} download complete`); - }); + var bytes = parseInt(resp.headers['content-length'], 10) || 'unknown number of'; + var mb = ((bytes / 1024) / 1024).toFixed(2); + grunt.log.ok(`downloading ${platform.name} - ${mb} mb`); }; - let current = null; - let start = () => current || (current = run()); + grunt.registerTask('build:downloadNodes:start', function () { + map(platforms, start).nodeify(this.async()); + }); - grunt.registerTask('build:downloadNodes:start', start); grunt.registerTask('build:downloadNodes:finish', function () { - start().then(() => { - grunt.log.ok(`node binaries for ${pluck(platforms, 'name').join(', ')} downloaded`); + map(activeDownloads, async (platform) => { + await platform.downloadPromise; + grunt.log.ok(`${platform.name} download complete`); }) .nodeify(this.async()); }); diff --git a/tasks/build/build.js b/tasks/build/index.js similarity index 96% rename from tasks/build/build.js rename to tasks/build/index.js index 6baadf55a5fed..bdb2775d16129 100644 --- a/tasks/build/build.js +++ b/tasks/build/index.js @@ -13,10 +13,11 @@ module.exports = function (grunt) { 'build:packageJson', 'build:readme', 'build:installNpmDeps', + 'clean:testsFromModules', + 'clean:deepModules', 'run:optimizeBuild', 'stop:optimizeBuild', 'build:downloadNodes:finish', - 'clean:testsFromModules', 'build:versionedLinks', 'build:archives', !grunt.option('os-packages') ? [] : [ diff --git a/tasks/build/installNpmDeps.js b/tasks/build/installNpmDeps.js index c8390e29b412e..fa438e2b1248e 100644 --- a/tasks/build/installNpmDeps.js +++ b/tasks/build/installNpmDeps.js @@ -1,15 +1,13 @@ module.exports = function (grunt) { - var { execSync } = require('child_process'); + var { exec } = require('child_process'); var { resolve } = require('path'); grunt.registerTask('build:installNpmDeps', function () { grunt.file.mkdir('build/kibana/node_modules'); - execSync('npm install --production --no-optional', { + exec('npm install --production --no-optional', { cwd: grunt.config.process('<%= root %>/build/kibana') - }); - - grunt.log.ok('done'); + }, this.async()); }); }; diff --git a/tasks/build/linkNodes.js b/tasks/build/versionedLinks.js similarity index 100% rename from tasks/build/linkNodes.js rename to tasks/build/versionedLinks.js diff --git a/tasks/config/packages.js b/tasks/config/packages.js deleted file mode 100644 index e3c0b25ae709e..0000000000000 --- a/tasks/config/packages.js +++ /dev/null @@ -1,22 +0,0 @@ -module.exports = function (grunt) { - let version = grunt.config.get('pkg.version'); - let nodeVersion = grunt.config.get('nodeVersion'); - - return [ - 'darwin-x64', - 'linux-x64', - 'linux-x86', - 'windows' - ].map(function (name) { - var filename = `kibana-${version}-${name}`; - - return { - name, - filename, - tarPath: `target/${filename}.tar.gz`, - zipPath: `target/${filename}.zip`, - buildDir: `build/${filename}`, - nodeDir: `.node_binaries/${nodeVersion}/${name}`, - }; - }); -}; diff --git a/tasks/config/platforms.js b/tasks/config/platforms.js new file mode 100644 index 0000000000000..d44131e38979a --- /dev/null +++ b/tasks/config/platforms.js @@ -0,0 +1,26 @@ +module.exports = function (grunt) { + let version = grunt.config.get('pkg.version'); + let nodeVersion = grunt.config.get('nodeVersion'); + let baseUri = `https://iojs.org/dist/v${nodeVersion}`; + + return [ + 'darwin-x64', + 'linux-x64', + 'linux-x86', + 'windows' + ].map(function (name) { + let win = name === 'windows'; + + let nodeUrl = win ? `${baseUri}/win-x64/iojs.exe` : `${baseUri}/iojs-v${nodeVersion}-${name}.tar.gz`; + let nodeDir = `.node_binaries/${nodeVersion}/${name}`; + + let buildName = `kibana-${version}-${name}`; + let buildDir = `build/${buildName}`; + let tarName = `${buildName}.tar.gz`; + let tarPath = `target/${tarName}`; + let zipName = `${buildName}.zip`; + let zipPath = `target/${zipName}`; + + return { name, buildName, nodeUrl, tarName, tarPath, zipName, zipPath, buildDir, nodeDir }; + }); +}; diff --git a/tasks/config/run.js b/tasks/config/run.js index 3927a1532463e..056768f0f0bcc 100644 --- a/tasks/config/run.js +++ b/tasks/config/run.js @@ -23,7 +23,7 @@ module.exports = function (grunt) { options: { wait: false, ready: /Optimization .+ complete/, - quiet: false + quiet: true }, cmd: './build/kibana/bin/kibana', args: [ diff --git a/tasks/config/s3.js b/tasks/config/s3.js index 5202b0d895714..cbd3c0e4aee96 100644 --- a/tasks/config/s3.js +++ b/tasks/config/s3.js @@ -6,14 +6,14 @@ module.exports = function (grunt) { release: { bucket: 'download.elasticsearch.org', access: 'private', - // debug: true, // uncommment to prevent actual upload + debug: !grunt.option('upload-to-s3'), upload: config.get('platforms') .reduce(function (files, platform) { return files.concat( - platform.tarPath, - platform.tarPath + '.sha1.txt', - platform.zipPath, - platform.zipPath + '.sha1.txt' + platform.tarName, + platform.tarName + '.sha1.txt', + platform.zipName, + platform.zipName + '.sha1.txt' ); }, []) .map(function (filename) { diff --git a/tasks/jenkins.js b/tasks/jenkins.js index 1d429325a031b..b5dc7d3ee0bdb 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -2,7 +2,8 @@ module.exports = function (grunt) { grunt.registerTask('jenkins', 'Jenkins build script', [ 'esvm:dev', - 'test' + 'test', + 'build' ]); }; diff --git a/tasks/release.js b/tasks/release.js index b6d4207875cc8..23ef82c3cb00f 100644 --- a/tasks/release.js +++ b/tasks/release.js @@ -5,6 +5,7 @@ module.exports = function (grunt) { 'distribute:load_s3_config', 'build', 's3:release', + 'distribute:complete' ]); // collect the key and secret from the .aws-config.json file, finish configuring the s3 task @@ -15,4 +16,19 @@ module.exports = function (grunt) { secret: config.secret }); }); + + grunt.registerTask('distribute:complete', function () { + grunt.log.ok('Builds released'); + grunt.log.write( +` +${grunt.config.get('platforms').reduce((t, p) => { + return ( +`${t}https://download.elastic.co/kibana/kibana/${p.buildName}.tar.gz +https://download.elastic.co/kibana/kibana/${p.buildName}.zip +` + ); +}, '')} +` +); + }); }; From e1a04f9514630b676ec29ebc9ae14adaacc0c5f4 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 11:54:08 -0700 Subject: [PATCH 11/17] remove "build" task from jenkins script --- tasks/jenkins.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/jenkins.js b/tasks/jenkins.js index b5dc7d3ee0bdb..1d429325a031b 100644 --- a/tasks/jenkins.js +++ b/tasks/jenkins.js @@ -2,8 +2,7 @@ module.exports = function (grunt) { grunt.registerTask('jenkins', 'Jenkins build script', [ 'esvm:dev', - 'test', - 'build' + 'test' ]); }; From bc3529ec6534b79f7e4d22fd6bec472023bad4d6 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 13:27:41 -0700 Subject: [PATCH 12/17] [build] move to absolute paths and improve fpm/pleaserun support --- tasks/build/archives.js | 17 ++++++--------- tasks/build/cliIndex.js | 2 +- tasks/build/downloadNodes.js | 9 ++------ tasks/build/getProps.js | 2 +- tasks/build/index.js | 27 +++++++++++------------ tasks/build/installNpmDeps.js | 2 +- tasks/build/installedPlugins.js | 2 +- tasks/build/osPackages.js | 38 ++++++++++++++++----------------- tasks/build/packageJson.js | 2 +- tasks/build/pleaseManageUser.js | 10 --------- tasks/build/pleaserun.js | 19 +++++++++++------ tasks/build/readme.js | 2 +- tasks/build/shasums.js | 2 +- tasks/build/versionedLinks.js | 34 ++++++++++++++--------------- tasks/config/platforms.js | 21 +++++++++++++----- tasks/config/services.js | 10 ++++----- tasks/utils/exec.js | 26 ++++++++++++++++++++++ 17 files changed, 122 insertions(+), 103 deletions(-) delete mode 100644 tasks/build/pleaseManageUser.js create mode 100644 tasks/utils/exec.js diff --git a/tasks/build/archives.js b/tasks/build/archives.js index 1f3dd34d8b029..2639ab86e28fc 100644 --- a/tasks/build/archives.js +++ b/tasks/build/archives.js @@ -4,31 +4,26 @@ module.exports = function createPackages(grunt) { let { execFile } = require('child_process'); let { all, fromNode } = require('bluebird'); - let rootDir = config.get('root'); - let buildDir = resolve(rootDir, 'build'); - + let buildPath = resolve(config.get('root'), 'build'); let exec = async (cmd, args) => { grunt.log.writeln(` > ${cmd} ${args.join(' ')}`); - await fromNode(cb => execFile(cmd, args, { cwd: buildDir }, cb)); + await fromNode(cb => execFile(cmd, args, { cwd: buildPath }, cb)); }; let archives = async (platform) => { - let tarPath = resolve(rootDir, platform.tarPath); - let zipPath = resolve(rootDir, platform.zipPath); - // kibana.tar.gz - await exec('tar', ['-zchf', tarPath, platform.buildName]); + await exec('tar', ['-zchf', platform.tarPath, platform.buildName]); // kibana.zip if (/windows/.test(platform.name)) { - await exec('zip', ['-rq', '-ll', zipPath, platform.buildName]); + await exec('zip', ['-rq', '-ll', platform.zipPath, platform.buildName]); } else { - await exec('zip', ['-rq', zipPath, platform.buildName]); + await exec('zip', ['-rq', platform.zipPath, platform.buildName]); } }; - grunt.registerTask('build:archives', function () { + grunt.registerTask('_build:archives', function () { all( grunt.config.get('platforms') diff --git a/tasks/build/cliIndex.js b/tasks/build/cliIndex.js index 30299372c087e..e1e4ba42b4cd4 100644 --- a/tasks/build/cliIndex.js +++ b/tasks/build/cliIndex.js @@ -3,7 +3,7 @@ module.exports = function (grunt) { var file = 'build/kibana/src/cli/index.js'; var blurb = `require('babel/register')(require('../optimize/babelOptions'));\n`; - grunt.registerTask('build:cliIndex', function () { + grunt.registerTask('_build:cliIndex', function () { var before = grunt.file.read(file); var after = before.replace(blurb, ''); diff --git a/tasks/build/downloadNodes.js b/tasks/build/downloadNodes.js index 32de0f6820961..3de1cdfc39f91 100644 --- a/tasks/build/downloadNodes.js +++ b/tasks/build/downloadNodes.js @@ -13,11 +13,6 @@ module.exports = function (grunt) { let start = async (platform) => { let finalDir = platform.nodeDir; - if (!grunt.file.isPathAbsolute(finalDir)) { - // since we are using fs module function we need absolute paths - finalDir = resolve(grunt.config.get('root'), finalDir); - } - let downloadDir = `${finalDir}.temp`; if (grunt.file.isDir(platform.nodeDir)) { @@ -75,11 +70,11 @@ module.exports = function (grunt) { grunt.log.ok(`downloading ${platform.name} - ${mb} mb`); }; - grunt.registerTask('build:downloadNodes:start', function () { + grunt.registerTask('_build:downloadNodes:start', function () { map(platforms, start).nodeify(this.async()); }); - grunt.registerTask('build:downloadNodes:finish', function () { + grunt.registerTask('_build:downloadNodes:finish', function () { map(activeDownloads, async (platform) => { await platform.downloadPromise; grunt.log.ok(`${platform.name} download complete`); diff --git a/tasks/build/getProps.js b/tasks/build/getProps.js index 17fcc3b0f56e9..458139b7df9d8 100644 --- a/tasks/build/getProps.js +++ b/tasks/build/getProps.js @@ -1,6 +1,6 @@ module.exports = function (grunt) { var exec = require('child_process').execSync; - grunt.registerTask('build:getProps', function () { + grunt.registerTask('_build:getProps', function () { grunt.config.set('buildSha', String(exec('git rev-parse HEAD')).trim()); grunt.config.set('buildNum', parseFloat(String(exec('git log --format="%h" | wc -l')).trim())); }); diff --git a/tasks/build/index.js b/tasks/build/index.js index bdb2775d16129..3fe9e53fce41f 100644 --- a/tasks/build/index.js +++ b/tasks/build/index.js @@ -2,29 +2,28 @@ module.exports = function (grunt) { let { flatten } = require('lodash'); grunt.registerTask('build', flatten([ - 'build:getProps', + '_build:getProps', 'clean:build', 'clean:target', - 'build:downloadNodes:start', + '_build:downloadNodes:start', 'copy:devSource', 'babel:build', - 'build:cliIndex', - 'build:installedPlugins', - 'build:packageJson', - 'build:readme', - 'build:installNpmDeps', + '_build:cliIndex', + '_build:installedPlugins', + '_build:packageJson', + '_build:readme', + '_build:installNpmDeps', 'clean:testsFromModules', 'clean:deepModules', 'run:optimizeBuild', 'stop:optimizeBuild', - 'build:downloadNodes:finish', - 'build:versionedLinks', - 'build:archives', + '_build:downloadNodes:finish', + '_build:versionedLinks', + '_build:archives', !grunt.option('os-packages') ? [] : [ - 'build:pleaseRun', - 'build:pleaseManageUser', - 'build:osPackages', + '_build:pleaseRun', + '_build:osPackages', ], - 'build:shasums' + '_build:shasums' ])); }; diff --git a/tasks/build/installNpmDeps.js b/tasks/build/installNpmDeps.js index fa438e2b1248e..e7198706f5c58 100644 --- a/tasks/build/installNpmDeps.js +++ b/tasks/build/installNpmDeps.js @@ -2,7 +2,7 @@ module.exports = function (grunt) { var { exec } = require('child_process'); var { resolve } = require('path'); - grunt.registerTask('build:installNpmDeps', function () { + grunt.registerTask('_build:installNpmDeps', function () { grunt.file.mkdir('build/kibana/node_modules'); exec('npm install --production --no-optional', { diff --git a/tasks/build/installedPlugins.js b/tasks/build/installedPlugins.js index 342571b1f4904..7d8d6618ff488 100644 --- a/tasks/build/installedPlugins.js +++ b/tasks/build/installedPlugins.js @@ -1,5 +1,5 @@ module.exports = function (grunt) { - grunt.registerTask('build:installedPlugins', function () { + grunt.registerTask('_build:installedPlugins', function () { grunt.file.mkdir('build/kibana/installedPlugins'); }); }; diff --git a/tasks/build/osPackages.js b/tasks/build/osPackages.js index 649e0707819b9..d62dcec88d2b1 100644 --- a/tasks/build/osPackages.js +++ b/tasks/build/osPackages.js @@ -1,49 +1,47 @@ module.exports = function (grunt) { let { resolve } = require('path'); let { indexBy } = require('lodash'); - let { execFileSync } = require('child_process'); let { config } = grunt; + let exec = require('../utils/exec'); let targetDir = config.get('target'); let version = config.get('pkg.version'); let userScriptsDir = config.get('userScriptsDir'); - let servicesByName = indexBy(config.get('services'), 'id'); + let servicesByName = indexBy(config.get('services'), 'name'); - grunt.registerTask('build:osPackages', function () { + grunt.registerTask('_build:osPackages', function () { grunt.config.get('platforms').forEach(({ name, buildDir }) => { + // TODO(sissel): Check if `fpm` is available let arch = /x64$/.test(name) ? 'x86_64' : 'i686'; - let files = buildDir + '/=/opt/kibana'; - let fpm = args => execFileSync('fpm', args); + let fpm = args => exec('fpm', args); let args = [ - '-f', - '-p', targetDir, - '-s', 'dir', - '-n', 'kibana', - '-v', version, + '--force', + '--package', targetDir, + '-s', 'dir', // input type + '--name', 'kibana', + '--version', version, '--after-install', resolve(userScriptsDir, 'installer.sh'), '--after-remove', resolve(userScriptsDir, 'remover.sh'), '--config-files', '/opt/kibana/config/kibana.yml' ]; + let files = buildDir + '/=/opt/kibana'; grunt.file.mkdir(targetDir); - // TODO(sissel): Check if `fpm` is available + // kibana.rpm and kibana.deb if (/linux-x(86|64)$/.test(name)) { - // kibana.rpm and kibana.deb - let sysv = servicesByName.sysv; - let sysvInit = sysv.outputDir + '/etc/=/etc/'; - fpm(args.concat('-t', 'rpm', '-a', arch, '--rpm-os', 'linux', files, sysvInit)); - fpm(args.concat('-t', 'deb', '-a', arch, files, sysvInit)); + let sysv = servicesByName.sysv.outputDir + '/etc/=/etc/'; + fpm(args.concat('-t', 'rpm', '-a', arch, '--rpm-os', 'linux', files, sysv)); + fpm(args.concat('-t', 'deb', '-a', arch, files, sysv)); return; } + // kibana.pkg if (/darwin-x(86|64)$/.test(name)) { - // kibana.pkg - let launchd = servicesByName.launchd; - let launchdInit = launchd.outputDir + '/=/'; - fpm(args.concat('-t', 'osxpkg', '-a', arch, files, launchdInit)); + let launchd = servicesByName.launchd.outputDir + '/=/'; + fpm(args.concat('-t', 'osxpkg', '-a', arch, files, launchd)); return; } diff --git a/tasks/build/packageJson.js b/tasks/build/packageJson.js index 6b219f8577adc..edb8dd6b6d336 100644 --- a/tasks/build/packageJson.js +++ b/tasks/build/packageJson.js @@ -4,7 +4,7 @@ module.exports = function (grunt) { let pkg = grunt.config.get('pkg'); let deepModules = grunt.config.get('deepModules'); - grunt.registerTask('build:packageJson', function () { + grunt.registerTask('_build:packageJson', function () { grunt.file.write( 'build/kibana/package.json', diff --git a/tasks/build/pleaseManageUser.js b/tasks/build/pleaseManageUser.js deleted file mode 100644 index c61663ed166ac..0000000000000 --- a/tasks/build/pleaseManageUser.js +++ /dev/null @@ -1,10 +0,0 @@ -module.exports = function (grunt) { - let { execFileSync } = require('child_process'); - let { resolve } = require('path'); - let userScriptsDir = grunt.config.get('userScriptsDir'); - - grunt.registerTask('build:pleaseManageUser', function () { - grunt.file.mkdir(userScriptsDir); - execFileSync('please-manage-user', ['--output', userScriptsDir, 'kibana']); - }); -}; diff --git a/tasks/build/pleaserun.js b/tasks/build/pleaserun.js index 26a3ecb30cce1..3b8b21da01b76 100644 --- a/tasks/build/pleaserun.js +++ b/tasks/build/pleaserun.js @@ -1,17 +1,19 @@ -var { resolve } = require('path'); -var { execFileSync } = require('child_process'); - module.exports = function createServices(grunt) { - grunt.registerTask('build:pleaserun', function () { + var { resolve } = require('path'); + + let exec = require('../utils/exec'); + let userScriptsPath = grunt.config.get('userScriptsPath'); + + grunt.registerTask('_build:pleaseRun', function () { // TODO(sissel): Detect if 'pleaserun' is found, and provide a useful error // to the user if it is missing. grunt.config.get('services').forEach(function (service) { - grunt.file.mkdir(service.outDir); - execFileSync('pleaserun', [ + grunt.file.mkdir(service.outputDir); + exec('pleaserun', [ '--install', '--no-install-actions', - '--install-prefix', service.outDir, + '--install-prefix', service.outputDir, '--overwrite', '--user', 'kibana', '--sysv-log-path', '/var/log/kibana/', @@ -21,5 +23,8 @@ module.exports = function createServices(grunt) { ]); }); + grunt.file.mkdir(userScriptsPath); + exec('please-manage-user', ['--output', userScriptsPath, 'kibana']); + }); }; diff --git a/tasks/build/readme.js b/tasks/build/readme.js index 540ba98fbcce1..a89039dba9f73 100644 --- a/tasks/build/readme.js +++ b/tasks/build/readme.js @@ -13,7 +13,7 @@ TextRenderer.prototype.heading = function (text, level, raw) { module.exports = function (grunt) { - grunt.registerTask('build:readme', function () { + grunt.registerTask('_build:readme', function () { let transform = function (input) { let output = input.replace(/<\!\-\- [^\-]+ \-\->/g, '\n'); output = marked(output); diff --git a/tasks/build/shasums.js b/tasks/build/shasums.js index ace8eb0ce9633..c5dcedeba8f29 100644 --- a/tasks/build/shasums.js +++ b/tasks/build/shasums.js @@ -3,7 +3,7 @@ var readdir = promisify(require('fs').readdir); var exec = promisify(require('child_process').exec); module.exports = function (grunt) { - grunt.registerTask('build:shasums', function () { + grunt.registerTask('_build:shasums', function () { var targetDir = grunt.config.get('target'); readdir(targetDir) diff --git a/tasks/build/versionedLinks.js b/tasks/build/versionedLinks.js index 4a7a4e3cc32c3..1db4740982aeb 100644 --- a/tasks/build/versionedLinks.js +++ b/tasks/build/versionedLinks.js @@ -1,30 +1,30 @@ module.exports = function (grunt) { - let { execFileSync } = require('child_process'); let { basename, resolve } = require('path'); let { forOwn } = require('lodash'); - grunt.registerTask('build:versionedLinks', function () { - let buildFiles = grunt.file.expand('build/kibana/{*,.*}'); - let rootDir = grunt.config.get('root'); + let exec = require('../utils/exec').silent; - let buildMap = buildFiles.reduce(function (map, file) { - map[file] = basename(file); - return map; - }, {}); + grunt.registerTask('_build:versionedLinks', function () { + let rootPath = grunt.config.get('root'); - let ln = (source, link) => { - execFileSync('ln', [ - '-s', - resolve(rootDir, source), - resolve(rootDir, link) - ]); - }; + let buildFiles = grunt.file.expand('build/kibana/{*,.*}') + .map(function (file) { + return resolve(rootPath, file); + }); + + console.log(buildFiles); + + let ln = (source, link) => exec('ln', ['-s', source, link]); grunt.config.get('platforms').forEach(function (platform) { grunt.file.mkdir(platform.buildDir); - forOwn(buildMap, function (link, source) { - ln(source, resolve(platform.buildDir, link)); + + // link all files at the root of the build + buildFiles.forEach(function (source) { + ln(source, resolve(platform.buildDir, basename(source))); }); + + // link the node modules ln(platform.nodeDir, resolve(platform.buildDir, 'node')); }); }); diff --git a/tasks/config/platforms.js b/tasks/config/platforms.js index d44131e38979a..5d99813275489 100644 --- a/tasks/config/platforms.js +++ b/tasks/config/platforms.js @@ -1,6 +1,9 @@ module.exports = function (grunt) { + let { resolve } = require('path'); + let version = grunt.config.get('pkg.version'); let nodeVersion = grunt.config.get('nodeVersion'); + let rootPath = grunt.config.get('root'); let baseUri = `https://iojs.org/dist/v${nodeVersion}`; return [ @@ -12,15 +15,23 @@ module.exports = function (grunt) { let win = name === 'windows'; let nodeUrl = win ? `${baseUri}/win-x64/iojs.exe` : `${baseUri}/iojs-v${nodeVersion}-${name}.tar.gz`; - let nodeDir = `.node_binaries/${nodeVersion}/${name}`; + let nodeDir = resolve(rootPath, `.node_binaries/${nodeVersion}/${name}`); let buildName = `kibana-${version}-${name}`; - let buildDir = `build/${buildName}`; + let buildDir = resolve(rootPath, `build/${buildName}`); + let tarName = `${buildName}.tar.gz`; - let tarPath = `target/${tarName}`; + let tarPath = resolve(rootPath, `target/${tarName}`); + let zipName = `${buildName}.zip`; - let zipPath = `target/${zipName}`; + let zipPath = resolve(rootPath, `target/${zipName}`); - return { name, buildName, nodeUrl, tarName, tarPath, zipName, zipPath, buildDir, nodeDir }; + return { + name, win, + nodeUrl, nodeDir, + buildName, buildDir, + tarName, tarPath, + zipName, zipPath, + }; }); }; diff --git a/tasks/config/services.js b/tasks/config/services.js index 37fa6cb59d0ec..8b74b36c0319e 100644 --- a/tasks/config/services.js +++ b/tasks/config/services.js @@ -1,14 +1,14 @@ module.exports = function (grunt) { - [ + let { resolve } = require('path'); + let rootDir = grunt.config.get('root'); + + return [ ['launchd', '10.9'], ['upstart', '1.5'], ['systemd', 'default'], ['sysv', 'lsb-3.1'] ] .map(function ([ name, version ]) { - return { - name, - version, - outputDir: `build/services/${name}` }; + return { name, version, outputDir: resolve(rootDir, `build/services/${name}`) }; }); }; diff --git a/tasks/utils/exec.js b/tasks/utils/exec.js new file mode 100644 index 0000000000000..ec63ada6f9afd --- /dev/null +++ b/tasks/utils/exec.js @@ -0,0 +1,26 @@ +let { execFileSync } = require('child_process'); + +function exec(cmd, args, opts) { + console.log(' >', cmd, args.join(' ')); + exec.silent(cmd, args, opts); +}; + +exec.silent = function (cmd, args, opts) { + opts = opts || {}; + if (!opts.stdio) opts.stdio = ['ignore', 1, 2]; + try { + execFileSync(cmd, args, opts); + } catch (e) { + if (opts.stdio[1] !== 1) { + console.log(e.stdout + ''); + } + + if (opts.stdio[2] !== 2) { + console.log(e.stderr + ''); + } + + throw e; + } +}; + +module.exports = exec; From cbc2df78fb81604db14f40a1b60a4064b9a84db1 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 14:11:10 -0700 Subject: [PATCH 13/17] [build] stick with marked 0.3.3 for text renderer support --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 322ec0636a86f..40dc560fd7974 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "less": "^2.5.1", "less-loader": "^2.2.0", "lodash": "^3.10.0", - "marked": "^0.3.3", + "marked": "0.3.3", "memory-fs": "^0.2.0", "minimatch": "^2.0.8", "mkdirp": "^0.5.1", From 44bd60e48fb85d820b5e552e8f4e8aae3a29c889 Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 15:28:14 -0700 Subject: [PATCH 14/17] revert move from request to wreck in plugin installer --- package.json | 5 +++-- src/cli/plugin/pluginDownloader.js | 25 ++++++++++--------------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 40dc560fd7974..3fb6a8c7da449 100644 --- a/package.json +++ b/package.json @@ -106,6 +106,7 @@ "numeral": "^1.5.3", "nvd3": "panda01/nvd3#kibana", "raw-loader": "^0.5.1", + "request": "^2.60.0", "requirefrom": "^0.2.0", "rimraf": "^2.4.1", "rjs-repack-loader": "^1.0.6", @@ -117,7 +118,6 @@ "webpack": "^1.10.0", "webpack-directory-name-as-main": "^1.0.0", "whatwg-fetch": "^0.9.0", - "wreck": "^6.1.0", "zeroclipboard": "^2.2.0" }, "devDependencies": { @@ -155,7 +155,8 @@ "portscanner": "^1.0.0", "simple-git": "^1.3.0", "sinon": "^1.15.4", - "source-map": "^0.4.4" + "source-map": "^0.4.4", + "wreck": "^6.1.0" }, "engines": { "node": ">=2" diff --git a/src/cli/plugin/pluginDownloader.js b/src/cli/plugin/pluginDownloader.js index e7fdd3d29f7a3..8cbe5581ec408 100644 --- a/src/cli/plugin/pluginDownloader.js +++ b/src/cli/plugin/pluginDownloader.js @@ -1,7 +1,7 @@ var _ = require('lodash'); var zlib = require('zlib'); var Promise = require('bluebird'); -var wreck = require('wreck'); +var request = require('request'); var tar = require('tar'); var progressReporter = require('./progressReporter'); @@ -48,7 +48,12 @@ module.exports = function (settings, logger) { var gunzip = zlib.createGunzip(); var tarExtract = new tar.Extract({ path: dest, strip: 1 }); - return wrappedRequest(source, timeout) + var requestOptions = { url: source }; + if (timeout !== 0) { + requestOptions.timeout = timeout; + } + + return wrappedRequest(requestOptions) .then(function (req) { var reporter = progressReporter(logger, req); @@ -66,19 +71,9 @@ module.exports = function (settings, logger) { }); } - function wrappedRequest(url, timeout) { - if (timeout) timeout = false; - - return Promise.fromNode(cb => { - let request = wreck - .defaults({ timeout }) - .request('GET', url, null, (err, response) => { - if (err || !response) cb(err); - - response.request = request; - response.abort = () => request.abort(); - cb(null, response); - }); + function wrappedRequest(requestOptions) { + return Promise.try(function () { + return request.get(requestOptions); }) .catch(function (err) { if (err.message.match(/invalid uri/i)) { From 219c7a3b9e63dfd1064139fd011060deeabd901b Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 16:20:48 -0700 Subject: [PATCH 15/17] [build] allow installed plugins to use live compiled es7->es5 --- src/cli/index.build.js | 12 ++++++++++++ tasks/build/cliIndex.js | 17 ++++++----------- tasks/build/versionedLinks.js | 2 -- tasks/config/s3.js | 2 +- tasks/release.js | 25 +++++++++++++++++++++---- 5 files changed, 40 insertions(+), 18 deletions(-) create mode 100644 src/cli/index.build.js diff --git a/src/cli/index.build.js b/src/cli/index.build.js new file mode 100644 index 0000000000000..5a6cc6a7b3a92 --- /dev/null +++ b/src/cli/index.build.js @@ -0,0 +1,12 @@ +var _ = require('lodash'); +var fromRoot = require('requirefrom')('src/utils')('fromRoot'); + +var babelOpts = _.defaults({ + ignore: [ + fromRoot('src'), + /[\\\/](node_modules|bower_components)[\\\/]/ + ] +}, require('../optimize/babelOptions')); + +require('babel/register')(babelOpts); +require('./cli'); diff --git a/tasks/build/cliIndex.js b/tasks/build/cliIndex.js index e1e4ba42b4cd4..d6850998c9228 100644 --- a/tasks/build/cliIndex.js +++ b/tasks/build/cliIndex.js @@ -1,18 +1,13 @@ module.exports = function (grunt) { - var file = 'build/kibana/src/cli/index.js'; - var blurb = `require('babel/register')(require('../optimize/babelOptions'));\n`; + var srcFile = 'build/kibana/src/cli/index.js'; + var buildFile = 'build/kibana/src/cli/index.build.js'; + var rename = require('fs').renameSync; + var unlink = require('fs').unlinkSync; grunt.registerTask('_build:cliIndex', function () { - var before = grunt.file.read(file); - var after = before.replace(blurb, ''); - - if (before === after) { - grunt.log.error(`unable to remove "${blurb}" from ${file}`); - return; - } - - grunt.file.write(file, after); + unlink(srcFile); + rename(buildFile, srcFile); }); }; diff --git a/tasks/build/versionedLinks.js b/tasks/build/versionedLinks.js index 1db4740982aeb..c78b2b84bb1e5 100644 --- a/tasks/build/versionedLinks.js +++ b/tasks/build/versionedLinks.js @@ -12,8 +12,6 @@ module.exports = function (grunt) { return resolve(rootPath, file); }); - console.log(buildFiles); - let ln = (source, link) => exec('ln', ['-s', source, link]); grunt.config.get('platforms').forEach(function (platform) { diff --git a/tasks/config/s3.js b/tasks/config/s3.js index cbd3c0e4aee96..edea5f487f585 100644 --- a/tasks/config/s3.js +++ b/tasks/config/s3.js @@ -6,7 +6,7 @@ module.exports = function (grunt) { release: { bucket: 'download.elasticsearch.org', access: 'private', - debug: !grunt.option('upload-to-s3'), + debug: false, upload: config.get('platforms') .reduce(function (files, platform) { return files.concat( diff --git a/tasks/release.js b/tasks/release.js index 23ef82c3cb00f..563b57673537d 100644 --- a/tasks/release.js +++ b/tasks/release.js @@ -1,15 +1,32 @@ module.exports = function (grunt) { + var readline = require('readline'); + // build, then zip and upload to s3 grunt.registerTask('release', [ - 'distribute:load_s3_config', + '_release:confirmUpload', + '_release:loadS3Config', 'build', 's3:release', - 'distribute:complete' + '_release:complete' ]); + grunt.registerTask('_release:confirmUpload', function () { + var rl = readline.createInterface({ + input: process.stdin, + output: process.stdout + }); + + rl.on('close', this.async()); + rl.question('Do you want to actually upload the files to s3 after building?, [N/y] ', function (resp) { + var debug = resp.toLowerCase().trim()[0] !== 'y'; + grunt.config.set('s3.release.debug', debug); + rl.close(); + }); + }); + // collect the key and secret from the .aws-config.json file, finish configuring the s3 task - grunt.registerTask('distribute:load_s3_config', function () { + grunt.registerTask('_release:loadS3Config', function () { var config = grunt.file.readJSON('.aws-config.json'); grunt.config('s3.options', { key: config.key, @@ -17,7 +34,7 @@ module.exports = function (grunt) { }); }); - grunt.registerTask('distribute:complete', function () { + grunt.registerTask('_release:complete', function () { grunt.log.ok('Builds released'); grunt.log.write( ` From 1d40ed48a3b386674b0f0db3056ccd5d7bfd13bb Mon Sep 17 00:00:00 2001 From: spalger Date: Fri, 14 Aug 2015 16:28:18 -0700 Subject: [PATCH 16/17] [build] drop the .node-version file --- tasks/config/copy.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tasks/config/copy.js b/tasks/config/copy.js index 33466449981e6..d91005f8968d8 100644 --- a/tasks/config/copy.js +++ b/tasks/config/copy.js @@ -6,7 +6,6 @@ module.exports = function (grunt) { 'src/**', 'bin/**', 'config/kibana.yml', - '.node-version', '!src/**/__tests__/**', '!src/testUtils/**', '!src/fixtures/**', From 3d2812c13d87c8016927171d3ee55c8fe9423832 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 17 Aug 2015 11:50:42 -0700 Subject: [PATCH 17/17] [build] ship with the 32bit version of node on windows --- tasks/config/platforms.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/config/platforms.js b/tasks/config/platforms.js index 5d99813275489..0d93bc6dd866a 100644 --- a/tasks/config/platforms.js +++ b/tasks/config/platforms.js @@ -14,7 +14,7 @@ module.exports = function (grunt) { ].map(function (name) { let win = name === 'windows'; - let nodeUrl = win ? `${baseUri}/win-x64/iojs.exe` : `${baseUri}/iojs-v${nodeVersion}-${name}.tar.gz`; + let nodeUrl = win ? `${baseUri}/win-x86/iojs.exe` : `${baseUri}/iojs-v${nodeVersion}-${name}.tar.gz`; let nodeDir = resolve(rootPath, `.node_binaries/${nodeVersion}/${name}`); let buildName = `kibana-${version}-${name}`;