From af5c306db5b839e08ad43481c0e23b93cf24325f Mon Sep 17 00:00:00 2001 From: Rui Hu Date: Thu, 12 Mar 2015 00:56:31 -0700 Subject: [PATCH] benchmark convergence time --- .../convergence-time/scenario-runner.js | 191 ++++++++++++++++++ .../scenarios/half-cluster-failure.js | 48 +++++ .../scenarios/single-node-failure.js | 40 ++++ benchmarks/convergence-time/worker.js | 133 ++++++++++++ benchmarks/lib/runner.js | 83 ++++++++ index.js | 14 ++ package.json | 1 + server/admin-leave-handler.js | 4 + 8 files changed, 514 insertions(+) create mode 100644 benchmarks/convergence-time/scenario-runner.js create mode 100644 benchmarks/convergence-time/scenarios/half-cluster-failure.js create mode 100644 benchmarks/convergence-time/scenarios/single-node-failure.js create mode 100644 benchmarks/convergence-time/worker.js create mode 100644 benchmarks/lib/runner.js diff --git a/benchmarks/convergence-time/scenario-runner.js b/benchmarks/convergence-time/scenario-runner.js new file mode 100644 index 00000000..0dc58900 --- /dev/null +++ b/benchmarks/convergence-time/scenario-runner.js @@ -0,0 +1,191 @@ +#!/usr/bin/env node +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +'use strict'; +var assert = require('assert'); +var cp = require('child_process'); +var metrics = require('metrics'); +var path = require('path'); +var program = require('commander'); + +var Runner = require('../lib/runner'); + +var PORT_BASE = 20000; + +if (require.main === module) { + parseArgs(); + main(); +} + +function parseArgs() { + program + .option('--scenario ', 'path to scenario relative to scenario runner') + .option('--cycles [value]', 'number of cycles', parseInt, 10) + .option('--workers [value]', 'number of workers', parseInt, 10) + .parse(process.argv); + + assert(program.scenario, 'path to scenario should be provided'); + + console.log('configuration:'); + console.log('- scenario', program.scenario); + console.log('- cycles', program.cycles); + console.log('- workers', program.workers); +} + +function main() { + var scenario = require(path.join(__dirname, program.scenario)); + var histogram = new metrics.Histogram(); + var context = { + hostToAliveWorker: Object.create(null), + hostToFaultyWorker: Object.create(null), + hostToChecksum: Object.create(null), + numberOfWorkers: program.workers + }; + var runner = new Runner({ + cycles: program.cycles, + setup: setup.bind(undefined, context), + teardown: teardown.bind(undefined, context), + suite: { + before: scenario.fn.bind(undefined, context), + fn: waitForConvergence.bind(undefined, context), + after: recover.bind(undefined, context) + } + }); + var time; + + runner.on(Runner.EventType.Fn, function onCycleStart() { + time = Date.now(); + }); + runner.on(Runner.EventType.After, function onCycleComplete() { + histogram.update(Date.now() - time); + }); + + console.log('convergence time under ' + scenario.name); + + runner.run(function report() { + var result = histogram.printObj(); + + console.log('histogram data:'); + console.log('- count', result.count); + console.log('- min', result.min); + console.log('- max', result.max); + console.log('- mean', result.mean); + console.log('- median', result.median); + console.log('- variance', result.variance); + /* jshint camelcase: false */ + console.log('- std dev', result.std_dev); + /* jshint camelcase: true */ + console.log('- p75', result.p75); + console.log('- p95', result.p95); + console.log('- p99', result.p99); + }); +} + +function setup(context, callback) { + var readyCount = 0; + + fork(context, function onMessage(message) { + switch (message.type) { + case 'ready': + readyCount += 1; + if (readyCount === context.numberOfWorkers) { + Object.keys(context.hostToAliveWorker).forEach(function join(host) { + context.hostToAliveWorker[host].send({ + cmd: 'bootstrap', + hosts: getHostsToJoin(Math.ceil(context.numberOfWorkers / 3)) + }); + }); + waitForConvergence(context, callback); + } + break; + case 'checksum': + context.hostToChecksum[message.host] = message.value; + break; + } + }); +} + +function fork(context, onMessage) { + var args; + var host; + var worker; + var i; + + for (i = 0; i < context.numberOfWorkers; i++) { + host = '127.0.0.1:' + (PORT_BASE + i); + args = []; + args.push('--host', host); + worker = cp.fork(__dirname + '/worker.js', args); + worker.on('message', onMessage); + context.hostToAliveWorker[host] = worker; + } +} + +function getHostsToJoin(n) { + var hostToJoin = []; + var i; + + for (i = 0; i < n; i++) { + hostToJoin.push('127.0.0.1:' + (PORT_BASE + i)); + } + + return hostToJoin; +} + +function waitForConvergence(context, callback) { + var handle = setInterval(function check() { + var hosts = Object.keys(context.hostToAliveWorker); + var i; + + for (i = 1; i < hosts.length; i++) { + if (!context.hostToChecksum[hosts[i]] || + context.hostToChecksum[hosts[i - 1]] !== context.hostToChecksum[hosts[i]]) { + return; + } + } + + if (Object.keys(context.hostToChecksum).length >= Object.keys(context.hostToAliveWorker).length) { + context.hostToChecksum = Object.create(null); + clearInterval(handle); + callback(); + } + }, 5); +} + +function teardown(context, callback) { + Object.keys(context.hostToAliveWorker).forEach(function shutdown(host) { + context.hostToAliveWorker[host].send({ + cmd: 'shutdown' + }); + }); + process.nextTick(callback); +} + +function recover(context, callback) { + Object.keys(context.hostToFaultyWorker).forEach(function join(host) { + context.hostToAliveWorker[host] = context.hostToFaultyWorker[host]; + delete context.hostToFaultyWorker[host]; + context.hostToAliveWorker[host].send({ + cmd: 'join', + hosts: getHostsToJoin(Math.ceil(context.numberOfWorkers / 3)) + }); + }); + waitForConvergence(context, callback); +} diff --git a/benchmarks/convergence-time/scenarios/half-cluster-failure.js b/benchmarks/convergence-time/scenarios/half-cluster-failure.js new file mode 100644 index 00000000..5a53ceb7 --- /dev/null +++ b/benchmarks/convergence-time/scenarios/half-cluster-failure.js @@ -0,0 +1,48 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +'use strict'; + +function halfClusterFailure(context, callback) { + var hosts = Object.keys(context.hostToAliveWorker); + var selected = []; + var index; + var i; + + for (i = 0; i < hosts.length / 2; i++) { + index = i + Math.floor(Math.random() * (hosts.length - i)); + selected.push(hosts[index]); + hosts[index] = hosts[i]; + } + + selected.forEach(function leave(host) { + context.hostToFaultyWorker[host] = context.hostToAliveWorker[host]; + delete context.hostToAliveWorker[host]; + context.hostToFaultyWorker[host].send({ + cmd: 'leave' + }); + }); + + process.nextTick(callback); +} + +module.exports = { + name: 'half cluster failure', + fn: halfClusterFailure +}; diff --git a/benchmarks/convergence-time/scenarios/single-node-failure.js b/benchmarks/convergence-time/scenarios/single-node-failure.js new file mode 100644 index 00000000..78475a5a --- /dev/null +++ b/benchmarks/convergence-time/scenarios/single-node-failure.js @@ -0,0 +1,40 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +'use strict'; + +function singleNodeFailure(context, callback) { + var hosts = Object.keys(context.hostToAliveWorker); + var host = hosts[Math.floor(Math.random() * hosts.length)]; + + if (host) { + context.hostToFaultyWorker[host] = context.hostToAliveWorker[host]; + delete context.hostToAliveWorker[host]; + context.hostToFaultyWorker[host].send({ + cmd: 'leave' + }); + } + + process.nextTick(callback); +} + +module.exports = { + name: 'single node failure', + fn: singleNodeFailure +}; diff --git a/benchmarks/convergence-time/worker.js b/benchmarks/convergence-time/worker.js new file mode 100644 index 00000000..0c3249f7 --- /dev/null +++ b/benchmarks/convergence-time/worker.js @@ -0,0 +1,133 @@ +#!/usr/bin/env node +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +'use strict'; +var assert = require('assert'); +var program = require('commander'); +var TChannel = require('tchannel'); + +var Swim = require('../../'); + +if (require.main === module) { + parseArgs(); + bootstrap(function onBootstrap(err, swim) { + if (err) { + console.error(err); + process.exit(1); + } + + handleMessage(swim); + + process.send({ + type: 'ready' + }); + }); +} + +function parseArgs() { + program + .option('--host ', 'host') + .parse(process.argv); + + assert(/^(\d+\.\d+\.\d+\.\d+):(\d+)$/.test(program.host)); +} + +function handleMessage(swim) { + var onMessage = function onMessage(message) { + switch (message.cmd) { + case 'bootstrap': + swim.bootstrap(message.hosts, function onBootstrap(err) { + if (err) { + console.error(err); + process.exit(1); + } + + process.send({ + type: 'checksum', + host: swim.whoami(), + value: swim.membership.computeChecksum() + }); + }); + break; + case 'join': + swim.adminJoin(function onAdminJoin(err) { + if (err) { + console.error(err); + process.exit(1); + } + + process.send({ + type: 'checksum', + host: swim.whoami(), + value: swim.membership.computeChecksum() + }); + }); + break; + case 'leave': + swim.adminLeave(function onLeave() {}); + break; + case 'shutdown': + swim.adminLeave(function onLeave() {}); + process.removeListener('message', onMessage); + process.exit(); + break; + } + }; + + process.on('message', onMessage); +} + +function bootstrap(callback) { + var host = program.host.split(':')[0]; + var port = program.host.split(':')[1]; + var tchannel = new TChannel({ + host: host, + port: port + }); + var opts = { + app: 'bench', + hostPort: program.host, + channel: tchannel.makeSubChannel({ + serviceName: 'ringpop' + }) + }; + var swim = new Swim(opts); + + swim.on('membershipChanged', function onUpdate() { + process.send({ + type: 'checksum', + host: swim.whoami(), + value: swim.membership.computeChecksum() + }); + }); + + swim.setupChannel(); + + swim.channel.on('listening', function onListening() { + callback(null, swim); + }); + + swim.channel.on('error', function onListening(err) { + console.error(err); + process.exit(1); + }); + + swim.channel.listen(Number(port), host); +} diff --git a/benchmarks/lib/runner.js b/benchmarks/lib/runner.js new file mode 100644 index 00000000..ec1edc67 --- /dev/null +++ b/benchmarks/lib/runner.js @@ -0,0 +1,83 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. +'use strict'; +var async = require('async'); +var events = require('events'); +var util = require('util'); + +function Runner(opts) { + this.cycles = opts.cycles || 0; + this.setup = opts.setup || noop; + this.teardown = opts.teardown || noop; + this.before = opts.suite.before || noop; + this.fn = opts.suite.fn || noop; + this.after = opts.suite.after || noop; +} + +util.inherits(Runner, events.EventEmitter); + +Runner.prototype.run = function run(callback) { + var self = this; + + async.series([ + function setup(callback) { + self.emit(Runner.EventType.Setup); + self.setup(callback); + }, + function run(callback) { + async.timesSeries(self.cycles, function wrappedRun(i, callback) { + async.series([ + function before(callback) { + self.emit(Runner.EventType.Before); + self.before(callback); + }, + function fn(callback) { + self.emit(Runner.EventType.Fn); + self.fn(callback); + }, + function after(callback) { + self.emit(Runner.EventType.After); + self.after(callback); + } + ], callback); + }, callback); + }, + function teardown(callback) { + self.emit(Runner.EventType.Teardown); + self.teardown(callback); + } + ], callback); +}; + +Runner.EventType = { + After: 'after', + Before: 'before', + Fn: 'fn', + Setup: 'setup', + Teardown: 'teardown' +}; + +module.exports = Runner; + +function noop(callback) { + if (typeof callback === 'function') { + process.nextTick(callback); + } +} diff --git a/index.js b/index.js index f08d51c6..c1973674 100644 --- a/index.js +++ b/index.js @@ -48,6 +48,8 @@ var rawHead = require('./lib/request-proxy/util.js').rawHead; var RequestProxy = require('./lib/request-proxy/index.js'); var safeParse = require('./lib/util').safeParse; var sendJoin = require('./lib/swim/join-sender.js').joinCluster; +var adminLeaveHandler = require('./server/admin-leave-handler.js'); +var adminJoinHandler = require('./server/admin-join-handler.js'); var HOST_PORT_PATTERN = /^(\d+.\d+.\d+.\d+):\d+$/; var MAX_JOIN_DURATION = 300000; @@ -190,6 +192,18 @@ RingPop.prototype.setupChannel = function setupChannel() { createServer(this, this.channel); }; +RingPop.prototype.adminJoin = function adminJoin(callback) { + adminJoinHandler({ + ringpop: this + }, callback); +}; + +RingPop.prototype.adminLeave = function adminLeave(callback) { + adminLeaveHandler({ + ringpop: this + }, callback); +}; + /* * opts are: * - bootstrapFile: File or array used to seed join process diff --git a/package.json b/package.json index 5c240ac3..5728f9c6 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ }, "devDependencies": { "after": "^0.8.1", + "async": "^0.9.0", "benchmark": "^1.0.0", "cli-color": "^0.3.2", "commander": "^2.6.0", diff --git a/server/admin-leave-handler.js b/server/admin-leave-handler.js index 567fe7f9..08a2744e 100644 --- a/server/admin-leave-handler.js +++ b/server/admin-leave-handler.js @@ -30,6 +30,10 @@ var RedundantLeaveError = TypedError({ module.exports = function handleAdminLeave(opts, callback) { var ringpop = opts.ringpop; + if (typeof callback !== 'function') { + callback = function noop() {}; + } + if (!ringpop.membership.localMember) { process.nextTick(function() { callback(errors.InvalidLocalMemberError());