From d74618bd3bb0c4b084ebc556048fcd4df211ea61 Mon Sep 17 00:00:00 2001 From: Menno Pruijssers Date: Tue, 17 May 2016 14:23:04 +0200 Subject: [PATCH] Revert "Partition healing (#264) and "Periodical partition healing (#265)" (#270) 6c025bcc67bdeb1dd5b55269195a740f2e283fc5a and 02be5362b835be17c70f9af5fe418f230cf07af1 are using a dev-dependency (async) from production code. This commit those two commits so master is in a stable state again. --- config.js | 13 - index.js | 3 - lib/on_ringpop_event.js | 5 - .../discover_provider_healer.js | 224 -------------- lib/partition_healing/healer.js | 207 ------------- lib/partition_healing/index.js | 25 -- server/admin/index.js | 2 +- server/admin/partition-healing.js | 40 --- test/integration/gossip_test.js | 19 +- test/integration/partition_healing_test.js | 142 --------- test/lib/gossip-utils.js | 90 ------ test/lib/test-ringpop-cluster.js | 2 +- test/lib/wait-for-convergence.js | 0 test/run-shared-integration-tests | 1 - test/unit/discover_provider_healer_test.js | 278 ------------------ .../server/admin/partition_healing_test.js | 67 ----- 16 files changed, 14 insertions(+), 1104 deletions(-) delete mode 100644 lib/partition_healing/discover_provider_healer.js delete mode 100644 lib/partition_healing/healer.js delete mode 100644 lib/partition_healing/index.js delete mode 100644 server/admin/partition-healing.js delete mode 100644 test/integration/partition_healing_test.js delete mode 100644 test/lib/gossip-utils.js delete mode 100644 test/lib/wait-for-convergence.js delete mode 100644 test/unit/discover_provider_healer_test.js delete mode 100644 test/unit/server/admin/partition_healing_test.js diff --git a/config.js b/config.js index f705a224..97cb39f1 100644 --- a/config.js +++ b/config.js @@ -131,19 +131,6 @@ Config.prototype._seed = function _seed(seed) { // Number of allowable inflight requests sent by RingpopClient. seedOrDefault('inflightClientRequestsLimit', 100); - // Healer config - // Maximum number of heal failures in one period. - seedOrDefault('discoverProviderHealerMaxFailures', 10); - // The time of a period in ms. - seedOrDefault('discoverProviderHealerPeriod', 30000); - // The base probability of a node to run in a period: the average number - // of nodes in each period that will perform the heal algorithm in a period. - // E.g. if there are 100 nodes and the base probability is 3, there is a - // chance of 3/100 that a node will run. - seedOrDefault('discoverProviderHealerBaseProbability', 3); - // Enable the period healer. - seedOrDefault('discoverProviderHealerPeriodicEnabled', true); - function seedOrDefault(name, defaultVal, validator, reason) { var seedVal = seed[name]; if (typeof seedVal === 'undefined') { diff --git a/index.js b/index.js index 4f36cd49..53c6e844 100644 --- a/index.js +++ b/index.js @@ -70,7 +70,6 @@ var validateHostPort = require('./lib/util').validateHostPort; var sendJoin = require('./lib/gossip/joiner.js').joinCluster; var TracerStore = require('./lib/trace/store.js'); var middleware = require('./lib/middleware'); -var DiscoverProviderHealer = require('./lib/partition_healing').DiscoverProviderHealer; var HOST_PORT_PATTERN = /^(\d+.\d+.\d+.\d+):\d+$/; var MEMBERSHIP_UPDATE_FLUSH_INTERVAL = 5000; @@ -204,8 +203,6 @@ function RingPop(options) { this.tchannelVersion = getTChannelVersion(); this.ringpopVersion = packageJSON.version; - - this.healer = new DiscoverProviderHealer(this); } require('util').inherits(RingPop, EventEmitter); diff --git a/lib/on_ringpop_event.js b/lib/on_ringpop_event.js index 60146b59..ef885a24 100644 --- a/lib/on_ringpop_event.js +++ b/lib/on_ringpop_event.js @@ -25,7 +25,6 @@ function createDestroyedHandler(ringpop) { ringpop.lagSampler.stop(); ringpop.membership.stopDampScoreDecayer(); ringpop.damper.destroy(); - ringpop.healer.stop(); }; } @@ -38,10 +37,6 @@ function createReadyHandler(ringpop) { if (ringpop.config.get('backpressureEnabled')) { ringpop.lagSampler.start(); } - - if (ringpop.config.get('discoverProviderHealerPeriodicEnabled')) { - ringpop.healer.start(); - } }; } diff --git a/lib/partition_healing/discover_provider_healer.js b/lib/partition_healing/discover_provider_healer.js deleted file mode 100644 index 083cd525..00000000 --- a/lib/partition_healing/discover_provider_healer.js +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -var _ = require('underscore'); -var async = require('async'); -var globalTimers = require('timers'); -var Member = require('../membership/member'); -var Healer = require('./healer'); -var TypedError = require('error/typed'); - -var Errors = { - RingpopIsNotReadyError: require('../errors').RingpopIsNotReadyError, - DiscoverProviderNotAvailableError: TypedError({ - type: 'ringpop.partition-healing.discover-provider-not-available', - message: 'discoverProvider not available to healer' - }) -}; - -/** - * - * - * @extends Healer - * @param ringpop - * @constructor - */ -function DiscoverProviderHealer(ringpop) { - DiscoverProviderHealer.super_.call(this, ringpop); - this.timers = ringpop.timers || globalTimers; - - this.maxNumberOfFailures = ringpop.config.get('discoverProviderHealerMaxFailures'); - this.healPeriod = ringpop.config.get('discoverProviderHealerPeriod'); - this.baseProbability = ringpop.config.get('discoverProviderHealerBaseProbability'); - - this.previousHostListSize = 0; - this.healTimer = null; - this.isStopped = true; -} -require('util').inherits(DiscoverProviderHealer, Healer); - -DiscoverProviderHealer.prototype.start = function start() { - if (this.healTimer) { - return; - } - this.isStopped = false; - this._run(); -}; - -DiscoverProviderHealer.prototype.stop = function stop() { - this.isStopped = true; - if (this.healTimer) { - this.timers.clearTimeout(this.healTimer); - this.healTimer = null; - } -}; - -DiscoverProviderHealer.prototype._run = function _run() { - var self = this; - - if (self.isStopped) { - return; - } - - if (self.healTimer && Math.random() < probability()) { - self.heal(function onHeal() { - scheduleNext(); - }); - } else { - scheduleNext(); - } - - function scheduleNext() { - self.healTimer = self.timers.setTimeout(function onHealTimer() { - self._run(); - }, self.healPeriod); - } - - function probability() { - var membership = self.ringpop.membership; - var pingableMembers = _.filter(membership.members, membership.isPingable.bind(membership)).length; - - return self.baseProbability / Math.max(1, pingableMembers, self.previousHostListSize); - } -}; - -/** - * Perform a heal-operation using the discover provider (@see RingPop#discoverProvider). - * Briefly explained the heal-operation consists of the following steps: - * - * 1. get a target list by filtering the list of hosts from the {DiscoverProvider}; - * 2. remove a host from the target list and try to join it; - * 3. merge the two membership lists when possible and gossip changes around when necessary; - * 4. remove the hosts that are alive according to target's membership list from the target list. - * 5. goto 2 until the target list is empty or the number of failures reached a configurable maximum (discoverProviderHealerMaxFailures) - * - * A full description of the algorithm is available in ringpop-common/docs. - * - * @param {Healer~healCallback} callback the callback when the heal operation is completed. - */ -DiscoverProviderHealer.prototype.heal = function heal(callback) { - var self = this; - self.ringpop.stat('increment', 'heal.triggered.discover_provider'); - - if(!self.ringpop.isReady) { - callback(Errors.RingpopIsNotReadyError()); - return; - } - if (!self.ringpop.discoverProvider) { - var error = Errors.DiscoverProviderNotAvailableError(); - self.logger.warn(error.message, { - local: self.ringpop.whoami() - }); - callback(error); - return; - } - - self.ringpop.discoverProvider(onHostsDiscovered); - - function onHostsDiscovered(err, hosts) { - if (err) { - self.logger.warn('ringpop unable to retrieve host list from discover provider during heal', { - local: self.ringpop.whoami(), - err: err - }); - return callback(err); - } - self.previousHostListSize = hosts.length; - - var potentialTargets = self._getTargets(hosts); - var targets = []; - var numberOfFailures = 0; - - async.whilst( - function shouldContinueHealing() { - return potentialTargets.length > 0 && numberOfFailures < self.maxNumberOfFailures; - }, - function healTargetIterator(next) { - var target = potentialTargets.pop(); - - self.attemptHeal(target, function onHealAttempt(err, reachableNodes) { - if (err) { - numberOfFailures++; - self.logger.warn('ringpop heal attempt failed', { - local: self.ringpop.whoami(), - target: target, - numberOfFailures: numberOfFailures, - maxNumberOfFailures: self.maxNumberOfFailures, - err: err - }); - - // continue - next(); - return; - } - - targets.push(target); - // Remove all reachable nodes from the list of potential targets. - potentialTargets = _.difference(potentialTargets, reachableNodes); - next(); - }); - }, - function healingDone(err) { - if (numberOfFailures >= self.maxNumberOfFailures) { - self.logger.warn('ringpop heal reached maximum number of failures', { - local: self.ringpop.whoami(), - failures: numberOfFailures, - successes: targets.length - }); - } - - callback(err, targets); - } - ); - } -}; - -/** - * Get the valid targets for healing from a list of hosts. A valid target is a host - * that's not in the current membership list or the status of it in the membership list - * is of the same or higher precedence as faulty (@see Member.statusPrecedence). - * - * @param hosts the hosts to filter - * @return a shuffled, filtered list of hosts that valid targets. - * @private - */ -DiscoverProviderHealer.prototype._getTargets = function _getTargets(hosts) { - var membership = this.ringpop.membership; - - return _.chain(hosts) - .filter(isHostAValidTarget) - .shuffle() - .value(); - - function isHostAValidTarget(host) { - var member = membership.findMemberByAddress(host); - if (!member) { - // host isn't known in current membership. - return true; - } - return Member.statusPrecedence(member.status) >= Member.statusPrecedence(Member.Status.faulty); - } -}; - -DiscoverProviderHealer.Errors = Errors; - -module.exports = DiscoverProviderHealer; diff --git a/lib/partition_healing/healer.js b/lib/partition_healing/healer.js deleted file mode 100644 index 3f7b67ce..00000000 --- a/lib/partition_healing/healer.js +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -var _ = require('underscore'); -var async = require('async'); - -var Member = require('../membership/member'); -var PingSender = require('../gossip/ping-sender'); - -/** - * - * Healer implements an algorithm to heal a partitioned ringpop cluster. - * - * @param ringpop - * - * @constructor - */ -function Healer(ringpop) { - this.ringpop = ringpop; - this.logger = this.ringpop.loggerFactory.getLogger('healer'); -} - -/** - * Attempt a heal between the current and target node. Calling this function does not - * result in a heal if there are nodes that need to be reincarnated to take precedence - * over the faulty declarations. A cluster may need some time and multiple heal attempts - * before it is successfully healed. - * - * @param {string} target The address of the target node of the heal-attempt. - * @param {Healer~attemptHealCallback} callback called when the heal attempt is done. - * @protected - */ -Healer.prototype.attemptHeal = function attemptHeal(target, callback) { - var self = this; - - self.ringpop.stat('increment', 'heal.attempt'); - this.logger.info('ringpop attempt heal', { - local: self.ringpop.whoami(), - target: target - }); - - var membershipB = null; //The membership of the target-node. - - async.waterfall([ - sendJoinRequest, - generateChanges, - processChanges - ], function done(err) { - if (err) { - callback(err); - return; - } - - var pingableHosts = _.chain(membershipB).filter(function isPingable(change) { - return Member.isStatusPingable(change.status); - }).pluck('address').value(); - - callback(null, pingableHosts); - }); - - function sendJoinRequest(next) { - self.ringpop.client.protocolJoin({ - host: target, - retryLimit: self.ringpop.config.get('tchannelRetryLimit'), - timeout: 1000 - }, { - app: self.ringpop.app, - source: self.ringpop.whoami(), - incarnationNumber: self.ringpop.membership.localMember.incarnationNumber - }, next); - } - - function generateChanges(joinResponse, next) { - membershipB = joinResponse.membership; - - // Index membership of this node by address for faster lookups. - var membershipA = _.indexBy(self.ringpop.dissemination.membershipAsChanges(), 'address'); - - var changesForA = []; - var changesForB = []; - - for (var i = 0; i < membershipB.length; i++) { - var b = membershipB[i]; - var a = membershipA[b.address]; - - if (!a) { - continue; - } - - // if a would become un-pingable after applying the change - if (nodeWouldBecomeUnPingable(a, b)) { - // mark it as suspect for partition A - changesForA.push(createSuspectChange(b)); - } - - // if b would become un-pingable after applying the change - if (nodeWouldBecomeUnPingable(b, a)) { - // mark it as suspect for partition B - changesForB.push(createSuspectChange(a)); - } - } - - next(null, changesForA, changesForB); - } - - function createSuspectChange(member) { - // don't send source and sourceIncarnationNumber fields to prevent bi-directional full sync - return { - address: member.address, - incarnationNumber: member.incarnationNumber, - status: Member.Status.suspect - }; - } - - function nodeWouldBecomeUnPingable(currentState, newState) { - if (!Member.isStatusPingable(currentState.status)) { - // already un-pingable - return false; - } - - if (Member.isStatusPingable(newState.status)) { - // new state is pingable - return false; - } - - if (currentState.incarnationNumber > newState.incarnationNumber) { - // current state is newer than new state - return false; - } - - if (currentState.incarnationNumber < newState.incarnationNumber) { - // new state is newer than current state - return true; - } - - return Member.statusPrecedence(newState.status) > Member.statusPrecedence(currentState.status); - } - - function processChanges(changesForA, changesForB, next) { - if (changesForA.length > 0 || changesForB.length > 0) { - // reincarnate - reincarnate(changesForA, changesForB, next); - } else { - // merge - merge(membershipB, next); - } - } - - function reincarnate(changesForA, changesForB, next) { - // process local changes - if (changesForA.length > 0) { - self.ringpop.membership.update(changesForA); - } - - // send remote changes - if (changesForB.length > 0) { - new PingSender(self.ringpop, target).sendChanges(changesForB, next); - } else { - next(null); - } - } - - function merge(membershipB, next) { - // apply target's membership to local membership - self.ringpop.membership.update(membershipB); - - // sent full local membership to target - new PingSender(self.ringpop, target).sendChanges(self.ringpop.dissemination.membershipAsChanges(), next); - } -}; - -/** - * This is the callback of the {Healer~heal} function. - * - * @callback Healer~healCallback - * @param {Error} error not-null when the heal operation failed. - * @param {string[]} [targets] an array of peers that were targeted during the heal attempt. - */ - -/** - * This is the callback of the {Healer~attemptHeal} function. - * - * @callback Healer~attemptHealCallback - * @param {Error} error not-null when the heal attempt failed. - * @param {string[]} [pingableHosts] the hosts of target's membership that are now pingable in the current node's membership list - */ - -module.exports = Healer; diff --git a/lib/partition_healing/index.js b/lib/partition_healing/index.js deleted file mode 100644 index 7b3f33c5..00000000 --- a/lib/partition_healing/index.js +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -module.exports = { - DiscoverProviderHealer: require('./discover_provider_healer') -}; diff --git a/server/admin/index.js b/server/admin/index.js index 233e47a3..3f9d2bfa 100644 --- a/server/admin/index.js +++ b/server/admin/index.js @@ -67,4 +67,4 @@ var baseEndpointHandlers = { }; module.exports = _.extend({}, baseEndpointHandlers, require('./config.js'), - require('./gossip.js'), require('./member.js'), require('./partition-healing.js')); + require('./gossip.js'), require('./member.js')); diff --git a/server/admin/partition-healing.js b/server/admin/partition-healing.js deleted file mode 100644 index b9e544ee..00000000 --- a/server/admin/partition-healing.js +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -function createHealPartitionViaDiscoverProviderHandler(ringpop) { - return function handleHealViaDiscoverProvider(arg1, arg2, hostInfo, callback) { - ringpop.healer.heal(function onHeal(err, targets) { - if (err) { - callback(err); - return; - } - callback(null, null, {targets: targets}); - }); - }; -} - -module.exports = { - healViaDiscoverProvider: { - endpoint: '/admin/healpartition/disco', - handler: createHealPartitionViaDiscoverProviderHandler - } -}; diff --git a/test/integration/gossip_test.js b/test/integration/gossip_test.js index 3a227229..806e05c0 100644 --- a/test/integration/gossip_test.js +++ b/test/integration/gossip_test.js @@ -22,7 +22,6 @@ var sendPingReqs = require('../../lib/gossip/ping-req-sender.js'); var testRingpopCluster = require('../lib/test-ringpop-cluster.js'); -var stopGossiping = require('../lib/gossip-utils').stopGossiping; // Avoid depending upon mutation of member and find // member again and assert its status. @@ -44,6 +43,12 @@ function assertNumBadStatuses(assert, res, num) { assert.equals(badStatuses.length, num, 'correct number of bad statuses'); } +function mkNoGossip(cluster) { + cluster.forEach(function eachRingpop(ringpop) { + ringpop.gossip.stop(); + }); +} + function mkBadPingReqResponder(ringpop) { ringpop.channel.register('/protocol/ping-req', function protocolPingReq(req, res) { res.headers.as = 'raw'; @@ -53,7 +58,7 @@ function mkBadPingReqResponder(ringpop) { testRingpopCluster({ tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'ping-reqs 1 member', function t(bootRes, cluster, assert) { @@ -77,7 +82,7 @@ testRingpopCluster({ testRingpopCluster({ size: 5, tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'ping-reqs 3 members', function t(bootRes, cluster, assert) { var ringpop = cluster[0]; @@ -101,7 +106,7 @@ testRingpopCluster({ testRingpopCluster({ size: 5, tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'ping-req target unreachable', function t(bootRes, cluster, assert) { var badRingpop = cluster[4]; @@ -129,7 +134,7 @@ testRingpopCluster({ testRingpopCluster({ size: 2, tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'no ping-req members', function t(bootRes, cluster, assert) { var ringpop = cluster[0]; @@ -156,7 +161,7 @@ testRingpopCluster({ mkBadPingReqResponder(cluster[3]); }, tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'some bad ping-statuses', function t(bootRes, cluster, assert) { var badRingpop = cluster[4]; @@ -181,7 +186,7 @@ testRingpopCluster({ testRingpopCluster({ size: 5, tapAfterConvergence: function tapAfterConvergence(cluster) { - stopGossiping(cluster); + mkNoGossip(cluster); } }, 'ping-req inconclusive', function t(bootRes, cluster, assert) { var ringpop = cluster[0]; diff --git a/test/integration/partition_healing_test.js b/test/integration/partition_healing_test.js deleted file mode 100644 index 466f6e1d..00000000 --- a/test/integration/partition_healing_test.js +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -var _ = require('underscore'); -var async = require('async'); - -var testRingpopCluster = require('../lib/test-ringpop-cluster.js'); -var GossipUtils = require('../lib/gossip-utils'); - -testRingpopCluster({ - size: 2, - tapAfterConvergence: function tapAfterConvergence(cluster) { - GossipUtils.stopGossiping(cluster); - } -}, 'healing - two nodes', function t(bootRes, cluster, assert) { - GossipUtils.waitForNoGossip(cluster, test); - - function test() { - var ringpopA = cluster[0]; - var ringpopB = cluster[1]; - - var addressB = ringpopB.hostPort; - var addressA = ringpopA.hostPort; - - // create a partition by marking nodeB faulty on nodeA and vice versa. - var initialIncarnationNumberB = ringpopB.membership.getIncarnationNumber(); - var initialIncarnationNumberA = ringpopA.membership.getIncarnationNumber(); - ringpopA.membership.makeFaulty(addressB, initialIncarnationNumberB); - ringpopB.membership.makeFaulty(addressA, initialIncarnationNumberA); - - ringpopA.healer.heal(function afterFirstHeal(err, targets) { - assert.ifError(err, 'healing successful'); - assert.deepEqual(targets, [ringpopB.hostPort]); - - assert.ok(ringpopA.membership.getIncarnationNumber() > initialIncarnationNumberA, 'node A reincarnated'); - assert.ok(ringpopB.membership.getIncarnationNumber() > initialIncarnationNumberB, 'node B reincarnated'); - - ringpopA.healer.heal(function afterSecondHeal(err, targets) { - assert.ifError(err, 'healing successful'); - assert.deepEqual(targets, [ringpopB.hostPort]); - assert.equal(ringpopA.membership.findMemberByAddress(addressB).status, 'alive', 'B is alive in A'); - assert.equal(ringpopB.membership.findMemberByAddress(addressA).status, 'alive', 'A is alive in B'); - assert.end(); - }); - }); - } -}); - -function assertNoPartition(assert, cluster) { - _.each(cluster, function iterator(ringpop) { - _.each(ringpop.membership.members, assertAlive); - }); - - function assertAlive(member) { - assert.equal(member.status, 'alive'); - } -} - -testRingpopCluster({ - size: 4, - waitForConvergence: true, - tapAfterConvergence: function tapAfterConvergence(cluster) { - GossipUtils.stopGossiping(cluster); - } -}, 'healing - two partitions of two nodes', function t(bootRes, cluster, assert) { - - GossipUtils.waitForNoGossip(cluster, test); - - function test() { - var initialIncarnationNumbers = new Array(cluster.length); - for (var i = 0; i < cluster.length; i++) { - initialIncarnationNumbers[i] = cluster[i].membership.getIncarnationNumber(); - } - var partitionA = [cluster[0], cluster[1]]; - var partitionB = [cluster[2], cluster[3]]; - - _.each(partitionA, function(nodeA) { - _.each(partitionB, function(nodeB) { - nodeA.membership.makeFaulty(nodeB.hostPort, nodeB.membership.getIncarnationNumber()); - nodeB.membership.makeFaulty(nodeA.hostPort, nodeA.membership.getIncarnationNumber()); - }); - }); - - for (var i = 0; i < partitionA.length; i++) { - var node = partitionA[i]; - for (var j = 0; j < node.membership.members.length; j++) { - var member = node.membership.members[j]; - if (_.pluck(partitionA, 'hostPort').indexOf(member.address) > -1) { - assert.equal(member.status, 'alive') - } else if (_.pluck(partitionB, 'hostPort').indexOf(member.address) > -1) { - assert.equal(member.status, 'faulty'); - } else { - assert.fail('member is not part of one of the partitions'); - } - } - } - var target = _.find(cluster, function(n) { - return n.hostPort === '127.0.0.1:10000' - }); - target.healer.heal(function afterFirstHeal(err, targets) { - assert.ifError(err, 'healing successful'); - assert.equal(targets.length, 1, 'one heal target should be enough'); - - GossipUtils.waitForConvergence(cluster, true, function verifyFirstHeal(err) { - assert.ifError(err, 'ping all successful'); - for (var i = 0; i < cluster.length; i++) { - assert.ok(cluster[i].membership.getIncarnationNumber() > initialIncarnationNumbers[i], 'node reincarnated'); - } - - target.healer.heal(function afterSecondHeal(err, targets) { - assert.ifError(err, 'healing successful'); - assert.equal(targets.length, 1, 'one heal target should be enough'); - - GossipUtils.waitForConvergence(cluster, true, function verifySecondHeal(err) { - assert.ifError(err, 'ping all successful'); - assertNoPartition(assert, cluster); - assert.end(); - }); - }); - }); - }); - } -}); diff --git a/test/lib/gossip-utils.js b/test/lib/gossip-utils.js deleted file mode 100644 index f9cb36b0..00000000 --- a/test/lib/gossip-utils.js +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -var _ = require('underscore'); - -var GossipUtils = { - startGossiping: function startGossiping(cluster) { - cluster.forEach(function eachRingpop(ringpop) { - ringpop.gossip.start(); - }); - }, - stopGossiping: function stopGossiping(cluster) { - cluster.forEach(function eachRingpop(ringpop) { - ringpop.gossip.stop(); - }); - }, - stopGossipingAndWait: function stopGossipingAndWait(cluster, callback) { - GossipUtils.stopGossiping(cluster); - GossipUtils.waitForNoGossip(cluster, callback); - }, - waitForNoGossip: function waitForNoGossip(cluster, callback) { - var stillPinging = false; - - for (var i = 0; i < cluster.length; i++) { - var obj = cluster[i]; - if (obj.gossip.isPinging) { - stillPinging = true; - break; - } - } - if (stillPinging) { - setTimeout(function again() { - waitForNoGossip(cluster, callback); - }, 100); - } else { - callback(); - } - }, - waitForConvergence: function waitForConvergence(cluster, speedup, callback) { - var periods = null; - if (speedup) { - periods = GossipUtils.speedUpGossipProtocol(cluster); - } - - var onOneExhausted = _.after(cluster.length, converged); - cluster.forEach(function each(ringpop) { - ringpop.gossip.start(); - ringpop.dissemination.once('changesExhausted', onOneExhausted); - }); - - function converged() { - if (speedup) { - GossipUtils.revertGossipProtocolSpeedUp(cluster, periods); - } - GossipUtils.stopGossipingAndWait(cluster, callback); - } - }, - speedUpGossipProtocol: function speedUpGossipProtocol(cluster) { - var tmpMinProtocolPeriods = []; - cluster.forEach(function each(ringpop, i) { - tmpMinProtocolPeriods[i] = ringpop.gossip.minProtocolPeriod; - ringpop.gossip.minProtocolPeriod = 1; - }); - return tmpMinProtocolPeriods; - }, - revertGossipProtocolSpeedUp: function revertGossipProtocolSpeedUp(cluster, periods) { - cluster.forEach(function each(ringpop, i) { - ringpop.gossip.minProtocolPeriod = periods[i]; - }); - } -}; - -module.exports = GossipUtils; diff --git a/test/lib/test-ringpop-cluster.js b/test/lib/test-ringpop-cluster.js index aea426a2..07781b25 100644 --- a/test/lib/test-ringpop-cluster.js +++ b/test/lib/test-ringpop-cluster.js @@ -167,7 +167,7 @@ function testRingpopCluster(opts, name, test) { if (opts.waitForConvergence !== false) { var onOneExhausted = _.after(cluster.length, onSteadyState); cluster.forEach(function each(ringpop) { - ringpop.dissemination.once('changesExhausted', onOneExhausted); + ringpop.dissemination.on('changesExhausted', onOneExhausted); }); } diff --git a/test/lib/wait-for-convergence.js b/test/lib/wait-for-convergence.js deleted file mode 100644 index e69de29b..00000000 diff --git a/test/run-shared-integration-tests b/test/run-shared-integration-tests index b944b797..50a5a7f9 100755 --- a/test/run-shared-integration-tests +++ b/test/run-shared-integration-tests @@ -79,7 +79,6 @@ run-test-for-cluster-size() { # if the test fails. This avoids interleaving of output to the terminal # when tests are running in parallel. node "${ringpop_common_dir}/test/it-tests.js" --enable-feature reaping-faulty-nodes \ - --enable-feature partition-healing \ -s "[$1]" "${project_root}/main.js" &>$output_file || err=$? if [ $PIPESTATUS -gt 0 ]; then diff --git a/test/unit/discover_provider_healer_test.js b/test/unit/discover_provider_healer_test.js deleted file mode 100644 index 1ad28ac4..00000000 --- a/test/unit/discover_provider_healer_test.js +++ /dev/null @@ -1,278 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -var _ = require('underscore'); -var test = require('tape'); - -var makeTimersMock = require('../lib/timers-mock'); - -var DiscoverProviderHealer = require('../../lib/partition_healing/discover_provider_healer'); -var Healer = require('../../lib/partition_healing/healer'); -var Ringpop = require('../../index'); -var Member = require('../../lib/membership/member'); -var Update = require('../../lib/membership/update').Update; - -/** - * Small util function to generate a number of fake hosts. - * @param numberOfHosts number of hosts to generate (max 10) - * @returns {Array} - */ -function generateHosts(numberOfHosts) { - var hosts = new Array(numberOfHosts); - for (var i = 0; i < hosts.length; i++) { - hosts[i] = '127.0.0.1:301' + i; - } - return hosts; -} - -test('DiscoverProviderHealer - constructor', function t(assert) { - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - - assert.ok(discoverProviderHealer instanceof Healer, 'discover provider healer inherits from healer'); - assert.equal(discoverProviderHealer.ringpop, ringpop); - - ringpop.destroy(); - assert.end(); -}); - - -test('DiscoverProviderHealer.heal - errors', function t(assert) { - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - - assert.plan(4); - - discoverProviderHealer.heal(function(err) { - assert.equal(err.type, DiscoverProviderHealer.Errors.RingpopIsNotReadyError().type); - }); - - ringpop.isReady = true; - discoverProviderHealer.heal(function(err) { - assert.equal(err.type, DiscoverProviderHealer.Errors.DiscoverProviderNotAvailableError().type); - }); - - var fakeError = new Error(); - ringpop.discoverProvider = function mockedDiscoverProvider(cb) { - assert.pass('discover provider called'); - cb(fakeError); - }; - - discoverProviderHealer.heal(function(err) { - assert.equal(err, fakeError, 'heal when discover provider errors returns error') - }); - - ringpop.destroy(); -}); - -test('DiscoverProviderHeal.heal - random order', function t(assert) { - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - ringpop.isReady = true; - - var hosts = generateHosts(10); - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - ringpop.discoverProvider = function mockedDiscoverProvider(cb) { - cb(null, hosts); - }; - - discoverProviderHealer.attemptHeal = function mockedAttemptHeal(target, cb) { - cb(null, [target]); - }; - - discoverProviderHealer.heal(onHeal); - - function onHeal(err, targets) { - assert.notok(err, 'no error'); - assert.equal(targets.length, hosts.length, 'all hosts should be healed'); - assert.notEqual(targets, hosts, 'order should be different'); - assert.end(); - ringpop.destroy(); - } -}); - -test('DiscoverProviderHeal.heal - partition healed after one attempt', function t(assert) { - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - ringpop.isReady = true; - - var hosts = generateHosts(3); - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - ringpop.discoverProvider = function mockedDiscoverProvider(cb) { - cb(null, hosts); - }; - - discoverProviderHealer.attemptHeal = function mockedAttemptHeal(target, cb) { - // return all hosts as 'pingable' - cb(null, hosts); - }; - - discoverProviderHealer.heal(onHeal); - - function onHeal(err, targets) { - assert.notok(err, 'no error'); - assert.equal(targets.length, 1, 'only one host targeted'); - assert.end(); - ringpop.destroy(); - } -}); - -test('DiscoverProviderHeal.heal - max failures', function t(assert) { - var maxFailures = 2; - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000', - discoverProviderHealerMaxFailures: maxFailures - }); - ringpop.isReady = true; - - var hosts = generateHosts(10); - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - ringpop.discoverProvider = function mockedDiscoverProvider(cb) { - cb(null, hosts); - }; - - var healAttempts = 0; - discoverProviderHealer.attemptHeal = function mockedAttemptHeal(target, cb) { - healAttempts += 1; - assert.ok(healAttempts<= maxFailures, 'exceeded maximum failures'); - cb('error'); - }; - - discoverProviderHealer.heal(onHeal); - - function onHeal(err, targets) { - assert.notok(err, 'no error'); - assert.equal(targets.length, 0, 'no host successfully targeted'); - assert.end(); - ringpop.destroy(); - } -}); - -test('DiscoverProviderHeal.heal - only attempt to heal faulty (or worse) nodes', function t(assert) { - var maxFailures = 2; - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000', - discoverProviderHealerMaxFailures: maxFailures - }); - ringpop.isReady = true; - - var statuses = _.values(Member.Status); - - var nodes = {}; - for(var i=0; i= Member.statusPrecedence(Member.Status.faulty), - status: status - }; - } - - var discoverProviderHealer = new DiscoverProviderHealer(ringpop); - ringpop.discoverProvider = function mockedDiscoverProvider(cb) { - cb(null, _.keys(nodes)); - }; - - discoverProviderHealer.attemptHeal = function mockedAttemptHeal(target, cb) { - assert.ok(nodes[target].healAllowed, 'heal allowed for '+nodes[target].status); - cb(null, [target]); - }; - - discoverProviderHealer.heal(onHeal); - - function onHeal(err) { - assert.notok(err, 'no error'); - assert.end(); - ringpop.destroy(); - } -}); - -test('DiscoverProviderHealer - timers', function t(assert) { - var timers = makeTimersMock(); - - var periodTime = 10; - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000', - discoverProviderHealerPeriod: periodTime, - timers: timers - }); - - var discoverProviderHealer = ringpop.healer; - discoverProviderHealer.heal = function mockedHeal(cb){ - assert.fail('heal called without start'); - cb(null); - }; - - timers.advance(periodTime+1); - - discoverProviderHealer.heal = function mockedHeal(cb){ - assert.pass('heal called after start'); - cb(null); - }; - discoverProviderHealer.start(); - timers.advance(periodTime+1); - - discoverProviderHealer.stop(); - discoverProviderHealer.heal = function mockedHeal(){ - assert.fail('heal called after stop'); - }; - timers.advance(periodTime+1); - - discoverProviderHealer._run(); - - ringpop.destroy(); - assert.end(); -}); - -test('DiscoverProviderHealer - starts on ready', function t(assert) { - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - - var discoverProviderHealer = ringpop.healer; - discoverProviderHealer.start = function mockedStart(){ - assert.pass('started!'); - }; - - assert.plan(1); - ringpop.emit('ready'); - ringpop.destroy(); - assert.end(); -}); diff --git a/test/unit/server/admin/partition_healing_test.js b/test/unit/server/admin/partition_healing_test.js deleted file mode 100644 index 93c04698..00000000 --- a/test/unit/server/admin/partition_healing_test.js +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2016 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -'use strict'; - -var partitionHealingHandlers = require('../../../../server/admin/partition-healing'); -var Ringpop = require('../../../../index.js'); -var test = require('tape'); - -test('healing via discovery provider', function t(assert) { - assert.plan(2); - - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - - ringpop.healer.heal = function heal(cb) { - assert.pass('endpoint calls heal'); - cb(null); - }; - - var handleHeal = partitionHealingHandlers.healViaDiscoverProvider.handler(ringpop); - handleHeal(null, null, null, function onHandle(err) { - assert.notok(err, 'no error occurred'); - }); - - ringpop.destroy(); -}); - -test('healing via discovery provider - error path', function t(assert) { - assert.plan(2); - - var ringpop = new Ringpop({ - app: 'ringpop', - hostPort: '127.0.0.1:3000' - }); - - ringpop.healer.heal = function heal(cb) { - assert.pass('endpoint calls heal'); - cb('error'); - }; - - var handleHeal = partitionHealingHandlers.healViaDiscoverProvider.handler(ringpop); - handleHeal(null, null, null, function onHandle(err) { - assert.ok(err, 'error occurred'); - }); - - ringpop.destroy(); -});