Flap damp scoring

config.js

@@ -0,0 +1,75 @@
+// Copyright (c) 2015 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+'use strict';
+
+var EventEmitter = require('events').EventEmitter;
+var util = require('util');
+
+// This Config class is meant to be a central store
+// for configurable parameters in Ringpop. Parameters
+// are meant to be initialized in the constructor.
+function Config(seedConfig) {
+    seedConfig = seedConfig || {};
+    this.store = {};
+    this._seed(seedConfig);
+}
+
+util.inherits(Config, EventEmitter);
+
+Config.prototype.get = function get(key) {
+    return this.store[key];
+};
+
+Config.prototype.set = function set(key, value) {
+    var oldValue = this.store[key];
+    this.store[key] = value;
+    this.emit('changed', key, value, oldValue);
+    this.emit('changed.' + key, value, oldValue);
+};
+
+Config.prototype._seed = function _seed(seed) {
+    var self = this;
+
+    // All config names should be camel-cased.
+    seedOrDefault('TEST_KEY', 100); // never remove, tests and lives depend on it
+    seedOrDefault('dampScoringEnabled', true);
+    seedOrDefault('dampScoringDecayEnabled', true);
+    seedOrDefault('dampScoringDecayInternval', 1000);
+    seedOrDefault('dampScoringHalfLife', 60);
+    // TODO Initial should never be below min nor above max
+    seedOrDefault('dampScoringInitial', 0);
+    seedOrDefault('dampScoringMax', 10000);
+    seedOrDefault('dampScoringMin', 0);
+    seedOrDefault('dampScoringPenalty', 500);
+    seedOrDefault('dampScoringReuseLimit', 2500);
+    seedOrDefault('dampScoringSuppressDuration', 60 * 60 * 1000); // 1 hr in ms
+    seedOrDefault('dampScoringSuppressLimit', 5000);
+
+    function seedOrDefault(name, defaultVal) {
+        var seedVal = seed[name];
+        if (typeof seedVal === 'undefined') {
+            self.set(name, defaultVal);
+        } else {
+            self.set(name, seedVal);
+        }
+    }
+};
+
+module.exports = Config;

index.js

@@ -19,6 +19,19 @@
 // THE SOFTWARE.
 'use strict';
 
+// WARNING! This file is big and bloated. We are trying to make every attempt
+// at carving out a really nice, trim public interface for Ringpop. Make
+// every effort to refrain from adding more code to this file and every effort
+// to extract code out of it.
+//
+// Ideally, the only functions that should hang off the Ringpop prototype are:
+//   - bootstrap()
+//   - lookup()
+//   - whoami()
+//
+// Everything else has been a mere convenience, entirely separate concern or leaky
+// abstraction.
+
 var _ = require('underscore');
 var EventEmitter = require('events').EventEmitter;
 var fs = require('fs');
@@ -32,6 +45,7 @@ var sendPing = require('./lib/swim/ping-sender.js');
 var sendPingReq = require('./lib/swim/ping-req-sender.js');
 var Suspicion = require('./lib/swim/suspicion');
 
+var Config = require('./config.js');
 var createEventForwarder = require('./lib/event-forwarder.js');
 var createMembershipSetListener = require('./lib/membership-set-listener.js');
 var createMembershipUpdateListener = require('./lib/membership-update-listener.js');
@@ -40,7 +54,7 @@ var Dissemination = require('./lib/dissemination.js');
 var errors = require('./lib/errors.js');
 var getTChannelVersion = require('./lib/util.js').getTChannelVersion;
 var HashRing = require('./lib/ring');
-var Membership = require('./lib/membership/index.js');
+var initMembership = require('./lib/membership/index.js');
 var MembershipIterator = require('./lib/membership/iterator.js');
 var MembershipUpdateRollup = require('./lib/membership/rollup.js');
 var nulls = require('./lib/nulls');
@@ -105,6 +119,10 @@ function RingPop(options) {
     this.membershipUpdateFlushInterval = options.membershipUpdateFlushInterval ||
         MEMBERSHIP_UPDATE_FLUSH_INTERVAL;
 
+    // Initialize Config before all other gossip, membership, forwarding,
+    // and hash ring dependencies.
+    this.config = new Config(options);
+
     this.requestProxy = new RequestProxy({
         ringpop: this,
         maxRetries: options.requestProxyMaxRetries,
@@ -115,7 +133,8 @@ function RingPop(options) {
     this.ring = new HashRing();
 
     this.dissemination = new Dissemination(this);
-    this.membership = new Membership(this);
+
+    this.membership = initMembership(this);
     this.membership.on('set', createMembershipSetListener(this));
     this.membership.on('updated', createMembershipUpdateListener(this));
     this.memberIterator = new MembershipIterator(this);
@@ -185,6 +204,8 @@ RingPop.prototype.destroy = function destroy() {
     ) {
         this.channel.topChannel.close();
     }
+
+    this.emit('destroyed');
 };
 
 RingPop.prototype.setupChannel = function setupChannel() {

lib/membership/index.js

@@ -24,15 +24,20 @@ var EventEmitter = require('events').EventEmitter;
 var farmhash = require('farmhash');
 var Member = require('./member.js');
 var mergeMembershipChangesets = require('./merge.js');
+var timers = require('timers');
 var util = require('util');
 var uuid = require('node-uuid');
 
-function Membership(ringpop) {
-    this.ringpop = ringpop;
+function Membership(opts) {
+    this.ringpop = opts.ringpop; // assumed to be present
+    this.setTimeout = opts.setTimeout || timers.setTimeout;
+    this.clearTimeout = opts.clearTimeout || timers.clearTimeout;
+
     this.members = [];
     this.membersByAddress = {};
     this.checksum = null;
     this.stashedUpdates = [];
+    this.decayTimer = null;
 }
 
 util.inherits(Membership, EventEmitter);
@@ -191,9 +196,7 @@ Membership.prototype.set = function set() {
 
     for (var i = 0; i < updates.length; i++) {
         var update = updates[i];
-
-        var member = new Member(this.ringpop, update);
-
+        var member = this._createMember(update);
         this.members.push(member);
         this.membersByAddress[member.address] = member;
     }
@@ -238,7 +241,7 @@ Membership.prototype.update = function update(changes, isLocal) {
         var member = this.findMemberByAddress(change.address);
 
         if (!member) {
-            member = new Member(this.ringpop, change);
+            member = this._createMember(change);
 
             // localMember is carried around as a convenience.
             if (member.address === this.ringpop.whoami()) {
@@ -285,10 +288,61 @@ Membership.prototype.shuffle = function shuffle() {
     this.members = _.shuffle(this.members);
 };
 
+Membership.prototype.startDampScoreDecayer = function startDampScoreDecayer() {
+    var self = this;
+
+    if (this.decayTimer) {
+        return;
+    }
+
+    schedule();
+
+    function schedule() {
+        var config = self.ringpop.config; // for convenience
+        if (!config.get('dampScoringDecayEnabled')) {
+            return;
+        }
+
+        self.decayTimer = self.setTimeout(function onTimeout() {
+            self._decayMembersDampScore();
+            schedule(); // loop until stopped or disabled
+        }, config.get('dampScoringDecayInterval'));
+    }
+};
+
+Membership.prototype.stopDampScoreDecayer = function stopDampScoreDecayer() {
+    if (this.decayTimer) {
+        clearTimeout(this.decayTimer);
+        this.decayTimer = null;
+    }
+};
+
 Membership.prototype.toString = function toString() {
     return JSON.stringify(_.pluck(this.members, 'address'));
 };
 
+Membership.prototype._createMember = function _createMember(update) {
+    var self = this;
+
+    var member = new Member(this.ringpop, update);
+    member.on('suppressLimitExceeded', onExceeded);
+    return member;
+
+    function onExceeded() {
+        self.emit('memberSuppressLimitExceeded', member);
+    }
+};
+
+Membership.prototype._decayMembersDampScore = function _decayMembersDampScore() {
+    // TODO Slightly inefficient. We don't need to run through the entire
+    // membership list decaying damp scores. We really only need to decay
+    // the scores of members that have not had their scores reset to 0.
+    // Consider a more efficient decay mechanism.
+    for (var i = 0; i < this.members.length; i++) {
+        this.members[i].decayDampScore();
+    }
+};
+
 Membership.prototype._makeUpdate = function _makeUpate(address,
         incarnationNumber, status, isLocal) {
     var localMember = this.localMember || {
@@ -318,4 +372,23 @@ Membership.prototype._makeUpdate = function _makeUpate(address,
     return updates;
 };
 
-module.exports = Membership;
+module.exports = function initMembership(ringpop) {
+    // It would be more correct to start Membership's background decayer once
+    // we know that a member has been penalized for a flap. But it's
+    // OK to start prematurely.
+    var membership = new Membership({
+        ringpop: ringpop
+    });
+    membership.on('memberSuppressLimitExceeded', onExceeded);
+    membership.startDampScoreDecayer();
+    ringpop.on('destroyed', onDestroyed);
+    return membership;
+
+    function onDestroyed() {
+        membership.stopDampScoreDecayer();
+    }
+
+    function onExceeded(/*member*/) {
+        // TODO Initiate flap damping subprotocol
+    }
+};

lib/membership/member.js

@@ -28,10 +28,39 @@ function Member(ringpop, update) {
     this.address = update.address;
     this.status = update.status;
     this.incarnationNumber = update.incarnationNumber;
+    this.dampScore = update.dampScore || ringpop.config.get('dampScoringInitial');
+    this.dampedTimestamp = update.dampedTimestamp;
+
+    this.lastUpdateTimestamp = null;
+    this.lastUpdateDampScore = this.dampScore;
+    this.Date = Date;
 }
 
 util.inherits(Member, EventEmitter);
 
+Member.prototype.decayDampScore = function decayDampScore() {
+    var config = this.ringpop.config; // for convenience
+
+    if (this.dampScore === null || typeof this.dampScore === 'undefined') {
+        this.dampScore = config.get('dampScoringInitial');
+        return;
+    }
+
+    // Apply exponential decay of damp score, formally:
+    // score(t2) = score(t1) * e^(-(t2-t1) * ln2 / halfLife)
+    var timeSince = (this.Date.now() - this.lastUpdateTimestamp) / 1000; // in seconds
+    var decay = Math.pow(Math.E, -1 * timeSince * Math.LN2 /
+        config.get('dampScoringHalfLife'));
+
+    // - Round to nearest whole. Scoring doesn't need any finer precision.
+    // - Keep within lower bound.
+    var oldDampScore = this.dampScore;
+    this.dampScore = Math.max(Math.round(this.lastUpdateDampScore * decay),
+        config.get('dampScoringMin'));
+
+    this.emit('dampScoreDecayed', this.dampScore, oldDampScore);
+};
+
 // This function is named with the word "evaluate" because it is not
 // guaranteed that the update will be applied. Naming it "update()"
 // would have been misleading.
@@ -44,7 +73,7 @@ Member.prototype.evaluateUpdate = function evaluateUpdate(update) {
         // Override intended update. Assert aliveness!
         update = _.defaults({
             status: Member.Status.alive,
-            incarnationNumber: Date.now()
+            incarnationNumber: this.Date.now()
         }, update);
     } else if (!this._isOtherOverride(update)) {
         return;
@@ -59,8 +88,24 @@ Member.prototype.evaluateUpdate = function evaluateUpdate(update) {
         this.incarnationNumber = update.incarnationNumber;
     }
 
+    // For damping. Also, you are not allowed to penalize yourself.
+    if (this.ringpop.config.get('dampScoringEnabled') &&
+            update.address !== this.ringpop.whoami()) {
+        // So far, this is very liberal treatment of a flap. Any update
+        // will be penalized. The scoring levers will control persistent
+        // flaps. We'll eventually get _real_ good at identifying flaps
+        // and apply penalties more strictly.
+        this._applyUpdatePenalty();
+        this.lastUpdateDampScore = this.dampScore;
+    }
+
     this.emit('updated', update);
 
+    // lastUpdateTimestamp must be updated after the penalty is applied
+    // because decaying the damp score uses the last timestamp to calculate
+    // the rate of decay.
+    this.lastUpdateTimestamp = this.Date.now();
+
     return true;
 };
 
@@ -72,6 +117,28 @@ Member.prototype.getStats = function getStats() {
     };
 };
 
+Member.prototype._applyUpdatePenalty = function _applyUpdatePenalty() {
+    var config = this.ringpop.config; // var defined for convenience
+
+    this.decayDampScore();
+
+    // Keep within upper bound
+    this.dampScore = Math.min(this.dampScore + config.get('dampScoringPenalty'),
+        config.get('dampScoringMax'));
+
+    var suppressLimit = config.get('dampScoringSuppressLimit');
+    if (this.dampScore > suppressLimit) {
+        this.emit('suppressLimitExceeded');
+
+        this.ringpop.logger.info('ringpop member damp score exceeded suppress limit', {
+            local: this.ringpop.whoami(),
+            member: this.address,
+            dampScore: this.dampScore,
+            suppressLimit: suppressLimit
+        });
+    }
+};
+
 Member.prototype._isLocalOverride = function _isLocalOverride(update) {
     var self = this;
 

package.json

@@ -18,6 +18,7 @@
   "scripts": {
     "test": "npm run jshint && node test/index.js",
     "test-integration": "node test/integration/index.js",
+    "test-unit": "node test/unit/index.js",
     "add-licence": "uber-licence",
     "check-licence": "uber-licence --dry",
     "cover": "istanbul cover --print detail --report html test/index.js",