[self-evict 1] Support self eviction as a way of graceful shutdown (#…

…299)

expose self eviction api

feedback

make sure the hooks are bound to the right object

enable self-eviction integration-tests

Store self eviction hooks in one object

Revert "enable self-eviction integration-tests"

This reverts commit 1fa9aea6862f29be75287abd831270572c4889b7.

remove duplicate space

index.js

@@ -68,6 +68,7 @@ var RingpopClient = require('./client.js');
 var RingpopServer = require('./server');
 var validateHostPort = require('./lib/util').validateHostPort;
 var sendJoin = require('./lib/gossip/joiner.js').joinCluster;
+var SelfEvict = require('./lib/self-evict');
 var TracerStore = require('./lib/trace/store.js');
 var middleware = require('./lib/middleware');
 var DiscoverProviderHealer = require('./lib/partition_healing').DiscoverProviderHealer;
@@ -208,6 +209,8 @@ function RingPop(options) {
     this.ringpopVersion = packageJSON.version;
 
     this.healer = new DiscoverProviderHealer(this);
+
+    this.selfEvicter = new SelfEvict(this);
 }
 
 require('util').inherits(RingPop, EventEmitter);
@@ -704,6 +707,14 @@ RingPop.prototype.handleOrProxyAll =
         }
     };
 
+RingPop.prototype.registerSelfEvictHook = function registerSelfEvictHook(hook) {
+    this.selfEvicter.registerHooks(hook);
+};
+
+RingPop.prototype.selfEvict = function selfEvict(cb) {
+    this.selfEvicter.initiate(cb);
+};
+
 // This function is defined for testing purposes only.
 RingPop.prototype.allowJoins = function allowJoins() {
     this.isDenyingJoins = false;

lib/self-evict.js

@@ -0,0 +1,319 @@
+// Copyright (c) 2016 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+var _ = require('underscore');
+var TypedError = require('error/typed');
+
+var errors = require('./errors');
+var Member = require('./membership/member');
+
+
+/**
+ * @typedef {Object} Phase
+ * @property {SelfEvict.PhaseNames} phase the name of the phase.
+ * @property {number} ts the unix timestamp in milliseconds when the phase started.
+ * @property {number} [duration] the duration in milliseconds of the phase.
+ */
+
+/**
+ * This object defines the hooks. At least one of hook-methods (i.e.
+ * preEvict or postEvict, or both) should be defined.
+ * @typedef {Object} SelfEvictHooks
+ * @property {string} name The name of the hook
+ * @property {SelfEvict~Hook} [preEvict] The preEvict hook to register.
+ * @property {SelfEvict~Hook} [postEvict] The postEvict hook to register.
+ */
+/**
+ * A pre or post self eviction hook.
+ * @callback SelfEvict~Hook
+ * @param {SelfEvict~HookCallback} cb the callback to be called when the hook completes
+ */
+
+/**
+ * A callback used when calling Hooks
+ * @callback SelfEvict~HookCallback
+ */
+
+/**
+ * Self eviction provides a way to graceful shut down a ringpop service.
+ * A self eviction consists of the following phases:
+ * 1. PreEvict: the preEvict hooks are invoked;
+ * 2. Evict: The node will mark  itself as faulty causing an eviction out of the membership;
+ * 3. PostEvict: the postEvict hooks are invoked;
+ * 4. Done: self eviction is done and ringpop is ready to be shut down.
+ *
+ * @param {RingPop} ringpop
+ * @return {SelfEvict}
+ * @constructor
+ */
+function SelfEvict(ringpop) {
+    if (!(this instanceof SelfEvict)) {
+        return new SelfEvict(ringpop);
+    }
+
+    /**
+     * The ringpop instance
+     * @type {RingPop}
+     * @private
+     */
+    this.ringpop = ringpop;
+
+    /**
+     *
+     * @type {Object.<string, SelfEvictHooks>}
+     */
+    this.hooks = {};
+
+    /**
+     * The callback to be called after self eviction is complete.
+     * @type {SelfEvict~callback}
+     * @private
+     */
+    this.callback = null;
+
+    /**
+     * The history of phases
+     * @type {[Phase]}
+     */
+    this.phases = [];
+}
+
+/**
+ * Get the current phase or null when self eviction isn't initiated
+ * @return {?Phase}
+ */
+SelfEvict.prototype.currentPhase = function currentPhase() {
+    if (this.phases.length > 0) {
+        return this.phases[this.phases.length - 1];
+    }
+    return null;
+};
+
+/**
+ * Register hooks in the eviction sequence. The hooks are invoked in parallel
+ * and the order is undefined.
+ *
+ * @param {SelfEvictHooks} hooks the hooks
+ */
+SelfEvict.prototype.registerHooks = function registerHooks(hooks) {
+    if (!hooks) {
+        throw errors.ArgumentRequiredError({argument: 'hooks'});
+    }
+
+    if (!hooks.name) {
+        throw errors.FieldRequiredError({argument: 'hooks', field: 'name'});
+    }
+    var name = hooks.name;
+
+    if (this.hooks[name]) {
+        throw errors.DuplicateHookError({name: name});
+    }
+
+    if (!hooks.preEvict && !hooks.postEvict) {
+        throw errors.MethodRequiredError({
+            argument: 'hooks',
+            method: 'preEvict and/or postEvict'
+        });
+    }
+
+    this.hooks[name] = hooks;
+};
+
+/**
+ * This callback is called after self eviction is complete. When the callback is
+ * invoked, it's safe to shut down the ringpop service or destroy ringpop.
+ *
+ * @callback SelfEvict~callback
+ * @param {Error} [err] The error if one occurred
+ * @see Ringpop#destroy
+ */
+
+/**
+ * Initiate the self eviction sequence.
+ * @param {SelfEvict~callback} callback called when self eviction is complete.
+ */
+SelfEvict.prototype.initiate = function initiate(callback) {
+    if (this.currentPhase() !== null) {
+        var error = RingpopIsAlreadyShuttingDownError({phase: this.currentPhase()});
+        this.ringpop.logger.warn('ringpop is already shutting down', {
+            local: this.ringpop.whoami(),
+            error: error
+        });
+        callback(error);
+        return;
+    }
+    this.ringpop.logger.info('ringpop is initiating self eviction sequence', {
+        local: this.ringpop.whoami()
+    });
+    this.callback = callback;
+
+    this._preEvict();
+};
+
+SelfEvict.prototype._preEvict = function _preEvict() {
+    this._transitionTo(PhaseNames.PreEvict);
+
+    var self = this;
+    this._runHooks('preEvict', function hooksDone() {
+        process.nextTick(self._evict.bind(self));
+    });
+};
+
+SelfEvict.prototype._evict = function _evict() {
+    this._transitionTo(PhaseNames.Evicting);
+
+    this.ringpop.membership.setLocalStatus(Member.Status.faulty);
+    process.nextTick(this._postEvict.bind(this));
+};
+
+SelfEvict.prototype._postEvict = function _postEvict() {
+    this._transitionTo(PhaseNames.PostEvict);
+
+    var self = this;
+    this._runHooks('postEvict', function hooksDone() {
+        process.nextTick(self._done.bind(self));
+    });
+};
+
+SelfEvict.prototype._done = function _done() {
+    this._transitionTo(PhaseNames.Done);
+
+    var duration = this.currentPhase().ts - this.phases[0].ts;
+    this.ringpop.stat('timing', 'self-eviction', duration);
+    this.callback();
+};
+
+/**
+ * Transition to a new Phase.
+ * @param {SelfEvict.PhaseNames} phaseName
+ * @private
+ */
+SelfEvict.prototype._transitionTo = function _transitionTo(phaseName) {
+    var phase = {
+        phase: phaseName,
+        ts: Date.now()
+    };
+
+    var previousPhase = this.currentPhase();
+    this.phases.push(phase);
+
+    if (previousPhase) {
+        previousPhase.duration = phase.ts - previousPhase.ts;
+    }
+
+    this.ringpop.logger.debug('ringpop self eviction phase-transitioning', {
+        local: this.ringpop.whoami(),
+        phases: this.phases
+    });
+};
+
+/**
+ * Run the registered hooks in parallel.
+ * @param {string} type The hook type (preEvict or postEvict).
+ * @param {Function} done the callback to call when all hooks are completed.
+ * @private
+ */
+SelfEvict.prototype._runHooks = function _runHooks(type, done) {
+    var self = this;
+    var ringpop = self.ringpop;
+
+    var names = _.filter(_.keys(this.hooks), function filter(name) {
+        return _.has(self.hooks[name], type);
+    });
+
+    if (names.length === 0) {
+        done();
+        return;
+    }
+
+    var logger = this.ringpop.logger;
+
+    var start = Date.now();
+
+    var afterHook = _.after(names.length, function afterHooks() {
+        logger.debug('ringpop self eviction done running hooks', {
+            local: ringpop.whoami(),
+            totalDuration: Date.now() - start,
+            phase: self.currentPhase()
+        });
+        process.nextTick(done);
+    });
+
+    logger.debug('ringpop self eviction running hooks', {
+        local: ringpop.whoami(),
+        phase: self.currentPhase(),
+        hooks: names
+    });
+
+    for (var i = 0; i < names.length; i++) {
+        var name = names[i];
+        var hook = this.hooks[name];
+
+        runHook(hook);
+    }
+
+    function runHook(hook) {
+        process.nextTick(function() {
+            var name = hook.name;
+            logger.debug('ringpop self eviction running hook', {
+                    local: ringpop.whoami(),
+                    hook: name
+                }
+            );
+
+            var hookStart = Date.now();
+            hook[type](function hookDone() {
+                logger.debug('ringpop self eviction hook done', {
+                        local: ringpop.whoami(),
+                        hook: name,
+                        duration: Date.now() - hookStart
+                    }
+                );
+                afterHook();
+            });
+        })
+    }
+};
+
+
+/**
+ * Enum for SelfEvict phases
+ * @readonly
+ * @enum {string}
+ */
+SelfEvict.PhaseNames = {
+    PreEvict: 'pre_evict',
+    Evicting: 'evicting',
+    PostEvict: 'post_evict',
+    Done: 'done'
+};
+var PhaseNames = SelfEvict.PhaseNames;
+
+
+/**
+ * @returns {Error}
+ */
+var RingpopIsAlreadyShuttingDownError = TypedError({
+    type: 'ringpop.self-evict.already-evicting',
+    message: 'ringpop is already evicting itself. currentPhase={phase}',
+    phase: null
+});
+
+module.exports = SelfEvict;

test/unit/index-test.js

@@ -481,3 +481,39 @@ test('suspicionTimeout backward compatibility', function t(assert) {
     ringpop.destroy();
     assert.end();
 });
+
+test('registerSelfEvictHook registers hook in self evicter', function t(assert) {
+    var ringpop = createRingpop();
+
+    var hookFixture = {};
+
+    ringpop.selfEvicter= {
+        registerHooks: function(hook) {
+            assert.pass('registerHooks called');
+            assert.equal(hook, hookFixture);
+        }
+    };
+    assert.plan(2);
+    ringpop.registerSelfEvictHook(hookFixture);
+
+    ringpop.destroy();
+    assert.end();
+});
+
+test('selfEvict initiates self evict sequence', function t(assert) {
+    var ringpop = createRingpop();
+
+    var cbFixture = function(){};
+
+    ringpop.selfEvicter= {
+        initiate: function(cb) {
+            assert.pass('initiate');
+            assert.equal(cb, cbFixture);
+        }
+    };
+    assert.plan(2);
+    ringpop.selfEvict(cbFixture);
+
+    ringpop.destroy();
+    assert.end();
+});