Skip to content

Commit

Permalink
Handle unexpected gm group alterations prior to removal of
Browse files Browse the repository at this point in the history
dead pids from queue
  • Loading branch information
Ayanda-D committed Aug 15, 2016
1 parent 53f10c9 commit a416060
Show file tree
Hide file tree
Showing 3 changed files with 31 additions and 6 deletions.
9 changes: 9 additions & 0 deletions src/rabbit_mirror_queue_coordinator.erl
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,15 @@ handle_cast({gm_deaths, DeadGMPids},
DeadPids),
rabbit_mirror_queue_misc:add_mirrors(QueueName, ExtraNodes, async),
noreply(State);
{ok, _MPid0, DeadPids, _ExtraNodes} ->
%% see rabbitmq-server#914;
%% Different slave is now master, stop current coordinator normally.
%% Initiating queue is now slave and the least we could do is report
%% deaths which we 'think' we saw.
%% NOTE: Reported deaths here, could be inconsistant.
rabbit_mirror_queue_misc:report_deaths(MPid, false, QueueName,
DeadPids),
{stop, normal, State};
{error, not_found} ->
{stop, normal, State}
end;
Expand Down
16 changes: 13 additions & 3 deletions src/rabbit_mirror_queue_misc.erl
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ remove_from_queue(QueueName, Self, DeadGMPids) ->
rabbit_misc:execute_mnesia_transaction(
fun () ->
%% Someone else could have deleted the queue before we
%% get here.
%% get here. Or, gm group could've altered. see rabbitmq-server#914
case mnesia:read({rabbit_queue, QueueName}) of
[] -> {error, not_found};
[Q = #amqqueue { pid = QPid,
Expand All @@ -90,15 +90,25 @@ remove_from_queue(QueueName, Self, DeadGMPids) ->
AlivePids = [Pid || {_GM, Pid} <- AliveGM],
Alive = [Pid || Pid <- [QPid | SPids],
lists:member(Pid, AlivePids)],
{QPid1, SPids1} = promote_slave(Alive),
{QPid1, SPids1} = case Alive of
[] ->
%% GM altered, & if all pids are
%% perceived as dead, rather do
%% do nothing here, & trust the
%% promoted slave to have updated
%% mnesia during the alteration.
{QPid, SPids};
_ -> promote_slave(Alive)
end,
Extra =
case {{QPid, SPids}, {QPid1, SPids1}} of
{Same, Same} ->
[];
_ when QPid =:= QPid1 orelse QPid1 =:= Self ->
%% Either master hasn't changed, so
%% we're ok to update mnesia; or we have
%% become the master.
%% become the master. If gm altered,
%% we have no choice but to proceed.
Q1 = Q#amqqueue{pid = QPid1,
slave_pids = SPids1,
gm_pids = AliveGM},
Expand Down
12 changes: 9 additions & 3 deletions src/rabbit_mirror_queue_slave.erl
Original file line number Diff line number Diff line change
Expand Up @@ -225,9 +225,15 @@ handle_call({gm_deaths, DeadGMPids}, From,
_ ->
%% master has changed to not us
gen_server2:reply(From, ok),
%% assertion, we don't need to add_mirrors/2 in this
%% branch, see last clause in remove_from_queue/2
[] = ExtraNodes,
%% see rabbitmq-server#914;
%% It's not always guaranteed that we won't have ExtraNodes.
%% If gm alters, master can change to not us with extra nodes,
%% in which case we attempt to add mirrors on those nodes.
case ExtraNodes of
[] -> void;
_ -> rabbit_mirror_queue_misc:add_mirrors(
QName, ExtraNodes, async)
end,
%% Since GM is by nature lazy we need to make sure
%% there is some traffic when a master dies, to
%% make sure all slaves get informed of the
Expand Down

0 comments on commit a416060

Please sign in to comment.