From 0e35bc61d33b39b2181b0df619274889e6a9b87d Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Wed, 10 Aug 2016 14:19:42 +0300 Subject: [PATCH 1/8] Check rabbit_node_monitor during health-check Tests + comment outlining the problem. The check itself is in separate commit to `rabbitmq-common`. --- src/rabbit_autoheal.erl | 11 +++++++++++ test/health_check_SUITE.erl | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/rabbit_autoheal.erl b/src/rabbit_autoheal.erl index 5865ba822715..db4d41221ea2 100644 --- a/src/rabbit_autoheal.erl +++ b/src/rabbit_autoheal.erl @@ -297,6 +297,17 @@ winner_finish(Notify) -> send(leader(), {autoheal_finished, node()}), not_healing. +%% XXX This can enter infinite loop, if mnesia was somehow restarted +%% outside of our control - i.e. somebody started app back by hand or +%% completely restarted node. One possible solution would be something +%% like this (but it needs some more pondering and is left for some +%% other patch): +%% - monitor top-level mnesia supervisors of all losers +%% - notify loosers about the fact that they are indeed loosers +%% - wait for all monitors to go 'DOWN' (+ maybe some timeout on the whole process) +%% - do one round of parallel rpc calls to check whether mnesia is still stoppend on all +%% loosers +%% - If everything is still stopped, continue autoheall process. Or cancel it otherwise. wait_for_mnesia_shutdown([Node | Rest] = AllNodes) -> case rpc:call(Node, mnesia, system_info, [is_running]) of no -> diff --git a/test/health_check_SUITE.erl b/test/health_check_SUITE.erl index 4d8f56e9d305..50abc97a0291 100644 --- a/test/health_check_SUITE.erl +++ b/test/health_check_SUITE.erl @@ -33,6 +33,8 @@ ,ignores_remote_alarms/1 ,detects_local_alarm/1 ,honors_timeout_argument/1 + ,detects_stuck_local_node_monitor/1 + ,ignores_stuck_remote_node_monitor/1 ]). all() -> @@ -47,6 +49,8 @@ groups() -> ,ignores_remote_alarms ,detects_local_alarm ,honors_timeout_argument + ,detects_stuck_local_node_monitor + ,ignores_stuck_remote_node_monitor ]}]. init_per_suite(Config) -> @@ -123,6 +127,21 @@ detects_local_alarm(Config) -> {match, _} = re:run(Str, "resource alarm.*in effect"), ok. +detects_stuck_local_node_monitor(Config) -> + [A|_] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:rpc(Config, A, sys, suspend, [rabbit_node_monitor]), + {error, 75, Str} = rabbit_ct_broker_helpers:rabbitmqctl(Config, A, ["-t", "5", "node_health_check"]), + {match, _} = re:run(Str, "operation node_health_check.*timed out"), + resume_sys_process(Config, A, rabbit_node_monitor), + ok. + +ignores_stuck_remote_node_monitor(Config) -> + [A, B] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename), + rabbit_ct_broker_helpers:rpc(Config, A, sys, suspend, [rabbit_node_monitor]), + {ok, _} = rabbit_ct_broker_helpers:rabbitmqctl(Config, B, ["-t", "5", "node_health_check"]), + resume_sys_process(Config, A, rabbit_node_monitor), + ok. + honors_timeout_argument(Config) -> [A|_] = open_channel_and_declare_queue_everywhere(Config), QPid = suspend_single_queue(Config, A), From 2482c9fe4b540d9ef9a66cd6ca3534015eeb88bc Mon Sep 17 00:00:00 2001 From: Alexey Lebedeff Date: Mon, 8 Aug 2016 19:31:27 +0300 Subject: [PATCH 2/8] Add support for listing only local queues Partially implements https://github.com/rabbitmq/rabbitmq-server/issues/851 - Made old `--online`/`--offline` options mutually exclusive between themselves and the new `--local` option - Added documentation both for the old and the new option - Fixed some ugly indentation in generated usage (only `set_policy` wrapped line remains unfixed) - Added integration test suite for `rabbitmqctl list_queues` --- docs/rabbitmqctl.1.xml | 42 ++++++- docs/usage.xsl | 10 +- include/rabbit_cli.hrl | 2 + src/rabbit_cli.erl | 35 +++--- src/rabbit_control_main.erl | 21 ++-- test/rabbitmqctl_integration_SUITE.erl | 146 +++++++++++++++++++++++++ test/unit_SUITE.erl | 45 +++++--- 7 files changed, 257 insertions(+), 44 deletions(-) create mode 100644 test/rabbitmqctl_integration_SUITE.erl diff --git a/docs/rabbitmqctl.1.xml b/docs/rabbitmqctl.1.xml index ec864af6cf9f..7dd926eba886 100644 --- a/docs/rabbitmqctl.1.xml +++ b/docs/rabbitmqctl.1.xml @@ -749,9 +749,7 @@ - - authenticate_user username password - + authenticate_user username password @@ -1237,13 +1235,49 @@ - list_queues -p vhost queueinfoitem ... + + list_queues -p vhost --offline--online--local queueinfoitem ... + Returns queue details. Queue details of the / virtual host are returned if the "-p" flag is absent. The "-p" flag can be used to override this default. + + Displayed queues can be filtered by their status or + location using one of the following mutually exclusive + options: + + + --offline + + + List only those durable queues that are not + currently running - i.e. they are located on + inaccessible nodes. + + + + + --online + + + List queues that are currently live. + + + + + --local + + + List only those queues whose master process is + located on the current node. + + + + + The queueinfoitem parameter is used to indicate which queue information items to include in the results. The column order in the diff --git a/docs/usage.xsl b/docs/usage.xsl index 586f830327e2..0d3cb6f235c4 100644 --- a/docs/usage.xsl +++ b/docs/usage.xsl @@ -8,7 +8,7 @@ encoding="UTF-8" indent="no"/> - + %% Generated, do not edit! @@ -68,6 +68,14 @@ usage() -> %QUOTE%Usage: + + [ + + + | + + ] + [] <> diff --git a/include/rabbit_cli.hrl b/include/rabbit_cli.hrl index a0d1ecfdd519..b1cf41261f10 100644 --- a/include/rabbit_cli.hrl +++ b/include/rabbit_cli.hrl @@ -29,6 +29,7 @@ -define(RAM_OPT, "--ram"). -define(OFFLINE_OPT, "--offline"). -define(ONLINE_OPT, "--online"). +-define(LOCAL_OPT, "--local"). -define(NODE_DEF(Node), {?NODE_OPT, {option, Node}}). @@ -46,6 +47,7 @@ -define(RAM_DEF, {?RAM_OPT, flag}). -define(OFFLINE_DEF, {?OFFLINE_OPT, flag}). -define(ONLINE_DEF, {?ONLINE_OPT, flag}). +-define(LOCAL_DEF, {?LOCAL_OPT, flag}). %% Subset of standartized exit codes from sysexits.h, see %% https://github.com/rabbitmq/rabbitmq-server/issues/396 for discussion. diff --git a/src/rabbit_cli.erl b/src/rabbit_cli.erl index 6b3548221747..ca00226ba6ec 100644 --- a/src/rabbit_cli.erl +++ b/src/rabbit_cli.erl @@ -18,7 +18,7 @@ -include("rabbit_cli.hrl"). -export([main/3, start_distribution/0, start_distribution/1, - parse_arguments/4, filter_opts/2, + parse_arguments/4, mutually_exclusive_flags/3, rpc_call/4, rpc_call/5, rpc_call/7]). %%---------------------------------------------------------------------------- @@ -42,8 +42,7 @@ [{string(), optdef()}], string(), [string()]) -> parse_result(). --spec filter_opts([{option_name(), option_value()}], [option_name()]) -> - [boolean()]. +-spec mutually_exclusive_flags([{option_name(), option_value()}], term(), [{option_name(), term()}]) -> {ok, term()} | {error, string()}. -spec rpc_call(node(), atom(), atom(), [any()]) -> any(). -spec rpc_call(node(), atom(), atom(), [any()], number()) -> any(). @@ -250,20 +249,22 @@ process_opts(Defs, C, [A | As], Found, KVs, Outs) -> {none, _, _} -> no_command end. -%% When we have a set of flags that are used for filtering, we want by -%% default to include every such option in our output. But if a user -%% explicitly specified any such flag, we want to include only items -%% which he has requested. -filter_opts(CurrentOptionValues, AllOptionNames) -> - Explicit = lists:map(fun(OptName) -> - proplists:get_bool(OptName, CurrentOptionValues) - end, - AllOptionNames), - case lists:member(true, Explicit) of - true -> - Explicit; - false -> - lists:duplicate(length(AllOptionNames), true) +mutually_exclusive_flags(CurrentOptionValues, Default, FlagsAndValues) -> + PresentFlags = lists:filtermap(fun({OptName, _} = _O) -> + proplists:get_bool(OptName, CurrentOptionValues) + end, + FlagsAndValues), + case PresentFlags of + [] -> + {ok, Default}; + [{_, Value}] -> + {ok, Value}; + _ -> + Names = [ [$', N, $'] || {N, _} <- PresentFlags ], + CommaSeparated = string:join(lists:droplast(Names), ", "), + AndOneMore = lists:last(Names), + Msg = io_lib:format("Options ~s and ~s are mutually exclusive", [CommaSeparated, AndOneMore]), + {error, lists:flatten(Msg)} end. %%---------------------------------------------------------------------------- diff --git a/src/rabbit_control_main.erl b/src/rabbit_control_main.erl index ea9d6a2030dc..92898c2a2cd7 100644 --- a/src/rabbit_control_main.erl +++ b/src/rabbit_control_main.erl @@ -74,7 +74,7 @@ {clear_policy, [?VHOST_DEF]}, {list_policies, [?VHOST_DEF]}, - {list_queues, [?VHOST_DEF, ?OFFLINE_DEF, ?ONLINE_DEF]}, + {list_queues, [?VHOST_DEF, ?OFFLINE_DEF, ?ONLINE_DEF, ?LOCAL_DEF]}, {list_exchanges, [?VHOST_DEF]}, {list_bindings, [?VHOST_DEF]}, {list_connections, [?VHOST_DEF]}, @@ -632,12 +632,19 @@ action(list_user_permissions, Node, Args = [_Username], _Opts, Inform, Timeout) true); action(list_queues, Node, Args, Opts, Inform, Timeout) -> - [Online, Offline] = rabbit_cli:filter_opts(Opts, [?ONLINE_OPT, ?OFFLINE_OPT]), - Inform("Listing queues", []), - VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), - ArgAtoms = default_if_empty(Args, [name, messages]), - call(Node, {rabbit_amqqueue, info_all, [VHostArg, ArgAtoms, Online, Offline]}, - ArgAtoms, Timeout); + case rabbit_cli:mutually_exclusive_flags( + Opts, all, [{?ONLINE_OPT, online} + ,{?OFFLINE_OPT, offline} + ,{?LOCAL_OPT, local}]) of + {ok, Filter} -> + Inform("Listing queues", []), + VHostArg = list_to_binary(proplists:get_value(?VHOST_OPT, Opts)), + ArgAtoms = default_if_empty(Args, [name, messages]), + call(Node, {rabbit_amqqueue, info_all, [VHostArg, ArgAtoms, Filter]}, + ArgAtoms, Timeout); + {error, ErrStr} -> + {error_string, ErrStr} + end; action(list_exchanges, Node, Args, Opts, Inform, Timeout) -> Inform("Listing exchanges", []), diff --git a/test/rabbitmqctl_integration_SUITE.erl b/test/rabbitmqctl_integration_SUITE.erl new file mode 100644 index 000000000000..9305781bda74 --- /dev/null +++ b/test/rabbitmqctl_integration_SUITE.erl @@ -0,0 +1,146 @@ +%% The contents of this file are subject to the Mozilla Public License +%% Version 1.1 (the "License"); you may not use this file except in +%% compliance with the License. You may obtain a copy of the License +%% at http://www.mozilla.org/MPL/ +%% +%% Software distributed under the License is distributed on an "AS IS" +%% basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See +%% the License for the specific language governing rights and +%% limitations under the License. +%% +%% The Original Code is RabbitMQ. +%% +%% The Initial Developer of the Original Code is GoPivotal, Inc. +%% Copyright (c) 2016 Pivotal Software, Inc. All rights reserved. +%% +-module(rabbitmqctl_integration_SUITE). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("amqp_client/include/amqp_client.hrl"). + +-export([all/0 + ,groups/0 + ,init_per_suite/1 + ,end_per_suite/1 + ,init_per_group/2 + ,end_per_group/2 + ,init_per_testcase/2 + ,end_per_testcase/2 + ]). + +-export([list_queues_local/1 + ,list_queues_offline/1 + ,list_queues_online/1 + ]). + +all() -> + [{group, list_queues}]. + +groups() -> + [{list_queues, [], + [list_queues_local + ,list_queues_online + ,list_queues_offline + ]}]. + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(list_queues, Config0) -> + NumNodes = 3, + Config = create_n_node_cluster(Config0, NumNodes), + Config1 = declare_some_queues(Config), + rabbit_ct_broker_helpers:stop_node(Config1, NumNodes - 1), + Config1; +init_per_group(_, Config) -> + Config. + +create_n_node_cluster(Config0, NumNodes) -> + Config1 = rabbit_ct_helpers:set_config( + Config0, [{rmq_nodes_count, NumNodes}, + {rmq_nodes_clustered, true}]), + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_broker_helpers:setup_steps() ++ + rabbit_ct_client_helpers:setup_steps()). + +declare_some_queues(Config) -> + Nodes = rabbit_ct_helpers:get_config(Config, rmq_nodes), + PerNodeQueues = [ declare_some_queues(Config, NodeNum) + || NodeNum <- lists:seq(0, length(Nodes)-1) ], + rabbit_ct_helpers:set_config(Config, {per_node_queues, PerNodeQueues}). + +declare_some_queues(Config, NodeNum) -> + {Conn, Chan} = rabbit_ct_client_helpers:open_connection_and_channel(Config, NodeNum), + NumQueues = 5, + Queues = [ list_to_binary(io_lib:format("queue-~b-on-node-~b", [QueueNum, NodeNum])) + || QueueNum <- lists:seq(1, NumQueues) ], + lists:foreach(fun (QueueName) -> + #'queue.declare_ok'{} = amqp_channel:call(Chan, #'queue.declare'{queue = QueueName, durable = true}) + end, Queues), + rabbit_ct_client_helpers:close_connection_and_channel(Conn, Chan), + Queues. + +end_per_group(list_queues, Config0) -> + Config1 = case rabbit_ct_helpers:get_config(Config0, save_config) of + undefined -> Config0; + C -> C + end, + rabbit_ct_helpers:run_steps(Config1, + rabbit_ct_client_helpers:teardown_steps() ++ + rabbit_ct_broker_helpers:teardown_steps()); +end_per_group(_, Config) -> + Config. + +init_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:testcase_started(Config0, Testcase). + +end_per_testcase(Testcase, Config0) -> + rabbit_ct_helpers:testcase_finished(Config0, Testcase). + +%%---------------------------------------------------------------------------- +%% Test cases +%%---------------------------------------------------------------------------- +list_queues_local(Config) -> + Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))), + Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))), + assert_ctl_queues(Config, 0, ["--local"], Node1Queues), + assert_ctl_queues(Config, 1, ["--local"], Node2Queues), + ok. + +list_queues_online(Config) -> + Node1Queues = lists:sort(lists:nth(1, ?config(per_node_queues, Config))), + Node2Queues = lists:sort(lists:nth(2, ?config(per_node_queues, Config))), + OnlineQueues = Node1Queues ++ Node2Queues, + assert_ctl_queues(Config, 0, ["--online"], OnlineQueues), + assert_ctl_queues(Config, 1, ["--online"], OnlineQueues), + ok. + +list_queues_offline(Config) -> + Node3Queues = lists:sort(lists:nth(3, ?config(per_node_queues, Config))), + OfflineQueues = Node3Queues, + assert_ctl_queues(Config, 0, ["--offline"], OfflineQueues), + assert_ctl_queues(Config, 1, ["--offline"], OfflineQueues), + ok. + +%%---------------------------------------------------------------------------- +%% Helpers +%%---------------------------------------------------------------------------- +assert_ctl_queues(Config, Node, Args, Expected0) -> + Expected = lists:sort(Expected0), + Got0 = run_list_queues(Config, Node, Args), + Got = lists:sort(lists:map(fun hd/1, Got0)), + case Got of + Expected -> + ok; + _ -> + ct:pal(error, "Listing queues on node ~p failed. Expected:~n~p~n~nGot:~n~p~n~n", + [Node, Expected, Got]), + exit({list_queues_unexpected_on, Node, Expected, Got}) + end. + +run_list_queues(Config, Node, Args) -> + rabbit_ct_broker_helpers:rabbitmqctl_list(Config, Node, ["list_queues"] ++ Args ++ ["name"]). diff --git a/test/unit_SUITE.erl b/test/unit_SUITE.erl index ba0f43f11e34..43e812fa3dd0 100644 --- a/test/unit_SUITE.erl +++ b/test/unit_SUITE.erl @@ -31,7 +31,7 @@ groups() -> [ {parallel_tests, [parallel], [ arguments_parser, - filtering_flags_parsing, + mutually_exclusive_flags_parsing, {basic_header_handling, [parallel], [ write_table_with_invalid_existing_type, invalid_existing_headers, @@ -135,26 +135,41 @@ check_parse_arguments(ExpRes, Fun, As) -> true = SortRes(ExpRes) =:= SortRes(Fun(As)). -filtering_flags_parsing(_Config) -> - Cases = [{[], [], []} - ,{[{"--online", true}], ["--offline", "--online", "--third-option"], [false, true, false]} - ,{[{"--online", true}, {"--third-option", true}, {"--offline", true}], ["--offline", "--online", "--third-option"], [true, true, true]} - ,{[], ["--offline", "--online", "--third-option"], [true, true, true]} - ], - lists:foreach(fun({Vals, Opts, Expect}) -> - case rabbit_cli:filter_opts(Vals, Opts) of - Expect -> +mutually_exclusive_flags_parsing(_Config) -> + Matcher = fun ({ok, Value}, {ok, Value}) -> true; + ({error, Value}, {error, Pattern}) -> + case re:run(Value, Pattern) of + {match, _} -> true; + _ -> false + end; + (_, _) -> false + end, + Spec = [{"--online", online} + ,{"--offline", offline} + ,{"--local", local}], + Default = all, + Cases =[{["--online"], {ok, online}} + ,{[], {ok, Default}} + ,{["--offline"], {ok, offline}} + ,{["--local"], {ok, local}} + ,{["--offline", "--local"], {error, "mutually exclusive"}} + ,{["--offline", "--online"], {error, "mutually exclusive"}} + ,{["--offline", "--local", "--online"], {error, "mutually exclusive"}} + ], + lists:foreach(fun({Opts, Expected}) -> + ExpandedOpts = [ {Opt, true} || Opt <- Opts ], + Got = rabbit_cli:mutually_exclusive_flags(ExpandedOpts, all, Spec), + case Matcher(Got, Expected) of + true -> ok; - Got -> - exit({no_match, Got, Expect, {args, Vals, Opts}}) + false -> + exit({no_match, Got, Expected, {opts, Opts}}) end - end, - Cases). + end, Cases). %% ------------------------------------------------------------------- %% basic_header_handling. %% ------------------------------------------------------------------- - -define(XDEATH_TABLE, [{<<"reason">>, longstr, <<"blah">>}, {<<"queue">>, longstr, <<"foo.bar.baz">>}, From 9c0823739580e3e9fcd382feda429408ba3d55aa Mon Sep 17 00:00:00 2001 From: jerryk Date: Fri, 12 Aug 2016 16:55:52 -0700 Subject: [PATCH 3/8] Fix trivial typo noticed in error message. --- packaging/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/Makefile b/packaging/Makefile index 31b85d9c1f09..02820bdb23e6 100644 --- a/packaging/Makefile +++ b/packaging/Makefile @@ -46,7 +46,7 @@ ifeq ($(SOURCE_DIST_FILE),) $(error Cannot find source archive; please specify SOURCE_DIST_FILE) endif ifneq ($(words $(SOURCE_DIST_FILE)),1) -$(error Multile source archives found; please specify SOURCE_DIST_FILE) +$(error Multiple source archives found; please specify SOURCE_DIST_FILE) endif ifeq ($(filter %.tar.xz %.txz,$(SOURCE_DIST_FILE)),) $(error The source archive must a tar.xz archive) From 15cb69740faf7c3df7ddbf0a6ce593f885f12e10 Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Mon, 15 Aug 2016 17:01:13 +0300 Subject: [PATCH 4/8] Docs wording --- docs/rabbitmqctl.1.xml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/rabbitmqctl.1.xml b/docs/rabbitmqctl.1.xml index 7dd926eba886..d9a9991ea31c 100644 --- a/docs/rabbitmqctl.1.xml +++ b/docs/rabbitmqctl.1.xml @@ -1254,8 +1254,7 @@ List only those durable queues that are not - currently running - i.e. they are located on - inaccessible nodes. + currently available (more specifically, their master node isn't). @@ -1263,7 +1262,7 @@ --online - List queues that are currently live. + List queues that are currently available (their master node is). From b28620837391de1b6208ddb096d1dcdb98e6ab8b Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Mon, 15 Aug 2016 10:02:32 -0600 Subject: [PATCH 5/8] Update iptables calls with --wait If iptables is currently being called outside of the ocf script, the iptables call will fail because it cannot get a lock. This change updates the iptables call to include the -w flag which will wait until the lock can be established and not just exit with an error. --- scripts/rabbitmq-server-ha.ocf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rabbitmq-server-ha.ocf b/scripts/rabbitmq-server-ha.ocf index cd07d0c1b0d3..9e07dc8490e0 100755 --- a/scripts/rabbitmq-server-ha.ocf +++ b/scripts/rabbitmq-server-ha.ocf @@ -691,9 +691,9 @@ block_client_access() # do not add temporary RMQ blocking rule, if it is already exist # otherwise, try to add a blocking rule with max of 5 retries local tries=5 - until $(iptables -nvL | grep -q 'temporary RMQ block') || [ $tries -eq 0 ]; do + until $(iptables -nvL --wait | grep -q 'temporary RMQ block') || [ $tries -eq 0 ]; do tries=$((tries-1)) - iptables -I INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ + iptables --wait -I INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ -m comment --comment 'temporary RMQ block' -j REJECT --reject-with tcp-reset sleep 1 done @@ -707,8 +707,8 @@ block_client_access() unblock_client_access() { # remove all temporary RMQ blocking rules, if there are more than one exist - for i in $(iptables -nvL --line-numbers | awk '/temporary RMQ block/ {print $1}'); do - iptables -D INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ + for i in $(iptables -nvL --wait --line-numbers | awk '/temporary RMQ block/ {print $1}'); do + iptables --wait -D INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ -m comment --comment 'temporary RMQ block' -j REJECT --reject-with tcp-reset done } From 6e1d718b0312b472fb658f4e9b0dac69cb2794be Mon Sep 17 00:00:00 2001 From: Alex Schultz Date: Mon, 15 Aug 2016 10:02:32 -0600 Subject: [PATCH 6/8] Update iptables calls with --wait If iptables is currently being called outside of the ocf script, the iptables call will fail because it cannot get a lock. This change updates the iptables call to include the -w flag which will wait until the lock can be established and not just exit with an error. --- scripts/rabbitmq-server-ha.ocf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/rabbitmq-server-ha.ocf b/scripts/rabbitmq-server-ha.ocf index cd07d0c1b0d3..9e07dc8490e0 100755 --- a/scripts/rabbitmq-server-ha.ocf +++ b/scripts/rabbitmq-server-ha.ocf @@ -691,9 +691,9 @@ block_client_access() # do not add temporary RMQ blocking rule, if it is already exist # otherwise, try to add a blocking rule with max of 5 retries local tries=5 - until $(iptables -nvL | grep -q 'temporary RMQ block') || [ $tries -eq 0 ]; do + until $(iptables -nvL --wait | grep -q 'temporary RMQ block') || [ $tries -eq 0 ]; do tries=$((tries-1)) - iptables -I INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ + iptables --wait -I INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ -m comment --comment 'temporary RMQ block' -j REJECT --reject-with tcp-reset sleep 1 done @@ -707,8 +707,8 @@ block_client_access() unblock_client_access() { # remove all temporary RMQ blocking rules, if there are more than one exist - for i in $(iptables -nvL --line-numbers | awk '/temporary RMQ block/ {print $1}'); do - iptables -D INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ + for i in $(iptables -nvL --wait --line-numbers | awk '/temporary RMQ block/ {print $1}'); do + iptables --wait -D INPUT -p tcp -m tcp --dport ${OCF_RESKEY_node_port} -m state --state NEW,RELATED,ESTABLISHED \ -m comment --comment 'temporary RMQ block' -j REJECT --reject-with tcp-reset done } From e86d65f1f5eb85087ce529921f1ff848cb010230 Mon Sep 17 00:00:00 2001 From: Diana Corbacho Date: Tue, 16 Aug 2016 16:31:12 +0100 Subject: [PATCH 7/8] Discard any unexpected messages, such as late replies from gen_server --- src/gm.erl | 5 ++++- test/gm_SUITE.erl | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/gm.erl b/src/gm.erl index 176e14537f2d..4add10063ae3 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -757,7 +757,10 @@ handle_info({'DOWN', MRef, process, _Pid, Reason}, catch lost_membership -> {stop, normal, State} - end. + end; +handle_info(_, State) -> + %% Discard any unexpected message, such as late replies from neighbour_call/2 + noreply(State). terminate(Reason, #state { module = Module, callback_args = Args }) -> Module:handle_terminate(Args, Reason). diff --git a/test/gm_SUITE.erl b/test/gm_SUITE.erl index f5ccf75b7032..e4c68a257a91 100644 --- a/test/gm_SUITE.erl +++ b/test/gm_SUITE.erl @@ -38,7 +38,8 @@ all() -> broadcast, confirmed_broadcast, member_death, - receive_in_order + receive_in_order, + unexpected_msg ]. init_per_suite(Config) -> @@ -114,6 +115,14 @@ receive_in_order(_Config) -> passed end). +unexpected_msg(_Config) -> + passed = with_two_members( + fun(Pid, _) -> + Pid ! {make_ref(), old_gen_server_answer}, + true = erlang:is_process_alive(Pid), + passed + end). + do_broadcast(Fun) -> with_two_members(broadcast_fun(Fun)). From 878b6a00ad9ade9adb0c400df6541e63f74b9ede Mon Sep 17 00:00:00 2001 From: Michael Klishin Date: Tue, 16 Aug 2016 19:10:37 +0300 Subject: [PATCH 8/8] Typo --- src/gm.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gm.erl b/src/gm.erl index 4add10063ae3..492117d17fe7 100644 --- a/src/gm.erl +++ b/src/gm.erl @@ -759,7 +759,7 @@ handle_info({'DOWN', MRef, process, _Pid, Reason}, {stop, normal, State} end; handle_info(_, State) -> - %% Discard any unexpected message, such as late replies from neighbour_call/2 + %% Discard any unexpected messages, such as late replies from neighbour_call/2 noreply(State). terminate(Reason, #state { module = Module, callback_args = Args }) ->