diff --git a/deps/rabbit/BUILD.bazel b/deps/rabbit/BUILD.bazel index 3bf4158a6eb3..66e1e984c02d 100644 --- a/deps/rabbit/BUILD.bazel +++ b/deps/rabbit/BUILD.bazel @@ -579,6 +579,11 @@ rabbitmq_integration_suite( size = "medium", ) +rabbitmq_integration_suite( + name = "peer_discovery_tmp_hidden_node_SUITE", + size = "large", +) + rabbitmq_integration_suite( name = "per_user_connection_channel_limit_partitions_SUITE", size = "large", diff --git a/deps/rabbit/app.bzl b/deps/rabbit/app.bzl index 5f53f508cb54..22361ca786d9 100644 --- a/deps/rabbit/app.bzl +++ b/deps/rabbit/app.bzl @@ -1132,6 +1132,15 @@ def test_suite_beam_files(name = "test_suite_beam_files"): erlc_opts = "//:test_erlc_opts", deps = ["//deps/amqp_client:erlang_app"], ) + erlang_bytecode( + name = "peer_discovery_tmp_hidden_node_SUITE_beam_files", + testonly = True, + srcs = ["test/peer_discovery_tmp_hidden_node_SUITE.erl"], + outs = ["test/peer_discovery_tmp_hidden_node_SUITE.beam"], + app_name = "rabbit", + erlc_opts = "//:test_erlc_opts", + deps = ["//deps/amqp_client:erlang_app", "//deps/rabbitmq_ct_helpers:erlang_app"], + ) erlang_bytecode( name = "per_user_connection_channel_limit_SUITE_beam_files", testonly = True, diff --git a/deps/rabbit/src/rabbit_peer_discovery.erl b/deps/rabbit/src/rabbit_peer_discovery.erl index a53ffe6083b3..7d49d087dc9c 100644 --- a/deps/rabbit/src/rabbit_peer_discovery.erl +++ b/deps/rabbit/src/rabbit_peer_discovery.erl @@ -30,7 +30,8 @@ group_leader_proxy/2]). -ifdef(TEST). --export([sort_nodes_and_props/1, +-export([query_node_props/1, + sort_nodes_and_props/1, join_selected_node/3]). -endif. @@ -388,16 +389,19 @@ query_node_props(Nodes) when Nodes =/= [] -> _ -> VMArgs1 end, - VMArgs3 = maybe_add_tls_arguments(VMArgs2), + VMArgs3 = maybe_add_proto_dist_arguments(VMArgs2), + VMArgs4 = maybe_add_inetrc_arguments(VMArgs3), + VMArgs5 = maybe_add_tls_arguments(VMArgs4), + PeerStartArg0 = #{name => PeerName, + args => VMArgs5, + connection => standard_io, + wait_boot => infinity}, PeerStartArg = case Context of #{nodename_type := longnames} -> - #{name => PeerName, - host => Suffix, - longnames => true, - args => VMArgs3}; + PeerStartArg0#{host => Suffix, + longnames => true}; _ -> - #{name => PeerName, - args => VMArgs3} + PeerStartArg0 end, ?LOG_DEBUG("Peer discovery: peer node arguments: ~tp", [PeerStartArg]), @@ -409,7 +413,7 @@ query_node_props(Nodes) when Nodes =/= [] -> [Peer], #{domain => ?RMQLOG_DOMAIN_PEER_DISC}), try - erpc:call(Peer, ?MODULE, do_query_node_props, [Nodes]) + peer:call(Pid, ?MODULE, do_query_node_props, [Nodes], 180000) after peer:stop(Pid) end; @@ -423,27 +427,40 @@ query_node_props(Nodes) when Nodes =/= [] -> query_node_props([]) -> []. -maybe_add_tls_arguments(VMArgs0) -> +maybe_add_proto_dist_arguments(VMArgs) -> case init:get_argument(proto_dist) of - {ok, [["inet_tls"]]} -> - add_tls_arguments(inet_tls, VMArgs0); - {ok, [["inet6_tls"]]} -> - add_tls_arguments(inet6_tls, VMArgs0); + {ok, [[Val]]} -> + %% See net_kernel.erl / protocol_childspecs/1. + Mod = list_to_existing_atom(Val ++ "_dist"), + ModDir = filename:dirname(code:which(Mod)), + ["-proto_dist", Val, "-pa", ModDir | VMArgs]; _ -> - VMArgs0 + VMArgs end. -add_tls_arguments(InetDistModule, VMArgs0) -> - VMArgs1 = case InetDistModule of - inet_tls -> - ProtoDistArg = ["-proto_dist", "inet_tls" | VMArgs0], - ["-pa", filename:dirname(code:which(inet_tls_dist)) - | ProtoDistArg]; - inet6_tls -> - ProtoDistArg = ["-proto_dist", "inet6_tls" | VMArgs0], - ["-pa", filename:dirname(code:which(inet6_tls_dist)) - | ProtoDistArg] - end, +maybe_add_inetrc_arguments(VMArgs) -> + %% If an inetrc file is configured, we need to use it for the temporary + %% hidden node too. + case application:get_env(kernel, inetrc) of + {ok, Val} -> + maybe_add_inetrc_arguments1(VMArgs, Val); + undefined -> + case os:getenv("ERL_INETRC") of + Val when is_list(Val) -> + maybe_add_inetrc_arguments1(VMArgs, Val); + false -> + VMArgs + end + end. + +maybe_add_inetrc_arguments1(VMArgs, Val) -> + %% The filename argument must be passed as a quoted string so that the + %% command line is correctly parsed as an Erlang string by the temporary + %% hidden node. + ValString = rabbit_misc:format("~0p", [Val]), + ["-kernel", "inetrc", ValString | VMArgs]. + +maybe_add_tls_arguments(VMArgs) -> %% In the next case, RabbitMQ has been configured with additional Erlang VM %% arguments such as this: %% @@ -494,14 +511,14 @@ add_tls_arguments(InetDistModule, VMArgs0) -> %% "/usr/local/lib/erlang/lib/ssl-11.0.3/ebin", %% "-proto_dist","inet_tls","-boot", %% "no_dot_erlang","-hidden"], - VMArgs2 = case init:get_argument(ssl_dist_opt) of + VMArgs1 = case init:get_argument(ssl_dist_opt) of {ok, SslDistOpts0} -> SslDistOpts1 = [["-ssl_dist_opt" | SslDistOpt] || SslDistOpt <- SslDistOpts0], SslDistOpts2 = lists:concat(SslDistOpts1), - SslDistOpts2 ++ VMArgs1; + SslDistOpts2 ++ VMArgs; _ -> - VMArgs1 + VMArgs end, %% In the next case, RabbitMQ has been configured with additional Erlang VM %% arguments such as this: @@ -511,13 +528,13 @@ add_tls_arguments(InetDistModule, VMArgs0) -> %% %% This code adds the `ssl_dist_optfile' argument to the peer node's %% argument list. - VMArgs3 = case init:get_argument(ssl_dist_optfile) of + VMArgs2 = case init:get_argument(ssl_dist_optfile) of {ok, [[SslDistOptfileArg]]} -> - ["-ssl_dist_optfile", SslDistOptfileArg | VMArgs2]; + ["-ssl_dist_optfile", SslDistOptfileArg | VMArgs1]; _ -> - VMArgs2 + VMArgs1 end, - VMArgs3. + VMArgs2. do_query_node_props(Nodes) when Nodes =/= [] -> %% Make sure all log messages are forwarded from this temporary hidden @@ -608,6 +625,10 @@ query_node_props1([], [], NodesAndProps, ProxyGroupLeader) -> query_node_props2([{Node, Members} | Rest], NodesAndProps, ProxyGroupLeader) -> try + erpc:call( + Node, logger, debug, + ["Peer discovery: temporary hidden node '~ts' queries properties " + "from node '~ts'", [node(), Node]]), StartTime = get_node_start_time(Node, microsecond, ProxyGroupLeader), IsReady = is_node_db_ready(Node, ProxyGroupLeader), NodeAndProps = {Node, Members, StartTime, IsReady}, @@ -638,6 +659,8 @@ query_node_props2([], NodesAndProps, ProxyGroupLeader) -> after 120_000 -> ok end, + ?assertEqual([], nodes()), + ?assert(length(NodesAndProps2) =< length(nodes(hidden))), NodesAndProps2. -spec get_node_start_time(Node, Unit, ProxyGroupLeader) -> StartTime when @@ -825,6 +848,13 @@ can_use_discovered_nodes(_DiscoveredNodes, []) -> %% %% @private +select_node_to_join([{Node, _Members, _StartTime, _IsReady} | _]) + when Node =:= node() -> + ?LOG_INFO( + "Peer discovery: node '~ts' selected for auto-clustering", + [Node], + #{domain => ?RMQLOG_DOMAIN_PEER_DISC}), + Node; select_node_to_join([{Node, _Members, _StartTime, IsReady} | _]) when IsReady =/= false -> ?LOG_INFO( diff --git a/deps/rabbit/test/peer_discovery_tmp_hidden_node_SUITE.erl b/deps/rabbit/test/peer_discovery_tmp_hidden_node_SUITE.erl new file mode 100644 index 000000000000..77338e10198d --- /dev/null +++ b/deps/rabbit/test/peer_discovery_tmp_hidden_node_SUITE.erl @@ -0,0 +1,379 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright (c) 2024 Broadcom. All Rights Reserved. The term “Broadcom” +%% refers to Broadcom Inc. and/or its subsidiaries. All rights reserved. +%% + +-module(peer_discovery_tmp_hidden_node_SUITE). + +-include_lib("kernel/include/logger.hrl"). +-include_lib("kernel/include/inet.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +-include_lib("rabbit_common/include/logging.hrl"). + +-export([suite/0, + all/0, + init_per_suite/1, + end_per_suite/1, + init_per_group/2, + end_per_group/2, + init_per_testcase/2, + end_per_testcase/2, + + do_setup_test_node/1, + + no_connection_between_peers_is_opened/1, + long_names_work/1, + ipv6_works/1, + inetrc_file_as_atom_works/1, + tls_dist_works/1 + ]). + +suite() -> + [{timetrap, {minutes, 15}}]. + +all() -> + [no_connection_between_peers_is_opened, + long_names_work, + ipv6_works, + inetrc_file_as_atom_works, + tls_dist_works]. + +%% ------------------------------------------------------------------- +%% Testsuite setup/teardown +%% ------------------------------------------------------------------- + +init_per_suite(Config) -> + rabbit_ct_helpers:log_environment(), + rabbit_ct_helpers:run_setup_steps(Config). + +end_per_suite(Config) -> + rabbit_ct_helpers:run_teardown_steps(Config). + +init_per_group(_Group, Config) -> + Config. + +end_per_group(_Group, Config) -> + Config. + +init_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_started(Config, Testcase), + Config. + +end_per_testcase(Testcase, Config) -> + rabbit_ct_helpers:testcase_finished(Config, Testcase). + +%% ------------------------------------------------------------------- +%% Testcases. +%% ------------------------------------------------------------------- + +no_connection_between_peers_is_opened(_Config) -> + PeerOptions = #{longnames => false}, + test_query_node_props(?FUNCTION_NAME, 2, PeerOptions). + +long_names_work(_Config) -> + PeerOptions = #{longnames => true}, + test_query_node_props(?FUNCTION_NAME, 2, PeerOptions). + +ipv6_works(Config) -> + PrivDir = ?config(priv_dir, Config), + InetrcFilename = filename:join(PrivDir, "inetrc-ipv6.erl"), + ct:pal("Inetrc filename:~n~0p", [InetrcFilename]), + Inetrc = [{inet6, true}], + InetrcContent = [io_lib:format("~p.~n", [Param]) || Param <- Inetrc], + ct:pal("Inetrc file content:~n---8<---~n~s---8<---", [InetrcContent]), + ok = file:write_file(InetrcFilename, InetrcContent), + InetrcArg = rabbit_misc:format("~0p", [InetrcFilename]), + + PeerOptions = #{host => "::1", + args => ["-proto_dist", "inet6_tcp", + "-kernel", "inetrc", InetrcArg]}, + test_query_node_props(?FUNCTION_NAME, 2, PeerOptions). + +inetrc_file_as_atom_works(_Config) -> + %% We can't write the inetrc file in `privdir' like we did in + %% `ipv6_works/1' because here we convert the filename to an atom and an + %% atom can't be more than 255 characters. It happens that in the + %% Buildbuddy CI worker, we reach a filename of 340+ characters. + %% + %% Instead, we write the file in the temporary directory. + %% + %% TEMP and TMP are used on Microsoft Windows, TMPDIR on Unix (but TMPDIR + %% might not be defined). + TmpDir = os:getenv("TEMP", os:getenv("TMP", os:getenv("TMPDIR", "/tmp"))), + InetrcFilename = filename:join(TmpDir, "inetrc-ipv6.erl"), + ct:pal("Inetrc filename:~n~0p", [InetrcFilename]), + Inetrc = [{inet6, true}], + InetrcContent = [io_lib:format("~p.~n", [Param]) || Param <- Inetrc], + ct:pal("Inetrc file content:~n---8<---~n~s---8<---", [InetrcContent]), + ok = file:write_file(InetrcFilename, InetrcContent), + InetrcArg = rabbit_misc:format("~0p", [list_to_atom(InetrcFilename)]), + + PeerOptions = #{host => "::1", + args => ["-proto_dist", "inet6_tcp", + "-kernel", "inetrc", InetrcArg]}, + test_query_node_props(?FUNCTION_NAME, 2, PeerOptions). + +tls_dist_works(Config) -> + CertsDir = ?config(rmq_certsdir, Config), + Password = ?config(rmq_certspwd, Config), + CACert = filename:join([CertsDir, "testca", "cacert.pem"]), + ServerCert = filename:join([CertsDir, "server", "cert.pem"]), + ServerKey = filename:join([CertsDir, "server", "key.pem"]), + SslOptions = [{server, + [{cacertfile, CACert}, + {certfile, ServerCert}, + {keyfile, ServerKey}, + {password, Password}, + {secure_renegotiate, true}, + {verify, verify_none}, + {fail_if_no_peer_cert, false}]}, + {client, + [{cacertfile, CACert}, + {secure_renegotiate, true}]}], + + PrivDir = ?config(priv_dir, Config), + SslOptFilename = filename:join(PrivDir, "ssl-options.erl"), + ct:pal("SSL options filename:~n~0p", [SslOptFilename]), + SslOptContent = rabbit_misc:format("~p.~n", [SslOptions]), + ct:pal("SSL options file content:~n---8<---~n~s---8<---", [SslOptContent]), + ok = file:write_file(SslOptFilename, SslOptContent), + + %% We need to read the certificate's Subject ID to see what hostname is + %% used in the certificate and use the same to start the test Erlang nodes. + %% We also need to pay attention if the name is short or long. + {ok, ServerCertBin} = file:read_file(ServerCert), + ct:pal("ServerCertBin = ~p", [ServerCertBin]), + [DecodedCert] = public_key:pem_decode(ServerCertBin), + ct:pal("DecodedCert = ~p", [DecodedCert]), + DecodedCert1 = element(2, DecodedCert), + {_SerialNr, {rdnSequence, IssuerAttrs}} = public_key:pkix_subject_id( + DecodedCert1), + ct:pal("IssuerAttrs = ~p", [IssuerAttrs]), + [ServerName] = [Value + || [#'AttributeTypeAndValue'{type = {2, 5, 4, 3}, + value = {utf8String, Value}}] + <- IssuerAttrs], + ct:pal("ServerName = ~p", [ServerName]), + UseLongnames = re:run(ServerName, "\\.", [{capture, none}]) =:= match, + + PeerOptions = #{host => binary_to_list(ServerName), + longnames => UseLongnames, + args => ["-proto_dist", "inet_tls", + "-ssl_dist_optfile", SslOptFilename]}, + test_query_node_props(?FUNCTION_NAME, 2, PeerOptions). + +test_query_node_props(Testcase, NodeCount, PeerOptions) -> + Peers = start_test_nodes(Testcase, NodeCount, PeerOptions), + try + do_test_query_node_props(Peers) + after + stop_test_nodes(Peers) + end. + +do_test_query_node_props(Peers) -> + %% Ensure no connection exists at the beginning. + ensure_no_connections_between_test_nodes(Peers), + + %% Query the remote node's properties. The return value should have the + %% properties of the peer node, otherwise it means that we failed to + %% contact it. + [NodeA, NodeB] = lists:sort(maps:keys(Peers)), + NodeAPid = maps:get(NodeA, Peers), + Ret = peer:call( + NodeAPid, + rabbit_peer_discovery, query_node_props, [[NodeB]], + infinity), + ct:pal("Discovered nodes properties:~n~p", [Ret]), + ?assertMatch([{NodeB, [NodeB], _, false}], Ret), + + %% Ensure no connection exists after the query. + ensure_no_connections_between_test_nodes(Peers). + +%% ------------------------------------------------------------------- +%% Helpers. +%% ------------------------------------------------------------------- + +start_test_nodes(Testcase, NodeCount, PeerOptions) -> + PeerOptions1 = PeerOptions#{ + %% We use an alternative connection channel, not the + %% regular Erlang distribution, because we want to test + %% the behavior of the temporary hidden node and + %% especially that it doesn't rely or create a connection + %% between the two nodes. + connection => standard_io, + wait_boot => infinity}, + TestEbin = filename:dirname(code:which(?MODULE)), + Args0 = maps:get(args, PeerOptions1, []), + Args1 = ["-pa", TestEbin | Args0], + Env0 = maps:get(env, PeerOptions1, []), + Env1 = [{"ERL_LIBS", os:getenv("ERL_LIBS")} | Env0], + PeerOptions2 = PeerOptions1#{args => Args1, + env => Env1}, + start_test_nodes(Testcase, 1, NodeCount, PeerOptions2, #{}). + +start_test_nodes(Testcase, NodeNumber, NodeCount, PeerOptions, Peers) + when NodeNumber =< NodeCount -> + PeerName0 = rabbit_misc:format("~s-~b", [Testcase, NodeNumber]), + PeerOptions1 = PeerOptions#{name => PeerName0}, + ct:pal("Starting peer with options: ~p", [PeerOptions1]), + case catch peer:start(PeerOptions1) of + {ok, PeerPid, PeerName} -> + ct:pal("Configuring peer '~ts'", [PeerName]), + setup_test_node(PeerPid, PeerOptions1), + Peers1 = Peers#{PeerName => PeerPid}, + start_test_nodes( + Testcase, NodeNumber + 1, NodeCount, PeerOptions, Peers1); + Error1 when not is_map_key(host, PeerOptions1) -> + ct:pal("Failed to started peer node:~n" + "Options: ~p~n" + "Error: ~p", [PeerOptions1, Error1]), + %% At least when running from a Buildbuddy CI worker, the network + %% configuration is incomplete and the host lacks an FQDN. This + %% breaks the start of an Erlang node with long names. + %% + %% To work around that, we mess with the network configuration (the + %% Erlang node runs as root) and try to determine a hostname we can + %% use. We then try again to start the node. + case determine_hostname(PeerOptions1) of + {ok, Host} -> + PeerOptions2 = PeerOptions1#{host => Host}, + start_test_nodes( + Testcase, NodeNumber, NodeCount, PeerOptions2, Peers); + {error, _} = Error2 -> + ct:pal("Failed to determine a usable hostname:~n" + "Options: ~p~n" + "Error: ~p", [PeerOptions1, Error2]), + stop_test_nodes(Peers), + erlang:throw(Error2) + end; + Error -> + ct:pal("Failed to started peer node:~n" + "Options: ~p~n" + "Error: ~p", [PeerOptions1, Error]), + stop_test_nodes(Peers), + erlang:throw(Error) + end; +start_test_nodes(_Testcase, _NodeNumber, _Count, _PeerOptions, Peers) -> + ct:pal("Peers: ~p", [Peers]), + Peers. + +determine_hostname(PeerOptions) -> + %% Please wear eye protection glasses to read what's next! + %% + %% The Buildbuddy CI worker network configuration lacks an FQDN and we need + %% one to start an Erlang node with a long name. To work around this, we + %% modify `/etc/hosts' to add an FQDN for 127.0.0.1 and ::1. + %% + %% 1. We read the existing file + %% 2. We modify it to add the FQDN + %% 3. We write the modified file + %% 4. We put a coin in the swear jar + HostsFilename = "/etc/hosts", + case file:read_file(HostsFilename) of + {ok, HostsFile} -> + HostsFile1 = re:replace( + HostsFile, + "^(127\\.0\\.0\\.1|::1)\\s+.*", + "& localhost.my.domain", + [{return, binary}, multiline, global]), + ct:pal( + "Changing ~s from:~n" + "---8<---~n" + "~s" + "---8<---~n" + "to:~n" + "---8<---~n" + "~s" + "---8<---~n", + [HostsFilename, HostsFile, HostsFile1]), + case file:write_file(HostsFilename, HostsFile1) of + ok -> determine_hostname1(PeerOptions); + {error, _} = Error -> Error + end; + {error, _} = Error -> + Error + end. + +determine_hostname1(PeerOptions) -> + %% Now that we proudly have an FQDN, we query the IP + %% addresses and get the hostname(s) associated with each. + %% + %% In the end, we return the first hostname that matches + %% the short/long name criteria. + case inet:getifaddrs() of + {ok, IFaces} -> + ct:pal("Network interfaces: ~p", [IFaces]), + IPv4Addrs = [IPv4Addr + || {_Name, Props} <- IFaces, + {addr, IPv4Addr} <- Props, + is_tuple(IPv4Addr) andalso size(IPv4Addr) =:= 4], + ct:pal("IPv4 addresses: ~p", [IPv4Addrs]), + determine_hostname2(IPv4Addrs, PeerOptions); + {error, _} = Error -> + Error + end. + +determine_hostname2([IPv4Addr | Rest], PeerOptions) -> + WantFQDN = maps:get(longnames, PeerOptions, false), + case inet:gethostbyaddr(IPv4Addr) of + {ok, #hostent{h_name = FQDN, h_aliases = ShortDNs}} -> + AllDNs = [FQDN | ShortDNs], + ct:pal( + "All domain names for IPv4 address ~p: ~p", + [IPv4Addr, AllDNs]), + ValidDNs = lists:filter( + fun(DN) -> + lists:member($., DN) =:= WantFQDN + end, AllDNs), + ct:pal( + "Valid domain names for IPv4 address ~p: ~p", + [IPv4Addr, ValidDNs]), + case ValidDNs of + [DN | _] -> + {ok, DN}; + [] -> + ct:pal( + "No valid hostnames found for IPv4 ~p: ~p", + [IPv4Addr, AllDNs]), + determine_hostname2(Rest, PeerOptions) + end; + {error, _} = Error -> + Error + end; +determine_hostname2([], _PeerOptions) -> + {error, no_valid_hostnames_found}. + +setup_test_node(PeerPid, PeerOptions) -> + peer:call(PeerPid, ?MODULE, do_setup_test_node, [PeerOptions]). + +do_setup_test_node(PeerOptions) -> + Context = case maps:get(longnames, PeerOptions, false) of + true -> #{nodename_type => longnames}; + false -> #{} + end, + logger:set_primary_config(level, debug), + meck:new(rabbit_prelaunch, [unstick, passthrough, no_link]), + meck:expect(rabbit_prelaunch, get_context, fun() -> Context end), + meck:new(rabbit_nodes, [unstick, passthrough, no_link]), + Nodes = [node()], + meck:expect(rabbit_nodes, all, fun() -> Nodes end), + meck:expect(rabbit_nodes, list_members, fun() -> Nodes end), + ok. + +stop_test_nodes(Peers) -> + maps:foreach( + fun(_PeerName, PeerPid) -> + peer:stop(PeerPid) + end, Peers). + +ensure_no_connections_between_test_nodes(Peers) -> + maps:foreach( + fun(_PeerName, PeerPid) -> + ?assertEqual([], peer:call(PeerPid, erlang, nodes, [])) + end, Peers). diff --git a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl index 3d8a3d5f6e7a..1db45308e426 100644 --- a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl +++ b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl @@ -618,7 +618,10 @@ ensure_ssl_certs(Config) -> {verify, Verify}, {fail_if_no_peer_cert, FailIfNoPeerCert} ]}]}), - set_config(Config1, {rmq_certsdir, CertsDir}); + set_config( + Config1, + [{rmq_certsdir, CertsDir}, + {rmq_certspwd, CertsPwd}]); _ -> {skip, "Failed to create SSL certificates"} end.