Make cross-sentinel connection failures non-fatal

src/uhs/sentinel/client.cpp

@@ -15,8 +15,8 @@ namespace cbdc::sentinel::rpc {
         : m_logger(std::move(logger)),
           m_client(std::move(endpoints)) {}
 
-    auto client::init() -> bool {
-        if(!m_client.init()) {
+    auto client::init(std::optional<bool> error_fatal) -> bool {
+        if(!m_client.init(error_fatal)) {
             m_logger->error("Failed to initialize sentinel RPC client");
             return false;
         }

src/uhs/sentinel/client.hpp

@@ -31,8 +31,11 @@ namespace cbdc::sentinel::rpc {
         auto operator=(client&&) -> client& = delete;
 
         /// Initializes the client. Establishes a connection to the sentinel.
+        /// \param error_fatal treat connection errors as fatal. See
+        ///                    tcp_client::init for further explanation.
         /// \return true if initialization succeeded.
-        auto init() -> bool;
+        /// \see \ref cbdc::rpc::tcp_client::init(std::optional<bool>)
+        auto init(std::optional<bool> error_fatal = std::nullopt) -> bool;
 
         /// Result type from execute_transaction.
         using execute_result_type

src/uhs/twophase/sentinel_2pc/controller.cpp

@@ -49,7 +49,7 @@ namespace cbdc::sentinel_2pc {
             auto client = std::make_unique<sentinel::rpc::client>(
                 std::vector<network::endpoint_t>{ep},
                 m_logger);
-            if(!client->init()) {
+            if(!client->init(false)) {
                 m_logger->warn("Failed to start sentinel client");
             }
             m_sentinel_clients.emplace_back(std::move(client));

src/util/rpc/tcp_client.hpp

@@ -53,11 +53,27 @@ namespace cbdc::rpc {
 
         /// Initializes the client. Connects to the server endpoints and
         /// starts the response handler thread.
-        /// \return false if there is only one endpoint and connecting failed.
-        ///         Otherwise true.
-        [[nodiscard]] auto init() -> bool {
+        /// \param error_fatal treat connection errors as fatal.
+        ///                    If this is set to true, failure to connect
+        ///                    to any of the endpoints will result in failure
+        ///                    to start the client and return false from this
+        ///                    function.
+        ///                    If this is set to false, any
+        ///                    connection error will be silently ignored and
+        ///                    the handler thread will be started. The client
+        ///                    will continue to retry connecting in the
+        ///                    background.
+        ///                    If this is std::nullopt, connection
+        ///                    errors are only treated as fatal when there is a
+        ///                    single endpoint.
+        /// \return false if there is a fatal connection error, true if the
+        /// client is connected and ready and the response handler is started.
+        [[nodiscard]] auto init(std::optional<bool> error_fatal = std::nullopt) -> bool {
+            if(!error_fatal) {
+                error_fatal = m_server_endpoints.size() <= 1;
+            }
             if(!m_net.cluster_connect(m_server_endpoints,
-                                      m_server_endpoints.size() <= 1)) {
+                                      error_fatal.value())) {
                 return false;
             }