feat: etcd cluster single node failure APISIX startup failure

apisix/cli/etcd.lua

@@ -32,6 +32,7 @@ local tonumber = tonumber
 local str_format = string.format
 local str_sub = string.sub
 local table_concat = table.concat
+local table_insert = table.insert
 
 local _M = {}
 
@@ -187,6 +188,7 @@ function _M.init(env, args)
     end
 
     -- check the etcd cluster version
+    local etcd_healthy_hosts = {}
     for index, host in ipairs(yaml_conf.etcd.host) do
         local version_url = host .. "/version"
         local errmsg
@@ -207,8 +209,8 @@ function _M.init(env, args)
         end
 
         if not res then
-            errmsg = str_format("request etcd endpoint \'%s\' error, %s\n", version_url, err)
-            util.die(errmsg)
+            print(str_format("request etcd endpoint \'%s\' error, %s\n", version_url, err))
+            goto continue
         end
 
         local body, _, err = dkjson.decode(res)
@@ -225,10 +227,18 @@ function _M.init(env, args)
                      env.min_etcd_version,
                      ", please upgrade your etcd cluster\n")
         end
+
+        table_insert(etcd_healthy_hosts, host)
+
+        :: continue ::
+    end
+
+    if host_count >= 2 and #etcd_healthy_hosts < 2 then
+        util.die("etcd cluster must have two or more healthy nodes\n")
     end
 
     local etcd_ok = false
-    for index, host in ipairs(yaml_conf.etcd.host) do
+    for index, host in ipairs(etcd_healthy_hosts) do
         local is_success = true
 
         local errmsg
@@ -358,7 +368,7 @@ function _M.init(env, args)
     end
 
     if not etcd_ok then
-        util.die("none of the configured etcd works well")
+        util.die("none of the configured etcd works well\n")
     end
 end
 

t/cli/test_etcd_ha.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+ETCD_NAME_0=etcd0
+ETCD_NAME_1=etcd1
+ETCD_NAME_2=etcd2
+
+echo '
+etcd:
+  host:
+    - "http://127.0.0.1:23790"
+    - "http://127.0.0.1:23791"
+    - "http://127.0.0.1:23792"
+' > conf/config.yaml
+
+docker-compose -f ./t/cli/docker-compose-etcd-cluster.yaml up -d
+
+# case 1: stop one etcd nodes (result: start successful)
+docker stop ${ETCD_NAME_0}
+
+make init && make run
+
+if [ "$?" != 0 ]; then
+    echo "FAIL: stop only one etcd node APISIX should start normally"
+    exit 1
+fi
+
+echo "OK: APISIX successfully to start, stop only one etcd node"
+
+make stop
+
+# case 2: stop two etcd nodes (result: start failure)
+docker stop ${ETCD_NAME_1}
+
+make init && make run
+
+if [ "$?" == 0 ]; then
+    echo "FAIL: etcd has stopped two nodes, APISIX should fail to start"
+    exit 1
+fi
+
+echo "OK: APISIX failed to start, etcd cluster must have two or more healthy nodes"
+
+
+# case 3: stop all etcd nodes (result: start failure)
+docker stop ${ETCD_NAME_2}
+
+make init && make run
+
+if [ "$?" == 0 ]; then
+    echo "FAIL: all etcd nodes have stopped, APISIX should not be able to start"
+    exit 1
+fi
+
+echo "OK: APISIX failed to start, all etcd nodes have stopped"
+
+# stop etcd docker container
+docker-compose down