diff --git a/apisix/control/v1.lua b/apisix/control/v1.lua index 17520f633df9..0aabf165dccf 100644 --- a/apisix/control/v1.lua +++ b/apisix/control/v1.lua @@ -16,6 +16,13 @@ -- local core = require("apisix.core") local plugin = require("apisix.plugin") +local get_routes = require("apisix.router").http_routes +local get_services = require("apisix.http.service").services +local upstream_mod = require("apisix.upstream") +local get_upstreams = upstream_mod.upstreams +local ipairs = ipairs +local str_format = string.format +local ngx_var = ngx.var local _M = {} @@ -49,11 +56,120 @@ function _M.schema() end +local function extra_checker_info(value, src_type) + local checker = value.checker + local upstream = value.checker_upstream + local host = upstream.checks and upstream.checks.active and upstream.checks.active.host + local port = upstream.checks and upstream.checks.active and upstream.checks.active.port + local nodes = upstream.nodes + local healthy_nodes = core.table.new(#nodes, 0) + for _, node in ipairs(nodes) do + local ok = checker:get_target_status(node.host, port or node.port, host) + if ok then + core.table.insert(healthy_nodes, node) + end + end + + local conf = value.value + return { + name = upstream_mod.get_healthchecker_name(value), + src_id = conf.id, + src_type = src_type, + nodes = nodes, + healthy_nodes = healthy_nodes, + } +end + + +local function iter_and_add_healthcheck_info(infos, values, src_type) + if not values then + return + end + + for _, value in core.config_util.iterate_values(values) do + if value.checker then + core.table.insert(infos, extra_checker_info(value, src_type)) + end + end +end + + +function _M.get_health_checkers() + local infos = {} + local routes = get_routes() + iter_and_add_healthcheck_info(infos, routes, "routes") + local services = get_services() + iter_and_add_healthcheck_info(infos, services, "services") + local upstreams = get_upstreams() + iter_and_add_healthcheck_info(infos, upstreams, "upstreams") + return 200, infos +end + + +local function iter_and_find_healthcheck_info(values, src_type, src_id) + if not values then + return nil, str_format("%s[%s] not found", src_type, src_id) + end + + for _, value in core.config_util.iterate_values(values) do + if value.value.id == src_id then + if not value.checker then + return nil, str_format("no checker for %s[%s]", src_type, src_id) + end + + return extra_checker_info(value, src_type) + end + end + + return nil, str_format("%s[%s] not found", src_type, src_id) +end + + +function _M.get_health_checker() + local uri_segs = core.utils.split_uri(ngx_var.uri) + core.log.info("healthcheck uri: ", core.json.delay_encode(uri_segs)) + + local src_type, src_id = uri_segs[4], uri_segs[5] + if not src_id then + return 404, {error_msg = str_format("missing src id for src type %s", src_type)} + end + + local values + if src_type == "routes" then + values = get_routes() + elseif src_type == "services" then + values = get_services() + elseif src_type == "upstreams" then + values = get_upstreams() + else + return 400, {error_msg = str_format("invalid src type %s", src_type)} + end + + local info, err = iter_and_find_healthcheck_info(values, src_type, src_id) + if not info then + return 404, {error_msg = err} + end + return 200, info +end + + return { -- /v1/schema { methods = {"GET"}, uris = {"/schema"}, handler = _M.schema, + }, + -- /v1/healthcheck + { + methods = {"GET"}, + uris = {"/healthcheck"}, + handler = _M.get_health_checkers, + }, + -- /v1/healthcheck/{src_type}/{src_id} + { + methods = {"GET"}, + uris = {"/healthcheck/*"}, + handler = _M.get_health_checker, } } diff --git a/apisix/upstream.lua b/apisix/upstream.lua index c0c0889925da..dd455d42da18 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -60,6 +60,12 @@ local function release_checker(healthcheck_parent) end +local function get_healthchecker_name(value) + return "upstream#" .. value.key +end +_M.get_healthchecker_name = get_healthchecker_name + + local function create_checker(upstream) if healthcheck == nil then healthcheck = require("resty.healthcheck") @@ -71,7 +77,7 @@ local function create_checker(upstream) end local checker, err = healthcheck.new({ - name = "upstream#" .. healthcheck_parent.key, + name = get_healthchecker_name(healthcheck_parent), shm_name = "upstream-healthcheck", checks = upstream.checks, }) diff --git a/doc/control-api.md b/doc/control-api.md index 683fadd50466..37473a757514 100644 --- a/doc/control-api.md +++ b/doc/control-api.md @@ -79,3 +79,102 @@ Return the jsonschema used by this APISIX instance in the format below: For `plugins` part, only enabled plugins will be returned. Some plugins may lack of fields like `consumer_schema` or `type`, it is dependended by the plugin's definition. + +### GET /v1/healthcheck + +Introduced since `v2.3`. + +Return current [health check](health-check.md) status in the format below: + +```json +[ + { + "healthy_nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + } + ], + "name": "upstream#/upstreams/1", + "nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + }, + { + "host": "127.0.0.2", + "port": 1988, + "weight": 1 + } + ], + "src_id": "1", + "src_type": "upstreams" + }, + { + "healthy_nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + } + ], + "name": "upstream#/routes/1", + "nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + }, + { + "host": "127.0.0.1", + "port": 1988, + "weight": 1 + } + ], + "src_id": "1", + "src_type": "routes" + } +] +``` + +Each entry contains fields below: + +* src_type: where the health checker comes from. The value is one of `["routes", "services", "upstreams"]`. +* src_id: the id of object which creates the health checker. For example, if Upstream +object with id 1 creates a health checker, the `src_type` is `upstreams` and the `src_id` is `1`. +* name: the name of the health checker. +* nodes: the target nodes of the health checker. +* healthy_nodes: the healthy node known by the health checker. + +User can also use `/v1/healthcheck/$src_type/$src_id` can get the status of a health checker. + +For example, `GET /v1/healthcheck/upstreams/1` returns: + +```json +{ + "healthy_nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + } + ], + "name": "upstream#/upstreams/1", + "nodes": [ + { + "host": "127.0.0.1", + "port": 1980, + "weight": 1 + }, + { + "host": "127.0.0.2", + "port": 1988, + "weight": 1 + } + ], + "src_id": "1", + "src_type": "upstreams" +} +``` diff --git a/doc/health-check.md b/doc/health-check.md index 748fbf499e26..fa2b04fc422b 100644 --- a/doc/health-check.md +++ b/doc/health-check.md @@ -22,6 +22,9 @@ Health Check of APISIX is based on [lua-resty-healthcheck](https://github.com/Kong/lua-resty-healthcheck), you can use it for upstream. +Note that we only start the health check when the upstream is hit by a request. +There won't be any health check if an upstream is configured but isn't in used. + The following is an example of health check: ```shell @@ -105,3 +108,5 @@ contains: `active` or `passive`. * `passive.unhealthy.tcp_failures`: Number of TCP failures in proxied traffic to consider a target unhealthy, as observed by passive health checks. * `passive.unhealthy.timeouts`: Number of timeouts in proxied traffic to consider a target unhealthy, as observed by passive health checks. * `passive.unhealthy.http_failures`: Number of HTTP failures in proxied traffic (as defined by `passive.unhealthy.http_statuses`) to consider a target unhealthy, as observed by passive health checks. + +The health check status can be fetched via `GET /v1/healthcheck` in [control API](./control-api.md). diff --git a/doc/zh-cn/health-check.md b/doc/zh-cn/health-check.md index a41ce86d113e..15bea36c6785 100644 --- a/doc/zh-cn/health-check.md +++ b/doc/zh-cn/health-check.md @@ -23,6 +23,9 @@ APISIX的健康检查使用[lua-resty-healthcheck](https://github.com/Kong/lua-resty-healthcheck)实现,你可以在upstream中使用它。 +注意只有在 upstream 被请求时才会开始健康检查。 +如果一个 upstream 被配置但没有被请求,那么就不会有健康检查。 + 下面是一个检查检查的例子: ```shell @@ -105,3 +108,5 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 * `passive.unhealthy.tcp_failures`: 如果TCP通讯失败次数超过 `tcp_failures` 次,则将upstream节点设置为 `unhealthy` 状态。 * `passive.unhealthy.timeouts`: 如果被动健康检查超时次数超过 `timeouts` 次,则将upstream节点设置为 `unhealthy` 状态。 * `passive.unhealthy.http_failures`: 如果被动健康检查的HTTP请求失败(由 `passive.unhealthy.http_statuses` 定义)的次数超过 `http_failures`次,则将upstream节点设置为 `unhealthy` 状态。 + +健康检查信息可以通过 [控制接口](./control_api.md) 中的 `GET /v1/healthcheck` 接口得到。 diff --git a/t/control/healthcheck.t b/t/control/healthcheck.t new file mode 100644 index 000000000000..be22420185b3 --- /dev/null +++ b/t/control/healthcheck.t @@ -0,0 +1,271 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); +log_level("info"); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->yaml_config) { + my $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 + config_center: yaml + enable_admin: false +_EOC_ + + $block->set_value("yaml_config", $yaml_config); + } + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: upstreams +--- apisix_yaml +routes: + - + uris: + - /hello + upstream_id: 1 +upstreams: + - nodes: + "127.0.0.1:1980": 1 + "127.0.0.2:1988": 1 + type: roundrobin + id: 1 + checks: + active: + http_path: "/status" + healthy: + interval: 1 + successes: 1 + unhealthy: + interval: 1 + http_failures: 1 +#END +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + + ngx.sleep(2) + + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res[1].nodes, function(a, b) + return a.host < b.host + end) + ngx.say(json.encode(res)) + + local code, body, res = t.test('/v1/healthcheck/upstreams/1', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res.nodes, function(a, b) + return a.host < b.host + end) + ngx.say(json.encode(res)) + } + } +--- grep_error_log eval +qr/unhealthy TCP increment \(.+\) for '[^']+'/ +--- grep_error_log_out +unhealthy TCP increment (1/2) for '(127.0.0.2:1988)' +unhealthy TCP increment (2/2) for '(127.0.0.2:1988)' +--- response_body +[{"healthy_nodes":[{"host":"127.0.0.1","port":1980,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.2","port":1988,"weight":1}],"src_id":"1","src_type":"upstreams"}] +{"healthy_nodes":[{"host":"127.0.0.1","port":1980,"weight":1}],"name":"upstream#/upstreams/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.2","port":1988,"weight":1}],"src_id":"1","src_type":"upstreams"} + + + +=== TEST 2: routes +--- apisix_yaml +routes: + - + id: 1 + uris: + - /hello + upstream: + nodes: + "127.0.0.1:1980": 1 + "127.0.0.1:1988": 1 + type: roundrobin + checks: + active: + http_path: "/status" + host: "127.0.0.1" + healthy: + interval: 1 + successes: 1 + unhealthy: + interval: 1 + http_failures: 1 +#END +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + + ngx.sleep(2) + + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res[1].nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(res)) + + local code, body, res = t.test('/v1/healthcheck/routes/1', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res.nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(res)) + } + } +--- grep_error_log eval +qr/unhealthy TCP increment \(.+\) for '[^']+'/ +--- grep_error_log_out +unhealthy TCP increment (1/2) for '127.0.0.1(127.0.0.1:1988)' +unhealthy TCP increment (2/2) for '127.0.0.1(127.0.0.1:1988)' +--- response_body +[{"healthy_nodes":[{"host":"127.0.0.1","port":1980,"weight":1}],"name":"upstream#/routes/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.1","port":1988,"weight":1}],"src_id":"1","src_type":"routes"}] +{"healthy_nodes":[{"host":"127.0.0.1","port":1980,"weight":1}],"name":"upstream#/routes/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.1","port":1988,"weight":1}],"src_id":"1","src_type":"routes"} + + + +=== TEST 3: services +--- apisix_yaml +routes: + - id: 1 + service_id: 1 + uris: + - /hello + +services: + - + id: 1 + upstream: + nodes: + "127.0.0.1:1980": 1 + "127.0.0.1:1988": 1 + type: roundrobin + checks: + active: + http_path: "/status" + host: "127.0.0.1" + port: 1988 + healthy: + interval: 1 + successes: 1 + unhealthy: + interval: 1 + http_failures: 1 +#END +--- config + location /t { + content_by_lua_block { + local json = require("toolkit.json") + local t = require("lib.test_admin") + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + local res, err = httpc:request_uri(uri, {method = "GET"}) + + ngx.sleep(2) + + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res[1].nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(res)) + + local code, body, res = t.test('/v1/healthcheck/services/1', + ngx.HTTP_GET) + res = json.decode(res) + table.sort(res.nodes, function(a, b) + return a.port < b.port + end) + ngx.say(json.encode(res)) + } + } +--- grep_error_log eval +qr/unhealthy TCP increment \(.+\) for '[^']+'/ +--- grep_error_log_out +unhealthy TCP increment (1/2) for '127.0.0.1(127.0.0.1:1988)' +unhealthy TCP increment (2/2) for '127.0.0.1(127.0.0.1:1988)' +--- response_body +[{"healthy_nodes":{},"name":"upstream#/services/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.1","port":1988,"weight":1}],"src_id":"1","src_type":"services"}] +{"healthy_nodes":{},"name":"upstream#/services/1","nodes":[{"host":"127.0.0.1","port":1980,"weight":1},{"host":"127.0.0.1","port":1988,"weight":1}],"src_id":"1","src_type":"services"} + + + +=== TEST 4: no checkers +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local code, body, res = t.test('/v1/healthcheck', + ngx.HTTP_GET) + ngx.print(res) + } + } +--- response_body +{} + + + +=== TEST 5: no checker +--- request +GET /v1/healthcheck/routes/1 +--- error_code: 404 +--- response_body +{"error_msg":"routes[1] not found"} + + + +=== TEST 6: invalid src type +--- request +GET /v1/healthcheck/route/1 +--- error_code: 400 +--- response_body +{"error_msg":"invalid src type route"}