From b57dc6a3b4363f12715ee3205d4978c6f62da7c1 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 14:30:32 +0800 Subject: [PATCH 01/15] feat(ssl): support get upstream cert frm ssl object --- apisix/admin/ssl.lua | 2 +- apisix/init.lua | 16 ++++ apisix/schema_def.lua | 32 ++++++- apisix/ssl.lua | 4 + apisix/ssl/router/radixtree_sni.lua | 17 ++++ apisix/upstream.lua | 40 ++++++++- docs/en/latest/admin-api.md | 8 +- docs/zh/latest/admin-api.md | 7 +- t/admin/ssl.t | 73 +++++++++++++++- t/admin/upstream.t | 129 ++++++++++++++++++++++++++++ t/node/upstream-mtls.t | 65 ++++++++++++++ 11 files changed, 380 insertions(+), 13 deletions(-) diff --git a/apisix/admin/ssl.lua b/apisix/admin/ssl.lua index 341e03004d1a..820b84a29b04 100644 --- a/apisix/admin/ssl.lua +++ b/apisix/admin/ssl.lua @@ -46,7 +46,7 @@ local function check_conf(id, conf, need_id) conf.id = id core.log.info("schema: ", core.json.delay_encode(core.schema.ssl)) - core.log.info("conf : ", core.json.delay_encode(conf)) + core.log.info("conf : ", core.json.delay_encode(conf)) local ok, err = apisix_ssl.check_ssl_conf(false, conf) if not ok then diff --git a/apisix/init.lua b/apisix/init.lua index d68e31ba5666..e950ea9009ec 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -512,6 +512,22 @@ function _M.http_access_phase() return pubsub_kafka.access(api_ctx) end + if api_ctx.matched_upstream and api_ctx.matched_upstream.tls_id then + local tls_id = api_ctx.matched_upstream.tls_id + local upstream_ssl = router.router_ssl.get_by_id(tls_id) + if not upstream_ssl or upstream_ssl.type ~= 1 then + if is_http then + return core.response.exit(502) + end + + return ngx_exit(1) + end + + core.log.info("matched ssl: ", + core.json.delay_encode(upstream_ssl, true)) + api_ctx.upstream_ssl = upstream_ssl + end + local code, err = set_upstream(route, api_ctx) if code then core.log.error("failed to set upstream: ", err) diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index 767c2fa63503..8510247c39b6 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -401,6 +401,7 @@ local upstream_schema = { minimum = 0, }, timeout = timeout_def, + tls_id = id_schema, tls = { type = "object", properties = { @@ -505,6 +506,14 @@ local upstream_schema = { {required = {"type", "nodes"}}, {required = {"type", "service_name", "discovery_type"}}, }, + dependencies = { + tls = { + ["not"] = {required = {"tls_id"}} + }, + tls_id = { + ["not"] = {required = {"tls"}} + } + } } -- TODO: add more nginx variable support @@ -722,6 +731,13 @@ _M.ssl = { type = "object", properties = { id = id_schema, + type = { + description = "ssl certificate type, " .. + "0 to server certificate, " .. + "1 to client certificate for upstream", + type = "integer", + enum = {0, 1} + }, cert = certificate_scheme, key = private_key_schema, sni = { @@ -772,10 +788,20 @@ _M.ssl = { create_time = timestamp_def, update_time = timestamp_def }, - oneOf = { - {required = {"sni", "key", "cert"}}, - {required = {"snis", "key", "cert"}} + ["if"] = { + properties = { + type = { + enum = {0}, + }, + }, + }, + ["then"] = { + oneOf = { + {required = {"sni", "key", "cert"}}, + {required = {"snis", "key", "cert"}} + } }, + ["else"] = {required = {"key", "cert"}} } diff --git a/apisix/ssl.lua b/apisix/ssl.lua index c0c47aec07fd..41a131c5252e 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -197,6 +197,10 @@ function _M.check_ssl_conf(in_dp, conf) return nil, err end + if conf.type == 1 then + return true + end + local numcerts = conf.certs and #conf.certs or 0 local numkeys = conf.keys and #conf.keys or 0 if numcerts ~= numkeys then diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index 70ac0faa32d1..df0fe85db124 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -45,6 +45,7 @@ local function create_router(ssl_items) for _, ssl in config_util.iterate_values(ssl_items) do if ssl.value ~= nil and + (ssl.value.type == nil or ssl.value.type == 1) and (ssl.value.status == nil or ssl.value.status == 1) then -- compatible with old version local j = 0 @@ -261,4 +262,20 @@ function _M.init_worker() end +function _M.get_by_id(ssl_id) + local ssl + local ssls = core.config.fetch_created_obj("/ssl") + if ssls then + ssl = ssls:get(tostring(ssl_id)) + end + + if not ssl then + core.log.error("failed to find ssl by id: ", ssl_id) + return nil + end + + return ssl.value +end + + return _M diff --git a/apisix/upstream.lua b/apisix/upstream.lua index a0e963b44f8a..51ff9a381a2a 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -329,15 +329,26 @@ function _M.set_by_route(route, api_ctx) end local scheme = up_conf.scheme - if (scheme == "https" or scheme == "grpcs") and up_conf.tls then + if (scheme == "https" or scheme == "grpcs") and + (api_ctx.upstream_ssl or up_conf.tls) then + + local client_cert, client_key + if up_conf.tls then + client_cert = up_conf.tls.client_cert + client_key = up_conf.tls.client_key + else + client_cert = api_ctx.upstream_ssl.cert + client_key = api_ctx.upstream_ssl.key + end + -- the sni here is just for logging local sni = api_ctx.var.upstream_host - local cert, err = apisix_ssl.fetch_cert(sni, up_conf.tls.client_cert) + local cert, err = apisix_ssl.fetch_cert(sni, client_cert) if not ok then return 503, err end - local key, err = apisix_ssl.fetch_pkey(sni, up_conf.tls.client_key) + local key, err = apisix_ssl.fetch_pkey(sni, client_key) if not ok then return 503, err end @@ -415,6 +426,29 @@ local function check_upstream_conf(in_dp, conf) return false, "invalid configuration: " .. err end + local ssl_id = conf.tls_id + if ssl_id then + local key = "/ssl/" .. ssl_id + local res, err = core.etcd.get(key) + if not res then + return nil, "failed to fetch ssl info by " + .. "ssl id [" .. ssl_id .. "]: " .. err + end + + if res.status ~= 200 then + return nil, "failed to fetch ssl info by " + .. "ssl id [" .. ssl_id .. "], " + .. "response code: " .. res.status + end + if res.body and res.body.node and + res.body.node.value and res.body.node.value.type ~= 1 then + + return nil, "failed to fetch ssl info by " + .. "ssl id [" .. ssl_id .. "], " + .. "wrong ssl type" + end + end + -- encrypt the key in the admin if conf.tls and conf.tls.client_key then conf.tls.client_key = apisix_ssl.aes_encrypt_pkey(conf.tls.client_key) diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 9c7bec756f64..4bb5bd899e01 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -541,8 +541,9 @@ In addition to the equalization algorithm selections, Upstream also supports pas | labels | optional | Attributes of the Upstream specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | optional | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | optional | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| tls.client_cert | optional | Sets the client certificate while connecting to a TLS Upstream. | | -| tls.client_key | optional | Sets the client private key while connecting to a TLS Upstream. | | +| tls.client_cert | optional, can't be used with `tls_id` | Sets the client certificate while connecting to a TLS Upstream. | | +| tls.client_key | optional, can't be used with `tls_id` | Sets the client private key while connecting to a TLS Upstream. | | +| tls_id | optional, can't be used with `tls.client_cert` `tls.client_key` | Set the referenced [SSL](#ssl) id. | | | keepalive_pool.size | optional | Sets `keepalive` directive dynamically. | | | keepalive_pool.idle_timeout | optional | Sets `keepalive_timeout` directive dynamically. | | | keepalive_pool.requests | optional | Sets `keepalive_requests` directive dynamically. | | @@ -570,6 +571,8 @@ You can set the `scheme` to `tls`, which means "TLS over TCP". To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields. +Or you can reference SSL object by `tls_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is 1. Only `cert` and `key` will be used in the SSL object. + To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields. Example Configuration: @@ -789,6 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | +| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream ` 0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index 7cffc2adb904..acb165e9c422 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -549,8 +549,9 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | -| tls.client_cert | 可选 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | | -| tls.client_key | 可选 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | | +| tls.client_cert | 可选，不能和 `tls_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | | +| tls.client_key | 可选，不能和 `tls_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | | +| tls_id | 可选，不能和 `tls.client_cert` `tls.client_key` 一起使用 | SSL | 启用的 ssl id，详见 [SSL](#ssl) | | |keepalive_pool.size | 可选 | 辅助 | 动态设置 `keepalive` 指令，细节见下文 | |keepalive_pool.idle_timeout | 可选 | 辅助 | 动态设置 `keepalive_timeout` 指令，细节见下文 | |keepalive_pool.requests | 可选 | 辅助 | 动态设置 `keepalive_requests` 指令，细节见下文 | @@ -577,6 +578,7 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。 他们的格式和 SSL 对象的 `cert` 和 `key` 一样。 +`tls_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 1 时才能被引用。SSL 对象中只有 `cert`和`key` 会被使用。 `keepalive_pool` 允许 upstream 对象有自己单独的连接池。 它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。 @@ -799,6 +801,7 @@ $ curl http://127.0.0.1:9080/get | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | +| type | 可选 | 辅助 | 标识证书的类型。 | `1` 表示证书是客户端证书，APISIX 访问上游时使用 ` 0` 表示证书是服务端证书，APISIX 验证客户端请求时使用 | | status | 可选 | 辅助 | 是否启用此 SSL，缺省 `1`。 | `1` 表示启用，`0` 表示禁用 | ssl 对象 json 配置内容： diff --git a/t/admin/ssl.t b/t/admin/ssl.t index 49d3ec8cd146..3e6326f41ed6 100644 --- a/t/admin/ssl.t +++ b/t/admin/ssl.t @@ -236,7 +236,7 @@ GET /t GET /t --- error_code: 400 --- response_body -{"error_msg":"invalid configuration: value should match only one schema, but matches none"} +{"error_msg":"invalid configuration: then clause did not match"} --- no_error_log [error] @@ -535,7 +535,7 @@ passed GET /t --- error_code: 400 --- response_body -{"error_msg":"invalid configuration: value should match only one schema, but matches none"} +{"error_msg":"invalid configuration: then clause did not match"} --- no_error_log [error] @@ -771,3 +771,72 @@ GET /t passed --- no_error_log [error] + + + +=== TEST 20: missing sni information +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + + local ssl_cert = t.read_file("t/certs/apisix.crt") + local ssl_key = t.read_file("t/certs/apisix.key") + local data = {cert = ssl_cert, key = ssl_key} + + local code, body = t.test('/apisix/admin/ssl/1', + ngx.HTTP_PUT, + core.json.encode(data), + [[{ + "node": { + "key": "/apisix/ssl/1" + }, + "action": "set" + }]] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"invalid configuration: then clause did not match"} +--- no_error_log +[error] + + + +=== TEST 21: type 1, missing sni information +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + + local ssl_cert = t.read_file("t/certs/apisix.crt") + local ssl_key = t.read_file("t/certs/apisix.key") + local data = {type = 1, cert = ssl_cert, key = ssl_key} + + local code, body = t.test('/apisix/admin/ssl/1', + ngx.HTTP_PUT, + core.json.encode(data), + [[{ + "node": { + "key": "/apisix/ssl/1" + }, + "action": "set" + }]] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /t +--- response_body chomp +passed diff --git a/t/admin/upstream.t b/t/admin/upstream.t index 96dcef3c482f..d17259141536 100644 --- a/t/admin/upstream.t +++ b/t/admin/upstream.t @@ -627,3 +627,132 @@ GET /t {"error_msg":"wrong upstream id, do not need it"} --- no_error_log [error] + + + +=== TEST 19: tls and tls_id cannot appear at the same time +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + + local ssl_cert = t.read_file("t/certs/apisix.crt") + local ssl_key = t.read_file("t/certs/apisix.key") + local data = { + nodes = { + ["127.0.0.1:8080"] = 1 + }, + type = "roundrobin", + tls_id = 1, + tls = { + client_cert = ssl_cert, + client_key = ssl_key + } + } + local code, body = t.test('/apisix/admin/upstreams', + ngx.HTTP_POST, + core.json.encode(data) + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body eval +qr/{"error_msg":"invalid configuration: failed to validate dependent schema for \\\"tls_id|tls\\\": value wasn't supposed to match schema"}/ +--- no_error_log +[error] + + + +=== TEST 20: tls_id does not exist +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin") + + local data = { + nodes = { + ["127.0.0.1:8080"] = 1 + }, + type = "roundrobin", + tls_id = 9999999 + } + local code, body = t.test('/apisix/admin/upstreams', + ngx.HTTP_POST, + core.json.encode(data) + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"failed to fetch ssl info by ssl id [9999999], response code: 404"} +--- no_error_log +[error] + + + +=== TEST 20: tls_id exist with wrong ssl type +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local ssl_key = t.read_file("t/certs/mtls_client.key") + local data = { + sni = "test.com", + cert = ssl_cert, + key = ssl_key + } + local code, body = t.test('/apisix/admin/ssl/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.print(body) + return + end + + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls_id = 1 + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.print(body) + return + end + } + } +--- request +GET /t +--- error_code: 400 +--- response_body +{"error_msg":"failed to fetch ssl info by ssl id [1], wrong ssl type"} +--- no_error_log +[error] diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 7af6d2e61785..630478a918a4 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -545,3 +545,68 @@ GET /t GET /hello_chunked --- response_body hello world + + + +=== 13: get cert by tls_id +--- ONLY +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local ssl_key = t.read_file("t/certs/mtls_client.key") + local data = { + type = 1, + cert = ssl_cert, + key = ssl_key + } + local code, body = t.test('/apisix/admin/ssl/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + local data = { + upstream = { + scheme = "https", + type = "roundrobin", + nodes = { + ["127.0.0.1:1983"] = 1, + }, + tls_id = 1 + }, + uri = "/hello" + } + local code, body = t.test('/apisix/admin/routes/1', + ngx.HTTP_PUT, + json.encode(data) + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + } + } +--- request +GET /t + + + +=== TEST 14: hit +--- LAST +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello +--- response_body +hello world From b3081a49230c5b43ede019bd6d586a55732e33d2 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 14:52:05 +0800 Subject: [PATCH 02/15] fix lint --- apisix/ssl/router/radixtree_sni.lua | 1 + docs/en/latest/admin-api.md | 4 ++-- docs/zh/latest/admin-api.md | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index df0fe85db124..36a3274e403c 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -26,6 +26,7 @@ local error = error local str_find = core.string.find local str_gsub = string.gsub local str_lower = string.lower +local tostring = tostring local ssl_certificates local radixtree_router local radixtree_router_ver diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 4bb5bd899e01..a422dbe05911 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -543,7 +543,7 @@ In addition to the equalization algorithm selections, Upstream also supports pas | update_time | optional | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | | tls.client_cert | optional, can't be used with `tls_id` | Sets the client certificate while connecting to a TLS Upstream. | | | tls.client_key | optional, can't be used with `tls_id` | Sets the client private key while connecting to a TLS Upstream. | | -| tls_id | optional, can't be used with `tls.client_cert` `tls.client_key` | Set the referenced [SSL](#ssl) id. | | +| tls_id | optional, can't be used with `tls.client_cert` and `tls.client_key` | Set the referenced [SSL](#ssl) id. | | | keepalive_pool.size | optional | Sets `keepalive` directive dynamically. | | | keepalive_pool.idle_timeout | optional | Sets `keepalive_timeout` directive dynamically. | | | keepalive_pool.requests | optional | Sets `keepalive_requests` directive dynamically. | | @@ -792,7 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream ` 0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | +| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; ` 0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index acb165e9c422..aade400b5259 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -551,7 +551,7 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | tls.client_cert | 可选，不能和 `tls_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | | | tls.client_key | 可选，不能和 `tls_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | | -| tls_id | 可选，不能和 `tls.client_cert` `tls.client_key` 一起使用 | SSL | 启用的 ssl id，详见 [SSL](#ssl) | | +| tls_id | 可选，不能和 `tls.client_cert`、`tls.client_key` 一起使用 | SSL | 设置引用的 ssl id，详见 [SSL](#ssl) | | |keepalive_pool.size | 可选 | 辅助 | 动态设置 `keepalive` 指令，细节见下文 | |keepalive_pool.idle_timeout | 可选 | 辅助 | 动态设置 `keepalive_timeout` 指令，细节见下文 | |keepalive_pool.requests | 可选 | 辅助 | 动态设置 `keepalive_requests` 指令，细节见下文 | @@ -801,7 +801,7 @@ $ curl http://127.0.0.1:9080/get | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | -| type | 可选 | 辅助 | 标识证书的类型。 | `1` 表示证书是客户端证书，APISIX 访问上游时使用 ` 0` 表示证书是服务端证书，APISIX 验证客户端请求时使用 | +| type | 可选 | 辅助 | 标识证书的类型。 | `1` 表示证书是客户端证书，APISIX 访问上游时使用；`0` 表示证书是服务端证书，APISIX 验证客户端请求时使用 | | status | 可选 | 辅助 | 是否启用此 SSL，缺省 `1`。 | `1` 表示启用，`0` 表示禁用 | ssl 对象 json 配置内容： From 69735f35e7d2bef333ffc058060d833d2950c59e Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 15:00:45 +0800 Subject: [PATCH 03/15] fix lint --- docs/en/latest/admin-api.md | 2 +- t/node/upstream-mtls.t | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index a422dbe05911..67ce20e02e37 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -792,7 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; ` 0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | +| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 630478a918a4..9c75b20588d5 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -549,7 +549,6 @@ hello world === 13: get cert by tls_id ---- ONLY --- config location /t { content_by_lua_block { @@ -602,7 +601,6 @@ GET /t === TEST 14: hit ---- LAST --- upstream_server_config ssl_client_certificate ../../certs/mtls_ca.crt; ssl_verify_client on; From 9eeeb83120c416bc62b512a6651167076622b576 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 15:10:20 +0800 Subject: [PATCH 04/15] reindex test case --- t/admin/upstream.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/admin/upstream.t b/t/admin/upstream.t index d17259141536..699283706cc5 100644 --- a/t/admin/upstream.t +++ b/t/admin/upstream.t @@ -702,7 +702,7 @@ GET /t -=== TEST 20: tls_id exist with wrong ssl type +=== TEST 21: tls_id exist with wrong ssl type --- config location /t { content_by_lua_block { From 6b4ab9067c1cbdbad8181639aea0bf996ce444c5 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 15:32:56 +0800 Subject: [PATCH 05/15] reindex --- t/node/upstream-mtls.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 9c75b20588d5..89363b66e712 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -548,7 +548,7 @@ hello world -=== 13: get cert by tls_id +=== TEST 13: get cert by tls_id --- config location /t { content_by_lua_block { From 249fa7cdd8613c96e641e2462d49498b3b781217 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 23:08:46 +0800 Subject: [PATCH 06/15] change tls_id to tls.clent_cert_id --- apisix/admin/ssl.lua | 2 +- apisix/init.lua | 34 ++++++++++++++++++---------------- apisix/schema_def.lua | 33 ++++++++++++++++++--------------- apisix/ssl.lua | 2 +- apisix/upstream.lua | 15 +++++++-------- docs/en/latest/admin-api.md | 10 +++++----- docs/zh/latest/admin-api.md | 11 ++++++----- t/admin/ssl.t | 4 ++-- t/admin/ssl2.t | 8 ++++---- t/admin/upstream.t | 20 ++++++++++++-------- t/node/upstream-mtls.t | 8 +++++--- 11 files changed, 79 insertions(+), 68 deletions(-) diff --git a/apisix/admin/ssl.lua b/apisix/admin/ssl.lua index 820b84a29b04..9a73107c9f10 100644 --- a/apisix/admin/ssl.lua +++ b/apisix/admin/ssl.lua @@ -46,7 +46,7 @@ local function check_conf(id, conf, need_id) conf.id = id core.log.info("schema: ", core.json.delay_encode(core.schema.ssl)) - core.log.info("conf : ", core.json.delay_encode(conf)) + core.log.info("conf: ", core.json.delay_encode(conf)) local ok, err = apisix_ssl.check_ssl_conf(false, conf) if not ok then diff --git a/apisix/init.lua b/apisix/init.lua index e950ea9009ec..f1f251e84dba 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -495,6 +495,24 @@ function _M.http_access_phase() or route_val.upstream end + if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and + api_ctx.matched_upstream.tls.client_cert_id then + + local cert_id = api_ctx.matched_upstream.tls.client_cert_id + local upstream_ssl = router.router_ssl.get_by_id(cert_id) + if not upstream_ssl or upstream_ssl.type ~= "client" then + if is_http then + return core.response.exit(502) + end + + return ngx_exit(1) + end + + core.log.info("matched ssl: ", + core.json.delay_encode(upstream_ssl, true)) + api_ctx.upstream_ssl = upstream_ssl + end + if enable_websocket then api_ctx.var.upstream_upgrade = api_ctx.var.http_upgrade api_ctx.var.upstream_connection = api_ctx.var.http_connection @@ -512,22 +530,6 @@ function _M.http_access_phase() return pubsub_kafka.access(api_ctx) end - if api_ctx.matched_upstream and api_ctx.matched_upstream.tls_id then - local tls_id = api_ctx.matched_upstream.tls_id - local upstream_ssl = router.router_ssl.get_by_id(tls_id) - if not upstream_ssl or upstream_ssl.type ~= 1 then - if is_http then - return core.response.exit(502) - end - - return ngx_exit(1) - end - - core.log.info("matched ssl: ", - core.json.delay_encode(upstream_ssl, true)) - api_ctx.upstream_ssl = upstream_ssl - end - local code, err = set_upstream(route, api_ctx) if code then core.log.error("failed to set upstream: ", err) diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index 8510247c39b6..a05bb7ab6c10 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -405,6 +405,7 @@ local upstream_schema = { tls = { type = "object", properties = { + client_cert_id = id_schema, client_cert = certificate_scheme, client_key = private_key_schema, verify = { @@ -415,8 +416,17 @@ local upstream_schema = { }, }, dependencies = { - client_cert = {"client_key"}, - client_key = {"client_cert"}, + client_cert = { + required = {"client_key"}, + ["not"] = {required = {"client_cert_id"}} + }, + client_key = { + required = {"client_cert"}, + ["not"] = {required = {"client_cert_id"}} + }, + client_cert_id = { + ["not"] = {required = {"client_client", "client_key"}} + } } }, keepalive_pool = { @@ -505,14 +515,6 @@ local upstream_schema = { oneOf = { {required = {"type", "nodes"}}, {required = {"type", "service_name", "discovery_type"}}, - }, - dependencies = { - tls = { - ["not"] = {required = {"tls_id"}} - }, - tls_id = { - ["not"] = {required = {"tls"}} - } } } @@ -733,10 +735,11 @@ _M.ssl = { id = id_schema, type = { description = "ssl certificate type, " .. - "0 to server certificate, " .. - "1 to client certificate for upstream", - type = "integer", - enum = {0, 1} + "server to server certificate, " .. + "client to client certificate for upstream", + type = "string", + default = "server", + enum = {"server", "client"} }, cert = certificate_scheme, key = private_key_schema, @@ -791,7 +794,7 @@ _M.ssl = { ["if"] = { properties = { type = { - enum = {0}, + enum = {"server"}, }, }, }, diff --git a/apisix/ssl.lua b/apisix/ssl.lua index 41a131c5252e..7d48f308502e 100644 --- a/apisix/ssl.lua +++ b/apisix/ssl.lua @@ -197,7 +197,7 @@ function _M.check_ssl_conf(in_dp, conf) return nil, err end - if conf.type == 1 then + if conf.type == "client" then return true end diff --git a/apisix/upstream.lua b/apisix/upstream.lua index 51ff9a381a2a..b5514375479f 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -329,16 +329,15 @@ function _M.set_by_route(route, api_ctx) end local scheme = up_conf.scheme - if (scheme == "https" or scheme == "grpcs") and - (api_ctx.upstream_ssl or up_conf.tls) then + if (scheme == "https" or scheme == "grpcs") and up_conf.tls then local client_cert, client_key - if up_conf.tls then - client_cert = up_conf.tls.client_cert - client_key = up_conf.tls.client_key - else + if up_conf.tls.client_cert_id then client_cert = api_ctx.upstream_ssl.cert client_key = api_ctx.upstream_ssl.key + else + client_cert = up_conf.tls.client_cert + client_key = up_conf.tls.client_key end -- the sni here is just for logging @@ -426,7 +425,7 @@ local function check_upstream_conf(in_dp, conf) return false, "invalid configuration: " .. err end - local ssl_id = conf.tls_id + local ssl_id = conf.tls and conf.tls.client_cert_id or nil if ssl_id then local key = "/ssl/" .. ssl_id local res, err = core.etcd.get(key) @@ -441,7 +440,7 @@ local function check_upstream_conf(in_dp, conf) .. "response code: " .. res.status end if res.body and res.body.node and - res.body.node.value and res.body.node.value.type ~= 1 then + res.body.node.value and res.body.node.value.type ~= "client" then return nil, "failed to fetch ssl info by " .. "ssl id [" .. ssl_id .. "], " diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 67ce20e02e37..3464104d715d 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -541,9 +541,9 @@ In addition to the equalization algorithm selections, Upstream also supports pas | labels | optional | Attributes of the Upstream specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | optional | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | optional | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| tls.client_cert | optional, can't be used with `tls_id` | Sets the client certificate while connecting to a TLS Upstream. | | -| tls.client_key | optional, can't be used with `tls_id` | Sets the client private key while connecting to a TLS Upstream. | | -| tls_id | optional, can't be used with `tls.client_cert` and `tls.client_key` | Set the referenced [SSL](#ssl) id. | | +| tls.client_cert | optional, can't be used with `tls.client_cert_id` | Sets the client certificate while connecting to a TLS Upstream. | | +| tls.client_key | optional, can't be used with `tls.client_cert_id` | Sets the client private key while connecting to a TLS Upstream. | | +| tls.client_cert_id | optional, can't be used with `tls.client_cert` and `tls.client_key` | Set the referenced [SSL](#ssl) id. | | | keepalive_pool.size | optional | Sets `keepalive` directive dynamically. | | | keepalive_pool.idle_timeout | optional | Sets `keepalive_timeout` directive dynamically. | | | keepalive_pool.requests | optional | Sets `keepalive_requests` directive dynamically. | | @@ -571,7 +571,7 @@ You can set the `scheme` to `tls`, which means "TLS over TCP". To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields. -Or you can reference SSL object by `tls_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is 1. Only `cert` and `key` will be used in the SSL object. +Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is 1. Only `cert` and `key` will be used in the SSL object. To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields. @@ -792,7 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| type | False | Certificate type | Identifies the type of certificate. | `1` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `0` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | +| type | False | Certificate type | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index aade400b5259..e1d2139a3d5e 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -549,9 +549,9 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | -| tls.client_cert | 可选，不能和 `tls_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | | -| tls.client_key | 可选，不能和 `tls_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | | -| tls_id | 可选，不能和 `tls.client_cert`、`tls.client_key` 一起使用 | SSL | 设置引用的 ssl id，详见 [SSL](#ssl) | | +| tls.client_cert | 可选，不能和 `tls.client_cert_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | | +| tls.client_key | 可选，不能和 `tls.client_cert_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | | +| tls.client_cert_id | 可选，不能和 `tls.client_cert`、`tls.client_key` 一起使用 | SSL | 设置引用的 ssl id，详见 [SSL](#ssl) | | |keepalive_pool.size | 可选 | 辅助 | 动态设置 `keepalive` 指令，细节见下文 | |keepalive_pool.idle_timeout | 可选 | 辅助 | 动态设置 `keepalive_timeout` 指令，细节见下文 | |keepalive_pool.requests | 可选 | 辅助 | 动态设置 `keepalive_requests` 指令，细节见下文 | @@ -578,7 +578,8 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。 他们的格式和 SSL 对象的 `cert` 和 `key` 一样。 -`tls_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 1 时才能被引用。SSL 对象中只有 `cert`和`key` 会被使用。 + +`tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用。SSL 对象中只有 `cert`和`key` 会被使用。 `keepalive_pool` 允许 upstream 对象有自己单独的连接池。 它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。 @@ -801,7 +802,7 @@ $ curl http://127.0.0.1:9080/get | labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} | | create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | | update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 | -| type | 可选 | 辅助 | 标识证书的类型。 | `1` 表示证书是客户端证书，APISIX 访问上游时使用；`0` 表示证书是服务端证书，APISIX 验证客户端请求时使用 | +| type | 可选 | 辅助 | 标识证书的类型，缺省为 `server`。 | `client` 表示证书是客户端证书，APISIX 访问上游时使用；`server` 表示证书是服务端证书，APISIX 验证客户端请求时使用 | | status | 可选 | 辅助 | 是否启用此 SSL，缺省 `1`。 | `1` 表示启用，`0` 表示禁用 | ssl 对象 json 配置内容： diff --git a/t/admin/ssl.t b/t/admin/ssl.t index 3e6326f41ed6..0232d21102fe 100644 --- a/t/admin/ssl.t +++ b/t/admin/ssl.t @@ -810,7 +810,7 @@ GET /t -=== TEST 21: type 1, missing sni information +=== TEST 21: type client, missing sni information --- config location /t { content_by_lua_block { @@ -819,7 +819,7 @@ GET /t local ssl_cert = t.read_file("t/certs/apisix.crt") local ssl_key = t.read_file("t/certs/apisix.key") - local data = {type = 1, cert = ssl_cert, key = ssl_key} + local data = {type = "client", cert = ssl_cert, key = ssl_key} local code, body = t.test('/apisix/admin/ssl/1', ngx.HTTP_PUT, diff --git a/t/admin/ssl2.t b/t/admin/ssl2.t index 752e25cd7ba4..865652ce2e89 100644 --- a/t/admin/ssl2.t +++ b/t/admin/ssl2.t @@ -71,7 +71,7 @@ __DATA__ } } --- response_body -{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1}}} +{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1,"type":"server"}}} @@ -104,7 +104,7 @@ __DATA__ } } --- response_body -{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1}}} +{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1,"type":"server"}}} @@ -137,7 +137,7 @@ __DATA__ } } --- response_body -{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1}}} +{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1,"type":"server"}}} @@ -172,7 +172,7 @@ __DATA__ } } --- response_body -{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1}}} +{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1,"type":"server"}}} diff --git a/t/admin/upstream.t b/t/admin/upstream.t index 699283706cc5..16bfb5157b7b 100644 --- a/t/admin/upstream.t +++ b/t/admin/upstream.t @@ -630,7 +630,7 @@ GET /t -=== TEST 19: tls and tls_id cannot appear at the same time +=== TEST 19: client_cert/client_key and client_cert_id cannot appear at the same time --- config location /t { content_by_lua_block { @@ -644,8 +644,8 @@ GET /t ["127.0.0.1:8080"] = 1 }, type = "roundrobin", - tls_id = 1, tls = { + client_cert_id = 1, client_cert = ssl_cert, client_key = ssl_key } @@ -663,13 +663,13 @@ GET /t GET /t --- error_code: 400 --- response_body eval -qr/{"error_msg":"invalid configuration: failed to validate dependent schema for \\\"tls_id|tls\\\": value wasn't supposed to match schema"}/ +qr/{"error_msg":"invalid configuration: property \\\"tls\\\" validation failed: failed to validate dependent schema for \\\"client_cert|client_key\\\": value wasn't supposed to match schema"}/ --- no_error_log [error] -=== TEST 20: tls_id does not exist +=== TEST 20: tls.client_cert_id does not exist --- config location /t { content_by_lua_block { @@ -681,7 +681,9 @@ qr/{"error_msg":"invalid configuration: failed to validate dependent schema for ["127.0.0.1:8080"] = 1 }, type = "roundrobin", - tls_id = 9999999 + tls = { + client_cert_id = 9999999 + } } local code, body = t.test('/apisix/admin/upstreams', ngx.HTTP_POST, @@ -702,7 +704,7 @@ GET /t -=== TEST 21: tls_id exist with wrong ssl type +=== TEST 21: tls.client_cert_id exist with wrong ssl type --- config location /t { content_by_lua_block { @@ -731,9 +733,11 @@ GET /t scheme = "https", type = "roundrobin", nodes = { - ["127.0.0.1:1983"] = 1, + ["127.0.0.1:1983"] = 1 }, - tls_id = 1 + tls = { + client_cert_id = 1 + } }, uri = "/hello" } diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 89363b66e712..36b016ac671e 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -548,7 +548,7 @@ hello world -=== TEST 13: get cert by tls_id +=== TEST 13: get cert by tls.client_cert_id --- config location /t { content_by_lua_block { @@ -557,7 +557,7 @@ hello world local ssl_cert = t.read_file("t/certs/mtls_client.crt") local ssl_key = t.read_file("t/certs/mtls_client.key") local data = { - type = 1, + type = "client", cert = ssl_cert, key = ssl_key } @@ -579,7 +579,9 @@ hello world nodes = { ["127.0.0.1:1983"] = 1, }, - tls_id = 1 + tls = { + client_cert_id = 1 + } }, uri = "/hello" } From 8074d8f2709b9416a61f736e553a9b67f404ea96 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 23:12:40 +0800 Subject: [PATCH 07/15] remove tls_id --- apisix/schema_def.lua | 1 - apisix/ssl/router/radixtree_sni.lua | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apisix/schema_def.lua b/apisix/schema_def.lua index a05bb7ab6c10..7d39b62aad76 100644 --- a/apisix/schema_def.lua +++ b/apisix/schema_def.lua @@ -401,7 +401,6 @@ local upstream_schema = { minimum = 0, }, timeout = timeout_def, - tls_id = id_schema, tls = { type = "object", properties = { diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index 36a3274e403c..cca10089d8e8 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -45,8 +45,7 @@ local function create_router(ssl_items) local idx = 0 for _, ssl in config_util.iterate_values(ssl_items) do - if ssl.value ~= nil and - (ssl.value.type == nil or ssl.value.type == 1) and + if ssl.value ~= nil and ssl.value.type == "server" and (ssl.value.status == nil or ssl.value.status == 1) then -- compatible with old version local j = 0 From 8b54302271cbe492a796ede7ff3a24415b007b15 Mon Sep 17 00:00:00 2001 From: soulbird Date: Thu, 9 Jun 2022 23:17:47 +0800 Subject: [PATCH 08/15] fix doc --- docs/en/latest/admin-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 3464104d715d..230721323459 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -571,7 +571,7 @@ You can set the `scheme` to `tls`, which means "TLS over TCP". To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields. -Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is 1. Only `cert` and `key` will be used in the SSL object. +Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`. Only `cert` and `key` will be used in the SSL object. To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields. From 9ad13ccd246b3c6a0397718f9231793dcddbe3b8 Mon Sep 17 00:00:00 2001 From: soulbird Date: Fri, 10 Jun 2022 08:37:13 +0800 Subject: [PATCH 09/15] fix test case --- t/node/upstream-mtls.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 36b016ac671e..c5ca59024e16 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -77,7 +77,7 @@ __DATA__ GET /t --- error_code: 400 --- response_body -{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: property \"client_key\" is required when \"client_cert\" is set"} +{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: failed to validate dependent schema for \"client_cert\": property \"client_key\" is required"} From 7386efe38b899db7dcd75fbc115430d73a3faddf Mon Sep 17 00:00:00 2001 From: soulbird Date: Fri, 10 Jun 2022 14:26:26 +0800 Subject: [PATCH 10/15] add more test cases --- apisix/init.lua | 5 +++ apisix/ssl/router/radixtree_sni.lua | 1 - apisix/upstream.lua | 2 +- t/node/upstream-mtls.t | 68 +++++++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 2 deletions(-) diff --git a/apisix/init.lua b/apisix/init.lua index f1f251e84dba..74184b018175 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -501,6 +501,11 @@ function _M.http_access_phase() local cert_id = api_ctx.matched_upstream.tls.client_cert_id local upstream_ssl = router.router_ssl.get_by_id(cert_id) if not upstream_ssl or upstream_ssl.type ~= "client" then + local err = upstream_ssl and + "ssl id [" .. cert_id .. "] not exits" or + "ssl type should be 'client'" + core.log.error("failed to get ssl cert, ", err) + if is_http then return core.response.exit(502) end diff --git a/apisix/ssl/router/radixtree_sni.lua b/apisix/ssl/router/radixtree_sni.lua index cca10089d8e8..891d8d21dd4c 100644 --- a/apisix/ssl/router/radixtree_sni.lua +++ b/apisix/ssl/router/radixtree_sni.lua @@ -270,7 +270,6 @@ function _M.get_by_id(ssl_id) end if not ssl then - core.log.error("failed to find ssl by id: ", ssl_id) return nil end diff --git a/apisix/upstream.lua b/apisix/upstream.lua index b5514375479f..4de23de3c915 100644 --- a/apisix/upstream.lua +++ b/apisix/upstream.lua @@ -425,7 +425,7 @@ local function check_upstream_conf(in_dp, conf) return false, "invalid configuration: " .. err end - local ssl_id = conf.tls and conf.tls.client_cert_id or nil + local ssl_id = conf.tls and conf.tls.client_cert_id if ssl_id then local key = "/ssl/" .. ssl_id local res, err = core.etcd.get(key) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index c5ca59024e16..8ad7506bf4ac 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -549,11 +549,13 @@ hello world === TEST 13: get cert by tls.client_cert_id +--- FIRST --- config location /t { content_by_lua_block { local t = require("lib.test_admin") local json = require("toolkit.json") + local ssl_cert = t.read_file("t/certs/mtls_client.crt") local ssl_key = t.read_file("t/certs/mtls_client.key") local data = { @@ -610,3 +612,69 @@ GET /t GET /hello --- response_body hello world + + + +=== TEST 15: change ssl object type +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local ssl_cert = t.read_file("t/certs/mtls_client.crt") + local ssl_key = t.read_file("t/certs/mtls_client.key") + local data = { + type = "server", + cert = ssl_cert, + key = ssl_key + } + local code, body = t.test('/apisix/admin/ssl/1', + ngx.HTTP_PUT, + json.encode(data) + ) + } + } +--- request +GET /t + + + +=== TEST 16: hit, ssl object type mismatch +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello +--- error_code: 502 + + + +=== TEST 17: delete ssl object +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local code, body = t.test('/apisix/admin/ssl/1', ngx.HTTP_DELETE) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + } + } +--- request +GET /t + + + +=== TEST 18: hit, ssl object not exits +--- upstream_server_config + ssl_client_certificate ../../certs/mtls_ca.crt; + ssl_verify_client on; +--- request +GET /hello +--- error_code: 502 From 94de007636e1ab3a62976f5c6ba43df852aa1053 Mon Sep 17 00:00:00 2001 From: soulbird Date: Fri, 10 Jun 2022 14:30:31 +0800 Subject: [PATCH 11/15] remove FIRST --- t/node/upstream-mtls.t | 1 - 1 file changed, 1 deletion(-) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index 8ad7506bf4ac..bba8b6ed254e 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -549,7 +549,6 @@ hello world === TEST 13: get cert by tls.client_cert_id ---- FIRST --- config location /t { content_by_lua_block { From e2bc606c5a2571fef0581634b216319681e07e81 Mon Sep 17 00:00:00 2001 From: soulbird Date: Fri, 10 Jun 2022 16:26:42 +0800 Subject: [PATCH 12/15] fix test cases --- apisix/init.lua | 6 +++--- t/node/upstream-mtls.t | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/apisix/init.lua b/apisix/init.lua index 74184b018175..f60511686c4d 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -502,9 +502,9 @@ function _M.http_access_phase() local upstream_ssl = router.router_ssl.get_by_id(cert_id) if not upstream_ssl or upstream_ssl.type ~= "client" then local err = upstream_ssl and - "ssl id [" .. cert_id .. "] not exits" or - "ssl type should be 'client'" - core.log.error("failed to get ssl cert, ", err) + "ssl type should be 'client'" or + "ssl id [" .. cert_id .. "] not exits" + core.log.error("failed to get ssl cert: ", err) if is_http then return core.response.exit(502) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index bba8b6ed254e..e592dba0dd0a 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -625,6 +625,7 @@ hello world local ssl_key = t.read_file("t/certs/mtls_client.key") local data = { type = "server", + sni = "test.com", cert = ssl_cert, key = ssl_key } @@ -632,6 +633,11 @@ hello world ngx.HTTP_PUT, json.encode(data) ) + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end } } --- request @@ -646,7 +652,8 @@ GET /t --- request GET /hello --- error_code: 502 - +--- error_log +failed to get ssl cert: ssl type should be 'client' === TEST 17: delete ssl object @@ -677,3 +684,6 @@ GET /t --- request GET /hello --- error_code: 502 +--- error_log +failed to get ssl cert: ssl id [1] not exits + From f110ee7a0a57a8b9201438901e70d94589598a7d Mon Sep 17 00:00:00 2001 From: soulbird Date: Fri, 10 Jun 2022 16:41:32 +0800 Subject: [PATCH 13/15] fix lint --- t/node/upstream-mtls.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/node/upstream-mtls.t b/t/node/upstream-mtls.t index e592dba0dd0a..c909dbc9a64f 100644 --- a/t/node/upstream-mtls.t +++ b/t/node/upstream-mtls.t @@ -656,6 +656,7 @@ GET /hello failed to get ssl cert: ssl type should be 'client' + === TEST 17: delete ssl object --- config location /t { @@ -686,4 +687,3 @@ GET /hello --- error_code: 502 --- error_log failed to get ssl cert: ssl id [1] not exits - From 98b3ff47e0e56ebc3b952cdf25df55799585ce37 Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 13 Jun 2022 13:17:00 +0800 Subject: [PATCH 14/15] fix doc --- docs/en/latest/admin-api.md | 4 ++-- docs/zh/latest/admin-api.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 230721323459..9511127b6126 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -571,7 +571,7 @@ You can set the `scheme` to `tls`, which means "TLS over TCP". To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields. -Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`. Only `cert` and `key` will be used in the SSL object. +Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`, otherwise the request will be rejected by APISIX. In addition, only `cert` and `key` will be used in the SSL object. To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields. @@ -792,7 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| type | False | Certificate type | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | +| type | False | Certificate position | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: diff --git a/docs/zh/latest/admin-api.md b/docs/zh/latest/admin-api.md index e1d2139a3d5e..367c318ac449 100644 --- a/docs/zh/latest/admin-api.md +++ b/docs/zh/latest/admin-api.md @@ -579,7 +579,7 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上 `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。 他们的格式和 SSL 对象的 `cert` 和 `key` 一样。 -`tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用。SSL 对象中只有 `cert`和`key` 会被使用。 +`tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用，否则请求会被 APISIX 拒绝。另外，SSL 对象中只有 `cert`和`key` 会被使用。 `keepalive_pool` 允许 upstream 对象有自己单独的连接池。 它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。 From b7570c9d439539f40dd18d194a34013e6cead09f Mon Sep 17 00:00:00 2001 From: soulbird Date: Mon, 13 Jun 2022 15:26:50 +0800 Subject: [PATCH 15/15] fix doc --- docs/en/latest/admin-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/latest/admin-api.md b/docs/en/latest/admin-api.md index 9511127b6126..d78eb0d6e512 100644 --- a/docs/en/latest/admin-api.md +++ b/docs/en/latest/admin-api.md @@ -792,7 +792,7 @@ Currently, the response is returned from etcd. | labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} | | create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 | | update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 | -| type | False | Certificate position | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | +| type | False | Auxiliary | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. | | status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable | Example Configuration: