feat(stream): accept tls over tcp

Signed-off-by: spacewander <[email protected]>

README.md

@@ -74,7 +74,7 @@ A/B testing, canary release, blue-green deployment, limit rate, defense against
   - Proxy Protocol
   - Proxy Dubbo: Dubbo Proxy based on Tengine.
   - HTTP(S) Forward Proxy
-  - [SSL](docs/en/latest/https.md): Dynamically load an SSL certificate.
+  - [SSL](docs/en/latest/certificate.md): Dynamically load an SSL certificate.
 
 - **Full Dynamic**
 

apisix/cli/ngx_tpl.lua

@@ -99,13 +99,22 @@ stream {
     }
 
     server {
-        {% for _, addr in ipairs(stream_proxy.tcp or {}) do %}
-        listen {*addr*} {% if enable_reuseport then %} reuseport {% end %} {% if proxy_protocol and proxy_protocol.enable_tcp_pp then %} proxy_protocol {% end %};
+        {% for _, item in ipairs(stream_proxy.tcp or {}) do %}
+        listen {*item.addr*} {% if item.tls then %} ssl {% end %} {% if enable_reuseport then %} reuseport {% end %} {% if proxy_protocol and proxy_protocol.enable_tcp_pp then %} proxy_protocol {% end %};
         {% end %}
         {% for _, addr in ipairs(stream_proxy.udp or {}) do %}
         listen {*addr*} udp {% if enable_reuseport then %} reuseport {% end %};
         {% end %}
 
+        {% if tcp_enable_ssl then %}
+        ssl_certificate      {* ssl.ssl_cert *};
+        ssl_certificate_key  {* ssl.ssl_cert_key *};
+
+        ssl_certificate_by_lua_block {
+            apisix.stream_ssl_phase()
+        }
+        {% end %}
+
         {% if proxy_protocol and proxy_protocol.enable_tcp_pp_to_upstream then %}
         proxy_protocol on;
         {% end %}

apisix/cli/ops.lua

@@ -192,6 +192,25 @@ local config_schema = {
                                     {
                                         type = "string",
                                     },
+                                    {
+                                        type = "object",
+                                        properties = {
+                                            addr = {
+                                                anyOf = {
+                                                    {
+                                                        type = "integer",
+                                                    },
+                                                    {
+                                                        type = "string",
+                                                    },
+                                                }
+                                            },
+                                            tls = {
+                                                type = "boolean",
+                                            }
+                                        },
+                                        required = {"addr"}
+                                    },
                                 },
                             },
                             uniqueItems = true,
@@ -430,6 +449,21 @@ Please modify "admin_key" in conf/config.yaml .
     yaml_conf.apisix.ssl.ssl_cert = "cert/ssl_PLACE_HOLDER.crt"
     yaml_conf.apisix.ssl.ssl_cert_key = "cert/ssl_PLACE_HOLDER.key"
 
+    local tcp_enable_ssl
+    -- compatible with the original style which only has the addr
+    if yaml_conf.apisix.stream_proxy and yaml_conf.apisix.stream_proxy.tcp then
+        local tcp = yaml_conf.apisix.stream_proxy.tcp
+        for i, item in ipairs(tcp) do
+            if type(item) ~= "table" then
+                tcp[i] = {addr = item}
+            else
+                if item.tls then
+                    tcp_enable_ssl = true
+                end
+            end
+        end
+    end
+
     local dubbo_upstream_multiplex_count = 32
     if yaml_conf.plugin_attr and yaml_conf.plugin_attr["dubbo-proxy"] then
         local dubbo_conf = yaml_conf.plugin_attr["dubbo-proxy"]
@@ -454,6 +488,7 @@ Please modify "admin_key" in conf/config.yaml .
         error_log = {level = "warn"},
         enabled_plugins = enabled_plugins,
         dubbo_upstream_multiplex_count = dubbo_upstream_multiplex_count,
+        tcp_enable_ssl = tcp_enable_ssl,
     }
 
     if not yaml_conf.apisix then

apisix/init.lua

@@ -736,6 +736,25 @@ function _M.http_control()
 end
 
 
+function _M.stream_ssl_phase()
+    local ngx_ctx = ngx.ctx
+    local api_ctx = ngx_ctx.api_ctx
+
+    if api_ctx == nil then
+        api_ctx = core.tablepool.fetch("api_ctx", 0, 32)
+        ngx_ctx.api_ctx = api_ctx
+    end
+
+    local ok, err = router.router_ssl.match_and_set(api_ctx)
+    if not ok then
+        if err then
+            core.log.error("failed to fetch ssl config: ", err)
+        end
+        ngx_exit(-1)
+    end
+end
+
+
 function _M.stream_init(args)
     core.log.info("enter stream_init")
 

apisix/router.lua

@@ -106,9 +106,15 @@ end
 
 
 function _M.stream_init_worker()
+    local router_ssl_name = "radixtree_sni"
+
     local router_stream = require("apisix.stream.router.ip_port")
     router_stream.stream_init_worker(filter)
     _M.router_stream = router_stream
+
+    local router_ssl = require("apisix.ssl.router." .. router_ssl_name)
+    router_ssl.init_worker()
+    _M.router_ssl = router_ssl
 end
 
 

ci/linux_apisix_current_luarocks_runner.sh

@@ -58,6 +58,9 @@ script() {
     # apisix cli test
     ./utils/set-dns.sh
 
+    # install test dependencies
+    sudo pip install requests
+
     for f in ./t/cli/test_*.sh; do
         sudo PATH="$PATH" "$f"
     done

conf/config-default.yaml

@@ -98,8 +98,9 @@ apisix:
     ssl: 'radixtree_sni'          # radixtree_sni: match route by SNI(base on radixtree)
   #stream_proxy:                  # TCP/UDP proxy
   #  tcp:                         # TCP proxy port list
-  #    - 9100
-  #    - "127.0.0.1:9101"
+  #    - addr: 9100
+  #      tls: true
+  #    - addr: "127.0.0.1:9101"
   #  udp:                         # UDP proxy port list
   #    - 9200
   #    - "127.0.0.1:9201"

docs/ar/README.md

@@ -79,7 +79,7 @@
   - بروتوكول الوكيل
   - الوكيل  Dubbo: Dubbo يعتمد على  Tengine.
   - HTTP(S)	وكيل إعادة التوجيه
-  - [SSL](docs/en/latest/https.md): تحميل شهادة SSL ديناميكيًا.
+  - [SSL](docs/en/latest/certificate.md): تحميل شهادة SSL ديناميكيًا.
 
 - **ديناميكية كاملة**
 

docs/en/latest/admin-api.md

@@ -812,7 +812,7 @@ Config Example:
 }
 ```
 
-More examples can be found in [HTTPS](./https.md).
+More examples can be found in [Certificate](./certificate.md).
 
 ## Global Rule
 

docs/en/latest/https.md → docs/en/latest/certificate.md

@@ -1,5 +1,5 @@
 ---
-title: HTTPS
+title: Certificate
 ---
 
 <!--

docs/en/latest/config.json

@@ -185,7 +185,7 @@
         },
         {
           "type": "doc",
-          "id": "https"
+          "id": "certificate"
         },
         {
           "type": "doc",

docs/en/latest/grpc-proxy.md

@@ -36,7 +36,7 @@ gRPC client -> APISIX -> gRPC/gRPCS server
 Here's an example, to proxying gRPC service by specified route:
 
 * attention: the `scheme` of the route's upstream must be `grpc` or `grpcs`.
-* attention: APISIX use TLS‑encrypted HTTP/2 to expose gRPC service, so need to [config SSL certificate](https.md)
+* attention: APISIX use TLS‑encrypted HTTP/2 to expose gRPC service, so need to [config SSL certificate](certificate.md)
 * attention: APISIX also support to expose gRPC service with plaintext HTTP/2, which does not rely on TLS, usually used to proxy gRPC service in intranet environment
 * the grpc server example：[grpc_server_example](https://github.com/iresty/grpc_server_example)
 

docs/en/latest/stream-proxy.md

@@ -23,12 +23,12 @@ title: Stream Proxy
 
 TCP is the protocol for many popular applications and services, such as LDAP, MySQL, and RTMP. UDP (User Datagram Protocol) is the protocol for many popular non-transactional applications, such as DNS, syslog, and RADIUS.
 
-APISIX can dynamic load balancing TCP/UDP proxy. In Nginx world, we call TCP/UDP proxy to stream proxy, we followed this statement.
+APISIX can dynamically load balancing TCP/UDP proxy. In Nginx world, we call TCP/UDP proxy to stream proxy, we followed this statement.
 
 ## How to enable stream proxy?
 
 Setting the `stream_proxy` option in `conf/config.yaml`, specify a list of addresses that require dynamic proxy.
-By default, no any stream proxy is enabled.
+By default, no stream proxy is enabled.
 
 ```yaml
 apisix:
@@ -82,6 +82,38 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
 }'
 ```
 
-It means APISIX will proxy the request to `127.0.0.1:1995` which the server address is `127.0.0.1` and the server port is equal `2000`.
+It means APISIX will proxy the request to `127.0.0.1:1995` which the server address is `127.0.0.1` and the server port is equal to `2000`.
 
 Read [Admin API's Stream Route section](./admin-api.md#stream-route) for the complete options list.
+
+## Accept TLS over TCP
+
+APISIX can accept TLS over TCP.
+
+First of all, we need to enable TLS for the TCP address:
+
+```yaml
+apisix:
+  stream_proxy: # TCP/UDP proxy
+    tcp: # TCP proxy address list
+      - addr: 9100
+        tls: true
+```
+
+Second, we need to configure certificate for the given SNI.
+See [Admin API's SSL section](./admin-api.md#ssl) for how to do.
+
+Third, we need to configure a stream route to match and proxy it to the upstream:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "remote_addr": "127.0.0.1",
+    "upstream": {
+        "nodes": {
+            "127.0.0.1:1995": 1
+        },
+        "type": "roundrobin"
+    }
+}'
+```

docs/es/latest/README.md

@@ -76,7 +76,7 @@ ensayos A/B, ensayos de despliegue de canarios (canary release), despliegue azul
   - Proxy de Protocolo
   - Proxy Dubbo: Proxy de Dubbo basado en Tengine.
   - Proxy de HTTP(S) hacia adelante
-  - [SSL](../../en/latest/https.md): Carga dinámica de certificado SSL.
+  - [SSL](../../en/latest/certificate.md): Carga dinámica de certificado SSL.
 
 - **Completamente Dinámico**
 

docs/zh/latest/README.md

@@ -78,7 +78,7 @@ A/B 测试、金丝雀发布(灰度发布)、蓝绿部署、限流限速、抵
   - Proxy Protocol
   - Dubbo 代理：基于 Tengine，可以实现 Dubbo 请求的代理。
   - HTTP(S) 反向代理
-  - [SSL](https.md)：动态加载 SSL 证书。
+  - [SSL](certificate.md)：动态加载 SSL 证书。
 
 - **全动态能力**
 

docs/zh/latest/admin-api.md

@@ -815,7 +815,7 @@ ssl 对象 json 配置内容：
 }
 ```
 
-更多的配置示例见 [HTTPS](./https.md)。
+更多的配置示例见 [证书](./certificate.md)。
 
 [Back to TOC](#目录)
 

docs/zh/latest/https.md → docs/zh/latest/certificate.md

@@ -1,5 +1,5 @@
 ---
-title: HTTPS
+title: 证书
 ---
 
 <!--

docs/zh/latest/config.json

@@ -168,7 +168,7 @@
         },
         {
           "type": "doc",
-          "id": "https"
+          "id": "certificate"
         },
         {
           "type": "doc",

docs/zh/latest/grpc-proxy.md

@@ -35,7 +35,7 @@ title: gRPC 代理
 在指定 Route 中，代理 gRPC 服务接口:
 
 * 注意：这个 Route 对应的 Upstream 的 `scheme` 必须设置为 `grpc` 或者 `grpcs`。
-* 注意： APISIX 使用 TLS 加密的 HTTP/2 暴露 gRPC 服务, 所以需要先 [配置 SSL 证书](https.md)；
+* 注意： APISIX 使用 TLS 加密的 HTTP/2 暴露 gRPC 服务, 所以需要先 [配置 SSL 证书](certificate.md)；
 * 注意： APISIX 也支持通过纯文本的 HTTP/2 暴露 gRPC 服务，这不需要依赖 SSL，通常用于内网环境代理gRPC服务
 * 下面例子所代理的 gRPC 服务可供参考：[grpc_server_example](https://github.com/api7/grpc_server_example)。
 

docs/zh/latest/stream-proxy.md

@@ -83,3 +83,35 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
 例子中 APISIX 会把服务器地址为 `127.0.0.1`, 端口为 `2000` 代理到上游地址 `127.0.0.1:1995`。
 
 完整的匹配选项列表参见 [Admin API 的 Stream Route](./admin-api.md#stream-route)。
+
+## 接收 TLS over TCP
+
+APISIX 支持接收 TLS over TCP.
+
+首先，我们需要给对应的 TCP 地址启用 TLS：
+
+```yaml
+apisix:
+  stream_proxy: # TCP/UDP proxy
+    tcp: # TCP proxy address list
+      - addr: 9100
+        tls: true
+```
+
+接着，我们需要为给定的 SNI 配置证书。
+具体步骤参考 [Admin API 的 SSL](./admin-api.md#ssl)。
+
+然后，我们需要配置一个 route，匹配连接并代理到上游：
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "remote_addr": "127.0.0.1",
+    "upstream": {
+        "nodes": {
+            "127.0.0.1:1995": 1
+        },
+        "type": "roundrobin"
+    }
+}'
+```