From 8057b979365cfa58e153c7cd80113fd77877f47b Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Sun, 28 Jan 2024 17:17:51 +0800 Subject: [PATCH 01/12] elasticsearch-logger && loggly --- apisix/plugins/elasticsearch-logger.lua | 23 +++++- .../en/latest/plugins/elasticsearch-logger.md | 5 ++ .../zh/latest/plugins/elasticsearch-logger.md | 4 + docs/zh/latest/plugins/loggly.md | 21 ++--- t/plugin/elasticsearch-logger.t | 81 +++++++++++++++++++ 5 files changed, 123 insertions(+), 11 deletions(-) diff --git a/apisix/plugins/elasticsearch-logger.lua b/apisix/plugins/elasticsearch-logger.lua index de29da2e0184..119046532e7d 100644 --- a/apisix/plugins/elasticsearch-logger.lua +++ b/apisix/plugins/elasticsearch-logger.lua @@ -75,7 +75,23 @@ local schema = { ssl_verify = { type = "boolean", default = true - } + }, + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, }, encrypt_fields = {"auth.password"}, oneOf = { @@ -168,6 +184,11 @@ local function send_to_elasticsearch(conf, entries) end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end + + function _M.log(conf, ctx) local entry = get_logger_entry(conf, ctx) diff --git a/docs/en/latest/plugins/elasticsearch-logger.md b/docs/en/latest/plugins/elasticsearch-logger.md index 06f70354f844..2d98a0c38064 100644 --- a/docs/en/latest/plugins/elasticsearch-logger.md +++ b/docs/en/latest/plugins/elasticsearch-logger.md @@ -48,6 +48,11 @@ When the Plugin is enabled, APISIX will serialize the request context informatio | auth.password | string | True | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) password. | | ssl_verify | boolean | False | true | When set to `true` enables SSL verification as per [OpenResty docs](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake). | | timeout | integer | False | 10 | Elasticsearch send data timeout in seconds. | +| include_req_body | boolean | False | false | When set to `true` includes the request body in the log. If the request body is too big to be kept in the memory, it can't be logged due to Nginx's limitations. | +| include_req_body_expr | array | False | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | + NOTE: `encrypt_fields = {"auth.password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/zh/latest/plugins/elasticsearch-logger.md b/docs/zh/latest/plugins/elasticsearch-logger.md index d97311b17ffb..7b0022e29ef0 100644 --- a/docs/zh/latest/plugins/elasticsearch-logger.md +++ b/docs/zh/latest/plugins/elasticsearch-logger.md @@ -49,6 +49,10 @@ description: 本文介绍了 API 网关 Apache APISIX 的 elasticsearch-logger | auth.password | string | 是 | | Elasticsearch [authentication](https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-up-authentication.html) 密码。 | | ssl_verify | boolean | 否 | true | 当设置为 `true` 时则启用 SSL 验证。更多信息请参考 [lua-nginx-module](https://github.com/openresty/lua-nginx-module#tcpsocksslhandshake)。 | | timeout | integer | 否 | 10 | 发送给 Elasticsearch 请求超时时间。 | +| include_req_body | boolean | 否 | false | 当设置为 `true` 时,包含请求体。**注意**:如果请求体无法完全存放在内存中,由于 NGINX 的限制,APISIX 无法将它记录下来。 | +| include_req_body_expr | array | 否 | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | 当设置为 `true` 时,包含响应体。 | +| include_resp_body_expr | array | 否 | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 注意:schema 中还定义了 `encrypt_fields = {"auth.password"}`,这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。 diff --git a/docs/zh/latest/plugins/loggly.md b/docs/zh/latest/plugins/loggly.md index 27d813c4a9bd..e44cfd42815f 100644 --- a/docs/zh/latest/plugins/loggly.md +++ b/docs/zh/latest/plugins/loggly.md @@ -35,16 +35,17 @@ description: API 网关 Apache APISIX loggly 插件可用于将日志转发到 S ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 描述 | -|------------------------|---------------|----------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------| -| customer_token | string | 是 | | 将日志发送到 Loggly 时使用的唯一标识符,以确保将日志发送到正确的组织帐户。 | -| severity | string (enum) | 否 | INFO | Syslog 日志事件的严重性级别。包括:`DEBUG`、`INFO`、`NOTICE`、`WARNING`、`ERR`、`CRIT`、`ALERT` 和 `EMEGR`。 | -| severity_map | object | 否 | nil | 一种将上游 HTTP 响应代码映射到 Syslog 中的方法。 `key-value`,其中 `key` 是 HTTP 响应代码,`value`是 Syslog 严重级别。例如`{"410": "CRIT"}`。 | -| tags | array | 否 | | 元数据将包含在任何事件日志中,以帮助进行分段和过滤。 | -| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| include_req_body | boolean | 否 | false | 当设置为 `true` 时,包含请求体。**注意**:如果请求体无法完全存放在内存中,由于 NGINX 的限制,APISIX 无法将它记录下来。 | -| include_resp_body | boolean | 否 | false | 当设置为 `true` 时,包含响应体。 | -| include_resp_body_expr | array | 否 | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | +| 名称 | 类型 | 必选项 | 默认值 | 描述 | +|-------------------------|---------------|-----|-------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| customer_token | string | 是 | | 将日志发送到 Loggly 时使用的唯一标识符,以确保将日志发送到正确的组织帐户。 | +| severity | string (enum) | 否 | INFO | Syslog 日志事件的严重性级别。包括:`DEBUG`、`INFO`、`NOTICE`、`WARNING`、`ERR`、`CRIT`、`ALERT` 和 `EMEGR`。 | +| severity_map | object | 否 | nil | 一种将上游 HTTP 响应代码映射到 Syslog 中的方法。 `key-value`,其中 `key` 是 HTTP 响应代码,`value`是 Syslog 严重级别。例如`{"410": "CRIT"}`。 | +| tags | array | 否 | | 元数据将包含在任何事件日志中,以帮助进行分段和过滤。 | +| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| include_req_body | boolean | 否 | false | 当设置为 `true` 时,包含请求体。**注意**:如果请求体无法完全存放在内存中,由于 NGINX 的限制,APISIX 无法将它记录下来。 | +| include_req_body_expr | array | 否 | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | 当设置为 `true` 时,包含响应体。 | +| include_resp_body_expr | array | 否 | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志或数据)。这样可以避免插件频繁地提交数据,默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/t/plugin/elasticsearch-logger.t b/t/plugin/elasticsearch-logger.t index 1706f5c289cd..ef2434d2a293 100644 --- a/t/plugin/elasticsearch-logger.t +++ b/t/plugin/elasticsearch-logger.t @@ -668,3 +668,84 @@ hello world --- wait: 2 --- error_log check elasticsearch custom body success + + + +=== TEST 17: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/elasticsearch-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, { + uri = "/hello", + upstream = { + type = "roundrobin", + nodes = { + ["127.0.0.1:1980"] = 1 + } + }, + plugins = { + ["elasticsearch-logger"] = { + endpoint_addr = "http://127.0.0.1:9201", + field = { + index = "services" + }, + batch_max_size = 1, + inactive_timeout = 1, + include_req_body = true + } + } + }) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 18: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/elasticsearch-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, { + uri = "/hello", + upstream = { + type = "roundrobin", + nodes = { + ["127.0.0.1:1980"] = 1 + } + }, + plugins = { + ["elasticsearch-logger"] = { + endpoint_addr = "http://127.0.0.1:9201", + field = { + index = "services" + }, + batch_max_size = 1, + inactive_timeout = 1, + include_req_body = true, + include_resp_body = true + } + } + }) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +"body":"hello world\n" From fd77c7633653f0a2f8d7f4284066fc4456f23302 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Sun, 28 Jan 2024 17:50:34 +0800 Subject: [PATCH 02/12] skywalking-logger --- apisix/plugins/skywalking-logger.lua | 22 +++++- docs/en/latest/plugins/skywalking-logger.md | 21 ++--- docs/zh/latest/plugins/skywalking-logger.md | 21 ++--- t/plugin/skywalking-logger.t | 87 +++++++++++++++++++++ 4 files changed, 132 insertions(+), 19 deletions(-) diff --git a/apisix/plugins/skywalking-logger.lua b/apisix/plugins/skywalking-logger.lua index 136b9dec1b34..32147e4c0ecb 100644 --- a/apisix/plugins/skywalking-logger.lua +++ b/apisix/plugins/skywalking-logger.lua @@ -39,6 +39,21 @@ local schema = { log_format = {type = "object"}, timeout = {type = "integer", minimum = 1, default = 3}, include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, }, required = {"endpoint_addr"}, } @@ -78,7 +93,7 @@ local function send_http_data(conf, log_message) local host = url_decoded.host local port = url_decoded.port - core.log.info("sending a batch logs to ", conf.endpoint_addr) + core.log.info("sending a batch logs to ", conf.endpoint_addr, ", log_message: ", log_message) local httpc = http.new() httpc:set_timeout(conf.timeout * 1000) @@ -116,6 +131,11 @@ local function send_http_data(conf, log_message) end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end + + function _M.log(conf, ctx) local log_body = log_util.get_log_entry(plugin_name, conf, ctx) local trace_context diff --git a/docs/en/latest/plugins/skywalking-logger.md b/docs/en/latest/plugins/skywalking-logger.md index b72ec5577e62..f1592e4ece53 100644 --- a/docs/en/latest/plugins/skywalking-logger.md +++ b/docs/en/latest/plugins/skywalking-logger.md @@ -35,15 +35,18 @@ If there is an existing tracing context, it sets up the trace-log correlation au ## Attributes -| Name | Type | Required | Default | Valid values | Description | -|-----------------------|---------|----------|------------------------|---------------|--------------------------------------------------------------------------------------------------------------| -| endpoint_addr | string | True | | | URI of the SkyWalking OAP server. | -| service_name | string | False | "APISIX" | | Service name for the SkyWalking reporter. | -| service_instance_name | string | False | "APISIX Instance Name" | | Service instance name for the SkyWalking reporter. Set it to `$hostname` to directly get the local hostname. | -| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | -| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. | -| name | string | False | "skywalking logger" | | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | -| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| Name | Type | Required | Default | Valid values | Description | +|-----------------------|---------|----------|------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| endpoint_addr | string | True | | | URI of the SkyWalking OAP server. | +| service_name | string | False | "APISIX" | | Service name for the SkyWalking reporter. | +| service_instance_name | string | False | "APISIX Instance Name" | | Service instance name for the SkyWalking reporter. Set it to `$hostname` to directly get the local hostname. | +| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | +| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. | +| name | string | False | "skywalking logger" | | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | +| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/zh/latest/plugins/skywalking-logger.md b/docs/zh/latest/plugins/skywalking-logger.md index 87cabb3b450a..238b29918564 100644 --- a/docs/zh/latest/plugins/skywalking-logger.md +++ b/docs/zh/latest/plugins/skywalking-logger.md @@ -37,15 +37,18 @@ description: 本文将介绍 API 网关 Apache APISIX 如何通过 skywalking-lo ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | -| ---------------------- | ------- | ------ | -------------------- | ------------- | ---------------------------------------------------------------- | -| endpoint_addr | string | 是 | | | SkyWalking OAP 服务器的 URI。 | -| service_name | string | 否 |"APISIX" | | SkyWalking 服务名称。 | -| service_instance_name | string | 否 |"APISIX Instance Name"| | SkyWalking 服务的实例名称。当设置为 `$hostname`会直接获取本地主机名。 | -| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | -| name | string | 否 | "skywalking logger" | | 标识 logger 的唯一标识符。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。 | -| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,将请求正文包含在日志中。 | +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ---------------------- | ------- | ------ | -------------------- |---------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| endpoint_addr | string | 是 | | | SkyWalking OAP 服务器的 URI。 | +| service_name | string | 否 |"APISIX" | | SkyWalking 服务名称。 | +| service_instance_name | string | 否 |"APISIX Instance Name"| | SkyWalking 服务的实例名称。当设置为 `$hostname`会直接获取本地主机名。 | +| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | +| name | string | 否 | "skywalking logger" | | 标识 logger 的唯一标识符。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。 | +| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,将请求正文包含在日志中。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志/数据)。这样可以避免插件频繁地提交数据,默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/t/plugin/skywalking-logger.t b/t/plugin/skywalking-logger.t index 900c5a07aa39..9662c0016e78 100644 --- a/t/plugin/skywalking-logger.t +++ b/t/plugin/skywalking-logger.t @@ -294,3 +294,90 @@ opentracing qr/\\\"serviceInstance\\\":\\\"\$hostname\\\"/ qr/\\\"serviceInstance\\\":\\\"\\\"/ --- wait: 0.5 + + + +=== TEST 13: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/skywalking-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "skywalking-logger": { + "endpoint_addr": "http://127.0.0.1:1986", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"{\\\"sample_payload\\\":\\\"hello\\\"}\" + + + +=== TEST 14: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/skywalking-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "skywalking-logger": { + "endpoint_addr": "http://127.0.0.1:1986", + "batch_max_size": 1, + "max_retry_count": 1, + "retry_delay": 2, + "buffer_duration": 2, + "inactive_timeout": 2, + "include_req_body": true, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"opentracing\\n\" From 01806dc9c879c364691414af4b676920c4148cb1 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Sun, 28 Jan 2024 19:22:47 +0800 Subject: [PATCH 03/12] fix skywalking-logger test --- apisix/plugins/skywalking-logger.lua | 2 +- t/plugin/skywalking-logger.t | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apisix/plugins/skywalking-logger.lua b/apisix/plugins/skywalking-logger.lua index 32147e4c0ecb..94ba65534a4c 100644 --- a/apisix/plugins/skywalking-logger.lua +++ b/apisix/plugins/skywalking-logger.lua @@ -93,7 +93,7 @@ local function send_http_data(conf, log_message) local host = url_decoded.host local port = url_decoded.port - core.log.info("sending a batch logs to ", conf.endpoint_addr, ", log_message: ", log_message) + core.log.info("sending a batch logs to ", conf.endpoint_addr) local httpc = http.new() httpc:set_timeout(conf.timeout * 1000) diff --git a/t/plugin/skywalking-logger.t b/t/plugin/skywalking-logger.t index 9662c0016e78..3d79aaadbf87 100644 --- a/t/plugin/skywalking-logger.t +++ b/t/plugin/skywalking-logger.t @@ -33,7 +33,10 @@ add_block_preprocessor(sub { location /v3/logs { content_by_lua_block { local core = require("apisix.core") - + ngx.req.read_body() + local data = ngx.req.get_body_data() + local headers = ngx.req.get_headers() + ngx.log(ngx.WARN, "skywalking-logger body: ", data) core.log.warn(core.json.encode(core.request.get_body(), true)) } } @@ -381,3 +384,4 @@ qr/\\\"serviceInstance\\\":\\\"\\\"/ } --- error_log \"body\":\"opentracing\\n\" + From 3c44bfd5b29b96f49b7abc86dc416a550fac78b0 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Sun, 28 Jan 2024 22:48:24 +0800 Subject: [PATCH 04/12] sls-logger & add tencent-cloud-cls doc --- apisix/plugins/sls-logger.lua | 24 +++++- docs/en/latest/plugins/sls-logger.md | 3 + docs/zh/latest/plugins/sls-logger.md | 3 + docs/zh/latest/plugins/tencent-cloud-cls.md | 2 + t/plugin/sls-logger.t | 88 +++++++++++++++++++++ 5 files changed, 118 insertions(+), 2 deletions(-) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index de2fbae67ffa..4e316d21f11c 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -33,6 +33,21 @@ local schema = { type = "object", properties = { include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, timeout = {type = "integer", minimum = 1, default= 5000}, log_format = {type = "object"}, host = {type = "string"}, @@ -81,7 +96,7 @@ local function send_tcp_data(route_conf, log_message) .. "] err: " .. err end - core.log.debug("sls logger send data ", log_message) + core.log.warn("sls logger send data ", log_message) ok, err = sock:send(log_message) if not ok then res = false @@ -125,10 +140,14 @@ local function handle_log(entries) if not data then return true end - + core.log.warn(234) return send_tcp_data(entries[1].route_conf, data) end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end + -- log phase in APISIX function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) @@ -146,6 +165,7 @@ function _M.log(conf, ctx) } local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str, structured_data) + core.log.info("collect_data:" .. core.json.encode(rf5424_data)) local process_context = { data = rf5424_data, diff --git a/docs/en/latest/plugins/sls-logger.md b/docs/en/latest/plugins/sls-logger.md index 47dc9449bbcf..94199a856730 100644 --- a/docs/en/latest/plugins/sls-logger.md +++ b/docs/en/latest/plugins/sls-logger.md @@ -46,6 +46,9 @@ It might take some time to receive the log data. It will be automatically sent a | access_key_id | True | AccessKey ID in Alibaba Cloud. See [Authorization](https://www.alibabacloud.com/help/en/log-service/latest/create-a-ram-user-and-authorize-the-ram-user-to-access-log-service) for more details. | | access_key_secret | True | AccessKey Secret in Alibaba Cloud. See [Authorization](https://www.alibabacloud.com/help/en/log-service/latest/create-a-ram-user-and-authorize-the-ram-user-to-access-log-service) for more details. | | include_req_body | True | When set to `true`, includes the request body in the log. | +| include_req_body_expr | No | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | No | When set to `true` includes the response body in the log. | +| include_resp_body_expr | No | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | name | False | Unique identifier for the batch processor. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | NOTE: `encrypt_fields = {"access_key_secret"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). diff --git a/docs/zh/latest/plugins/sls-logger.md b/docs/zh/latest/plugins/sls-logger.md index 7bd85c81beaf..64613a34a66c 100644 --- a/docs/zh/latest/plugins/sls-logger.md +++ b/docs/zh/latest/plugins/sls-logger.md @@ -43,6 +43,9 @@ title: sls-logger | access_key_id | 必须的 | AccessKey ID。建议使用阿里云子账号 AK,详情请参见 [授权](https://help.aliyun.com/document_detail/47664.html?spm=a2c4g.11186623.2.15.49301b47lfvxXP#task-xsk-ttc-ry)。| | access_key_secret | 必须的 | AccessKey Secret。建议使用阿里云子账号 AK,详情请参见 [授权](https://help.aliyun.com/document_detail/47664.html?spm=a2c4g.11186623.2.15.49301b47lfvxXP#task-xsk-ttc-ry)。| | include_req_body | 可选的 | 是否包含请求体。| +| include_req_body_expr | 可选的 | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | 可选的 | 当设置为 `true` 时,日志中将包含响应体。 | +| include_resp_body_expr | 可选的 | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | |name| 可选的 | 批处理名字。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。| 注意:schema 中还定义了 `encrypt_fields = {"access_key_secret"}`,这意味着该字段将会被加密存储在 etcd 中。具体参考 [加密存储字段](../plugin-develop.md#加密存储字段)。 diff --git a/docs/zh/latest/plugins/tencent-cloud-cls.md b/docs/zh/latest/plugins/tencent-cloud-cls.md index 88bff5b06619..0e45141d3d3f 100644 --- a/docs/zh/latest/plugins/tencent-cloud-cls.md +++ b/docs/zh/latest/plugins/tencent-cloud-cls.md @@ -42,7 +42,9 @@ description: API 网关 Apache APISIX tencent-cloud-cls 插件可用于将日志 | secret_key | string | 是 | | | 云 API 密钥的 key。 | | sample_ratio | number | 否 | 1 | [0.00001, 1] | 采样的比例。设置为 `1` 时,将对所有请求进行采样。 | | include_req_body | boolean | 否 | false | [false, true]| 当设置为 `true` 时,日志中将包含请求体。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | | include_resp_body | boolean | 否 | false | [false, true]| 当设置为 `true` 时,日志中将包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | | global_tag | object | 否 | | | kv 形式的 JSON 数据,可以写入每一条日志,便于在 CLS 中检索。 | | log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index 940ddf6a2be8..e7623cbddab5 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -385,3 +385,91 @@ passed GET /hello --- response_body hello world + + + +=== TEST 14: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "sls-logger": { + "host": "127.0.0.1", + "port": 10009, + "project": "your_project", + "logstore": "your_logstore", + "access_key_id": "your_access_key_id", + "access_key_secret": "your_access_key_secret", + "timeout": 30000, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"{\\\"sample_payload\\\":\\\"hello\\\" + + + +=== TEST 14: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "sls-logger": { + "host": "127.0.0.1", + "port": 10009, + "project": "your_project", + "logstore": "your_logstore", + "access_key_id": "your_access_key_id", + "access_key_secret": "your_access_key_secret", + "timeout": 30000, + "include_req_body": true, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"hello world\\n\" From 9c4bcafa1b22e58cf36d682b67ced8e986068729 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Sun, 28 Jan 2024 23:27:39 +0800 Subject: [PATCH 05/12] syslog --- apisix/plugins/syslog.lua | 20 +++++- apisix/plugins/syslog/init.lua | 2 +- docs/en/latest/plugins/syslog.md | 5 +- docs/zh/latest/plugins/syslog.md | 5 +- t/plugin/syslog.t | 102 +++++++++++++++++++++++++++++-- 5 files changed, 125 insertions(+), 9 deletions(-) diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua index 155ea7b41bba..8d190ea18bbb 100644 --- a/apisix/plugins/syslog.lua +++ b/apisix/plugins/syslog.lua @@ -34,7 +34,22 @@ local schema = { pool_size = {type = "integer", minimum = 5, default = 5}, tls = {type = "boolean", default = false}, log_format = {type = "object"}, - include_req_body = {type = "boolean", default = false} + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, }, required = {"host", "port"} } @@ -68,6 +83,9 @@ function _M.check_schema(conf, schema_type) return core.schema.check(schema, conf) end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) diff --git a/apisix/plugins/syslog/init.lua b/apisix/plugins/syslog/init.lua index 0ab34f8054e9..25552efd52c1 100644 --- a/apisix/plugins/syslog/init.lua +++ b/apisix/plugins/syslog/init.lua @@ -88,7 +88,7 @@ function _M.push_entry(conf, ctx, entry) local rfc5424_data = rfc5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str) - + core.log.info("collect_data:" .. core.json.encode(rfc5424_data)) if batch_processor_manager:add_entry(conf, rfc5424_data) then return end diff --git a/docs/en/latest/plugins/syslog.md b/docs/en/latest/plugins/syslog.md index 1a7e5e4a8f79..11142807dd03 100644 --- a/docs/en/latest/plugins/syslog.md +++ b/docs/en/latest/plugins/syslog.md @@ -46,7 +46,10 @@ Logs can be set as JSON objects. | sock_type | string | False | "tcp" | ["tcp", "udp] | Transport layer protocol to use. | | pool_size | integer | False | 5 | [5, ...] | Keep-alive pool size used by `sock:keepalive`. | | log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | -| include_req_body | boolean | False | false | | When set to `true` includes the request body in the log. | +| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/zh/latest/plugins/syslog.md b/docs/zh/latest/plugins/syslog.md index 4707bba8b6fa..fd847f12211f 100644 --- a/docs/zh/latest/plugins/syslog.md +++ b/docs/zh/latest/plugins/syslog.md @@ -49,7 +49,10 @@ description: API 网关 Apache APISIX syslog 插件可用于将日志推送到 S | retry_delay | integer | 否 | | [0, ...] | 重试连接到日志服务器或重试向日志服务器发送日志消息之前的时间延迟(以毫秒为单位)。 | | pool_size | integer | 否 | 5 | [5, ...] | `sock:keepalive` 使用的 Keepalive 池大小。 | | log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| include_req_body | boolean | 否 | false | | 当设置为 `true` 时包括请求体。 | +| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时包括请求体。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志/数据)。这样可以避免插件频繁地提交数据,默认情况下批处理器每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 5e13aa301c56..38e83b8c1b5e 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -477,7 +477,99 @@ passed -=== TEST 15: access +=== TEST 15: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5140, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.say(body) + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body\":\"{\\\"sample_payload\\\":\\\"hello\\\"}\" + + + +=== TEST 16: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5140, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.say(body) + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body\":\"hello world\\n\" + + + +=== TEST 17: access --- extra_init_by_lua local syslog = require("apisix.plugins.syslog.init") local json = require("apisix.core.json") @@ -500,7 +592,7 @@ push_entry is called with data -=== TEST 16: check log +=== TEST 18: check log --- exec tail -n 1 ci/pod/vector/syslog-tcp.log --- response_body eval @@ -508,7 +600,7 @@ qr/.*vip.*/ -=== TEST 17: test udp mode +=== TEST 19: test udp mode --- config location /t { content_by_lua_block { @@ -548,13 +640,13 @@ passed -=== TEST 18: hit +=== TEST 20: hit --- request GET /hello -=== TEST 19: check log +=== TEST 21: check log --- exec tail -n 1 ci/pod/vector/syslog-udp.log --- response_body eval From 0b0897a0d6da4a34eed7701ab9313c7fe966728a Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Mon, 29 Jan 2024 00:08:55 +0800 Subject: [PATCH 06/12] feat: Added 'include body' related plug-in configuration --- apisix/plugins/sls-logger.lua | 4 +- apisix/plugins/tcp-logger.lua | 21 ++++++- apisix/plugins/udp-logger.lua | 21 ++++++- docs/en/latest/plugins/tcp-logger.md | 5 +- docs/en/latest/plugins/udp-logger.md | 5 +- docs/zh/latest/plugins/tcp-logger.md | 5 +- docs/zh/latest/plugins/udp-logger.md | 5 +- t/plugin/skywalking-logger.t | 1 - t/plugin/sls-logger.t | 2 +- t/plugin/tcp-logger.t | 90 +++++++++++++++++++++++++- t/plugin/udp-logger.t | 94 +++++++++++++++++++++++++++- 11 files changed, 241 insertions(+), 12 deletions(-) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 4e316d21f11c..52d1781d5659 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -96,7 +96,7 @@ local function send_tcp_data(route_conf, log_message) .. "] err: " .. err end - core.log.warn("sls logger send data ", log_message) + core.log.debug("sls logger send data ", log_message) ok, err = sock:send(log_message) if not ok then res = false @@ -140,7 +140,7 @@ local function handle_log(entries) if not data then return true end - core.log.warn(234) + return send_tcp_data(entries[1].route_conf, data) end diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua index 444afe1d968a..1522e0059901 100644 --- a/apisix/plugins/tcp-logger.lua +++ b/apisix/plugins/tcp-logger.lua @@ -33,7 +33,22 @@ local schema = { tls_options = {type = "string"}, timeout = {type = "integer", minimum = 1, default= 1000}, log_format = {type = "object"}, - include_req_body = {type = "boolean", default = false} + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, }, required = {"host", "port"} } @@ -77,6 +92,7 @@ local function send_tcp_data(conf, log_message) sock:settimeout(conf.timeout) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) + core.log.info("sending a log_message: ", log_message) local ok, err = sock:connect(conf.host, conf.port) if not ok then @@ -108,6 +124,9 @@ local function send_tcp_data(conf, log_message) return res, err_msg end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua index 7d76a4b02eae..47858521fdd5 100644 --- a/apisix/plugins/udp-logger.lua +++ b/apisix/plugins/udp-logger.lua @@ -31,7 +31,22 @@ local schema = { port = {type = "integer", minimum = 0}, timeout = {type = "integer", minimum = 1, default = 3}, log_format = {type = "object"}, - include_req_body = {type = "boolean", default = false} + include_req_body = {type = "boolean", default = false}, + include_req_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, + include_resp_body = { type = "boolean", default = false }, + include_resp_body_expr = { + type = "array", + minItems = 1, + items = { + type = "array" + } + }, }, required = {"host", "port"} } @@ -70,6 +85,7 @@ local function send_udp_data(conf, log_message) sock:settimeout(conf.timeout * 1000) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) + core.log.info("sending logs: ", log_message) local ok, err = sock:setpeername(conf.host, conf.port) @@ -94,6 +110,9 @@ local function send_udp_data(conf, log_message) return res, err_msg end +function _M.body_filter(conf, ctx) + log_util.collect_body(conf, ctx) +end function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) diff --git a/docs/en/latest/plugins/tcp-logger.md b/docs/en/latest/plugins/tcp-logger.md index e5bffac3500e..b163029cd143 100644 --- a/docs/en/latest/plugins/tcp-logger.md +++ b/docs/en/latest/plugins/tcp-logger.md @@ -46,7 +46,10 @@ This plugin also allows to push logs as a batch to your external TCP server. It | log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | | tls | boolean | False | false | | When set to `true` performs SSL verification. | | tls_options | string | False | | | TLS options. | -| include_req_body | boolean | False | false | | When set to `true` includes the request body in the log. | +| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/udp-logger.md b/docs/en/latest/plugins/udp-logger.md index e3acd0030a00..9af8c98dece1 100644 --- a/docs/en/latest/plugins/udp-logger.md +++ b/docs/en/latest/plugins/udp-logger.md @@ -44,7 +44,10 @@ This plugin also allows to push logs as a batch to your external UDP server. It | timeout | integer | False | 3 | [1,...] | Timeout for the upstream to send data. | | log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | | name | string | False | "udp logger" | | Unique identifier for the batch processor. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. processor. | -| include_req_body | boolean | False | false | | When set to `true` includes the request body in the log. | +| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| include_req_body_expr | array | No | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | +| include_resp_body | boolean | No | false | [false, true] | When set to `true` includes the response body in the log. | +| include_resp_body_expr | array | No | | | Filter for when the `include_resp_body` attribute is set to `true`. Response body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/zh/latest/plugins/tcp-logger.md b/docs/zh/latest/plugins/tcp-logger.md index 3984fb1d407a..17203a3e5ec0 100644 --- a/docs/zh/latest/plugins/tcp-logger.md +++ b/docs/zh/latest/plugins/tcp-logger.md @@ -43,7 +43,10 @@ description: 本文介绍了 API 网关 Apache APISIX 如何使用 tcp-logger | log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | | tls | boolean | 否 | false | | 用于控制是否执行 SSL 验证。 | | tls_options | string | 否 | | | TLS 选项。 | -| include_req_body | boolean | 否 | | | 当设置为 `true` 时,日志中将包含请求体。 | +| include_req_body | boolean | 否 | | [false, true] | 当设置为 `true` 时,日志中将包含请求体。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | [false, true]| 当设置为 `true` 时,日志中将包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志/数据)。这样可以避免插件频繁地提交数据,默认情况下批处理器每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/docs/zh/latest/plugins/udp-logger.md b/docs/zh/latest/plugins/udp-logger.md index 00f00d641703..a8ca60778551 100644 --- a/docs/zh/latest/plugins/udp-logger.md +++ b/docs/zh/latest/plugins/udp-logger.md @@ -42,7 +42,10 @@ description: 本文介绍了 API 网关 Apache APISIX 如何使用 udp-logger | timeout | integer | 否 | 1000 | [1,...] | 发送数据超时间。 | | log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | | name | string | 否 | "udp logger" | | 标识 logger 的唯一标识符。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。 | -| include_req_body | boolean | 否 | | | 当设置为 `true` 时,日志中将包含请求体。 | +| include_req_body | boolean | 否 | | [false, true] | 当设置为 `true` 时,日志中将包含请求体。 | +| include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | +| include_resp_body | boolean | 否 | false | [false, true]| 当设置为 `true` 时,日志中将包含响应体。 | +| include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志和数据)。这样可以避免插件频繁地提交数据,默认情况下批处理器每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/t/plugin/skywalking-logger.t b/t/plugin/skywalking-logger.t index 3d79aaadbf87..6ab87be26947 100644 --- a/t/plugin/skywalking-logger.t +++ b/t/plugin/skywalking-logger.t @@ -384,4 +384,3 @@ qr/\\\"serviceInstance\\\":\\\"\\\"/ } --- error_log \"body\":\"opentracing\\n\" - diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index e7623cbddab5..3c7590b32155 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -432,7 +432,7 @@ hello world -=== TEST 14: add plugin with 'include_resp_body' setting, collect response log +=== TEST 15: add plugin with 'include_resp_body' setting, collect response log --- config location /t { content_by_lua_block { diff --git a/t/plugin/tcp-logger.t b/t/plugin/tcp-logger.t index 3ef774f813d1..9e538a3a4bcf 100644 --- a/t/plugin/tcp-logger.t +++ b/t/plugin/tcp-logger.t @@ -513,7 +513,95 @@ opentracing -=== TEST 15: check tls log +=== TEST 15: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) + local code, body1 = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tcp-logger": { + "host": "127.0.0.1", + "port": 43000, + "tls": true, + "batch_max_size": 1, + "include_req_body": true, + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 16: add plugin with 'include_resp_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) + local code, body1 = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tcp-logger": { + "host": "127.0.0.1", + "port": 43000, + "tls": true, + "batch_max_size": 1, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"opentracing\n" + + + +=== TEST 17: check tls log --- exec tail -n 1 ci/pod/vector/tls-datas.log --- response_body eval diff --git a/t/plugin/udp-logger.t b/t/plugin/udp-logger.t index 44744823634b..ba4f8f6d32ed 100644 --- a/t/plugin/udp-logger.t +++ b/t/plugin/udp-logger.t @@ -440,7 +440,99 @@ passed -=== TEST 12: check log format from logging server +=== TEST 12: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "udp-logger": { + "host": "127.0.0.1", + "port": 8127, + "tls": false, + "batch_max_size": 1, + "inactive_timeout": 1, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 13: add plugin with 'include_resp_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "udp-logger": { + "host": "127.0.0.1", + "port": 8127, + "tls": false, + "batch_max_size": 1, + "inactive_timeout": 1, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"hello world\n" + + + +=== TEST 14: check log format from logging server --- exec tail -n 1 ci/pod/vector/udp.log --- response_body eval From 1812f2ae0fa0d2ec3c01945872b23d8447210103 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Tue, 30 Jan 2024 23:08:38 +0800 Subject: [PATCH 07/12] feat: add option to include request body and response body in log util --- apisix/plugins/sls-logger.lua | 1 - apisix/plugins/syslog/init.lua | 2 +- apisix/plugins/tcp-logger.lua | 1 - apisix/plugins/udp-logger.lua | 1 - .../en/latest/plugins/elasticsearch-logger.md | 1 - docs/en/latest/plugins/skywalking-logger.md | 18 ++-- docs/zh/latest/plugins/loggly.md | 20 ++-- docs/zh/latest/plugins/skywalking-logger.md | 18 ++-- t/plugin/sls-logger.t | 88 --------------- t/plugin/syslog.t | 102 +----------------- t/plugin/tcp-logger.t | 90 +--------------- t/plugin/udp-logger.t | 94 +--------------- 12 files changed, 36 insertions(+), 400 deletions(-) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 52d1781d5659..60fd2c017a7d 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -165,7 +165,6 @@ function _M.log(conf, ctx) } local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str, structured_data) - core.log.info("collect_data:" .. core.json.encode(rf5424_data)) local process_context = { data = rf5424_data, diff --git a/apisix/plugins/syslog/init.lua b/apisix/plugins/syslog/init.lua index 25552efd52c1..0ab34f8054e9 100644 --- a/apisix/plugins/syslog/init.lua +++ b/apisix/plugins/syslog/init.lua @@ -88,7 +88,7 @@ function _M.push_entry(conf, ctx, entry) local rfc5424_data = rfc5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str) - core.log.info("collect_data:" .. core.json.encode(rfc5424_data)) + if batch_processor_manager:add_entry(conf, rfc5424_data) then return end diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua index 1522e0059901..284247dd9ffb 100644 --- a/apisix/plugins/tcp-logger.lua +++ b/apisix/plugins/tcp-logger.lua @@ -92,7 +92,6 @@ local function send_tcp_data(conf, log_message) sock:settimeout(conf.timeout) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) - core.log.info("sending a log_message: ", log_message) local ok, err = sock:connect(conf.host, conf.port) if not ok then diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua index 47858521fdd5..db62f4ad9f35 100644 --- a/apisix/plugins/udp-logger.lua +++ b/apisix/plugins/udp-logger.lua @@ -85,7 +85,6 @@ local function send_udp_data(conf, log_message) sock:settimeout(conf.timeout * 1000) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) - core.log.info("sending logs: ", log_message) local ok, err = sock:setpeername(conf.host, conf.port) diff --git a/docs/en/latest/plugins/elasticsearch-logger.md b/docs/en/latest/plugins/elasticsearch-logger.md index 2d98a0c38064..098dc27d335b 100644 --- a/docs/en/latest/plugins/elasticsearch-logger.md +++ b/docs/en/latest/plugins/elasticsearch-logger.md @@ -53,7 +53,6 @@ When the Plugin is enabled, APISIX will serialize the request context informatio | include_resp_body | boolean | False | false | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | - NOTE: `encrypt_fields = {"auth.password"}` is also defined in the schema, which means that the field will be stored encrypted in etcd. See [encrypted storage fields](../plugin-develop.md#encrypted-storage-fields). This Plugin supports using batch processors to aggregate and process entries (logs/data) in a batch. This avoids the need for frequently submitting the data. The batch processor submits data every `5` seconds or when the data in the queue reaches `1000`. See [Batch Processor](../batch-processor.md#configuration) for more information or setting your custom configuration. diff --git a/docs/en/latest/plugins/skywalking-logger.md b/docs/en/latest/plugins/skywalking-logger.md index f1592e4ece53..7150c0bb2f24 100644 --- a/docs/en/latest/plugins/skywalking-logger.md +++ b/docs/en/latest/plugins/skywalking-logger.md @@ -35,15 +35,15 @@ If there is an existing tracing context, it sets up the trace-log correlation au ## Attributes -| Name | Type | Required | Default | Valid values | Description | -|-----------------------|---------|----------|------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| endpoint_addr | string | True | | | URI of the SkyWalking OAP server. | -| service_name | string | False | "APISIX" | | Service name for the SkyWalking reporter. | -| service_instance_name | string | False | "APISIX Instance Name" | | Service instance name for the SkyWalking reporter. Set it to `$hostname` to directly get the local hostname. | -| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | -| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. | -| name | string | False | "skywalking logger" | | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | -| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | +| Name | Type | Required | Default | Valid values | Description | +|-----------------------|---------|----------|------------------------|---------------|--------------------------------------------------------------------------------------------------------------| +| endpoint_addr | string | True | | | URI of the SkyWalking OAP server. | +| service_name | string | False | "APISIX" | | Service name for the SkyWalking reporter. | +| service_instance_name | string | False | "APISIX Instance Name" | | Service instance name for the SkyWalking reporter. Set it to `$hostname` to directly get the local hostname. | +| log_format | object | False | | | Log format declared as key value pairs in JSON format. Values only support strings. [APISIX](../apisix-variable.md) or [Nginx](http://nginx.org/en/docs/varindex.html) variables can be used by prefixing the string with `$`. | +| timeout | integer | False | 3 | [1,...] | Time to keep the connection alive for after sending a request. | +| name | string | False | "skywalking logger" | | Unique identifier to identify the logger. If you use Prometheus to monitor APISIX metrics, the name is exported in `apisix_batch_process_entries`. | +| include_req_body | boolean | False | false | [false, true] | When set to `true` includes the request body in the log. | | include_req_body_expr | array | False | | | Filter for when the `include_req_body` attribute is set to `true`. Request body is only logged when the expression set here evaluates to `true`. See [lua-resty-expr](https://github.com/api7/lua-resty-expr) for more. | | include_resp_body | boolean | False | false | [false, true] | When set to `true` includes the response body in the log. | | include_resp_body_expr | array | False | | | When the `include_resp_body` attribute is set to `true`, use this to filter based on [lua-resty-expr](https://github.com/api7/lua-resty-expr). If present, only logs the response if the expression evaluates to `true`. | diff --git a/docs/zh/latest/plugins/loggly.md b/docs/zh/latest/plugins/loggly.md index e44cfd42815f..5bf92cf5a9ae 100644 --- a/docs/zh/latest/plugins/loggly.md +++ b/docs/zh/latest/plugins/loggly.md @@ -35,17 +35,17 @@ description: API 网关 Apache APISIX loggly 插件可用于将日志转发到 S ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 描述 | -|-------------------------|---------------|-----|-------|---------------------------------------------------------------------------------------------------------------------------------------------------| -| customer_token | string | 是 | | 将日志发送到 Loggly 时使用的唯一标识符,以确保将日志发送到正确的组织帐户。 | -| severity | string (enum) | 否 | INFO | Syslog 日志事件的严重性级别。包括:`DEBUG`、`INFO`、`NOTICE`、`WARNING`、`ERR`、`CRIT`、`ALERT` 和 `EMEGR`。 | -| severity_map | object | 否 | nil | 一种将上游 HTTP 响应代码映射到 Syslog 中的方法。 `key-value`,其中 `key` 是 HTTP 响应代码,`value`是 Syslog 严重级别。例如`{"410": "CRIT"}`。 | -| tags | array | 否 | | 元数据将包含在任何事件日志中,以帮助进行分段和过滤。 | -| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| include_req_body | boolean | 否 | false | 当设置为 `true` 时,包含请求体。**注意**:如果请求体无法完全存放在内存中,由于 NGINX 的限制,APISIX 无法将它记录下来。 | +| 名称 | 类型 | 必选项 | 默认值 | 描述 | +|------------------------|---------------|----------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------| +| customer_token | string | 是 | | 将日志发送到 Loggly 时使用的唯一标识符,以确保将日志发送到正确的组织帐户。 | +| severity | string (enum) | 否 | INFO | Syslog 日志事件的严重性级别。包括:`DEBUG`、`INFO`、`NOTICE`、`WARNING`、`ERR`、`CRIT`、`ALERT` 和 `EMEGR`。 | +| severity_map | object | 否 | nil | 一种将上游 HTTP 响应代码映射到 Syslog 中的方法。 `key-value`,其中 `key` 是 HTTP 响应代码,`value`是 Syslog 严重级别。例如`{"410": "CRIT"}`。 | +| tags | array | 否 | | 元数据将包含在任何事件日志中,以帮助进行分段和过滤。 | +| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| include_req_body | boolean | 否 | false | 当设置为 `true` 时,包含请求体。**注意**:如果请求体无法完全存放在内存中,由于 NGINX 的限制,APISIX 无法将它记录下来。 | | include_req_body_expr | array | 否 | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | -| include_resp_body | boolean | 否 | false | 当设置为 `true` 时,包含响应体。 | -| include_resp_body_expr | array | 否 | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | +| include_resp_body | boolean | 否 | false | 当设置为 `true` 时,包含响应体。 | +| include_resp_body_expr | array | 否 | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | 该插件支持使用批处理器来聚合并批量处理条目(日志或数据)。这样可以避免插件频繁地提交数据,默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据,如需了解批处理器相关参数设置,请参考 [Batch-Processor](../batch-processor.md#配置)。 diff --git a/docs/zh/latest/plugins/skywalking-logger.md b/docs/zh/latest/plugins/skywalking-logger.md index 238b29918564..8ef7417582c4 100644 --- a/docs/zh/latest/plugins/skywalking-logger.md +++ b/docs/zh/latest/plugins/skywalking-logger.md @@ -37,15 +37,15 @@ description: 本文将介绍 API 网关 Apache APISIX 如何通过 skywalking-lo ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | -| ---------------------- | ------- | ------ | -------------------- |---------------|--------------------------------------------------------------------------------------------------------------------------------------------------| -| endpoint_addr | string | 是 | | | SkyWalking OAP 服务器的 URI。 | -| service_name | string | 否 |"APISIX" | | SkyWalking 服务名称。 | -| service_instance_name | string | 否 |"APISIX Instance Name"| | SkyWalking 服务的实例名称。当设置为 `$hostname`会直接获取本地主机名。 | -| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | -| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | -| name | string | 否 | "skywalking logger" | | 标识 logger 的唯一标识符。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。 | -| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,将请求正文包含在日志中。 | +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ---------------------- | ------- | ------ | -------------------- | ------------- | ---------------------------------------------------------------- | +| endpoint_addr | string | 是 | | | SkyWalking OAP 服务器的 URI。 | +| service_name | string | 否 |"APISIX" | | SkyWalking 服务名称。 | +| service_instance_name | string | 否 |"APISIX Instance Name"| | SkyWalking 服务的实例名称。当设置为 `$hostname`会直接获取本地主机名。 | +| log_format | object | 否 | | | 以 JSON 格式的键值对来声明日志格式。对于值部分,仅支持字符串。如果是以 `$` 开头,则表明是要获取 [APISIX 变量](../apisix-variable.md) 或 [NGINX 内置变量](http://nginx.org/en/docs/varindex.html)。 | +| timeout | integer | 否 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | +| name | string | 否 | "skywalking logger" | | 标识 logger 的唯一标识符。如果您使用 Prometheus 监视 APISIX 指标,名称将以 `apisix_batch_process_entries` 导出。 | +| include_req_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,将请求正文包含在日志中。 | | include_req_body_expr | array | 否 | | | 当 `include_req_body` 属性设置为 `true` 时的过滤器。只有当此处设置的表达式求值为 `true` 时,才会记录请求体。有关更多信息,请参阅 [lua-resty-expr](https://github.com/api7/lua-resty-expr) 。 | | include_resp_body | boolean | 否 | false | [false, true] | 当设置为 `true` 时,包含响应体。 | | include_resp_body_expr | array | 否 | | | 当 `include_resp_body` 属性设置为 `true` 时进行过滤响应体,并且只有当此处设置的表达式计算结果为 `true` 时,才会记录响应体。更多信息,请参考 [lua-resty-expr](https://github.com/api7/lua-resty-expr)。 | diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index 3c7590b32155..940ddf6a2be8 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -385,91 +385,3 @@ passed GET /hello --- response_body hello world - - - -=== TEST 14: add plugin with 'include_req_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) - - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "sls-logger": { - "host": "127.0.0.1", - "port": 10009, - "project": "your_project", - "logstore": "your_logstore", - "access_key_id": "your_access_key_id", - "access_key_secret": "your_access_key_secret", - "timeout": 30000, - "include_req_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - end - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- error_log -\"body\":\"{\\\"sample_payload\\\":\\\"hello\\\" - - - -=== TEST 15: add plugin with 'include_resp_body' setting, collect response log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "sls-logger": { - "host": "127.0.0.1", - "port": 10009, - "project": "your_project", - "logstore": "your_logstore", - "access_key_id": "your_access_key_id", - "access_key_secret": "your_access_key_secret", - "timeout": 30000, - "include_req_body": true, - "include_resp_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - end - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- error_log -\"body\":\"hello world\\n\" diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 38e83b8c1b5e..5e13aa301c56 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -477,99 +477,7 @@ passed -=== TEST 15: add plugin with 'include_req_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "syslog": { - "batch_max_size": 1, - "flush_limit": 1, - "host" : "127.0.0.1", - "port" : 5140, - "include_req_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - ngx.say(body) - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body\":\"{\\\"sample_payload\\\":\\\"hello\\\"}\" - - - -=== TEST 16: add plugin with 'include_resp_body' setting, collect response log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "syslog": { - "batch_max_size": 1, - "flush_limit": 1, - "host" : "127.0.0.1", - "port" : 5140, - "include_resp_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - ngx.say(body) - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body\":\"hello world\\n\" - - - -=== TEST 17: access +=== TEST 15: access --- extra_init_by_lua local syslog = require("apisix.plugins.syslog.init") local json = require("apisix.core.json") @@ -592,7 +500,7 @@ push_entry is called with data -=== TEST 18: check log +=== TEST 16: check log --- exec tail -n 1 ci/pod/vector/syslog-tcp.log --- response_body eval @@ -600,7 +508,7 @@ qr/.*vip.*/ -=== TEST 19: test udp mode +=== TEST 17: test udp mode --- config location /t { content_by_lua_block { @@ -640,13 +548,13 @@ passed -=== TEST 20: hit +=== TEST 18: hit --- request GET /hello -=== TEST 21: check log +=== TEST 19: check log --- exec tail -n 1 ci/pod/vector/syslog-udp.log --- response_body eval diff --git a/t/plugin/tcp-logger.t b/t/plugin/tcp-logger.t index 9e538a3a4bcf..3ef774f813d1 100644 --- a/t/plugin/tcp-logger.t +++ b/t/plugin/tcp-logger.t @@ -513,95 +513,7 @@ opentracing -=== TEST 15: add plugin with 'include_req_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) - local code, body1 = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "tcp-logger": { - "host": "127.0.0.1", - "port": 43000, - "tls": true, - "batch_max_size": 1, - "include_req_body": true, - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1982": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say("fail") - return - end - - local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body":"{\"sample_payload\":\"hello\"}" - - - -=== TEST 16: add plugin with 'include_resp_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) - local code, body1 = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "tcp-logger": { - "host": "127.0.0.1", - "port": 43000, - "tls": true, - "batch_max_size": 1, - "include_resp_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1982": 1 - }, - "type": "roundrobin" - }, - "uri": "/opentracing" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say("fail") - return - end - - local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body":"opentracing\n" - - - -=== TEST 17: check tls log +=== TEST 15: check tls log --- exec tail -n 1 ci/pod/vector/tls-datas.log --- response_body eval diff --git a/t/plugin/udp-logger.t b/t/plugin/udp-logger.t index ba4f8f6d32ed..44744823634b 100644 --- a/t/plugin/udp-logger.t +++ b/t/plugin/udp-logger.t @@ -440,99 +440,7 @@ passed -=== TEST 12: add plugin with 'include_req_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "udp-logger": { - "host": "127.0.0.1", - "port": 8127, - "tls": false, - "batch_max_size": 1, - "inactive_timeout": 1, - "include_req_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body":"{\"sample_payload\":\"hello\"}" - - - -=== TEST 13: add plugin with 'include_resp_body' setting, collect request log ---- config - location /t { - content_by_lua_block { - local t = require("lib.test_admin").test - t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) - local code, body = t('/apisix/admin/routes/1', - ngx.HTTP_PUT, - [[{ - "plugins": { - "udp-logger": { - "host": "127.0.0.1", - "port": 8127, - "tls": false, - "batch_max_size": 1, - "inactive_timeout": 1, - "include_resp_body": true - } - }, - "upstream": { - "nodes": { - "127.0.0.1:1980": 1 - }, - "type": "roundrobin" - }, - "uri": "/hello" - }]] - ) - - if code >= 300 then - ngx.status = code - ngx.say(body) - return - end - - - local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") - } - } ---- request -GET /t ---- error_log -"body":"hello world\n" - - - -=== TEST 14: check log format from logging server +=== TEST 12: check log format from logging server --- exec tail -n 1 ci/pod/vector/udp.log --- response_body eval From 8eaa742f86c3d0c890b181ad2f68c67d3ad92577 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Tue, 30 Jan 2024 23:10:58 +0800 Subject: [PATCH 08/12] fix code style --- apisix/plugins/sls-logger.lua | 2 ++ apisix/plugins/syslog.lua | 2 ++ apisix/plugins/tcp-logger.lua | 2 ++ apisix/plugins/udp-logger.lua | 2 ++ 4 files changed, 8 insertions(+) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 60fd2c017a7d..6ca437f0755f 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -144,10 +144,12 @@ local function handle_log(entries) return send_tcp_data(entries[1].route_conf, data) end + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end + -- log phase in APISIX function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) diff --git a/apisix/plugins/syslog.lua b/apisix/plugins/syslog.lua index 8d190ea18bbb..fa160feeb45d 100644 --- a/apisix/plugins/syslog.lua +++ b/apisix/plugins/syslog.lua @@ -83,10 +83,12 @@ function _M.check_schema(conf, schema_type) return core.schema.check(schema, conf) end + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end + function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) syslog.push_entry(conf, ctx, entry) diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua index 284247dd9ffb..401e1772c2fc 100644 --- a/apisix/plugins/tcp-logger.lua +++ b/apisix/plugins/tcp-logger.lua @@ -123,10 +123,12 @@ local function send_tcp_data(conf, log_message) return res, err_msg end + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end + function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua index db62f4ad9f35..3b898b017ad9 100644 --- a/apisix/plugins/udp-logger.lua +++ b/apisix/plugins/udp-logger.lua @@ -109,10 +109,12 @@ local function send_udp_data(conf, log_message) return res, err_msg end + function _M.body_filter(conf, ctx) log_util.collect_body(conf, ctx) end + function _M.log(conf, ctx) local entry = log_util.get_log_entry(plugin_name, conf, ctx) From f31327d0888ad9acb382b1914b35fafee25797d6 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Thu, 1 Feb 2024 12:34:02 +0800 Subject: [PATCH 09/12] add test case --- apisix/plugins/sls-logger.lua | 2 +- apisix/plugins/syslog/init.lua | 2 +- apisix/plugins/tcp-logger.lua | 1 + apisix/plugins/udp-logger.lua | 1 + t/plugin/sls-logger.t | 87 +++++++++++++++++++++++++++++++ t/plugin/syslog.t | 92 +++++++++++++++++++++++++++++++++ t/plugin/tcp-logger.t | 88 +++++++++++++++++++++++++++++++ t/plugin/udp-logger.t | 94 +++++++++++++++++++++++++++++++++- 8 files changed, 364 insertions(+), 3 deletions(-) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index 6ca437f0755f..a47009ec326d 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -167,7 +167,7 @@ function _M.log(conf, ctx) } local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str, structured_data) - + core.log.info("collect_data:" .. core.json.encode(rf5424_data)) local process_context = { data = rf5424_data, route_conf = conf diff --git a/apisix/plugins/syslog/init.lua b/apisix/plugins/syslog/init.lua index 0ab34f8054e9..8a3d90e38b08 100644 --- a/apisix/plugins/syslog/init.lua +++ b/apisix/plugins/syslog/init.lua @@ -88,7 +88,7 @@ function _M.push_entry(conf, ctx, entry) local rfc5424_data = rfc5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str) - + core.log.info("collect_data:" .. rfc5424_data) if batch_processor_manager:add_entry(conf, rfc5424_data) then return end diff --git a/apisix/plugins/tcp-logger.lua b/apisix/plugins/tcp-logger.lua index 401e1772c2fc..e95ba9339e95 100644 --- a/apisix/plugins/tcp-logger.lua +++ b/apisix/plugins/tcp-logger.lua @@ -92,6 +92,7 @@ local function send_tcp_data(conf, log_message) sock:settimeout(conf.timeout) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) + core.log.info("sending log_message: ", log_message) local ok, err = sock:connect(conf.host, conf.port) if not ok then diff --git a/apisix/plugins/udp-logger.lua b/apisix/plugins/udp-logger.lua index 3b898b017ad9..75e8bba31740 100644 --- a/apisix/plugins/udp-logger.lua +++ b/apisix/plugins/udp-logger.lua @@ -85,6 +85,7 @@ local function send_udp_data(conf, log_message) sock:settimeout(conf.timeout * 1000) core.log.info("sending a batch logs to ", conf.host, ":", conf.port) + core.log.info("sending log_message: ", log_message) local ok, err = sock:setpeername(conf.host, conf.port) diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index 940ddf6a2be8..070f8dbcddc5 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -385,3 +385,90 @@ passed GET /hello --- response_body hello world + + + +=== TEST 14: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) + + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "sls-logger": { + "host": "127.0.0.1", + "port": 10009, + "project": "your_project", + "logstore": "your_logstore", + "access_key_id": "your_access_key_id", + "access_key_secret": "your_access_key_secret", + "timeout": 30000, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"{\\\"sample_payload\\\":\\\"hello\\\" + + + +=== TEST 15: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/sls-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "sls-logger": { + "host": "127.0.0.1", + "port": 10009, + "project": "your_project", + "logstore": "your_logstore", + "access_key_id": "your_access_key_id", + "access_key_secret": "your_access_key_secret", + "timeout": 30000, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- error_log +\"body\":\"hello world\\n\" diff --git a/t/plugin/syslog.t b/t/plugin/syslog.t index 5e13aa301c56..9dbb03af83c0 100644 --- a/t/plugin/syslog.t +++ b/t/plugin/syslog.t @@ -559,3 +559,95 @@ GET /hello tail -n 1 ci/pod/vector/syslog-udp.log --- response_body eval qr/.*upstream.*/ + + + +=== TEST 20: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5140, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.say(body) + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 21: add plugin with 'include_resp_body' setting, collect response log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/syslog', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "syslog": { + "batch_max_size": 1, + "flush_limit": 1, + "host" : "127.0.0.1", + "port" : 5140, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + ngx.say(body) + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"hello world\n" diff --git a/t/plugin/tcp-logger.t b/t/plugin/tcp-logger.t index 3ef774f813d1..b3c29ee5116c 100644 --- a/t/plugin/tcp-logger.t +++ b/t/plugin/tcp-logger.t @@ -518,3 +518,91 @@ opentracing tail -n 1 ci/pod/vector/tls-datas.log --- response_body eval qr/.*route_id.*1.*/ + + + +=== TEST 16: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) + local code, body1 = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tcp-logger": { + "host": "127.0.0.1", + "port": 43000, + "tls": true, + "batch_max_size": 1, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 17: add plugin with 'include_resp_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/tcp-logger', ngx.HTTP_DELETE) + local code, body1 = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "tcp-logger": { + "host": "127.0.0.1", + "port": 43000, + "tls": true, + "batch_max_size": 1, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say("fail") + return + end + + local code, _, body = t("/opentracing", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"opentracing\n" diff --git a/t/plugin/udp-logger.t b/t/plugin/udp-logger.t index 44744823634b..ba4f8f6d32ed 100644 --- a/t/plugin/udp-logger.t +++ b/t/plugin/udp-logger.t @@ -440,7 +440,99 @@ passed -=== TEST 12: check log format from logging server +=== TEST 12: add plugin with 'include_req_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "udp-logger": { + "host": "127.0.0.1", + "port": 8127, + "tls": false, + "batch_max_size": 1, + "inactive_timeout": 1, + "include_req_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"{\"sample_payload\":\"hello\"}" + + + +=== TEST 13: add plugin with 'include_resp_body' setting, collect request log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + t('/apisix/admin/plugin_metadata/udp-logger', ngx.HTTP_DELETE) + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "udp-logger": { + "host": "127.0.0.1", + "port": 8127, + "tls": false, + "batch_max_size": 1, + "inactive_timeout": 1, + "include_resp_body": true + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + + + local code, _, body = t("/hello", "POST", "{\"sample_payload\":\"hello\"}") + } + } +--- request +GET /t +--- error_log +"body":"hello world\n" + + + +=== TEST 14: check log format from logging server --- exec tail -n 1 ci/pod/vector/udp.log --- response_body eval From c5224c18f3980593d9335265eff7bd5d8ee0c104 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Thu, 1 Feb 2024 12:41:39 +0800 Subject: [PATCH 10/12] Resolving conflicts --- t/plugin/elasticsearch-logger.t | 57 +++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/t/plugin/elasticsearch-logger.t b/t/plugin/elasticsearch-logger.t index ef2434d2a293..5174a5103b45 100644 --- a/t/plugin/elasticsearch-logger.t +++ b/t/plugin/elasticsearch-logger.t @@ -671,7 +671,59 @@ check elasticsearch custom body success -=== TEST 17: add plugin with 'include_req_body' setting, collect request log +=== TEST 17: using unsupported field (type) for elasticsearch v8 should work normally +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', ngx.HTTP_PUT, { + uri = "/hello", + upstream = { + type = "roundrobin", + nodes = { + ["127.0.0.1:1980"] = 1 + } + }, + plugins = { + ["elasticsearch-logger"] = { + endpoint_addr = "http://127.0.0.1:9201", + field = { + index = "services", + type = "collector" + }, + auth = { + username = "elastic", + password = "123456" + }, + batch_max_size = 1, + inactive_timeout = 1 + } + } + }) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 18: test route (auth success) +--- request +GET /hello +--- wait: 2 +--- response_body +hello world +--- no_error_log +Action/metadata line [1] contains an unknown parameter [_type] + + + +=== TEST 19: add plugin with 'include_req_body' setting, collect request log --- config location /t { content_by_lua_block { @@ -711,7 +763,7 @@ check elasticsearch custom body success -=== TEST 18: add plugin with 'include_resp_body' setting, collect response log +=== TEST 20: add plugin with 'include_resp_body' setting, collect response log --- config location /t { content_by_lua_block { @@ -749,3 +801,4 @@ check elasticsearch custom body success } --- error_log "body":"hello world\n" + From e27691da79ef836b05ab0890a9481d8cc9440bbb Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Thu, 1 Feb 2024 12:51:52 +0800 Subject: [PATCH 11/12] Cancel log encode --- apisix/plugins/sls-logger.lua | 2 +- t/plugin/sls-logger.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apisix/plugins/sls-logger.lua b/apisix/plugins/sls-logger.lua index a47009ec326d..114a209690e5 100644 --- a/apisix/plugins/sls-logger.lua +++ b/apisix/plugins/sls-logger.lua @@ -167,7 +167,7 @@ function _M.log(conf, ctx) } local rf5424_data = rf5424.encode("SYSLOG", "INFO", ctx.var.host, "apisix", ctx.var.pid, json_str, structured_data) - core.log.info("collect_data:" .. core.json.encode(rf5424_data)) + core.log.info("collect_data:" .. rf5424_data) local process_context = { data = rf5424_data, route_conf = conf diff --git a/t/plugin/sls-logger.t b/t/plugin/sls-logger.t index 070f8dbcddc5..af6ae667c34f 100644 --- a/t/plugin/sls-logger.t +++ b/t/plugin/sls-logger.t @@ -428,7 +428,7 @@ hello world } } --- error_log -\"body\":\"{\\\"sample_payload\\\":\\\"hello\\\" +"body":"{\"sample_payload\":\"hello\"} @@ -471,4 +471,4 @@ hello world } } --- error_log -\"body\":\"hello world\\n\" +"body":"hello world\n" From fa53f3bbc6a14d27389f6a82482c8925c19c9608 Mon Sep 17 00:00:00 2001 From: baiyun <337531158@qq.com> Date: Thu, 1 Feb 2024 13:48:17 +0800 Subject: [PATCH 12/12] fix error --- t/plugin/udp-logger.t | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/t/plugin/udp-logger.t b/t/plugin/udp-logger.t index ba4f8f6d32ed..b20248b191c5 100644 --- a/t/plugin/udp-logger.t +++ b/t/plugin/udp-logger.t @@ -440,7 +440,15 @@ passed -=== TEST 12: add plugin with 'include_req_body' setting, collect request log +=== TEST 12: check log format from logging server +--- exec +tail -n 1 ci/pod/vector/udp.log +--- response_body eval +qr/.*logger format in plugin.*/ + + + +=== TEST 13: add plugin with 'include_req_body' setting, collect request log --- config location /t { content_by_lua_block { @@ -486,7 +494,7 @@ GET /t -=== TEST 13: add plugin with 'include_resp_body' setting, collect request log +=== TEST 14: add plugin with 'include_resp_body' setting, collect request log --- config location /t { content_by_lua_block { @@ -529,11 +537,3 @@ GET /t GET /t --- error_log "body":"hello world\n" - - - -=== TEST 14: check log format from logging server ---- exec -tail -n 1 ci/pod/vector/udp.log ---- response_body eval -qr/.*logger format in plugin.*/