From 3d9e2fdbae78d2f4c821e81ae3e90f9545ee55ce Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Mon, 1 Aug 2022 14:20:11 +0800 Subject: [PATCH] feat(prometheus): extra labels from nginx var for http request metrics (#7549) close #4273 Co-authored-by: tzssangglass --- apisix/plugins/prometheus/exporter.lua | 57 ++++++++-- conf/config-default.yaml | 14 +++ docs/en/latest/plugins/prometheus.md | 23 ++++ t/plugin/prometheus4.t | 140 +++++++++++++++++++++++++ 4 files changed, 225 insertions(+), 9 deletions(-) mode change 100644 => 100755 conf/config-default.yaml create mode 100644 t/plugin/prometheus4.t diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index e6b61b26b604..45ff94c3f631 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -39,6 +39,8 @@ local get_protos = require("apisix.plugins.grpc-transcode.proto").protos local service_fetch = require("apisix.http.service").get local latency_details = require("apisix.utils.log-util").latency_details_in_ms local xrpc = require("apisix.stream.xrpc") +local unpack = unpack +local next = next local ngx_capture @@ -65,6 +67,31 @@ local function gen_arr(...) return inner_tab_arr end +local extra_labels_tbl = {} + +local function extra_labels(name, ctx) + clear_tab(extra_labels_tbl) + + local attr = plugin.plugin_attr("prometheus") + local metrics = attr.metrics + + if metrics and metrics[name] and metrics[name].extra_labels then + local labels = metrics[name].extra_labels + for _, kv in ipairs(labels) do + local val, v = next(kv) + if ctx then + val = ctx.var[v:sub(2)] + if val == nil then + val = "" + end + end + core.table.insert(extra_labels_tbl, val) + end + end + + return extra_labels_tbl +end + local _M = {} @@ -138,15 +165,17 @@ function _M.http_init(prometheus_enabled_in_stream) -- no consumer in request. metrics.status = prometheus:counter("http_status", "HTTP status codes per service in APISIX", - {"code", "route", "matched_uri", "matched_host", "service", "consumer", "node"}) + {"code", "route", "matched_uri", "matched_host", "service", "consumer", "node", + unpack(extra_labels("http_status"))}) metrics.latency = prometheus:histogram("http_latency", "HTTP request latency in milliseconds per service in APISIX", - {"type", "route", "service", "consumer", "node"}, DEFAULT_BUCKETS) + {"type", "route", "service", "consumer", "node", unpack(extra_labels("http_latency"))}, + DEFAULT_BUCKETS) metrics.bandwidth = prometheus:counter("bandwidth", "Total bandwidth in bytes consumed per service in APISIX", - {"type", "route", "service", "consumer", "node"}) + {"type", "route", "service", "consumer", "node", unpack(extra_labels("bandwidth"))}) if prometheus_enabled_in_stream then init_stream_metrics() @@ -208,25 +237,35 @@ function _M.http_log(conf, ctx) metrics.status:inc(1, gen_arr(vars.status, route_id, matched_uri, matched_host, - service_id, consumer_name, balancer_ip)) + service_id, consumer_name, balancer_ip, + unpack(extra_labels("http_status", ctx)))) local latency, upstream_latency, apisix_latency = latency_details(ctx) + local latency_extra_label_values = extra_labels("http_latency", ctx) + metrics.latency:observe(latency, - gen_arr("request", route_id, service_id, consumer_name, balancer_ip)) + gen_arr("request", route_id, service_id, consumer_name, balancer_ip, + unpack(latency_extra_label_values))) if upstream_latency then metrics.latency:observe(upstream_latency, - gen_arr("upstream", route_id, service_id, consumer_name, balancer_ip)) + gen_arr("upstream", route_id, service_id, consumer_name, balancer_ip, + unpack(latency_extra_label_values))) end metrics.latency:observe(apisix_latency, - gen_arr("apisix", route_id, service_id, consumer_name, balancer_ip)) + gen_arr("apisix", route_id, service_id, consumer_name, balancer_ip, + unpack(latency_extra_label_values))) + + local bandwidth_extra_label_values = extra_labels("bandwidth", ctx) metrics.bandwidth:inc(vars.request_length, - gen_arr("ingress", route_id, service_id, consumer_name, balancer_ip)) + gen_arr("ingress", route_id, service_id, consumer_name, balancer_ip, + unpack(bandwidth_extra_label_values))) metrics.bandwidth:inc(vars.bytes_sent, - gen_arr("egress", route_id, service_id, consumer_name, balancer_ip)) + gen_arr("egress", route_id, service_id, consumer_name, balancer_ip, + unpack(bandwidth_extra_label_values))) end diff --git a/conf/config-default.yaml b/conf/config-default.yaml old mode 100644 new mode 100755 index 0c088d144785..c565b4869b1c --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -522,6 +522,20 @@ plugin_attr: export_addr: ip: 127.0.0.1 port: 9091 + #metrics: + # http_status: + # # extra labels from nginx variables + # extra_labels: + # # the label name doesn't need to be the same as variable name + # # below labels are only examples, you could add any valid variables as you need + # - upstream_addr: $upstream_addr + # - upstream_status: $upstream_status + # http_latency: + # extra_labels: + # - upstream_addr: $upstream_addr + # bandwidth: + # extra_labels: + # - upstream_addr: $upstream_addr server-info: report_ttl: 60 # live time for server info in etcd (unit: second) dubbo-proxy: diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 8f0b4383e357..4b5e97a5263d 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -53,6 +53,29 @@ plugin_attr: export_uri: /apisix/metrics ``` +### Specifying `metrics` + +For http request related metrics, you could specify extra labels, which match the APISIX variables. + +If you specify label for nonexist APISIX variable, the label value would be "". + +Currently, only below metrics are supported: + +* http_status +* http_latency +* bandwidth + +Here is a configuration example: + +```yaml title="conf/config.yaml" +plugin_attr: + prometheus: + metrics: + http_status: + extra_labels: + - upstream_addr: $upstream_addr + - upstream_status: $upstream_status + ## API This Plugin will add the API endpoint `/apisix/prometheus/metrics` or your custom export URI for exposing the metrics. diff --git a/t/plugin/prometheus4.t b/t/plugin/prometheus4.t new file mode 100644 index 000000000000..93302028e68b --- /dev/null +++ b/t/plugin/prometheus4.t @@ -0,0 +1,140 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +BEGIN { + if ($ENV{TEST_NGINX_CHECK_LEAK}) { + $SkipReason = "unavailable for the hup tests"; + + } else { + $ENV{TEST_NGINX_USE_HUP} = 1; + undef $ENV{TEST_NGINX_USE_STAP}; + } +} + +use t::APISIX 'no_plan'; + +add_block_preprocessor(sub { + my ($block) = @_; + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: pre-create public API route +--- config + location /t { + content_by_lua_block { + + local t = require("lib.test_admin").test + local code = t('/apisix/admin/routes/metrics', + ngx.HTTP_PUT, + [[{ + "plugins": { + "public-api": {} + }, + "uri": "/apisix/prometheus/metrics" + }]] + ) + if code >= 300 then + ngx.status = code + return + end + } + } + + + +=== TEST 2: set route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/10', + ngx.HTTP_PUT, + [[{ + "plugins": { + "prometheus": {} + }, + "upstream": { + "nodes": { + "127.0.0.1:1980": 1 + }, + "type": "roundrobin" + }, + "uri": "/hello" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 3: client request +--- yaml_config +plugin_attr: + prometheus: + metrics: + bandwidth: + extra_labels: + - upstream_addr: $upstream_addr + - upstream_status: $upstream_status +--- request +GET /hello + + + +=== TEST 4: fetch the prometheus metric data +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_bandwidth\{type="egress",route="10",service="",consumer="",node="127.0.0.1",upstream_addr="127.0.0.1:1980",upstream_status="200"\} \d+/ + + + +=== TEST 5: client request, label with nonexist ngx variable +--- yaml_config +plugin_attr: + prometheus: + metrics: + http_status: + extra_labels: + - dummy: $dummy +--- request +GET /hello + + + +=== TEST 6: fetch the prometheus metric data, with nonexist ngx variable +--- request +GET /apisix/prometheus/metrics +--- response_body eval +qr/apisix_http_status\{code="200",route="10",matched_uri="\/hello",matched_host="",service="",consumer="",node="127.0.0.1",dummy=""\} \d+/