feat: Add ability to inject headers via prefix to otel traces (#7822)

Co-authored-by: tzssangglass <[email protected]>

apisix/plugins/opentelemetry.lua

@@ -53,6 +53,7 @@ local lrucache = core.lrucache.new({
     type = 'plugin', count = 128, ttl = 24 * 60 * 60,
 })
 
+local asterisk = string.byte("*", 1)
 
 local attr_schema = {
     type = "object",
@@ -169,6 +170,13 @@ local schema = {
                 type = "string",
                 minLength = 1,
             }
+        },
+        additional_header_prefix_attributes = {
+            type = "array",
+            items = {
+                type = "string",
+                minLength = 1,
+            }
         }
     }
 }
@@ -273,6 +281,27 @@ local function create_tracer_obj(conf)
 end
 
 
+local function inject_attributes(attributes, wanted_attributes, source, with_prefix)
+    for _, key in ipairs(wanted_attributes) do
+        local is_key_a_match = #key >= 2 and key:byte(-1) == asterisk and with_prefix
+
+        if is_key_a_match then
+            local prefix = key:sub(0, -2)
+            for possible_key, value in pairs(source) do
+                if core.string.has_prefix(possible_key, prefix) then
+                    core.table.insert(attributes, attr.string(possible_key, value))
+                end
+            end
+        else
+            local val = source[key]
+            if val then
+                core.table.insert(attributes, attr.string(key, val))
+            end
+        end
+    end
+end
+
+
 function _M.rewrite(conf, api_ctx)
     local tracer, err = core.lrucache.plugin_ctx(lrucache, api_ctx, nil, create_tracer_obj, conf)
     if not tracer then
@@ -286,13 +315,18 @@ function _M.rewrite(conf, api_ctx)
         attr.string("service", api_ctx.service_name),
         attr.string("route", api_ctx.route_name),
     }
+
     if conf.additional_attributes then
-        for _, key in ipairs(conf.additional_attributes) do
-            local val = api_ctx.var[key]
-            if val then
-                core.table.insert(attributes, attr.string(key, val))
-            end
-        end
+        inject_attributes(attributes, conf.additional_attributes, api_ctx.var, false)
+    end
+
+    if conf.additional_header_prefix_attributes then
+        inject_attributes(
+            attributes,
+            conf.additional_header_prefix_attributes,
+            core.request.headers(api_ctx),
+            true
+        )
     end
 
     local ctx = tracer:start(upstream_context, api_ctx.var.request_uri, {

docs/en/latest/plugins/opentelemetry.md

@@ -46,6 +46,8 @@ The Plugin only supports binary-encoded [OLTP over HTTP](https://opentelemetry.i
 | sampler.options.root.options.fraction | number        | False    | 0                                               | [0, 1]                                                       | Root sampling probability for `trace_id_ratio`.                                                                                                                                                                                              |
 | additional_attributes                 | array[string] | False    |                                                 |                                                              | Variables and its values which will be appended to the trace span.                                                                                                                                                                           |
 | additional_attributes[0]              | string        | True     |                                                 |                                                              | APISIX or Nginx variables. For example, `http_header` or `route_id`.                                                                                                                                                                         |
+| additional_header_prefix_attributes   | array[string] | False    |                                                 |                                                              | Headers or headers prefixes to be appended to the trace span's attributes.                                                                                                                                                                   |
+| additional_header_prefix_attributes[0]| string        | True     |                                                 |                                                              | Request headers. For example, `x-my-header"` or `x-my-headers-*` to include all headers with the prefix `x-my-headers-`.                                                                                                                     |
 
 ### Configuring the collector
 

t/plugin/opentelemetry2.t

@@ -142,3 +142,88 @@ plugin body_filter phase
 opentelemetry context current
 opentelemetry context current
 opentelemetry export span
+
+
+
+=== TEST 3: set additional_attributes with match
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "name": "route_name",
+                    "plugins": {
+                        "opentelemetry": {
+                            "sampler": {
+                                "name": "always_on"
+                            },
+                            "additional_header_prefix_attributes": [
+                                "x-my-header-*"
+                            ]
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/attributes"
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 4: opentelemetry expands headers
+--- extra_init_by_lua
+    local otlp = require("opentelemetry.trace.exporter.otlp")
+    otlp.export_spans = function(self, spans)
+        if (#spans ~= 1) then
+            ngx.log(ngx.ERR, "unexpected spans length: ", #spans)
+            return
+        end
+
+        local attributes_names = {}
+        local attributes = {}
+        local span = spans[1]
+        for _, attribute in ipairs(span.attributes) do
+            if attribute.key == "hostname" then
+                -- remove any randomness
+                goto skip
+            end
+            table.insert(attributes_names, attribute.key)
+            attributes[attribute.key] = attribute.value.string_value or ""
+            ::skip::
+        end
+        table.sort(attributes_names)
+        for _, attribute in ipairs(attributes_names) do
+            ngx.log(ngx.INFO, "attribute " .. attribute .. ": \"" .. attributes[attribute] .. "\"")
+        end
+
+        ngx.log(ngx.INFO, "opentelemetry export span")
+    end
+--- request
+GET /attributes
+--- more_headers
+x-my-header-name: william
+x-my-header-nick: bill
+--- wait: 1
+--- error_code: 404
+--- grep_error_log eval
+qr/attribute .+?:.[^,]*/
+--- grep_error_log_out
+attribute route: "route_name"
+attribute service: ""
+attribute x-my-header-name: "william"
+attribute x-my-header-nick: "bill"