feat: Support store secrets in secrets manager for auth plugin via kms components

apisix/admin/kms.lua

@@ -14,10 +14,14 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
+local require = require
+
 local core = require("apisix.core")
 local utils = require("apisix.admin.utils")
+
 local type = type
 local tostring = tostring
+local pcall = pcall
 
 
 local _M = {
@@ -46,7 +50,12 @@ local function check_conf(id, conf, need_id, typ)
     conf.id = id
 
     core.log.info("conf: ", core.json.delay_encode(conf))
-    local ok, err = core.schema.check(core.schema["kms_" .. typ], conf)
+    local ok, kms_service = pcall(require, "apisix.kms." .. typ)
+    if not ok then
+        return false, {error_msg = "invalid kms service: " .. typ}
+    end
+
+    local ok, err = core.schema.check(kms_service.schema, conf)
     if not ok then
         return nil, {error_msg = "invalid configuration: " .. err}
     end

apisix/consumer.lua

@@ -15,6 +15,7 @@
 -- limitations under the License.
 --
 local core           = require("apisix.core")
+local kms            = require("apisix.kms")
 local plugin         = require("apisix.plugin")
 local plugin_checker = require("apisix.plugin").plugin_checker
 local error          = error
@@ -103,7 +104,7 @@ local function create_consume_cache(consumers_conf, key_attr)
     for _, consumer in ipairs(consumers_conf.nodes) do
         core.log.info("consumer node: ", core.json.delay_encode(consumer))
         local new_consumer = core.table.clone(consumer)
-        new_consumer.auth_conf = core.utils.retrieve_secrets_ref(new_consumer.auth_conf)
+        new_consumer.auth_conf = kms.fetch_secrets(new_consumer.auth_conf)
         consumer_names[new_consumer.auth_conf[key_attr]] = new_consumer
     end
 

apisix/core/env.lua

@@ -27,10 +27,13 @@ local find        = string.find
 local sub         = string.sub
 local str         = ffi.string
 
-local _M = {}
-
 local ENV_PREFIX = "$ENV://"
 
+local _M = {
+    PREFIX = ENV_PREFIX
+}
+
+
 local apisix_env_vars = {}
 
 ffi.cdef [[
@@ -39,32 +42,35 @@ ffi.cdef [[
 
 
 function _M.init()
-  local e = ffi.C.environ
-  if not e then
-    log.warn("could not access environment variables")
-    return
-  end
-
-  local i = 0
-  while e[i] ~= nil do
-    local var = str(e[i])
-    local p = find(var, "=")
-    if p then
-        apisix_env_vars[sub(var, 1, p - 1)] = sub(var, p + 1)
+    local e = ffi.C.environ
+    if not e then
+        log.warn("could not access environment variables")
+        return
     end
 
-    i = i + 1
-  end
+    local i = 0
+    while e[i] ~= nil do
+        local var = str(e[i])
+        local p = find(var, "=")
+        if p then
+            apisix_env_vars[sub(var, 1, p - 1)] = sub(var, p + 1)
+        end
+
+        i = i + 1
+    end
 end
 
 
-local function is_env_uri(env_uri)
+local function parse_env_uri(env_uri)
     -- Avoid the error caused by has_prefix to cause a crash.
-    return type(env_uri) == "string" and string.has_prefix(upper(env_uri), ENV_PREFIX)
-end
+    if type(env_uri) ~= "string" then
+        return nil, "error env_uri type: " .. type(env_uri)
+    end
 
+    if not string.has_prefix(upper(env_uri), ENV_PREFIX) then
+        return nil, "error env_uri prefix: " .. env_uri
+    end
 
-local function parse_env_uri(env_uri)
     local path = sub(env_uri, #ENV_PREFIX + 1)
     local idx = find(path, "/")
     if not idx then
@@ -80,18 +86,17 @@ local function parse_env_uri(env_uri)
 end
 
 
-function _M.get(env_uri)
-    if not is_env_uri(env_uri) then
-        return nil
+function _M.fetch_by_uri(env_uri)
+    local opts, err = parse_env_uri(env_uri)
+    if not opts then
+        return nil, err
     end
 
-    local opts = parse_env_uri(env_uri)
     local main_value = apisix_env_vars[opts.key] or os.getenv(opts.key)
     if main_value and opts.sub_key ~= "" then
         local vt, err = json.decode(main_value)
         if not vt then
-          log.warn("decode failed, err: ", err, " value: ", main_value)
-          return nil
+            return nil, "decode failed, err: " .. (err or "") .. ", value: " .. main_value
         end
         return vt[opts.sub_key]
     end

apisix/core/utils.lua

@@ -25,8 +25,6 @@ local rfind_char     = core_str.rfind_char
 local table          = require("apisix.core.table")
 local log            = require("apisix.core.log")
 local string         = require("apisix.core.string")
-local env            = require("apisix.core.env")
-local lrucache       = require("apisix.core.lrucache")
 local dns_client     = require("apisix.core.dns.client")
 local ngx_re         = require("ngx.re")
 local ipmatcher      = require("resty.ipmatcher")
@@ -37,7 +35,6 @@ local sub_str        = string.sub
 local str_byte       = string.byte
 local tonumber       = tonumber
 local tostring       = tostring
-local pairs          = pairs
 local re_gsub        = ngx.re.gsub
 local type           = type
 local io_popen       = io.popen
@@ -332,57 +329,4 @@ end
 _M.resolve_var = resolve_var
 
 
-local secrets_lrucache = lrucache.new({
-    ttl = 300, count = 512
-})
-
-local retrieve_secrets_ref
-do
-    local retrieve_ref
-    function retrieve_ref(refs)
-        for k, v in pairs(refs) do
-            local typ = type(v)
-            if typ == "string" then
-                refs[k] = env.get(v) or v
-            elseif typ == "table" then
-                retrieve_ref(v)
-            end
-        end
-        return refs
-    end
-
-    local function retrieve(refs)
-        log.info("retrieve secrets refs")
-
-        local new_refs = table.deepcopy(refs)
-        return retrieve_ref(new_refs)
-    end
-
-    function retrieve_secrets_ref(refs, cache, key, version)
-        if not refs or type(refs) ~= "table" then
-            return nil
-        end
-        if not cache then
-            return retrieve(refs)
-        end
-        return secrets_lrucache(key, version, retrieve, refs)
-    end
-end
--- Retrieve all secrets ref in the given table
----
--- Retrieve all secrets ref in the given table,
--- and then replace them with the values from the environment variables.
---
--- @function core.utils.retrieve_secrets_ref
--- @tparam table refs The table to be retrieved.
--- @tparam boolean cache Whether to use lrucache to cache results.
--- @tparam string key The cache key for lrucache.
--- @tparam string version The cache version for lrucache.
--- @treturn table The table after the reference is replaced.
--- @usage
--- local new_refs = core.utils.retrieve_secrets_ref(refs) -- "no cache"
--- local new_refs = core.utils.retrieve_secrets_ref(refs, true, key, ver) -- "cache"
-_M.retrieve_secrets_ref = retrieve_secrets_ref
-
-
 return _M

apisix/init.lua

@@ -37,6 +37,7 @@ local admin_init      = require("apisix.admin.init")
 local get_var         = require("resty.ngxvar").fetch
 local router          = require("apisix.router")
 local apisix_upstream = require("apisix.upstream")
+local apisix_kms      = require("apisix.kms")
 local set_upstream    = apisix_upstream.set_by_route
 local apisix_ssl      = require("apisix.ssl")
 local upstream_util   = require("apisix.utils.upstream")
@@ -150,6 +151,7 @@ function _M.http_init_worker()
     plugin_config.init_worker()
     require("apisix.consumer").init_worker()
     consumer_group.init_worker()
+    apisix_kms.init_worker()
 
     apisix_upstream.init_worker()
     require("apisix.plugins.ext-plugin.init").init_worker()