feat: support aws secret manager

apisix-master-0.rockspec

@@ -81,7 +81,8 @@ dependencies = {
     "lua-resty-ldap = 0.1.0-0",
     "lua-resty-t1k = 1.1.5",
     "brotli-ffi = 0.3-1",
-    "lua-ffi-zlib = 0.6-0"
+    "lua-ffi-zlib = 0.6-0",
+    "api7-lua-resty-aws == 2.0.1-1",
 }
 
 build = {

apisix/secret/aws.lua

@@ -0,0 +1,135 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+--- AWS Tools.
+local core = require("apisix.core")
+local http = require("resty.http")
+local aws = require("resty.aws")
+
+local sub = core.string.sub
+local find = core.string.find
+local env = core.env
+local unpack = unpack
+
+local schema = {
+    type = "object",
+    properties = {
+        access_key_id = {
+            type = "string",
+        },
+        secret_access_key = {
+            type = "string",
+        },
+        session_token = {
+            type = "string",
+        },
+        region = {
+            type = "string",
+            default = "us-east-1",
+        },
+        endpoint_url = core.schema.uri_def,
+    },
+    required = {"access_key_id", "secret_access_key"},
+}
+
+local _M = {
+    schema = schema
+}
+
+local function make_request_to_aws(conf, key)
+    local aws_instance = aws()
+
+    local region = conf.region
+
+    local access_key_id = env.fetch_by_uri(conf.access_key_id) or conf.access_key_id
+
+    local secret_access_key = env.fetch_by_uri(conf.secret_access_key) or conf.secret_access_key
+
+    local session_token = env.fetch_by_uri(conf.session_token) or conf.session_token
+
+    local credentials = aws_instance:Credentials({
+        accessKeyId = access_key_id,
+        secretAccessKey = secret_access_key,
+        sessionToken = session_token,
+    })
+
+    local default_endpoint = "https://secretsmanager." .. region .. ".amazonaws.com"
+    local scheme, host, port, _, _ = unpack(http:parse_uri(conf.endpoint_url or default_endpoint))
+    local endpoint = scheme .. "://" .. host
+
+    local sm = aws_instance:SecretsManager({
+        credentials = credentials,
+        endpoint = endpoint,
+        region = region,
+        port = port,
+    })
+
+    local res, err = sm:getSecretValue({
+        SecretId = key,
+        VersionStage = "AWSCURRENT",
+    })
+
+    if not res then
+        return nil, err
+    end
+
+    if res.status ~= 200 then
+        local data = core.json.encode(res.body)
+        if data then
+            return nil, "invalid status code " .. res.status .. ", " .. data
+        end
+
+        return nil, "invalid status code " .. res.status
+    end
+
+    return res.body.SecretString
+end
+
+-- key is the aws secretId
+function _M.get(conf, key)
+    core.log.info("fetching data from aws for key: ", key)
+
+    local idx = find(key, '/')
+
+    local main_key = idx and sub(key, 1, idx - 1) or key
+    if main_key == "" then
+        return nil, "can't find main key, key: " .. key
+    end
+
+    local sub_key = idx and sub(key, idx + 1) or nil
+
+    core.log.info("main: ", main_key, sub_key and ", sub: " .. sub_key or "")
+
+    local res, err = make_request_to_aws(conf, main_key)
+    if not res then
+        return nil, "failed to retrtive data from aws secret manager: " .. err
+    end
+
+    if not sub_key then
+        return res
+    end
+
+    local data, err = core.json.decode(res)
+    if not data then
+        return nil, "failed to decode result, res: " .. res .. ", err: " .. err
+    end
+
+    return data[sub_key]
+end
+
+
+return _M

ci/init-common-test-service.sh

@@ -19,3 +19,9 @@
 # prepare vault kv engine
 sleep 3s
 docker exec -i vault sh -c "VAULT_TOKEN='root' VAULT_ADDR='http://0.0.0.0:8200' vault secrets enable -path=kv -version=1 kv"
+
+# prepare localstack
+sleep 3s
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name apisix-key --description 'APISIX Secret' --secret-string '{\"jack\":\"value\"}'"
+sleep 3s
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name apisix-mysql --description 'APISIX Secret' --secret-string 'secret'"

ci/pod/docker-compose.common.yml

@@ -102,3 +102,12 @@ services:
       VAULT_DEV_ROOT_TOKEN_ID: root
       VAULT_DEV_LISTEN_ADDRESS: 0.0.0.0:8200
     command: [ "vault", "server", "-dev" ]
+
+
+  ## LocalStack
+  localstack:
+    image: localstack/localstack
+    container_name: localstack
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway

docs/en/latest/terminology/secret.md

@@ -38,7 +38,8 @@ Its working principle is shown in the figure:
 APISIX currently supports storing secrets in the following ways:
 
 - [Environment Variables](#use-environment-variables-to-manage-secrets)
-- [HashiCorp Vault](#use-vault-to-manage-secrets)
+- [HashiCorp Vault](#use-hashicorp-vault-to-manage-secrets)
+- [AWS Secrets Manager](#use-aws-secrets-manager-to-manage-secrets)
 
 You can use APISIX Secret functions by specifying format variables in the consumer configuration of the following plugins, such as `key-auth`.
 
@@ -190,3 +191,105 @@ curl http://127.0.0.1:9180/apisix/admin/consumers \
 ```
 
 Through the above two steps, when the user request hits the `key-auth` plugin, the real value of the key in the Vault will be obtained through the APISIX Secret component.
+
+## Use AWS Secrets Manager to manage secrets
+
+Managing secrets with AWS Secrets Manager is a secure and convenient way to store and manage sensitive information. This method allows you to save secret information in AWS Secrets Manager and reference these secrets in a specific format when configuring APISIX plugins.
+
+APISIX currently supports two authentication methods: using [long-term credentials](https://docs.aws.amazon.com/sdkref/latest/guide/access-iam-users.html) and [short-term credentials](https://docs.aws.amazon.com/sdkref/latest/guide/access-temp-idc.html).
+
+### Usage
+
+```
+$secret://$manager/$id/$secret_name/$key
+```
+
+- manager: secrets management service, could be the HashiCorp Vault, AWS, etc.
+- id: APISIX Secrets resource ID, which needs to be consistent with the one specified when adding the APISIX Secrets resource
+- secret_name: the secret name in the secrets management service
+- key: get the value of a property when the value of the secret is a JSON string
+
+### Required Parameters
+
+| Name | Required | Default Value | Description |
+| --- | --- | --- | --- |
+| access_key_id | True |  | AWS Access Key ID |
+| secret_access_key | True |  | AWS Secret Access Key |
+| session_token | False |  | Temporary access credential information |
+| region | False | us-east-1 | AWS Region |
+| endpoint_url | False | https://secretsmanager.{region}.amazonaws.com | AWS Secret Manager URL |
+
+### Example: use in key-auth plugin
+
+Here, we use the key-auth plugin as an example to demonstrate how to manage secrets through AWS Secrets Manager.
+
+Step 1: Create the corresponding key in the AWS secrets manager. Here, [localstack](https://www.localstack.cloud/) is used for as the example environment, and you can use the following command:
+
+```shell
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name jack --description 'APISIX Secret' --secret-string '{\"auth-key\":\"value\"}'"
+```
+
+Step 2: Add APISIX Secrets resources through the Admin API, configure the connection information such as the address of AWS Secrets Manager.
+
+You can store the critical key information in environment variables to ensure the configuration information is secure, and reference it where it is used:
+
+```shell
+export AWS_ACCESS_KEY_ID=<access_key_id>
+export AWS_SECRET_ACCESS_KEY=<secrets_access_key>
+export AWS_SESSION_TOKEN=<token>
+export AWS_REGION=<aws-region>
+```
+
+Alternatively, you can also specify all the information directly in the configuration:
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/secrets/aws/1 \
+-H "X-API-KEY: $admin_key" -X PUT -d '
+{
+    "endpoint_url": "http://127.0.0.1:4566",
+    "region": "us-east-1",
+    "access_key_id": "access",
+    "secret_access_key": "secret",
+    "session_token": "token"
+}'
+```
+
+If you use APISIX Standalone mode, you can add the following configuration in `apisix.yaml` configuration file:
+
+```yaml
+secrets:
+  - id: aws/1
+    endpoint_url: http://127.0.0.1:4566
+    region: us-east-1
+    access_key_id: access
+    secret_access_key: secret
+    session_token: token
+```
+
+Step 3: Reference the APISIX Secrets resource in the `key-auth` plugin and fill in the key information:
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/consumers \
+-H "X-API-KEY: $admin_key" -X PUT -d '
+{
+    "username": "jack",
+    "plugins": {
+        "key-auth": {
+            "key": "$secret://aws/1/jack/auth-key"
+        }
+    }
+}'
+```
+
+Through the above two steps, when the user request hits the `key-auth` plugin, the real value of the key in the Vault will be obtained through the APISIX Secret component.
+
+### Verification
+
+You can verify this with the following command:
+
+```shell
+#Replace the following your_route with the actual route path.
+curl -i http://127.0.0.1:9080/your_route -H 'apikey: value'
+```
+
+This will verify whether the `key-auth` plugin is correctly using the key from AWS Secrets Manager.

docs/zh/latest/terminology/secret.md

@@ -39,6 +39,7 @@ APISIX 目前支持通过以下方式存储密钥：
 
 - [环境变量](#使用环境变量管理密钥)
 - [HashiCorp Vault](#使用-vault-管理密钥)
+- [AWS Secrets Manager](#使用-aws-secrets-manager-管理密钥)
 
 你可以在以下插件的 consumer 配置中通过指定格式的变量来使用 APISIX Secret 功能，比如 `key-auth` 插件。
 
@@ -191,3 +192,106 @@ curl http://127.0.0.1:9180/apisix/admin/consumers \
 ```
 
 通过上面两步操作，当用户请求命中 `key-auth` 插件时，会通过 APISIX Secret 组件获取到 key 在 Vault 中的真实值。
+
+## 使用 AWS Secrets Manager 管理密钥
+
+使用 AWS Secrets Manager 管理密钥是一种安全且便捷的方式来存储和管理敏感信息。通过这种方式，你可以将密钥信息保存在 AWS Secret Manager 中，并在配置 APISIX 插件时通过特定的格式引用这些密钥。
+
+APISIX 目前支持两种访问方式： [长期凭证的访问方式](https://docs.aws.amazon.com/zh_cn/sdkref/latest/guide/access-iam-users.html) 和 [短期凭证的访问方式](https://docs.aws.amazon.com/zh_cn/sdkref/latest/guide/access-temp-idc.html)。
+
+### 引用方式
+
+在 APISIX 中引用密钥时，可以使用以下格式：
+
+```
+$secret://$manager/$id/$secret_name/$key
+```
+
+- manager: 密钥管理服务，可以是 Vault、AWS 等
+- APISIX Secret 资源 ID，需要与添加 APISIX Secret 资源时指定的 ID 保持一致
+- secret_name: 密钥管理服务中的密钥名称
+- key：当密钥的值是 JSON 字符串时，获取某个属性的值
+
+### 相关参数
+
+| 名称 | 必选项 | 默认值 | 描述 |
+| --- | --- | --- | --- |
+| access_key_id | 是 |  | AWS 访问密钥 ID |
+| secret_access_key | 是 |  | AWS 访问密钥 |
+| session_token | 否 |  | 临时访问凭证信息 |
+| region | 否 | us-east-1 | AWS 区域 |
+| endpoint_url | 否 | https://secretsmanager.{region}.amazonaws.com | AWS Secret Manager 地址 |
+
+### 示例：在 key-auth 插件中使用
+
+这里以 key-auth 插件的使用为例，展示如何通过 AWS Secret Manager 管理密钥：
+
+第一步：在 AWS Secret Manager 中创建对应的密钥，这里使用 [localstack](https://www.localstack.cloud/) 模拟，可以使用如下命令：
+
+```shell
+docker exec -i localstack sh -c "awslocal secretsmanager create-secret --name jack --description 'APISIX Secret' --secret-string '{\"auth-key\":\"value\"}'"
+```
+
+第二步：通过 Admin API 添加 Secret 资源，配置 AWS Secret Manager 的地址等连接信息：
+
+你可以在环境变量中存储关键密钥信息，保证配置信息是安全的，在使用到地方进行引用：
+
+```shell
+export AWS_ACCESS_KEY_ID=<access_key_id>
+export AWS_SECRET_ACCESS_KEY=<secrets_access_key>
+export AWS_SESSION_TOKEN=<token>
+```
+
+当然，你也可以通过直接在配置中指定所有信息内容：
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/secrets/aws/1 \
+-H "X-API-KEY: $admin_key" -X PUT -d '
+{
+    "endpoint_url": "http://127.0.0.1:4566",
+    "region": "us-east-1",
+    "access_key_id": "access",
+    "secret_access_key": "secret",
+    "session_token": "token"
+}'
+```
+
+如果使用 APISIX Standalone 版本，则可以在 `apisix.yaml`  文件中添加如下配置：
+
+```yaml
+secrets:
+  - id: aws/1
+    endpoint_url: http://127.0.0.1:4566
+    region: us-east-1
+    access_key_id: access
+    secret_access_key: secret
+    session_token: token
+```
+
+第三步：在 `key-auth` 插件中引用 APISIX Secret 资源，填充秘钥信息：
+
+```shell
+curl http://127.0.0.1:9180/apisix/admin/consumers \
+-H "X-API-KEY: $admin_key" -X PUT -d '
+{
+    "username": "jack",
+    "plugins": {
+        "key-auth": {
+            "key": "$secret://aws/1/jack/auth-key"
+        }
+    }
+}'
+```
+
+通过上面两步操作，当用户请求命中 `key-auth` 插件时，会通过 APISIX Secret 组件获取到 key 在 AWS Secret Manager 中的真实值。
+
+### 验证
+
+你可以通过如下指令进行验证：
+
+```shell
+# 示例：将下面的 your_route 替换为实际的路由路径
+curl -i http://127.0.0.1:9080/your_route -H 'apikey: value'
+```
+
+这将验证 key-auth 插件是否正确地使用 AWS Secret Manager 中的密钥。