From f2a8bf14a0a90d4becc54e67a0011c46ec78d6d0 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 10:04:47 +0800
Subject: [PATCH 01/13] feat(plugin): Add new plugin bot-restriction for bot
 spider restrction

---
 apisix/plugins/bot-restriction.lua        | 179 ++++++
 conf/config-default.yaml                  |   1 +
 docs/en/latest/plugins/bot-restriction.md | 130 ++++
 docs/zh/latest/plugins/bot-restriction.md | 123 ++++
 t/plugin/bot-restriction.t                | 699 ++++++++++++++++++++++
 5 files changed, 1132 insertions(+)
 create mode 100644 apisix/plugins/bot-restriction.lua
 create mode 100644 docs/en/latest/plugins/bot-restriction.md
 create mode 100644 docs/zh/latest/plugins/bot-restriction.md
 create mode 100644 t/plugin/bot-restriction.t

diff --git a/apisix/plugins/bot-restriction.lua b/apisix/plugins/bot-restriction.lua
new file mode 100644
index 000000000000..42e890bba434
--- /dev/null
+++ b/apisix/plugins/bot-restriction.lua
@@ -0,0 +1,179 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local ipairs = ipairs
+local core = require("apisix.core")
+local stringx = require('pl.stringx')
+local str_strip = stringx.strip
+local re_find = ngx.re.find
+
+local MATCH_NONE = 0
+local MATCH_ALLOW = 1
+local MATCH_DENY = 2
+local MATCH_BOT = 3
+
+local lrucache_useragent = core.lrucache.new({ ttl = 300, count = 1024 })
+
+local schema = {
+    type = "object",
+    properties = {
+        message = {
+            type = "string",
+            minLength = 1,
+            maxLength = 1024,
+            default = "Not allowed"
+        },
+        whitelist = {
+            type = "array",
+            minItems = 1
+        },
+        blacklist = {
+            type = "array",
+            minItems = 1
+        },
+    },
+    additionalProperties = false,
+}
+
+local plugin_name = "bot-restriction"
+
+local _M = {
+    version = 0.1,
+    priority = 3000,
+    name = plugin_name,
+    schema = schema,
+}
+
+-- List taken from https://github.com/ua-parser/uap-core/blob/master/regexes.yaml
+local well_known_bots = {
+    [[(Pingdom\.com_bot_version_)(\d+)\.(\d+)]],
+    [[(facebookexternalhit)/(\d+)\.(\d+)]],
+    [[Google.{0,50}/\+/web/snippet]],
+    [[(NewRelicPinger)/(\d+)\.(\d+)],
+    [[\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|c]]
+            .. [[ore)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)]],
+    [[ PTST/\d+(?:\.)?\d+$]],
+    [[/((?:Ant-)?Nutch|[A-z]+[Bb]ot|[A-z]+[Ss]pider|Axtaris|fetchurl|Isara|ShopSalad|T]]
+            .. [[ailsweep)[ \-](\d+)(?:\.(\d+)(?:\.(\d+))?)?]],
+    [[\b(008|Altresium|Argus|BaiduMobaider|BoardReader|DNSGroup|DataparkSearch|EDI|Goo]]
+            .. [[dzer|Grub|INGRID|Infohelfer|LinkedInBot|LOOQ|Nutch|OgScrper|PathDefender|Peew|Po]]
+            .. [[stPost|Steeler|Twitterbot|VSE|WebCrunch|WebZIP|Y!J-BR[A-Z]|YahooSeeker|envolk|sp]]
+            .. [[roose|wminer)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)]],
+    [[(MSIE) (\d+)\.(\d+)([a-z]\d|[a-z]|);.{0,200} MSIECrawler]],
+    [[(Google-HTTP-Java-Client|Apache-HttpClient|Go-http-client|scalaj-http|http%20cli]]
+            .. [[ent|Python-urllib|HttpMonitor|TLSProber|WinHTTP|JNLP|okhttp|aihttp|reqwest|axios]]
+            .. [[|unirest-(?:java|python|ruby|nodejs|php|net))(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|]]
+            .. [[)|)]],
+    [[(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Jo]]
+            .. [[b Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends C]]
+            .. [[rawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|14]]
+            .. [[70\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot]]
+            .. [[-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|arch]]
+            .. [[iver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bing]]
+            .. [[bot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardRead]]
+            .. [[er Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYe]]
+            .. [[ti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo ]]
+            .. [[HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConver]]
+            .. [[a|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngin]]
+            .. [[e|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Go]]
+            .. [[oglebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|Hid]]
+            .. [[denMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobil]]
+            .. [[e|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchi]]
+            .. [[ve|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Lingu]]
+            .. [[ee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_]]
+            .. [[bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:]]
+            .. [[-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGato]]
+            .. [[r[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|]]
+            .. [[PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobo]]
+            .. [[t|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scoot]]
+            .. [[er|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimpleP]]
+            .. [[ie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy]]
+            .. [[|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tin]]
+            .. [[y Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voy]]
+            .. [[ager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|]]
+            .. [[Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(]]
+            .. [[?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|Yottaa]]
+            .. [[Monitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)]]
+            .. [[(?:\.(\d+)(?:\.(\d+)|)|)|)]],
+    [[(?:\/[A-Za-z0-9\.]+|) {0,5}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexe]]
+            .. [[r|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)]]
+            .. [[|)|)]],
+    [[(?:\/[A-Za-z0-9\.]+|) {0,5}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexe]]
+            .. [[r|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50})) (\d+)(?:\.(\d+)(?:\.(\d+)|)|]]
+            .. [[)]],
+    [[((?:[A-z0-9]{1,50}|[A-z\-]{1,50} ?|)(?: the |)(?:[Ss][Pp][Ii][Dd][Ee][Rr]|[Ss]cr]]
+            .. [[ape|[Cc][Rr][Aa][Ww][Ll])[A-z0-9]{0,50})(?:(?:[ /]| v)(\d+)(?:\.(\d+)|)(?:\.(\d+]]
+            .. [[)|)|)]],
+}
+
+local function match_user_agent(user_agent, conf)
+    user_agent = str_strip(user_agent)
+    if conf.whitelist then
+        for _, rule in ipairs(conf.whitelist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_ALLOW
+            end
+        end
+    end
+
+    if conf.blacklist then
+        for _, rule in ipairs(conf.blacklist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_DENY
+            end
+        end
+    end
+
+    for _, rule in ipairs(well_known_bots) do
+        if re_find(user_agent, rule, "jo") then
+            return MATCH_BOT
+        end
+    end
+
+    return MATCH_NONE
+end
+
+function _M.check_schema(conf)
+    local ok, err = core.schema.check(schema, conf)
+
+    if not ok then
+        return false, err
+    end
+
+    return true
+end
+
+function _M.access(conf, _)
+    local headers = ngx.req.get_headers()
+    local user_agent = headers["user-agent"]
+    -- ignore multiple instances of request headers
+    if type(user_agent) == "table" then
+        return
+    end
+    if not user_agent then
+        return
+    end
+    local match, err = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
+    if err then
+        return
+    end
+
+    if match > MATCH_ALLOW then
+        return 403, { message = conf.message }
+    end
+end
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index eedf77febdfa..9cbe1963deef 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -252,6 +252,7 @@ plugins:                          # plugin list (sorted by priority)
   - batch-requests                 # priority: 4010
   - cors                           # priority: 4000
   - ip-restriction                 # priority: 3000
+  - bot-restriction                # priority: 3000
   - referer-restriction            # priority: 2990
   - uri-blocker                    # priority: 2900
   - request-validation             # priority: 2800
diff --git a/docs/en/latest/plugins/bot-restriction.md b/docs/en/latest/plugins/bot-restriction.md
new file mode 100644
index 000000000000..c133015beba0
--- /dev/null
+++ b/docs/en/latest/plugins/bot-restriction.md
@@ -0,0 +1,130 @@
+---
+title: bot-restriction
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## Summary
+
+- [**Name**](#name)
+- [**Attributes**](#attributes)
+- [**How To Enable**](#how-to-enable)
+- [**Test Plugin**](#test-plugin)
+- [**Disable Plugin**](#disable-plugin)
+
+## Name
+
+The `bot-restriction` can restrict access to a Service or a Route by either
+`whitelisting` or `blacklisting` or `most well-known` bots.
+
+## Attributes
+
+| Name      | Type          | Requirement | Default | Valid | Description                              |
+| --------- | ------------- | ----------- | ------- | ----- | ---------------------------------------- |
+| whitelist | array[string] | optional    |         |       | List of User-Agent of whitelist. |
+| blacklist | array[string] | optional    |         |       | List of User-Agent of blacklist. |
+| message | string | optional             | Not allowed. | [1, 1024] | Message of deny reason. |
+
+Any of `whitelist` or `blacklist` can be optional, and can work together in this order:
+whitelist->blacklist->default well-known User-Agent list.
+
+The message can be user-defined.
+
+## How To Enable
+
+Creates a route or service object, and enable plugin `bot-restriction`.
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "uri": "/index.html",
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:1980": 1
+        }
+    },
+    "plugins": {
+        "bot-restriction": {
+             "whitelist": [
+                 "my-bot1",
+                 "(Baiduspider)/(\\d+)\\.(\\d+)"
+             ],
+             "blacklist": [
+                 "my-bot2",
+                 "(Twitterspider)/(\\d+)\\.(\\d+)"
+             ]
+        }
+    }
+}'
+```
+
+Default returns `{"message":"Not allowed"}` when rejected. If you want to use a custom message, you can configure it in the plugin section.
+
+```json
+"plugins": {
+    "bot-restriction": {
+        "blacklist": [
+            "my-bot2",
+            "(Twitterspider)/(\\d+)\\.(\\d+)"
+        ],
+        "message": "Do you want to do something bad?"
+    }
+}
+```
+
+## Test Plugin
+
+Requests from normal User-Agent:
+
+```shell
+$ curl http://127.0.0.1:9080/index.html -i
+HTTP/1.1 200 OK
+...
+```
+
+Requests from bot User-Agent:
+
+```shell
+$ curl http://127.0.0.1:9080/index.html --header 'User-Agent: Twitterspider/2.0'
+HTTP/1.1 403 Forbidden
+```
+
+## Disable Plugin
+
+When you want to disable the `bot-restriction` plugin, it is very simple,
+you can delete the corresponding json configuration in the plugin configuration,
+no need to restart the service, it will take effect immediately:
+
+```shell
+$ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value='
+{
+    "uri": "/index.html",
+    "plugins": {},
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "39.97.63.215:80": 1
+        }
+    }
+}'
+```
+
+The `bot-restriction` plugin has been disabled now. It works for other plugins.
\ No newline at end of file
diff --git a/docs/zh/latest/plugins/bot-restriction.md b/docs/zh/latest/plugins/bot-restriction.md
new file mode 100644
index 000000000000..b0b2c601f365
--- /dev/null
+++ b/docs/zh/latest/plugins/bot-restriction.md
@@ -0,0 +1,123 @@
+---
+title: bot-restriction
+---
+
+<!--
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+-->
+
+## 目录
+
+- [**名字**](#名字)
+- [**属性**](#属性)
+- [**如何启用**](#如何启用)
+- [**测试插件**](#测试插件)
+- [**禁用插件**](#禁用插件)
+
+## 名字
+
+`bot-restriction` 可以通过以下方式限制对服务或接口的访问，可以将指定 User-Agent 列入白名单或黑名单，同时此插件也将对常见的爬虫 UA 进行了检查。
+
+## 属性
+
+| 参数名    | 类型          | 可选项 | 默认值 | 有效值 | 描述                             |
+| --------- | ------------- | ------ | ------ | ------ | -------------------------------- |
+| whitelist | array[string] | 可选   |        |        | 加入白名单的 User-Agent |
+| blacklist | array[string] | 可选   |        |        | 加入黑名单的 User-Agent |
+| message | string | 可选   | Not allowed. | [1, 1024] | 在未允许的 User-Agent 访问的情况下返回的信息 |
+
+白名单或黑名单可以同时启用，此插件对 User-Agent 的检查先后顺序依次如下：白名单、黑名单、内置的场景爬虫 User-Agent。 `message`可以由用户自定义。
+
+## 如何启用
+
+下面是一个示例，在指定的 route 上开启了 `bot-restriction` 插件:
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "uri": "/index.html",
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "127.0.0.1:1980": 1
+        }
+    },
+    "plugins": {
+        "bot-restriction": {
+             "whitelist": [
+                 "my-bot1",
+                 "(Baiduspider)/(\\d+)\\.(\\d+)"
+             ],
+             "blacklist": [
+                 "my-bot2",
+                 "(Twitterspider)/(\\d+)\\.(\\d+)"
+             ]
+        }
+    }
+}'
+```
+
+当未允许的 User-Agent 访问时，默认返回`{"message":"Not allowed"}`。如果你想使用自定义的`message`，可以在插件部分进行配置:
+
+```json
+"plugins": {
+    "bot-restriction": {
+        "blacklist": [
+            "my-bot2",
+            "(Twitterspider)/(\\d+)\\.(\\d+)"
+        ],
+        "message": "Do you want to do something bad?"
+    }
+}
+```
+
+## 测试插件
+
+通过正常的 UA 访问：
+
+```shell
+$ curl http://127.0.0.1:9080/index.html --header 'User-Agent: YourApp/2.0.0'
+HTTP/1.1 200 OK
+```
+
+通过爬虫 User-Agent 访问：
+
+```shell
+$ curl http://127.0.0.1:9080/index.html --header 'User-Agent: Twitterspider/2.0'
+HTTP/1.1 403 Forbidden
+```
+
+## 禁用插件
+
+当你想去掉 `bot-restriction` 插件的时候，很简单，在插件的配置中把对应的 json 配置删除即可，无须重启服务，即刻生效：
+
+```shell
+$ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1  -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value='
+{
+    "uri": "/index.html",
+    "plugins": {},
+    "upstream": {
+        "type": "roundrobin",
+        "nodes": {
+            "39.97.63.215:80": 1
+        }
+    }
+}'
+```
+
+现在就已移除 `bot-restriction` 插件，其它插件的开启和移除也类似。
diff --git a/t/plugin/bot-restriction.t b/t/plugin/bot-restriction.t
new file mode 100644
index 000000000000..118998f2d3ea
--- /dev/null
+++ b/t/plugin/bot-restriction.t
@@ -0,0 +1,699 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+no_long_string();
+no_shuffle();
+no_root_location();
+
+add_block_preprocessor(sub {
+    my ($block) = @_;
+
+    if (!$block->request) {
+        $block->set_value("request", "GET /t");
+    }
+
+    if (!$block->no_error_log && !$block->error_log) {
+        $block->set_value("no_error_log", "[error]\n[alert]");
+    }
+});
+
+run_tests;
+
+__DATA__
+
+=== TEST 1: empty conf
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.bot-restriction")
+            local ok, err = plugin.check_schema({})
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say(require("toolkit.json").encode(conf))
+        }
+    }
+--- error_code: 200
+
+
+
+=== TEST 2: set whitelist, blacklist and user-defined message
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.bot-restriction")
+            local conf = {
+               whitelist = {
+                    "my-bot1",
+                    "my-bot2"
+               },
+               blacklist = {
+                    "my-bot1",
+                    "my-bot2"
+               },
+               message = "User-Agent Forbidden",
+            }
+            local ok, err = plugin.check_schema(conf)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say(require("toolkit.json").encode(conf))
+        }
+    }
+--- response_body
+{"blacklist":["my-bot1","my-bot2"],"message":"User-Agent Forbidden","whitelist":["my-bot1","my-bot2"]}
+
+
+
+=== TEST 3: whitelist not array
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.bot-restriction")
+            local conf = {
+                whitelist = "my-bot1",
+            }
+            local ok, err = plugin.check_schema(conf)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- response_body
+property "whitelist" validation failed: wrong type: expected array, got string
+done
+
+
+
+=== TEST 4: blacklist not array
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.bot-restriction")
+            local conf = {
+                blacklist = 100,
+            }
+            local ok, err = plugin.check_schema(conf)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- response_body
+property "blacklist" validation failed: wrong type: expected array, got number
+done
+
+
+
+=== TEST 5: message not string
+--- config
+    location /t {
+        content_by_lua_block {
+            local plugin = require("apisix.plugins.bot-restriction")
+            local conf = {
+                message = 100,
+            }
+            local ok, err = plugin.check_schema(conf)
+            if not ok then
+                ngx.say(err)
+            end
+
+            ngx.say("done")
+        }
+    }
+--- response_body
+property "message" validation failed: wrong type: expected string, got number
+done
+
+
+
+=== TEST 6: set blacklist
+
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "blacklist": [
+                                     "my-bot1",
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                 ]
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 7: hit route and user-agent in blacklist
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:my-bot1
+--- error_code: 403
+
+
+
+=== TEST 8: hit route and user-agent in blacklist with multiple
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:my-bot1
+User-Agent:my-bot1
+--- error_code: 200
+
+
+
+=== TEST 9: hit route and user-agent match blacklist regex
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:Baiduspider/3.0
+--- error_code: 403
+
+
+
+=== TEST 10: hit route and user-agent not in blacklist
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:foo/bar
+--- error_code: 200
+
+
+
+=== TEST 11: set whitelist
+
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "whitelist": [
+                                     "my-bot1",
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                 ]
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 12: hit route and user-agent in whitelist
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:my-bot1
+--- error_code: 200
+
+
+
+=== TEST 13: hit route and user-agent match whitelist regex
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:Baiduspider/3.0
+--- error_code: 200
+
+
+
+=== TEST 14: hit route and user-agent not in whitelist
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:foo/bar
+--- error_code: 200
+
+
+
+=== TEST 15: set rules to default
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 16: hit route and user-agent in default list
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:Twitterbot/1.0
+--- error_code: 403
+
+
+
+=== TEST 17: set config: user-agent in both whitelist and blacklist
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "whitelist": [
+                                     "foo/bar",
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                 ],
+                                 "blacklist": [
+                                     "foo/bar",
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                 ]
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 18: hit route and user-agent in both whitelist and blacklist, part 1
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:foo/bar
+--- error_code: 200
+
+
+
+=== TEST 19: hit route and user-agent in both whitelist and blacklist, part 2
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:Baiduspider/1.0
+--- error_code: 200
+
+
+
+=== TEST 20: set config: user-agent in both whitelist and default deny list
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "whitelist": [
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                 ]
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 21: hit route and user-agent in both whitelist and default deny list
+
+--- request
+GET /hello
+--- more_headers
+User-Agent:Baiduspider/1.0
+--- error_code: 200
+
+
+
+=== TEST 22: message that do not reach the minimum range
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "message": ""
+                            }
+                        }
+                }]]
+                )
+
+            ngx.say(body)
+        }
+    }
+--- response_body_like eval
+qr/string too short, expected at least 1, got 0/
+
+
+
+=== TEST 23: exceeds the maximum limit of message
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local json = require("toolkit.json")
+
+            local data = {
+                uri = "/hello",
+                upstream = {
+                    type = "roundrobin",
+                    nodes = {
+                        ["127.0.0.1:1980"] = 1,
+                    }
+                },
+                plugins = {
+                    ["bot-restriction"] = {
+                        message = ("-1Aa#"):rep(205)
+                    }
+                }
+            }
+
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                json.encode(data)
+            )
+
+            ngx.say(body)
+        }
+    }
+--- response_body_like eval
+qr/string too long, expected at most 1024, got 1025/
+
+
+
+=== TEST 24: set custom message
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "bot-restriction": {
+                                 "message": "Do you want to do something bad?"
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+
+--- response_body
+passed
+
+
+
+=== TEST 25: test custom message
+--- request
+GET /hello
+--- more_headers
+User-Agent:Twitterbot/1.0
+--- error_code: 403
+--- response_body
+{"message":"Do you want to do something bad?"}
+
+
+
+=== TEST 26: test remove bot-restriction part 1
+--- config
+    location /enable {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                        "bot-restriction": {
+                        }
+                    }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+
+    location /disable {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- request
+GET /enable
+--- error_code: 200
+
+
+
+=== TEST 27: test remove bot-restriction part 2
+--- request
+GET /hello
+--- more_headers
+User-Agent:Twitterbot/1.0
+--- error_code: 403
+
+
+
+=== TEST 28: test remove bot-restriction part 3, remove plugin
+--- config
+    location /disable {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- request
+GET /disable
+--- error_code: 200
+
+
+
+=== TEST 29: test remove bot-restriction part 4, check bot User-Agent
+--- request
+GET /hello
+--- more_headers
+User-Agent:Twitterbot/1.0
+--- response_body
+hello world
+
+
+
+=== TEST 30: set disable=true
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                ngx.HTTP_PUT,
+                [[{
+                    "uri": "/hello",
+                    "plugins": {
+                        "bot-restriction": {
+                            "blacklist": [
+                                "foo"
+                            ],
+                            "disable": true
+                        }
+                    }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
\ No newline at end of file

From 830b621e45e5be65232389f752d2bd800d24bc09 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 13:04:13 +0800
Subject: [PATCH 02/13] fix doc

---
 docs/en/latest/plugins/bot-restriction.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/latest/plugins/bot-restriction.md b/docs/en/latest/plugins/bot-restriction.md
index c133015beba0..7b6fb3bae6ef 100644
--- a/docs/en/latest/plugins/bot-restriction.md
+++ b/docs/en/latest/plugins/bot-restriction.md
@@ -127,4 +127,4 @@ $ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1 -H 'X-API-KEY: edd1c9f03433
 }'
 ```
 
-The `bot-restriction` plugin has been disabled now. It works for other plugins.
\ No newline at end of file
+The `bot-restriction` plugin has been disabled now. It works for other plugins.

From bfe2d7ccc53d6b91b876bb64b3ccc8fa707bc20c Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 13:07:21 +0800
Subject: [PATCH 03/13] fix doc

---
 docs/en/latest/config.json | 1 +
 docs/zh/latest/config.json | 1 +
 2 files changed, 2 insertions(+)

diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index 2f088781efff..d1b123a48014 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -73,6 +73,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
+            "plugins/bot-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]
diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json
index 9cda9ea311b5..5a84f7a9b024 100644
--- a/docs/zh/latest/config.json
+++ b/docs/zh/latest/config.json
@@ -71,6 +71,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
+            "plugins/bot-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]

From cb6d5d800ed43e0a8fb191531a9cde4ff6e82619 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 14:26:43 +0800
Subject: [PATCH 04/13] Fix lint

---
 apisix/plugins/bot-restriction.lua | 14 +++++++-------
 t/admin/plugins.t                  |  2 +-
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/apisix/plugins/bot-restriction.lua b/apisix/plugins/bot-restriction.lua
index 42e890bba434..249181b69f9d 100644
--- a/apisix/plugins/bot-restriction.lua
+++ b/apisix/plugins/bot-restriction.lua
@@ -52,7 +52,7 @@ local plugin_name = "bot-restriction"
 
 local _M = {
     version = 0.1,
-    priority = 3000,
+    priority = 2999,
     name = plugin_name,
     schema = schema,
 }
@@ -156,14 +156,14 @@ function _M.check_schema(conf)
     return true
 end
 
-function _M.access(conf, _)
-    local headers = ngx.req.get_headers()
-    local user_agent = headers["user-agent"]
-    -- ignore multiple instances of request headers
-    if type(user_agent) == "table" then
+function _M.access(conf, ctx)
+    local user_agent = core.request.header(ctx, "User-Agent")
+
+    if not user_agent then
         return
     end
-    if not user_agent then
+    -- ignore multiple instances of request headers
+    if type(user_agent) == "table" then
         return
     end
     local match, err = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index 61553a35e343..cd279e635a94 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -40,7 +40,7 @@ __DATA__
 --- request
 GET /apisix/admin/plugins/list
 --- response_body_like eval
-qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
+qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","bot-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
 --- no_error_log
 [error]
 

From 8be944ea13a3bdd34e7f271dc30ec8ae3998bfb1 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 14:30:11 +0800
Subject: [PATCH 05/13] fix lint

---
 apisix/plugins/bot-restriction.lua | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apisix/plugins/bot-restriction.lua b/apisix/plugins/bot-restriction.lua
index 249181b69f9d..6b8d63719e94 100644
--- a/apisix/plugins/bot-restriction.lua
+++ b/apisix/plugins/bot-restriction.lua
@@ -17,6 +17,7 @@
 local ipairs = ipairs
 local core = require("apisix.core")
 local stringx = require('pl.stringx')
+local type = type
 local str_strip = stringx.strip
 local re_find = ngx.re.find
 

From dc331cdf2f731f5546cedb6a47d29a5729b7ae37 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 14:37:15 +0800
Subject: [PATCH 06/13] Fix lint

---
 t/plugin/bot-restriction.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/plugin/bot-restriction.t b/t/plugin/bot-restriction.t
index 118998f2d3ea..cfae1fbf517f 100644
--- a/t/plugin/bot-restriction.t
+++ b/t/plugin/bot-restriction.t
@@ -696,4 +696,4 @@ hello world
         }
     }
 --- response_body
-passed
\ No newline at end of file
+passed

From 7ea7c8729e348ee9ec3682535955989606196a50 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 16:52:44 +0800
Subject: [PATCH 07/13] Fix Review

---
 apisix/plugins/bot-restriction.lua |  5 +----
 conf/config-default.yaml           |  2 +-
 t/plugin/bot-restriction.t         | 26 --------------------------
 3 files changed, 2 insertions(+), 31 deletions(-)

diff --git a/apisix/plugins/bot-restriction.lua b/apisix/plugins/bot-restriction.lua
index 6b8d63719e94..b01374bc915c 100644
--- a/apisix/plugins/bot-restriction.lua
+++ b/apisix/plugins/bot-restriction.lua
@@ -167,10 +167,7 @@ function _M.access(conf, ctx)
     if type(user_agent) == "table" then
         return
     end
-    local match, err = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
-    if err then
-        return
-    end
+    local match = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
 
     if match > MATCH_ALLOW then
         return 403, { message = conf.message }
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 9cbe1963deef..19969dc78e4a 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -252,7 +252,7 @@ plugins:                          # plugin list (sorted by priority)
   - batch-requests                 # priority: 4010
   - cors                           # priority: 4000
   - ip-restriction                 # priority: 3000
-  - bot-restriction                # priority: 3000
+  - bot-restriction                # priority: 2999
   - referer-restriction            # priority: 2990
   - uri-blocker                    # priority: 2900
   - request-validation             # priority: 2800
diff --git a/t/plugin/bot-restriction.t b/t/plugin/bot-restriction.t
index cfae1fbf517f..0f6712cc9a27 100644
--- a/t/plugin/bot-restriction.t
+++ b/t/plugin/bot-restriction.t
@@ -151,7 +151,6 @@ done
 
 
 === TEST 6: set blacklist
-
 --- config
     location /t {
         content_by_lua_block {
@@ -587,31 +586,6 @@ User-Agent:Twitterbot/1.0
             ngx.say(body)
         }
     }
-
-    location /disable {
-        content_by_lua_block {
-            local t = require("lib.test_admin").test
-            local code, body = t('/apisix/admin/routes/1',
-                 ngx.HTTP_PUT,
-                 [[{
-                        "uri": "/hello",
-                        "upstream": {
-                            "type": "roundrobin",
-                            "nodes": {
-                                "127.0.0.1:1980": 1
-                            }
-                        },
-                        "plugins": {
-                        }
-                }]]
-                )
-
-            if code >= 300 then
-                ngx.status = code
-            end
-            ngx.say(body)
-        }
-    }
 --- request
 GET /enable
 --- error_code: 200

From 0c6dcb2d5bde3d5aad8cf22b1df74e21626d6e61 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 12 Jul 2021 18:49:00 +0800
Subject: [PATCH 08/13] rename plugin to ua-restriction

---
 apisix/plugins/bot-restriction.lua            | 177 --------------
 apisix/plugins/ua-restriction.lua             | 121 ++++++++++
 conf/config-default.yaml                      |   2 +-
 docs/en/latest/config.json                    |   2 +-
 .../{bot-restriction.md => ua-restriction.md} |  28 +--
 docs/zh/latest/config.json                    |   2 +-
 .../{bot-restriction.md => ua-restriction.md} |  26 +-
 t/admin/plugins.t                             |   2 +-
 .../{bot-restriction.t => ua-restriction.t}   | 223 ++++++------------
 9 files changed, 218 insertions(+), 365 deletions(-)
 delete mode 100644 apisix/plugins/bot-restriction.lua
 create mode 100644 apisix/plugins/ua-restriction.lua
 rename docs/en/latest/plugins/{bot-restriction.md => ua-restriction.md} (77%)
 rename docs/zh/latest/plugins/{bot-restriction.md => ua-restriction.md} (74%)
 rename t/plugin/{bot-restriction.t => ua-restriction.t} (70%)

diff --git a/apisix/plugins/bot-restriction.lua b/apisix/plugins/bot-restriction.lua
deleted file mode 100644
index b01374bc915c..000000000000
--- a/apisix/plugins/bot-restriction.lua
+++ /dev/null
@@ -1,177 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one or more
--- contributor license agreements.  See the NOTICE file distributed with
--- this work for additional information regarding copyright ownership.
--- The ASF licenses this file to You under the Apache License, Version 2.0
--- (the "License"); you may not use this file except in compliance with
--- the License.  You may obtain a copy of the License at
---
---     http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-local ipairs = ipairs
-local core = require("apisix.core")
-local stringx = require('pl.stringx')
-local type = type
-local str_strip = stringx.strip
-local re_find = ngx.re.find
-
-local MATCH_NONE = 0
-local MATCH_ALLOW = 1
-local MATCH_DENY = 2
-local MATCH_BOT = 3
-
-local lrucache_useragent = core.lrucache.new({ ttl = 300, count = 1024 })
-
-local schema = {
-    type = "object",
-    properties = {
-        message = {
-            type = "string",
-            minLength = 1,
-            maxLength = 1024,
-            default = "Not allowed"
-        },
-        whitelist = {
-            type = "array",
-            minItems = 1
-        },
-        blacklist = {
-            type = "array",
-            minItems = 1
-        },
-    },
-    additionalProperties = false,
-}
-
-local plugin_name = "bot-restriction"
-
-local _M = {
-    version = 0.1,
-    priority = 2999,
-    name = plugin_name,
-    schema = schema,
-}
-
--- List taken from https://github.com/ua-parser/uap-core/blob/master/regexes.yaml
-local well_known_bots = {
-    [[(Pingdom\.com_bot_version_)(\d+)\.(\d+)]],
-    [[(facebookexternalhit)/(\d+)\.(\d+)]],
-    [[Google.{0,50}/\+/web/snippet]],
-    [[(NewRelicPinger)/(\d+)\.(\d+)],
-    [[\b(Boto3?|JetS3t|aws-(?:cli|sdk-(?:cpp|go|java|nodejs|ruby2?|dotnet-(?:\d{1,2}|c]]
-            .. [[ore)))|s3fs)/(\d+)\.(\d+)(?:\.(\d+)|)]],
-    [[ PTST/\d+(?:\.)?\d+$]],
-    [[/((?:Ant-)?Nutch|[A-z]+[Bb]ot|[A-z]+[Ss]pider|Axtaris|fetchurl|Isara|ShopSalad|T]]
-            .. [[ailsweep)[ \-](\d+)(?:\.(\d+)(?:\.(\d+))?)?]],
-    [[\b(008|Altresium|Argus|BaiduMobaider|BoardReader|DNSGroup|DataparkSearch|EDI|Goo]]
-            .. [[dzer|Grub|INGRID|Infohelfer|LinkedInBot|LOOQ|Nutch|OgScrper|PathDefender|Peew|Po]]
-            .. [[stPost|Steeler|Twitterbot|VSE|WebCrunch|WebZIP|Y!J-BR[A-Z]|YahooSeeker|envolk|sp]]
-            .. [[roose|wminer)/(\d+)(?:\.(\d+)|)(?:\.(\d+)|)]],
-    [[(MSIE) (\d+)\.(\d+)([a-z]\d|[a-z]|);.{0,200} MSIECrawler]],
-    [[(Google-HTTP-Java-Client|Apache-HttpClient|Go-http-client|scalaj-http|http%20cli]]
-            .. [[ent|Python-urllib|HttpMonitor|TLSProber|WinHTTP|JNLP|okhttp|aihttp|reqwest|axios]]
-            .. [[|unirest-(?:java|python|ruby|nodejs|php|net))(?:[ /](\d+)(?:\.(\d+)|)(?:\.(\d+)|]]
-            .. [[)|)]],
-    [[(CSimpleSpider|Cityreview Robot|CrawlDaddy|CrawlFire|Finderbots|Index crawler|Jo]]
-            .. [[b Roboter|KiwiStatus Spider|Lijit Crawler|QuerySeekerSpider|ScollSpider|Trends C]]
-            .. [[rawler|USyd-NLP-Spider|SiteCat Webbot|BotName\/\$BotVersion|123metaspider-Bot|14]]
-            .. [[70\.net crawler|50\.nu|8bo Crawler Bot|Aboundex|Accoona-[A-z]{1,30}-Agent|AdsBot]]
-            .. [[-Google(?:-[a-z]{1,30}|)|altavista|AppEngine-Google|archive.{0,30}\.org_bot|arch]]
-            .. [[iver|Ask Jeeves|[Bb]ai[Dd]u[Ss]pider(?:-[A-Za-z]{1,30})(?:-[A-Za-z]{1,30}|)|bing]]
-            .. [[bot|BingPreview|blitzbot|BlogBridge|Bloglovin|BoardReader Blog Indexer|BoardRead]]
-            .. [[er Favicon Fetcher|boitho.com-dc|BotSeer|BUbiNG|\b\w{0,30}favicon\w{0,30}\b|\bYe]]
-            .. [[ti(?:-[a-z]{1,30}|)|Catchpoint(?: bot|)|[Cc]harlotte|Checklinks|clumboot|Comodo ]]
-            .. [[HTTP\(S\) Crawler|Comodo-Webinspector-Crawler|ConveraCrawler|CRAWL-E|CrawlConver]]
-            .. [[a|Daumoa(?:-feedfetcher|)|Feed Seeker Bot|Feedbin|findlinks|Flamingo_SearchEngin]]
-            .. [[e|FollowSite Bot|furlbot|Genieo|gigabot|GomezAgent|gonzo1|(?:[a-zA-Z]{1,30}-|)Go]]
-            .. [[oglebot(?:-[a-zA-Z]{1,30}|)|Google SketchUp|grub-client|gsa-crawler|heritrix|Hid]]
-            .. [[denMarket|holmes|HooWWWer|htdig|ia_archiver|ICC-Crawler|Icarus6j|ichiro(?:/mobil]]
-            .. [[e|)|IconSurf|IlTrovatore(?:-Setaccio|)|InfuzApp|Innovazion Crawler|InternetArchi]]
-            .. [[ve|IP2[a-z]{1,30}Bot|jbot\b|KaloogaBot|Kraken|Kurzor|larbin|LEIA|LesnikBot|Lingu]]
-            .. [[ee Bot|LinkAider|LinkedInBot|Lite Bot|Llaut|lycos|Mail\.RU_Bot|masscan|masidani_]]
-            .. [[bot|Mediapartners-Google|Microsoft .{0,30} Bot|mogimogi|mozDex|MJ12bot|msnbot(?:]]
-            .. [[-media {0,2}|)|msrbot|Mtps Feed Aggregation System|netresearch|Netvibes|NewsGato]]
-            .. [[r[^/]{0,30}|^NING|Nutch[^/]{0,30}|Nymesis|ObjectsSearch|OgScrper|Orbiter|OOZBOT|]]
-            .. [[PagePeeker|PagesInventory|PaxleFramework|Peeplo Screenshot Bot|PlantyNet_WebRobo]]
-            .. [[t|Pompos|Qwantify|Read%20Later|Reaper|RedCarpet|Retreiver|Riddler|Rival IQ|scoot]]
-            .. [[er|Scrapy|Scrubby|searchsight|seekbot|semanticdiscovery|SemrushBot|Simpy|SimpleP]]
-            .. [[ie|SEOstats|SimpleRSS|SiteCon|Slackbot-LinkExpanding|Slack-ImgProxy|Slurp|snappy]]
-            .. [[|Speedy Spider|Squrl Java|Stringer|TheUsefulbot|ThumbShotsBot|Thumbshots\.ru|Tin]]
-            .. [[y Tiny RSS|Twitterbot|WhatsApp|URL2PNG|Vagabondo|VoilaBot|^vortex|Votay bot|^voy]]
-            .. [[ager|WASALive.Bot|Web-sniffer|WebThumb|WeSEE:[A-z]{1,30}|WhatWeb|WIRE|WordPress|]]
-            .. [[Wotbox|www\.almaden\.ibm\.com|Xenu(?:.s|) Link Sleuth|Xerka [A-z]{1,30}Bot|yacy(]]
-            .. [[?:bot|)|YahooSeeker|Yahoo! Slurp|Yandex\w{1,30}|YodaoBot(?:-[A-z]{1,30}|)|Yottaa]]
-            .. [[Monitor|Yowedo|^Zao|^Zao-Crawler|ZeBot_www\.ze\.bz|ZooShot|ZyBorg)(?:[ /]v?(\d+)]]
-            .. [[(?:\.(\d+)(?:\.(\d+)|)|)|)]],
-    [[(?:\/[A-Za-z0-9\.]+|) {0,5}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexe]]
-            .. [[r|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50}))[/ ](\d+)(?:\.(\d+)(?:\.(\d+)]]
-            .. [[|)|)]],
-    [[(?:\/[A-Za-z0-9\.]+|) {0,5}([A-Za-z0-9 \-_\!\[\]:]{0,50}(?:[Aa]rchiver|[Ii]ndexe]]
-            .. [[r|[Ss]craper|[Bb]ot|[Ss]pider|[Cc]rawl[a-z]{0,50})) (\d+)(?:\.(\d+)(?:\.(\d+)|)|]]
-            .. [[)]],
-    [[((?:[A-z0-9]{1,50}|[A-z\-]{1,50} ?|)(?: the |)(?:[Ss][Pp][Ii][Dd][Ee][Rr]|[Ss]cr]]
-            .. [[ape|[Cc][Rr][Aa][Ww][Ll])[A-z0-9]{0,50})(?:(?:[ /]| v)(\d+)(?:\.(\d+)|)(?:\.(\d+]]
-            .. [[)|)|)]],
-}
-
-local function match_user_agent(user_agent, conf)
-    user_agent = str_strip(user_agent)
-    if conf.whitelist then
-        for _, rule in ipairs(conf.whitelist) do
-            if re_find(user_agent, rule, "jo") then
-                return MATCH_ALLOW
-            end
-        end
-    end
-
-    if conf.blacklist then
-        for _, rule in ipairs(conf.blacklist) do
-            if re_find(user_agent, rule, "jo") then
-                return MATCH_DENY
-            end
-        end
-    end
-
-    for _, rule in ipairs(well_known_bots) do
-        if re_find(user_agent, rule, "jo") then
-            return MATCH_BOT
-        end
-    end
-
-    return MATCH_NONE
-end
-
-function _M.check_schema(conf)
-    local ok, err = core.schema.check(schema, conf)
-
-    if not ok then
-        return false, err
-    end
-
-    return true
-end
-
-function _M.access(conf, ctx)
-    local user_agent = core.request.header(ctx, "User-Agent")
-
-    if not user_agent then
-        return
-    end
-    -- ignore multiple instances of request headers
-    if type(user_agent) == "table" then
-        return
-    end
-    local match = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
-
-    if match > MATCH_ALLOW then
-        return 403, { message = conf.message }
-    end
-end
-
-return _M
diff --git a/apisix/plugins/ua-restriction.lua b/apisix/plugins/ua-restriction.lua
new file mode 100644
index 000000000000..eaba09ff7375
--- /dev/null
+++ b/apisix/plugins/ua-restriction.lua
@@ -0,0 +1,121 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local ipairs = ipairs
+local core = require("apisix.core")
+local stringx = require('pl.stringx')
+local type = type
+local str_strip = stringx.strip
+local re_find = ngx.re.find
+
+local MATCH_NONE = 0
+local MATCH_ALLOW = 1
+local MATCH_DENY = 2
+
+local lrucache_useragent = core.lrucache.new({ ttl = 300, count = 4096 })
+
+local schema = {
+    type = "object",
+    properties = {
+        message = {
+            type = "string",
+            minLength = 1,
+            maxLength = 1024,
+            default = "Not allowed"
+        },
+        allowlist = {
+            type = "array",
+            minItems = 1
+        },
+        denylist = {
+            type = "array",
+            minItems = 1
+        },
+    },
+    anyOf = {
+        {required = {"allowlist"}},
+        {required = {"denylist"}},
+    },
+    minProperties = 1,
+    additionalProperties = false,
+}
+
+local plugin_name = "ua-restriction"
+
+local _M = {
+    version = 0.1,
+    priority = 2999,
+    name = plugin_name,
+    schema = schema,
+}
+
+local function match_user_agent(user_agent, conf)
+    user_agent = str_strip(user_agent)
+    if conf.allowlist then
+        for _, rule in ipairs(conf.allowlist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_ALLOW
+            end
+        end
+    end
+
+    if conf.denylist then
+        for _, rule in ipairs(conf.denylist) do
+            if re_find(user_agent, rule, "jo") then
+                return MATCH_DENY
+            end
+        end
+    end
+
+    return MATCH_NONE
+end
+
+function _M.check_schema(conf)
+    local ok, err = core.schema.check(schema, conf)
+
+    if not ok then
+        return false, err
+    end
+
+    return true
+end
+
+function _M.access(conf, ctx)
+    local user_agent = core.request.header(ctx, "User-Agent")
+
+    if not user_agent then
+        return
+    end
+    local match = MATCH_NONE
+    if type(user_agent) == "table" then
+        for _, v in ipairs(user_agent) do
+            if type(v) == "string" then
+                match = lrucache_useragent(v, conf, match_user_agent, v, conf)
+                if match > MATCH_ALLOW then
+                    break
+                end
+            end
+        end
+    else
+        match = lrucache_useragent(user_agent, conf, match_user_agent, user_agent, conf)
+    end
+
+    if match > MATCH_ALLOW then
+        return 403, { message = conf.message }
+    end
+end
+
+return _M
diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 19969dc78e4a..a103928b965f 100644
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -252,7 +252,7 @@ plugins:                          # plugin list (sorted by priority)
   - batch-requests                 # priority: 4010
   - cors                           # priority: 4000
   - ip-restriction                 # priority: 3000
-  - bot-restriction                # priority: 2999
+  - ua-restriction                 # priority: 2999
   - referer-restriction            # priority: 2990
   - uri-blocker                    # priority: 2900
   - request-validation             # priority: 2800
diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json
index d1b123a48014..72565a642304 100644
--- a/docs/en/latest/config.json
+++ b/docs/en/latest/config.json
@@ -73,7 +73,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
-            "plugins/bot-restriction",
+            "plugins/ua-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]
diff --git a/docs/en/latest/plugins/bot-restriction.md b/docs/en/latest/plugins/ua-restriction.md
similarity index 77%
rename from docs/en/latest/plugins/bot-restriction.md
rename to docs/en/latest/plugins/ua-restriction.md
index 7b6fb3bae6ef..3bf335e70e3f 100644
--- a/docs/en/latest/plugins/bot-restriction.md
+++ b/docs/en/latest/plugins/ua-restriction.md
@@ -1,5 +1,5 @@
 ---
-title: bot-restriction
+title: ua-restriction
 ---
 
 <!--
@@ -31,25 +31,23 @@ title: bot-restriction
 
 ## Name
 
-The `bot-restriction` can restrict access to a Service or a Route by either
-`whitelisting` or `blacklisting` or `most well-known` bots.
+The `ua-restriction` can restrict access to a Service or a Route by either `allowlist` or `denylist` `User-Agent`.
 
 ## Attributes
 
 | Name      | Type          | Requirement | Default | Valid | Description                              |
 | --------- | ------------- | ----------- | ------- | ----- | ---------------------------------------- |
-| whitelist | array[string] | optional    |         |       | List of User-Agent of whitelist. |
-| blacklist | array[string] | optional    |         |       | List of User-Agent of blacklist. |
+| allowlist | array[string] | optional    |         |       | List of User-Agent of allowlist. |
+| denylist | array[string] | optional    |         |       | List of User-Agent of denylist. |
 | message | string | optional             | Not allowed. | [1, 1024] | Message of deny reason. |
 
-Any of `whitelist` or `blacklist` can be optional, and can work together in this order:
-whitelist->blacklist->default well-known User-Agent list.
+Any of `allowlist` or `denylist` can be optional, and can work together in this order: allowlist->denylist
 
 The message can be user-defined.
 
 ## How To Enable
 
-Creates a route or service object, and enable plugin `bot-restriction`.
+Creates a route or service object, and enable plugin `ua-restriction`.
 
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@@ -62,12 +60,12 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
         }
     },
     "plugins": {
-        "bot-restriction": {
-             "whitelist": [
+        "ua-restriction": {
+             "allowlist": [
                  "my-bot1",
                  "(Baiduspider)/(\\d+)\\.(\\d+)"
              ],
-             "blacklist": [
+             "denylist": [
                  "my-bot2",
                  "(Twitterspider)/(\\d+)\\.(\\d+)"
              ]
@@ -80,8 +78,8 @@ Default returns `{"message":"Not allowed"}` when rejected. If you want to use a
 
 ```json
 "plugins": {
-    "bot-restriction": {
-        "blacklist": [
+    "ua-restriction": {
+        "denylist": [
             "my-bot2",
             "(Twitterspider)/(\\d+)\\.(\\d+)"
         ],
@@ -109,7 +107,7 @@ HTTP/1.1 403 Forbidden
 
 ## Disable Plugin
 
-When you want to disable the `bot-restriction` plugin, it is very simple,
+When you want to disable the `ua-restriction` plugin, it is very simple,
 you can delete the corresponding json configuration in the plugin configuration,
 no need to restart the service, it will take effect immediately:
 
@@ -127,4 +125,4 @@ $ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1 -H 'X-API-KEY: edd1c9f03433
 }'
 ```
 
-The `bot-restriction` plugin has been disabled now. It works for other plugins.
+The `ua-restriction` plugin has been disabled now. It works for other plugins.
diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json
index 5a84f7a9b024..524777800336 100644
--- a/docs/zh/latest/config.json
+++ b/docs/zh/latest/config.json
@@ -71,7 +71,7 @@
             "plugins/cors",
             "plugins/uri-blocker",
             "plugins/ip-restriction",
-            "plugins/bot-restriction",
+            "plugins/ua-restriction",
             "plugins/referer-restriction",
             "plugins/consumer-restriction"
           ]
diff --git a/docs/zh/latest/plugins/bot-restriction.md b/docs/zh/latest/plugins/ua-restriction.md
similarity index 74%
rename from docs/zh/latest/plugins/bot-restriction.md
rename to docs/zh/latest/plugins/ua-restriction.md
index b0b2c601f365..3126d9c3d0cf 100644
--- a/docs/zh/latest/plugins/bot-restriction.md
+++ b/docs/zh/latest/plugins/ua-restriction.md
@@ -1,5 +1,5 @@
 ---
-title: bot-restriction
+title: ua-restriction
 ---
 
 <!--
@@ -31,21 +31,21 @@ title: bot-restriction
 
 ## 名字
 
-`bot-restriction` 可以通过以下方式限制对服务或接口的访问，可以将指定 User-Agent 列入白名单或黑名单，同时此插件也将对常见的爬虫 UA 进行了检查。
+`ua-restriction` 可以通过以下方式限制对服务或接口的访问，可以将指定 `User-Agent` 列入白名单或黑名单。
 
 ## 属性
 
 | 参数名    | 类型          | 可选项 | 默认值 | 有效值 | 描述                             |
 | --------- | ------------- | ------ | ------ | ------ | -------------------------------- |
-| whitelist | array[string] | 可选   |        |        | 加入白名单的 User-Agent |
-| blacklist | array[string] | 可选   |        |        | 加入黑名单的 User-Agent |
+| allowlist | array[string] | 可选   |        |        | 加入白名单的 User-Agent |
+| denylist | array[string] | 可选   |        |        | 加入黑名单的 User-Agent |
 | message | string | 可选   | Not allowed. | [1, 1024] | 在未允许的 User-Agent 访问的情况下返回的信息 |
 
-白名单或黑名单可以同时启用，此插件对 User-Agent 的检查先后顺序依次如下：白名单、黑名单、内置的场景爬虫 User-Agent。 `message`可以由用户自定义。
+白名单或黑名单可以同时启用，此插件对 User-Agent 的检查先后顺序依次如下：白名单、黑名单。`message`可以由用户自定义。
 
 ## 如何启用
 
-下面是一个示例，在指定的 route 上开启了 `bot-restriction` 插件:
+下面是一个示例，在指定的 route 上开启了 `ua-restriction` 插件:
 
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
@@ -58,12 +58,12 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
         }
     },
     "plugins": {
-        "bot-restriction": {
-             "whitelist": [
+        "ua-restriction": {
+             "allowlist": [
                  "my-bot1",
                  "(Baiduspider)/(\\d+)\\.(\\d+)"
              ],
-             "blacklist": [
+             "denylist": [
                  "my-bot2",
                  "(Twitterspider)/(\\d+)\\.(\\d+)"
              ]
@@ -76,8 +76,8 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
 
 ```json
 "plugins": {
-    "bot-restriction": {
-        "blacklist": [
+    "ua-restriction": {
+        "denylist": [
             "my-bot2",
             "(Twitterspider)/(\\d+)\\.(\\d+)"
         ],
@@ -104,7 +104,7 @@ HTTP/1.1 403 Forbidden
 
 ## 禁用插件
 
-当你想去掉 `bot-restriction` 插件的时候，很简单，在插件的配置中把对应的 json 配置删除即可，无须重启服务，即刻生效：
+当你想去掉 `ua-restriction` 插件的时候，很简单，在插件的配置中把对应的 json 配置删除即可，无须重启服务，即刻生效：
 
 ```shell
 $ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1  -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d value='
@@ -120,4 +120,4 @@ $ curl http://127.0.0.1:2379/v2/keys/apisix/routes/1  -H 'X-API-KEY: edd1c9f0343
 }'
 ```
 
-现在就已移除 `bot-restriction` 插件，其它插件的开启和移除也类似。
+现在就已移除 `ua-restriction` 插件，其它插件的开启和移除也类似。
diff --git a/t/admin/plugins.t b/t/admin/plugins.t
index cd279e635a94..1bc0995b4e01 100644
--- a/t/admin/plugins.t
+++ b/t/admin/plugins.t
@@ -40,7 +40,7 @@ __DATA__
 --- request
 GET /apisix/admin/plugins/list
 --- response_body_like eval
-qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","bot-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
+qr/\["client-control","ext-plugin-pre-req","zipkin","request-id","fault-injection","serverless-pre-function","batch-requests","cors","ip-restriction","ua-restriction","referer-restriction","uri-blocker","request-validation","openid-connect","wolf-rbac","hmac-auth","basic-auth","jwt-auth","key-auth","consumer-restriction","authz-keycloak","proxy-mirror","proxy-cache","proxy-rewrite","api-breaker","limit-conn","limit-count","limit-req","server-info","traffic-split","redirect","response-rewrite","grpc-transcode","prometheus","echo","http-logger","sls-logger","tcp-logger","kafka-logger","syslog","udp-logger","example-plugin","serverless-post-function","ext-plugin-post-req"\]/
 --- no_error_log
 [error]
 
diff --git a/t/plugin/bot-restriction.t b/t/plugin/ua-restriction.t
similarity index 70%
rename from t/plugin/bot-restriction.t
rename to t/plugin/ua-restriction.t
index 0f6712cc9a27..9949fafac14b 100644
--- a/t/plugin/bot-restriction.t
+++ b/t/plugin/ua-restriction.t
@@ -42,7 +42,7 @@ __DATA__
 --- config
     location /t {
         content_by_lua_block {
-            local plugin = require("apisix.plugins.bot-restriction")
+            local plugin = require("apisix.plugins.ua-restriction")
             local ok, err = plugin.check_schema({})
             if not ok then
                 ngx.say(err)
@@ -51,21 +51,22 @@ __DATA__
             ngx.say(require("toolkit.json").encode(conf))
         }
     }
---- error_code: 200
+--- response_body_like eval
+qr/object matches none of the requireds/
 
 
 
-=== TEST 2: set whitelist, blacklist and user-defined message
+=== TEST 2: set allowlist, denylist and user-defined message
 --- config
     location /t {
         content_by_lua_block {
-            local plugin = require("apisix.plugins.bot-restriction")
+            local plugin = require("apisix.plugins.ua-restriction")
             local conf = {
-               whitelist = {
+               allowlist = {
                     "my-bot1",
                     "my-bot2"
                },
-               blacklist = {
+               denylist = {
                     "my-bot1",
                     "my-bot2"
                },
@@ -80,17 +81,17 @@ __DATA__
         }
     }
 --- response_body
-{"blacklist":["my-bot1","my-bot2"],"message":"User-Agent Forbidden","whitelist":["my-bot1","my-bot2"]}
+{"allowlist":["my-bot1","my-bot2"],"denylist":["my-bot1","my-bot2"],"message":"User-Agent Forbidden"}
 
 
 
-=== TEST 3: whitelist not array
+=== TEST 3: allowlist not array
 --- config
     location /t {
         content_by_lua_block {
-            local plugin = require("apisix.plugins.bot-restriction")
+            local plugin = require("apisix.plugins.ua-restriction")
             local conf = {
-                whitelist = "my-bot1",
+                allowlist = "my-bot1",
             }
             local ok, err = plugin.check_schema(conf)
             if not ok then
@@ -101,18 +102,18 @@ __DATA__
         }
     }
 --- response_body
-property "whitelist" validation failed: wrong type: expected array, got string
+property "allowlist" validation failed: wrong type: expected array, got string
 done
 
 
 
-=== TEST 4: blacklist not array
+=== TEST 4: denylist not array
 --- config
     location /t {
         content_by_lua_block {
-            local plugin = require("apisix.plugins.bot-restriction")
+            local plugin = require("apisix.plugins.ua-restriction")
             local conf = {
-                blacklist = 100,
+                denylist = 100,
             }
             local ok, err = plugin.check_schema(conf)
             if not ok then
@@ -123,7 +124,7 @@ done
         }
     }
 --- response_body
-property "blacklist" validation failed: wrong type: expected array, got number
+property "denylist" validation failed: wrong type: expected array, got number
 done
 
 
@@ -132,7 +133,7 @@ done
 --- config
     location /t {
         content_by_lua_block {
-            local plugin = require("apisix.plugins.bot-restriction")
+            local plugin = require("apisix.plugins.ua-restriction")
             local conf = {
                 message = 100,
             }
@@ -150,7 +151,7 @@ done
 
 
 
-=== TEST 6: set blacklist
+=== TEST 6: set denylist
 --- config
     location /t {
         content_by_lua_block {
@@ -166,8 +167,8 @@ done
                             }
                         },
                         "plugins": {
-                            "bot-restriction": {
-                                 "blacklist": [
+                            "ua-restriction": {
+                                 "denylist": [
                                      "my-bot1",
                                      "(Baiduspider)/(\\d+)\\.(\\d+)"
                                  ]
@@ -187,8 +188,7 @@ passed
 
 
 
-=== TEST 7: hit route and user-agent in blacklist
-
+=== TEST 7: hit route and user-agent in denylist
 --- request
 GET /hello
 --- more_headers
@@ -197,19 +197,17 @@ User-Agent:my-bot1
 
 
 
-=== TEST 8: hit route and user-agent in blacklist with multiple
-
+=== TEST 8: hit route and user-agent in denylist with multiple
 --- request
 GET /hello
 --- more_headers
 User-Agent:my-bot1
-User-Agent:my-bot1
---- error_code: 200
-
+User-Agent:my-bot2
+--- error_code: 403
 
 
-=== TEST 9: hit route and user-agent match blacklist regex
 
+=== TEST 9: hit route and user-agent match denylist regex
 --- request
 GET /hello
 --- more_headers
@@ -218,8 +216,7 @@ User-Agent:Baiduspider/3.0
 
 
 
-=== TEST 10: hit route and user-agent not in blacklist
-
+=== TEST 10: hit route and user-agent not in denylist
 --- request
 GET /hello
 --- more_headers
@@ -228,8 +225,7 @@ User-Agent:foo/bar
 
 
 
-=== TEST 11: set whitelist
-
+=== TEST 11: set allowlist
 --- config
     location /t {
         content_by_lua_block {
@@ -245,8 +241,8 @@ User-Agent:foo/bar
                             }
                         },
                         "plugins": {
-                            "bot-restriction": {
-                                 "whitelist": [
+                            "ua-restriction": {
+                                 "allowlist": [
                                      "my-bot1",
                                      "(Baiduspider)/(\\d+)\\.(\\d+)"
                                  ]
@@ -266,8 +262,7 @@ passed
 
 
 
-=== TEST 12: hit route and user-agent in whitelist
-
+=== TEST 12: hit route and user-agent in allowlist
 --- request
 GET /hello
 --- more_headers
@@ -276,8 +271,7 @@ User-Agent:my-bot1
 
 
 
-=== TEST 13: hit route and user-agent match whitelist regex
-
+=== TEST 13: hit route and user-agent match allowlist regex
 --- request
 GET /hello
 --- more_headers
@@ -286,8 +280,7 @@ User-Agent:Baiduspider/3.0
 
 
 
-=== TEST 14: hit route and user-agent not in whitelist
-
+=== TEST 14: hit route and user-agent not in allowlist
 --- request
 GET /hello
 --- more_headers
@@ -296,50 +289,7 @@ User-Agent:foo/bar
 
 
 
-=== TEST 15: set rules to default
---- config
-    location /t {
-        content_by_lua_block {
-            local t = require("lib.test_admin").test
-            local code, body = t('/apisix/admin/routes/1',
-                 ngx.HTTP_PUT,
-                 [[{
-                        "uri": "/hello",
-                        "upstream": {
-                            "type": "roundrobin",
-                            "nodes": {
-                                "127.0.0.1:1980": 1
-                            }
-                        },
-                        "plugins": {
-                            "bot-restriction": {
-                            }
-                        }
-                }]]
-                )
-
-            if code >= 300 then
-                ngx.status = code
-            end
-            ngx.say(body)
-        }
-    }
---- response_body
-passed
-
-
-
-=== TEST 16: hit route and user-agent in default list
-
---- request
-GET /hello
---- more_headers
-User-Agent:Twitterbot/1.0
---- error_code: 403
-
-
-
-=== TEST 17: set config: user-agent in both whitelist and blacklist
+=== TEST 15: set config: user-agent in both allowlist and denylist
 --- config
     location /t {
         content_by_lua_block {
@@ -355,12 +305,12 @@ User-Agent:Twitterbot/1.0
                             }
                         },
                         "plugins": {
-                            "bot-restriction": {
-                                 "whitelist": [
+                            "ua-restriction": {
+                                 "allowlist": [
                                      "foo/bar",
                                      "(Baiduspider)/(\\d+)\\.(\\d+)"
                                  ],
-                                 "blacklist": [
+                                 "denylist": [
                                      "foo/bar",
                                      "(Baiduspider)/(\\d+)\\.(\\d+)"
                                  ]
@@ -380,8 +330,7 @@ passed
 
 
 
-=== TEST 18: hit route and user-agent in both whitelist and blacklist, part 1
-
+=== TEST 16: hit route and user-agent in both allowlist and denylist, part 1
 --- request
 GET /hello
 --- more_headers
@@ -390,8 +339,7 @@ User-Agent:foo/bar
 
 
 
-=== TEST 19: hit route and user-agent in both whitelist and blacklist, part 2
-
+=== TEST 17: hit route and user-agent in both allowlist and denylist, part 2
 --- request
 GET /hello
 --- more_headers
@@ -400,7 +348,7 @@ User-Agent:Baiduspider/1.0
 
 
 
-=== TEST 20: set config: user-agent in both whitelist and default deny list
+=== TEST 18: message that do not reach the minimum range
 --- config
     location /t {
         content_by_lua_block {
@@ -416,53 +364,7 @@ User-Agent:Baiduspider/1.0
                             }
                         },
                         "plugins": {
-                            "bot-restriction": {
-                                 "whitelist": [
-                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
-                                 ]
-                            }
-                        }
-                }]]
-                )
-
-            if code >= 300 then
-                ngx.status = code
-            end
-            ngx.say(body)
-        }
-    }
---- response_body
-passed
-
-
-
-=== TEST 21: hit route and user-agent in both whitelist and default deny list
-
---- request
-GET /hello
---- more_headers
-User-Agent:Baiduspider/1.0
---- error_code: 200
-
-
-
-=== TEST 22: message that do not reach the minimum range
---- config
-    location /t {
-        content_by_lua_block {
-            local t = require("lib.test_admin").test
-            local code, body = t('/apisix/admin/routes/1',
-                 ngx.HTTP_PUT,
-                 [[{
-                        "uri": "/hello",
-                        "upstream": {
-                            "type": "roundrobin",
-                            "nodes": {
-                                "127.0.0.1:1980": 1
-                            }
-                        },
-                        "plugins": {
-                            "bot-restriction": {
+                            "ua-restriction": {
                                  "message": ""
                             }
                         }
@@ -477,7 +379,7 @@ qr/string too short, expected at least 1, got 0/
 
 
 
-=== TEST 23: exceeds the maximum limit of message
+=== TEST 19: exceeds the maximum limit of message
 --- config
     location /t {
         content_by_lua_block {
@@ -493,7 +395,10 @@ qr/string too short, expected at least 1, got 0/
                     }
                 },
                 plugins = {
-                    ["bot-restriction"] = {
+                    ["ua-restriction"] = {
+                        denylist = {
+                           "my-bot1",
+                        },
                         message = ("-1Aa#"):rep(205)
                     }
                 }
@@ -512,7 +417,7 @@ qr/string too long, expected at most 1024, got 1025/
 
 
 
-=== TEST 24: set custom message
+=== TEST 20: set custom message
 --- config
     location /t {
         content_by_lua_block {
@@ -528,8 +433,11 @@ qr/string too long, expected at most 1024, got 1025/
                             }
                         },
                         "plugins": {
-                            "bot-restriction": {
-                                 "message": "Do you want to do something bad?"
+                            "ua-restriction": {
+                                "denylist": [
+                                    "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                ],
+                                "message": "Do you want to do something bad?"
                             }
                         }
                 }]]
@@ -547,18 +455,18 @@ passed
 
 
 
-=== TEST 25: test custom message
+=== TEST 21: test custom message
 --- request
 GET /hello
 --- more_headers
-User-Agent:Twitterbot/1.0
+User-Agent:Baiduspider/1.0
 --- error_code: 403
 --- response_body
 {"message":"Do you want to do something bad?"}
 
 
 
-=== TEST 26: test remove bot-restriction part 1
+=== TEST 22: test remove ua-restriction part 1, enable
 --- config
     location /enable {
         content_by_lua_block {
@@ -574,9 +482,12 @@ User-Agent:Twitterbot/1.0
                             }
                         },
                         "plugins": {
-                        "bot-restriction": {
+                            "ua-restriction": {
+                                "denylist": [
+                                     "(Baiduspider)/(\\d+)\\.(\\d+)"
+                                ]
+                            }
                         }
-                    }
                 }]]
                 )
 
@@ -592,16 +503,16 @@ GET /enable
 
 
 
-=== TEST 27: test remove bot-restriction part 2
+=== TEST 23: test remove ua-restriction part 2
 --- request
 GET /hello
 --- more_headers
-User-Agent:Twitterbot/1.0
+User-Agent:Baiduspider/1.0
 --- error_code: 403
 
 
 
-=== TEST 28: test remove bot-restriction part 3, remove plugin
+=== TEST 24: test remove ua-restriction part 3, remove plugin
 --- config
     location /disable {
         content_by_lua_block {
@@ -633,17 +544,17 @@ GET /disable
 
 
 
-=== TEST 29: test remove bot-restriction part 4, check bot User-Agent
+=== TEST 25: test remove ua-restriction part 4, check spider User-Agent
 --- request
 GET /hello
 --- more_headers
-User-Agent:Twitterbot/1.0
+User-Agent:Baiduspider/1.0
 --- response_body
 hello world
 
 
 
-=== TEST 30: set disable=true
+=== TEST 26: set disable=true
 --- config
     location /t {
         content_by_lua_block {
@@ -653,8 +564,8 @@ hello world
                 [[{
                     "uri": "/hello",
                     "plugins": {
-                        "bot-restriction": {
-                            "blacklist": [
+                        "ua-restriction": {
+                            "denylist": [
                                 "foo"
                             ],
                             "disable": true

From 420e676a5fe219f164433a11b76f676223f3cf69 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Mon, 19 Jul 2021 19:31:22 +0800
Subject: [PATCH 09/13] fix Review

---
 apisix/plugins/ua-restriction.lua        | 1 -
 docs/en/latest/plugins/ua-restriction.md | 2 +-
 docs/zh/latest/plugins/ua-restriction.md | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/apisix/plugins/ua-restriction.lua b/apisix/plugins/ua-restriction.lua
index eaba09ff7375..344ef523e37d 100644
--- a/apisix/plugins/ua-restriction.lua
+++ b/apisix/plugins/ua-restriction.lua
@@ -49,7 +49,6 @@ local schema = {
         {required = {"allowlist"}},
         {required = {"denylist"}},
     },
-    minProperties = 1,
     additionalProperties = false,
 }
 
diff --git a/docs/en/latest/plugins/ua-restriction.md b/docs/en/latest/plugins/ua-restriction.md
index 3bf335e70e3f..25de1e346af0 100644
--- a/docs/en/latest/plugins/ua-restriction.md
+++ b/docs/en/latest/plugins/ua-restriction.md
@@ -39,7 +39,7 @@ The `ua-restriction` can restrict access to a Service or a Route by either `allo
 | --------- | ------------- | ----------- | ------- | ----- | ---------------------------------------- |
 | allowlist | array[string] | optional    |         |       | List of User-Agent of allowlist. |
 | denylist | array[string] | optional    |         |       | List of User-Agent of denylist. |
-| message | string | optional             | Not allowed. | [1, 1024] | Message of deny reason. |
+| message | string | optional             | Not allowed. | length range: [1, 1024] | Message of deny reason. |
 
 Any of `allowlist` or `denylist` can be optional, and can work together in this order: allowlist->denylist
 
diff --git a/docs/zh/latest/plugins/ua-restriction.md b/docs/zh/latest/plugins/ua-restriction.md
index 3126d9c3d0cf..091c026c2b6d 100644
--- a/docs/zh/latest/plugins/ua-restriction.md
+++ b/docs/zh/latest/plugins/ua-restriction.md
@@ -39,7 +39,7 @@ title: ua-restriction
 | --------- | ------------- | ------ | ------ | ------ | -------------------------------- |
 | allowlist | array[string] | 可选   |        |        | 加入白名单的 User-Agent |
 | denylist | array[string] | 可选   |        |        | 加入黑名单的 User-Agent |
-| message | string | 可选   | Not allowed. | [1, 1024] | 在未允许的 User-Agent 访问的情况下返回的信息 |
+| message | string | 可选   | Not allowed. | 长度限制：[1, 1024] | 在未允许的 User-Agent 访问的情况下返回的信息 |
 
 白名单或黑名单可以同时启用，此插件对 User-Agent 的检查先后顺序依次如下：白名单、黑名单。`message`可以由用户自定义。
 

From f7a61bd9e802e4176727422b54c71d46afae38a0 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Tue, 20 Jul 2021 15:46:20 +0800
Subject: [PATCH 10/13] Fix Review

---
 t/plugin/ua-restriction.t | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/t/plugin/ua-restriction.t b/t/plugin/ua-restriction.t
index 9949fafac14b..4c1c6000e0e9 100644
--- a/t/plugin/ua-restriction.t
+++ b/t/plugin/ua-restriction.t
@@ -330,7 +330,7 @@ passed
 
 
 
-=== TEST 16: hit route and user-agent in both allowlist and denylist, part 1
+=== TEST 16: hit route and user-agent in both allowlist and denylist, pass(part 1)
 --- request
 GET /hello
 --- more_headers
@@ -339,7 +339,7 @@ User-Agent:foo/bar
 
 
 
-=== TEST 17: hit route and user-agent in both allowlist and denylist, part 2
+=== TEST 17: hit route and user-agent in both allowlist and denylist, pass(part 2)
 --- request
 GET /hello
 --- more_headers

From 9ace564f0a66b987a0cc8cace1e4935464d92542 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Tue, 20 Jul 2021 17:52:13 +0800
Subject: [PATCH 11/13] Fix Test

---
 t/plugin/ua-restriction.t | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/t/plugin/ua-restriction.t b/t/plugin/ua-restriction.t
index 4c1c6000e0e9..9c53d0bf4e9c 100644
--- a/t/plugin/ua-restriction.t
+++ b/t/plugin/ua-restriction.t
@@ -466,7 +466,7 @@ User-Agent:Baiduspider/1.0
 
 
 
-=== TEST 22: test remove ua-restriction part 1, enable
+=== TEST 22: test remove ua-restriction, enable(part 1)
 --- config
     location /enable {
         content_by_lua_block {
@@ -503,7 +503,7 @@ GET /enable
 
 
 
-=== TEST 23: test remove ua-restriction part 2
+=== TEST 23: test remove ua-restriction, fail(part 2)
 --- request
 GET /hello
 --- more_headers
@@ -512,7 +512,7 @@ User-Agent:Baiduspider/1.0
 
 
 
-=== TEST 24: test remove ua-restriction part 3, remove plugin
+=== TEST 24: test remove ua-restriction, remove plugin(part 3)
 --- config
     location /disable {
         content_by_lua_block {
@@ -544,7 +544,7 @@ GET /disable
 
 
 
-=== TEST 25: test remove ua-restriction part 4, check spider User-Agent
+=== TEST 25: test remove ua-restriction, check spider User-Agent(part 4)
 --- request
 GET /hello
 --- more_headers

From a0fb6dd677a4a336593b4ba2e253249abd039477 Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Tue, 20 Jul 2021 22:27:29 +0800
Subject: [PATCH 12/13] Fix ci

---
 t/plugin/ua-restriction.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/t/plugin/ua-restriction.t b/t/plugin/ua-restriction.t
index 9c53d0bf4e9c..ef165c014ba8 100644
--- a/t/plugin/ua-restriction.t
+++ b/t/plugin/ua-restriction.t
@@ -466,7 +466,7 @@ User-Agent:Baiduspider/1.0
 
 
 
-=== TEST 22: test remove ua-restriction, enable(part 1)
+=== TEST 22: test remove ua-restriction, add denylist(part 1)
 --- config
     location /enable {
         content_by_lua_block {

From 6837ed15e4b58c443f58b2dff318422bc44a373f Mon Sep 17 00:00:00 2001
From: "Arthur.Zhang" <zhangya_no1@qq.com>
Date: Wed, 21 Jul 2021 16:33:40 +0800
Subject: [PATCH 13/13] Add  bypass_missing and fix test case

---
 apisix/plugins/ua-restriction.lua        |  24 +--
 docs/en/latest/plugins/ua-restriction.md |  10 +-
 docs/zh/latest/plugins/ua-restriction.md |   4 +-
 t/plugin/ua-restriction.t                | 209 ++++++++++++++++++++---
 4 files changed, 206 insertions(+), 41 deletions(-)

diff --git a/apisix/plugins/ua-restriction.lua b/apisix/plugins/ua-restriction.lua
index 344ef523e37d..95bc3d47a7f2 100644
--- a/apisix/plugins/ua-restriction.lua
+++ b/apisix/plugins/ua-restriction.lua
@@ -30,11 +30,9 @@ local lrucache_useragent = core.lrucache.new({ ttl = 300, count = 4096 })
 local schema = {
     type = "object",
     properties = {
-        message = {
-            type = "string",
-            minLength = 1,
-            maxLength = 1024,
-            default = "Not allowed"
+        bypass_missing = {
+            type = "boolean",
+            default = false,
         },
         allowlist = {
             type = "array",
@@ -44,10 +42,12 @@ local schema = {
             type = "array",
             minItems = 1
         },
-    },
-    anyOf = {
-        {required = {"allowlist"}},
-        {required = {"denylist"}},
+        message = {
+            type = "string",
+            minLength = 1,
+            maxLength = 1024,
+            default = "Not allowed"
+        },
     },
     additionalProperties = false,
 }
@@ -96,7 +96,11 @@ function _M.access(conf, ctx)
     local user_agent = core.request.header(ctx, "User-Agent")
 
     if not user_agent then
-        return
+        if conf.bypass_missing then
+            return
+        else
+            return 403, { message = conf.message }
+        end
     end
     local match = MATCH_NONE
     if type(user_agent) == "table" then
diff --git a/docs/en/latest/plugins/ua-restriction.md b/docs/en/latest/plugins/ua-restriction.md
index 25de1e346af0..7855ccff21c5 100644
--- a/docs/en/latest/plugins/ua-restriction.md
+++ b/docs/en/latest/plugins/ua-restriction.md
@@ -31,14 +31,15 @@ title: ua-restriction
 
 ## Name
 
-The `ua-restriction` can restrict access to a Service or a Route by either `allowlist` or `denylist` `User-Agent`.
+The `ua-restriction` can restrict access to a Service or a Route by `allowlist` and `denylist` `User-Agent` header.
 
 ## Attributes
 
 | Name      | Type          | Requirement | Default | Valid | Description                              |
 | --------- | ------------- | ----------- | ------- | ----- | ---------------------------------------- |
-| allowlist | array[string] | optional    |         |       | List of User-Agent of allowlist. |
-| denylist | array[string] | optional    |         |       | List of User-Agent of denylist. |
+| bypass_missing  | boolean       | optional    | false   |       | Whether to bypass the check when the User-Agent header is missing |
+| allowlist | array[string] | optional    |         |       | A list of allowed User-Agent headers. |
+| denylist | array[string] | optional    |         |       | A list of denied User-Agent headers. |
 | message | string | optional             | Not allowed. | length range: [1, 1024] | Message of deny reason. |
 
 Any of `allowlist` or `denylist` can be optional, and can work together in this order: allowlist->denylist
@@ -61,6 +62,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
     },
     "plugins": {
         "ua-restriction": {
+             "bypass_missing": true,
              "allowlist": [
                  "my-bot1",
                  "(Baiduspider)/(\\d+)\\.(\\d+)"
@@ -98,7 +100,7 @@ HTTP/1.1 200 OK
 ...
 ```
 
-Requests from bot User-Agent:
+Requests with the bot User-Agent:
 
 ```shell
 $ curl http://127.0.0.1:9080/index.html --header 'User-Agent: Twitterspider/2.0'
diff --git a/docs/zh/latest/plugins/ua-restriction.md b/docs/zh/latest/plugins/ua-restriction.md
index 091c026c2b6d..a13b80658112 100644
--- a/docs/zh/latest/plugins/ua-restriction.md
+++ b/docs/zh/latest/plugins/ua-restriction.md
@@ -31,12 +31,13 @@ title: ua-restriction
 
 ## 名字
 
-`ua-restriction` 可以通过以下方式限制对服务或接口的访问，可以将指定 `User-Agent` 列入白名单或黑名单。
+`ua-restriction` 可以通过将指定 `User-Agent` 列入白名单或黑名单的方式来限制对服务或接口的访问。
 
 ## 属性
 
 | 参数名    | 类型          | 可选项 | 默认值 | 有效值 | 描述                             |
 | --------- | ------------- | ------ | ------ | ------ | -------------------------------- |
+| bypass_missing  | boolean       | 可选    | false   |       | User-Agent 不存在时是否绕过检查 |
 | allowlist | array[string] | 可选   |        |        | 加入白名单的 User-Agent |
 | denylist | array[string] | 可选   |        |        | 加入黑名单的 User-Agent |
 | message | string | 可选   | Not allowed. | 长度限制：[1, 1024] | 在未允许的 User-Agent 访问的情况下返回的信息 |
@@ -59,6 +60,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
     },
     "plugins": {
         "ua-restriction": {
+            "bypass_missing": true,
              "allowlist": [
                  "my-bot1",
                  "(Baiduspider)/(\\d+)\\.(\\d+)"
diff --git a/t/plugin/ua-restriction.t b/t/plugin/ua-restriction.t
index ef165c014ba8..77c68748edea 100644
--- a/t/plugin/ua-restriction.t
+++ b/t/plugin/ua-restriction.t
@@ -38,12 +38,24 @@ run_tests;
 
 __DATA__
 
-=== TEST 1: empty conf
+=== TEST 1: set allowlist, denylist, bypass_missing and user-defined message
 --- config
     location /t {
         content_by_lua_block {
             local plugin = require("apisix.plugins.ua-restriction")
-            local ok, err = plugin.check_schema({})
+            local conf = {
+               bypass_missing = true,
+               allowlist = {
+                    "my-bot1",
+                    "my-bot2"
+               },
+               denylist = {
+                    "my-bot1",
+                    "my-bot2"
+               },
+               message = "User-Agent Forbidden",
+            }
+            local ok, err = plugin.check_schema(conf)
             if not ok then
                 ngx.say(err)
             end
@@ -51,37 +63,30 @@ __DATA__
             ngx.say(require("toolkit.json").encode(conf))
         }
     }
---- response_body_like eval
-qr/object matches none of the requireds/
+--- response_body
+{"allowlist":["my-bot1","my-bot2"],"bypass_missing":true,"denylist":["my-bot1","my-bot2"],"message":"User-Agent Forbidden"}
 
 
 
-=== TEST 2: set allowlist, denylist and user-defined message
+=== TEST 2: bypass_missing not boolean
 --- config
     location /t {
         content_by_lua_block {
             local plugin = require("apisix.plugins.ua-restriction")
             local conf = {
-               allowlist = {
-                    "my-bot1",
-                    "my-bot2"
-               },
-               denylist = {
-                    "my-bot1",
-                    "my-bot2"
-               },
-               message = "User-Agent Forbidden",
+                bypass_missing = "foo",
             }
             local ok, err = plugin.check_schema(conf)
             if not ok then
                 ngx.say(err)
             end
 
-            ngx.say(require("toolkit.json").encode(conf))
+            ngx.say("done")
         }
     }
 --- response_body
-{"allowlist":["my-bot1","my-bot2"],"denylist":["my-bot1","my-bot2"],"message":"User-Agent Forbidden"}
+property "bypass_missing" validation failed: wrong type: expected boolean, got string
+done
 
 
 
@@ -194,16 +199,20 @@ GET /hello
 --- more_headers
 User-Agent:my-bot1
 --- error_code: 403
+--- response_body
+{"message":"Not allowed"}
 
 
 
-=== TEST 8: hit route and user-agent in denylist with multiple
+=== TEST 8: hit route and user-agent in denylist with multiple user-agent
 --- request
 GET /hello
 --- more_headers
 User-Agent:my-bot1
 User-Agent:my-bot2
 --- error_code: 403
+--- response_body
+{"message":"Not allowed"}
 
 
 
@@ -213,6 +222,8 @@ GET /hello
 --- more_headers
 User-Agent:Baiduspider/3.0
 --- error_code: 403
+--- response_body
+{"message":"Not allowed"}
 
 
 
@@ -222,6 +233,8 @@ GET /hello
 --- more_headers
 User-Agent:foo/bar
 --- error_code: 200
+--- response_body
+hello world
 
 
 
@@ -268,6 +281,8 @@ GET /hello
 --- more_headers
 User-Agent:my-bot1
 --- error_code: 200
+--- response_body
+hello world
 
 
 
@@ -277,6 +292,8 @@ GET /hello
 --- more_headers
 User-Agent:Baiduspider/3.0
 --- error_code: 200
+--- response_body
+hello world
 
 
 
@@ -286,6 +303,8 @@ GET /hello
 --- more_headers
 User-Agent:foo/bar
 --- error_code: 200
+--- response_body
+hello world
 
 
 
@@ -336,6 +355,8 @@ GET /hello
 --- more_headers
 User-Agent:foo/bar
 --- error_code: 200
+--- response_body
+hello world
 
 
 
@@ -345,10 +366,140 @@ GET /hello
 --- more_headers
 User-Agent:Baiduspider/1.0
 --- error_code: 200
+--- response_body
+hello world
+
+
+
+=== TEST 18: bypass_missing test, using default, reset conf(part1)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "ua-restriction": {
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 19: bypass_missing test, using default, send request without User-Agent(part2)
+--- request
+GET /hello
+--- error_code: 403
+--- response_body
+{"message":"Not allowed"}
+
+
+
+=== TEST 20: bypass_missing test, set to true(part1)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "ua-restriction": {
+                                "bypass_missing": true
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 21: bypass_missing test, set to true, send request without User-Agent(part2)
+--- request
+GET /hello
+--- error_code: 200
+--- response_body
+hello world
+
+
+
+=== TEST 22: bypass_missing test, set to false(part1)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                        "uri": "/hello",
+                        "upstream": {
+                            "type": "roundrobin",
+                            "nodes": {
+                                "127.0.0.1:1980": 1
+                            }
+                        },
+                        "plugins": {
+                            "ua-restriction": {
+                                "bypass_missing": false
+                            }
+                        }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 23: bypass_missing test, set to false, send request without User-Agent(part2)
+--- request
+GET /hello
+--- error_code: 403
+--- response_body
+{"message":"Not allowed"}
 
 
 
-=== TEST 18: message that do not reach the minimum range
+=== TEST 24: message that do not reach the minimum range
 --- config
     location /t {
         content_by_lua_block {
@@ -379,7 +530,7 @@ qr/string too short, expected at least 1, got 0/
 
 
 
-=== TEST 19: exceeds the maximum limit of message
+=== TEST 25: exceeds the maximum limit of message
 --- config
     location /t {
         content_by_lua_block {
@@ -417,7 +568,7 @@ qr/string too long, expected at most 1024, got 1025/
 
 
 
-=== TEST 20: set custom message
+=== TEST 26: set custom message
 --- config
     location /t {
         content_by_lua_block {
@@ -455,7 +606,7 @@ passed
 
 
 
-=== TEST 21: test custom message
+=== TEST 27: test custom message
 --- request
 GET /hello
 --- more_headers
@@ -466,7 +617,7 @@ User-Agent:Baiduspider/1.0
 
 
 
-=== TEST 22: test remove ua-restriction, add denylist(part 1)
+=== TEST 28: test remove ua-restriction, add denylist(part 1)
 --- config
     location /enable {
         content_by_lua_block {
@@ -500,19 +651,23 @@ User-Agent:Baiduspider/1.0
 --- request
 GET /enable
 --- error_code: 200
+--- response_body
+passed
 
 
 
-=== TEST 23: test remove ua-restriction, fail(part 2)
+=== TEST 29: test remove ua-restriction, fail(part 2)
 --- request
 GET /hello
 --- more_headers
 User-Agent:Baiduspider/1.0
 --- error_code: 403
+--- response_body
+{"message":"Not allowed"}
 
 
 
-=== TEST 24: test remove ua-restriction, remove plugin(part 3)
+=== TEST 30: test remove ua-restriction, remove plugin(part 3)
 --- config
     location /disable {
         content_by_lua_block {
@@ -541,10 +696,12 @@ User-Agent:Baiduspider/1.0
 --- request
 GET /disable
 --- error_code: 200
+--- response_body
+passed
 
 
 
-=== TEST 25: test remove ua-restriction, check spider User-Agent(part 4)
+=== TEST 31: test remove ua-restriction, check spider User-Agent(part 4)
 --- request
 GET /hello
 --- more_headers
@@ -554,7 +711,7 @@ hello world
 
 
 
-=== TEST 26: set disable=true
+=== TEST 32: set disable=true
 --- config
     location /t {
         content_by_lua_block {