From bea7e490146b54427b4515789938696e77d1e538 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Wed, 1 Apr 2020 23:05:40 +0800 Subject: [PATCH 01/20] feat: add api aggreate --- apisix/init.lua | 5 + bin/apisix | 6 + lua/apisix/http/aggregate.lua | 83 ++++++++ t/APISIX.pm | 6 + t/http/aggregate.t | 391 ++++++++++++++++++++++++++++++++++ 5 files changed, 491 insertions(+) create mode 100644 lua/apisix/http/aggregate.lua create mode 100644 t/http/aggregate.t diff --git a/apisix/init.lua b/apisix/init.lua index 3a081575fc2d..8b7d2792a244 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -17,6 +17,7 @@ local require = require local core = require("apisix.core") local plugin = require("apisix.plugin") +local aggregate = require("apisix.http.aggregate").aggregate local service_fetch = require("apisix.http.service").get local admin_init = require("apisix.admin.init") local get_var = require("resty.ngxvar").fetch @@ -630,5 +631,9 @@ function _M.stream_log_phase() run_plugin("log") end +function _M.http_api_aggregate() + local code, body = aggregate() + core.response.exit(code, body) +end return _M diff --git a/bin/apisix b/bin/apisix index 83712c4c9bca..8490bf91a30f 100755 --- a/bin/apisix +++ b/bin/apisix @@ -380,6 +380,12 @@ http { } {% end %} + location /apisix/aggregate { + content_by_lua_block { + apisix.http_api_aggregate() + } + } + ssl_certificate_by_lua_block { apisix.http_ssl_phase() } diff --git a/lua/apisix/http/aggregate.lua b/lua/apisix/http/aggregate.lua new file mode 100644 index 000000000000..f9ca2df3d81b --- /dev/null +++ b/lua/apisix/http/aggregate.lua @@ -0,0 +1,83 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local http = require("resty.http") +local ngx = ngx + + +local _M = { + version = 0.1, +} + +local function check_input(data) + if not data.pipeline then + return 400, {message = "missing 'pipeline' in input"} + end + local type_timeout = type(data.timeout) + if type_timeout ~= "number" and type_timeout ~= "nil" then + return 400, {message = "'timeout' should be number"} + end + if not data.timeout or data.timeout == 0 then + data.timeout = 30000 + end +end + +function _M.aggregate(service_id) + ngx.req.read_body() + local req_body = ngx.req.get_body_data() + local data, err = core.json.decode(req_body) + local code, body = check_input(data) + if not data then + core.log.error("invalid request body: ", req_body, " err: ", err) + return 400, {message = "invalid request body", req_body = req_body} + end + + if code then + return code, body + end + + local httpc = http.new() + core.log.info(data.timeout) + httpc:set_timeout(data.timeout) + httpc:connect("127.0.0.1", ngx.var.server_port) + local responses, err = httpc:request_pipeline(data.pipeline) + if not responses then + return 400, {message = "request failed", err = err} + end + + local aggregated_resp = {} + for i,r in ipairs(responses) do + if not r.status then + core.table.insert(aggregated_resp, { + status = 500, + reason = "target timeout" + }) + end + local sub_resp = { + status = r.status, + reason = r.reason, + headers = r.headers, + } + if r.has_body then + sub_resp.body = r:read_body() + end + core.table.insert(aggregated_resp, sub_resp) + end + return 200, aggregated_resp +end + +return _M diff --git a/t/APISIX.pm b/t/APISIX.pm index 6b19152469e7..051e58d1bc95 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -297,6 +297,12 @@ _EOC_ } } + location /apisix/aggregate { + content_by_lua_block { + apisix.http_api_aggregate() + } + } + location / { set \$upstream_mirror_host ''; set \$upstream_scheme 'http'; diff --git a/t/http/aggregate.t b/t/http/aggregate.t new file mode 100644 index 000000000000..fb82018a5bbc --- /dev/null +++ b/t/http/aggregate.t @@ -0,0 +1,391 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +no_long_string(); +no_root_location(); +log_level("info"); + +run_tests; + +__DATA__ + +=== TEST 1: sanity +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=], + [=[[ + { + "status": 200, + "body":"B", + "headers": { + "X-Res": "B", + "X-Header1": "hello", + "X-Header2": "world" + } + }, + { + "status": 201, + "body":"C", + "headers": { + "X-Res": "C", + "X-Method": "PUT" + } + }, + { + "status": 202, + "body":"D", + "headers": { + "X-Res": "D" + } + } + ]]=] + ) + + ngx.status = code + ngx.say(body) + } + } + + location = /b { + content_by_lua_block { + ngx.status = 200 + ngx.header["X-Header1"] = ngx.req.get_headers()["Header1"] + ngx.header["X-Header2"] = ngx.req.get_headers()["Header2"] + ngx.header["X-Res"] = "B" + ngx.print("B") + } + } + location = /c { + content_by_lua_block { + ngx.status = 201 + ngx.header["X-Res"] = "C" + ngx.header["X-Method"] = ngx.req.get_method() + ngx.print("C") + } + } + location = /d { + content_by_lua_block { + ngx.status = 202 + ngx.header["X-Res"] = "D" + ngx.print("D") + } + } +--- request +GET /aggregate +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 2: missing pipeling +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline1":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"message":"missing 'pipeline' in input"} +--- no_error_log +[error] + + + +=== TEST 3: timeout is not number +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "timeout": "200", + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"message":"'timeout' should be number"} +--- no_error_log +[error] + + + +=== TEST 4: different response time +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "timeout": 2000, + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=], + [=[[ + { + "status": 200 + }, + { + "status": 201 + }, + { + "status": 202 + } + ]]=] + ) + + ngx.status = code + ngx.say(body) + } + } + + location = /b { + content_by_lua_block { + ngx.sleep(0.02) + ngx.status = 200 + } + } + location = /c { + content_by_lua_block { + ngx.sleep(0.05) + ngx.status = 201 + } + } + location = /d { + content_by_lua_block { + ngx.sleep(1) + ngx.status = 202 + } + } +--- request +GET /aggregate +--- response_body +passed +--- no_error_log +[error] + + + +=== TEST 5: last request timeout +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "timeout": 100, + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=], + [=[[ + { + "status": 200 + }, + { + "status": 201 + }, + { + "status": 500, + "reason": "target timeout" + } + ]]=] + ) + + ngx.status = code + ngx.say(body) + } + } + + location = /b { + content_by_lua_block { + ngx.status = 200 + } + } + location = /c { + content_by_lua_block { + ngx.status = 201 + } + } + location = /d { + content_by_lua_block { + ngx.sleep(1) + ngx.status = 202 + } + } +--- request +GET /aggregate +--- response_body +passed +--- error_log +timeout + + + +=== TEST 6: first request timeout +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "timeout": 100, + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=], + [=[[ + { + "status": 500, + "reason": "target timeout" + } + ]]=] + ) + + ngx.status = code + ngx.say(body) + } + } + + location = /b { + content_by_lua_block { + ngx.sleep(1) + ngx.status = 200 + } + } + location = /c { + content_by_lua_block { + ngx.status = 201 + } + } + location = /d { + content_by_lua_block { + ngx.status = 202 + } + } +--- request +GET /aggregate +--- response_body +passed +--- error_log +timeout From 560dbf76328ce53ad2c6329786a95f0cf80a623e Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Wed, 1 Apr 2020 23:26:10 +0800 Subject: [PATCH 02/20] fix: move file and timeout code --- {lua/apisix => apisix}/http/aggregate.lua | 2 +- t/http/aggregate.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) rename {lua/apisix => apisix}/http/aggregate.lua (98%) diff --git a/lua/apisix/http/aggregate.lua b/apisix/http/aggregate.lua similarity index 98% rename from lua/apisix/http/aggregate.lua rename to apisix/http/aggregate.lua index f9ca2df3d81b..96ff0f9e103d 100644 --- a/lua/apisix/http/aggregate.lua +++ b/apisix/http/aggregate.lua @@ -63,7 +63,7 @@ function _M.aggregate(service_id) for i,r in ipairs(responses) do if not r.status then core.table.insert(aggregated_resp, { - status = 500, + status = 504, reason = "target timeout" }) end diff --git a/t/http/aggregate.t b/t/http/aggregate.t index fb82018a5bbc..af508e94218e 100644 --- a/t/http/aggregate.t +++ b/t/http/aggregate.t @@ -294,7 +294,7 @@ passed "status": 201 }, { - "status": 500, + "status": 504, "reason": "target timeout" } ]]=] @@ -356,7 +356,7 @@ timeout }]=], [=[[ { - "status": 500, + "status": 504, "reason": "target timeout" } ]]=] From c16b85ba24d44c46f9451dde7daf9e55aec4054d Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Thu, 2 Apr 2020 11:19:51 +0800 Subject: [PATCH 03/20] feat: add global headers and query for pipeline --- apisix/http/aggregate.lua | 44 +++++++++++++++++++++++++++++++++++---- t/http/aggregate.t | 27 ++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/apisix/http/aggregate.lua b/apisix/http/aggregate.lua index 96ff0f9e103d..73216bda5b5f 100644 --- a/apisix/http/aggregate.lua +++ b/apisix/http/aggregate.lua @@ -14,10 +14,12 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local core = require("apisix.core") -local http = require("resty.http") -local ngx = ngx - +local core = require("apisix.core") +local http = require("resty.http") +local ngx = ngx +local ipairs = ipairs +local pairs = pairs +local type = type local _M = { version = 0.1, @@ -36,6 +38,38 @@ local function check_input(data) end end +local function set_base_header(data) + if not data.headers then + return + end + + for i,req in ipairs(data.pipeline) do + if not req.headers then + req.headers = data.headers + else + for k, v in pairs(data.headers) do + req.headers[k] = v + end + end + end +end + +local function set_base_query(data) + if not data.query then + return + end + + for i,req in ipairs(data.pipeline) do + if not req.query then + req.query = data.query + else + for k, v in pairs(data.query) do + req.query[k] = v + end + end + end +end + function _M.aggregate(service_id) ngx.req.read_body() local req_body = ngx.req.get_body_data() @@ -54,6 +88,8 @@ function _M.aggregate(service_id) core.log.info(data.timeout) httpc:set_timeout(data.timeout) httpc:connect("127.0.0.1", ngx.var.server_port) + set_base_header(data) + set_base_query(data) local responses, err = httpc:request_pipeline(data.pipeline) if not responses then return 400, {message = "request failed", err = err} diff --git a/t/http/aggregate.t b/t/http/aggregate.t index af508e94218e..d6da338a4be3 100644 --- a/t/http/aggregate.t +++ b/t/http/aggregate.t @@ -34,6 +34,12 @@ __DATA__ local code, body = t('/apisix/aggregate', ngx.HTTP_POST, [=[{ + "query": { + "base": "base_query" + }, + "headers": { + "Base-Header": "base" + }, "pipeline":[ { "path": "/b", @@ -45,7 +51,10 @@ __DATA__ "path": "/c", "method": "PUT" },{ - "path": "/d" + "path": "/d", + "query": { + "one": "thing" + } }] }]=], [=[[ @@ -53,6 +62,8 @@ __DATA__ "status": 200, "body":"B", "headers": { + "Base-Header": "base", + "Base-Query": "base_query", "X-Res": "B", "X-Header1": "hello", "X-Header2": "world" @@ -62,6 +73,8 @@ __DATA__ "status": 201, "body":"C", "headers": { + "Base-Header": "base", + "Base-Query": "base_query", "X-Res": "C", "X-Method": "PUT" } @@ -70,7 +83,10 @@ __DATA__ "status": 202, "body":"D", "headers": { - "X-Res": "D" + "Base-Header": "base", + "Base-Query": "base_query", + "X-Res": "D", + "X-Query-One": "thing" } } ]]=] @@ -84,6 +100,8 @@ __DATA__ location = /b { content_by_lua_block { ngx.status = 200 + ngx.header["Base-Header"] = ngx.req.get_headers()["Base-Header"] + ngx.header["Base-Query"] = ngx.var.arg_base ngx.header["X-Header1"] = ngx.req.get_headers()["Header1"] ngx.header["X-Header2"] = ngx.req.get_headers()["Header2"] ngx.header["X-Res"] = "B" @@ -93,6 +111,8 @@ __DATA__ location = /c { content_by_lua_block { ngx.status = 201 + ngx.header["Base-Header"] = ngx.req.get_headers()["Base-Header"] + ngx.header["Base-Query"] = ngx.var.arg_base ngx.header["X-Res"] = "C" ngx.header["X-Method"] = ngx.req.get_method() ngx.print("C") @@ -101,6 +121,9 @@ __DATA__ location = /d { content_by_lua_block { ngx.status = 202 + ngx.header["Base-Header"] = ngx.req.get_headers()["Base-Header"] + ngx.header["Base-Query"] = ngx.var.arg_base + ngx.header["X-Query-One"] = ngx.var.arg_one ngx.header["X-Res"] = "D" ngx.print("D") } From 813d291d299cef579ccffb37e39a9643a4023d75 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Thu, 2 Apr 2020 11:35:26 +0800 Subject: [PATCH 04/20] fix: specify header and query should be high priority --- apisix/http/aggregate.lua | 8 ++++++-- t/http/aggregate.t | 20 ++++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/apisix/http/aggregate.lua b/apisix/http/aggregate.lua index 73216bda5b5f..f5828cfe25bf 100644 --- a/apisix/http/aggregate.lua +++ b/apisix/http/aggregate.lua @@ -48,7 +48,9 @@ local function set_base_header(data) req.headers = data.headers else for k, v in pairs(data.headers) do - req.headers[k] = v + if not req.headers[k] then + req.headers[k] = v + end end end end @@ -64,7 +66,9 @@ local function set_base_query(data) req.query = data.query else for k, v in pairs(data.query) do - req.query[k] = v + if not req.query[k] then + req.query[k] = v + end end end end diff --git a/t/http/aggregate.t b/t/http/aggregate.t index d6da338a4be3..68a21337059d 100644 --- a/t/http/aggregate.t +++ b/t/http/aggregate.t @@ -35,17 +35,20 @@ __DATA__ ngx.HTTP_POST, [=[{ "query": { - "base": "base_query" + "base": "base_query", + "conflict": "query_value" }, "headers": { - "Base-Header": "base" + "Base-Header": "base", + "Conflict-Header": "header_value" }, "pipeline":[ { "path": "/b", "headers": { "Header1": "hello", - "Header2": "world" + "Header2": "world", + "Conflict-Header": "b-header-value" } },{ "path": "/c", @@ -53,7 +56,8 @@ __DATA__ },{ "path": "/d", "query": { - "one": "thing" + "one": "thing", + "conflict": "d_value" } }] }]=], @@ -66,7 +70,8 @@ __DATA__ "Base-Query": "base_query", "X-Res": "B", "X-Header1": "hello", - "X-Header2": "world" + "X-Header2": "world", + "X-Conflict-Header": "b-header-value" } }, { @@ -86,7 +91,8 @@ __DATA__ "Base-Header": "base", "Base-Query": "base_query", "X-Res": "D", - "X-Query-One": "thing" + "X-Query-One": "thing", + "X-Query-Conflict": "d_value" } } ]]=] @@ -104,6 +110,7 @@ __DATA__ ngx.header["Base-Query"] = ngx.var.arg_base ngx.header["X-Header1"] = ngx.req.get_headers()["Header1"] ngx.header["X-Header2"] = ngx.req.get_headers()["Header2"] + ngx.header["X-Conflict-Header"] = ngx.req.get_headers()["Conflict-Header"] ngx.header["X-Res"] = "B" ngx.print("B") } @@ -124,6 +131,7 @@ __DATA__ ngx.header["Base-Header"] = ngx.req.get_headers()["Base-Header"] ngx.header["Base-Query"] = ngx.var.arg_base ngx.header["X-Query-One"] = ngx.var.arg_one + ngx.header["X-Query-Conflict"] = ngx.var.arg_conflict ngx.header["X-Res"] = "D" ngx.print("D") } From 99b443ba9a184a2432447a61c66109554f2a6a12 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Thu, 2 Apr 2020 17:08:04 +0800 Subject: [PATCH 05/20] refactor: using plugin to implement the features --- apisix/init.lua | 6 ---- .../api-aggregate.lua} | 33 +++++++++++++++---- bin/apisix | 6 ---- conf/config.yaml | 1 + t/APISIX.pm | 6 ---- t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 1 + .../aggregate.t => plugin/api-aggregate.t} | 0 8 files changed, 30 insertions(+), 25 deletions(-) rename apisix/{http/aggregate.lua => plugins/api-aggregate.lua} (84%) rename t/{http/aggregate.t => plugin/api-aggregate.t} (100%) diff --git a/apisix/init.lua b/apisix/init.lua index 8b7d2792a244..e73f50137747 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -17,7 +17,6 @@ local require = require local core = require("apisix.core") local plugin = require("apisix.plugin") -local aggregate = require("apisix.http.aggregate").aggregate local service_fetch = require("apisix.http.service").get local admin_init = require("apisix.admin.init") local get_var = require("resty.ngxvar").fetch @@ -631,9 +630,4 @@ function _M.stream_log_phase() run_plugin("log") end -function _M.http_api_aggregate() - local code, body = aggregate() - core.response.exit(code, body) -end - return _M diff --git a/apisix/http/aggregate.lua b/apisix/plugins/api-aggregate.lua similarity index 84% rename from apisix/http/aggregate.lua rename to apisix/plugins/api-aggregate.lua index f5828cfe25bf..11e10b6a8797 100644 --- a/apisix/http/aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -21,8 +21,19 @@ local ipairs = ipairs local pairs = pairs local type = type +local plugin_name = "api-aggregate" + +local schema = { + type = "object", + properties = { + } +} + local _M = { version = 0.1, + priority = 4010, + name = plugin_name, + schema = schema } local function check_input(data) @@ -74,18 +85,18 @@ local function set_base_query(data) end end -function _M.aggregate(service_id) +local function aggregate(service_id) ngx.req.read_body() local req_body = ngx.req.get_body_data() local data, err = core.json.decode(req_body) - local code, body = check_input(data) if not data then core.log.error("invalid request body: ", req_body, " err: ", err) - return 400, {message = "invalid request body", req_body = req_body} + core.response.exit(400, {message = "invalid request body", req_body = req_body}) end + local code, body = check_input(data) if code then - return code, body + core.response.exit(code, body) end local httpc = http.new() @@ -96,7 +107,7 @@ function _M.aggregate(service_id) set_base_query(data) local responses, err = httpc:request_pipeline(data.pipeline) if not responses then - return 400, {message = "request failed", err = err} + core.response.exit(400, {message = "request failed", err = err}) end local aggregated_resp = {} @@ -117,7 +128,17 @@ function _M.aggregate(service_id) end core.table.insert(aggregated_resp, sub_resp) end - return 200, aggregated_resp + core.response.exit(200, aggregated_resp) +end + +function _M.api() + return { + { + methods = {"POST"}, + uri = "/apisix/aggregate", + handler = aggregate, + } + } end return _M diff --git a/bin/apisix b/bin/apisix index 8490bf91a30f..83712c4c9bca 100755 --- a/bin/apisix +++ b/bin/apisix @@ -380,12 +380,6 @@ http { } {% end %} - location /apisix/aggregate { - content_by_lua_block { - apisix.http_api_aggregate() - } - } - ssl_certificate_by_lua_block { apisix.http_ssl_phase() } diff --git a/conf/config.yaml b/conf/config.yaml index 3eccbca27b89..952b60e64629 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -147,5 +147,6 @@ plugins: # plugin list - proxy-mirror - kafka-logger - cors + - api-aggregate stream_plugins: - mqtt-proxy diff --git a/t/APISIX.pm b/t/APISIX.pm index 051e58d1bc95..6b19152469e7 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -297,12 +297,6 @@ _EOC_ } } - location /apisix/aggregate { - content_by_lua_block { - apisix.http_api_aggregate() - } - } - location / { set \$upstream_mirror_host ''; set \$upstream_scheme 'http'; diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 20ee9ed88f7c..8b9fe0dee758 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -30,7 +30,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors"\]/ +qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","api-aggregate"\]/ --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index 0acdb8244347..056ea53912cc 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -57,6 +57,7 @@ qr/loaded plugin and sort by priority: [-\d]+ name: [\w-]+/ --- grep_error_log_out loaded plugin and sort by priority: 11000 name: fault-injection loaded plugin and sort by priority: 10000 name: serverless-pre-function +loaded plugin and sort by priority: 4010 name: api-aggregate loaded plugin and sort by priority: 4000 name: cors loaded plugin and sort by priority: 3000 name: ip-restriction loaded plugin and sort by priority: 2599 name: openid-connect diff --git a/t/http/aggregate.t b/t/plugin/api-aggregate.t similarity index 100% rename from t/http/aggregate.t rename to t/plugin/api-aggregate.t From 8cf00923902afff1eef967eace2313581311af31 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Thu, 2 Apr 2020 17:10:20 +0800 Subject: [PATCH 06/20] fix: keep init.lua format --- apisix/init.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/apisix/init.lua b/apisix/init.lua index e73f50137747..3a081575fc2d 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -630,4 +630,5 @@ function _M.stream_log_phase() run_plugin("log") end + return _M From 2ab4bcfbb935364b7887ce99876db99bea745cd2 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Fri, 3 Apr 2020 09:55:39 +0800 Subject: [PATCH 07/20] fix: change reason when for timeout --- apisix/plugins/api-aggregate.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/api-aggregate.lua index 11e10b6a8797..46ea223e4039 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -115,7 +115,7 @@ local function aggregate(service_id) if not r.status then core.table.insert(aggregated_resp, { status = 504, - reason = "target timeout" + reason = "upstream timeout" }) end local sub_resp = { From 0b827914f1e72f49a066f90f363d73c4c182f74b Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Fri, 3 Apr 2020 15:10:40 +0800 Subject: [PATCH 08/20] fix: add nil check for request body and append invaild body test case --- apisix/plugins/api-aggregate.lua | 11 ++++--- t/plugin/api-aggregate.t | 55 ++++++++++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/api-aggregate.lua index 46ea223e4039..7a4b65f79236 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -25,8 +25,7 @@ local plugin_name = "api-aggregate" local schema = { type = "object", - properties = { - } + additionalProperties = false, } local _M = { @@ -85,13 +84,15 @@ local function set_base_query(data) end end -local function aggregate(service_id) +local function aggregate() ngx.req.read_body() local req_body = ngx.req.get_body_data() + if not req_body then + core.response.exit(400, {message = "no request body, you should give at least one pipeline setting"}) + end local data, err = core.json.decode(req_body) if not data then - core.log.error("invalid request body: ", req_body, " err: ", err) - core.response.exit(400, {message = "invalid request body", req_body = req_body}) + core.response.exit(400, {message = "invalid request body", req_body = req_body, err = err}) end local code, body = check_input(data) diff --git a/t/plugin/api-aggregate.t b/t/plugin/api-aggregate.t index 68a21337059d..4a9753e499dc 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/api-aggregate.t @@ -326,7 +326,7 @@ passed }, { "status": 504, - "reason": "target timeout" + "reason": "upstream timeout" } ]]=] ) @@ -388,7 +388,7 @@ timeout [=[[ { "status": 504, - "reason": "target timeout" + "reason": "upstream timeout" } ]]=] ) @@ -420,3 +420,54 @@ GET /aggregate passed --- error_log timeout + + + +=== TEST 7: no body in request +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + nil, + nil + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"message":"no request body, you should give at least one pipeline setting"} +--- no_error_log +[error] + + + +=== TEST 8: invalid body +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + "invaild json string" + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"Expected value but found invalid token at character 1","req_body":"invaild json string","message":"invalid request body"} +--- no_error_log +[error] From 5964eff6a31103ae10da59738edc064bac2fe8e4 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Fri, 10 Apr 2020 16:30:43 +0800 Subject: [PATCH 09/20] refactor: use default schema check to ensure validation --- apisix/plugins/api-aggregate.lua | 80 ++++++++++++++-- t/plugin/api-aggregate.t | 151 ++++++++++++++++++++++++++++++- 2 files changed, 220 insertions(+), 11 deletions(-) diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/api-aggregate.lua index 7a4b65f79236..11be15672570 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -28,6 +28,66 @@ local schema = { additionalProperties = false, } +local req_schema = { + type = "object", + properties = { + query = { + description = "pipeline query string", + type = "object" + }, + headers = { + description = "pipeline header", + type = "object" + }, + timeout = { + description = "pipeline timeout(ms)", + type = "integer", + default = 30000, + }, + pipeline = { + type = "array", + minItems = 1, + items = { + type = "object", + properties = { + version = { + description = "HTTP version", + type = "number", + enum = {1.0, 1.1}, + default = 1.1, + }, + method = { + description = "HTTP method", + type = "string", + enum = {"GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", + "OPTIONS", "CONNECT", "TRACE"}, + default = "GET" + }, + path = { + type = "string", + minLength = 1, + }, + query = { + description = "request header", + type = "object", + }, + headers = { + description = "request query string", + type = "object", + }, + ssl_verify = { + type = "boolean", + default = false + }, + } + } + } + }, + anyOf = { + {required = {"pipeline"}}, + }, +} + local _M = { version = 0.1, priority = 4010, @@ -35,16 +95,18 @@ local _M = { schema = schema } -local function check_input(data) - if not data.pipeline then - return 400, {message = "missing 'pipeline' in input"} +function _M.check_schema(conf) + local ok, err = core.schema.check(schema, conf) + if not ok then + return false, err end - local type_timeout = type(data.timeout) - if type_timeout ~= "number" and type_timeout ~= "nil" then - return 400, {message = "'timeout' should be number"} - end - if not data.timeout or data.timeout == 0 then - data.timeout = 30000 + return true +end + +local function check_input(data) + local ok, err = core.schema.check(req_schema, data) + if not ok then + return 400, {message = "bad request body", err = err} end end diff --git a/t/plugin/api-aggregate.t b/t/plugin/api-aggregate.t index 4a9753e499dc..ae5e653ec9f5 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/api-aggregate.t @@ -178,7 +178,7 @@ passed GET /aggregate --- error_code: 400 --- response_body -{"message":"missing 'pipeline' in input"} +{"err":"object matches none of the requireds: [\"pipeline\"]","message":"bad request body"} --- no_error_log [error] @@ -218,7 +218,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"message":"'timeout' should be number"} +{"err":"property \"timeout\" validation failed: wrong type: expected integer, got string","message":"bad request body"} --- no_error_log [error] @@ -471,3 +471,150 @@ GET /aggregate {"err":"Expected value but found invalid token at character 1","req_body":"invaild json string","message":"invalid request body"} --- no_error_log [error] + + + +=== TEST 9: invalid pipeline's path +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[ + { + "path": "" + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"path\" validation failed: string too short, expected at least 1, got 0","message":"bad request body"} +--- no_error_log +[error] + + + +=== TEST 10: invalid pipeline's method +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[{ + "path": "/c", + "method": "put" + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"method\" validation failed: matches non of the enum values","message":"bad request body"} +--- no_error_log +[error] + + + +=== TEST 11: invalid pipeline's version +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[{ + "path": "/d", + "version":1.2 + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"version\" validation failed: matches non of the enum values","message":"bad request body"} +--- no_error_log +[error] + + + +=== TEST 12: invalid pipeline's ssl +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[{ + "path": "/d", + "ssl_verify":1.2 + }] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"ssl_verify\" validation failed: wrong type: expected boolean, got number","message":"bad request body"} +--- no_error_log +[error] + + + +=== TEST 13: invalid pipeline's number +--- config + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "pipeline":[] + }]=] + ) + + ngx.status = code + ngx.print(body) + } + } +--- request +GET /aggregate +--- error_code: 400 +--- response_body +{"err":"property \"pipeline\" validation failed: expect array to have at least 1 items","message":"bad request body"} +--- no_error_log +[error] \ No newline at end of file From 29c9b210650545d94e73e4f7b20d982146b5957a Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Fri, 10 Apr 2020 16:31:30 +0800 Subject: [PATCH 10/20] fix: lint warnings --- apisix/plugins/api-aggregate.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/api-aggregate.lua index 11be15672570..1eee47380984 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -19,7 +19,6 @@ local http = require("resty.http") local ngx = ngx local ipairs = ipairs local pairs = pairs -local type = type local plugin_name = "api-aggregate" From 3549860393c2a4d64ce328b57c6ef9ecffc66289 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Mon, 13 Apr 2020 16:57:21 +0800 Subject: [PATCH 11/20] doc: append aggregate doc --- README.md | 1 + README_CN.md | 1 + doc/README.md | 1 + doc/README_CN.md | 1 + doc/plugins/api-aggregate-cn.md | 135 ++++++++++++++++++++++++++++++++ doc/plugins/api-aggregate.md | 135 ++++++++++++++++++++++++++++++++ 6 files changed, 274 insertions(+) create mode 100644 doc/plugins/api-aggregate-cn.md create mode 100644 doc/plugins/api-aggregate.md diff --git a/README.md b/README.md index 1d6bc1d13070..03b1f1bb885c 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ A/B testing, canary release, blue-green deployment, limit rate, defense against - IPv6: Use IPv6 to match route. - Support [TTL](doc/admin-api-cn.md#route) - [Support priority](doc/router-radixtree.md#3-match-priority) + - [Support Http API-Aggregation](doc/plugins/api-aggregate.md) - **Security** - Authentications: [key-auth](doc/plugins/key-auth.md), [JWT](doc/plugins/jwt-auth.md), [basic-auth](doc/plugins/basic-auth.md), [wolf-rbac](doc/plugins/wolf-rbac.md) diff --git a/README_CN.md b/README_CN.md index 1296f26a0159..e31ddcf0417d 100644 --- a/README_CN.md +++ b/README_CN.md @@ -81,6 +81,7 @@ A/B 测试、金丝雀发布(灰度发布)、蓝绿部署、限流限速、抵 - IPv6:支持使用 IPv6 格式匹配路由 - 支持路由的[自动过期(TTL)](doc/admin-api-cn.md#route) - [支持路由的优先级](doc/router-radixtree.md#3-match-priority) + - [支持 Http 接口聚合](doc/plugins/api-aggregate-cn.md) - **安全防护** - 多种身份认证方式: [key-auth](doc/plugins/key-auth-cn.md), [JWT](doc/plugins/jwt-auth-cn.md), [basic-auth](doc/plugins/basic-auth-cn.md), [wolf-rbac](doc/plugins/wolf-rbac-cn.md)。 diff --git a/doc/README.md b/doc/README.md index 40da08e2423a..5c24dafa654d 100644 --- a/doc/README.md +++ b/doc/README.md @@ -64,6 +64,7 @@ Plugins * [proxy-mirror](plugins/proxy-mirror.md): Provides the ability to mirror client requests. * [kafka-logger](plugins/kafka-logger.md): Log requests to External Kafka servers. * [cors](plugins/cors.md): Enbale cors for you api. +* [api-aggregate](plugins/api-aggregate.md): Allow you aggregate http api via http pipeline. Deploy to the Cloud ======= diff --git a/doc/README_CN.md b/doc/README_CN.md index bf3141400251..6542edff507b 100644 --- a/doc/README_CN.md +++ b/doc/README_CN.md @@ -65,3 +65,4 @@ Reference document * [tcp-logger](plugins/tcp-logger.md): 将请求记录到TCP服务器 * [kafka-logger](plugins/kafka-logger-cn.md): 将请求记录到外部Kafka服务器。 * [cors](plugins/cors-cn.md): 为你的API启用CORS. +* [api-aggregate](plugins/api-aggregate-cn.md): 以 **http pipeline** 的方式聚合 `http` 请求 diff --git a/doc/plugins/api-aggregate-cn.md b/doc/plugins/api-aggregate-cn.md new file mode 100644 index 000000000000..c08dd87fc63f --- /dev/null +++ b/doc/plugins/api-aggregate-cn.md @@ -0,0 +1,135 @@ + + +# [English](api-aggregate.md) + +# 目录 + +- [**简介**](#简介) +- [**属性**](#属性) +- [**如何启用**](#如何启用) +- [**聚合接口请求/响应**](#聚合接口请求/响应) +- [**测试插件**](#测试插件) +- [**禁用插件**](#禁用插件) + +## 简介 + +`api-aggregate` 插件可以让你在以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式在网关聚合多个http请求。 + +## 属性 + +无 + +## 如何启用 + +本插件默认启用。 + +## 聚合接口请求/响应 +插件会为 `apisix` 创建一个 `/apisix/aggregate` 的聚合接口来处理你的聚合请求。 + +### 接口请求参数: + +| 参数名 | 类型 | 可选 | 默认值 | 描述 | +| --- | --- | --- | --- | --- | +| query | Object | Yes | | 给所有请求都携带的 `QueryString` | +| headers | Object | Yes | | 给所有请求都携带的 `Header` | +| timeout | Number | Yes | 3000 | 聚合请求的超时时间,单位为 `ms` | +| pipeline | [HttpRequest](#Request) | No | | Http 请求的详细信息 | + +#### HttpRequest +| 参数名 | 类型 | 可选 | 默认值 | 描述 | +| --- | --- | --- | --- | --- | +| version | Enum | Yes | 1.1 | 请求用的 `http` 协议版本,可以使用 `1.0` or `1.1` | +| method | Enum | Yes | GET | 请求使用的 `http` 方法,例如:`GET`. | +| query | Object | Yes | | 独立请求所携带的 `QueryString`, 如果 `Key` 和全局的有冲突,以此设置为主。 | +| headers | Object | Yes | | 独立请求所携带的 `Header`, 如果 `Key` 和全局的有冲突,以此设置为主。 | +| path | String | No | | 请求路径 | +| body | String | Yes | | 请求体 | + +### 接口响应参数: +返回值为一个 [HttpResponse](#HttpResponse) 的 `数组`。 + +#### HttpResponse +| 参数名 | 类型 | 描述 | +| --- | --- | --- | --- | --- | +| status | Integer | Http 请求的状态码 | +| reason | String | Http 请求的返回信息 | +| body | String | Http 请求的响应体 | +| headers | Object | Http 请求的响应头 | + +## 测试插件 + +你可以将要访问的请求信息传到网关的聚合接口( `/apisix/aggregate` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 +```shell +curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ +--header 'Content-Type: application/json' \ +--d '{ + "headers": { + "Content-Type": "application/json", + "admin-jwt":"xxxx" + }, + "timeout": 500, + "pipeline": [ + { + "method": "POST", + "path": "/community.GiftSrv/GetGifts", + "body": "test" + }, + { + "method": "POST", + "path": "/community.GiftSrv/GetGifts", + "body": "test2" + } + ] +}' +``` + +返回如下: +```json +[ + { + "status": 200, + "reason": "OK", + "body": "{\"ret\":500,\"msg\":\"error\",\"game_info\":null,\"gift\":[],\"to_gets\":0,\"get_all_msg\":\"\"}", + "headers": { + "Connection": "keep-alive", + "Date": "Sat, 11 Apr 2020 17:53:20 GMT", + "Content-Type": "application/json", + "Content-Length": "81", + "Server": "APISIX web server" + } + }, + { + "status": 200, + "reason": "OK", + "body": "{\"ret\":500,\"msg\":\"error\",\"game_info\":null,\"gift\":[],\"to_gets\":0,\"get_all_msg\":\"\"}", + "headers": { + "Connection": "keep-alive", + "Date": "Sat, 11 Apr 2020 17:53:20 GMT", + "Content-Type": "application/json", + "Content-Length": "81", + "Server": "APISIX web server" + } + } +] +``` + +## 禁用插件 + +正常来说你不需要禁用本插件,如果有特殊情况,请从 `/conf/config.yaml` 的 `plugins` 节点中移除即可。 diff --git a/doc/plugins/api-aggregate.md b/doc/plugins/api-aggregate.md new file mode 100644 index 000000000000..ef9c738438bf --- /dev/null +++ b/doc/plugins/api-aggregate.md @@ -0,0 +1,135 @@ + + +# [Chinese](api-aggregate-cn.md) + +# Summary + +- [**Description**](#Description) +- [**Attributes**](#Attributes) +- [**How To Enable**](#how-to-Enable) +- [**Aggregation Api Request/Response**](#aggregation-api-request/response) +- [**Test Plugin**](#test-plugin) +- [**Disable Plugin**](#disable-plugin) + +## Description + +`api-aggregate` can let you aggregate multiple request on `apisix` via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). + +## Attributes + +None + +## How To Enable + +Default enbaled + +## Aggregation Api Request/Response +The plugin will create a api in `apisix` to handle your aggregation request. + +### Aggregate Api Request: + +| ParameterName | Type | Optional | Default | Description | +| --- | --- | --- | --- | --- | +| query | Object | Yes | | Specify `QueryString` for all request | +| headers | Object | Yes | | Specify `Header` for all request | +| timeout | Number | Yes | 3000 | Aggregate Api timeout in `ms` | +| pipeline | [HttpRequest](#Request) | No | | Request's detail | + +#### HttpRequest +| ParameterName | Type | Optional | Default | Description | +| --- | --- | --- | --- | --- | +| version | Enum | Yes | 1.1 | http version: `1.0` or `1.1` | +| method | Enum | Yes | GET | http method, such as:`GET`. | +| query | Object | Yes | | request's `QueryString`, if `Key` is conflicted with global `query`, this setting's value will be setted.| +| headers | Object | Yes | | request's `Header`, if `Key` is conflicted with global `headers`, this setting's value will be setted.| +| path | String | No | | http request's path | +| body | String | Yes | | http request's body | + +### Aggregate Api Response: +Response is `Array` of [HttpResponse](#HttpResponse). + +#### HttpResponse +| ParameterName | Type | Description | +| --- | --- | --- | --- | --- | +| status | Integer | http status code | +| reason | String | http reason phrase | +| body | String | http response body | +| headers | Object | http response headers | + +## Test Plugin + +You can pass your request detail to aggregation api( `/apisix/aggregate` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: +```shell +curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ +--header 'Content-Type: application/json' \ +--d '{ + "headers": { + "Content-Type": "application/json", + "admin-jwt":"xxxx" + }, + "timeout": 500, + "pipeline": [ + { + "method": "POST", + "path": "/community.GiftSrv/GetGifts", + "body": "test" + }, + { + "method": "POST", + "path": "/community.GiftSrv/GetGifts", + "body": "test2" + } + ] +}' +``` + +返回如下: +```json +[ + { + "status": 200, + "reason": "OK", + "body": "{\"ret\":500,\"msg\":\"error\",\"game_info\":null,\"gift\":[],\"to_gets\":0,\"get_all_msg\":\"\"}", + "headers": { + "Connection": "keep-alive", + "Date": "Sat, 11 Apr 2020 17:53:20 GMT", + "Content-Type": "application/json", + "Content-Length": "81", + "Server": "APISIX web server" + } + }, + { + "status": 200, + "reason": "OK", + "body": "{\"ret\":500,\"msg\":\"error\",\"game_info\":null,\"gift\":[],\"to_gets\":0,\"get_all_msg\":\"\"}", + "headers": { + "Connection": "keep-alive", + "Date": "Sat, 11 Apr 2020 17:53:20 GMT", + "Content-Type": "application/json", + "Content-Length": "81", + "Server": "APISIX web server" + } + } +] +``` + +## Disable Plugin + +Normally, you don't need to disable this plugin.If you does need please remove it from the `plugins` section of`/conf/config.yaml`. From 3acd1b5c866a63557cc3cb7f6e1c629ede0a32c5 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Mon, 13 Apr 2020 17:01:13 +0800 Subject: [PATCH 12/20] fix: lint error --- t/plugin/api-aggregate.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/plugin/api-aggregate.t b/t/plugin/api-aggregate.t index ae5e653ec9f5..5cf8a4e0668b 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/api-aggregate.t @@ -617,4 +617,4 @@ GET /aggregate --- response_body {"err":"property \"pipeline\" validation failed: expect array to have at least 1 items","message":"bad request body"} --- no_error_log -[error] \ No newline at end of file +[error] From ef3b0e3f9833ba501d4ab2138c322b2591f4e6d9 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Mon, 13 Apr 2020 20:42:46 +0800 Subject: [PATCH 13/20] trigger travis --- t/plugin/api-aggregate.t | 1 - 1 file changed, 1 deletion(-) diff --git a/t/plugin/api-aggregate.t b/t/plugin/api-aggregate.t index 5cf8a4e0668b..82982bdb8910 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/api-aggregate.t @@ -606,7 +606,6 @@ GET /aggregate "pipeline":[] }]=] ) - ngx.status = code ngx.print(body) } From b1ec1386d485a13772a6b19985dba43bd716de12 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Mon, 13 Apr 2020 20:46:41 +0800 Subject: [PATCH 14/20] fix: remove unnecessary line --- doc/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/README.md b/doc/README.md index 9db43f15c86c..d77881ccfc56 100644 --- a/doc/README.md +++ b/doc/README.md @@ -65,7 +65,6 @@ Plugins * [kafka-logger](plugins/kafka-logger.md): Log requests to External Kafka servers. * [cors](plugins/cors.md): Enable CORS(Cross-origin resource sharing) for your API. * [api-aggregate](plugins/api-aggregate.md): Allow you aggregate http api via http pipeline. -* [cors](plugins/cors.md): Enable CORS(Cross-origin resource sharing) for your API. Deploy to the Cloud ======= From a498091a3ed8d5e394ed54995bac2c0b9d8d3362 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Mon, 13 Apr 2020 22:11:44 +0800 Subject: [PATCH 15/20] fix: lint error --- doc/plugins/api-aggregate-cn.md | 10 +++++----- doc/plugins/api-aggregate.md | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/doc/plugins/api-aggregate-cn.md b/doc/plugins/api-aggregate-cn.md index c08dd87fc63f..14f3a07983e9 100644 --- a/doc/plugins/api-aggregate-cn.md +++ b/doc/plugins/api-aggregate-cn.md @@ -80,11 +80,11 @@ curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ --header 'Content-Type: application/json' \ --d '{ - "headers": { - "Content-Type": "application/json", - "admin-jwt":"xxxx" - }, - "timeout": 500, + "headers": { + "Content-Type": "application/json", + "admin-jwt":"xxxx" + }, + "timeout": 500, "pipeline": [ { "method": "POST", diff --git a/doc/plugins/api-aggregate.md b/doc/plugins/api-aggregate.md index ef9c738438bf..acb31a79408e 100644 --- a/doc/plugins/api-aggregate.md +++ b/doc/plugins/api-aggregate.md @@ -75,16 +75,16 @@ Response is `Array` of [HttpResponse](#HttpResponse). ## Test Plugin -You can pass your request detail to aggregation api( `/apisix/aggregate` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: +You can pass your request detail to aggregation api( `/apisix/aggregate` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: ```shell curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ --header 'Content-Type: application/json' \ --d '{ - "headers": { - "Content-Type": "application/json", - "admin-jwt":"xxxx" - }, - "timeout": 500, + "headers": { + "Content-Type": "application/json", + "admin-jwt":"xxxx" + }, + "timeout": 500, "pipeline": [ { "method": "POST", From 95ffdd8b6941ce59d2be0b6c87fb6106247d5f9d Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Wed, 15 Apr 2020 19:26:19 +0800 Subject: [PATCH 16/20] fix: code review --- apisix/plugins/api-aggregate.lua | 39 +++++++++++++++---- doc/plugins/api-aggregate-cn.md | 6 +-- doc/plugins/api-aggregate.md | 6 +-- t/plugin/api-aggregate.t | 67 ++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 13 deletions(-) diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/api-aggregate.lua index 1eee47380984..779cc5055658 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/api-aggregate.lua @@ -14,11 +14,12 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local core = require("apisix.core") -local http = require("resty.http") -local ngx = ngx -local ipairs = ipairs -local pairs = pairs +local core = require("apisix.core") +local http = require("resty.http") +local ngx = ngx +local io_open = io.open +local ipairs = ipairs +local pairs = pairs local plugin_name = "api-aggregate" @@ -94,6 +95,7 @@ local _M = { schema = schema } + function _M.check_schema(conf) local ok, err = core.schema.check(schema, conf) if not ok then @@ -102,6 +104,7 @@ function _M.check_schema(conf) return true end + local function check_input(data) local ok, err = core.schema.check(req_schema, data) if not ok then @@ -109,6 +112,7 @@ local function check_input(data) end end + local function set_base_header(data) if not data.headers then return @@ -127,6 +131,7 @@ local function set_base_header(data) end end + local function set_base_query(data) if not data.query then return @@ -145,12 +150,31 @@ local function set_base_query(data) end end + +local function getFile(file_name) + local f = io_open(file_name, 'r') + if f then + return f:read("*all") + end + + return +end + + local function aggregate() ngx.req.read_body() local req_body = ngx.req.get_body_data() if not req_body then - core.response.exit(400, {message = "no request body, you should give at least one pipeline setting"}) + local file_name = ngx.req.get_body_file() + if file_name then + req_body = getFile(file_name) + end + + if not req_body then + core.response.exit(400, {message = "no request body, you should give at least one pipeline setting"}) + end end + local data, err = core.json.decode(req_body) if not data then core.response.exit(400, {message = "invalid request body", req_body = req_body, err = err}) @@ -162,7 +186,6 @@ local function aggregate() end local httpc = http.new() - core.log.info(data.timeout) httpc:set_timeout(data.timeout) httpc:connect("127.0.0.1", ngx.var.server_port) set_base_header(data) @@ -193,6 +216,7 @@ local function aggregate() core.response.exit(200, aggregated_resp) end + function _M.api() return { { @@ -203,4 +227,5 @@ function _M.api() } end + return _M diff --git a/doc/plugins/api-aggregate-cn.md b/doc/plugins/api-aggregate-cn.md index 14f3a07983e9..b2e03a663ad6 100644 --- a/doc/plugins/api-aggregate-cn.md +++ b/doc/plugins/api-aggregate-cn.md @@ -77,7 +77,7 @@ 你可以将要访问的请求信息传到网关的聚合接口( `/apisix/aggregate` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 ```shell -curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/aggregate' \ --header 'Content-Type: application/json' \ --d '{ "headers": { @@ -87,12 +87,12 @@ curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ "timeout": 500, "pipeline": [ { - "method": "POST", + "method": "POST", "path": "/community.GiftSrv/GetGifts", "body": "test" }, { - "method": "POST", + "method": "POST", "path": "/community.GiftSrv/GetGifts", "body": "test2" } diff --git a/doc/plugins/api-aggregate.md b/doc/plugins/api-aggregate.md index acb31a79408e..d8ef90e98185 100644 --- a/doc/plugins/api-aggregate.md +++ b/doc/plugins/api-aggregate.md @@ -77,7 +77,7 @@ Response is `Array` of [HttpResponse](#HttpResponse). You can pass your request detail to aggregation api( `/apisix/aggregate` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: ```shell -curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/aggregate' \ --header 'Content-Type: application/json' \ --d '{ "headers": { @@ -87,12 +87,12 @@ curl --location --request POST 'http://100.109.220.139/apisix/aggregate' \ "timeout": 500, "pipeline": [ { - "method": "POST", + "method": "POST", "path": "/community.GiftSrv/GetGifts", "body": "test" }, { - "method": "POST", + "method": "POST", "path": "/community.GiftSrv/GetGifts", "body": "test2" } diff --git a/t/plugin/api-aggregate.t b/t/plugin/api-aggregate.t index 82982bdb8910..1fb3622724f4 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/api-aggregate.t @@ -617,3 +617,70 @@ GET /aggregate {"err":"property \"pipeline\" validation failed: expect array to have at least 1 items","message":"bad request body"} --- no_error_log [error] + + + +=== TEST 14: when client body has been wrote to temp file +--- config + client_body_in_file_only on; + location = /aggregate { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + local code, body = t('/apisix/aggregate', + ngx.HTTP_POST, + [=[{ + "timeout": 100, + "pipeline":[ + { + "path": "/b", + "headers": { + "Header1": "hello", + "Header2": "world" + } + },{ + "path": "/c", + "method": "PUT" + },{ + "path": "/d" + }] + }]=], + [=[[ + { + "status": 200 + }, + { + "status": 201 + }, + { + "status": 202 + } + ]]=] + ) + + ngx.status = code + ngx.say(body) + } + } + + location = /b { + content_by_lua_block { + ngx.status = 200 + } + } + location = /c { + content_by_lua_block { + ngx.status = 201 + } + } + location = /d { + content_by_lua_block { + ngx.status = 202 + } + } +--- request +GET /aggregate +--- response_body +passed +--- no_error_log +[error] From dbb76251245e8a7c5aff4adafbe006c360f1d710 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Thu, 23 Apr 2020 18:33:00 +0800 Subject: [PATCH 17/20] refactor: rename plugin to batch-requets --- README.md | 2 +- README_CN.md | 2 +- .../{api-aggregate.lua => batch-requests.lua} | 4 +-- conf/config.yaml | 2 +- doc/README.md | 2 +- doc/README_CN.md | 2 +- ...i-aggregate-cn.md => batch-requests-cn.md} | 14 +++++----- .../{api-aggregate.md => batch-requests.md} | 18 ++++++------ t/admin/plugins.t | 2 +- t/debug/debug-mode.t | 2 +- .../{api-aggregate.t => batch-requests.t} | 28 +++++++++---------- 11 files changed, 39 insertions(+), 39 deletions(-) rename apisix/plugins/{api-aggregate.lua => batch-requests.lua} (98%) rename doc/plugins/{api-aggregate-cn.md => batch-requests-cn.md} (83%) rename doc/plugins/{api-aggregate.md => batch-requests.md} (84%) rename t/plugin/{api-aggregate.t => batch-requests.t} (95%) diff --git a/README.md b/README.md index 4298605b4f6f..e2054d9be18a 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ A/B testing, canary release, blue-green deployment, limit rate, defense against - IPv6: Use IPv6 to match route. - Support [TTL](doc/admin-api-cn.md#route) - [Support priority](doc/router-radixtree.md#3-match-priority) - - [Support Http API-Aggregation](doc/plugins/api-aggregate.md) + - [Support Batch Http Requests](doc/plugins/batch-requests.md) - **Security** - Authentications: [key-auth](doc/plugins/key-auth.md), [JWT](doc/plugins/jwt-auth.md), [basic-auth](doc/plugins/basic-auth.md), [wolf-rbac](doc/plugins/wolf-rbac.md) diff --git a/README_CN.md b/README_CN.md index 752ef8fd1e73..f7fac00fcfeb 100644 --- a/README_CN.md +++ b/README_CN.md @@ -81,7 +81,7 @@ A/B 测试、金丝雀发布(灰度发布)、蓝绿部署、限流限速、抵 - IPv6:支持使用 IPv6 格式匹配路由 - 支持路由的[自动过期(TTL)](doc/admin-api-cn.md#route) - [支持路由的优先级](doc/router-radixtree.md#3-match-priority) - - [支持 Http 接口聚合](doc/plugins/api-aggregate-cn.md) + - [支持批量 Http 请求](doc/plugins/batch-requests-cn.md) - **安全防护** - 多种身份认证方式: [key-auth](doc/plugins/key-auth-cn.md), [JWT](doc/plugins/jwt-auth-cn.md), [basic-auth](doc/plugins/basic-auth-cn.md), [wolf-rbac](doc/plugins/wolf-rbac-cn.md)。 diff --git a/apisix/plugins/api-aggregate.lua b/apisix/plugins/batch-requests.lua similarity index 98% rename from apisix/plugins/api-aggregate.lua rename to apisix/plugins/batch-requests.lua index 779cc5055658..f183f88693b2 100644 --- a/apisix/plugins/api-aggregate.lua +++ b/apisix/plugins/batch-requests.lua @@ -21,7 +21,7 @@ local io_open = io.open local ipairs = ipairs local pairs = pairs -local plugin_name = "api-aggregate" +local plugin_name = "batch-requests" local schema = { type = "object", @@ -221,7 +221,7 @@ function _M.api() return { { methods = {"POST"}, - uri = "/apisix/aggregate", + uri = "/apisix/batch", handler = aggregate, } } diff --git a/conf/config.yaml b/conf/config.yaml index f949781a9c37..04402fe5024a 100644 --- a/conf/config.yaml +++ b/conf/config.yaml @@ -145,6 +145,6 @@ plugins: # plugin list - proxy-mirror - kafka-logger - cors - - api-aggregate + - batch-requests stream_plugins: - mqtt-proxy diff --git a/doc/README.md b/doc/README.md index d77881ccfc56..238e1983cb1f 100644 --- a/doc/README.md +++ b/doc/README.md @@ -64,7 +64,7 @@ Plugins * [proxy-mirror](plugins/proxy-mirror.md): Provides the ability to mirror client requests. * [kafka-logger](plugins/kafka-logger.md): Log requests to External Kafka servers. * [cors](plugins/cors.md): Enable CORS(Cross-origin resource sharing) for your API. -* [api-aggregate](plugins/api-aggregate.md): Allow you aggregate http api via http pipeline. +* [batch-requests](plugins/batch-requests.md): Allow you send mutiple http api via **http pipeline**. Deploy to the Cloud ======= diff --git a/doc/README_CN.md b/doc/README_CN.md index 6542edff507b..1fc08c5abccd 100644 --- a/doc/README_CN.md +++ b/doc/README_CN.md @@ -65,4 +65,4 @@ Reference document * [tcp-logger](plugins/tcp-logger.md): 将请求记录到TCP服务器 * [kafka-logger](plugins/kafka-logger-cn.md): 将请求记录到外部Kafka服务器。 * [cors](plugins/cors-cn.md): 为你的API启用CORS. -* [api-aggregate](plugins/api-aggregate-cn.md): 以 **http pipeline** 的方式聚合 `http` 请求 +* [batch-requests](plugins/batch-requests-cn.md): 以 **http pipeline** 的方式在网关一次性发起多个 `http` 请求。 diff --git a/doc/plugins/api-aggregate-cn.md b/doc/plugins/batch-requests-cn.md similarity index 83% rename from doc/plugins/api-aggregate-cn.md rename to doc/plugins/batch-requests-cn.md index b2e03a663ad6..7deeab8c2ce6 100644 --- a/doc/plugins/api-aggregate-cn.md +++ b/doc/plugins/batch-requests-cn.md @@ -17,20 +17,20 @@ # --> -# [English](api-aggregate.md) +# [English](batch-requests.md) # 目录 - [**简介**](#简介) - [**属性**](#属性) - [**如何启用**](#如何启用) -- [**聚合接口请求/响应**](#聚合接口请求/响应) +- [**批量接口请求/响应**](#批量接口请求/响应) - [**测试插件**](#测试插件) - [**禁用插件**](#禁用插件) ## 简介 -`api-aggregate` 插件可以让你在以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式在网关聚合多个http请求。 +`batch-requests` 插件可以一次接受多个请求并以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式在网关发起多个http请求,合并结果后再返回客户端,这在客户端需要访问多个接口时可以显著地提升请求性能。 ## 属性 @@ -40,8 +40,8 @@ 本插件默认启用。 -## 聚合接口请求/响应 -插件会为 `apisix` 创建一个 `/apisix/aggregate` 的聚合接口来处理你的聚合请求。 +## 批量接口请求/响应 +插件会为 `apisix` 创建一个 `/apisix/batch` 的接口来处理你的批量请求。 ### 接口请求参数: @@ -75,9 +75,9 @@ ## 测试插件 -你可以将要访问的请求信息传到网关的聚合接口( `/apisix/aggregate` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 +你可以将要访问的请求信息传到网关的批量请求接口( `/apisix/batch` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 ```shell -curl --location --request POST 'http://127.0.0.1:9080/apisix/aggregate' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/batch' \ --header 'Content-Type: application/json' \ --d '{ "headers": { diff --git a/doc/plugins/api-aggregate.md b/doc/plugins/batch-requests.md similarity index 84% rename from doc/plugins/api-aggregate.md rename to doc/plugins/batch-requests.md index d8ef90e98185..80e872070f75 100644 --- a/doc/plugins/api-aggregate.md +++ b/doc/plugins/batch-requests.md @@ -17,20 +17,20 @@ # --> -# [Chinese](api-aggregate-cn.md) +# [Chinese](batch-requests-cn.md) # Summary - [**Description**](#Description) - [**Attributes**](#Attributes) - [**How To Enable**](#how-to-Enable) -- [**Aggregation Api Request/Response**](#aggregation-api-request/response) +- [**Batch Api Request/Response**](#batch-api-request/response) - [**Test Plugin**](#test-plugin) - [**Disable Plugin**](#disable-plugin) ## Description -`api-aggregate` can let you aggregate multiple request on `apisix` via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). +`batch-requests` can accept mutiple request and send them from `apisix` via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining),and return a aggregated response to client,this can significantly improve performance when the client needs to access multiple APIs. ## Attributes @@ -40,10 +40,10 @@ None Default enbaled -## Aggregation Api Request/Response +## Batch Api Request/Response The plugin will create a api in `apisix` to handle your aggregation request. -### Aggregate Api Request: +### Batch Api Request: | ParameterName | Type | Optional | Default | Description | | --- | --- | --- | --- | --- | @@ -62,7 +62,7 @@ The plugin will create a api in `apisix` to handle your aggregation request. | path | String | No | | http request's path | | body | String | Yes | | http request's body | -### Aggregate Api Response: +### Batch Api Response: Response is `Array` of [HttpResponse](#HttpResponse). #### HttpResponse @@ -75,9 +75,9 @@ Response is `Array` of [HttpResponse](#HttpResponse). ## Test Plugin -You can pass your request detail to aggregation api( `/apisix/aggregate` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: +You can pass your request detail to batch api( `/apisix/batch` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: ```shell -curl --location --request POST 'http://127.0.0.1:9080/apisix/aggregate' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/batch' \ --header 'Content-Type: application/json' \ --d '{ "headers": { @@ -100,7 +100,7 @@ curl --location --request POST 'http://127.0.0.1:9080/apisix/aggregate' \ }' ``` -返回如下: +response as below: ```json [ { diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 8b9fe0dee758..11939872ff4c 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -30,7 +30,7 @@ __DATA__ --- request GET /apisix/admin/plugins/list --- response_body_like eval -qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","api-aggregate"\]/ +qr/\["limit-req","limit-count","limit-conn","key-auth","basic-auth","prometheus","node-status","jwt-auth","zipkin","ip-restriction","grpc-transcode","serverless-pre-function","serverless-post-function","openid-connect","proxy-rewrite","redirect","response-rewrite","fault-injection","udp-logger","wolf-rbac","proxy-cache","tcp-logger","proxy-mirror","kafka-logger","cors","batch-requests"\]/ --- no_error_log [error] diff --git a/t/debug/debug-mode.t b/t/debug/debug-mode.t index 056ea53912cc..dcd66e50d4ad 100644 --- a/t/debug/debug-mode.t +++ b/t/debug/debug-mode.t @@ -57,7 +57,7 @@ qr/loaded plugin and sort by priority: [-\d]+ name: [\w-]+/ --- grep_error_log_out loaded plugin and sort by priority: 11000 name: fault-injection loaded plugin and sort by priority: 10000 name: serverless-pre-function -loaded plugin and sort by priority: 4010 name: api-aggregate +loaded plugin and sort by priority: 4010 name: batch-requests loaded plugin and sort by priority: 4000 name: cors loaded plugin and sort by priority: 3000 name: ip-restriction loaded plugin and sort by priority: 2599 name: openid-connect diff --git a/t/plugin/api-aggregate.t b/t/plugin/batch-requests.t similarity index 95% rename from t/plugin/api-aggregate.t rename to t/plugin/batch-requests.t index 1fb3622724f4..13e3c49c958b 100644 --- a/t/plugin/api-aggregate.t +++ b/t/plugin/batch-requests.t @@ -31,7 +31,7 @@ __DATA__ content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "query": { @@ -151,7 +151,7 @@ passed content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline1":[ @@ -190,7 +190,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "timeout": "200", @@ -230,7 +230,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "timeout": 2000, @@ -299,7 +299,7 @@ passed content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "timeout": 100, @@ -367,7 +367,7 @@ timeout content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "timeout": 100, @@ -429,7 +429,7 @@ timeout content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, nil, nil @@ -455,7 +455,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, "invaild json string" ) @@ -480,7 +480,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline":[ @@ -510,7 +510,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -540,7 +540,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -570,7 +570,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -600,7 +600,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "pipeline":[] @@ -627,7 +627,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/aggregate', + local code, body = t('/apisix/batch', ngx.HTTP_POST, [=[{ "timeout": 100, From bc6be4883a7c07151a37e64820feb9bf51082192 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Tue, 28 Apr 2020 18:11:40 +0800 Subject: [PATCH 18/20] refactor: fix as reviews --- apisix/plugins/batch-requests.lua | 22 +++++++++------- doc/plugins/batch-requests-cn.md | 6 ++--- doc/plugins/batch-requests.md | 4 +-- t/plugin/batch-requests.t | 42 +++++++++++++++---------------- 4 files changed, 39 insertions(+), 35 deletions(-) diff --git a/apisix/plugins/batch-requests.lua b/apisix/plugins/batch-requests.lua index f183f88693b2..33bc68c4cfd6 100644 --- a/apisix/plugins/batch-requests.lua +++ b/apisix/plugins/batch-requests.lua @@ -108,12 +108,12 @@ end local function check_input(data) local ok, err = core.schema.check(req_schema, data) if not ok then - return 400, {message = "bad request body", err = err} + return 400, {error_msg = "bad request body: " .. err} end end -local function set_base_header(data) +local function set_common_header(data) if not data.headers then return end @@ -132,7 +132,7 @@ local function set_base_header(data) end -local function set_base_query(data) +local function set_common_query(data) if not data.query then return end @@ -161,7 +161,7 @@ local function getFile(file_name) end -local function aggregate() +local function batch_requests() ngx.req.read_body() local req_body = ngx.req.get_body_data() if not req_body then @@ -187,9 +187,13 @@ local function aggregate() local httpc = http.new() httpc:set_timeout(data.timeout) - httpc:connect("127.0.0.1", ngx.var.server_port) - set_base_header(data) - set_base_query(data) + local ok, err = httpc:connect("127.0.0.1", ngx.var.server_port) + if not ok then + core.response.exit(500, {message = "connect to apisix failed", err = err}) + end + + set_common_header(data) + set_common_query(data) local responses, err = httpc:request_pipeline(data.pipeline) if not responses then core.response.exit(400, {message = "request failed", err = err}) @@ -221,8 +225,8 @@ function _M.api() return { { methods = {"POST"}, - uri = "/apisix/batch", - handler = aggregate, + uri = "/apisix/batch-requests", + handler = batch_requests, } } end diff --git a/doc/plugins/batch-requests-cn.md b/doc/plugins/batch-requests-cn.md index 7deeab8c2ce6..dc06e862bdef 100644 --- a/doc/plugins/batch-requests-cn.md +++ b/doc/plugins/batch-requests-cn.md @@ -41,7 +41,7 @@ 本插件默认启用。 ## 批量接口请求/响应 -插件会为 `apisix` 创建一个 `/apisix/batch` 的接口来处理你的批量请求。 +插件会为 `apisix` 创建一个 `/apisix/batch-requests` 的接口来处理你的批量请求。 ### 接口请求参数: @@ -75,9 +75,9 @@ ## 测试插件 -你可以将要访问的请求信息传到网关的批量请求接口( `/apisix/batch` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 +你可以将要访问的请求信息传到网关的批量请求接口( `/apisix/batch-requests` ),网关会以 [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining) 的方式自动帮你完成请求。 ```shell -curl --location --request POST 'http://127.0.0.1:9080/apisix/batch' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/batch-requests' \ --header 'Content-Type: application/json' \ --d '{ "headers": { diff --git a/doc/plugins/batch-requests.md b/doc/plugins/batch-requests.md index 80e872070f75..081c5904ddac 100644 --- a/doc/plugins/batch-requests.md +++ b/doc/plugins/batch-requests.md @@ -75,9 +75,9 @@ Response is `Array` of [HttpResponse](#HttpResponse). ## Test Plugin -You can pass your request detail to batch api( `/apisix/batch` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: +You can pass your request detail to batch api( `/apisix/batch-requests` ), `apisix` can automatically complete requests via [http pipeline](https://en.wikipedia.org/wiki/HTTP_pipelining). Such as: ```shell -curl --location --request POST 'http://127.0.0.1:9080/apisix/batch' \ +curl --location --request POST 'http://127.0.0.1:9080/apisix/batch-requests' \ --header 'Content-Type: application/json' \ --d '{ "headers": { diff --git a/t/plugin/batch-requests.t b/t/plugin/batch-requests.t index 13e3c49c958b..dde064ef9618 100644 --- a/t/plugin/batch-requests.t +++ b/t/plugin/batch-requests.t @@ -31,7 +31,7 @@ __DATA__ content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "query": { @@ -151,7 +151,7 @@ passed content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline1":[ @@ -178,7 +178,7 @@ passed GET /aggregate --- error_code: 400 --- response_body -{"err":"object matches none of the requireds: [\"pipeline\"]","message":"bad request body"} +{"error_msg":"bad request body: object matches none of the requireds: [\"pipeline\"]"} --- no_error_log [error] @@ -190,7 +190,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "timeout": "200", @@ -218,7 +218,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"timeout\" validation failed: wrong type: expected integer, got string","message":"bad request body"} +{"error_msg":"bad request body: property \"timeout\" validation failed: wrong type: expected integer, got string"} --- no_error_log [error] @@ -230,7 +230,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "timeout": 2000, @@ -299,7 +299,7 @@ passed content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "timeout": 100, @@ -367,7 +367,7 @@ timeout content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "timeout": 100, @@ -429,7 +429,7 @@ timeout content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, nil, nil @@ -455,7 +455,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, "invaild json string" ) @@ -480,7 +480,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline":[ @@ -498,7 +498,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"path\" validation failed: string too short, expected at least 1, got 0","message":"bad request body"} +{"error_msg":"bad request body: property \"pipeline\" validation failed: failed to validate item 1: property \"path\" validation failed: string too short, expected at least 1, got 0"} --- no_error_log [error] @@ -510,7 +510,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -528,7 +528,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"method\" validation failed: matches non of the enum values","message":"bad request body"} +{"error_msg":"bad request body: property \"pipeline\" validation failed: failed to validate item 1: property \"method\" validation failed: matches non of the enum values"} --- no_error_log [error] @@ -540,7 +540,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -558,7 +558,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"version\" validation failed: matches non of the enum values","message":"bad request body"} +{"error_msg":"bad request body: property \"pipeline\" validation failed: failed to validate item 1: property \"version\" validation failed: matches non of the enum values"} --- no_error_log [error] @@ -570,7 +570,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline":[{ @@ -588,7 +588,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"pipeline\" validation failed: failed to validate item 1: property \"ssl_verify\" validation failed: wrong type: expected boolean, got number","message":"bad request body"} +{"error_msg":"bad request body: property \"pipeline\" validation failed: failed to validate item 1: property \"ssl_verify\" validation failed: wrong type: expected boolean, got number"} --- no_error_log [error] @@ -600,7 +600,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "pipeline":[] @@ -614,7 +614,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"property \"pipeline\" validation failed: expect array to have at least 1 items","message":"bad request body"} +{"error_msg":"bad request body: property \"pipeline\" validation failed: expect array to have at least 1 items"} --- no_error_log [error] @@ -627,7 +627,7 @@ GET /aggregate content_by_lua_block { local core = require("apisix.core") local t = require("lib.test_admin").test - local code, body = t('/apisix/batch', + local code, body = t('/apisix/batch-requests', ngx.HTTP_POST, [=[{ "timeout": 100, From 6c11f4256b1fb02b4d5f1461c5610613c26270e8 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Wed, 29 Apr 2020 10:53:12 +0800 Subject: [PATCH 19/20] refactor: as reviews --- apisix/plugins/batch-requests.lua | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/apisix/plugins/batch-requests.lua b/apisix/plugins/batch-requests.lua index 33bc68c4cfd6..386d5dec6b47 100644 --- a/apisix/plugins/batch-requests.lua +++ b/apisix/plugins/batch-requests.lua @@ -151,10 +151,12 @@ local function set_common_query(data) end -local function getFile(file_name) +local function get_file(file_name) local f = io_open(file_name, 'r') if f then - return f:read("*all") + local req_body = f:read("*all") + f:close() + return req_body end return @@ -167,11 +169,13 @@ local function batch_requests() if not req_body then local file_name = ngx.req.get_body_file() if file_name then - req_body = getFile(file_name) + req_body = get_file(file_name) end if not req_body then - core.response.exit(400, {message = "no request body, you should give at least one pipeline setting"}) + core.response.exit(400, { + message = "no request body, you should give at least one pipeline setting" + }) end end From 8e4882a923dbb8e319c5b8f38211b3df18b0d872 Mon Sep 17 00:00:00 2001 From: ShiningRush <277040271@qq.com> Date: Wed, 29 Apr 2020 11:38:29 +0800 Subject: [PATCH 20/20] refactor: rename message field --- apisix/plugins/batch-requests.lua | 24 +++++++++++++----------- t/plugin/batch-requests.t | 4 ++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/apisix/plugins/batch-requests.lua b/apisix/plugins/batch-requests.lua index 386d5dec6b47..34d784a89f97 100644 --- a/apisix/plugins/batch-requests.lua +++ b/apisix/plugins/batch-requests.lua @@ -174,14 +174,16 @@ local function batch_requests() if not req_body then core.response.exit(400, { - message = "no request body, you should give at least one pipeline setting" + error_msg = "no request body, you should give at least one pipeline setting" }) end end local data, err = core.json.decode(req_body) if not data then - core.response.exit(400, {message = "invalid request body", req_body = req_body, err = err}) + core.response.exit(400, { + error_msg = "invalid request body: " .. req_body .. ", err: " .. err + }) end local code, body = check_input(data) @@ -193,31 +195,31 @@ local function batch_requests() httpc:set_timeout(data.timeout) local ok, err = httpc:connect("127.0.0.1", ngx.var.server_port) if not ok then - core.response.exit(500, {message = "connect to apisix failed", err = err}) + core.response.exit(500, {error_msg = "connect to apisix failed: " .. err}) end set_common_header(data) set_common_query(data) local responses, err = httpc:request_pipeline(data.pipeline) if not responses then - core.response.exit(400, {message = "request failed", err = err}) + core.response.exit(400, {error_msg = "request failed: " .. err}) end local aggregated_resp = {} - for i,r in ipairs(responses) do - if not r.status then + for _, resp in ipairs(responses) do + if not resp.status then core.table.insert(aggregated_resp, { status = 504, reason = "upstream timeout" }) end local sub_resp = { - status = r.status, - reason = r.reason, - headers = r.headers, + status = resp.status, + reason = resp.reason, + headers = resp.headers, } - if r.has_body then - sub_resp.body = r:read_body() + if resp.has_body then + sub_resp.body = resp:read_body() end core.table.insert(aggregated_resp, sub_resp) end diff --git a/t/plugin/batch-requests.t b/t/plugin/batch-requests.t index dde064ef9618..9c784a6bd481 100644 --- a/t/plugin/batch-requests.t +++ b/t/plugin/batch-requests.t @@ -443,7 +443,7 @@ timeout GET /aggregate --- error_code: 400 --- response_body -{"message":"no request body, you should give at least one pipeline setting"} +{"error_msg":"no request body, you should give at least one pipeline setting"} --- no_error_log [error] @@ -468,7 +468,7 @@ GET /aggregate GET /aggregate --- error_code: 400 --- response_body -{"err":"Expected value but found invalid token at character 1","req_body":"invaild json string","message":"invalid request body"} +{"error_msg":"invalid request body: invaild json string, err: Expected value but found invalid token at character 1"} --- no_error_log [error]