Skip to content

Commit

Permalink
feat: allow setting separate keepalive pool
Browse files Browse the repository at this point in the history
Signed-off-by: spacewander <[email protected]>
  • Loading branch information
spacewander committed Jun 30, 2021
1 parent c074fe7 commit e1ec980
Show file tree
Hide file tree
Showing 5 changed files with 281 additions and 12 deletions.
32 changes: 31 additions & 1 deletion apisix/balancer.lua
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ local balancer = require("ngx.balancer")
local core = require("apisix.core")
local priority_balancer = require("apisix.balancer.priority")
local ipairs = ipairs
local enable_keepalive = balancer.enable_keepalive
local set_more_tries = balancer.set_more_tries
local get_last_failure = balancer.get_last_failure
local set_timeouts = balancer.set_timeouts
Expand Down Expand Up @@ -253,6 +254,34 @@ end
_M.pick_server = pick_server


local set_current_peer
do
local pool_opt = {}

function set_current_peer(server, ctx)
local up_conf = ctx.upstream_conf
local keepalive_pool = up_conf.keepalive_pool

if keepalive_pool and enable_keepalive then
local idle_timeout = keepalive_pool.idle_timeout
local size = keepalive_pool.size
local requests = keepalive_pool.requests

pool_opt.pool_size = size
local ok, err = balancer.set_current_peer(server.host, server.port,
pool_opt)
if not ok then
return ok, err
end

return balancer.enable_keepalive(idle_timeout, requests)
end

return balancer.set_current_peer(server.host, server.port)
end
end


function _M.run(route, ctx)
local server, err

Expand Down Expand Up @@ -283,7 +312,8 @@ function _M.run(route, ctx)
end

core.log.info("proxy request to ", server.host, ":", server.port)
local ok, err = balancer.set_current_peer(server.host, server.port)

local ok, err = set_current_peer(server, ctx)
if not ok then
core.log.error("failed to set server peer [", server.host, ":",
server.port, "] err: ", err)
Expand Down
16 changes: 10 additions & 6 deletions apisix/cli/ngx_tpl.lua
Original file line number Diff line number Diff line change
Expand Up @@ -271,25 +271,29 @@ http {
# http configuration snippet ends
upstream apisix_backend {
keepalive {* http.upstream.keepalive *};
keepalive_requests {* http.upstream.keepalive_requests *};
keepalive_timeout {* http.upstream.keepalive_timeout *};
# we put the static configuration above so that we can override it in the Lua code
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
}
{% if enabled_plugins["dubbo-proxy"] then %}
upstream apisix_dubbo_backend {
multi {* dubbo_upstream_multiplex_count *};
keepalive {* http.upstream.keepalive *};
keepalive_requests {* http.upstream.keepalive_requests *};
keepalive_timeout {* http.upstream.keepalive_timeout *};
}
# we put the static configuration above so that we can override it in the Lua code
{% if enabled_plugins["dubbo-proxy"] then %}
upstream apisix_dubbo_backend {
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
multi {* dubbo_upstream_multiplex_count *};
keepalive 320;
}
{% end %}
Expand Down
20 changes: 20 additions & 0 deletions apisix/schema_def.lua
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,26 @@ local upstream_schema = {
},
required = {"client_cert", "client_key"},
},
keepalive_pool = {
type = "object",
properties = {
size = {
type = "integer",
default = 320,
minimum = 1,
},
idle_timeout = {
type = "number",
default = 60,
minimum = 0,
},
requests = {
type = "integer",
default = 1000,
minimum = 1,
},
},
},
type = {
description = "algorithms of load balancing",
type = "string",
Expand Down
12 changes: 7 additions & 5 deletions t/APISIX.pm
Original file line number Diff line number Diff line change
Expand Up @@ -148,12 +148,13 @@ if ($version =~ m/\/mod_dubbo/) {
$dubbo_upstream = <<_EOC_;
upstream apisix_dubbo_backend {
server 0.0.0.1;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
multi 1;
keepalive 320;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
}
_EOC_
Expand Down Expand Up @@ -447,11 +448,12 @@ _EOC_
upstream apisix_backend {
server 0.0.0.1;
keepalive 32;
balancer_by_lua_block {
apisix.http_balancer_phase()
}
keepalive 32;
}
$dubbo_upstream
Expand Down
213 changes: 213 additions & 0 deletions t/node/upstream-keepalive-pool.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
use t::APISIX;

my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
my $version = eval { `$nginx_binary -V 2>&1` };

if ($version !~ m/\/apisix-nginx-module/) {
plan(skip_all => "apisix-nginx-module not installed");
} else {
plan('no_plan');
}

repeat_each(1);
log_level('debug');
no_root_location();
no_shuffle();

add_block_preprocessor(sub {
my ($block) = @_;

if (!defined $block->request) {
$block->set_value("request", "GET /t");
}

if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
$block->set_value("no_error_log", "[error]");
}
});

run_tests();

__DATA__
=== TEST 1: bad pool size
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/upstreams/1',
ngx.HTTP_PUT,
[[{
"type": "roundrobin",
"nodes": {
"127.0.0.1:1983": 1
},
"keepalive_pool": {
"size": 0
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.print(body)
}
}
--- error_code: 400
--- response_body
{"error_msg":"invalid configuration: property \"keepalive_pool\" validation failed: property \"size\" validation failed: expected 0 to be greater than 1"}
=== TEST 2: set route/upstream
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/upstreams/1',
ngx.HTTP_PUT,
[[{
"type": "roundrobin",
"nodes": {
"127.0.0.1:1980": 1
},
"keepalive_pool": {
"size": 4,
"idle_timeout": 8,
"requests": 16
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.print(body)
return
end
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri":"/hello",
"upstream_id": 1
}]])
if code >= 300 then
ngx.status = code
end
ngx.print(body)
}
}
=== TEST 3: hit
--- config
location /t {
content_by_lua_block {
local http = require "resty.http"
local uri = "http://127.0.0.1:" .. ngx.var.server_port
.. "/hello"
for i = 1, 3 do
local httpc = http.new()
local res, err = httpc:request_uri(uri)
if not res then
ngx.say(err)
return
end
ngx.print(res.body)
end
}
}
--- response_body
hello world
hello world
hello world
--- grep_error_log eval
qr/lua balancer: keepalive .*/
--- grep_error_log_out eval
qr/^lua balancer: keepalive create pool, crc32: \S+, size: 4
lua balancer: keepalive no free connection, cpool: \S+
lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
lua balancer: keepalive reusing connection \S+, requests: 1, cpool: \S+
lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
lua balancer: keepalive reusing connection \S+, requests: 2, cpool: \S+
lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
$/
=== TEST 4: only reuse one time
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/upstreams/1',
ngx.HTTP_PUT,
[[{
"type": "roundrobin",
"nodes": {
"127.0.0.1:1980": 1
},
"keepalive_pool": {
"size": 1,
"idle_timeout": 8,
"requests": 2
}
}]]
)
if code >= 300 then
ngx.status = code
end
ngx.print(body)
}
}
=== TEST 5: hit
--- config
location /t {
content_by_lua_block {
local http = require "resty.http"
local uri = "http://127.0.0.1:" .. ngx.var.server_port
.. "/hello"
for i = 1, 3 do
local httpc = http.new()
local res, err = httpc:request_uri(uri)
if not res then
ngx.say(err)
return
end
ngx.print(res.body)
end
}
}
--- response_body
hello world
hello world
hello world
--- grep_error_log eval
qr/lua balancer: keepalive .*/
--- grep_error_log_out eval
qr/^lua balancer: keepalive create pool, crc32: \S+, size: 1
lua balancer: keepalive no free connection, cpool: \S+
lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
lua balancer: keepalive reusing connection \S+, requests: 1, cpool: \S+
lua balancer: keepalive not saving connection \S+, cpool: \S+, connections: 0
lua balancer: keepalive free pool \S+, crc32: \S+
lua balancer: keepalive create pool, crc32: \S+, size: 1
lua balancer: keepalive no free connection, cpool: \S+
lua balancer: keepalive saving connection \S+, cpool: \S+, connections: 1
$/

0 comments on commit e1ec980

Please sign in to comment.