From a9480b352413c411f87f7925dc969f6d2549b5bb Mon Sep 17 00:00:00 2001 From: sskserk Date: Wed, 9 Mar 2022 15:55:13 +0100 Subject: [PATCH 1/7] nginx 1.19.10 keepalive_time parameter --- internal/ingress/controller/config/config.go | 5 +++++ rootfs/etc/nginx/template/nginx.tmpl | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/ingress/controller/config/config.go b/internal/ingress/controller/config/config.go index f37516e78c..6a436d8efa 100644 --- a/internal/ingress/controller/config/config.go +++ b/internal/ingress/controller/config/config.go @@ -468,6 +468,10 @@ type Configuration struct { // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive UpstreamKeepaliveConnections int `json:"upstream-keepalive-connections,omitempty"` + // Sets the maximum time during which requests can be processed through one keepalive connection + // https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time + UpstreamKeepaliveTime string `json:"upstream-keepalive-time,omitempty"` + // Sets a timeout during which an idle keepalive connection to an upstream server will stay open. // http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_timeout UpstreamKeepaliveTimeout int `json:"upstream-keepalive-timeout,omitempty"` @@ -892,6 +896,7 @@ func NewDefault() Configuration { ServiceUpstream: false, }, UpstreamKeepaliveConnections: 320, + UpstreamKeepaliveTime: "1h", UpstreamKeepaliveTimeout: 60, UpstreamKeepaliveRequests: 10000, LimitConnZoneVariable: defaultLimitConnZoneVariable, diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 2ee76831cc..cd49dbd4f5 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -156,7 +156,7 @@ http { {{ else }} modsecurity_rules_file /etc/nginx/modsecurity/modsecurity.conf; {{ end }} - + {{ if $all.Cfg.EnableOWASPCoreRules }} modsecurity_rules_file /etc/nginx/owasp-modsecurity-crs/nginx-modsecurity.conf; {{ end }} @@ -508,7 +508,7 @@ http { {{ if (gt $cfg.UpstreamKeepaliveConnections 0) }} keepalive {{ $cfg.UpstreamKeepaliveConnections }}; - + keepalive_time {{ $cfg.UpstreamKeepaliveTime }}; keepalive_timeout {{ $cfg.UpstreamKeepaliveTimeout }}s; keepalive_requests {{ $cfg.UpstreamKeepaliveRequests }}; {{ end }} From 9781b98a96b66a22c63355e458c38b154d6bf91a Mon Sep 17 00:00:00 2001 From: sskserk Date: Tue, 15 Mar 2022 09:13:11 +0100 Subject: [PATCH 2/7] nginx v1.19.10 base image --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b1c90e0f1e..f323c10308 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ endif REGISTRY ?= gcr.io/k8s-staging-ingress-nginx -BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20210926-g5662db450@sha256:1ef404b5e8741fe49605a1f40c3fdd8ef657aecdb9526ea979d1672eeabd0cd9 +BASE_IMAGE ?= gcr.io/k8s-staging-ingress-nginx/nginx:v20220314-controller-v1.1.2-8-g01b92b8b3@sha256:68d88ad12e5ab880b0bcaf02b04032c294de62b8bfb0e9994c99bbfe3c0955dc GOARCH=$(ARCH) From ca9bbe2b79498193c084f1edd7683245810f5fad Mon Sep 17 00:00:00 2001 From: sskserk Date: Tue, 15 Mar 2022 11:09:10 +0100 Subject: [PATCH 3/7] keepalive_time documentation --- .../user-guide/nginx-configuration/configmap.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/user-guide/nginx-configuration/configmap.md b/docs/user-guide/nginx-configuration/configmap.md index b48cc1028e..4a776916c9 100755 --- a/docs/user-guide/nginx-configuration/configmap.md +++ b/docs/user-guide/nginx-configuration/configmap.md @@ -111,6 +111,7 @@ The following table shows a configuration option's name, type, and the default v |[variables-hash-bucket-size](#variables-hash-bucket-size)|int|128| |[variables-hash-max-size](#variables-hash-max-size)|int|2048| |[upstream-keepalive-connections](#upstream-keepalive-connections)|int|320| +|[upstream-keepalive-time](#upstream-keepalive-time)|string|"1h"| |[upstream-keepalive-timeout](#upstream-keepalive-timeout)|int|60| |[upstream-keepalive-requests](#upstream-keepalive-requests)|int|10000| |[limit-conn-zone-variable](#limit-conn-zone-variable)|string|"$binary_remote_addr"| @@ -223,13 +224,13 @@ Enables the return of the header Server from the backend instead of the generic Enables Ingress to parse and add *-snippet annotations/directives created by the user. _**default:**_ `true` -Warning: We recommend enabling this option only if you TRUST users with permission to create Ingress objects, as this +Warning: We recommend enabling this option only if you TRUST users with permission to create Ingress objects, as this may allow a user to add restricted configurations to the final nginx.conf file ## annotation-value-word-blocklist -Contains a comma-separated value of chars/words that are well known of being used to abuse Ingress configuration -and must be blocked. Related to [CVE-2021-25742](https://github.com/kubernetes/ingress-nginx/issues/7837) +Contains a comma-separated value of chars/words that are well known of being used to abuse Ingress configuration +and must be blocked. Related to [CVE-2021-25742](https://github.com/kubernetes/ingress-nginx/issues/7837) When an annotation is detected with a value that matches one of the blocked bad words, the whole Ingress won't be configured. @@ -769,6 +770,14 @@ _References:_ [http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) +## upstream-keepalive-time + +Sets the maximum time during which requests can be processed through one keepalive connection. + _**default:**_ "1h" + +_References:_ +[http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time](http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive_time) + ## upstream-keepalive-timeout Sets a timeout during which an idle keepalive connection to an upstream server will stay open. @@ -1258,7 +1267,7 @@ Configure `memcached` client for [Global Rate Limiting](https://github.com/kuber * `global-rate-limit-memcached-host`: IP/FQDN of memcached server to use. Required to enable Global Rate Limiting. * `global-rate-limit-memcached-port`: port of memcached server to use. Defaults default memcached port of `11211`. * `global-rate-limit-memcached-connect-timeout`: configure timeout for connect, send and receive operations. Unit is millisecond. Defaults to 50ms. -* `global-rate-limit-memcached-max-idle-timeout`: configure timeout for cleaning idle connections. Unit is millisecond. Defaults to 50ms. +* `global-rate-limit-memcached-max-idle-timeout`: configure timeout for cleaning idle connections. Unit is millisecond. Defaults to 50ms. * `global-rate-limit-memcached-pool-size`: configure number of max connections to keep alive. Make sure your `memcached` server can handle `global-rate-limit-memcached-pool-size * worker-processes * ` simultaneous connections. From 9230814fb7956ed39c972d0ce1332936cec83553 Mon Sep 17 00:00:00 2001 From: sskserk Date: Mon, 21 Mar 2022 09:26:08 +0100 Subject: [PATCH 4/7] base image --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f323c10308..5624a3fc8e 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ endif REGISTRY ?= gcr.io/k8s-staging-ingress-nginx -BASE_IMAGE ?= gcr.io/k8s-staging-ingress-nginx/nginx:v20220314-controller-v1.1.2-8-g01b92b8b3@sha256:68d88ad12e5ab880b0bcaf02b04032c294de62b8bfb0e9994c99bbfe3c0955dc +BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20220318-controller-v1.1.2-21-ge51c15160@sha256:ee001455750923c131bff706f20cd95078a78c9538ab0c15f754fd9af7fe9656 GOARCH=$(ARCH) From 64061fd5fe782f87d710c9b3e65782357c960e2f Mon Sep 17 00:00:00 2001 From: sskserk Date: Wed, 23 Mar 2022 23:56:34 +0100 Subject: [PATCH 5/7] restore base image --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5624a3fc8e..b1c90e0f1e 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ endif REGISTRY ?= gcr.io/k8s-staging-ingress-nginx -BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20220318-controller-v1.1.2-21-ge51c15160@sha256:ee001455750923c131bff706f20cd95078a78c9538ab0c15f754fd9af7fe9656 +BASE_IMAGE ?= k8s.gcr.io/ingress-nginx/nginx:v20210926-g5662db450@sha256:1ef404b5e8741fe49605a1f40c3fdd8ef657aecdb9526ea979d1672eeabd0cd9 GOARCH=$(ARCH) From ad8012c278f56c3a991e9df704efb468f85a0e17 Mon Sep 17 00:00:00 2001 From: sskserk Date: Wed, 13 Apr 2022 09:25:02 +0200 Subject: [PATCH 6/7] e2e test --- test/e2e/settings/keep-alive.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/e2e/settings/keep-alive.go b/test/e2e/settings/keep-alive.go index 5a2b5189ef..b61696ebc7 100644 --- a/test/e2e/settings/keep-alive.go +++ b/test/e2e/settings/keep-alive.go @@ -74,6 +74,15 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { }) }) + ginkgo.It("should set keepalive time to upstream server", func() { + f.UpdateNginxConfigMapData("upstream-keepalive-time", "1h") + + f.WaitForNginxConfiguration(func(server string) bool { + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_time\s*1h;`, server) + return match + }) + }) + ginkgo.It("should set the request count to upstream server through one keep alive connection", func() { f.UpdateNginxConfigMapData("upstream-keepalive-requests", "200") From ffc6495d3bd13395325d58ab60401e54c224555f Mon Sep 17 00:00:00 2001 From: sskserk Date: Wed, 13 Apr 2022 14:56:13 +0200 Subject: [PATCH 7/7] replace default value in test --- test/e2e/settings/keep-alive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/settings/keep-alive.go b/test/e2e/settings/keep-alive.go index b61696ebc7..6ef09b78c0 100644 --- a/test/e2e/settings/keep-alive.go +++ b/test/e2e/settings/keep-alive.go @@ -75,10 +75,10 @@ var _ = framework.DescribeSetting("keep-alive keep-alive-requests", func() { }) ginkgo.It("should set keepalive time to upstream server", func() { - f.UpdateNginxConfigMapData("upstream-keepalive-time", "1h") + f.UpdateNginxConfigMapData("upstream-keepalive-time", "75s") f.WaitForNginxConfiguration(func(server string) bool { - match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_time\s*1h;`, server) + match, _ := regexp.MatchString(`upstream\supstream_balancer\s\{[\s\S]*keepalive_time\s*75s;`, server) return match }) })