Merge pull request kubernetes-retired#1350 from aledbf/nginx-backlog

[nginx-ingress-controller]: Improve performance (listen backlog=net.core.somaxconn)

controllers/gce/controller/controller.go

@@ -439,20 +439,22 @@ func (lbc *LoadBalancerController) syncNodes(key string) error {
 	return nil
 }
 
-func nodeReady(node api.Node) bool {
-	for ix := range node.Status.Conditions {
-		condition := &node.Status.Conditions[ix]
-		if condition.Type == api.NodeReady {
-			return condition.Status == api.ConditionTrue
+func getNodeReadyPredicate() cache.NodeConditionPredicate {
+	return func(node *api.Node) bool {
+		for ix := range node.Status.Conditions {
+			condition := &node.Status.Conditions[ix]
+			if condition.Type == api.NodeReady {
+				return condition.Status == api.ConditionTrue
+			}
 		}
+		return false
 	}
-	return false
 }
 
 // getReadyNodeNames returns names of schedulable, ready nodes from the node lister.
 func (lbc *LoadBalancerController) getReadyNodeNames() ([]string, error) {
 	nodeNames := []string{}
-	nodes, err := lbc.nodeLister.NodeCondition(nodeReady).List()
+	nodes, err := lbc.nodeLister.NodeCondition(getNodeReadyPredicate()).List()
 	if err != nil {
 		return nodeNames, err
 	}

controllers/gce/controller/utils.go

@@ -365,7 +365,7 @@ func getZone(n api.Node) string {
 
 // GetZoneForNode returns the zone for a given node by looking up its zone label.
 func (t *GCETranslator) GetZoneForNode(name string) (string, error) {
-	nodes, err := t.nodeLister.NodeCondition(nodeReady).List()
+	nodes, err := t.nodeLister.NodeCondition(getNodeReadyPredicate()).List()
 	if err != nil {
 		return "", err
 	}
@@ -382,7 +382,7 @@ func (t *GCETranslator) GetZoneForNode(name string) (string, error) {
 // ListZones returns a list of zones this Kubernetes cluster spans.
 func (t *GCETranslator) ListZones() ([]string, error) {
 	zones := sets.String{}
-	readyNodes, err := t.nodeLister.NodeCondition(nodeReady).List()
+	readyNodes, err := t.nodeLister.NodeCondition(getNodeReadyPredicate()).List()
 	if err != nil {
 		return zones.List(), err
 	}

controllers/nginx/nginx.tmpl

@@ -276,7 +276,7 @@ http {
 
     # default server, including healthcheck
     server {
-        listen 8080 default_server reuseport;
+        listen 8080 default_server reuseport backlog={{ .backlogSize }};
 
         location /healthz {
             access_log off;

controllers/nginx/nginx/template.go

@@ -61,6 +61,7 @@ func (ngx *Manager) loadTemplate() {
 
 func (ngx *Manager) writeCfg(cfg config.Configuration, ingressCfg IngressConfig) (bool, error) {
 	conf := make(map[string]interface{})
+	conf["backlogSize"] = sysctlSomaxconn()
 	conf["upstreams"] = ingressCfg.Upstreams
 	conf["servers"] = ingressCfg.Servers
 	conf["tcpUpstreams"] = ingressCfg.TCPUpstreams

controllers/nginx/nginx/utils.go

@@ -28,6 +28,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/mitchellh/mapstructure"
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/util/sysctl"
 
 	"k8s.io/contrib/ingress/controllers/nginx/nginx/config"
 )
@@ -220,3 +221,16 @@ func diff(b1, b2 []byte) (data []byte, err error) {
 	}
 	return
 }
+
+// sysctlSomaxconn returns the value of net.core.somaxconn, i.e.
+// maximum number of connections that can be queued for acceptance
+// http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
+func sysctlSomaxconn() int {
+	maxConns, err := sysctl.GetSysctl("net.core.somaxconn")
+	if err != nil || maxConns < 512 {
+		glog.Warningf("system net.core.somaxconn=%v. Using NGINX default (511)", maxConns)
+		return 511
+	}
+
+	return maxConns
+}