Skip to content

Commit 698c084

Browse files
authored
Merge pull request #258 from rikatz/nginx-sticky-annotations
Nginx sticky annotations
2 parents a41ee3f + 4be502e commit 698c084

File tree

10 files changed

+406
-10
lines changed

10 files changed

+406
-10
lines changed

controllers/nginx/configuration.md

+19-1
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ The following annotations are supported:
5252
|[ingress.kubernetes.io/upstream-max-fails](#custom-nginx-upstream-checks)|number|
5353
|[ingress.kubernetes.io/upstream-fail-timeout](#custom-nginx-upstream-checks)|number|
5454
|[ingress.kubernetes.io/whitelist-source-range](#whitelist-source-range)|CIDR|
55+
|[ingress.kubernetes.io/affinity](#session-affinity)|true or false|
56+
|[ingress.kubernetes.io/session-cookie-name](#cookie-affinity)|string|
57+
|[ingress.kubernetes.io/session-cookie-hash](#cookie-affinity)|string|
5558

5659

5760

@@ -179,7 +182,22 @@ To configure this setting globally for all Ingress rules, the `whitelist-source-
179182

180183
*Note:* Adding an annotation to an Ingress rule overrides any global restriction.
181184

182-
Please check the [whitelist](examples/whitelist/README.md) example.
185+
Please check the [whitelist](examples/affinity/cookie/nginx/README.md) example.
186+
187+
188+
### Session Affinity
189+
190+
The annotation `ingress.kubernetes.io/affinity` enables and sets the affinity type in all Upstreams of an Ingress. This way, a request will always be directed to the same upstream server.
191+
192+
193+
#### Cookie affinity
194+
If you use the ``cookie`` type you can also specify the name of the cookie that will be used to route the requests with the annotation `ingress.kubernetes.io/session-cookie-name`. The default is to create a cookie named 'route'.
195+
196+
In case of NGINX the annotation `ingress.kubernetes.io/session-cookie-hash` defines which algorithm will be used to 'hash' the used upstream. Default value is `md5` and possible values are `md5`, `sha1` and `index`.
197+
The `index` option is not hashed, an in-memory index is used instead, it's quicker and the overhead is shorter Warning: the matching against upstream servers list is inconsistent. So, at reload, if upstreams servers has changed, index values are not guaranted to correspond to the same server as before! USE IT WITH CAUTION and only if you need to!
198+
199+
In NGINX this feature is implemented by the third party module [nginx-sticky-module-ng](https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng). The workflow used to define which upstream server will be used is explained [here]https://bitbucket.org/nginx-goodies/nginx-sticky-module-ng/raw/08a395c66e425540982c00482f55034e1fee67b6/docs/sticky.pdf
200+
183201

184202

185203
### **Allowed parameters in configuration ConfigMap**

controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl

+2-2
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,8 @@ http {
185185

186186
{{range $name, $upstream := $backends}}
187187
upstream {{$upstream.Name}} {
188-
{{ if $cfg.EnableStickySessions }}
189-
sticky hash=sha1 httponly;
188+
{{ if eq $upstream.SessionAffinity.AffinityType "cookie" }}
189+
sticky hash={{$upstream.SessionAffinity.CookieSessionAffinity.Hash}} name={{$upstream.SessionAffinity.CookieSessionAffinity.Name}} httponly;
190190
{{ else }}
191191
least_conn;
192192
{{ end }}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sessionaffinity
18+
19+
import (
20+
"regexp"
21+
22+
"github.com/golang/glog"
23+
24+
"k8s.io/kubernetes/pkg/apis/extensions"
25+
26+
"k8s.io/ingress/core/pkg/ingress/annotations/parser"
27+
)
28+
29+
const (
30+
annotationAffinityType = "ingress.kubernetes.io/affinity"
31+
// If a cookie with this name exists,
32+
// its value is used as an index into the list of available backends.
33+
annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
34+
defaultAffinityCookieName = "INGRESSCOOKIE"
35+
// This is the algorithm used by nginx to generate a value for the session cookie, if
36+
// one isn't supplied and affintiy is set to "cookie".
37+
annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
38+
defaultAffinityCookieHash = "md5"
39+
)
40+
41+
var (
42+
affinityCookieHashRegex = regexp.MustCompile(`^(index|md5|sha1)$`)
43+
)
44+
45+
// AffinityConfig describes the per ingress session affinity config
46+
type AffinityConfig struct {
47+
// The type of affinity that will be used
48+
AffinityType string `json:"type"`
49+
CookieConfig
50+
}
51+
52+
// CookieConfig describes the Config of cookie type affinity
53+
type CookieConfig struct {
54+
// The name of the cookie that will be used in case of cookie affinity type.
55+
Name string `json:"name"`
56+
// The hash that will be used to encode the cookie in case of cookie affinity type
57+
Hash string `json:"hash"`
58+
}
59+
60+
// CookieAffinityParse gets the annotation values related to Cookie Affinity
61+
// It also sets default values when no value or incorrect value is found
62+
func CookieAffinityParse(ing *extensions.Ingress) *CookieConfig {
63+
64+
sn, err := parser.GetStringAnnotation(annotationAffinityCookieName, ing)
65+
66+
if err != nil || sn == "" {
67+
glog.V(3).Infof("Ingress %v: No value found in annotation %v. Using the default %v", ing.Name, annotationAffinityCookieName, defaultAffinityCookieName)
68+
sn = defaultAffinityCookieName
69+
}
70+
71+
sh, err := parser.GetStringAnnotation(annotationAffinityCookieHash, ing)
72+
73+
if err != nil || !affinityCookieHashRegex.MatchString(sh) {
74+
glog.V(3).Infof("Invalid or no annotation value found in Ingress %v: %v. Setting it to default %v", ing.Name, annotationAffinityCookieHash, defaultAffinityCookieHash)
75+
sh = defaultAffinityCookieHash
76+
}
77+
78+
return &CookieConfig{
79+
Name: sn,
80+
Hash: sh,
81+
}
82+
}
83+
84+
// NewParser creates a new Affinity annotation parser
85+
func NewParser() parser.IngressAnnotation {
86+
return affinity{}
87+
}
88+
89+
type affinity struct {
90+
}
91+
92+
// ParseAnnotations parses the annotations contained in the ingress
93+
// rule used to configure the affinity directives
94+
func (a affinity) Parse(ing *extensions.Ingress) (interface{}, error) {
95+
96+
var cookieAffinityConfig *CookieConfig
97+
cookieAffinityConfig = &CookieConfig{}
98+
99+
// Check the type of affinity that will be used
100+
at, err := parser.GetStringAnnotation(annotationAffinityType, ing)
101+
if err != nil {
102+
at = ""
103+
}
104+
105+
switch at {
106+
case "cookie":
107+
cookieAffinityConfig = CookieAffinityParse(ing)
108+
109+
default:
110+
glog.V(3).Infof("No default affinity was found for Ingress %v", ing.Name)
111+
112+
}
113+
return &AffinityConfig{
114+
AffinityType: at,
115+
CookieConfig: *cookieAffinityConfig,
116+
}, nil
117+
118+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
Copyright 2016 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package sessionaffinity
18+
19+
import (
20+
"testing"
21+
22+
"k8s.io/kubernetes/pkg/api"
23+
"k8s.io/kubernetes/pkg/apis/extensions"
24+
"k8s.io/kubernetes/pkg/util/intstr"
25+
)
26+
27+
func buildIngress() *extensions.Ingress {
28+
defaultBackend := extensions.IngressBackend{
29+
ServiceName: "default-backend",
30+
ServicePort: intstr.FromInt(80),
31+
}
32+
33+
return &extensions.Ingress{
34+
ObjectMeta: api.ObjectMeta{
35+
Name: "foo",
36+
Namespace: api.NamespaceDefault,
37+
},
38+
Spec: extensions.IngressSpec{
39+
Backend: &extensions.IngressBackend{
40+
ServiceName: "default-backend",
41+
ServicePort: intstr.FromInt(80),
42+
},
43+
Rules: []extensions.IngressRule{
44+
{
45+
Host: "foo.bar.com",
46+
IngressRuleValue: extensions.IngressRuleValue{
47+
HTTP: &extensions.HTTPIngressRuleValue{
48+
Paths: []extensions.HTTPIngressPath{
49+
{
50+
Path: "/foo",
51+
Backend: defaultBackend,
52+
},
53+
},
54+
},
55+
},
56+
},
57+
},
58+
},
59+
}
60+
}
61+
62+
func TestIngressAffinityCookieConfig(t *testing.T) {
63+
ing := buildIngress()
64+
65+
data := map[string]string{}
66+
data[annotationAffinityType] = "cookie"
67+
data[annotationAffinityCookieHash] = "sha123"
68+
data[annotationAffinityCookieName] = "INGRESSCOOKIE"
69+
ing.SetAnnotations(data)
70+
71+
affin, _ := NewParser().Parse(ing)
72+
nginxAffinity, ok := affin.(*AffinityConfig)
73+
if !ok {
74+
t.Errorf("expected a Config type")
75+
}
76+
77+
if nginxAffinity.AffinityType != "cookie" {
78+
t.Errorf("expected cookie as sticky-type but returned %v", nginxAffinity.AffinityType)
79+
}
80+
81+
if nginxAffinity.CookieConfig.Hash != "md5" {
82+
t.Errorf("expected md5 as sticky-hash but returned %v", nginxAffinity.CookieConfig.Hash)
83+
}
84+
85+
if nginxAffinity.CookieConfig.Name != "INGRESSCOOKIE" {
86+
t.Errorf("expected route as sticky-name but returned %v", nginxAffinity.CookieConfig.Name)
87+
}
88+
}

core/pkg/ingress/controller/annotations.go

+11-3
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"k8s.io/ingress/core/pkg/ingress/annotations/ratelimit"
3434
"k8s.io/ingress/core/pkg/ingress/annotations/rewrite"
3535
"k8s.io/ingress/core/pkg/ingress/annotations/secureupstream"
36+
"k8s.io/ingress/core/pkg/ingress/annotations/sessionaffinity"
3637
"k8s.io/ingress/core/pkg/ingress/annotations/sslpassthrough"
3738
"k8s.io/ingress/core/pkg/ingress/errors"
3839
"k8s.io/ingress/core/pkg/ingress/resolver"
@@ -62,6 +63,7 @@ func newAnnotationExtractor(cfg extractorConfig) annotationExtractor {
6263
"RateLimit": ratelimit.NewParser(),
6364
"Redirect": rewrite.NewParser(cfg),
6465
"SecureUpstream": secureupstream.NewParser(),
66+
"SessionAffinity": sessionaffinity.NewParser(),
6567
"SSLPassthrough": sslpassthrough.NewParser(),
6668
},
6769
}
@@ -96,9 +98,10 @@ func (e *annotationExtractor) Extract(ing *extensions.Ingress) map[string]interf
9698
}
9799

98100
const (
99-
secureUpstream = "SecureUpstream"
100-
healthCheck = "HealthCheck"
101-
sslPassthrough = "SSLPassthrough"
101+
secureUpstream = "SecureUpstream"
102+
healthCheck = "HealthCheck"
103+
sslPassthrough = "SSLPassthrough"
104+
sessionAffinity = "SessionAffinity"
102105
)
103106

104107
func (e *annotationExtractor) SecureUpstream(ing *extensions.Ingress) bool {
@@ -115,3 +118,8 @@ func (e *annotationExtractor) SSLPassthrough(ing *extensions.Ingress) bool {
115118
val, _ := e.annotations[sslPassthrough].Parse(ing)
116119
return val.(bool)
117120
}
121+
122+
func (e *annotationExtractor) SessionAffinity(ing *extensions.Ingress) *sessionaffinity.AffinityConfig {
123+
val, _ := e.annotations[sessionAffinity].Parse(ing)
124+
return val.(*sessionaffinity.AffinityConfig)
125+
}

core/pkg/ingress/controller/annotations_test.go

+43-4
Original file line numberDiff line numberDiff line change
@@ -28,10 +28,13 @@ import (
2828
)
2929

3030
const (
31-
annotationSecureUpstream = "ingress.kubernetes.io/secure-backends"
32-
annotationUpsMaxFails = "ingress.kubernetes.io/upstream-max-fails"
33-
annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout"
34-
annotationPassthrough = "ingress.kubernetes.io/ssl-passthrough"
31+
annotationSecureUpstream = "ingress.kubernetes.io/secure-backends"
32+
annotationUpsMaxFails = "ingress.kubernetes.io/upstream-max-fails"
33+
annotationUpsFailTimeout = "ingress.kubernetes.io/upstream-fail-timeout"
34+
annotationPassthrough = "ingress.kubernetes.io/ssl-passthrough"
35+
annotationAffinityType = "ingress.kubernetes.io/affinity"
36+
annotationAffinityCookieName = "ingress.kubernetes.io/session-cookie-name"
37+
annotationAffinityCookieHash = "ingress.kubernetes.io/session-cookie-hash"
3538
)
3639

3740
type mockCfg struct {
@@ -179,3 +182,39 @@ func TestSSLPassthrough(t *testing.T) {
179182
}
180183
}
181184
}
185+
186+
func TestAffinitySession(t *testing.T) {
187+
ec := newAnnotationExtractor(mockCfg{})
188+
ing := buildIngress()
189+
190+
fooAnns := []struct {
191+
annotations map[string]string
192+
affinitytype string
193+
hash string
194+
name string
195+
}{
196+
{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "md5", annotationAffinityCookieName: "route"}, "cookie", "md5", "route"},
197+
{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "xpto", annotationAffinityCookieName: "route1"}, "cookie", "md5", "route1"},
198+
{map[string]string{annotationAffinityType: "cookie", annotationAffinityCookieHash: "", annotationAffinityCookieName: ""}, "cookie", "md5", "INGRESSCOOKIE"},
199+
{map[string]string{}, "", "", ""},
200+
{nil, "", "", ""},
201+
}
202+
203+
for _, foo := range fooAnns {
204+
ing.SetAnnotations(foo.annotations)
205+
r := ec.SessionAffinity(ing)
206+
t.Logf("Testing pass %v %v %v", foo.affinitytype, foo.hash, foo.name)
207+
if r == nil {
208+
t.Errorf("Returned nil but expected a SessionAffinity.AffinityConfig")
209+
continue
210+
}
211+
212+
if r.CookieConfig.Hash != foo.hash {
213+
t.Errorf("Returned %v but expected %v for Hash", r.CookieConfig.Hash, foo.hash)
214+
}
215+
216+
if r.CookieConfig.Name != foo.name {
217+
t.Errorf("Returned %v but expected %v for Name", r.CookieConfig.Name, foo.name)
218+
}
219+
}
220+
}

core/pkg/ingress/controller/controller.go

+9
Original file line numberDiff line numberDiff line change
@@ -723,6 +723,7 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
723723

724724
secUpstream := ic.annotations.SecureUpstream(ing)
725725
hz := ic.annotations.HealthCheck(ing)
726+
affinity := ic.annotations.SessionAffinity(ing)
726727

727728
var defBackend string
728729
if ing.Spec.Backend != nil {
@@ -762,6 +763,14 @@ func (ic *GenericController) createUpstreams(data []interface{}) map[string]*ing
762763
if !upstreams[name].Secure {
763764
upstreams[name].Secure = secUpstream
764765
}
766+
if upstreams[name].SessionAffinity.AffinityType == "" {
767+
upstreams[name].SessionAffinity.AffinityType = affinity.AffinityType
768+
if affinity.AffinityType == "cookie" {
769+
upstreams[name].SessionAffinity.CookieSessionAffinity.Name = affinity.CookieConfig.Name
770+
upstreams[name].SessionAffinity.CookieSessionAffinity.Hash = affinity.CookieConfig.Hash
771+
}
772+
}
773+
765774
svcKey := fmt.Sprintf("%v/%v", ing.GetNamespace(), path.Backend.ServiceName)
766775
endp, err := ic.serviceEndpoints(svcKey, path.Backend.ServicePort.String(), hz)
767776
if err != nil {

core/pkg/ingress/types.go

+20
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,26 @@ type Backend struct {
154154
Secure bool `json:"secure"`
155155
// Endpoints contains the list of endpoints currently running
156156
Endpoints []Endpoint `json:"endpoints"`
157+
// StickySession contains the StickyConfig object with stickness configuration
158+
159+
SessionAffinity SessionAffinityConfig
160+
}
161+
162+
// SessionAffinityConfig describes different affinity configurations for new sessions.
163+
// Once a session is mapped to a backend based on some affinity setting, it
164+
// retains that mapping till the backend goes down, or the ingress controller
165+
// restarts. Exactly one of these values will be set on the upstream, since multiple
166+
// affinity values are incompatible. Once set, the backend makes no guarantees
167+
// about honoring updates.
168+
type SessionAffinityConfig struct {
169+
AffinityType string `json:"name"`
170+
CookieSessionAffinity CookieSessionAffinity
171+
}
172+
173+
// CookieSessionAffinity defines the structure used in Affinity configured by Cookies.
174+
type CookieSessionAffinity struct {
175+
Name string `json:"name"`
176+
Hash string `json:"hash"`
157177
}
158178

159179
// Endpoint describes a kubernetes endpoint in a backend

0 commit comments

Comments
 (0)