Skip to content

Commit f6af1ca

Browse files
aledbfaledbf
authored
Merge pull request #614 from aledbf/refactor-passthrough
Refactor nginx ssl passthrough
2 parents 91c0006 + 590bc0d commit f6af1ca

File tree

15 files changed

+708
-122
lines changed

15 files changed

+708
-122
lines changed

‎Godeps/Godeps.json

+8
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎controllers/nginx/pkg/cmd/controller/nginx.go

+90-5
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@ import (
2626
"os"
2727
"os/exec"
2828
"strconv"
29+
"strings"
2930
"syscall"
3031
"time"
3132

3233
"github.com/golang/glog"
3334
"github.com/spf13/pflag"
3435

36+
proxyproto "github.com/armon/go-proxyproto"
3537
api_v1 "k8s.io/client-go/pkg/api/v1"
3638

37-
"strings"
38-
3939
"k8s.io/ingress/controllers/nginx/pkg/config"
4040
ngx_template "k8s.io/ingress/controllers/nginx/pkg/template"
4141
"k8s.io/ingress/controllers/nginx/pkg/version"
@@ -53,6 +53,8 @@ const (
5353

5454
defaultStatusModule statusModule = "default"
5555
vtsStatusModule statusModule = "vts"
56+
57+
errNoChild = "wait: no child processes"
5658
)
5759

5860
var (
@@ -81,8 +83,40 @@ func newNGINXController() ingress.Controller {
8183
configmap: &api_v1.ConfigMap{},
8284
isIPV6Enabled: isIPv6Enabled(),
8385
resolver: h,
86+
proxy: &proxy{},
87+
}
88+
89+
listener, err := net.Listen("tcp", ":443")
90+
if err != nil {
91+
glog.Fatalf("%v", err)
8492
}
8593

94+
proxyList := &proxyproto.Listener{Listener: listener}
95+
96+
// start goroutine that accepts tcp connections in port 443
97+
go func() {
98+
for {
99+
var conn net.Conn
100+
var err error
101+
102+
if n.isProxyProtocolEnabled {
103+
// we need to wrap the listener in order to decode
104+
// proxy protocol before handling the connection
105+
conn, err = proxyList.Accept()
106+
} else {
107+
conn, err = listener.Accept()
108+
}
109+
110+
if err != nil {
111+
glog.Warningf("unexpected error accepting tcp connection: %v", err)
112+
continue
113+
}
114+
115+
glog.V(3).Infof("remote adress %s to local %s", conn.RemoteAddr(), conn.LocalAddr())
116+
go n.proxy.Handle(conn)
117+
}
118+
}()
119+
86120
var onChange func()
87121
onChange = func() {
88122
template, err := ngx_template.NewTemplate(tmplPath, onChange)
@@ -121,7 +155,8 @@ type NGINXController struct {
121155

122156
storeLister ingress.StoreLister
123157

124-
binary string
158+
binary string
159+
resolver []net.IP
125160

126161
cmdArgs []string
127162

@@ -134,7 +169,10 @@ type NGINXController struct {
134169
// returns true if IPV6 is enabled in the pod
135170
isIPV6Enabled bool
136171

137-
resolver []net.IP
172+
// returns true if proxy protocol es enabled
173+
isProxyProtocolEnabled bool
174+
175+
proxy *proxy
138176
}
139177

140178
// Start start a new NGINX master process running in foreground.
@@ -306,7 +344,7 @@ func (n NGINXController) testTemplate(cfg []byte) error {
306344
return err
307345
}
308346
out, err := exec.Command(n.binary, "-t", "-c", tmpfile.Name()).CombinedOutput()
309-
if err != nil {
347+
if err != nil && err.Error() != errNoChild {
310348
// this error is different from the rest because it must be clear why nginx is not working
311349
oe := fmt.Sprintf(`
312350
-------------------------------------------------------------------------------
@@ -324,6 +362,20 @@ Error: %v
324362
// SetConfig sets the configured configmap
325363
func (n *NGINXController) SetConfig(cmap *api_v1.ConfigMap) {
326364
n.configmap = cmap
365+
366+
n.isProxyProtocolEnabled = false
367+
if cmap == nil {
368+
return
369+
}
370+
371+
val, ok := cmap.Data["use-proxy-protocol"]
372+
if ok {
373+
b, err := strconv.ParseBool(val)
374+
if err == nil {
375+
n.isProxyProtocolEnabled = b
376+
return
377+
}
378+
}
327379
}
328380

329381
// SetListers sets the configured store listers in the generic ingress controller
@@ -446,6 +498,39 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er
446498
return nil, err
447499
}
448500

501+
servers := []*server{}
502+
for _, pb := range ingressCfg.PassthroughBackends {
503+
svc := pb.Service
504+
if svc == nil {
505+
glog.Warningf("missing service for PassthroughBackends %v", pb.Backend)
506+
continue
507+
}
508+
port, err := strconv.Atoi(pb.Port.String())
509+
if err != nil {
510+
for _, sp := range svc.Spec.Ports {
511+
if sp.Name == pb.Port.String() {
512+
port = int(sp.Port)
513+
break
514+
}
515+
}
516+
} else {
517+
for _, sp := range svc.Spec.Ports {
518+
if sp.Port == int32(port) {
519+
port = int(sp.Port)
520+
break
521+
}
522+
}
523+
}
524+
525+
servers = append(servers, &server{
526+
Hostname: pb.Hostname,
527+
IP: svc.Spec.ClusterIP,
528+
Port: port,
529+
})
530+
}
531+
532+
n.proxy.ServerList = servers
533+
449534
return content, nil
450535
}
451536

‎controllers/nginx/pkg/cmd/controller/tcp.go

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"io"
6+
"net"
7+
8+
"github.com/golang/glog"
9+
"github.com/paultag/sniff/parser"
10+
)
11+
12+
type server struct {
13+
Hostname string
14+
IP string
15+
Port int
16+
}
17+
18+
type proxy struct {
19+
ServerList []*server
20+
Default *server
21+
}
22+
23+
func (p *proxy) Get(host string) *server {
24+
for _, s := range p.ServerList {
25+
if s.Hostname == host {
26+
return s
27+
}
28+
}
29+
30+
return &server{
31+
Hostname: "localhost",
32+
IP: "127.0.0.1",
33+
Port: 442,
34+
}
35+
}
36+
37+
func (p *proxy) Handle(conn net.Conn) {
38+
defer conn.Close()
39+
data := make([]byte, 4096)
40+
41+
length, err := conn.Read(data)
42+
if err != nil {
43+
glog.V(4).Infof("error reading the first 4k of the connection: %s", err)
44+
return
45+
}
46+
47+
var proxy *server
48+
hostname, err := parser.GetHostname(data[:])
49+
if err == nil {
50+
glog.V(3).Infof("parsed hostname from TLS Client Hello: %s", hostname)
51+
proxy = p.Get(hostname)
52+
if proxy == nil {
53+
return
54+
}
55+
} else {
56+
proxy = p.Default
57+
if proxy == nil {
58+
return
59+
}
60+
}
61+
62+
clientConn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", proxy.IP, proxy.Port))
63+
if err != nil {
64+
return
65+
}
66+
defer clientConn.Close()
67+
68+
_, err = clientConn.Write(data[:length])
69+
if err != nil {
70+
clientConn.Close()
71+
}
72+
pipe(clientConn, conn)
73+
}
74+
75+
func pipe(client, server net.Conn) {
76+
doCopy := func(s, c net.Conn, cancel chan<- bool) {
77+
io.Copy(s, c)
78+
cancel <- true
79+
}
80+
81+
cancel := make(chan bool, 2)
82+
83+
go doCopy(server, client, cancel)
84+
go doCopy(client, server, cancel)
85+
86+
select {
87+
case <-cancel:
88+
return
89+
}
90+
}

‎controllers/nginx/pkg/config/config.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ const (
5050

5151
logFormatUpstream = `%v - [$proxy_add_x_forwarded_for] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status`
5252

53-
logFormatStream = `[$time_local] $protocol [$ssl_preread_server_name] [$stream_upstream] $status $bytes_sent $bytes_received $session_time`
53+
logFormatStream = `[$time_local] $protocol $status $bytes_sent $bytes_received $session_time`
5454

5555
// http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_buffer_size
5656
// Sets the size of the buffer used for sending data.

‎controllers/nginx/pkg/template/template.go

+15-44
Original file line numberDiff line numberDiff line change
@@ -130,21 +130,20 @@ var (
130130
}
131131
return true
132132
},
133-
"buildLocation": buildLocation,
134-
"buildAuthLocation": buildAuthLocation,
135-
"buildAuthResponseHeaders": buildAuthResponseHeaders,
136-
"buildProxyPass": buildProxyPass,
137-
"buildRateLimitZones": buildRateLimitZones,
138-
"buildRateLimit": buildRateLimit,
139-
"buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams,
140-
"buildResolvers": buildResolvers,
141-
"isLocationAllowed": isLocationAllowed,
142-
"buildLogFormatUpstream": buildLogFormatUpstream,
143-
"contains": strings.Contains,
144-
"hasPrefix": strings.HasPrefix,
145-
"hasSuffix": strings.HasSuffix,
146-
"toUpper": strings.ToUpper,
147-
"toLower": strings.ToLower,
133+
"buildLocation": buildLocation,
134+
"buildAuthLocation": buildAuthLocation,
135+
"buildAuthResponseHeaders": buildAuthResponseHeaders,
136+
"buildProxyPass": buildProxyPass,
137+
"buildRateLimitZones": buildRateLimitZones,
138+
"buildRateLimit": buildRateLimit,
139+
"buildResolvers": buildResolvers,
140+
"isLocationAllowed": isLocationAllowed,
141+
"buildLogFormatUpstream": buildLogFormatUpstream,
142+
"contains": strings.Contains,
143+
"hasPrefix": strings.HasPrefix,
144+
"hasSuffix": strings.HasSuffix,
145+
"toUpper": strings.ToUpper,
146+
"toLower": strings.ToLower,
148147
}
149148
)
150149

@@ -169,34 +168,6 @@ func buildResolvers(a interface{}) string {
169168
return strings.Join(r, " ")
170169
}
171170

172-
func buildSSLPassthroughUpstreams(b interface{}, sslb interface{}) string {
173-
backends := b.([]*ingress.Backend)
174-
sslBackends := sslb.([]*ingress.SSLPassthroughBackend)
175-
buf := bytes.NewBuffer(make([]byte, 0, 10))
176-
177-
// multiple services can use the same upstream.
178-
// avoid duplications using a map[name]=true
179-
u := make(map[string]bool)
180-
for _, passthrough := range sslBackends {
181-
if u[passthrough.Backend] {
182-
continue
183-
}
184-
u[passthrough.Backend] = true
185-
fmt.Fprintf(buf, "upstream %v {\n", passthrough.Backend)
186-
for _, backend := range backends {
187-
if backend.Name == passthrough.Backend {
188-
for _, server := range backend.Endpoints {
189-
fmt.Fprintf(buf, "\t\tserver %v:%v;\n", server.Address, server.Port)
190-
}
191-
break
192-
}
193-
}
194-
fmt.Fprint(buf, "\t}\n\n")
195-
}
196-
197-
return buf.String()
198-
}
199-
200171
// buildLocation produces the location string, if the ingress has redirects
201172
// (specified through the ingress.kubernetes.io/rewrite-to annotation)
202173
func buildLocation(input interface{}) string {
@@ -283,7 +254,7 @@ func buildProxyPass(b interface{}, loc interface{}) string {
283254

284255
for _, backend := range backends {
285256
if backend.Name == location.Backend {
286-
if backend.Secure {
257+
if backend.Secure || backend.SSLPassthrough {
287258
proto = "https"
288259
}
289260
break

0 commit comments

Comments
 (0)