diff --git a/assert/assert.go b/assert/assert.go new file mode 100644 index 000000000..ad574dc79 --- /dev/null +++ b/assert/assert.go @@ -0,0 +1,25 @@ +// Package assert provides a simple assert framework. +package assert + +import ( + "fmt" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// Equal provides an assertEqual function +func Equal(t *testing.T) func(got, want interface{}) { + return EqualDepth(t, 1, "") +} + +func EqualDepth(t *testing.T, calldepth int, desc string) func(got, want interface{}) { + return func(got, want interface{}) { + _, file, line, _ := runtime.Caller(calldepth) + if !reflect.DeepEqual(got, want) { + fmt.Printf("\t%s:%d: %s: got %v want %v\n", filepath.Base(file), line, desc, got, want) + t.Fail() + } + } +} diff --git a/config/config.go b/config/config.go index 03c14b3aa..0dd44ebd9 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "net/http" + "regexp" "time" ) @@ -60,6 +61,8 @@ type Proxy struct { ClientIPHeader string TLSHeader string TLSHeaderValue string + GZIPContentTypesValue string + GZIPContentTypes *regexp.Regexp } type Runtime struct { diff --git a/config/load.go b/config/load.go index f248c8124..9c2a64c51 100644 --- a/config/load.go +++ b/config/load.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "os" + "regexp" "runtime" "strings" "time" @@ -100,6 +101,7 @@ func load(p *properties.Properties) (cfg *Config, err error) { f.StringVar(&cfg.Proxy.ClientIPHeader, "proxy.header.clientip", Default.Proxy.ClientIPHeader, "header for the request ip") f.StringVar(&cfg.Proxy.TLSHeader, "proxy.header.tls", Default.Proxy.TLSHeader, "header for TLS connections") f.StringVar(&cfg.Proxy.TLSHeaderValue, "proxy.header.tls.value", Default.Proxy.TLSHeaderValue, "value for TLS connection header") + f.StringVar(&cfg.Proxy.GZIPContentTypesValue, "proxy.gzip.contenttype", Default.Proxy.GZIPContentTypesValue, "regexp of content types to compress") f.StringSliceVar(&cfg.ListenerValue, "proxy.addr", Default.ListenerValue, "listener config") f.KVSliceVar(&cfg.CertSourcesValue, "proxy.cs", Default.CertSourcesValue, "certificate sources") f.DurationVar(&cfg.Proxy.ReadTimeout, "proxy.readtimeout", Default.Proxy.ReadTimeout, "read timeout for incoming requests") @@ -171,6 +173,13 @@ func load(p *properties.Properties) (cfg *Config, err error) { return nil, err } + if cfg.Proxy.GZIPContentTypesValue != "" { + cfg.Proxy.GZIPContentTypes, err = regexp.Compile(cfg.Proxy.GZIPContentTypesValue) + if err != nil { + return nil, fmt.Errorf("invalid expression for content types: %s", err) + } + } + // handle deprecations // deprecate := func(name, msg string) { // if f.IsSet(name) { diff --git a/config/load_test.go b/config/load_test.go index 3466ae972..782664e60 100644 --- a/config/load_test.go +++ b/config/load_test.go @@ -3,6 +3,7 @@ package config import ( "net/http" "reflect" + "regexp" "testing" "time" @@ -29,6 +30,7 @@ proxy.maxconn = 666 proxy.header.clientip = clientip proxy.header.tls = tls proxy.header.tls.value = tls-true +proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\\+(json|xml))$ registry.backend = something registry.file.path = /foo/bar registry.static.routes = route add svc / http://127.0.0.1:6666/ @@ -91,6 +93,8 @@ aws.apigw.cert.cn = furb ClientIPHeader: "clientip", TLSHeader: "tls", TLSHeaderValue: "tls-true", + GZIPContentTypesValue: `^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`, + GZIPContentTypes: regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))$`), }, Registry: Registry{ Backend: "something", diff --git a/fabio.properties b/fabio.properties index ce951a8a2..f7993215f 100644 --- a/fabio.properties +++ b/fabio.properties @@ -337,6 +337,8 @@ # # The remoteIP is taken from http.Request.RemoteAddr. # +# The default is +# # proxy.header.clientip = @@ -351,6 +353,24 @@ # proxy.header.tls.value = +# proxy.gzip.contenttype configures which responses should be compressed. +# +# By default, responses sent to the client are not compressed even if the +# client accepts compressed responses by setting the 'Accept-Encoding: gzip' +# header. By setting this value responses are compressed if the Content-Type +# header of the response matches and the response is not already compressed. +# The list of compressable content types is defined as a regular expression. +# The regular expression must follow the rules outlined in golang.org/pkg/regexp. +# +# A typical example is +# +# proxy.gzip.contenttype = ^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))(;.*)?$ +# +# The default is +# +# proxy.gzip.contenttype = + + # registry.backend configures which backend is used. # Supported backends are: consul, static, file # diff --git a/proxy/gzip/content_type_test.go b/proxy/gzip/content_type_test.go new file mode 100644 index 000000000..9b1aaafb5 --- /dev/null +++ b/proxy/gzip/content_type_test.go @@ -0,0 +1,35 @@ +package gzip + +import "testing" + +// TestContentTypes tests the content-type regexp that is used as +// an example in fabio.properties +func TestContentTypes(t *testing.T) { + tests := []string{ + "text/foo", + "text/foo; charset=UTF-8", + "text/plain", + "text/plain; charset=UTF-8", + "application/json", + "application/json; charset=UTF-8", + "application/javascript", + "application/javascript; charset=UTF-8", + "application/font-woff", + "application/font-woff; charset=UTF-8", + "application/xml", + "application/xml; charset=UTF-8", + "vendor/vendor.foo+json", + "vendor/vendor.foo+json; charset=UTF-8", + "vendor/vendor.foo+xml", + "vendor/vendor.foo+xml; charset=UTF-8", + } + + for _, tt := range tests { + tt := tt // capture loop var + t.Run(tt, func(t *testing.T) { + if !contentTypes.MatchString(tt) { + t.Fatalf("%q does not match content types regexp", tt) + } + }) + } +} diff --git a/proxy/gzip/gzip_handler.go b/proxy/gzip/gzip_handler.go new file mode 100644 index 000000000..046eb5b14 --- /dev/null +++ b/proxy/gzip/gzip_handler.go @@ -0,0 +1,104 @@ +// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed + +// Package gzip provides an HTTP handler which compresses responses +// if the client supports this, the response is compressable and +// not already compressed. +// +// Based on https://github.com/smancke/handler/gzip +package gzip + +import ( + "compress/gzip" + "io" + "net/http" + "regexp" + "strings" + "sync" +) + +const ( + headerVary = "Vary" + headerAcceptEncoding = "Accept-Encoding" + headerContentEncoding = "Content-Encoding" + headerContentType = "Content-Type" + headerContentLength = "Content-Length" + encodingGzip = "gzip" +) + +var gzipWriterPool = sync.Pool{ + New: func() interface{} { return gzip.NewWriter(nil) }, +} + +// NewGzipHandler wraps an existing handler to transparently gzip the response +// body if the client supports it (via the Accept-Encoding header) and the +// response Content-Type matches the contentTypes expression. +func NewGzipHandler(h http.Handler, contentTypes *regexp.Regexp) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add(headerVary, headerAcceptEncoding) + + if acceptsGzip(r) { + gzWriter := NewGzipResponseWriter(w, contentTypes) + defer gzWriter.Close() + h.ServeHTTP(gzWriter, r) + } else { + h.ServeHTTP(w, r) + } + }) +} + +type GzipResponseWriter struct { + writer io.Writer + gzipWriter *gzip.Writer + contentTypes *regexp.Regexp + http.ResponseWriter +} + +func NewGzipResponseWriter(w http.ResponseWriter, contentTypes *regexp.Regexp) *GzipResponseWriter { + return &GzipResponseWriter{ResponseWriter: w, contentTypes: contentTypes} +} + +func (grw *GzipResponseWriter) WriteHeader(code int) { + if grw.writer == nil { + if isCompressable(grw.Header(), grw.contentTypes) { + grw.Header().Del(headerContentLength) + grw.Header().Set(headerContentEncoding, encodingGzip) + grw.gzipWriter = gzipWriterPool.Get().(*gzip.Writer) + grw.gzipWriter.Reset(grw.ResponseWriter) + + grw.writer = grw.gzipWriter + } else { + grw.writer = grw.ResponseWriter + } + } + grw.ResponseWriter.WriteHeader(code) +} + +func (grw *GzipResponseWriter) Write(b []byte) (int, error) { + if grw.writer == nil { + if _, ok := grw.Header()[headerContentType]; !ok { + // Set content-type if not present. Otherwise golang would make application/gzip out of that. + grw.Header().Set(headerContentType, http.DetectContentType(b)) + } + grw.WriteHeader(http.StatusOK) + } + return grw.writer.Write(b) +} + +func (grw *GzipResponseWriter) Close() { + if grw.gzipWriter != nil { + grw.gzipWriter.Close() + gzipWriterPool.Put(grw.gzipWriter) + } +} + +func isCompressable(header http.Header, contentTypes *regexp.Regexp) bool { + // don't compress if it is already encoded + if header.Get(headerContentEncoding) != "" { + return false + } + return contentTypes.MatchString(header.Get(headerContentType)) +} + +func acceptsGzip(r *http.Request) bool { + return strings.Contains(r.Header.Get(headerAcceptEncoding), encodingGzip) +} diff --git a/proxy/gzip/gzip_handler_test.go b/proxy/gzip/gzip_handler_test.go new file mode 100644 index 000000000..c3dde0214 --- /dev/null +++ b/proxy/gzip/gzip_handler_test.go @@ -0,0 +1,134 @@ +// Copyright (c) 2016 Sebastian Mancke and eBay, both MIT licensed +package gzip + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + "net/http" + "net/http/httptest" + "regexp" + "strconv" + "testing" + + "github.com/eBay/fabio/assert" +) + +var contentTypes = regexp.MustCompile(`^(text/.*|application/(javascript|json|font-woff|xml)|.*\+(json|xml))(;.*)?$`) + +func Test_GzipHandler_CompressableType(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Type"), "text/plain; charset=utf-8") + assertEqual(resp.Header.Get("Content-Encoding"), "gzip") + + gzBytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + assertEqual(resp.Header.Get("Content-Length"), strconv.Itoa(len(gzBytes))) + + reader, err := gzip.NewReader(bytes.NewBuffer(gzBytes)) + assertEqual(err, nil) + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_NotCompressingTwice(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_already_compressed_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "gzip") + + reader, err := gzip.NewReader(resp.Body) + assertEqual(err, nil) + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_CompressableType_NoAccept(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_text_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "none") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "") + + bytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + + assertEqual(string(bytes), "Hello World") +} + +func Test_GzipHandler_NonCompressableType(t *testing.T) { + server := httptest.NewServer(NewGzipHandler(test_binary_handler(), contentTypes)) + + assertEqual := assert.Equal(t) + + r, err := http.NewRequest("GET", server.URL, nil) + assertEqual(err, nil) + r.Header.Set("Accept-Encoding", "gzip") + + resp, err := http.DefaultClient.Do(r) + assertEqual(err, nil) + + assertEqual(resp.Header.Get("Content-Encoding"), "") + + bytes, err := ioutil.ReadAll(resp.Body) + assertEqual(err, nil) + + assertEqual(bytes, []byte{42}) +} + +func test_text_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b := []byte("Hello World") + w.Header().Set("Content-Length", strconv.Itoa(len(b))) + w.Write(b) + }) +} + +func test_binary_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/jpg") + w.Write([]byte{42}) + }) +} + +func test_already_compressed_handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Encoding", "gzip") + gzWriter := gzip.NewWriter(w) + gzWriter.Write([]byte("Hello World")) + gzWriter.Close() + }) +} diff --git a/proxy/proxy.go b/proxy/proxy.go index 58a59c24a..cd199258e 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -6,6 +6,7 @@ import ( "github.com/eBay/fabio/config" "github.com/eBay/fabio/metrics" + "github.com/eBay/fabio/proxy/gzip" ) // httpProxy is a dynamic reverse proxy for HTTP and HTTPS protocols. @@ -60,6 +61,10 @@ func (p *httpProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { h = newHTTPProxy(t.URL, p.tr, time.Duration(0)) } + if p.cfg.GZIPContentTypes != nil { + h = gzip.NewGzipHandler(h, p.cfg.GZIPContentTypes) + } + start := time.Now() h.ServeHTTP(w, r) p.requests.UpdateSince(start) diff --git a/proxy/proxy_integration_test.go b/proxy/proxy_integration_test.go index 47e94af89..a9d5e2764 100644 --- a/proxy/proxy_integration_test.go +++ b/proxy/proxy_integration_test.go @@ -1,10 +1,13 @@ package proxy import ( + "bytes" + "compress/gzip" "net" "net/http" "net/http/httptest" "net/url" + "regexp" "testing" "github.com/eBay/fabio/config" @@ -55,3 +58,115 @@ func TestProxyNoRouteStaus(t *testing.T) { t.Fatalf("got %d want %d", got, want) } } + +func TestProxyGzipHandler(t *testing.T) { + tests := []struct { + desc string + content http.HandlerFunc + acceptEncoding string + contentEncoding string + wantResponse []byte + }{ + { + desc: "plain body - compressed response", + content: plainHandler("text/plain"), + acceptEncoding: "gzip", + contentEncoding: "gzip", + wantResponse: gzipContent, + }, + { + desc: "plain body - compressed response (with charset)", + content: plainHandler("text/plain; charset=UTF-8"), + acceptEncoding: "gzip", + contentEncoding: "gzip", + wantResponse: gzipContent, + }, + { + desc: "compressed body - compressed response", + content: gzipHandler("text/plain; charset=UTF-8"), + acceptEncoding: "gzip", + contentEncoding: "gzip", + wantResponse: gzipContent, + }, + { + desc: "plain body - plain response", + content: plainHandler("text/plain"), + acceptEncoding: "", + contentEncoding: "", + wantResponse: plainContent, + }, + { + desc: "compressed body - plain response", + content: gzipHandler("text/plain"), + acceptEncoding: "", + contentEncoding: "", + wantResponse: plainContent, + }, + { + desc: "plain body - plain response (no match)", + content: plainHandler("text/javascript"), + acceptEncoding: "gzip", + contentEncoding: "", + wantResponse: plainContent, + }, + } + + for _, tt := range tests { + tt := tt // capture loop var + t.Run(tt.desc, func(t *testing.T) { + server := httptest.NewServer(tt.content) + defer server.Close() + + table := make(route.Table) + table.AddRoute("mock", "/", server.URL, 1, nil) + route.SetTable(table) + + tr := &http.Transport{Dial: (&net.Dialer{}).Dial} + proxy := NewHTTPProxy(tr, config.Proxy{GZIPContentTypes: regexp.MustCompile("^text/plain(;.*)?$")}) + req := &http.Request{RequestURI: "/", RemoteAddr: "2.2.2.2:2222", Header: http.Header{"Accept-Encoding": []string{tt.acceptEncoding}}, URL: &url.URL{}} + rec := httptest.NewRecorder() + proxy.ServeHTTP(rec, req) + + if got, want := rec.Code, 200; got != want { + t.Fatalf("got code %d want %d", got, want) + } + if got, want := rec.Header().Get("Content-Encoding"), tt.contentEncoding; got != want { + t.Errorf("got content-encoding %q want %q", got, want) + } + if got, want := rec.Body.Bytes(), tt.wantResponse; !bytes.Equal(got, want) { + t.Errorf("got body %q want %q", got, want) + } + }) + } +} + +var plainContent = []byte("Hello World") +var gzipContent = compress(plainContent) + +func plainHandler(contentType string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", contentType) + w.Write(plainContent) + } +} + +func gzipHandler(contentType string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Encoding", "gzip") + w.Write(gzipContent) + } +} + +// compress returns the gzip compressed content of b. +func compress(b []byte) []byte { + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write(b); err != nil { + panic(err) + } + if err := w.Close(); err != nil { + panic(err) + } + return buf.Bytes() +}