Merge pull request #916 from weaveworks/854-reports-codec

Improve codec performance

.gitignore

@@ -55,3 +55,5 @@ experimental/_integration/_integration
 *npm-debug.log
 app/static.go
 prog/static.go
+vendor/github.com/2opremio/go-1/codec/codecgen/codecgen
+*.codecgen.go

Makefile

@@ -15,12 +15,17 @@ DOCKER_VERSION=1.6.2
 DOCKER_DISTRIB=.pkg/docker-$(DOCKER_VERSION).tgz
 DOCKER_DISTRIB_URL=https://get.docker.com/builds/Linux/x86_64/docker-$(DOCKER_VERSION).tgz
 RUNSVINIT=vendor/runsvinit/runsvinit
+CODECGEN_DIR=vendor/github.com/2opremio/go-1/codec/codecgen
+CODECGEN_EXE=$(CODECGEN_DIR)/codecgen
+GET_CODECGEN_DEPS=$(shell find $(1) -maxdepth 1 -type f -name '*.go' -not -name '*_test.go' -not -name '*.codecgen.go' -not -name '*.generated.go')
+CODECGEN_TARGETS=report/report.codecgen.go render/render.codecgen.go render/detailed/detailed.codecgen.go
 RM=--rm
 RUN_FLAGS=-ti
 BUILD_IN_CONTAINER=true
 GO ?= env GO15VENDOREXPERIMENT=1 go
 GO_BUILD_INSTALL_DEPS=-i
-GO_BUILD_FLAGS=$(GO_BUILD_INSTALL_DEPS) -ldflags "-extldflags \"-static\" -X main.version=$(SCOPE_VERSION)" -tags netgo
+GO_BUILD_TAGS=-tags netgo
+GO_BUILD_FLAGS=$(GO_BUILD_INSTALL_DEPS) -ldflags "-extldflags \"-static\" -X main.version=$(SCOPE_VERSION)" $(GO_BUILD_TAGS)
 
 all: $(SCOPE_EXPORT)
 
@@ -41,6 +46,7 @@ $(RUNSVINIT): vendor/runsvinit/*.go
 
 $(SCOPE_EXE): $(shell find ./ -path ./vendor -prune -o -type f -name *.go) prog/static.go
 
+
 ifeq ($(BUILD_IN_CONTAINER),true)
 
 $(SCOPE_EXE) $(RUNSVINIT) lint tests shell: $(SCOPE_BACKEND_BUILD_UPTODATE)
@@ -55,7 +61,7 @@ $(SCOPE_EXE) $(RUNSVINIT) lint tests shell: $(SCOPE_BACKEND_BUILD_UPTODATE)
 
 else
 
-$(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE)
+$(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE) $(CODECGEN_TARGETS)
 	time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D)
 	@strings $@ | grep cgo_stub\\\.go >/dev/null || { \
 	        rm $@; \
@@ -66,6 +72,15 @@ $(SCOPE_EXE): $(SCOPE_BACKEND_BUILD_UPTODATE)
 	        false; \
 	    }
 
+report/report.codecgen.go: $(call GET_CODECGEN_DEPS,report/)
+render/render.codecgen.go: $(call GET_CODECGEN_DEPS,render/)
+render/detailed/detailed.codecgen.go: $(call GET_CODECGEN_DEPS,render/detailed/)
+%.codecgen.go: $(CODECGEN_EXE)
+	cd $(@D) && env -u GOARCH -u GOOS $(shell pwd)/$(CODECGEN_EXE) -u -o $(@F) $(notdir $(filter-out $<,$^))
+
+$(CODECGEN_EXE): $(CODECGEN_DIR)/*.go
+	env -u GOARCH -u GOOS $(GO) build $(GO_BUILD_TAGS) -o $@ ./$(@D)
+
 $(RUNSVINIT):
 	time $(GO) build $(GO_BUILD_FLAGS) -o $@ ./$(@D)
 
@@ -123,10 +138,11 @@ clean:
 	$(GO) clean ./...
 	$(SUDO) docker rmi $(SCOPE_UI_BUILD_IMAGE) $(SCOPE_BACKEND_BUILD_IMAGE) >/dev/null 2>&1 || true
 	rm -rf $(SCOPE_EXPORT) $(SCOPE_UI_BUILD_UPTODATE) $(SCOPE_BACKEND_BUILD_UPTODATE) \
-		$(SCOPE_EXE) $(RUNSVINIT) prog/static.go client/build/app.js docker/weave .pkg
+		$(SCOPE_EXE) $(RUNSVINIT) prog/static.go client/build/app.js docker/weave .pkg \
+		$(CODECGEN_TARGETS) $(CODECGEN_EXE)
 
 deps:
-	$(GO) get -u -f -tags netgo \
+	$(GO) get -u -f $(GO_BUILD_TAGS) \
 		github.com/FiloSottile/gvt \
 		github.com/mattn/goveralls \
 		github.com/mjibson/esc \

app/api_report_test.go

@@ -1,11 +1,11 @@
 package app_test
 
 import (
-	"encoding/json"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/gorilla/mux"
+	"github.com/ugorji/go/codec"
 
 	"github.com/weaveworks/scope/app"
 	"github.com/weaveworks/scope/report"
@@ -25,8 +25,9 @@ func TestAPIReport(t *testing.T) {
 	var body = getRawJSON(t, ts, "/api/report")
 	// fmt.Printf("Body: %v\n", string(body))
 	var r report.Report
-	err := json.Unmarshal(body, &r)
-	if err != nil {
+
+	decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+	if err := decoder.Decode(&r); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 }

app/api_topologies_test.go

@@ -2,13 +2,12 @@ package app_test
 
 import (
 	"bytes"
-	"encoding/gob"
-	"encoding/json"
 	"net/http/httptest"
 	"testing"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/ugorji/go/codec"
 	"k8s.io/kubernetes/pkg/api"
 
 	"github.com/weaveworks/scope/app"
@@ -24,7 +23,8 @@ func TestAPITopology(t *testing.T) {
 	body := getRawJSON(t, ts, "/api/topology")
 
 	var topologies []app.APITopologyDesc
-	if err := json.Unmarshal(body, &topologies); err != nil {
+	decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+	if err := decoder.Decode(&topologies); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 	equals(t, 3, len(topologies))
@@ -61,7 +61,8 @@ func TestAPITopologyAddsKubernetes(t *testing.T) {
 	body := getRawJSON(t, ts, "/api/topology")
 
 	var topologies []app.APITopologyDesc
-	if err := json.Unmarshal(body, &topologies); err != nil {
+	decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+	if err := decoder.Decode(&topologies); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 	equals(t, 3, len(topologies))
@@ -84,13 +85,15 @@ func TestAPITopologyAddsKubernetes(t *testing.T) {
 		},
 	}).GetNode()
 	buf := &bytes.Buffer{}
-	if err := gob.NewEncoder(buf).Encode(rpt); err != nil {
+	encoder := codec.NewEncoder(buf, &codec.MsgpackHandle{})
+	if err := encoder.Encode(rpt); err != nil {
 		t.Fatalf("GOB encoding error: %s", err)
 	}
 	checkRequest(t, ts, "POST", "/api/report", buf.Bytes())
 
 	body = getRawJSON(t, ts, "/api/topology")
-	if err := json.Unmarshal(body, &topologies); err != nil {
+	decoder = codec.NewDecoderBytes(body, &codec.JsonHandle{})
+	if err := decoder.Decode(&topologies); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 	equals(t, 4, len(topologies))

app/api_topology.go

@@ -4,8 +4,10 @@ import (
 	"net/http"
 	"time"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/websocket"
 
+	"github.com/weaveworks/scope/common/xfer"
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/render/detailed"
 )
@@ -108,7 +110,9 @@ func handleWebsocket(
 		if err := conn.SetWriteDeadline(time.Now().Add(websocketTimeout)); err != nil {
 			return
 		}
-		if err := conn.WriteJSON(diff); err != nil {
+
+		if err := xfer.WriteJSONtoWS(conn, diff); err != nil {
+			log.Errorf("cannot serialize topology diff: %s", err)
 			return
 		}
 

app/api_topology_test.go

@@ -1,13 +1,13 @@
 package app_test
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/url"
 	"reflect"
 	"testing"
 
 	"github.com/gorilla/websocket"
+	"github.com/ugorji/go/codec"
 
 	"github.com/weaveworks/scope/app"
 	"github.com/weaveworks/scope/render"
@@ -22,21 +22,24 @@ func TestAll(t *testing.T) {
 
 	body := getRawJSON(t, ts, "/api/topology")
 	var topologies []app.APITopologyDesc
-	if err := json.Unmarshal(body, &topologies); err != nil {
+	decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+	if err := decoder.Decode(&topologies); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 
 	getTopology := func(topologyURL string) {
 		body := getRawJSON(t, ts, topologyURL)
 		var topology app.APITopology
-		if err := json.Unmarshal(body, &topology); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&topology); err != nil {
 			t.Fatalf("JSON parse error: %s", err)
 		}
 
 		for _, node := range topology.Nodes {
 			body := getRawJSON(t, ts, fmt.Sprintf("%s/%s", topologyURL, url.QueryEscape(node.ID)))
 			var node app.APINode
-			if err := json.Unmarshal(body, &node); err != nil {
+			decoder = codec.NewDecoderBytes(body, &codec.JsonHandle{})
+			if err := decoder.Decode(&node); err != nil {
 				t.Fatalf("JSON parse error: %s", err)
 			}
 		}
@@ -56,7 +59,8 @@ func TestAPITopologyContainers(t *testing.T) {
 	{
 		body := getRawJSON(t, ts, "/api/topology/containers")
 		var topo app.APITopology
-		if err := json.Unmarshal(body, &topo); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&topo); err != nil {
 			t.Fatal(err)
 		}
 		want := expected.RenderedContainers.Copy()
@@ -78,7 +82,8 @@ func TestAPITopologyProcesses(t *testing.T) {
 	{
 		body := getRawJSON(t, ts, "/api/topology/processes/"+expected.ServerProcessID)
 		var node app.APINode
-		if err := json.Unmarshal(body, &node); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&node); err != nil {
 			t.Fatal(err)
 		}
 		equals(t, expected.ServerProcessID, node.Node.ID)
@@ -91,7 +96,8 @@ func TestAPITopologyProcesses(t *testing.T) {
 		body := getRawJSON(t, ts, "/api/topology/processes-by-name/"+
 			url.QueryEscape(fixture.Client1Name))
 		var node app.APINode
-		if err := json.Unmarshal(body, &node); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&node); err != nil {
 			t.Fatal(err)
 		}
 		equals(t, fixture.Client1Name, node.Node.ID)
@@ -108,7 +114,8 @@ func TestAPITopologyHosts(t *testing.T) {
 	{
 		body := getRawJSON(t, ts, "/api/topology/hosts")
 		var topo app.APITopology
-		if err := json.Unmarshal(body, &topo); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&topo); err != nil {
 			t.Fatal(err)
 		}
 
@@ -119,7 +126,8 @@ func TestAPITopologyHosts(t *testing.T) {
 	{
 		body := getRawJSON(t, ts, "/api/topology/hosts/"+expected.ServerHostRenderedID)
 		var node app.APINode
-		if err := json.Unmarshal(body, &node); err != nil {
+		decoder := codec.NewDecoderBytes(body, &codec.JsonHandle{})
+		if err := decoder.Decode(&node); err != nil {
 			t.Fatal(err)
 		}
 		equals(t, expected.ServerHostRenderedID, node.Node.ID)
@@ -155,7 +163,8 @@ func TestAPITopologyWebsocket(t *testing.T) {
 	_, p, err := ws.ReadMessage()
 	ok(t, err)
 	var d render.Diff
-	if err := json.Unmarshal(p, &d); err != nil {
+	decoder := codec.NewDecoderBytes(p, &codec.JsonHandle{})
+	if err := decoder.Decode(&d); err != nil {
 		t.Fatalf("JSON parse error: %s", err)
 	}
 	equals(t, 7, len(d.Add))

app/benchmark_internal_test.go

@@ -1,13 +1,14 @@
 package app
 
 import (
-	"encoding/json"
 	"flag"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"testing"
 
+	"github.com/ugorji/go/codec"
+
 	"github.com/weaveworks/scope/render"
 	"github.com/weaveworks/scope/report"
 	"github.com/weaveworks/scope/test/fixture"
@@ -34,7 +35,8 @@ func loadReport() (report.Report, error) {
 		return fixture.Report, err
 	}
 	rpt := report.MakeReport()
-	err = json.Unmarshal(b, &rpt)
+	decoder := codec.NewDecoderBytes(b, &codec.JsonHandle{})
+	err = decoder.Decode(&rpt)
 	return rpt, err
 }
 

app/controls_test.go

@@ -1,7 +1,6 @@
 package app_test
 
 import (
-	"encoding/json"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -10,6 +9,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/ugorji/go/codec"
 
 	"github.com/weaveworks/scope/app"
 	"github.com/weaveworks/scope/common/xfer"
@@ -59,9 +59,11 @@ func TestControl(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer resp.Body.Close()
 
 	var response xfer.Response
-	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+	decoder := codec.NewDecoder(resp.Body, &codec.JsonHandle{})
+	if err := decoder.Decode(&response); err != nil {
 		t.Fatal(err)
 	}
 

app/router.go

@@ -3,13 +3,13 @@ package app
 import (
 	"compress/gzip"
 	"encoding/gob"
-	"encoding/json"
 	"net/http"
 	"net/url"
 	"strings"
 
 	"github.com/PuerkitoBio/ghost/handlers"
 	"github.com/gorilla/mux"
+	"github.com/ugorji/go/codec"
 
 	"github.com/weaveworks/scope/common/hostname"
 	"github.com/weaveworks/scope/common/xfer"
@@ -133,8 +133,11 @@ func RegisterReportPostHandler(a Adder, router *mux.Router) {
 
 		decoder := gob.NewDecoder(reader).Decode
 		if strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
-			decoder = json.NewDecoder(reader).Decode
+			decoder = codec.NewDecoder(reader, &codec.JsonHandle{}).Decode
+		} else if strings.HasPrefix(r.Header.Get("Content-Type"), "application/msgpack") {
+			decoder = codec.NewDecoder(reader, &codec.MsgpackHandle{}).Decode
 		}
+
 		if err := decoder(&rpt); err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
 			return

app/router_test.go

@@ -3,7 +3,6 @@ package app_test
 import (
 	"bytes"
 	"encoding/gob"
-	"encoding/json"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -12,6 +11,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/ugorji/go/codec"
 
 	"github.com/weaveworks/scope/app"
 	"github.com/weaveworks/scope/test"
@@ -83,5 +83,14 @@ func TestReportPostHandler(t *testing.T) {
 		err := gob.NewEncoder(buf).Encode(v)
 		return buf.Bytes(), err
 	})
-	test("application/json", json.Marshal)
+	test("application/json", func(v interface{}) ([]byte, error) {
+		buf := &bytes.Buffer{}
+		err := codec.NewEncoder(buf, &codec.JsonHandle{}).Encode(v)
+		return buf.Bytes(), err
+	})
+	test("application/msgpack", func(v interface{}) ([]byte, error) {
+		buf := &bytes.Buffer{}
+		err := codec.NewEncoder(buf, &codec.MsgpackHandle{}).Encode(v)
+		return buf.Bytes(), err
+	})
 }

app/scope_test.go

@@ -52,6 +52,7 @@ func checkRequest(t *testing.T, ts *httptest.Server, method, path string, body [
 	if err != nil {
 		t.Fatalf("Error getting %s: %s %s", method, path, err)
 	}
+	req.Header.Set("Content-Type", "application/msgpack")
 
 	client := &http.Client{}
 	res, err := client.Do(req)