test: load testing tooling and initial observations (#225)

Signed-off-by: Kavindu Dodanduwa <[email protected]>

.github/workflows/build.yaml

@@ -89,6 +89,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
+          file: ./build.Dockerfile
           outputs: type=docker,dest=${{ github.workspace }}/flagd-local.tar
           tags: flagd-local:test
 

.github/workflows/release-please.yaml

@@ -63,7 +63,7 @@ jobs:
         with:
           builder: ${{ steps.buildx.outputs.name }}
           context: .
-          file: ./Dockerfile
+          file: ./build.Dockerfile
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |

Dockerfile → build.Dockerfile

@@ -1,3 +1,4 @@
+# Main Dockerfile for flagd builds
 # Build the manager binary
 FROM --platform=$BUILDPLATFORM golang:1.18-alpine AS builder
 

profile.Dockerfile

@@ -0,0 +1,34 @@
+# Dockerfile with pprof profiler
+# Build the manager binary
+FROM --platform=$BUILDPLATFORM golang:1.18-alpine AS builder
+
+WORKDIR /workspace
+ARG TARGETOS
+ARG TARGETARCH
+ARG VERSION
+ARG COMMIT
+ARG DATE
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+# cache deps before building and copying source so that we don't need to re-download as much
+# and so that source changes don't invalidate our downloaded layer
+RUN go mod download
+
+# Copy the go source
+COPY main.go main.go
+COPY profiler.go profiler.go
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+
+# Build with profiler
+RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build -a -ldflags "-X main.version=${VERSION} -X main.commit=${COMMIT} -X main.date=${DATE}" -o flagd main.go profiler.go
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /
+COPY --from=builder /workspace/flagd .
+USER 65532:65532
+
+ENTRYPOINT ["/flagd"]

profiler.go

@@ -0,0 +1,19 @@
+//go:build profile
+
+package main
+
+import (
+	"net/http"
+	_ "net/http/pprof"
+)
+
+/*
+Enable pprof profiler for flagd. Build controlled by the build tag "profile".
+*/
+func init() {
+	// Go routine to server PProf
+	go func() {
+		server := http.Server{Addr: ":6060", Handler: nil}
+		server.ListenAndServe()
+	}()
+}

tests/README.MD

@@ -0,0 +1,5 @@
+## Tests
+
+This folder contains testing resources for flagd.
+
+- [Load Tests](/tests/loadtest)

tests/loadtest/.gitignore

@@ -0,0 +1,2 @@
+# Ignore random FF jsons generated from ff_gen.go
+random.json

tests/loadtest/README.MD

@@ -0,0 +1,55 @@
+## Load Testing
+
+This folder contains resources for flagd load testing. 
+
+- ff_gen.go : simple, random feature flag generation utility.
+- sample_k6.js : sample K6 load test script
+
+### Profiling
+
+It's possible to utilize `profiler.go` included with flagd source to profile flagd during
+load test. Profiling is enabled through [go pprof package](https://pkg.go.dev/net/http/pprof).
+
+To enable pprof profiling, build a docker image with the `profile.Dockerfile`
+
+ex:- `docker build . -f ./profile.Dockerfile -t flagdprofile`
+
+This image now exposes port `6060` for pprof data.
+
+### Example test run
+
+First, let's create random feature flags using `ff_gen.go` utility. To generate 100 boolean feature flags,
+run the command
+
+`go run ff_gen.go -c 100 -t boolean`
+
+This command generates `random.json`in the same directory.
+
+Then, let's start pprof profiler enabled flagd docker container with newly generated feature flags.
+
+`docker run -p 8013:8013 -p 6060:6060 --rm -it -v $(pwd):/etc/flagd flagdprofile start --uri file:./etc/flagd/random.json`
+
+Finally, you can run the K6 test script to load test the flagd container.
+
+`k6 run sample_k6.js`
+
+To observe the pprof date, you can either visit [http://localhost:6060/debug/pprof/](http://localhost:6060/debug/pprof/)
+or use go pprof tool. Example tool usages are given below,  
+
+- Analyze heap in command line: `go tool pprof http://localhost:6060/debug/pprof/heap`
+- Analyze heap in UI mode: `go tool pprof --http=:9090  http://localhost:6060/debug/pprof/heap`
+
+### Performance observations
+
+flagd performs well under heavy loads. Consider the following results observed against the HTTP API of flagd,
+
+![](../../images/loadTestResults.png)
+
+flagd is able to serve ~20K HTTP requests/second with just 64MB memory and 1 CPU. And the impact of flag type
+is minimal. There was no memory pressure observed throughout the test runs.
+
+#### Note on observations
+
+Above observations were made on a single system. Hence, throughput does not account for network delays.
+Also, there were no background syncs or context evaluations performed.
+

tests/loadtest/ff_gen.go

@@ -0,0 +1,108 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+)
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+const (
+	BOOL   = "boolean"
+	STRING = "string"
+)
+
+/*
+A simple random feature flag generator for testing purposes. Output is saved to "random.json".
+
+Configurable options:
+
+	-c : feature flag count (ex:go run ff_gen.go -c 500)
+	-t : type of feature flag (ex:go run ff_gen.go -t string). Support "boolean" and "string"
+*/
+func main() {
+	// Get flag count
+	var flagCount int
+	flag.IntVar(&flagCount, "c", 100, "Number of flags to generate")
+
+	// Get flag type : Boolean, String
+	var flagType string
+	flag.StringVar(&flagType, "t", BOOL, "Type of flags to generate")
+
+	flag.Parse()
+
+	if flagType != STRING && flagType != BOOL {
+		fmt.Printf("Invalid type %s. Falling back to default %s", flagType, BOOL)
+		flagType = BOOL
+	}
+
+	root := Flags{}
+	root.Flags = make(map[string]Flag)
+
+	switch flagType {
+	case BOOL:
+		root.setBoolFlags(flagCount)
+	case STRING:
+		root.setStringFlags(flagCount)
+	}
+
+	bytes, err := json.Marshal(root)
+	if err != nil {
+		fmt.Printf("Json error: %s ", err.Error())
+		return
+	}
+
+	err = os.WriteFile("./random.json", bytes, 444)
+	if err != nil {
+		fmt.Printf("File write error: %s ", err.Error())
+		return
+	}
+}
+
+func (f *Flags) setBoolFlags(toGen int) {
+	for i := 0; i < toGen; i++ {
+		variant := make(map[string]any)
+		variant["on"] = true
+		variant["off"] = false
+
+		f.Flags[fmt.Sprintf("flag%d", i)] = Flag{
+			State:          "ENABLED",
+			DefaultVariant: randomSelect("on", "off"),
+			Variants:       variant,
+		}
+	}
+}
+
+func (f *Flags) setStringFlags(toGen int) {
+	for i := 0; i < toGen; i++ {
+		variant := make(map[string]any)
+		variant["key1"] = "value1"
+		variant["key2"] = "value2"
+
+		f.Flags[fmt.Sprintf("flag%d", i)] = Flag{
+			State:          "ENABLED",
+			DefaultVariant: randomSelect("key1", "key2"),
+			Variants:       variant,
+		}
+	}
+}
+
+type Flags struct {
+	Flags map[string]Flag `json:"flags"`
+}
+
+type Flag struct {
+	State          string         `json:"state"`
+	DefaultVariant string         `json:"defaultVariant"`
+	Variants       map[string]any `json:"variants"`
+}
+
+func randomSelect(chooseFrom ...string) string {
+	return chooseFrom[rand.Intn(len(chooseFrom))]
+}

tests/loadtest/go.mod

@@ -0,0 +1,3 @@
+module tests.loadtest
+
+go 1.19

tests/loadtest/sample_k6.js

@@ -0,0 +1,42 @@
+import http from 'k6/http';
+
+/*
+* Sample K6 (https://k6.io/) load test script
+* */
+
+// K6 options - Load generation pattern: Ramp up, hold and teardown
+export const options = {
+    stages: [{duration: '10s', target: 50}, {duration: '30s', target: 50}, {duration: '10s', target: 0},]
+}
+
+// Flag prefix - See ff_gen.go to match
+export const prefix = "flag"
+
+// Custom options : Number of FFs flagd serves and type of the FFs being served
+export const customOptions = {
+    ffCount: 100,
+    type: "boolean"
+}
+
+export default function () {
+    // Randomly select flag to evaluate
+    let flag = prefix + Math.floor((Math.random() * customOptions.ffCount))
+
+    let resp = http.post(genUrl(customOptions.type), JSON.stringify({
+        flagKey: flag, context: {}
+    }), {headers: {'Content-Type': 'application/json'}});
+
+    // Handle and report errors
+    if (resp.status !== 200) {
+        console.log("Error response - FlagId : " + flag + " Response :" + JSON.stringify(resp.body))
+    }
+}
+
+export function genUrl(type) {
+    switch (type) {
+        case "boolean":
+            return "http://localhost:8013/schema.v1.Service/ResolveBoolean"
+        case "string":
+            return "http://localhost:8013/schema.v1.Service/ResolveString"
+    }
+}