Build out sidecar infrastructure (#125)

* save work * more setup * pass valkey version * Fix build variables * embed versions * scan new dockerfiles * set package versions * set workdir instead of 'cd' * fix gosec, and image tag * make the linter happy * try adding trivy in scan * fix dockerfile arg * minor fix * test * set tags * test * Scan only valkey * missing : * force it * no metadata neeeded now * clean-up image builder * sidecar to be a cobra binary * fix misspelling and error check * fix valkey building * just need to test * eliminate need for bitnami image * appease the linter gods
hyperspike · Jan 8, 2025 · 3f4b318 · 3f4b318
1 parent cd325e4
commit 3f4b318
Show file tree

Hide file tree

Showing 23 changed files with 824 additions and 129 deletions.
diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml
@@ -13,6 +13,7 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  VALKEY_VERSION: 8.0.1
 
 jobs:
   build-and-push-image:
@@ -35,16 +36,26 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    - name: Extract metadata (tags, labels) for Docker
-      id: meta
+    - name: Extract metadata (Controller tags, labels) for Docker
+      id: meta_controller
       uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
       with:
         images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ env.RELEASE_VERSION }}
+    - name: Extract metadata (Sidecar tags, labels) for Docker
+      id: meta_sidecar
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+      with:
+        images: ${{ env.REGISTRY }}/hyperspike/valkey-sidecar:${{ env.RELEASE_VERSION }}
+    - name: Extract metadata (Valkey tags, labels) for Docker
+      id: meta_valkey
+      uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
+      with:
+        images: ${{ env.REGISTRY }}/hyperspike/valkey:${{ env.VALKEY_VERSION }}
 
     - name: Setup Go ${{ matrix.go-version }}
       uses: actions/setup-go@v5
       with:
-        go-version: 1.22
+        go-version: 1.23
     # You can test your matrix by printing the current Go version
     - name: Display Go version
       run: go version
@@ -54,20 +65,47 @@ jobs:
 
     - name: Build and push Docker image
       uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc
-      id: docker_build
+      id: docker_build_controller
+      with:
+        file: Dockerfile.controller
+        context: .
+        push: true
+        visibility: public
+        tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ env.RELEASE_VERSION }}
+        labels: ${{ steps.meta_manager.outputs.labels }}
+    - name: Build and push Sidecar image
+      uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
+      id: docker_build_sidecar
       with:
+        file: Dockerfile.sidecar
         context: .
         push: true
         visibility: public
-        tags: ${{ steps.meta.outputs.tags }}
-        labels: ${{ steps.meta.outputs.labels }}
+        tags: ${{ env.REGISTRY }}/hyperspike/valkey-sidecar:${{ env.RELEASE_VERSION }}
+        labels: ${{ steps.meta_sidecar.outputs.labels }}
+    - name: Build and push Valkey image
+      uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
+      id: docker_build_valkey
+      with:
+        file: Dockerfile.valkey
+        context: .
+        push: true
+        visibility: public
+        tags: ${{ env.REGISTRY }}/hyperspike/valkey:${{ env.VALKEY_VERSION }}
+        labels: ${{ steps.meta_valkey.outputs.labels }}
 
     - name: Set up Cosign
       uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3.7.0
 
-    - name: Sign image with GitHub OIDC Token
+    - name: Sign Controller image with GitHub OIDC Token
+      run: |
+        cosign sign --yes ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ env.RELEASE_VERSION }}@${{ steps.docker_build_controller.outputs.digest }}
+    - name: Sign Sidecar image with GitHub OIDC Token
+      run: |
+        cosign sign --yes ${{ env.REGISTRY }}/hyperspike/valkey-sidecar:${{ env.RELEASE_VERSION }}@${{ steps.docker_build_sidecar.outputs.digest }}
+    - name: Sign Valkey image with GitHub OIDC Token
       run: |
-        cosign sign --yes ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}${{ env.RELEASE_VERSION }}@${{ steps.docker_build.outputs.digest }}
+        cosign sign --yes ${{ env.REGISTRY }}/hyperspike/valkey:${{ env.VALKEY_VERSION }}@${{ steps.docker_build_valkey.outputs.digest }}
 
     - name: Attest
       uses: actions/attest-build-provenance@v2

diff --git a/.github/workflows/scan.yaml b/.github/workflows/scan.yaml
@@ -8,6 +8,9 @@ on:
     - main
   pull_request:
 
+env:
+  REGISTRY: ghcr.io
+
 permissions:
   contents: read
   security-events: write
@@ -20,7 +23,13 @@ jobs:
     - uses: actions/checkout@v4
     - uses: hadolint/[email protected]
       with:
-        dockerfile: Dockerfile
+        dockerfile: Dockerfile.valkey
+    - uses: hadolint/[email protected]
+      with:
+        dockerfile: Dockerfile.controller
+    - uses: hadolint/[email protected]
+      with:
+        dockerfile: Dockerfile.sidecar
   gosec:
     runs-on: ubuntu-latest
     steps:
@@ -68,3 +77,27 @@ jobs:
 
         # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
         # skip-build-cache: true
+  trivy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build the Valkey image
+      uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75
+      id: docker_build_valkey
+      with:
+        file: Dockerfile.valkey
+        context: .
+        push: false
+        tags:  ${{ env.REGISTRY }}/hyperspike/valkey:${{ github.SHA }}
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      with:
+        image-ref: ${{ env.REGISTRY }}/hyperspike/valkey:${{ github.SHA }}
+        format: 'sarif'
+        output: 'trivy-results.sarif'
+    - name: Upload Trivy scan results to GitHub Security tab
+      uses: github/codeql-action/upload-sarif@v3
+      if: always()
+      with:
+        sarif_file: trivy-results.sarif
diff --git a/.gitignore b/.gitignore
@@ -32,6 +32,7 @@ go.work
 .ingress.yaml
 blank.yaml
 cilium/
-manager
+/manager
+/sidecar
 valkey-operator/
 valkey-operator-*-chart.tgz
diff --git a/.golangci.yml b/.golangci.yml
@@ -39,3 +39,6 @@ linters:
     - unconvert
     - unparam
     - unused
+linters-settings:
+  lll:
+    line-length: 256
diff --git a/Dockerfile b/Dockerfile
diff --git a/Dockerfile.controller b/Dockerfile.controller
@@ -0,0 +1,5 @@
+FROM gcr.io/distroless/static:nonroot
+COPY manager /manager
+USER 65532:65532
+
+ENTRYPOINT ["/manager"]
diff --git a/Dockerfile.sidecar b/Dockerfile.sidecar
@@ -0,0 +1,6 @@
+FROM gcr.io/distroless/static:nonroot
+
+COPY sidecar /sidecar
+USER 65532:65532
+
+ENTRYPOINT ["/sidecar"]
diff --git a/Dockerfile.valkey b/Dockerfile.valkey
@@ -0,0 +1,35 @@
+FROM alpine:3.21.1 AS builder
+
+ARG VALKEY_VERSION=8.0.1
+
+WORKDIR /home/valkey
+
+RUN apk add --no-cache --virtual .build-deps \
+	git=2.47.1-r0 \
+	coreutils=9.5-r2 \
+	linux-headers=6.6-r1 \
+	musl-dev=1.2.5-r8 \
+	openssl-dev=3.3.2-r4 \
+	gcc=14.2.0-r4 \
+	curl=8.11.1-r0 \
+	make=4.4.1-r2 \
+	&& curl -L https://github.com/valkey-io/valkey/archive/refs/tags/${VALKEY_VERSION}.tar.gz -o valkey.tar.gz \
+	&& tar -xzf valkey.tar.gz --strip-components=1 \
+	&& make PREFIX=/usr BUILD_TLS=yes \
+	&& make install BUILD_TLS=yes PREFIX=/home/valkey/build
+
+FROM alpine:3.21.1 AS valkey
+
+RUN apk add --no-cache \
+	openssl=3.3.2-r4 \
+	ca-certificates=20241121-r1 \
+	&& addgroup -S valkey -g 1009 \
+	&& adduser -S -G valkey valkey -u 1009 \
+	&& mkdir /etc/valkey \
+	&& chown valkey:valkey /etc/valkey \
+	&& mkdir /var/lib/valkey \
+	&& chown valkey:valkey /var/lib/valkey
+
+COPY --from=builder /home/valkey/build/ /usr/
+
+USER valkey
diff --git a/Makefile b/Makefile
@@ -1,5 +1,8 @@
 # Image URL to use all building/pushing image targets
-IMG ?= controller:latest
+REGISTRY ?= ghcr.io/hyperspike
+IMG_CONTROLLER ?= $(REGISTRY)/valkey-operator:$(VERSION)
+IMG_SIDECAR ?= $(REGISTRY)/valkey-sidecar:$(VERSION)
+IMG_VALKEY ?= $(REGISTRY)/valkey:$(VALKEY_VERSION)
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -13,6 +16,7 @@ GO := $(shell which go)
 MINIKUBE := $(shell which minikube)
 KUBECTL := $(shell which kubectl)
 VERSION ?= $(shell  if [ ! -z $$(git tag --points-at HEAD) ] ; then git tag --points-at HEAD|cat ; else  git rev-parse --short HEAD|cat; fi )
+DATE ?= $(shell date -u  +'%Y%m%d')
 SHA ?= $(shell git rev-parse --short HEAD)
 PKG ?= hyperspike.io/valkey-operator
 
@@ -30,6 +34,7 @@ SHELL = /usr/bin/env bash -o pipefail
 K8S_VERSION ?= 1.32.0
 ENVTEST_K8S_VERSION = $(K8S_VERSION)
 CILIUM_VERSION ?= 1.16.5
+VALKEY_VERSION ?= 8.0.2
 
 V ?= 0
 ifeq ($(V), 1)
@@ -106,26 +111,47 @@ manager: manifests generate fmt vet ## Build manager binary.
 		-trimpath \
 		-gcflags all="-N -l -trimpath=/src -trimpath=$(PWD)" \
 		-asmflags all="-trimpath=/src -trimpath=$(PWD)" \
-		-ldflags "-s -w -X $(PKG)/cmd.Version=$(VERSION) -X $(PKG)/cmd.Commit=$(SHA)" \
+		-ldflags "-s -w -X main.BuildDate=$(DATE) -X main.Version=$(VERSION) -X main.Commit=$(SHA) \
+			-X $(PKG)/cfg.DefaultSidecarImage=$(IMG_SIDECAR) -X $(PKG)/cfg.DefaultValkeyImage=$(IMG_VALKEY)" \
 		-installsuffix cgo \
-		-o $@ cmd/main.go
+		-o $@ ./cmd/manager/
 
-build: manager
+sidecar: manifests generate fmt vet ## Build sidecar binary.
+	$QCGO_ENABLED=0 GOOS=linux GOARCH=amd64 $(GO) build $(VV) \
+		-trimpath \
+		-gcflags all="-N -l -trimpath=/src -trimpath=$(PWD)" \
+		-asmflags all="-trimpath=/src -trimpath=$(PWD)" \
+		-ldflags "-s -w -X main.BuildDate=$(DATE) -X main.Version=$(VERSION) -X main.Commit=$(SHA) \
+			-X $(PKG)/cfg.DefaultSidecarImage=$(IMG_SIDECAR) -X $(PKG)/cfg.DefaultValkeyImage=$(IMG_VALKEY)" \
+		-installsuffix cgo \
+		-o $@ ./cmd/sidecar/
+
+build: manager sidecar ## Build manager and sidecar binaries.
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
-	go run ./cmd/main.go
+	go run ./cmd/manager/main.go
 
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
-.PHONY: docker-build
-docker-build: manager ## Build docker image with the manager.
-	$(CONTAINER_TOOL) build -t ${IMG} .
+.PHONY: docker-build docker-build-manager docker-build-sidecar docker-build-valkey
+docker-build-manager: manager ## Build docker image with the manager.
+	$(CONTAINER_TOOL) build -t ${IMG_CONTROLLER} -f Dockerfile.controller .
+
+docker-build-sidecar: sidecar ## Build docker image with the sidecar binary.
+	$(CONTAINER_TOOL) build -t ${IMG_SIDECAR} -f Dockerfile.sidecar .
+
+docker-build-valkey: ## Build docker image with the valkey binary.
+	$(CONTAINER_TOOL) build -t ${IMG_VALKEY} --build-arg VALKEY_VERSION=$(VALKEY_VERSION) -f Dockerfile.valkey .
+
+docker-build: docker-build-manager docker-build-sidecar docker-build-valkey ## Build docker image with the manager, sidecar and valkey binaries.
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
-	$(CONTAINER_TOOL) push ${IMG}
+	$(CONTAINER_TOOL) push ${IMG_CONTROLLER}
+	$(CONTAINER_TOOL) push ${IMG_SIDECAR}
+	$(CONTAINER_TOOL) push ${IMG_VALKEY}
 
 # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple
 # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
@@ -137,17 +163,17 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
 .PHONY: docker-buildx
 docker-buildx: ## Build and push docker image for the manager for cross-platform support
 	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
-	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile.controller > Dockerfile.controller.cross
 	- $(CONTAINER_TOOL) buildx create --name valkey-operator-builder
 	$(CONTAINER_TOOL) buildx use valkey-operator-builder
-	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG_CONTROLLER} -f Dockerfile.controller.cross .
 	- $(CONTAINER_TOOL) buildx rm valkey-operator-builder
-	rm Dockerfile.cross
+	rm Dockerfile.controller.cross
 
 .PHONY: build-installer
 build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment.
 	$Qmkdir -p dist
-	$Qcd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$Qcd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_CONTROLLER}
 	$Q$(KUSTOMIZE) build config/default > dist/install.yaml
 
 ##@ Deployment
@@ -166,7 +192,7 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 
 .PHONY: deploy
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_CONTROLLER}
 	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
 
 .PHONY: undeploy
@@ -223,7 +249,7 @@ HELM_VERSION ?= v3.15.4
 GOSEC_VERSION ?= v2.20.0
 
 helm-gen: manifests kustomize helmify ## Generate Helm chart from Kustomize manifests
-	$Qcd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$Qcd config/manager && $(KUSTOMIZE) edit set image controller=${IMG_CONTROLLER}
 	$Q$(KUSTOMIZE) build config/default | $(HELMIFY) -crd-dir valkey-operator
 	$Qsed s@\\\(app.kubernetes.io/name\\\)@\'\\\1\'@ -i valkey-operator/templates/deployment.yaml
 	$Qsed s@\\\(app.kubernetes.io/instance\\\)@\'\\\1\'@ -i valkey-operator/templates/deployment.yaml

diff --git a/cfg/config.go b/cfg/config.go
@@ -1,8 +1,15 @@
 package cfg
 
+var (
+	// Default Settings
+	DefaultSidecarImage string
+	DefaultValkeyImage  string
+	DefaultNodes        int = 3
+)
+
 type Config struct {
 	// The default clusterwide prometheus exporter image to use
-	ExporterImage string `json:"exporterImage"`
+	SidecarImage string `json:"exporterImage"`
 	// The default clusterwide valkey image to use
 	ValkeyImage string `json:"valkeyImage"`
 	// The default number of nodes to use
@@ -11,8 +18,8 @@ type Config struct {
 
 func Defaults() *Config {
 	return &Config{
-		ExporterImage: "docker.io/bitnami/redis-exporter:1.63.0-debian-12-r0",
-		ValkeyImage:   "docker.io/bitnami/valkey-cluster:8.0.1-debian-12-r0",
-		Nodes:         3,
+		SidecarImage: DefaultSidecarImage,
+		ValkeyImage:  DefaultValkeyImage,
+		Nodes:        DefaultNodes,
 	}
 }