Merge pull request #2580 from PaulFarver/feature/sops-support

Sops support

Makefile

@@ -12,6 +12,7 @@ SHFMT_VERSION := 2.6.4
 include docker/kubectl.version
 include docker/kustomize.version
 include docker/helm.version
+include docker/sops.version
 
 # NB default target architecture is amd64. If you would like to try the
 # other one -- pass an ARCH variable, e.g.,
@@ -53,15 +54,15 @@ release-bins: $(GENERATED_TEMPLATES_FILE)
 clean:
 	go clean
 	rm -rf ./build
-	rm -f test/bin/kubectl test/bin/helm test/bin/kind test/bin/kustomize
+	rm -f test/bin/kubectl test/bin/helm test/bin/kind test/bin/sops test/bin/kustomize
 
 realclean: clean
 	rm -rf ./cache
 
-test: test/bin/helm test/bin/kubectl test/bin/kustomize $(GENERATED_TEMPLATES_FILE)
+test: test/bin/helm test/bin/kubectl test/bin/sops test/bin/kustomize $(GENERATED_TEMPLATES_FILE)
 	PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | sort -u)
 
-e2e: lint-e2e test/bin/helm test/bin/kubectl test/e2e/bats $(GOBIN)/fluxctl build/.flux.done
+e2e: lint-e2e test/bin/helm test/bin/kubectl test/bin/sops test/e2e/bats $(GOBIN)/fluxctl build/.flux.done
 	PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.bash
 
 E2E_BATS_FILES := test/e2e/*.bats
@@ -85,7 +86,7 @@ build/.%.done: docker/Dockerfile.%
 		-f build/docker/$*/Dockerfile.$* ./build/docker/$*
 	touch $@
 
-build/.flux.done: build/fluxd build/kubectl build/kustomize docker/ssh_config docker/kubeconfig docker/known_hosts.sh
+build/.flux.done: build/fluxd build/kubectl build/sops build/kustomize docker/ssh_config docker/kubeconfig docker/known_hosts.sh
 
 build/fluxd: $(FLUXD_DEPS)
 build/fluxd: cmd/fluxd/*.go
@@ -96,11 +97,13 @@ test/bin/kubectl: cache/$(CURRENT_OS_ARCH)/kubectl-$(KUBECTL_VERSION)
 build/helm: cache/linux-$(ARCH)/helm-$(HELM_VERSION)
 test/bin/helm: cache/$(CURRENT_OS_ARCH)/helm-$(HELM_VERSION)
 build/kustomize: cache/linux-amd64/kustomize-$(KUSTOMIZE_VERSION)
+build/sops: cache/linux-amd64/sops-$(SOPS_VERSION)
 test/bin/kustomize: cache/$(CURRENT_OS_ARCH)/kustomize-$(KUSTOMIZE_VERSION)
 test/bin/shellcheck: cache/$(CURRENT_OS_ARCH)/shellcheck-$(SHELLCHECK_VERSION)
 test/bin/shfmt: cache/$(CURRENT_OS_ARCH)/shfmt-$(SHFMT_VERSION)
+test/bin/sops: cache/$(CURRENT_OS_ARCH)/sops-$(SOPS_VERSION)
 
-build/kubectl test/bin/kubectl build/kustomize test/bin/kustomize build/helm test/bin/helm test/bin/shellcheck test/bin/shfmt:
+build/kubectl test/bin/kubectl build/kustomize test/bin/kustomize build/helm test/bin/helm test/bin/shellcheck test/bin/shfmt build/sops test/bin/sops:
 	mkdir -p $(@D)
 	cp $< $@
 	if [ `basename $@` = "build" -a $(CURRENT_OS_ARCH) = "linux-$(ARCH)" ]; then strip $@; fi
@@ -135,6 +138,11 @@ cache/%/shfmt-$(SHFMT_VERSION):
 	mkdir -p cache/$*
 	curl --fail -L -o $@ "https://github.com/mvdan/sh/releases/download/v$(SHFMT_VERSION)/shfmt_v$(SHFMT_VERSION)_`echo $* | tr - _`"
 
+cache/%/sops-$(SOPS_VERSION): docker/sops.version
+	mkdir -p cache/$*
+	curl --fail -L -o $@ "https://github.com/mozilla/sops/releases/download/$(SOPS_VERSION)/sops-$(SOPS_VERSION).`echo $* | cut -f1 -d"-"`"
+	[ $* != "linux-amd64" ] || echo "$(SOPS_CHECKSUM)  $@" | shasum -a 256 -c
+
 test/e2e/bats: cache/bats-core-$(BATS_COMMIT).tar.gz
 	mkdir -p $@
 	tar -C $@ --strip-components 1 -xzf $< 

cmd/fluxd/main.go

@@ -120,6 +120,7 @@ func main() {
 		gitSetAuthor = fs.Bool("git-set-author", false, "if set, the author of git commits will reflect the user who initiated the commit and will differ from the git committer.")
 		gitLabel     = fs.String("git-label", "", "label to keep track of sync progress; overrides both --git-sync-tag and --git-notes-ref")
 		gitSecret    = fs.Bool("git-secret", false, `if set, git-secret will be run on every git checkout. A gpg key must be imported using  --git-gpg-key-import or by mounting a keyring containing it directly`)
+		sopsEnabled  = fs.Bool("sops", false, `if set, decrypt sops-encrypted manifest files with sops before applying them. Be aware that manifests generated with .flux.yaml are not automatically decrypted. Provide decryption keys in the same way you would provide them for the sops binary`)
 		// Old git config; still used if --git-label is not supplied, but --git-label is preferred.
 		gitSyncTag     = fs.String("git-sync-tag", defaultGitSyncTag, fmt.Sprintf("tag to use to mark sync progress for this cluster (only relevant when --sync-state=%s)", fluxsync.GitTagStateMode))
 		gitNotesRef    = fs.String("git-notes-ref", defaultGitNotesRef, "ref to use for keeping commit annotations in git notes")
@@ -360,6 +361,10 @@ func main() {
 		logger.Log("warning", fmt.Sprintf("--git-secret is enabled but there is no GPG key(s) provided using --git-gpg-key-import, we assume you mounted the keyring directly and continue"))
 	}
 
+	if *sopsEnabled && len(*gitImportGPG) == 0 {
+		logger.Log("warning", fmt.Sprintf("--sops is enabled but there is no GPG key(s) provided using --git-gpg-key-import, we assume that the means of decryption has been provided in another way"))
+	}
+
 	// Mechanical components.
 
 	// When we can receive from this channel, it indicates that we
@@ -508,7 +513,11 @@ func main() {
 			logger.Log("err", err)
 			os.Exit(1)
 		}
-		k8sManifests = kubernetes.NewManifests(namespacer, logger)
+		if *sopsEnabled {
+			k8sManifests = kubernetes.NewSopsManifests(namespacer, logger)
+		} else {
+			k8sManifests = kubernetes.NewManifests(namespacer, logger)
+		}
 	}
 
 	// Wrap the procedure for collecting images to scan
@@ -643,6 +652,7 @@ func main() {
 		"notes-ref", *gitNotesRef,
 		"set-author", *gitSetAuthor,
 		"git-secret", *gitSecret,
+		"sops", *sopsEnabled,
 	)
 
 	var jobs *job.Queue

docker/Dockerfile.flux

@@ -16,6 +16,7 @@ COPY ./ssh_config /etc/ssh/ssh_config
 
 COPY ./kubectl /usr/local/bin/
 COPY ./kustomize /usr/local/bin
+COPY ./sops /usr/local/bin
 
 # These are pretty static
 LABEL maintainer="Flux CD <https://github.com/fluxcd/flux/issues>" \

docker/sops.version

@@ -0,0 +1,2 @@
+SOPS_VERSION=v3.5.0
+SOPS_CHECKSUM=610fca9687d1326ef2e1a66699a740f5dbd5ac8130190275959da737ec52f096

docs/references/daemon.md

@@ -94,6 +94,7 @@ Version controlling of cluster manifests provides reproducibility and a historic
 | --ssh-keygen-type                                |                                    | -t argument to ssh-keygen (default unspecified)
 | **manifest generation**
 | --manifest-generation                            | false                              | search for .flux.yaml files to generate manifests
+| --sops                                           | false                              | decrypt sops encrypted manifest files with sops before applying them. Be aware that manifests generated with .flux.yaml are not decrypted. This is due to the way sops handles yaml streams. It is recommended that generating manifests from encrypted files is handles with the included sops binary. Provide keys for sops in the same way as providing them for the binary, for example with --git-gpg-key-import. A full description of how to supply sops with a key can be found in the [sops documentation](https://github.com/mozilla/sops#usage)
 
 ## More information
 

go.mod

@@ -6,9 +6,10 @@ go 1.13
 replace github.com/docker/distribution => github.com/2opremio/distribution v0.0.0-20190419185413-6c9727e5e5de
 
 require (
+	github.com/Azure/go-autorest v12.2.0+incompatible // indirect
 	github.com/Jeffail/gabs v1.4.0
 	github.com/Masterminds/semver v1.4.2
-	github.com/aws/aws-sdk-go v1.19.11
+	github.com/aws/aws-sdk-go v1.23.13
 	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
 	github.com/cheggaaa/pb/v3 v3.0.2
 	github.com/docker/distribution v2.7.1+incompatible
@@ -37,6 +38,7 @@ require (
 	github.com/weaveworks/common v0.0.0-20190410110702-87611edc252e
 	github.com/weaveworks/go-checkpoint v0.0.0-20170503165305-ebbb8b0518ab
 	github.com/whilp/git-urls v0.0.0-20160530060445-31bac0d230fa
+	go.mozilla.org/sops/v3 v3.5.0
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4