Mk include 2 (#8)

* Squashed 'mk-include/' content from commit 9bdd40520

git-subtree-dir: mk-include
git-subtree-split: 9bdd405204f66179031f7f42275ab38641b495b4

* Updated makefile for mk-include

* Changing to jfrog

* Updating release branch

Dockerfile

@@ -20,7 +20,7 @@ RUN tar -xzvf /home/gradle/core/build/distributions/kafka_*-SNAPSHOT.tgz --strip
 
 ##########
 
-FROM 368821881613.dkr.ecr.us-west-2.amazonaws.com/confluentinc/cc-base:v2.1.0
+FROM confluent-docker.jfrog.io/confluentinc/cc-base:v2.1.0
 
 ARG version
 ARG confluent_version

Makefile

@@ -1,54 +1,9 @@
-IMAGE_NAME := confluentinc/cc-kafka
-IMAGE_VERSION=$(shell git describe --tags --always --long --dirty)
-DOCKER_REPO ?= 368821881613.dkr.ecr.us-west-2.amazonaws.com
-BASE_IMAGE := ${DOCKER_REPO}/confluentinc/cc-base
+IMAGE_NAME := cc-kafka
+BASE_IMAGE := confluent-docker.jfrog.io/confluentinc/cc-base
 BASE_VERSION := v2.1.0
-AWS_PROFILE ?= default
-AWS_REGION ?= us-west-2
-GIT_SHA=$(shell git rev-parse HEAD)
-GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD)
-CONFLUENT_VERSION ?= 5.1.0-SNAPSHOT
-LIFECYCLE_POLICY ?= '{"rules":[{"rulePriority":10,"description":"keeps 50 latest tagged images","selection":{"tagStatus":"tagged","countType":"imageCountMoreThan","countNumber":50,"tagPrefixList":["v"]},"action":{"type":"expire"}},{"rulePriority":20,"description":"keeps 5 latest untagged images","selection":{"tagStatus":"untagged","countType":"imageCountMoreThan","countNumber":5},"action":{"type":"expire"}},{"rulePriority":30,"description":"keeps latest 20 numeric-tagged images","selection":{"tagStatus":"tagged","countType":"imageCountMoreThan","tagPrefixList":["0","1","2","3","4","5","6","7","8","9"],"countNumber":20},"action":{"type":"expire"}},{"rulePriority":40,"description":"keeps latest 20 a-f tagged images","selection":{"tagStatus":"tagged","countType":"imageCountMoreThan","tagPrefixList":["a","b","c","d","e","f"],"countNumber":20},"action":{"type":"expire"}}]}'
+RELEASE_BRANCH := ce-trunk
 
-show-args:
-	@echo 'IMAGE_NAME: $(IMAGE_NAME)'
-	@echo 'IMAGE_VERSION: $(IMAGE_VERSION)'
-	@echo 'DOCKER_REPO: $(DOCKER_REPO)'
-	@echo 'BASE_IMAGE: $(BASE_IMAGE)'
-	@echo 'BASE_VERSION: $(BASE_VERSION)'
-
-FORCE:
-
-build-docker: pull-base
-	docker build --no-cache --build-arg version=$(IMAGE_VERSION) --build-arg kafka_version=$(KAFKA_VERSION) --build-arg CC_BASE_VERSION=$(BASE_VERSION) --build-arg confluent_version=$(CONFLUENT_VERSION) --build-arg git_sha=$(GIT_SHA) --build-arg git_branch=$(GIT_BRANCH) -t $(IMAGE_NAME):$(IMAGE_VERSION) .
-
-repo-login:
-	@eval "$(shell aws ecr get-login --no-include-email --region us-west-2 --profile default)"
-
-create-repo:
-	aws ecr describe-repositories --region us-west-2 --repository-name $(IMAGE_NAME) || aws ecr create-repository --region us-west-2 --repository-name $(IMAGE_NAME)
-	aws ecr put-lifecycle-policy --region us-west-2 --repository-name $(IMAGE_NAME) --lifecycle-policy-text $(LIFECYCLE_POLICY) || echo "Failed to put lifecycle policy on $(IMAGE_NAME) repo"
-
-pull-base: repo-login
-ifneq ($(BASE_IMAGE),$(_empty))
-	docker image ls -f reference="$(BASE_IMAGE):$(BASE_VERSION)" | grep -Eq "$(BASE_IMAGE)[ ]*$(BASE_VERSION)" || \
-		docker pull $(BASE_IMAGE):$(BASE_VERSION)
-endif
-
-push-docker: create-repo push-docker-latest push-docker-version
-
-push-docker-latest: tag-docker-latest
-	@echo 'push latest to $(DOCKER_REPO)'
-	docker push $(DOCKER_REPO)/$(IMAGE_NAME):latest
-
-push-docker-version: tag-docker-version
-	@echo 'push $(IMAGE_VERSION) to $(DOCKER_REPO)'
-	docker push $(DOCKER_REPO)/$(IMAGE_NAME):$(IMAGE_VERSION)
-
-tag-docker-latest:
-	@echo 'create docker tag latest'
-	docker tag $(IMAGE_NAME):$(IMAGE_VERSION) $(DOCKER_REPO)/$(IMAGE_NAME):latest
-
-tag-docker-version:
-	@echo 'create docker tag $(IMAGE_VERSION)'
-	docker tag $(IMAGE_NAME):$(IMAGE_VERSION) $(DOCKER_REPO)/$(IMAGE_NAME):$(IMAGE_VERSION)
+include ./mk-include/cc-begin.mk
+include ./mk-include/cc-semver.mk
+include ./mk-include/cc-docker.mk
+include ./mk-include/cc-end.mk

mk-include/README.md

@@ -0,0 +1,45 @@
+# Confluent Cloud Makefile Includes
+This is a set of Makefile include targets that are used in cloud applications.
+
+## Install
+Add this repo to your repo with the command:
+```shell
+git subtree add --prefix mk-include [email protected]:confluentinc/cc-mk-include.git master --squash
+```
+
+Then update your makefile like so:
+
+### Go + Docker + Helm Service
+```make
+SERVICE_NAME := cc-scraper
+MAIN_GO := cmd/scraper/main.go
+BASE_IMAGE := golang
+BASE_VERSION := 1.9
+
+include ./mk-include/cc-begin.mk
+include ./mk-include/cc-semver.mk
+include ./mk-include/cc-go.mk
+include ./mk-include/cc-docker.mk
+include ./mk-include/cc-helm.mk
+include ./mk-include/cc-end.mk
+```
+
+### Docker + Helm Only Service
+```make
+IMAGE_NAME := cc-example
+BASE_IMAGE := confluent-docker.jfrog.io/confluentinc/caas-base-alpine
+BASE_VERSION := v0.6.1
+
+include ./mk-include/cc-begin.mk
+include ./mk-include/cc-semver.mk
+include ./mk-include/cc-docker.mk
+include ./mk-include/cc-helm.mk
+include ./mk-include/cc-end.mk
+```
+
+## Updating
+Once you have the make targets installed, you can update at any time by running
+
+```shell
+make update-mk-include
+```

mk-include/cc-begin.mk

@@ -0,0 +1,59 @@
+# Include this file first
+_empty :=
+_space := $(_empty) $(empty)
+
+INIT_CI_TARGETS += ci-install-cpd
+RELEASE_TARGETS += $(_empty)
+BUILD_TARGETS += $(_empty)
+TEST_TARGETS += $(_empty)
+CLEAN_TARGETS += $(_empty)
+
+# If this variable is set, release will run make $(RELEASE_MAKE_TARGETS)
+RELEASE_MAKE_TARGETS +=
+
+ifeq ($(SEMAPHORE),true)
+CI_BIN := $(SEMAPHORE_CACHE_DIR)/bin
+else ifeq ($(BUILDKITE),true)
+CI_BIN := /tmp/bin
+endif
+
+CPD_VERSION := v0.2.0
+
+# Git stuff
+BRANCH_NAME ?= $(shell test -d .git && git rev-parse --abbrev-ref HEAD)
+# Set RELEASE_BRANCH if we're on master or vN.N.x
+RELEASE_BRANCH := $(shell echo $(BRANCH_NAME) | grep -E 'master|v[0-9]+\.[0-9]+\.x')
+# assume the remote name is origin by default
+GIT_REMOTE_NAME ?= origin
+
+ifeq ($(CI),true)
+_ := $(shell test -d $(CI_BIN) || mkdir -p $(CI_BIN))
+export PATH = $(CI_BIN):$(shell printenv PATH)
+endif
+
+.PHONY: update-mk-include
+update-mk-include:
+	git subtree pull --prefix mk-include [email protected]:confluentinc/cc-mk-include.git master --squash
+
+.PHONY: bats
+bats:
+	find . -name *.bats -exec bats {} \;
+
+.PHONY: ci-install-cpd
+ci-install-cpd:
+	@(test -f $(CI_BIN)/cpd && $(CI_BIN)/cpd version | grep -q $(CPD_VERSION) ) ||\
+		aws --profile default s3 cp s3://cloud-confluent-bin/cpd/cpd-$(CPD_VERSION)-linux-amd64 $(CI_BIN)/cpd
+	chmod +x $(CI_BIN)/cpd
+
+$(HOME)/.netrc:
+	@echo .netrc missing, prompting for user input
+	@echo Enter Github credentials, if you use 2 factor authentcation generate a personal access token for the password: https://github.com/settings/tokens
+	$(eval user := $(shell bash -c 'read -p "GitHub Username: " user; echo $$user'))
+	$(eval pass := $(shell bash -c 'read -p "GitHub Password: " pass; echo $$pass'))
+	@printf "machine github.com\n\tlogin $(user)\n\tpassword $(pass)" > $(HOME)/.netrc
+
+.netrc: $(HOME)/.netrc
+	cp $(HOME)/.netrc .netrc
+
+.ssh: $(HOME)/.ssh
+	cp -R $(HOME)/.ssh .ssh

mk-include/cc-deb.mk

@@ -0,0 +1,48 @@
+DEB_PKG_NAME ?= $(SERVICE_NAME)
+DEB_OUTDIR ?= deb
+RELEASE_MAKE_TARGETS += publish-deb-docker
+
+# Check if version includes a - which means it's not a released version
+ifeq ($(findstring -,$(VERSION_NO_V)), "")
+APT_SUITE := stable
+else
+APT_SUITE := testing
+endif
+
+.aws:
+	cp -r ~/.aws .aws
+
+.PHONY: publish-deb-docker
+## Build and publish deb inside packaging docker container
+publish-deb-docker: .netrc .aws
+	docker build -f Dockerfile.deb-package --build-arg version=$(VERSION_NO_V) -t build-deb .
+	docker image rm -f build-deb
+	rm -rf .aws
+
+.PHONY: build-deb
+## Build deb package using fpm
+build-deb:
+	cd mk-include/packaging && \
+		gem install bundler && \
+		bundle install && \
+		fpm \
+			-s dir \
+			-t deb \
+			-n $(DEB_PKG_NAME) \
+			-v $(VERSION_NO_V) \
+			-p $(DEB_OUTDIR)/$(DEB_PKG_NAME)_$(VERSION_NO_V)_amd64.deb \
+			$(BUILDROOT)=/
+
+.PHONY: publish-deb
+## Upload deb to s3://cloud-confluent-apt
+publish-deb: build-deb
+	cd mk-include/packaging && \
+		deb-s3 upload \
+			--arch amd64 \
+			--suite $(APT_SUITE) \
+			--lock \
+			--fail-if-exists \
+			--visibility private \
+			--s3-region us-west-2 \
+			--bucket cloud-confluent-apt \
+			$(DEB_OUTDIR)/$(DEB_PKG_NAME)_$(VERSION_NO_V)_amd64.deb

mk-include/cc-docker.mk

@@ -0,0 +1,122 @@
+_empty :=
+_space := $(_empty) $(empty)
+
+# Base Stuff
+BASE_IMAGE ?=
+BASE_VERSION ?=
+
+# Image Name
+IMAGE_NAME ?= unknown
+ifeq ($(IMAGE_NAME),unknown)
+$(error IMAGE_NAME must be set)
+endif
+
+# Image Version
+#  If we're on CI and a release branch, build with the bumped version
+ifeq ($(CI),true)
+ifneq ($(RELEASE_BRANCH),$(_empty))
+IMAGE_VERSION ?= $(BUMPED_VERSION)
+else
+IMAGE_VERSION ?= $(VERSION)
+endif
+else
+IMAGE_VERSION ?= $(VERSION)
+endif
+
+IMAGE_REPO ?= confluentinc
+ifeq ($(IMAGE_REPO),$(_empty))
+BUILD_TAG ?= $(IMAGE_NAME):$(IMAGE_VERSION)
+BUILD_TAG_LATEST ?= $(IMAGE_NAME):latest
+else
+BUILD_TAG ?= $(IMAGE_REPO)/$(IMAGE_NAME):$(IMAGE_VERSION)
+BUILD_TAG_LATEST ?= $(IMAGE_REPO)/$(IMAGE_NAME):latest
+endif
+
+DOCKER_REPO ?= confluent-docker.jfrog.io
+
+# Set targers for standard commands
+RELEASE_TARGETS += push-docker
+BUILD_TARGETS += build-docker
+CLEAN_TARGETS += clean-images
+
+.PHONY: show-docker
+## Show docker variables
+show-docker:
+	@echo "BASE_IMAGE: $(BASE_IMAGE)"
+	@echo "BASE_VERSION: $(BASE_VERSION)"
+	@echo "IMAGE_NAME: $(IMAGE_NAME)"
+	@echo "IMAGE_VERSION: $(IMAGE_VERSION)"
+	@echo "IMAGE_REPO: $(IMAGE_REPO)"
+	@echo "BUILD_TAG: $(BUILD_TAG)"
+	@echo "BUILD_TAG_LATEST: $(BUILD_TAG_LATEST)"
+	@echo "DOCKER_REPO: $(DOCKER_REPO)"
+
+.PHONY: docker-login
+## Login to docker Artifactory
+docker-login:
+ifeq ($(DOCKER_USER)$(DOCKER_APIKEY),$(_empty))
+	@echo confluent-docker.jfrog.io not logged in, Username and Password not found in environment, prompting for login:
+	@jq -e '.auths."confluent-docker.jfrog.io"' $(HOME)/.docker/config.json ||\
+		docker login confluent-docker.jfrog.io
+else
+	@jq -e '.auths."confluent-docker.jfrog.io"' $(HOME)/.docker/config.json ||\
+		docker login confluent-docker.jfrog.io --username $(DOCKER_USER) --password $(DOCKER_APIKEY)
+endif
+
+.PHONY: docker-pull-base
+## Pull the base docker image
+docker-pull-base: docker-login
+ifneq ($(BASE_IMAGE),$(_empty))
+	docker image ls -f reference="$(BASE_IMAGE):$(BASE_VERSION)" | grep -Eq "$(BASE_IMAGE)[ ]*$(BASE_VERSION)" || \
+		docker pull $(BASE_IMAGE):$(BASE_VERSION)
+endif
+
+.PHONY: build-docker
+ifeq ($(BUILD_DOCKER_OVERRIDE),)
+## Build just the docker image
+build-docker: .netrc .ssh docker-pull-base
+	docker build --no-cache --build-arg version=$(IMAGE_VERSION) -t $(BUILD_TAG) .
+	rm -f .netrc
+else
+## Build just the docker image
+build-docker: $(BUILD_DOCKER_OVERRIDE)
+endif
+
+.PHONY: tag-docker
+tag-docker: tag-docker-latest tag-docker-version
+
+.PHONY: tag-docker-latest
+tag-docker-latest:
+	@echo 'create docker tag latest'
+	docker tag $(BUILD_TAG) $(DOCKER_REPO)/$(BUILD_TAG_LATEST)
+
+.PHONY: tag-docker-version
+tag-docker-version:
+	@echo 'create docker tag $(IMAGE_VERSION)'
+	docker tag $(BUILD_TAG) $(DOCKER_REPO)/$(BUILD_TAG)
+
+.PHONY: push-docker
+ifeq ($(PUSH_DOCKER_OVERRIDE),)
+push-docker: push-docker-latest push-docker-version
+else
+push-docker: $(PUSH_DOCKER_OVERRIDE)
+endif
+
+.PHONY: push-docker-latest
+push-docker-latest: tag-docker-latest
+	@echo 'push latest to $(DOCKER_REPO)'
+	docker push $(DOCKER_REPO)/$(BUILD_TAG_LATEST)
+
+.PHONY: push-docker-version
+## Push the current version of docker to artifactory
+push-docker-version: tag-docker-version
+	@echo 'push $(IMAGE_VERSION) to $(DOCKER_REPO)'
+	docker push $(DOCKER_REPO)/$(BUILD_TAG)
+
+.PHONY: clean-images
+clean-images:
+	docker images -q -f label=io.confluent.caas=true -f reference='*$(IMAGE_NAME)' | uniq | xargs docker rmi -f
+
+.PHONY: clean-all
+clean-all:
+	docker images -q -f label=io.confluent.caas=true | uniq | xargs docker rmi -f