release-tools: update

Commit summary: 7100c12 Only set staging registry when running canary job d129462 Document new method for adding CI jobs are new K8s versions e73c2ce Use staging registry for canary tests 2c09846 Add cleanup instructions to release-notes generation 0979c09 prow.sh: fix E2E suite for Kubernetes >= 1.18 3b4a2f1 prow.sh: fix installing Go for Kubernetes 1.19.0 82d108a switch to Go 1.15 843bddc Add steps on promoting release images 1fdf2d5 cloud build: bump timeout in Prow job 5a54e67 filter-junit: Fix gofmt error 36ea4ff filter-junit: Fix golint error 43e50d6 prow.sh: enable building arm64 image 3df86b7 cloud build: k8s-staging-sig-storage db0c2a7 cloud build: initialize support for running commands in Dockerfile 340e082 build.make: optional inclusion of Windows in multiarch images 5231f05 build.make: properly declare push-multiarch 4569f27 build.make: fix push-multiarch ambiguity bd41690 cloud build: initial set of shared files 6f2322e Update patch release notes generation command d8c76fe Support local snapshot RBAC for pull jobs ea1f94a update release tools instructions 7edc146 Update snapshotter to version 2.0.1 3863a0f build for multiple platforms only in CI, add s390x
kubernetes-csi · Oct 6, 2020 · b53d94c · b53d94c
2 parents 7d4f792 + a0f195c
commit b53d94c
Show file tree

Hide file tree

Showing 7 changed files with 237 additions and 50 deletions.
diff --git a/release-tools/SIDECAR_RELEASE_PROCESS.md b/release-tools/SIDECAR_RELEASE_PROCESS.md
@@ -39,29 +39,39 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 1. Changes can then be updated in all the sidecar repos and hostpath driver repo
    by following the [update
    instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating).
-1. New pull and CI jobs are configured by
+1. New pull and CI jobs are configured by adding new K8s versions to the top of
    [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh).
-   New pull jobs that have been unverified should be initially made optional.
-   [Example](https://github.com/kubernetes/test-infra/pull/15055)
+   New pull jobs that have been unverified should be initially made optional by
+   setting the new K8s version as
+   [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40).
 1. Once new pull and CI jobs have been verified, and the new Kubernetes version
    is released, we can make the optional jobs required, and also remove the
    Kubernetes versions that are no longer supported.
 
 ## Release Process
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
-1. Download [K8s release notes
+1. Download v2.8+ [K8s release notes
   generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
-    ```
-    GITHUB_TOKEN=<token> ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \
-    --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md
-    ```
-    * `--start-sha` should point to the last release from the same branch. For
-    example:
-        * `1.X-1.0` tag when releasing `1.X.0`
-        * `1.X.Y-1` tag when releasing `1.X.Y`
+    * Clean up old cached information (also needed if you are generating release
+      notes for multiple repos)
+      ```bash
+      rm -rf /tmp/k8s-repo
+      ```
+    * For new minor releases on master:
+        ```bash
+        GITHUB_TOKEN=<token> release-notes --discover=mergebase-to-latest
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
+    * For new patch releases on a release branch:
+        ```bash
+        GITHUB_TOKEN=<token> release-notes --discover=patch-to-latest --branch=release-1.1
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
 1. Compare the generated output to the new commits for the release to check if
    any notable change missed a release note.
 1. Reword release notes as needed. Make sure to check notes for breaking
@@ -82,6 +92,12 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
    [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
 1. If release was a new major/minor version, create a new `release-<minor>`
    branch at that commit.
+1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
+1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
+   the [k8s image
+   repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage),
+   run `./generate.sh > images.yaml`, and send a PR with the updated images.
+   Once merged, the image promoter will copy the images from staging to prod.
 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
    and feature pages with the new released version.
 1. After all the sidecars have been released, update

diff --git a/release-tools/build.make b/release-tools/build.make
@@ -60,23 +60,30 @@ else
 TESTARGS =
 endif
 
-ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
-
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
-build-%: check-go-version-go
+# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# separated by semicolon. An empty variable or empty entry (= just a
+# semicolon) builds for the default platform of the current Go
+# toolchain.
+BUILD_PLATFORMS =
+
+# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
+# defined by BUILD_PLATFORMS.
+$(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
-	CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
-	if [ "$$ARCH" = "amd64" ]; then \
-		CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
-		CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \
-	fi
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
+			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done
 
-container-%: build-%
+$(CMDS:%=container-%): container-%: build-%
 	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
 
-push-%: container-%
+$(CMDS:%=push-%): push-%: container-%
 	set -ex; \
 	push_image () { \
 		docker tag $*:latest $(IMAGE_NAME):$$tag; \
@@ -98,6 +105,77 @@ build: $(CMDS:%=build-%)
 container: $(CMDS:%=container-%)
 push: $(CMDS:%=push-%)
 
+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# It is currently optional: if no such file exists, Windows images are not included,
+# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that
+# Windows binaries can be built before adding a Dockerfile for it.
+#
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
+	set -ex; \
+	DOCKER_CLI_EXPERIMENTAL=enabled; \
+	export DOCKER_CLI_EXPERIMENTAL; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
+	if ! [ -f "$$dockerfile_windows" ]; then \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+	fi; \
+	pushMultiArch () { \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
+	}; \
+	if [ $(PULL_BASE_REF) = "master" ]; then \
+			: "creating or overwriting canary image"; \
+			pushMultiArch canary; \
+	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
+			: "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
+	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
+			: "creating release image"; \
+			pushMultiArch $(PULL_BASE_REF); \
+	else \
+			: "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \
+			exit 1; \
+	fi
+
+.PHONY: check-pull-base-ref
+check-pull-base-ref:
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
+
+.PHONY: push-multiarch
+push-multiarch: $(CMDS:%=push-multiarch-%)
+
 clean:
 	-rm -rf bin
 

diff --git a/release-tools/cloudbuild.sh b/release-tools/cloudbuild.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# shellcheck disable=SC1091
+. release-tools/prow.sh
+
+gcr_cloud_build
diff --git a/release-tools/cloudbuild.yaml b/release-tools/cloudbuild.yaml
@@ -0,0 +1,46 @@
+# A configuration file for multi-arch image building with the Google cloud build service.
+#
+# Repos using this file must:
+# - import csi-release-tools
+# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml
+# - add a .cloudbuild.sh which can be a custom file or a symlink
+#   to release-tools/cloudbuild.sh
+# - accept "binary" as build argument in their Dockerfile(s) (see
+#   https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6)
+#   because binaries will get built for different architectures and then
+#   get copied from the built host into the container image
+#
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# for more details on image pushing process in Kubernetes.
+#
+# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
+
+# This must be specified in seconds. If omitted, defaults to 600s (10 mins).
+timeout: 1800s
+# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  # The image must contain bash and curl. Ideally it should also contain
+  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # but that just speeds up the build and is not required.
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+    entrypoint: ./.cloudbuild.sh
+    env:
+    - GIT_TAG=${_GIT_TAG}
+    - PULL_BASE_REF=${_PULL_BASE_REF}
+    - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT}
+    - HOME=/root
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution.
+  _GIT_TAG: '12345'
+  # _PULL_BASE_REF will contain the ref that was pushed to trigger this build -
+  # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
+  _PULL_BASE_REF: 'master'
+  # The default gcr.io staging project for Kubernetes-CSI
+  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL).
+  # Might be overridden in the Prow build job for a repo which wants
+  # images elsewhere.
+  _STAGING_PROJECT: 'k8s-staging-sig-storage'
diff --git a/release-tools/filter-junit.go b/release-tools/filter-junit.go
@@ -15,10 +15,10 @@ limitations under the License.
 */
 
 /*
- * This command filters a JUnit file such that only tests with a name
- * matching a regular expression are passed through. By concatenating
- * multiple input files it is possible to merge them into a single file.
- */
+This command filters a JUnit file such that only tests with a name
+matching a regular expression are passed through. By concatenating
+multiple input files it is possible to merge them into a single file.
+*/
 package main
 
 import (