From 3863a0f674b70d87f34be95cad6dd464118713fa Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 4 Mar 2020 11:19:09 +0100
Subject: [PATCH 01/24] build for multiple platforms only in CI, add s390x

Developers should not be forced to build for all platforms by
default. We also don't want to copy-and-paste the go invocation for
each new platform.

To address both, the target platform(s) are now configurable via
BUILD_PLATFORMS and additional platforms are only enabled in the Prow
CI.

For now this serves as a test that the source actually compiles for
multiple platforms. Building images for different target platforms is a
different problem.
---
 build.make | 21 ++++++++++++++-------
 prow.sh    |  4 +++-
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/build.make b/build.make
index 3bf3391c2..5c9793c2b 100644
--- a/build.make
+++ b/build.make
@@ -60,18 +60,25 @@ else
 TESTARGS =
 endif
 
-ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
-
 # Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
 # to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.
 
+# BUILD_PLATFORMS contains a set of <os> <arch> <suffix> triplets,
+# separated by semicolon. An empty variable or empty entry (= just a
+# semicolon) builds for the default platform of the current Go
+# toolchain.
+BUILD_PLATFORMS =
+
+# This builds each command (= the sub-directories of ./cmd) for the target platform(s)
+# defined by BUILD_PLATFORMS.
 build-%: check-go-version-go
 	mkdir -p bin
-	CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
-	if [ "$$ARCH" = "amd64" ]; then \
-		CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
-		CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \
-	fi
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
+			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done
 
 container-%: build-%
 	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
diff --git a/prow.sh b/prow.sh
index d54dea518..30cb38428 100755
--- a/prow.sh
+++ b/prow.sh
@@ -85,6 +85,8 @@ get_versioned_variable () {
     echo "$value"
 }
 
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because
 # setting it globally can break other go usages (like "go get <some command>"
@@ -1026,7 +1028,7 @@ main () {
     images=
     if ${CSI_PROW_BUILD_JOB}; then
         # A successful build is required for testing.
-        run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed"
+        run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed"
         # We don't want test failures to prevent E2E testing below, because the failure
         # might have been minor or unavoidable, for example when experimenting with
         # changes in "release-tools" in a PR (that fails the "is release-tools unmodified"

From 7edc1461eacefed7f3112ac65ebefbdd2c4a64d1 Mon Sep 17 00:00:00 2001
From: Grant Griffiths <grant@portworx.com>
Date: Wed, 11 Mar 2020 10:48:03 -0700
Subject: [PATCH 02/24] Update snapshotter to version 2.0.1

Signed-off-by: Grant Griffiths <grant@portworx.com>
---
 prow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prow.sh b/prow.sh
index 30cb38428..86b2bf058 100755
--- a/prow.sh
+++ b/prow.sh
@@ -342,7 +342,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub
 configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"
 
 # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment
-configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag"
+configvar CSI_SNAPSHOTTER_VERSION 'v2.0.1' "external-snapshotter version tag"
 
 # Some tests are known to be unusable in a KinD cluster. For example,
 # stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply

From ea1f94aad7b211ecd18f05794582c60869bf9251 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Fri, 17 Apr 2020 09:02:07 -0700
Subject: [PATCH 03/24] update release tools instructions

---
 SIDECAR_RELEASE_PROCESS.md | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md
index 91a1e1b82..c8aaa4e8b 100644
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -54,14 +54,21 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
   generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
-    ```
-    GITHUB_TOKEN=<token> ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \
-    --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md
-    ```
-    * `--start-sha` should point to the last release from the same branch. For
-    example:
-        * `1.X-1.0` tag when releasing `1.X.0`
-        * `1.X.Y-1` tag when releasing `1.X.Y`
+    * For new minor releases on master:
+        ```
+        GITHUB_TOKEN=<token> release-notes --discover=mergebase-to-latest
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
+    * For new patch releases on a release branch:
+        ```
+        GITHUB_TOKEN=<token> release-notes --branch=release-1.1
+        --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b
+        --github-org=kubernetes-csi --github-repo=external-provisioner
+        --required-author="" --output out.md
+        ```
+        * `--start-rev` should point to the last patch release from the release branch.
+        * `--end-sha` should point to the latest commit from the release branch.
 1. Compare the generated output to the new commits for the release to check if
    any notable change missed a release note.
 1. Reword release notes as needed. Make sure to check notes for breaking

From d8c76fee3e64e38a3427c9cf9df0f069801e3d74 Mon Sep 17 00:00:00 2001
From: Grant Griffiths <grant@portworx.com>
Date: Wed, 15 Apr 2020 19:11:33 -0700
Subject: [PATCH 04/24] Support local snapshot RBAC for pull jobs

Signed-off-by: Grant Griffiths <grant@portworx.com>
---
 prow.sh | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/prow.sh b/prow.sh
index 86b2bf058..0b962f044 100755
--- a/prow.sh
+++ b/prow.sh
@@ -1064,18 +1064,24 @@ main () {
                 # always pulling the image
                 # (https://github.com/kubernetes-sigs/kind/issues/328).
                 docker tag "$i:latest" "$i:csiprow" || die "tagging the locally built container image for $i failed"
-            done
 
-            if [ -e deploy/kubernetes/rbac.yaml ]; then
-                # This is one of those components which has its own RBAC rules (like external-provisioner).
-                # We are testing a locally built image and also want to test with the the current,
-                # potentially modified RBAC rules.
-                if [ "$(echo "$cmds" | wc -w)" != 1 ]; then
-                    die "ambiguous deploy/kubernetes/rbac.yaml: need exactly one command, got: $cmds"
+                # For components with multiple cmds, the RBAC file should be in the following format:
+                #   rbac-$cmd.yaml
+                # If this file cannot be found, we can default to the standard location:
+                #   deploy/kubernetes/rbac.yaml
+                rbac_file_path=$(find . -type f -name "rbac-$i.yaml")
+                if [ "$rbac_file_path" == "" ]; then
+                    rbac_file_path="$(pwd)/deploy/kubernetes/rbac.yaml"
                 fi
-                e=$(echo "$cmds" | tr '[:lower:]' '[:upper:]' | tr - _)
-                images="$images ${e}_RBAC=$(pwd)/deploy/kubernetes/rbac.yaml"
-            fi
+                
+                if [ -e "$rbac_file_path" ]; then
+                    # This is one of those components which has its own RBAC rules (like external-provisioner).
+                    # We are testing a locally built image and also want to test with the the current,
+                    # potentially modified RBAC rules.
+                    e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _)
+                    images="$images ${e}_RBAC=$rbac_file_path"
+                fi
+            done
         fi
 
         if tests_need_non_alpha_cluster; then

From 6f2322e80813459de15eb2aa0d17ddd53cb3efa8 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Thu, 23 Apr 2020 10:21:49 -0700
Subject: [PATCH 05/24] Update patch release notes generation command

---
 SIDECAR_RELEASE_PROCESS.md | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md
index c8aaa4e8b..6960607ef 100644
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -50,7 +50,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 ## Release Process
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
-1. Download [K8s release notes
+1. Download v2.8+ [K8s release notes
   generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
@@ -62,13 +62,10 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
         ```
     * For new patch releases on a release branch:
         ```
-        GITHUB_TOKEN=<token> release-notes --branch=release-1.1
-        --start-rev=v1.1.1 --end-sha=f0a9219b29cc9053047c39d149ce9b22bc7b918b
+        GITHUB_TOKEN=<token> release-notes --discover=patch-to-latest --branch=release-1.1
         --github-org=kubernetes-csi --github-repo=external-provisioner
         --required-author="" --output out.md
         ```
-        * `--start-rev` should point to the last patch release from the release branch.
-        * `--end-sha` should point to the latest commit from the release branch.
 1. Compare the generated output to the new commits for the release to check if
    any notable change missed a release note.
 1. Reword release notes as needed. Make sure to check notes for breaking

From bd416901d4b7cad7bc70a39f508b68e2a9fff1f6 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Thu, 28 May 2020 10:22:20 +0200
Subject: [PATCH 06/24] cloud build: initial set of shared files

The approach taken here extends the existing support for
cross-compiling binaries on the build host and specifying the Go
compiler: Go is installed if needed (as in Prow testing), binaries are
build on the host, then one image is created for each platform, and
finally those are combined into a single multi-architecture image.
---
 build.make      | 63 +++++++++++++++++++++++++++++++++++++++++++++++++
 cloudbuild.sh   |  6 +++++
 cloudbuild.yaml | 44 ++++++++++++++++++++++++++++++++++
 prow.sh         | 14 +++++++++++
 4 files changed, 127 insertions(+)
 create mode 100755 cloudbuild.sh
 create mode 100644 cloudbuild.yaml

diff --git a/build.make b/build.make
index 5c9793c2b..e3a44f96f 100644
--- a/build.make
+++ b/build.make
@@ -105,6 +105,69 @@ build: $(CMDS:%=build-%)
 container: $(CMDS:%=container-%)
 push: $(CMDS:%=push-%)
 
+# Additional parameters are needed when pushing to a local registry,
+# see https://github.com/docker/buildx/issues/94.
+# However, that then runs into https://github.com/docker/cli/issues/2396.
+#
+# What works for local testing is:
+# make push-multiarch PULL_BASE_REF=master REGISTRY_NAME=<your account on dockerhub.io> BUILD_PLATFORMS="linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x"
+DOCKER_BUILDX_CREATE_ARGS ?=
+
+# This target builds a multiarch image for one command using Moby BuildKit builder toolkit.
+# Docker Buildx is included in Docker 19.03.
+#
+# ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# BUILD_PLATFORMS determines which individual images are included in the multiarch image.
+# PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
+# the tag for the resulting multiarch image.
+push-multiarch-%: check-pull-base-ref build-%
+	set -ex; \
+	DOCKER_CLI_EXPERIMENTAL=enabled; \
+	export DOCKER_CLI_EXPERIMENTAL; \
+	docker buildx create $(DOCKER_BUILDX_CREATE_ARGS) --use --name multiarchimage-buildertest; \
+	trap "docker buildx rm multiarchimage-buildertest" EXIT; \
+	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
+	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
+	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
+	pushMultiArch () { \
+		tag=$$1; \
+		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \
+			docker buildx build --push \
+				--tag $(IMAGE_NAME):$$arch-$$os-$$tag \
+				--platform=$$os/$$arch \
+				--file $$(eval echo \$${dockerfile_$$os}) \
+				--build-arg binary=./bin/$*$$suffix \
+				--label revision=$(REV) \
+				.; \
+		done; \
+		images=$$(echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do echo $(IMAGE_NAME):$$arch-$$os-$$tag; done); \
+		docker manifest create --amend $(IMAGE_NAME):$$tag $$images; \
+		docker manifest push -p $(IMAGE_NAME):$$tag; \
+	}; \
+	if [ $(PULL_BASE_REF) = "master" ]; then \
+			: "creating or overwriting canary image"; \
+			pushMultiArch canary; \
+	elif echo $(PULL_BASE_REF) | grep -q -e 'release-*' ; then \
+			: "creating or overwriting canary image for release branch"; \
+			release_canary_tag=$$(echo $(PULL_BASE_REF) | cut -f2 -d '-')-canary; \
+			pushMultiArch $$release_canary_tag; \
+	elif docker pull $(IMAGE_NAME):$(PULL_BASE_REF) 2>&1 | tee /dev/stderr | grep -q "manifest for $(IMAGE_NAME):$(PULL_BASE_REF) not found"; then \
+			: "creating release image"; \
+			pushMultiArch $(PULL_BASE_REF); \
+	else \
+			: "ERROR: release image $(IMAGE_NAME):$(PULL_BASE_REF) already exists: a new tag is required!"; \
+			exit 1; \
+	fi
+
+.PHONY: check-pull-base-ref
+check-pull-base-ref:
+	if ! [ "$(PULL_BASE_REF)" ]; then \
+		echo >&2 "ERROR: PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name."; \
+		exit 1; \
+	fi
+
+push-multiarch: $(CMDS:%=push-multiarch-%)
+
 clean:
 	-rm -rf bin
 
diff --git a/cloudbuild.sh b/cloudbuild.sh
new file mode 100755
index 000000000..3ba11ecad
--- /dev/null
+++ b/cloudbuild.sh
@@ -0,0 +1,6 @@
+#! /bin/bash
+
+# shellcheck disable=SC1091
+. release-tools/prow.sh
+
+gcr_cloud_build
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
new file mode 100644
index 000000000..1def14994
--- /dev/null
+++ b/cloudbuild.yaml
@@ -0,0 +1,44 @@
+# A configuration file for multi-arch image building with the Google cloud build service.
+#
+# Repos using this file must:
+# - import csi-release-tools
+# - add a symlink cloudbuild.yaml -> release-tools/cloudbuild.yaml
+# - add a .cloudbuild.sh which can be a custom file or a symlink
+#   to release-tools/cloudbuild.sh
+# - accept "binary" as build argument in their Dockerfile(s) (see
+#   https://github.com/pohly/node-driver-registrar/blob/3018101987b0bb6da2a2657de607174d6e3728f7/Dockerfile#L4-L6)
+#   because binaries will get built for different architectures and then
+#   get copied from the built host into the container image
+#
+# See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
+# for more details on image pushing process in Kubernetes.
+
+# This must be specified in seconds. If omitted, defaults to 600s (10 mins).
+timeout: 1200s
+# This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
+# or any new substitutions added in the future.
+options:
+  substitution_option: ALLOW_LOOSE
+steps:
+  # The image must contain bash and curl. Ideally it should also contain
+  # the desired version of Go (currently defined in release-tools/travis.yml),
+  # but that just speeds up the build and is not required.
+  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20200421-a2bf5f8'
+    entrypoint: ./.cloudbuild.sh
+    env:
+    - GIT_TAG=${_GIT_TAG}
+    - PULL_BASE_REF=${_PULL_BASE_REF}
+    - REGISTRY_NAME=gcr.io/${_STAGING_PROJECT}
+    - HOME=/root
+substitutions:
+  # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
+  # can be used as a substitution.
+  _GIT_TAG: '12345'
+  # _PULL_BASE_REF will contain the ref that was pushed to trigger this build -
+  # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
+  _PULL_BASE_REF: 'master'
+  # The default gcr.io staging project for Kubernetes-CSI
+  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL).
+  # Might be overridden in the Prow build job for a repo which wants
+  # images elsewhere.
+  _STAGING_PROJECT: 'k8s-staging-csi'
diff --git a/prow.sh b/prow.sh
index 0b962f044..93964e18e 100755
--- a/prow.sh
+++ b/prow.sh
@@ -1189,3 +1189,17 @@ main () {
 
     return "$ret"
 }
+
+# This function can be called by a repo's top-level cloudbuild.sh:
+# it handles environment set up in the GCR cloud build and then
+# invokes "make push-multiarch" to do the actual image building.
+gcr_cloud_build () {
+    # Register gcloud as a Docker credential helper.
+    # Required for "docker buildx build --push".
+    gcloud auth configure-docker
+
+    # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value.
+    REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v')
+
+    run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make push-multiarch REV="${REV}" REGISTRY_NAME="${REGISTRY_NAME}" BUILD_PLATFORMS="${CSI_PROW_BUILD_PLATFORMS}"
+}

From 4569f27a8cf57336ea13d8ef3dc4423c76712913 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 3 Jun 2020 11:03:47 +0200
Subject: [PATCH 07/24] build.make: fix push-multiarch ambiguity

"make push-multiarch" matched both push-multiarch and push-%.  This
seems to be none-deterministic and in at least one
repo (external-provisioner), make picked the wildcard rule which then
failed because there is no "multiarch" command.

This ambiguity gets resolved by instantiating the wildcard rules only
for existing commands. The advantage also is that "make
push-no-such-command" will fail with an obvious "No rule to make
target 'push-no-such-command'" instead of attempting to build the
command.
---
 build.make | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/build.make b/build.make
index e3a44f96f..0e4a9baaf 100644
--- a/build.make
+++ b/build.make
@@ -71,7 +71,7 @@ BUILD_PLATFORMS =
 
 # This builds each command (= the sub-directories of ./cmd) for the target platform(s)
 # defined by BUILD_PLATFORMS.
-build-%: check-go-version-go
+$(CMDS:%=build-%): build-%: check-go-version-go
 	mkdir -p bin
 	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o "./bin/$*$$suffix" ./cmd/$*); then \
@@ -80,10 +80,10 @@ build-%: check-go-version-go
 		fi; \
 	done
 
-container-%: build-%
+$(CMDS:%=container-%): container-%: build-%
 	docker build -t $*:latest -f $(shell if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi) --label revision=$(REV) .
 
-push-%: container-%
+$(CMDS:%=push-%): push-%: container-%
 	set -ex; \
 	push_image () { \
 		docker tag $*:latest $(IMAGE_NAME):$$tag; \
@@ -120,7 +120,7 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # BUILD_PLATFORMS determines which individual images are included in the multiarch image.
 # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
 # the tag for the resulting multiarch image.
-push-multiarch-%: check-pull-base-ref build-%
+$(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	set -ex; \
 	DOCKER_CLI_EXPERIMENTAL=enabled; \
 	export DOCKER_CLI_EXPERIMENTAL; \

From 5231f05d8b2f7a6b656fb27d84abcb758000842b Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 3 Jun 2020 11:04:29 +0200
Subject: [PATCH 08/24] build.make: properly declare push-multiarch

It's not a real file and thus should better be marked as phony.
---
 build.make | 1 +
 1 file changed, 1 insertion(+)

diff --git a/build.make b/build.make
index 0e4a9baaf..83d903e08 100644
--- a/build.make
+++ b/build.make
@@ -166,6 +166,7 @@ check-pull-base-ref:
 		exit 1; \
 	fi
 
+.PHONY: push-multiarch
 push-multiarch: $(CMDS:%=push-multiarch-%)
 
 clean:

From 340e082f04321801f51cdff42576bd55dcfc3683 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 3 Jun 2020 11:05:05 +0200
Subject: [PATCH 09/24] build.make: optional inclusion of Windows in multiarch
 images

Most repos inherit the default BUILD_PLATFORMS, which includes
Windows, but don't have the necessary Dockerfile.Windows yet. To
simplify the rollout of multiarch image builds, Windows binary
building continues to be tested (i.e. BUILD_PLATFORMS remains
unchanged), but push-multiarch skips Windows if the Dockerfile.Windows
is missing.
---
 build.make | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/build.make b/build.make
index 83d903e08..c17a1c194 100644
--- a/build.make
+++ b/build.make
@@ -117,6 +117,10 @@ DOCKER_BUILDX_CREATE_ARGS ?=
 # Docker Buildx is included in Docker 19.03.
 #
 # ./cmd/<command>/Dockerfile[.Windows] is used if found, otherwise Dockerfile[.Windows].
+# It is currently optional: if no such file exists, Windows images are not included,
+# even when Windows is listed in BUILD_PLATFORMS. That way, projects can test that
+# Windows binaries can be built before adding a Dockerfile for it.
+#
 # BUILD_PLATFORMS determines which individual images are included in the multiarch image.
 # PULL_BASE_REF must be set to 'master', 'release-x.y', or a tag name, and determines
 # the tag for the resulting multiarch image.
@@ -129,6 +133,9 @@ $(CMDS:%=push-multiarch-%): push-multiarch-%: check-pull-base-ref build-%
 	dockerfile_linux=$$(if [ -e ./cmd/$*/Dockerfile ]; then echo ./cmd/$*/Dockerfile; else echo Dockerfile; fi); \
 	dockerfile_windows=$$(if [ -e ./cmd/$*/Dockerfile.Windows ]; then echo ./cmd/$*/Dockerfile.Windows; else echo Dockerfile.Windows; fi); \
 	if [ '$(BUILD_PLATFORMS)' ]; then build_platforms='$(BUILD_PLATFORMS)'; else build_platforms="linux amd64"; fi; \
+	if ! [ -f "$$dockerfile_windows" ]; then \
+		build_platforms="$$(echo "$$build_platforms" | sed -e 's/windows *[^ ]* *.exe//g' -e 's/; *;/;/g')"; \
+	fi; \
 	pushMultiArch () { \
 		tag=$$1; \
 		echo "$$build_platforms" | tr ';' '\n' | while read -r os arch suffix; do \

From db0c2a7dc80d5b6d65ac8e4e8c64026f342fd987 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Thu, 4 Jun 2020 10:52:04 +0200
Subject: [PATCH 10/24] cloud build: initialize support for running commands in
 Dockerfile

If the Dockerfile needs to run some command, that step fails unless
QEMU is set up properly first:
      failed to solve: rpc error: code = Unknown desc = failed to load
      LLB: runtime execution on platform linux/ppc64le not supported
---
 prow.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/prow.sh b/prow.sh
index 93964e18e..32ec871a3 100755
--- a/prow.sh
+++ b/prow.sh
@@ -1198,6 +1198,12 @@ gcr_cloud_build () {
     # Required for "docker buildx build --push".
     gcloud auth configure-docker
 
+    if find . -name Dockerfile | grep -v ^./vendor | xargs --no-run-if-empty cat | grep -q ^RUN; then
+        # Needed for "RUN" steps on non-linux/amd64 platforms.
+        # See https://github.com/multiarch/qemu-user-static#getting-started
+        (set -x; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes)
+    fi
+
     # Extract tag-n-hash value from GIT_TAG (form vYYYYMMDD-tag-n-hash) for REV value.
     REV=v$(echo "$GIT_TAG" | cut -f3- -d 'v')
 

From 3df86b7d437d7126a1ffb79ff05b0e467748b920 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Fri, 12 Jun 2020 15:48:26 +0200
Subject: [PATCH 11/24] cloud build: k8s-staging-sig-storage

As discussed in https://github.com/kubernetes/k8s.io/pull/943, we want
to consolidate under k8s-staging-sig-storage.
---
 cloudbuild.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index 1def14994..a22c0e7ac 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -12,6 +12,8 @@
 #
 # See https://github.com/kubernetes/test-infra/blob/master/config/jobs/image-pushing/README.md
 # for more details on image pushing process in Kubernetes.
+#
+# To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
 
 # This must be specified in seconds. If omitted, defaults to 600s (10 mins).
 timeout: 1200s
@@ -38,7 +40,7 @@ substitutions:
   # a branch like 'master' or 'release-0.2', or a tag like 'v0.2'.
   _PULL_BASE_REF: 'master'
   # The default gcr.io staging project for Kubernetes-CSI
-  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-csi/GLOBAL).
+  # (=> https://console.cloud.google.com/gcr/images/k8s-staging-sig-storage/GLOBAL).
   # Might be overridden in the Prow build job for a repo which wants
   # images elsewhere.
-  _STAGING_PROJECT: 'k8s-staging-csi'
+  _STAGING_PROJECT: 'k8s-staging-sig-storage'

From 43e50d6f6471f476acd5c90cd3788e198a86d9f8 Mon Sep 17 00:00:00 2001
From: Yibo Cai <yibo.cai@arm.com>
Date: Tue, 16 Jun 2020 10:45:09 +0800
Subject: [PATCH 12/24] prow.sh: enable building arm64 image

---
 prow.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/prow.sh b/prow.sh
index 32ec871a3..9778635f9 100755
--- a/prow.sh
+++ b/prow.sh
@@ -85,7 +85,7 @@ get_versioned_variable () {
     echo "$value"
 }
 
-configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
+configvar CSI_PROW_BUILD_PLATFORMS "linux amd64; windows amd64 .exe; linux ppc64le -ppc64le; linux s390x -s390x; linux arm64 -arm64" "Go target platforms (= GOOS + GOARCH) and file suffix of the resulting binaries"
 
 # If we have a vendor directory, then use it. We must be careful to only
 # use this for "make" invocations inside the project's repo itself because

From 36ea4ffae0fabb9404ad1588be285fe739b6e565 Mon Sep 17 00:00:00 2001
From: Animesh Kumar <animuz111@gmail.com>
Date: Wed, 17 Jun 2020 18:05:08 +0530
Subject: [PATCH 13/24] filter-junit: Fix golint error

Signed-off-by: Animesh Kumar <animuz111@gmail.com>
---
 filter-junit.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/filter-junit.go b/filter-junit.go
index 2f51be00e..1da224a00 100644
--- a/filter-junit.go
+++ b/filter-junit.go
@@ -15,9 +15,9 @@ limitations under the License.
 */
 
 /*
- * This command filters a JUnit file such that only tests with a name
- * matching a regular expression are passed through. By concatenating
- * multiple input files it is possible to merge them into a single file.
+This command filters a JUnit file such that only tests with a name
+matching a regular expression are passed through. By concatenating
+multiple input files it is possible to merge them into a single file.
  */
 package main
 

From 5a54e67d80858259793f0b970c227153db1f4c06 Mon Sep 17 00:00:00 2001
From: Animesh Kumar <animuz111@gmail.com>
Date: Thu, 18 Jun 2020 13:07:40 +0530
Subject: [PATCH 14/24] filter-junit: Fix gofmt error

Signed-off-by: Animesh Kumar <animuz111@gmail.com>
---
 filter-junit.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/filter-junit.go b/filter-junit.go
index 1da224a00..cf1cb4abf 100644
--- a/filter-junit.go
+++ b/filter-junit.go
@@ -18,7 +18,7 @@ limitations under the License.
 This command filters a JUnit file such that only tests with a name
 matching a regular expression are passed through. By concatenating
 multiple input files it is possible to merge them into a single file.
- */
+*/
 package main
 
 import (

From 1fdf2d53cddecb5e847ad810d0097e1e44a7b5f6 Mon Sep 17 00:00:00 2001
From: namrata-ibm <Namrata.Bhave@ibm.com>
Date: Wed, 8 Jul 2020 08:03:24 -0400
Subject: [PATCH 15/24] cloud build: bump timeout in Prow job

---
 cloudbuild.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cloudbuild.yaml b/cloudbuild.yaml
index a22c0e7ac..8f678924e 100644
--- a/cloudbuild.yaml
+++ b/cloudbuild.yaml
@@ -16,7 +16,7 @@
 # To promote release images, see https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage.
 
 # This must be specified in seconds. If omitted, defaults to 600s (10 mins).
-timeout: 1200s
+timeout: 1800s
 # This prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
 # or any new substitutions added in the future.
 options:

From 843bddca1493740fe733d44f749e8ba1e450ad72 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Wed, 5 Aug 2020 13:29:03 -0700
Subject: [PATCH 16/24] Add steps on promoting release images

---
 SIDECAR_RELEASE_PROCESS.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md
index 6960607ef..f6434c825 100644
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -86,6 +86,12 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
    [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
 1. If release was a new major/minor version, create a new `release-<minor>`
    branch at that commit.
+1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
+1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
+   the [k8s image
+   repo](https://github.com/kubernetes/k8s.io/tree/master/k8s.gcr.io/images/k8s-staging-sig-storage),
+   run `./generate.sh > images.yaml`, and send a PR with the updated images.
+   Once merged, the image promoter will copy the images from staging to prod.
 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
    and feature pages with the new released version.
 1. After all the sidecars have been released, update

From 82d108acdc4dd4cbd6d8d97534000dd5d8ea7e8d Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Wed, 12 Aug 2020 09:42:55 +0200
Subject: [PATCH 17/24] switch to Go 1.15

Go 1.15 was released and is the major version that Kubernetes 1.19.0
is going to use. There are probably bugs in the older 1.13.3 that were
fixed, so we should update.
---
 travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/travis.yml b/travis.yml
index bfd7647ba..1ab13aef2 100644
--- a/travis.yml
+++ b/travis.yml
@@ -6,7 +6,7 @@ git:
   depth: false
 matrix:
   include:
-  - go: 1.13.3
+  - go: 1.15
 before_script:
 - mkdir -p bin
 - wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep

From 3b4a2f1d956845ab094e61d8701eab2080226cd3 Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Fri, 14 Aug 2020 14:21:51 +0200
Subject: [PATCH 18/24] prow.sh: fix installing Go for Kubernetes 1.19.0

Kubernetes 1.19.0 uses Go 1.15, but refers to it as 1.15.0. This broke
both the check whether we need to install 1.15 (because "go version"
reports 1.15, which didn't match 1.15.0) and then downloading the
release archive (because the URL also only uses 1.15).
---
 prow.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/prow.sh b/prow.sh
index 9778635f9..8ce80cf87 100755
--- a/prow.sh
+++ b/prow.sh
@@ -513,6 +513,10 @@ go_version_for_kubernetes () (
     if ! [ "$go_version" ]; then
         die "Unable to determine Go version for Kubernetes $version from hack/lib/golang.sh."
     fi
+    # Strip the trailing .0. Kubernetes includes it, Go itself doesn't.
+    # Ignore: See if you can use ${variable//search/replace} instead.
+    # shellcheck disable=SC2001
+    go_version="$(echo "$go_version" | sed -e 's/\.0$//')"
     echo "$go_version"
 )
 

From 0979c091026916e3593022951c08d4f79bdb2bdf Mon Sep 17 00:00:00 2001
From: Patrick Ohly <patrick.ohly@intel.com>
Date: Fri, 14 Aug 2020 14:23:51 +0200
Subject: [PATCH 19/24] prow.sh: fix E2E suite for Kubernetes >= 1.18

It used to be necessary to override from where the E2E suite came on a
case-by-case basis (initially, testing was using a more recent suite
against an older Kubernetes). This should never become necessary again
and the lack of a specific entry for 1.18 already had the unintended
effect that Kubernetes 1.18 was tested with the suite from master, so
overall it is better to always use the E2E suite which matches
Kubernetes.
---
 prow.sh | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/prow.sh b/prow.sh
index 8ce80cf87..05d397a97 100755
--- a/prow.sh
+++ b/prow.sh
@@ -218,17 +218,18 @@ configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image ove
 # all generated files are present.
 #
 # CSI_PROW_E2E_REPO=none disables E2E testing.
-# TOOO: remove versioned variables and make e2e version match k8s version
-configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x"
-configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x"
-configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x"
-# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases
-configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version
-configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions
-configvar CSI_PROW_E2E_IMPORT_PATH_LATEST k8s.io/kubernetes "E2E package for Kubernetes >= 1.13.x" # currently the same for all versions
-configvar CSI_PROW_E2E_VERSION "$(get_versioned_variable CSI_PROW_E2E_VERSION "${csi_prow_kubernetes_version_suffix}")"  "E2E version"
-configvar CSI_PROW_E2E_REPO "$(get_versioned_variable CSI_PROW_E2E_REPO "${csi_prow_kubernetes_version_suffix}")" "E2E repo"
-configvar CSI_PROW_E2E_IMPORT_PATH "$(get_versioned_variable CSI_PROW_E2E_IMPORT_PATH "${csi_prow_kubernetes_version_suffix}")" "E2E package"
+tag_from_version () {
+    version="$1"
+    shift
+    case "$version" in
+        latest) echo "master";;
+        release-*) echo "$version";;
+        *) echo "v$version";;
+    esac
+}
+configvar CSI_PROW_E2E_VERSION "$(tag_from_version "${CSI_PROW_KUBERNETES_VERSION}")"  "E2E version"
+configvar CSI_PROW_E2E_REPO "https://github.com/kubernetes/kubernetes" "E2E repo"
+configvar CSI_PROW_E2E_IMPORT_PATH "k8s.io/kubernetes" "E2E package"
 
 # csi-sanity testing from the csi-test repo can be run against the installed
 # CSI driver. For this to work, deploying the driver must expose the Unix domain

From 2c098465dd81ade3fbab1dd251f65d2ec4abc1a5 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Mon, 24 Aug 2020 16:44:47 -0700
Subject: [PATCH 20/24] Add cleanup instructions to release-notes generation

---
 SIDECAR_RELEASE_PROCESS.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md
index f6434c825..91e837c9f 100644
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -54,14 +54,19 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
   generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes)
 1. Generate release notes for the release. Replace arguments with the relevant
   information.
+    * Clean up old cached information (also needed if you are generating release
+      notes for multiple repos)
+      ```bash
+      rm -rf /tmp/k8s-repo
+      ```
     * For new minor releases on master:
-        ```
+        ```bash
         GITHUB_TOKEN=<token> release-notes --discover=mergebase-to-latest
         --github-org=kubernetes-csi --github-repo=external-provisioner
         --required-author="" --output out.md
         ```
     * For new patch releases on a release branch:
-        ```
+        ```bash
         GITHUB_TOKEN=<token> release-notes --discover=patch-to-latest --branch=release-1.1
         --github-org=kubernetes-csi --github-repo=external-provisioner
         --required-author="" --output out.md

From e73c2ce5312662816b180c5b03eb490322051fd7 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Fri, 25 Sep 2020 17:41:33 -0700
Subject: [PATCH 21/24] Use staging registry for canary tests

---
 prow.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/prow.sh b/prow.sh
index 05d397a97..1812e3a7f 100755
--- a/prow.sh
+++ b/prow.sh
@@ -211,6 +211,10 @@ configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell functi
 # still use that name.
 configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images"
 
+# Image registry to use for canary images.
+# Only valid if CSI_PROW_DRIVER_CANARY is set.
+configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "registry for canary images"
+
 # The E2E testing can come from an arbitrary repo. The expectation is that
 # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836)
 # after setting KUBECONFIG. As a special case, if the repository is Kubernetes,
@@ -693,7 +697,7 @@ install_csi_driver () {
     fi
 
     if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then
-        images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}"
+        images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY} IMAGE_REGISTRY=${CSI_PROW_DRIVER_CANARY_REGISTRY}"
     fi
     # Ignore: Double quote to prevent globbing and word splitting.
     # It's intentional here for $images.

From d1294628784b0b37b9ce30aceab3725cd32c153e Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Fri, 25 Sep 2020 17:49:38 -0700
Subject: [PATCH 22/24] Document new method for adding CI jobs are new K8s
 versions

---
 SIDECAR_RELEASE_PROCESS.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md
index 91e837c9f..4575eb819 100644
--- a/SIDECAR_RELEASE_PROCESS.md
+++ b/SIDECAR_RELEASE_PROCESS.md
@@ -39,10 +39,11 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 1. Changes can then be updated in all the sidecar repos and hostpath driver repo
    by following the [update
    instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating).
-1. New pull and CI jobs are configured by
+1. New pull and CI jobs are configured by adding new K8s versions to the top of
    [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh).
-   New pull jobs that have been unverified should be initially made optional.
-   [Example](https://github.com/kubernetes/test-infra/pull/15055)
+   New pull jobs that have been unverified should be initially made optional by
+   setting the new K8s version as
+   [experimental](https://github.com/kubernetes/test-infra/blob/a1858f46d6014480b130789df58b230a49203a64/config/jobs/kubernetes-csi/gen-jobs.sh#L40).
 1. Once new pull and CI jobs have been verified, and the new Kubernetes version
    is released, we can make the optional jobs required, and also remove the
    Kubernetes versions that are no longer supported.

From 7d4f792feac2a00b61655b2dcfd527caac92796b Mon Sep 17 00:00:00 2001
From: Jiawei Wang <jiaweiwang@google.com>
Date: Tue, 6 Oct 2020 11:18:22 -0700
Subject: [PATCH 23/24] Upgrade csi-lib-utils to 0.7.1

---
 go.mod                                                 |  2 +-
 go.sum                                                 |  4 ++--
 .../kubernetes-csi/csi-lib-utils/metrics/metrics.go    | 10 +++++++++-
 vendor/modules.txt                                     |  2 +-
 4 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/go.mod b/go.mod
index 17a63c646..cd5aa2e0a 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.12
 require (
 	github.com/container-storage-interface/spec v1.2.0
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/kubernetes-csi/csi-lib-utils v0.7.0
+	github.com/kubernetes-csi/csi-lib-utils v0.7.1
 	google.golang.org/grpc v1.26.0
 	k8s.io/api v0.17.0
 	k8s.io/apimachinery v0.17.1-beta.0
diff --git a/go.sum b/go.sum
index c0308ef6c..c9d2dfbce 100644
--- a/go.sum
+++ b/go.sum
@@ -113,8 +113,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kubernetes-csi/csi-lib-utils v0.7.0 h1:t1cS7HTD7z5D7h9iAdjWuHtMxJPb9s1fIv34rxytzqs=
-github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0=
+github.com/kubernetes-csi/csi-lib-utils v0.7.1 h1:xd/mTX6j8TXFBxaRD8XX3cQ+jTNHxnA4OfZFDthL/to=
+github.com/kubernetes-csi/csi-lib-utils v0.7.1/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
diff --git a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go
index 2e252d654..1529d10df 100644
--- a/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go
+++ b/vendor/github.com/kubernetes-csi/csi-lib-utils/metrics/metrics.go
@@ -92,6 +92,10 @@ func NewCSIMetricsManager(driverName string) CSIMetricsManager {
 		),
 	}
 
+	// https://github.com/open-telemetry/opentelemetry-collector/issues/969
+	// Add process_start_time_seconds into the metric to let the start time be parsed correctly
+	metrics.RegisterProcessStartTime(cmm.registry.RawRegister)
+
 	cmm.SetDriverName(driverName)
 	cmm.registerMetrics()
 	return &cmm
@@ -171,7 +175,11 @@ func VerifyMetricsMatch(expectedMetrics, actualMetrics string, metricToIgnore st
 		wantScanner.Scan()
 		wantLine := strings.TrimSpace(wantScanner.Text())
 		gotLine := strings.TrimSpace(gotScanner.Text())
-		if wantLine != gotLine && (metricToIgnore == "" || !strings.HasPrefix(gotLine, metricToIgnore)) {
+		if wantLine != gotLine &&
+			(metricToIgnore == "" || !strings.HasPrefix(gotLine, metricToIgnore)) &&
+			// We should ignore the comments from metricToIgnore, otherwise the verification will
+			// fail because of the comments.
+			!strings.HasPrefix(gotLine, "#") {
 			return fmt.Errorf("\r\nMetric Want: %q\r\nMetric Got:  %q\r\n", wantLine, gotLine)
 		}
 	}
diff --git a/vendor/modules.txt b/vendor/modules.txt
index e9211baeb..2d9c226d6 100644
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -41,7 +41,7 @@ github.com/hashicorp/golang-lru/simplelru
 github.com/imdario/mergo
 # github.com/json-iterator/go v1.1.8
 github.com/json-iterator/go
-# github.com/kubernetes-csi/csi-lib-utils v0.7.0
+# github.com/kubernetes-csi/csi-lib-utils v0.7.1
 github.com/kubernetes-csi/csi-lib-utils/connection
 github.com/kubernetes-csi/csi-lib-utils/leaderelection
 github.com/kubernetes-csi/csi-lib-utils/metrics

From 7100c1209eba08b3444ad7cb84f31e225e0e3889 Mon Sep 17 00:00:00 2001
From: Michelle Au <msau@google.com>
Date: Tue, 6 Oct 2020 11:25:19 -0700
Subject: [PATCH 24/24] Only set staging registry when running canary job

---
 prow.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/prow.sh b/prow.sh
index 1812e3a7f..b069a318f 100755
--- a/prow.sh
+++ b/prow.sh
@@ -212,7 +212,7 @@ configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell functi
 configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images"
 
 # Image registry to use for canary images.
-# Only valid if CSI_PROW_DRIVER_CANARY is set.
+# Only valid if CSI_PROW_DRIVER_CANARY == "canary".
 configvar CSI_PROW_DRIVER_CANARY_REGISTRY "gcr.io/k8s-staging-sig-storage" "registry for canary images"
 
 # The E2E testing can come from an arbitrary repo. The expectation is that
@@ -697,7 +697,11 @@ install_csi_driver () {
     fi
 
     if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then
+      if [ "${CSI_PROW_DRIVER_CANARY}" == "canary" ]; then
         images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY} IMAGE_REGISTRY=${CSI_PROW_DRIVER_CANARY_REGISTRY}"
+      else
+        images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}"
+      fi
     fi
     # Ignore: Double quote to prevent globbing and word splitting.
     # It's intentional here for $images.