Skip to content

Commit

Permalink
release-tools: update to Go 1.13.4
Browse files Browse the repository at this point in the history
This brings in the switch to Go 1.13.4, which affects the content of
the vendor directory.

It also caused the API tests to fail because of the incorrect import
path: go 1.13.4 (correctly!) pulled the v2.2.0 release of csi-test and
tried to build with that, which then failed due to the API
changes. Somehow that worked with older go.
  • Loading branch information
pohly committed Nov 15, 2019
2 parents a68ba27 + 4fcafec commit a69270c
Show file tree
Hide file tree
Showing 40 changed files with 218 additions and 9,612 deletions.
2 changes: 1 addition & 1 deletion hack/_apitest/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package apitest
import (
"testing"

"github.com/kubernetes-csi/csi-test/pkg/sanity"
"github.com/kubernetes-csi/csi-test/v3/pkg/sanity"
)

func TestMyDriver(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion hack/_apitest2/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import (
"path"
"testing"

"github.com/kubernetes-csi/csi-test/pkg/sanity"
"github.com/kubernetes-csi/csi-test/v3/pkg/sanity"
)

// TestMyDriverWithCustomTargetPaths verifies that CreateTargetDir and
Expand Down
2 changes: 1 addition & 1 deletion hack/_embedded/embedded_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package embedded
import (
"testing"

"github.com/kubernetes-csi/csi-test/pkg/sanity"
"github.com/kubernetes-csi/csi-test/v3/pkg/sanity"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
Expand Down
61 changes: 18 additions & 43 deletions release-tools/build.make
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
# including build.make.
REGISTRY_NAME=quay.io/k8scsi

# Can be set to -mod=vendor to ensure that the "vendor" directory is used.
GOFLAGS_VENDOR=

# Revision that gets built into each binary via the main.version
# string. Uses the `git describe` output based on the most recent
# version tag with a short revision suffix or, if nothing has been
Expand Down Expand Up @@ -62,11 +65,11 @@ ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))
# Specific packages can be excluded from each of the tests below by setting the *_FILTER_CMD variables
# to something like "| grep -v 'github.com/kubernetes-csi/project/pkg/foobar'". See usage below.

build-%:
build-%: check-go-version-go
mkdir -p bin
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$*
if [ "$$ARCH" = "amd64" ]; then \
CGO_ENABLED=0 GOOS=windows go build -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \
fi

container-%: build-%
Expand Down Expand Up @@ -97,19 +100,19 @@ push: $(CMDS:%=push-%)
clean:
-rm -rf bin

test:
test: check-go-version-go

.PHONY: test-go
test: test-go
test-go:
@ echo; echo "### $@:"
go test `go list ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v -e 'vendor' -e '/test/e2e$$' $(TEST_GO_FILTER_CMD)` $(TESTARGS)

.PHONY: test-vet
test: test-vet
test-vet:
@ echo; echo "### $@:"
go vet `go list ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`
go test $(GOFLAGS_VENDOR) `go list $(GOFLAGS_VENDOR) ./... | grep -v vendor $(TEST_VET_FILTER_CMD)`

.PHONY: test-fmt
test: test-fmt
Expand Down Expand Up @@ -154,43 +157,7 @@ test-fmt:
test: test-vendor
test-vendor:
@ echo; echo "### $@:"
@ if [ -f Gopkg.toml ]; then \
echo "Repo uses 'dep' for vendoring."; \
case "$$(dep version 2>/dev/null | grep 'version *:')" in \
*v0.[56789]*) dep check && echo "vendor up-to-date" || false;; \
*) echo "skipping check, dep >= 0.5 required";; \
esac; \
elif [ -f go.mod ]; then \
echo "Repo uses 'go mod'."; \
if [ "$${JOB_NAME}" ] && \
( [ "$${JOB_TYPE}" != "presubmit" ] || \
[ $$( (git diff "${PULL_BASE_SHA}..HEAD" -- go.mod go.sum vendor release-tools; \
git diff "${PULL_BASE_SHA}..HEAD" | grep -e '^@@.*@@ import (' -e '^[+-]import') | \
wc -l) -eq 0 ] ); then \
echo "Skipping vendor check because the Prow pre-submit job does not affect dependencies."; \
elif ! GO111MODULE=on go mod tidy; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- go.mod go.sum | wc -l) -gt 0 ]; then \
echo "ERROR: go module files *not* up-to-date, they did get modified by 'GO111MODULE=on go mod tidy':"; \
git diff -- go.mod go.sum; \
false; \
elif [ -d vendor ]; then \
if ! GO111MODULE=on go mod vendor; then \
echo "ERROR: vendor check failed."; \
false; \
elif [ $$(git status --porcelain -- vendor | wc -l) -gt 0 ]; then \
echo "ERROR: vendor directory *not* up-to-date, it did get modified by 'GO111MODULE=on go mod vendor':"; \
git status -- vendor; \
git diff -- vendor; \
false; \
else \
echo "Go dependencies and vendor directory up-to-date."; \
fi; \
else \
echo "Go dependencies up-to-date."; \
fi; \
fi
@ ./release-tools/verify-vendor.sh

.PHONY: test-subtree
test: test-subtree
Expand All @@ -216,3 +183,11 @@ test-shellcheck:
./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
done; \
exit $$ret

# Targets in the makefile can depend on check-go-version-<path to go binary>
# to trigger a warning if the x.y version of that binary does not match
# what the project uses. Make ensures that this is only checked once per
# invocation.
.PHONY: check-go-version-%
check-go-version-%:
./release-tools/verify-go-version.sh "$*"
58 changes: 23 additions & 35 deletions release-tools/prow.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ get_versioned_variable () {
echo "$value"
}

# If we have a vendor directory, then use it. We must be careful to only
# use this for "make" invocations inside the project's repo itself because
# setting it globally can break other go usages (like "go get <some command>"
# which is disabled with GOFLAGS=-mod=vendor).
configvar GOFLAGS_VENDOR "$( [ -d vendor ] && echo '-mod=vendor' )" "Go flags for using the vendor directory"

# Go versions can be specified seperately for different tasks
# If the pre-installed Go is missing or a different
# version, the required version here will get installed
Expand All @@ -101,7 +107,8 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version
# kind version to use. If the pre-installed version is different,
# the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/
# (if available), otherwise it is built from source.
configvar CSI_PROW_KIND_VERSION "v0.5.0" "kind"
# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39
configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind"

# ginkgo test runner version to use. If the pre-installed version is
# different, the desired version is built from source.
Expand Down Expand Up @@ -136,7 +143,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes"
#
# If the version is prefixed with "release-", then nothing
# is overridden.
override_k8s_version "1.13.10"
override_k8s_version "1.14.6"
override_k8s_version "1.15.3"

Expand Down Expand Up @@ -183,7 +189,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip
#
# When no deploy script is found (nothing in `deploy` directory,
# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed.
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0-rc8" "hostpath driver"
configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver"
configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo"
configvar CSI_PROW_DEPLOYMENT "" "deployment"
configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name"
Expand All @@ -200,9 +206,10 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image"
# all generated files are present.
#
# CSI_PROW_E2E_REPO=none disables E2E testing.
configvar CSI_PROW_E2E_VERSION_1_13 v1.14.0 "E2E version for Kubernetes 1.13.x" # we can't use the one from 1.13.x because it didn't have --storage.testdriver
# TOOO: remove versioned variables and make e2e version match k8s version
configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x"
configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x"
configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x"
# TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases
configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version
configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions
Expand Down Expand Up @@ -292,11 +299,6 @@ regex_join () {
# alpha in previous Kubernetes releases. This was considered too
# error prone. Therefore we use E2E tests that match the Kubernetes
# version that is getting tested.
#
# However, for 1.13.x testing we have to use the E2E tests from 1.14
# because 1.13 didn't have --storage.testdriver yet, so for that (and only
# that version) we have to define alpha tests differently.
configvar CSI_PROW_E2E_ALPHA_1_13 '\[Feature: \[Testpattern:.Dynamic.PV..block.volmode.\] should.create.and.delete.block.persistent.volumes' "alpha tests for Kubernetes 1.13" # Raw block was an alpha feature in 1.13.
configvar CSI_PROW_E2E_ALPHA_LATEST '\[Feature:' "alpha tests for Kubernetes >= 1.14" # there's no need to update this, adding a new case for CSI_PROW_E2E for a new Kubernetes is enough
configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi_prow_kubernetes_version_suffix}")" "alpha tests"

Expand All @@ -312,12 +314,12 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi
# kubernetes-csi components must be updated, either by disabling
# the failing test for "latest" or by updating the test and not running
# it anymore for older releases.
configvar CSI_PROW_E2E_ALPHA_GATES_1_13 'VolumeSnapshotDataSource=true,BlockVolume=true,CSIBlockVolume=true' "alpha feature gates for Kubernetes 1.13"
configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14"
configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15"
configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16"
# TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and
# add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST.
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes"
configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates"

# Some tests are known to be unusable in a KinD cluster. For example,
Expand Down Expand Up @@ -681,16 +683,16 @@ EOF

}

# Gets logs of all containers in the default namespace. When passed -f, kubectl will
# Gets logs of all containers in all namespaces. When passed -f, kubectl will
# keep running and capture new output. Prints the pid of all background processes.
# The caller must kill (when using -f) and/or wait for them.
#
# May be called multiple times and thus appends.
start_loggers () {
kubectl get pods -o go-template --template='{{range .items}}{{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r pod containers; do
kubectl get pods --all-namespaces -o go-template --template='{{range .items}}{{.metadata.namespace}} {{.metadata.name}} {{range .spec.containers}}{{.name}} {{end}}{{"\n"}}{{end}}' | while read -r namespace pod containers; do
for container in $containers; do
mkdir -p "${ARTIFACTS}/$pod"
kubectl logs "$@" "$pod" "$container" >>"${ARTIFACTS}/$pod/$container.log" &
mkdir -p "${ARTIFACTS}/$namespace/$pod"
kubectl logs -n "$namespace" "$@" "$pod" "$container" >>"${ARTIFACTS}/$namespace/$pod/$container.log" &
echo "$!"
done
done
Expand Down Expand Up @@ -723,22 +725,6 @@ install_sanity () (
run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed"
)

# Whether the hostpath driver supports raw block devices depends on which version
# we are testing. It would be much nicer if we could determine that by querying the
# installed driver's capabilities instead of having to do a version check.
hostpath_supports_block () {
local result
result="$(docker exec csi-prow-control-plane docker image ls --format='{{.Repository}} {{.Tag}} {{.ID}}' | grep hostpath | while read -r repo tag id; do
if [ "$tag" == "v1.0.1" ]; then
# Old version because the revision label is missing: didn't have support yet.
echo "false"
return
fi
done)"
# If not set, then it must be a newer driver with support.
echo "${result:-true}"
}
# The default implementation of this function generates a external
# driver test configuration for the hostpath driver.
#
Expand All @@ -755,12 +741,14 @@ SnapshotClass:
DriverInfo:
Name: ${CSI_PROW_HOSTPATH_DRIVER_NAME}
Capabilities:
block: $(hostpath_supports_block)
block: true
persistence: true
dataSource: true
multipods: true
nodeExpansion: true
controllerExpansion: true
snapshotDataSource: true
singleNodeVolume: true
EOF
}

Expand Down Expand Up @@ -946,7 +934,7 @@ main () {
images=
if ${CSI_PROW_BUILD_JOB}; then
# A successful build is required for testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all || die "'make all' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make all' failed"
# We don't want test failures to prevent E2E testing below, because the failure
# might have been minor or unavoidable, for example when experimenting with
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
Expand All @@ -956,13 +944,13 @@ main () {
warn "installing 'dep' failed, cannot test vendoring"
ret=1
fi
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test 2>&1 | make_test_to_junit; then
if ! run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make -k test "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" 2>&1 | make_test_to_junit; then
warn "'make test' failed, proceeding anyway"
ret=1
fi
fi
# Required for E2E testing.
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container || die "'make container' failed"
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
fi

if tests_need_kind; then
Expand Down
8 changes: 5 additions & 3 deletions release-tools/travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,20 @@ language: go
sudo: required
services:
- docker
git:
depth: false
matrix:
include:
- go: 1.12.4
- go: 1.13.3
before_script:
- mkdir -p bin
- wget https://github.com/golang/dep/releases/download/v0.5.1/dep-linux-amd64 -O bin/dep
- chmod u+x bin/dep
- export PATH=$PWD/bin:$PATH
script:
- make -k all test
- make -k all test GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' )
after_success:
- if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}" quay.io;
make push;
make push GOFLAGS_VENDOR=$( [ -d vendor ] && echo '-mod=vendor' );
fi
23 changes: 23 additions & 0 deletions release-tools/update-vendor.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

if [ -f Gopkg.toml ]; then
echo "Repo uses 'dep' for vendoring."
(set -x; dep ensure)
elif [ -f go.mod ]; then
release-tools/verify-go-version.sh "go"
(set -x; env GO111MODULE=on go mod tidy && env GO111MODULE=on go mod vendor)
fi
51 changes: 51 additions & 0 deletions release-tools/verify-go-version.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/usr/bin/env bash

# Copyright 2019 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

GO="$1"

if [ ! "$GO" ]; then
echo >&2 "usage: $0 <path to go binary>"
exit 1
fi

die () {
echo "ERROR: $*"
exit 1
}

version=$("$GO" version) || die "determining version of $GO failed"
# shellcheck disable=SC2001
majorminor=$(echo "$version" | sed -e 's/.*go\([0-9]*\)\.\([0-9]*\).*/\1.\2/')
# shellcheck disable=SC2001
expected=$(grep "^ *- go:" "release-tools/travis.yml" | sed -e 's/.*go: *\([0-9]*\)\.\([0-9]*\).*/\1.\2/')

if [ "$majorminor" != "$expected" ]; then
cat >&2 <<EOF
======================================================
WARNING
This projects is tested with Go v$expected.
Your current Go version is v$majorminor.
This may or may not be close enough.
In particular test-gofmt and test-vendor
are known to be sensitive to the version of
Go.
======================================================
EOF
fi
Loading

0 comments on commit a69270c

Please sign in to comment.