Merge pull request #3698 from weaveworks/update-backend-build

build: update CI to use Go 1.13

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 defaults: &defaults
   working_directory: /go/src/github.com/weaveworks/scope
   docker:
-    - image: weaveworks/scope-backend-build:master-fda40b83
+    - image: weaveworks/scope-backend-build:master-14d4ecc3
 
 client-defaults: &client-defaults
   working_directory: /home/weave/scope

Makefile

@@ -129,7 +129,6 @@ tests: $(CODECGEN_TARGETS) prog/staticui/staticui.go prog/externalui/externalui.
 
 lint:
 	./tools/lint
-	./tools/shell-lint tools
 
 prog/staticui/staticui.go:
 	mkdir -p prog/staticui

tools/.circleci/config.yml

@@ -0,0 +1,35 @@
+version: 2
+jobs:
+  build:
+    # Use 'machine' type so we can run the lint step with directory mounted
+    machine:
+      docker_layer_caching: true
+    working_directory: /home/circleci/src/github.com/weaveworks/build-tools
+    environment:
+      GOPATH: /home/circleci/
+    steps:
+    - checkout
+    - run: cd build; make
+    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint .
+    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./shell-lint .
+    - run: docker run --rm -v "$PWD/cover:/go/src/cover" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD/socks:/go/src/socks" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy"
+    - run: docker run --rm -v "$PWD/runner:/go/src/runner" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make
+
+    - deploy:
+        command: |
+          if [ "${CIRCLE_BRANCH}" == "master" ]; then
+            cd build
+            docker login -u $DOCKER_REGISTRY_USER -p $DOCKER_REGISTRY_PASSWORD
+            for image in $(make images); do
+              # Push all tags - latest and git-tag
+              docker push "${image}"
+
+              # Tag the built images with something derived from the base images in
+              # their respective Dockerfiles. So "FROM golang:1.8.0-stretch" as a
+              # base image would lead to a tag of "1.8.0-stretch"
+              IMG_TAG=$(make "imagetag-${image#weaveworks/build-}")
+              docker tag "${image}:latest" "${image}:${IMG_TAG}"
+              docker push "${image}:${IMG_TAG}"
+            done
+          fi

tools/build/Makefile

@@ -3,7 +3,7 @@
 
 # Boiler plate for bulding Docker containers.
 # All this must go at top of file I'm afraid.
-IMAGE_PREFIX := quay.io/weaveworks/build-
+IMAGE_PREFIX := weaveworks/build-
 IMAGE_TAG := $(shell ../image-tag)
 GIT_REVISION := $(shell git rev-parse HEAD)
 UPTODATE := .uptodate

tools/build/golang/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.11.1-stretch
+FROM golang:1.13.1-stretch
 RUN apt-get update && \
     apt-get install -y \
       curl \

tools/cover/Makefile

@@ -4,7 +4,7 @@ all: cover
 
 cover: *.go
 	go get -tags netgo ./$(@D)
-	go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D)
+	go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D)
 
 clean:
 	rm -rf cover

tools/image-tag

@@ -6,4 +6,7 @@ set -o pipefail
 
 WORKING_SUFFIX=$(if git status --porcelain | grep -qE '^(?:[^?][^ ]|[^ ][^?])\s'; then echo "-WIP"; else echo ""; fi)
 BRANCH_PREFIX=$(git rev-parse --abbrev-ref HEAD)
-echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short HEAD)$WORKING_SUFFIX"
+
+# Fix the object name prefix length to 8 characters to have it consistent across the system.
+# See https://git-scm.com/docs/git-rev-parse#Documentation/git-rev-parse.txt---shortlength
+echo "${BRANCH_PREFIX//\//-}-$(git rev-parse --short=8 HEAD)$WORKING_SUFFIX"

tools/lint

@@ -22,7 +22,6 @@ LINT_IGNORE_FILE=${LINT_IGNORE_FILE:-".lintignore"}
 
 IGNORE_LINT_COMMENT=
 IGNORE_SPELLINGS=
-PARALLEL=
 while true; do
     case "$1" in
         -nocomment)
@@ -37,10 +36,6 @@ while true; do
             IGNORE_SPELLINGS="$2,$IGNORE_SPELLINGS"
             shift 2
             ;;
-        -p)
-            PARALLEL=1
-            shift 1
-            ;;
         *)
             break
             ;;
@@ -65,6 +60,7 @@ spell_check() {
 }
 
 lint_go() {
+    # This function is called on a whole directory containing Go files
     local filename="$1"
     local lint_result=0
 
@@ -73,7 +69,7 @@ lint_go() {
         echo "${filename}: run gofmt -s -w ${filename}"
     fi
 
-    go tool vet "${filename}" || lint_result=$?
+    go vet "${filename}" || lint_result=$?
 
     # golint is completely optional.  If you don't like it
     # don't have it installed.
@@ -185,7 +181,7 @@ lint() {
 
     case "$mimetype.$ext" in
         text/x-shellscript.*) lint_sh "${filename}" || lint_result=1 ;;
-        *.go) lint_go "${filename}" || lint_result=1 ;;
+        *.go) ;; # done at directory level
         *.tf) lint_tf "${filename}" || lint_result=1 ;;
         *.md) lint_md "${filename}" || lint_result=1 ;;
         *.py) lint_py "${filename}" || lint_result=1 ;;
@@ -208,7 +204,7 @@ lint_files() {
     while read -r filename; do
         lint "${filename}" || lint_result=1
     done
-    exit $lint_result
+    return $lint_result
 }
 
 matches_any() {
@@ -239,18 +235,33 @@ filter_out() {
     fi
 }
 
-list_files() {
+lint_directory() {
+    local dirname="$1"
+    local lint_result=0
+    # This test is just checking if there are any Go files in the directory
+    if compgen -G "$dirname/*.go" >/dev/null; then
+        lint_go "${dirname}" || lint_result=1
+    fi
+    find . -maxdepth 1 "$dirname" | filter_out "$LINT_IGNORE_FILE" | lint_files
+    return $lint_result
+}
+
+lint_directories() {
+    local lint_result=0
+    while read -r dirname; do
+        lint_directory "${dirname}" || lint_result=1
+    done
+    exit $lint_result
+}
+
+list_directories() {
     if [ $# -gt 0 ]; then
-        find "$@" \( -name vendor -o -name .git \) -prune -o -type f
-    else
-        git ls-files --exclude-standard | grep -vE '(^|/)vendor/'
+        find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
     fi
 }
 
 if [ $# = 1 ] && [ -f "$1" ]; then
     lint "$1"
-elif [ -n "$PARALLEL" ]; then
-    list_files "$@" | filter_out "$LINT_IGNORE_FILE" | xargs -n1 -P16 "$0"
 else
-    list_files "$@" | filter_out "$LINT_IGNORE_FILE" | lint_files
+    list_directories "$@" | lint_directories
 fi

tools/runner/Makefile

@@ -4,7 +4,7 @@ all: runner
 
 runner: *.go
 	go get -tags netgo ./$(@D)
-	go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D)
+	go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D)
 
 clean:
 	rm -rf runner

tools/socks/Makefile

@@ -21,7 +21,7 @@ $(IMAGE_TAR): Dockerfile $(PROXY_EXE)
 
 $(PROXY_EXE): *.go
 	go get -tags netgo ./$(@D)
-	go build -ldflags "-extldflags \"-static\" -linkmode=external" -tags netgo -o $@ ./$(@D)
+	go build -ldflags "-extldflags \"-static\"" -tags netgo -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 clean:

tools/test

@@ -60,7 +60,7 @@ while [ $# -gt 0 ]; do
     esac
 done
 
-GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout $TIMEOUT $VERBOSE)
+GO_TEST_ARGS=(-tags "${TAGS[@]}" -cpu 4 -timeout "$TIMEOUT" $VERBOSE)
 
 if [ -n "$SLOW" ] || [ -n "$CIRCLECI" ]; then
     SLOW=true
@@ -100,9 +100,6 @@ fi
 
 PACKAGE_BASE=$(go list -e ./)
 
-# Speed up the tests by compiling and installing their dependencies first.
-go test -i "${GO_TEST_ARGS[@]}" "${TESTDIRS[@]}"
-
 run_test() {
     local dir=$1
     if [ -z "$NO_GO_GET" ]; then
@@ -118,7 +115,7 @@ run_test() {
         ) | paste -s -d, -)
         local output
         output=$(mktemp "$coverdir/unit.XXXXXXXXXX")
-        local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" -coverprofile=$output -coverpkg=$COVERPKGS)
+        local GO_TEST_ARGS_RUN=("${GO_TEST_ARGS[@]}" "-coverprofile=$output" "-coverpkg=$COVERPKGS")
     fi
 
     local START