Squashed 'tools/' changes from f041a74ff..e6dd5e8f0

e6dd5e8f0 Merge pull request #168 from weaveworks/downgrade-werkzeug
144aa64c0 Restruct scheduler to use compatible version of werkzeug library
51a217660 Merge pull request #166 from weaveworks/165-rename-circleci-to-wksctl
bfd7a2bbe Renames circleci project wks -> wksctl afte code move
a5f7be3cc Note that the API token was revoked
15e4cfb94 Merge pull request #164 from weaveworks/improve-lint
dc8722589 Fix lint errors in Python code
8cb6a62a2 Set failure state if lint_files fails
c635ce412 Simplify the no-arguments case
167b7ed57 Skip entire directories that match filter patterns
1caa455a6 Refactor: read filter patterns once at start
4cc7911e5 Merge pull request #163 from weaveworks/fix-lint
393808dab Merge pull request #162 from weaveworks/golang-python3
3c5fcd16e Make lint run on CircleCI
f0936a1e9 lint all directories if not given a list
68f62a393 Fix typo in lint script
9ef44b057 Stop installing promtool
882df5ec5 Install python 3 instead of obsolete v2.7
988002377 Update Go to latest 1.13.3
2a2df5278 Merge pull request #161 from murali-reddy/terraform-update
bf8ed23b9 update google_compute_instance arguments as per the latest documentation

git-subtree-dir: tools
git-subtree-split: e6dd5e8f04e513bbc646fd85f85e73c5f68df89b

tools/.circleci/config.yml

@@ -10,11 +10,11 @@ jobs:
     steps:
     - checkout
     - run: cd build; make
-    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./lint .
-    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c ./shell-lint .
-    - run: docker run --rm -v "$PWD/cover:/go/src/cover" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make
-    - run: docker run --rm -v "$PWD/socks:/go/src/socks" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy"
-    - run: docker run --rm -v "$PWD/runner:/go/src/runner" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:$PWD" -w "$PWD" --entrypoint sh weaveworks/build-golang -c "./shell-lint ."
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/cover" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/socks" --entrypoint sh weaveworks/build-golang -c "make proxy"
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src/runner" --entrypoint sh weaveworks/build-golang -c make
+    - run: docker run --rm -v "$PWD:/go/src" -w "/go/src" --entrypoint sh weaveworks/build-golang -c "./lint ./build ./config_management ./cover ./dependencies ./integration ./provisioning ./runner ./scheduler ./socks"
 
     - deploy:
         command: |

tools/build/golang/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.1-stretch
+FROM golang:1.13.5-stretch
 RUN apt-get update && \
     apt-get install -y \
       curl \
@@ -8,13 +8,16 @@ RUN apt-get update && \
       libprotobuf-dev \
       make \
       protobuf-compiler \
-      python-pip \
+      python3 \
+      python3-pip \
       python-requests \
       python-yaml \
+      libssl-dev \
+      python-openssl \
       shellcheck \
       unzip && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
-RUN pip install attrs pyhcl yapf==0.16.2 flake8==3.3.0
+RUN pip3 install attrs==19.2.0 pyhcl yapf==0.16.2 flake8==3.3.0
 RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt_v1.3.0_linux_amd64 && \
 	echo "b1925c2c405458811f0c227266402cf1868b4de529f114722c2e3a5af4ac7bb2  shfmt" | sha256sum -c && \
 	chmod +x shfmt && \
@@ -34,7 +37,7 @@ RUN go get -tags netgo \
 		github.com/golang/protobuf/protoc-gen-go \
 		github.com/kisielk/errcheck \
 		github.com/mjibson/esc \
-		github.com/prometheus/prometheus/cmd/promtool && \
+		&& \
 		rm -rf /go/pkg /go/src
 RUN mkdir protoc && \
 	cd protoc && \

tools/dependencies/cross_versions.py

@@ -80,8 +80,9 @@ def cross_versions(config):
 def main(argv):
     try:
         config = _validate_input(argv)
-        print(linesep.join('\t'.join(triple)
-                           for triple in cross_versions(config)))
+        print(
+            linesep.join('\t'.join(triple)
+                         for triple in cross_versions(config)))
     except Exception as e:
         print(str(e))
         exit(_ERROR_RUNTIME)

tools/lint

@@ -219,12 +219,16 @@ matches_any() {
     return 1
 }
 
-filter_out() {
+read_patterns() {
     local patterns_file="$1"
     if [ -n "$patterns_file" ] && [ -r "$patterns_file" ]; then
-        local patterns
-        patterns=$(sed '/^#.*$/d ; /^\s*$/d' "$patterns_file") # Remove blank lines and comments before we start iterating.
-        [ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$patterns"
+        sed '/^#.*$/d ; /^\s*$/d' "$patterns_file" # Remove blank lines and comments.
+    fi
+}
+
+filter_out() {
+    local patterns="$1"
+    if [ -n "$patterns" ]; then
         local filtered_out=()
         while read -r filename; do
             matches_any "$filename" "$patterns" && filtered_out+=("$filename") || echo "$filename"
@@ -238,11 +242,13 @@ filter_out() {
 lint_directory() {
     local dirname="$1"
     local lint_result=0
+    matches_any "$dirname" "$PATTERNS" && return 0
+    [ -n "$DEBUG" ] && echo >&2 "> Linting directory: $dirname"
     # This test is just checking if there are any Go files in the directory
     if compgen -G "$dirname/*.go" >/dev/null; then
         lint_go "${dirname}" || lint_result=1
     fi
-    find . -maxdepth 1 "$dirname" | filter_out "$LINT_IGNORE_FILE" | lint_files
+    find "$dirname" -maxdepth 1 | filter_out "$PATTERNS" | lint_files || lint_result=1
     return $lint_result
 }
 
@@ -255,11 +261,12 @@ lint_directories() {
 }
 
 list_directories() {
-    if [ $# -gt 0 ]; then
-        find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
-    fi
+    find "$@" \( -name vendor -o -name .git -o -name .cache -o -name .pkg \) -prune -o -type d
 }
 
+PATTERNS=$(read_patterns "$LINT_IGNORE_FILE")
+[ -n "$DEBUG" ] && echo >&2 "> Filters:" && echo >&2 "$PATTERNS"
+
 if [ $# = 1 ] && [ -f "$1" ]; then
     lint "$1"
 else

tools/provisioning/gcp/main.tf

@@ -18,8 +18,10 @@ resource "google_compute_instance" "tf_test_vm" {
   zone         = "${var.gcp_zone}"
   count        = "${var.num_hosts}"
 
-  disk {
-    image = "${var.gcp_image}"
+  boot_disk {
+    initialize_params {
+      image = "${var.gcp_image}"
+    }
   }
 
   tags = [

tools/provisioning/gcp/outputs.tf

@@ -3,11 +3,11 @@ output "username" {
 }
 
 output "public_ips" {
-  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip}"]
+  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip}"]
 }
 
 output "private_ips" {
-  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.address}"]
+  value = ["${google_compute_instance.tf_test_vm.*.network_interface.0.network_ip}"]
 }
 
 output "hostnames" {
@@ -24,7 +24,7 @@ output "hostnames" {
 output "private_etc_hosts" {
   value = "${join("\n", 
     "${formatlist("%v %v.%v.%v", 
-      google_compute_instance.tf_test_vm.*.network_interface.0.address, 
+      google_compute_instance.tf_test_vm.*.network_interface.0.network_ip,
       google_compute_instance.tf_test_vm.*.name, 
       google_compute_instance.tf_test_vm.*.zone, 
       var.app
@@ -36,7 +36,7 @@ output "private_etc_hosts" {
 output "public_etc_hosts" {
   value = "${join("\n", 
     "${formatlist("%v %v.%v.%v", 
-      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip, 
+      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip,
       google_compute_instance.tf_test_vm.*.name, 
       google_compute_instance.tf_test_vm.*.zone, 
       var.app
@@ -47,8 +47,8 @@ output "public_etc_hosts" {
 output "ansible_inventory" {
   value = "${format("[all]\n%s", join("\n",
     "${formatlist("%v private_ip=%v",
-      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.assigned_nat_ip,
-      google_compute_instance.tf_test_vm.*.network_interface.0.address
+      google_compute_instance.tf_test_vm.*.network_interface.0.access_config.0.nat_ip,
+      google_compute_instance.tf_test_vm.*.network_interface.0.network_ip
     )}"
   ))}"
 }

tools/scheduler/main.py

@@ -82,14 +82,14 @@ def avg(test):
     test_times_dict = dict(test_times)
     test_times.sort(key=operator.itemgetter(1))
 
-    shards = {i: [] for i in xrange(shard_count)}
+    shards = {i: [] for i in range(shard_count)}
     while test_times:
         test_name, time = test_times.pop()
 
         # find shortest shard and put it in that
         s, _ = min(
             ((i, sum(test_times_dict[t] for t in shards[i]))
-             for i in xrange(shard_count)),
+             for i in range(shard_count)),
             key=operator.itemgetter(1))
 
         shards[s].append(test_name)
@@ -123,9 +123,6 @@ def _matches_any_regex(name, regexes):
             return matches
 
 
-# See also: https://circleci.com/account/api
-CIRCLE_CI_API_TOKEN = 'cffb83afd920cfa109cbd3e9eecb7511a2d18bb9'
-
 # N.B.: When adding a project below, please ensure:
 # - its CircleCI project is either public, or is followed by the user attached
 #   to the above API token
@@ -137,8 +134,7 @@ def _matches_any_regex(name, regexes):
     ('weaveworks/weave', 'positive-cocoa-90213', 'us-central1-a', True, None),
     ('weaveworks/scope', 'scope-integration-tests', 'us-central1-a', False,
      None),
-    ('weaveworks/wks', 'wks-tests', 'us-central1-a', True,
-     CIRCLE_CI_API_TOKEN),
+    ('weaveworks/wksctl', 'wks-tests', 'us-central1-a', True, None),
 ]
 
 

tools/scheduler/requirements.txt

@@ -1,2 +1,3 @@
 flask==1.0.0
 google-api-python-client==1.6.7
+werkzeug<1.0