diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..5bcb44d1f3 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,115 @@ +version: 2 +defaults: &defaults + docker: + - image: weaveworks/weavebuild:circle20-2e6504d3 + working_directory: /go/src/github.com/weaveworks/weave + +jobs: + lint: + <<: *defaults + steps: + - checkout + - run: git submodule update --init + - run: make DEBUG=1 BUILD_IN_CONTAINER=false lint + + unit-test: + <<: *defaults + parallelism: 2 + steps: + - checkout + - run: COVERDIR=test/coverage make BUILD_IN_CONTAINER=false tests + - persist_to_workspace: + root: . + paths: + - test/coverage + + build: + <<: *defaults + steps: + - checkout + - setup_remote_docker + - run: git submodule update --init + - run: make COVERAGE=true BUILD_IN_CONTAINER=false SUDO= exes all + - persist_to_workspace: + root: . + paths: + - weave.tar.gz + - tools/runner/runner + - test/tls/tls + + smoke-tests: + machine: + image: circleci/classic:201709-01 + working_directory: /home/circleci/src/github.com/weaveworks/weave + environment: + SRCDIR: /home/circleci/src/github.com/weaveworks/weave + CIRCLE_ARTIFACTS: /tmp/artifacts + parallelism: 2 + steps: + - checkout + - run: mkdir $CIRCLE_ARTIFACTS + - run: git submodule update --init + - attach_workspace: + at: . + # kick off creation of test VMs + - run: bin/provision_test_vms.sh + # when VMs are ready, copy built software to them + - run: bin/circle-test-pre + - run: + command: bin/circle-test-smoke + no_output_timeout: 360s + # Destroy testing VMs: + - run: + command: bin/circle-destroy-vms + background: true + - persist_to_workspace: + root: . + paths: + - test/coverage + - store_artifacts: + path: /tmp/artifacts + + gen-coverage: + <<: *defaults + environment: + CIRCLE_ARTIFACTS: /tmp/artifacts + steps: + - checkout + - run: mkdir $CIRCLE_ARTIFACTS + - attach_workspace: + at: . + - run: cd test; ./gen_coverage_reports.sh + - run: goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=test/profile.cov -service=circleci + - run: cp test/coverage.* $CIRCLE_ARTIFACTS + - store_artifacts: + path: /tmp/artifacts + + deploy: + <<: *defaults + steps: + - setup_remote_docker + # Rebuild without coverage and push to dockerhub + - deploy: bin/circle-deploy-master + +workflows: + version: 2 + build_test: + jobs: + - lint + - unit-test + - build + - smoke-tests: + requires: + - lint + - unit-test + - build + - gen-coverage: + requires: + - unit-test + - smoke-tests + - deploy: + requires: + - smoke-tests + filters: + branches: + only: master diff --git a/.lintignore b/.lintignore index f6b9efd061..9412318290 100644 --- a/.lintignore +++ b/.lintignore @@ -4,7 +4,6 @@ # avoid breaking the build on make lint: bin/circle-deploy-issues bin/circle-deploy-master -bin/circle-teardown-pre bin/circle-test-teardown bin/circle-test-unit bin/install-wordepress diff --git a/Makefile b/Makefile index 770ba4d186..6351d35977 100644 --- a/Makefile +++ b/Makefile @@ -246,6 +246,7 @@ endif # It also makes sure the multiarch hooks are reqistered in the kernel so the QEMU emulation works $(BUILD_UPTODATE): build/* $(SUDO) docker build -t $(BUILD_IMAGE) build/ + $(SUDO) docker tag $(BUILD_IMAGE) $(BUILD_IMAGE):$(shell tools/image-tag) $(SUDO) docker run --rm --privileged multiarch/qemu-user-static:register --reset touch $@ diff --git a/bin/circle-dependencies-post b/bin/circle-dependencies-post deleted file mode 100755 index 1b705fc638..0000000000 --- a/bin/circle-dependencies-post +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e - -source "$STATE" - -if [ -n "$TEST_AND_PUBLISH" ]; then - cd "$SRCDIR" - git submodule update --init - cd "$SRCDIR/build" - ../tools/rebuild-image weaveworks/weavebuild . Dockerfile build.sh - touch "$SRCDIR/.build.uptodate" - sudo chown -R ubuntu /usr/local/go - cd "$SRCDIR" - make testrunner - make COVERAGE=true RM= exes all -fi diff --git a/bin/circle-dependencies-post-sources b/bin/circle-dependencies-post-sources deleted file mode 100755 index f6875ced92..0000000000 --- a/bin/circle-dependencies-post-sources +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash - -set -e - -echo "TEST_AND_PUBLISH=1" >"$STATE" - -# Avoid doing a full build if we can. Note if the PR doesn't exist -# when the build is kicked off, we can't run this check so we do a -# full build. Subsequent pushes to the same branch will have the desired behaviour -if [ -n "$CI_PULL_REQUEST" -a "$CI_PULL_REQUEST" = "$CI_PULL_REQUESTS" ]; then - if PR=$(echo "$CI_PULL_REQUEST" | grep -oP '(?<=^https://github.com/weaveworks/weave/pull/)[0-9]+$'); then - echo "GH_PR=$PR" >>"$STATE" - if FILES=$(curl -s "https://api.github.com/repos/weaveworks/weave/pulls/$PR/files" | jq -r '.[]|.filename'); then - echo "PR touches [$FILES]" - if ! echo "$FILES" | grep -v ^site/; then - echo "PR contains site changes only - skipping tests" - echo "TEST_AND_PUBLISH=" >>"$STATE" - else - echo "PR contains non-site changes - running tests" - fi - else - echo "Unable to get files for PR $PR" >&2 - fi - else - echo "Unable to extract PR number from $CI_PULL_REQUEST" >&2 - fi -fi - -source "$STATE" - -mkdir -p "$(dirname "$SRCDIR")" && cp -r "$(pwd)/" "$SRCDIR" - -if [ -n "$TEST_AND_PUBLISH" ]; then - cd "$SRCDIR" - git submodule update --init -fi diff --git a/bin/circle-deploy-master b/bin/circle-deploy-master index 4c0efb5678..5a8dc34694 100755 --- a/bin/circle-deploy-master +++ b/bin/circle-deploy-master @@ -2,12 +2,8 @@ set -e -source "$STATE" - -if [ -n "$TEST_AND_PUBLISH" ] ; then - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS - cd $SRCDIR - make clean-bin - # CircleCI doesn't let us install the qemu module to do ARM docker builds - make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only -fi +docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS +cd $SRCDIR +make clean-bin +# CircleCI doesn't let us install the qemu module to do ARM docker builds +make publish ALL_ARCHITECTURES=amd64 UPDATE_LATEST=latest-only SUDO= BUILD_IN_CONTAINER=false diff --git a/bin/circle-destroy-vms b/bin/circle-destroy-vms new file mode 100755 index 0000000000..853d3c38c6 --- /dev/null +++ b/bin/circle-destroy-vms @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e + +source "$SRCDIR/bin/circle-env" +cd "$SRCDIR/test" +./run-integration-tests.sh destroy diff --git a/bin/circle-env b/bin/circle-env new file mode 100644 index 0000000000..aa716bc092 --- /dev/null +++ b/bin/circle-env @@ -0,0 +1,11 @@ +# Set environment variables here because CircleCI can't expand $FOO in 'env' section + +export PATH="$PATH:$HOME/.local/bin:/opt/google-cloud-sdk/bin" +export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +# Base name of VMs for integration tests: +export NAME=test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX +export TEST_VMS_PROV_AND_CONF_LOCK_FILE=$CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt +export TEST_VMS_READY_LOCK_FILE=$CIRCLE_ARTIFACTS/test_vms_ready.txt +# Google Cloud Platorm images for integration tests: +export USE_IMAGE=1 +export DISK_NAME_PREFIX=test-$CIRCLE_BUILD_NUM-0 diff --git a/bin/circle-teardown-pre b/bin/circle-teardown-pre deleted file mode 100755 index d11b1111f4..0000000000 --- a/bin/circle-teardown-pre +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -e - -source "$STATE" - -if [ -n "$TEST_AND_PUBLISH" ] ; then - if [ "$CIRCLE_NODE_INDEX" = "0" ] ; then - go get github.com/mattn/goveralls - goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci - cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS - fi -fi diff --git a/bin/circle-test-pre b/bin/circle-test-pre index 67254cf927..040f85662e 100755 --- a/bin/circle-test-pre +++ b/bin/circle-test-pre @@ -9,22 +9,21 @@ function signal_failure() { } trap signal_failure ERR -source "$STATE" source "$(dirname "$0")/wait_for.sh" -if [ -n "$TEST_AND_PUBLISH" ]; then - [ -n "$SECRET_KEY" ] || { - echo "Cannot run smoke tests: no secret key" - exit 1 - } +[ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 +} + +source "$SRCDIR/bin/circle-env" - # Provisioning from image typically take 90 seconds, - # but if we are creating a GCP image, it can take more than 10 minutes: - wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" +# Provisioning from image typically take 90 seconds, +# but if we are creating a GCP image, it can take more than 10 minutes: +wait_for 900 "$TEST_VMS_PROV_AND_CONF_LOCK_FILE" - # Upload built binaries to testing VMs: - cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. - ./run-integration-tests.sh setup >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 - echo "OK" >"$TEST_VMS_READY_LOCK_FILE" - echo "Test VMs now ready. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" -fi +# Upload built binaries to testing VMs: +cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. +./run-integration-tests.sh setup +echo "OK" >"$TEST_VMS_READY_LOCK_FILE" +echo "Test VMs now ready. $(date)." diff --git a/bin/circle-test-smoke b/bin/circle-test-smoke index 250841ebe7..d14cce51ac 100755 --- a/bin/circle-test-smoke +++ b/bin/circle-test-smoke @@ -2,22 +2,21 @@ set -e -source "$STATE" source "$(dirname "$0")/wait_for.sh" -if [ -n "$TEST_AND_PUBLISH" ]; then - [ -n "$SECRET_KEY" ] || { - echo "Cannot run smoke tests: no secret key" - exit 1 - } +[ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 +} - # Provisioning from image typically take 90 seconds, - # but if we are creating a GCP image, it can take more than 10 minutes: - wait_for 900 "$TEST_VMS_READY_LOCK_FILE" +source "$SRCDIR/bin/circle-env" - # Run integration tests: - export COVERAGE=true - export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" - cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs. - ./run-integration-tests.sh test -fi +# Provisioning from image typically take 90 seconds, +# but if we are creating a GCP image, it can take more than 10 minutes: +wait_for 900 "$TEST_VMS_READY_LOCK_FILE" + +# Run integration tests: +export COVERAGE=true +export WEAVE_NET_SANITY_CHECKS_FILES="$CIRCLE_ARTIFACTS/weave_net_sanity_check_*.log" +cd "$SRCDIR/test" # Ensures we generate code coverage files in the right folder, and use the Terraform state of previously provisionned VMs. +./run-integration-tests.sh test diff --git a/bin/circle-test-teardown b/bin/circle-test-teardown deleted file mode 100755 index c2e6ae25fa..0000000000 --- a/bin/circle-test-teardown +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -source "$STATE" - -if [ -n "$TEST_AND_PUBLISH" ] ; then - cd $SRCDIR/test - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh) -fi diff --git a/bin/circle-test-unit b/bin/circle-test-unit deleted file mode 100755 index b9ac9a5dc1..0000000000 --- a/bin/circle-test-unit +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -e - -source "$STATE" - -if [ -n "$TEST_AND_PUBLISH" ] ; then - cd $SRCDIR - make DEBUG=1 lint - COVERDIR=test/coverage make RM= tests -fi diff --git a/bin/provision_test_vms.sh b/bin/provision_test_vms.sh index b5641cf629..b42a32cdc3 100755 --- a/bin/provision_test_vms.sh +++ b/bin/provision_test_vms.sh @@ -9,8 +9,6 @@ function signal_failure() { } trap signal_failure ERR -source "$STATE" - function install_terraform() { curl -fsS https://releases.hashicorp.com/terraform/0.8.5/terraform_0.8.5_linux_amd64.zip | gunzip >terraform && chmod +x terraform && sudo mv terraform /usr/bin } @@ -20,24 +18,23 @@ function install_ansible() { sudo apt-get install -qq -y python-pip python-dev libffi-dev libssl-dev \ && pip install --user -U setuptools cffi \ && pip install --user ansible - export PATH="$PATH:$HOME/.local/bin" } -if [ -n "$TEST_AND_PUBLISH" ]; then - [ -n "$SECRET_KEY" ] || { - echo "Cannot run smoke tests: no secret key" - exit 1 - } +[ -n "$SECRET_KEY" ] || { + echo "Cannot run smoke tests: no secret key" + exit 1 +} + +source "$SRCDIR/bin/circle-env" - install_terraform >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 - install_ansible >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 +install_terraform +install_ansible - # Only attempt to create GCP image in first container, wait for it to be created otherwise: - [ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0 +# Only attempt to create GCP image in first container, wait for it to be created otherwise: +[ "$CIRCLE_NODE_INDEX" != "0" ] && export CREATE_IMAGE=0 - # Provision and configure testing VMs: - cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. - ./run-integration-tests.sh configure >>"$TEST_VMS_SETUP_OUTPUT_FILE" 2>&1 - echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE" - echo "Test VMs now provisioned and configured. $(date)." >>"$TEST_VMS_SETUP_OUTPUT_FILE" -fi +# Provision and configure testing VMs: +cd "$SRCDIR/test" # Ensures we generate Terraform state files in the right folder, for later use by integration tests. +./run-integration-tests.sh configure +echo "OK" >"$TEST_VMS_PROV_AND_CONF_LOCK_FILE" +echo "Test VMs now provisioned and configured. $(date)." diff --git a/bin/wait_for.sh b/bin/wait_for.sh index d9148cb194..39bb669bbf 100755 --- a/bin/wait_for.sh +++ b/bin/wait_for.sh @@ -16,6 +16,6 @@ wait_for() { if ! ((i % 10)); then echo "[$i seconds]: Waiting for $lock_file to be created..."; fi sleep 1 done - echo "Timed out waiting for test VMs to be ready. See details in: $TEST_VMS_SETUP_OUTPUT_FILE" >&2 + echo "Timed out waiting for test VMs to be ready." >&2 return 1 } diff --git a/build/Dockerfile b/build/Dockerfile index 67e9c33911..d89965835f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -50,6 +50,8 @@ RUN curl -fsSLo shfmt https://github.com/mvdan/sh/releases/download/v1.3.0/shfmt # Install common Go tools RUN go get \ + github.com/weaveworks/build-tools/cover \ + github.com/mattn/goveralls \ github.com/golang/lint/golint \ github.com/fzipp/gocyclo \ github.com/fatih/hclfmt \ @@ -86,5 +88,8 @@ RUN curl -sSL http://www.tcpdump.org/release/libpcap-${LIBPCAP_CROSS_VERSION}.ta && ln -sf libpcap.so.${MAJOR_VER} /usr/local/lib/libpcap.so \ && make clean; done +# Install Docker +RUN curl -fsSL https://get.docker.com | VERSION=18.06.0-ce /bin/sh + COPY build.sh / ENTRYPOINT ["sh", "/build.sh"] diff --git a/circle.yml b/circle.yml deleted file mode 100644 index e1cd09834b..0000000000 --- a/circle.yml +++ /dev/null @@ -1,66 +0,0 @@ -general: - branches: - ignore: - - gh\-pages - -machine: - pre: - - curl -sSL https://s3.amazonaws.com/circle-downloads/install-circleci-docker.sh | bash -s -- 1.10.0 - services: - - docker - environment: - GOPATH: /home/ubuntu:$GOPATH - SRCDIR: /home/ubuntu/src/github.com/weaveworks/weave - PATH: $PATH:$HOME/.local/bin:/opt/google-cloud-sdk/bin - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - STATE: /home/ubuntu/state.env - # Base name of VMs for integration tests: - NAME: test-$CIRCLE_BUILD_NUM-$CIRCLE_NODE_INDEX - TEST_VMS_SETUP_OUTPUT_FILE: $CIRCLE_ARTIFACTS/test_vms_setup_output.txt - TEST_VMS_PROV_AND_CONF_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_provisioned_and_configured.txt - TEST_VMS_READY_LOCK_FILE: $CIRCLE_ARTIFACTS/test_vms_ready.txt - # Google Cloud Platorm images for integration tests: - USE_IMAGE: 1 - DISK_NAME_PREFIX: test-$CIRCLE_BUILD_NUM-0 - -dependencies: - cache_directories: - - "~/docker" - post: - - bin/circle-dependencies-post-sources - # Provision & configure testing VMs: - - "while [ ! -d $SRCDIR ]; do sleep 1; done; $SRCDIR/bin/provision_test_vms.sh": - parallel: true - background: true - - bin/circle-dependencies-post - -test: - pre: - - "$SRCDIR/bin/circle-test-pre": - parallel: true - background: true - override: - - "$SRCDIR/bin/circle-test-unit": - parallel: true - - "$SRCDIR/bin/circle-test-smoke": - parallel: true - timeout: 360 - post: - - "$SRCDIR/bin/circle-test-teardown": - parallel: true - -teardown: - pre: - # Destroy testing VMs: - - "cd $SRCDIR/test && ./run-integration-tests.sh destroy": - parallel: true - background: true - - "$SRCDIR/bin/circle-teardown-pre": - parallel: true - -deployment: - master: - branch: master - owner: weaveworks - commands: - - "$SRCDIR/bin/circle-deploy-master" diff --git a/test/840_weave_kube_3_test.sh b/test/840_weave_kube_3_test.sh index 0efde493c0..2e7a5e8a8d 100755 --- a/test/840_weave_kube_3_test.sh +++ b/test/840_weave_kube_3_test.sh @@ -214,11 +214,17 @@ spec: access: deny EOF +# Allow some time for the policy change to take effect +sleep 1 + assert_raises "$SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null" # remove the access for nettest-deny run_on $HOST1 "$KUBECTL delete netpol allow-nettest-deny" +# Allow some time for the policy change to take effect +sleep 1 + # nettest-deny should still not be able to reach nettest pods assert_raises "! $SSH $HOST1 $KUBECTL exec $denyPodName -- curl -s -S -f -m 2 http://$DOMAIN:8080/status >/dev/null" diff --git a/test/gen_coverage_reports.sh b/test/gen_coverage_reports.sh index 8f9f381697..27a2e46983 100755 --- a/test/gen_coverage_reports.sh +++ b/test/gen_coverage_reports.sh @@ -3,13 +3,8 @@ set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -if [ -n "$CIRCLECI" ]; then - for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do - scp node$i:/home/ubuntu/src/github.com/weaveworks/weave/test/coverage/* ./coverage/ || true - done -fi +# 'cover' tool is from github.com/weaveworks/build-tools/cover -go get github.com/weaveworks/build-tools/cover cover ./coverage/* >profile.cov go tool cover -html=profile.cov -o coverage.html go tool cover -func=profile.cov -o coverage.txt