From a6ee996adc13b07ef385d491e33f42610460a5a3 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Tue, 28 Jan 2020 14:11:03 +0000 Subject: [PATCH 01/18] moving build pipeline to use devcontainer --- azure-pipelines.yaml | 137 ++++++++++++++++--------------------------- ci.sh | 38 ++++++++++++ 2 files changed, 88 insertions(+), 87 deletions(-) create mode 100644 ci.sh diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 91809d1..4b2018b 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -7,103 +7,66 @@ pool: vmImage: 'Ubuntu 16.04' variables: - GOROOT: '/usr/local/go1.12' # Go installation path - GOPATH: '$(System.DefaultWorkingDirectory)/gopath' # Go workspace path - GOBIN: '$(GOPATH)/bin' # Go binaries path - MODULE_PATH: '$(GOPATH)/src/$(Build.Repository.Name)' # Path to the module's code IMAGE_NAME: 'candidate/k8s/azure-databricks/operator:$(Build.SourceVersion)' # ACR_NAME: - set this value in Azure Devops variables # AZURE_SUBSCRIPTION: - set this value in Azure Devops variables # DATABRICKS_HOST: - set this value in Azure Devops variables # DATABRICKS_TOKEN: - set this value in Azure Devops variables -steps: +jobs: +- job: Build + steps: + # Cache the docker image file + - task: CacheBeta@0 + inputs: + key: go-cache | go.sum + path: ".gocache" + restoreKeys: go-cache + displayName: Cache go mod cache -- checkout: self # self represents the repo where the initial Pipelines YAML file was found - lfs: "true" + # Cache the docker image file + - task: CacheBeta@0 + inputs: + key: docker-image | .devcontainer/** + path: ".dockercache" + restoreKeys: docker-image + cacheHitVar: DOCKER_CACHE_HIT + displayName: Cache docker layers -- script: | - set -x - ls -l /usr/local/go* - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(MODULE_PATH)' - shopt -s extglob - mv !(gopath) '$(MODULE_PATH)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - go version - go env - displayName: 'Set up the Go workspace' - env: - GO111MODULE: on + - script: | + bash -f ./ci.sh + displayName: 'Run CI' -- script: | - set -x - mkdir -p '$(MODULE_PATH)/bin' - os=$(go env GOOS) - arch=$(go env GOARCH) - # download kubebuilder - curl -sL https://go.kubebuilder.io/dl/2.2.0/${os}/${arch} | tar -xz -C $(MODULE_PATH)/bin - mv $(MODULE_PATH)/bin/kubebuilder_2.2.0_${os}_${arch}/bin/* $(MODULE_PATH)/bin/ - # download kustomize - curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.4.0/kustomize_v3.4.0_${os}_${arch}.tar.gz | tar -xz -C /tmp/ - mv /tmp/kustomize $(MODULE_PATH)/bin/kustomize - # set permission - chmod a+x $(MODULE_PATH)/bin/* - # export path - export PATH=$PATH:$(MODULE_PATH)/bin - # Install golangci-linter - curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.21.0 - # install test dependencies - go mod download - make install-test-dependency - # run test - make test - continueOnError: 'false' - displayName: 'Install kubebuilder, kustomize and Run unit tests' - env: - GO111MODULE: on - KUBEBUILDER_ASSETS: $(MODULE_PATH)/bin - DATABRICKS_HOST: $(DATABRICKS_HOST) - DATABRICKS_TOKEN: $(DATABRICKS_TOKEN) - workingDirectory: '$(MODULE_PATH)' + - task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: Cobertura + summaryFileLocation: $(System.DefaultWorkingDirectory)/**/cover.xml + reportDirectory: $(System.DefaultWorkingDirectory)/**/cover - -- task: PublishCodeCoverageResults@1 - inputs: - codeCoverageTool: Cobertura - summaryFileLocation: $(System.DefaultWorkingDirectory)/**/cover.xml - reportDirectory: $(System.DefaultWorkingDirectory)/**/cover + - script: docker build -t $(IMAGE_NAME) . # add options to this command to meet your needs + condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: 'Docker build' + - task: AzureCLI@1 + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + displayName: Push to Azure Container Registry + inputs: + azureSubscription: $(AZURE_SUBSCRIPTION) + scriptLocation: inlineScript + failOnStandardError: false + inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) . -- script: docker build -t $(IMAGE_NAME) . # add options to this command to meet your needs - condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: 'Docker build' - workingDirectory: '$(MODULE_PATH)' + - script: | + ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml + set -x + echo $(IMAGE_NAME) > $(Build.ArtifactStagingDirectory)/azure-databricks-operator.txt + ls $(Build.ArtifactStagingDirectory) + continueOnError: 'false' + displayName: 'Prepare manifests for publish' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) -- task: AzureCLI@1 - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: Push to Azure Container Registry - inputs: - azureSubscription: $(AZURE_SUBSCRIPTION) - scriptLocation: inlineScript - failOnStandardError: false - workingDirectory: '$(MODULE_PATH)' - inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) . - -- script: | - ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml - set -x - echo $(IMAGE_NAME) > $(Build.ArtifactStagingDirectory)/azure-databricks-operator.txt - ls $(Build.ArtifactStagingDirectory) - continueOnError: 'false' - displayName: 'Prepare manifests for publish' - workingDirectory: '$(MODULE_PATH)' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - -- task: PublishBuildArtifacts@1 - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - artifactName: drop + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + artifactName: drop diff --git a/ci.sh b/ci.sh new file mode 100644 index 0000000..54149f2 --- /dev/null +++ b/ci.sh @@ -0,0 +1,38 @@ +#! /bin/bash + +set -e +set -x + +# Get storage drive details +docker info + +# Create .dockercache directory +mkdir -p ./.dockercache/ + +# Import devcontainer from cache to speed up build +if [ -f ".dockercache/devcontainer.tar" ]; +then + echo "-------> Restoring docker image" + time docker load -i .dockercache/devcontainer.tar +fi + +echo "-------> Building devcontainer" +# Use the devcontainer to run the build as it has all the environment setup that we need +time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer + +# Create a directory for go mod cache +mkdir -p ${PWD}/.gocache + +echo "-------> Building code and running tests" +# Run `make` to build and test the code +time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" devcontainer -c "make install-test-dependency test" + +# Ensure .gocache permmissions correct for build to save cache +sudo chown -R $USER ${PWD} + +# If the current cached image is out of date save devcontainer so it can be cached +if [ $DOCKER_CACHE_HIT != "true" ]; +then + echo "-------> Saving docker image" + time docker image save -o ./.dockercache/devcontainer.tar devcontainer +fi From 3cbc41462d720afb7407812fde19dc70752fa87d Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Tue, 28 Jan 2020 15:53:55 +0000 Subject: [PATCH 02/18] fixed gopls reference in container --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 4fb1085..a62a13b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -38,7 +38,7 @@ RUN apt-get update \ github.com/cweill/gotests/... \ golang.org/x/tools/cmd/goimports \ golang.org/x/lint/golint \ - golang.org/x/tools/cmd/gopls \ + golang.org/x/tools/gopls \ github.com/alecthomas/gometalinter \ honnef.co/go/tools/... \ github.com/golangci/golangci-lint/cmd/golangci-lint \ From 7282f35e5b84d0e699d9bf96494879bbffb7e2ae Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Tue, 28 Jan 2020 16:07:55 +0000 Subject: [PATCH 03/18] removing context from the devcontainer setup --- .devcontainer/devcontainer.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b95d5d0..a329c9f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,6 @@ "name": "Go", "dockerFile": "./Dockerfile", "workspaceFolder": "/workspace", - "context": "..", "workspaceMount": "src=${localWorkspaceFolder},dst=/workspace,type=bind", "runArgs": [ // Mount the env file From e852fff2b822a3e09b4680db9db0586b1176528f Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Tue, 28 Jan 2020 16:21:14 +0000 Subject: [PATCH 04/18] fixed makefile path --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a62a13b..472cb6b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -82,7 +82,7 @@ ENV DATABRICKS_HOST "" ENV DATABRICKS_TOKEN "" ENV KUBECONFIG="/root/.kube/kind-config-kind" -COPY ./Makefile ./ +COPY Makefile ./ RUN make install-kind RUN make install-kubebuilder RUN make install-kustomize From 129991236ed886e08436d3ce34fcc0e0bffd37df Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Tue, 28 Jan 2020 16:29:19 +0000 Subject: [PATCH 05/18] fixing makefile path --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 472cb6b..aef17c3 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -82,7 +82,7 @@ ENV DATABRICKS_HOST "" ENV DATABRICKS_TOKEN "" ENV KUBECONFIG="/root/.kube/kind-config-kind" -COPY Makefile ./ +COPY ../Makefile ./ RUN make install-kind RUN make install-kubebuilder RUN make install-kustomize From 5c3161429d8f02a4409b8b250c2693c74d67405c Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 09:36:05 +0000 Subject: [PATCH 06/18] trying to fix makefile issue --- .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 1 + ci.sh | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index aef17c3..a62a13b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -82,7 +82,7 @@ ENV DATABRICKS_HOST "" ENV DATABRICKS_TOKEN "" ENV KUBECONFIG="/root/.kube/kind-config-kind" -COPY ../Makefile ./ +COPY ./Makefile ./ RUN make install-kind RUN make install-kubebuilder RUN make install-kustomize diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a329c9f..c12b3e9 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,6 +3,7 @@ "name": "Go", "dockerFile": "./Dockerfile", "workspaceFolder": "/workspace", + "context": "..", "workspaceMount": "src=${localWorkspaceFolder},dst=/workspace,type=bind", "runArgs": [ // Mount the env file diff --git a/ci.sh b/ci.sh index 54149f2..6766c1a 100644 --- a/ci.sh +++ b/ci.sh @@ -16,6 +16,9 @@ then time docker load -i .dockercache/devcontainer.tar fi +# Copy the makefile into the container folder +cp ./Makefile ./.dockercache/Makefile + echo "-------> Building devcontainer" # Use the devcontainer to run the build as it has all the environment setup that we need time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer From 3bc5400e33c030fd1c49662043be58502cf485db Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 09:51:04 +0000 Subject: [PATCH 07/18] fixing typo in path --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 6766c1a..790c76c 100644 --- a/ci.sh +++ b/ci.sh @@ -17,7 +17,7 @@ then fi # Copy the makefile into the container folder -cp ./Makefile ./.dockercache/Makefile +cp ./Makefile ./.devcontainer/Makefile echo "-------> Building devcontainer" # Use the devcontainer to run the build as it has all the environment setup that we need From 115312c9f4367c7b41fbafb7f75744023cbf415d Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 15:14:19 +0000 Subject: [PATCH 08/18] prevent formatting gocache files and setting env --- Makefile | 6 +----- ci.sh | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index d320c8e..1e0b3cc 100644 --- a/Makefile +++ b/Makefile @@ -79,10 +79,6 @@ endif cd config/manager && kustomize edit set image controller=${IMG} kustomize build config/default | kubectl apply -f - kustomize build config/default > operatorsetup.yaml - - - - # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen @@ -90,7 +86,7 @@ manifests: controller-gen # Run go fmt against code fmt: - find . -name '*.go' | grep -v vendor | xargs gofmt -s -w + find . -name '*.go' | grep -v 'vendor|.gocache' | xargs gofmt -s -w # Run go vet against code vet: diff --git a/ci.sh b/ci.sh index 790c76c..351d805 100644 --- a/ci.sh +++ b/ci.sh @@ -28,7 +28,7 @@ mkdir -p ${PWD}/.gocache echo "-------> Building code and running tests" # Run `make` to build and test the code -time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" devcontainer -c "make install-test-dependency test" +time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env-file "./.devcontainer/.env" devcontainer -c "make test" # Ensure .gocache permmissions correct for build to save cache sudo chown -R $USER ${PWD} From df124bb51d576b2c0f214435ea614cf18d2836cb Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 15:33:00 +0000 Subject: [PATCH 09/18] fixing env variables --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 351d805..75d3b42 100644 --- a/ci.sh +++ b/ci.sh @@ -28,7 +28,7 @@ mkdir -p ${PWD}/.gocache echo "-------> Building code and running tests" # Run `make` to build and test the code -time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env-file "./.devcontainer/.env" devcontainer -c "make test" +time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=${DATABRICKS_HOST} --env DATABRICKS_TOKEN=${DATABRICKS_TOKEN} devcontainer -c "make test" # Ensure .gocache permmissions correct for build to save cache sudo chown -R $USER ${PWD} From 3f914d0b2e6b4dd95caf10002fb24db1b7000f67 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 15:49:56 +0000 Subject: [PATCH 10/18] exclude gocache from linting --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1e0b3cc..089cd75 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ manifests: controller-gen # Run go fmt against code fmt: - find . -name '*.go' | grep -v 'vendor|.gocache' | xargs gofmt -s -w + find . -name '*.go' | grep -v -E 'vendor|.gocache' | xargs gofmt -s -w # Run go vet against code vet: From 3b3b93c3fbb4217b915b4cc161e84d1583ae0ff9 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 16:05:10 +0000 Subject: [PATCH 11/18] fixed env variable syntax --- ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 75d3b42..7a0869e 100644 --- a/ci.sh +++ b/ci.sh @@ -28,7 +28,7 @@ mkdir -p ${PWD}/.gocache echo "-------> Building code and running tests" # Run `make` to build and test the code -time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=${DATABRICKS_HOST} --env DATABRICKS_TOKEN=${DATABRICKS_TOKEN} devcontainer -c "make test" +time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" # Ensure .gocache permmissions correct for build to save cache sudo chown -R $USER ${PWD} From f84d9bf84cf8aee01faca978b9d4c93b1e8f6a4e Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 29 Jan 2020 16:22:26 +0000 Subject: [PATCH 12/18] patching up passing env variables :facepalm: --- .devcontainer/devcontainer.json | 2 +- azure-pipelines.yaml | 4 ++++ ci.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index c12b3e9..b95d5d0 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ "name": "Go", "dockerFile": "./Dockerfile", "workspaceFolder": "/workspace", - "context": "..", + "context": "..", "workspaceMount": "src=${localWorkspaceFolder},dst=/workspace,type=bind", "runArgs": [ // Mount the env file diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 4b2018b..f7e0492 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -36,6 +36,10 @@ jobs: - script: | bash -f ./ci.sh displayName: 'Run CI' + env: + GO111MODULE: on + DATABRICKS_HOST: $(DATABRICKS_HOST) + DATABRICKS_TOKEN: $(DATABRICKS_TOKEN) - task: PublishCodeCoverageResults@1 inputs: diff --git a/ci.sh b/ci.sh index 7a0869e..75d3b42 100644 --- a/ci.sh +++ b/ci.sh @@ -28,7 +28,7 @@ mkdir -p ${PWD}/.gocache echo "-------> Building code and running tests" # Run `make` to build and test the code -time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" +time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=${DATABRICKS_HOST} --env DATABRICKS_TOKEN=${DATABRICKS_TOKEN} devcontainer -c "make test" # Ensure .gocache permmissions correct for build to save cache sudo chown -R $USER ${PWD} From a5cafa969bde7f9225a64e75fd52a1c0dc18c91d Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 5 Feb 2020 12:19:31 +0000 Subject: [PATCH 13/18] PR feedback to move sh logic to yaml --- .devcontainer/Dockerfile | 3 +++ azure-pipelines.yaml | 30 +++++++++++++++++++++-------- ci.sh | 41 ---------------------------------------- 3 files changed, 25 insertions(+), 49 deletions(-) delete mode 100644 ci.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index a62a13b..20b4428 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -87,6 +87,9 @@ RUN make install-kind RUN make install-kubebuilder RUN make install-kustomize RUN make install-test-dependency +# tidy up go packages +RUN rm -rf /go/src/ && rm -rf /go/pkg + ENV SHELL /bin/bash # Save command line history diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index f7e0492..7a83e4e 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -17,7 +17,7 @@ jobs: - job: Build steps: # Cache the docker image file - - task: CacheBeta@0 + - task: Cache@2 inputs: key: go-cache | go.sum path: ".gocache" @@ -25,7 +25,7 @@ jobs: displayName: Cache go mod cache # Cache the docker image file - - task: CacheBeta@0 + - task: Cache@2 inputs: key: docker-image | .devcontainer/** path: ".dockercache" @@ -34,12 +34,26 @@ jobs: displayName: Cache docker layers - script: | - bash -f ./ci.sh - displayName: 'Run CI' - env: - GO111MODULE: on - DATABRICKS_HOST: $(DATABRICKS_HOST) - DATABRICKS_TOKEN: $(DATABRICKS_TOKEN) + docker info + docker load -i ./dockercache/devcontainer.tar + + - script: | + # Copy the makefile into the container folder + cp ./Makefile ./.devcontainer/Makefile + + echo "-------> Building devcontainer" + time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer + # Create a directory for go mod cache + mkdir -p $(System.DefaultWorkingDirectory)/.gocache + + echo "-------> Building code and running tests" + # Run `make` to build and test the code + time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" + + - script: + echo "-------> Saving docker image" + time docker image save -o ./.dockercache/devcontainer.tar devcontainer + condition: ne(variables.DOCKER_CACHE_HIT, 'true') - task: PublishCodeCoverageResults@1 inputs: diff --git a/ci.sh b/ci.sh deleted file mode 100644 index 75d3b42..0000000 --- a/ci.sh +++ /dev/null @@ -1,41 +0,0 @@ -#! /bin/bash - -set -e -set -x - -# Get storage drive details -docker info - -# Create .dockercache directory -mkdir -p ./.dockercache/ - -# Import devcontainer from cache to speed up build -if [ -f ".dockercache/devcontainer.tar" ]; -then - echo "-------> Restoring docker image" - time docker load -i .dockercache/devcontainer.tar -fi - -# Copy the makefile into the container folder -cp ./Makefile ./.devcontainer/Makefile - -echo "-------> Building devcontainer" -# Use the devcontainer to run the build as it has all the environment setup that we need -time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer - -# Create a directory for go mod cache -mkdir -p ${PWD}/.gocache - -echo "-------> Building code and running tests" -# Run `make` to build and test the code -time docker run -v ${PWD}/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v ${PWD}:/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=${DATABRICKS_HOST} --env DATABRICKS_TOKEN=${DATABRICKS_TOKEN} devcontainer -c "make test" - -# Ensure .gocache permmissions correct for build to save cache -sudo chown -R $USER ${PWD} - -# If the current cached image is out of date save devcontainer so it can be cached -if [ $DOCKER_CACHE_HIT != "true" ]; -then - echo "-------> Saving docker image" - time docker image save -o ./.dockercache/devcontainer.tar devcontainer -fi From 3b99dbd1f71aa5bb86c366b50b6cafd1df7d6bf4 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 5 Feb 2020 12:22:07 +0000 Subject: [PATCH 14/18] fixing indentation issue --- azure-pipelines.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 7a83e4e..ac92667 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -34,23 +34,23 @@ jobs: displayName: Cache docker layers - script: | - docker info - docker load -i ./dockercache/devcontainer.tar + docker info + docker load -i ./dockercache/devcontainer.tar - script: | - # Copy the makefile into the container folder - cp ./Makefile ./.devcontainer/Makefile + # Copy the makefile into the container folder + cp ./Makefile ./.devcontainer/Makefile - echo "-------> Building devcontainer" - time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer - # Create a directory for go mod cache - mkdir -p $(System.DefaultWorkingDirectory)/.gocache + echo "-------> Building devcontainer" + time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer + # Create a directory for go mod cache + mkdir -p $(System.DefaultWorkingDirectory)/.gocache - echo "-------> Building code and running tests" - # Run `make` to build and test the code + echo "-------> Building code and running tests" + # Run `make` to build and test the code time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" - - script: + - script: | echo "-------> Saving docker image" time docker image save -o ./.dockercache/devcontainer.tar devcontainer condition: ne(variables.DOCKER_CACHE_HIT, 'true') From 212c85ec45673b174e10859fa8c44ffad9547658 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 5 Feb 2020 12:26:43 +0000 Subject: [PATCH 15/18] fixing indentation again --- azure-pipelines.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index ac92667..a8b606b 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -48,7 +48,7 @@ jobs: echo "-------> Building code and running tests" # Run `make` to build and test the code - time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" + time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" - script: | echo "-------> Saving docker image" From 09d4db2f1b1202dfc9b9ce6d57264cf616c87b25 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Wed, 5 Feb 2020 12:33:32 +0000 Subject: [PATCH 16/18] minor fixes --- azure-pipelines.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index a8b606b..231f7d1 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -36,6 +36,8 @@ jobs: - script: | docker info docker load -i ./dockercache/devcontainer.tar + condition: eq(variables.DOCKER_CACHE_HIT, 'true') + displayName: Load cached devcontainer image - script: | # Copy the makefile into the container folder @@ -49,11 +51,13 @@ jobs: echo "-------> Building code and running tests" # Run `make` to build and test the code time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" + displayName: Build and Test - script: | echo "-------> Saving docker image" time docker image save -o ./.dockercache/devcontainer.tar devcontainer condition: ne(variables.DOCKER_CACHE_HIT, 'true') + displayName: Save devcontainer image - task: PublishCodeCoverageResults@1 inputs: From 87d2d8c7f04be27d6c93e7b7ec41abd27602fab2 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Fri, 14 Feb 2020 09:38:44 +0000 Subject: [PATCH 17/18] fixing issue with missing cache dir --- azure-pipelines.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index 231f7d1..b90f11e 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -40,6 +40,9 @@ jobs: displayName: Load cached devcontainer image - script: | + # Create dockercache directory + mkdir -p ./.dockercache/ + # Copy the makefile into the container folder cp ./Makefile ./.devcontainer/Makefile From 0364e786b7019abf53116dd4d77b2c581a4613b6 Mon Sep 17 00:00:00 2001 From: Dave Storey Date: Fri, 14 Feb 2020 10:13:45 +0000 Subject: [PATCH 18/18] permissions issue fix --- azure-pipelines.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/azure-pipelines.yaml b/azure-pipelines.yaml index b90f11e..b35c87c 100644 --- a/azure-pipelines.yaml +++ b/azure-pipelines.yaml @@ -54,6 +54,8 @@ jobs: echo "-------> Building code and running tests" # Run `make` to build and test the code time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" + + sudo chown -R $USER $(System.DefaultWorkingDirectory) displayName: Build and Test - script: |