Moving build pipeline to use devcontainer (#152)

* moving build pipeline to use devcontainer

.devcontainer/Dockerfile

@@ -38,7 +38,7 @@ RUN apt-get update \
         github.com/cweill/gotests/... \
         golang.org/x/tools/cmd/goimports \
         golang.org/x/lint/golint \
-        golang.org/x/tools/cmd/gopls \
+        golang.org/x/tools/gopls \
         github.com/alecthomas/gometalinter \
         honnef.co/go/tools/... \
         github.com/golangci/golangci-lint/cmd/golangci-lint \
@@ -86,6 +86,9 @@ RUN make install-kind
 RUN make install-kubebuilder
 RUN make install-kustomize
 RUN make install-test-dependency
+# tidy up go packages
+RUN rm -rf /go/src/ && rm -rf /go/pkg
+
 ENV SHELL /bin/bash
 
 # Save command line history 

Makefile

@@ -84,10 +84,6 @@ endif
 	cd config/manager && kustomize edit set image controller=${IMG}
 	kustomize build config/default | kubectl apply -f -
 	kustomize build config/default > operatorsetup.yaml
-
-
-
-
 
 create-dbrickssettings-secret:
 	kubectl --namespace ${OPERATOR_NAMESPACE} \
@@ -101,7 +97,7 @@ manifests: controller-gen
 
 # Run go fmt against code
 fmt:
-	find . -name '*.go' | grep -v vendor | xargs gofmt -s -w
+	find . -name '*.go' | grep -v -E 'vendor|.gocache' | xargs gofmt -s -w
 
 # Run go vet against code
 vet:

azure-pipelines.yaml

@@ -7,103 +7,93 @@ pool:
   vmImage: 'Ubuntu 16.04'
 
 variables:
-  GOROOT: '/usr/local/go1.12' # Go installation path
-  GOPATH: '$(System.DefaultWorkingDirectory)/gopath' # Go workspace path
-  GOBIN:  '$(GOPATH)/bin' # Go binaries path
-  MODULE_PATH: '$(GOPATH)/src/$(Build.Repository.Name)' # Path to the module's code
   IMAGE_NAME: 'candidate/k8s/azure-databricks/operator:$(Build.SourceVersion)'
   # ACR_NAME:           - set this value in Azure Devops variables
   # AZURE_SUBSCRIPTION: - set this value in Azure Devops variables
   # DATABRICKS_HOST:    - set this value in Azure Devops variables
   # DATABRICKS_TOKEN:   - set this value in Azure Devops variables
 
-steps:
+jobs:
+- job: Build
+  steps:
+  # Cache the docker image file
+  - task: Cache@2
+    inputs:
+      key: go-cache | go.sum
+      path: ".gocache"
+      restoreKeys: go-cache 
+    displayName: Cache go mod cache
 
-- checkout: self  # self represents the repo where the initial Pipelines YAML file was found
-  lfs: "true"
+  # Cache the docker image file
+  - task: Cache@2
+    inputs:
+      key: docker-image | .devcontainer/**
+      path: ".dockercache"
+      restoreKeys: docker-image
+      cacheHitVar: DOCKER_CACHE_HIT
+    displayName: Cache docker layers
 
-- script: |
-    set -x
-    ls -l /usr/local/go*
-    mkdir -p '$(GOBIN)'
-    mkdir -p '$(GOPATH)/pkg'
-    mkdir -p '$(MODULE_PATH)'
-    shopt -s extglob
-    mv !(gopath) '$(MODULE_PATH)'
-    echo '##vso[task.prependpath]$(GOBIN)'
-    echo '##vso[task.prependpath]$(GOROOT)/bin'
-    go version
-    go env
-  displayName: 'Set up the Go workspace'
-  env:
-    GO111MODULE: on
+  - script: |
+      docker info
+      docker load -i ./dockercache/devcontainer.tar
+    condition: eq(variables.DOCKER_CACHE_HIT, 'true')
+    displayName: Load cached devcontainer image
 
-- script: |
-    set -x
-    mkdir -p '$(MODULE_PATH)/bin'
-    os=$(go env GOOS)
-    arch=$(go env GOARCH)
-    # download kubebuilder
-    curl -sL https://go.kubebuilder.io/dl/2.2.0/${os}/${arch} | tar -xz -C $(MODULE_PATH)/bin
-    mv $(MODULE_PATH)/bin/kubebuilder_2.2.0_${os}_${arch}/bin/* $(MODULE_PATH)/bin/
-    # download kustomize
-    curl -sL https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv3.4.0/kustomize_v3.4.0_${os}_${arch}.tar.gz | tar -xz -C /tmp/
-    mv /tmp/kustomize $(MODULE_PATH)/bin/kustomize
-    # set permission
-    chmod a+x $(MODULE_PATH)/bin/*
-    # export path
-    export PATH=$PATH:$(MODULE_PATH)/bin
-    # Install golangci-linter
-    curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.21.0
-    # install test dependencies
-    go mod download
-    make install-test-dependency
-    # run test
-    make test
-  continueOnError: 'false'
-  displayName: 'Install kubebuilder, kustomize and Run unit tests'
-  env:
-    GO111MODULE: on
-    KUBEBUILDER_ASSETS: $(MODULE_PATH)/bin
-    DATABRICKS_HOST: $(DATABRICKS_HOST)
-    DATABRICKS_TOKEN: $(DATABRICKS_TOKEN)
-  workingDirectory: '$(MODULE_PATH)'
+  - script: |
+      # Create dockercache directory
+      mkdir -p ./.dockercache/
 
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: Cobertura 
-    summaryFileLocation: $(System.DefaultWorkingDirectory)/**/cover.xml
-    reportDirectory: $(System.DefaultWorkingDirectory)/**/cover
+      # Copy the makefile into the container folder
+      cp ./Makefile ./.devcontainer/Makefile
 
+      echo "-------> Building devcontainer"
+      time docker build --cache-from devcontainer:latest -t devcontainer -f ./.devcontainer/Dockerfile ./.devcontainer
+      # Create a directory for go mod cache
+      mkdir -p $(System.DefaultWorkingDirectory)/.gocache
 
-- script: docker build -t $(IMAGE_NAME) .  # add options to this command to meet your needs
-  condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master'))
-  displayName: 'Docker build'
-  workingDirectory: '$(MODULE_PATH)'
+      echo "-------> Building code and running tests"
+      # Run `make` to build and test the code
+      time docker run -v $(System.DefaultWorkingDirectory)/.gocache:/go/pkg/ -v /var/run/docker.sock:/var/run/docker.sock -v $(System.DefaultWorkingDirectory):/src --workdir /src --entrypoint /bin/bash --network="host" --env DATABRICKS_HOST=$(DATABRICKS_HOST) --env DATABRICKS_TOKEN=$(DATABRICKS_TOKEN) devcontainer -c "make test" 
+   
+      sudo chown -R $USER $(System.DefaultWorkingDirectory)
+    displayName: Build and Test
 
-- task: AzureCLI@1
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-  displayName: Push to Azure Container Registry
-  inputs:
-    azureSubscription: $(AZURE_SUBSCRIPTION)
-    scriptLocation: inlineScript
-    failOnStandardError: false
-    workingDirectory: '$(MODULE_PATH)'
-    inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) .
+  - script: |
+      echo "-------> Saving docker image"
+      time docker image save -o ./.dockercache/devcontainer.tar devcontainer
+    condition: ne(variables.DOCKER_CACHE_HIT, 'true')
+    displayName: Save devcontainer image
 
-- script: |
-    ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml
-    set -x
-    echo $(IMAGE_NAME) > $(Build.ArtifactStagingDirectory)/azure-databricks-operator.txt
-    ls $(Build.ArtifactStagingDirectory)
-  continueOnError: 'false'
-  displayName: 'Prepare manifests for publish'
-  workingDirectory: '$(MODULE_PATH)'
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) 
+  - task: PublishCodeCoverageResults@1
+    inputs:
+      codeCoverageTool: Cobertura 
+      summaryFileLocation: $(System.DefaultWorkingDirectory)/**/cover.xml
+      reportDirectory: $(System.DefaultWorkingDirectory)/**/cover
 
-- task: PublishBuildArtifacts@1
-  condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-  inputs:
-    pathtoPublish: $(Build.ArtifactStagingDirectory)
-    artifactName: drop
+  - script: docker build -t $(IMAGE_NAME) .  # add options to this command to meet your needs
+    condition: and(succeeded(), ne(variables['Build.SourceBranch'], 'refs/heads/master'))
+    displayName: 'Docker build'
+
+  - task: AzureCLI@1
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    displayName: Push to Azure Container Registry
+    inputs:
+      azureSubscription: $(AZURE_SUBSCRIPTION)
+      scriptLocation: inlineScript
+      failOnStandardError: false
+      inlineScript: az acr build --registry $(ACR_NAME) --image $(IMAGE_NAME) .
+
+  - script: |
+      ./bin/kustomize build config/default > $(Build.ArtifactStagingDirectory)/setup.yaml
+      set -x
+      echo $(IMAGE_NAME) > $(Build.ArtifactStagingDirectory)/azure-databricks-operator.txt
+      ls $(Build.ArtifactStagingDirectory)
+    continueOnError: 'false'
+    displayName: 'Prepare manifests for publish'
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) 
+
+  - task: PublishBuildArtifacts@1
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    inputs:
+      pathtoPublish: $(Build.ArtifactStagingDirectory)
+      artifactName: drop