GHA: Docker build caching and other speed improvements (DefectDojo#3659)

* GHA: docker caching & other speedups

.github/workflows/flake8.yml

@@ -2,6 +2,7 @@ name: Flake8
 # pull requests:
 #      run on pull_request_target instead of just pull_request as we need write access to update the status check
 on:
+  workflow_dispatch:
   pull_request_target:
   push:
 

.github/workflows/integration-tests.yml

@@ -3,6 +3,7 @@ name: Integration tests
 # push:
 #      run on every push, which is when something gets merged also
 on:
+  workflow_dispatch:
   pull_request:
   push:
     branches:
@@ -11,43 +12,106 @@ on:
       - release/**
       - hotfix/**
 
+env:
+  DD_DOCKER_REPO: defectdojo
+
 jobs:
+  build:
+    # build with docker so we can use layer caching
+    name: Build Image
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        docker-image: [django, nginx, integration-tests]
+    steps:
+      # - name: Login to DockerHub
+      #   uses: docker/login-action@v1
+      #   with:
+      #     username: ${{ secrets.DOCKERHUB_USERNAME }}
+      #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Read Docker Image Identifiers
+        id: read-docker-image-identifiers
+        run: echo "IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        env:
+          docker-image: ${{ matrix.docker-image }}
+        with:
+          path: /tmp/.buildx-cache-${{ env.docker-image }}
+          key: ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}-${{ github.sha }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}-${{ github.sha }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}
+
+      - name: Build
+        id: docker_build
+        uses: docker/build-push-action@v2
+        env:
+          docker-image: ${{ matrix.docker-image }}
+        with:
+          context: .
+          push: false
+          tags: |
+            ${{ env.DD_DOCKER_REPO }}/defectdojo-${{ env.docker-image }}:latest
+          file: Dockerfile.${{ env.docker-image }}
+          outputs: type=docker,dest=${{ env.docker-image }}_img
+          cache-from: type=local,src=/tmp/.buildx-cache-${{ env.docker-image }}
+          cache-to: type=local,dest=/tmp/.buildx-cache-${{ env.docker-image }}
+
+      # export docker images to be used in next jobs below
+      - name: Upload image ${{ matrix.docker-image }} as artifact
+        uses:  actions/upload-artifact@v2
+        with:
+          name: ${{ matrix.docker-image }}
+          path: ${{ matrix.docker-image }}_img
+          retention-days: 1
+
   integration_tests:
+    # run tests with docker-compose
+    name: integration tests
+    needs: build
     runs-on: ubuntu-latest
     steps:
 
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Set integration-test mode
-        run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
+      # load docker images from build jobs
+      - name: Load images from artifacts
+        uses: actions/download-artifact@v2
 
-      # Pull the latest image to build, and avoid caching pull-only images.
-      # (docker pull is faster than caching in most cases.)
-      - run: docker-compose pull
-      # In this step, this action saves a list of existing images,
-      # the cache is created without them in the post run.
-      # It also restores the cache if it exists.
-      - uses: satackey/action-docker-layer-caching@master
-        # Ignore the failure of a step and avoid terminating the job.
-        continue-on-error: true
+      - name: Load docker images
+        run: |-
+             docker load -i nginx/nginx_img
+             docker load -i django/django_img
+             docker load -i integration-tests/integration-tests_img
+             docker images
 
-      - name: Build the stack
-        run: docker-compose build
+      - name: Set integration-test mode
+        run: ln -s docker-compose.override.integration_tests.yml docker-compose.override.yml
 
       # phased startup so we can use the exit code from integrationtest container
-      - name: Start MySQL
-        run: docker-compose up -d
-
-      - name: Initialize
-        run: docker-compose up --exit-code-from initializer initializer
 
       - name: Start Dojo
         # implicity starts uwsgi and rabbitmq
-        run: docker-compose up -d nginx celerybeat celeryworker
+        run: docker-compose up -d mysql nginx celerybeat celeryworker
+
+      - name: Initialize
+        run: docker-compose up --exit-code-from initializer initializer
 
       - name: Integration tests
-        run: docker-compose up --exit-code-from integrationtest integrationtest
+        run: docker-compose up --exit-code-from integration-tests integration-tests
 
       - name: Logs
         if: failure()

.github/workflows/k8s-testing.yml

@@ -37,61 +37,78 @@ env:
     --set createPostgresqlSecret=true \
    "
 jobs:
-  build_image:
-    name: Docker image build
+  build:
+    name: Build Image
+
     runs-on: ubuntu-latest
+
     strategy:
       matrix:
-        component: [django, nginx]
+        docker-image: [django, nginx]
+
     steps:
       # - name: Login to DockerHub
       #   uses: docker/login-action@v1
       #   with:
       #     username: ${{ secrets.DOCKERHUB_USERNAME }}
       #     password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Checkout
         uses: actions/checkout@v2
+
       - name: Read Docker Image Identifiers
         id: read-docker-image-identifiers
         run: echo "IMAGE_REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_ENV
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
-      # - name: Log into containers
-        # uses: docker/login-action@v1
-        # with:
-        #   registry: ${{ env.GITHUB_CACHE_REPO }}
-        #   username: ${{ github.actor }}
-        #   password: ${{ secrets.PAT }}
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        env:
+          docker-image: ${{ matrix.docker-image }}
+        with:
+          path: /tmp/.buildx-cache-${{ env.docker-image }}
+          key: ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}-${{ github.sha }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}-${{ github.sha }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ github.workflow }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}
+
       - name: Build
         id: docker_build
         uses: docker/build-push-action@v2
+        env:
+          docker-image: ${{ matrix.docker-image }}
         with:
           context: .
           push: false
           tags: |
-            ${{ env.DD_DOCKER_REPO }}/defectdojo-${{ matrix.component }}:latest
-            ${{ env.GITHUB_CACHE_REPO }}/${{ env.IMAGE_REPOSITORY }}/${{ matrix.component }}:cache
-          file: Dockerfile.${{ matrix.component }}
-          outputs: type=docker,dest=${{ matrix.component }}_img
-          # cache-to: type=registry,ref=${{ env.GITHUB_CACHE_REPO }}/${{ env.IMAGE_REPOSITORY }}/${{ matrix.component }}:cache,mode=max
-          # cache-from: type=registry,ref=${{ env.GITHUB_CACHE_REPO }}/${{ env.IMAGE_REPOSITORY }}/${{ matrix.component }}:cache
-      - name: Upload image ${{ matrix.component }} as artifact
+            ${{ env.DD_DOCKER_REPO }}/defectdojo-${{ env.docker-image }}:latest
+          file: Dockerfile.${{ env.docker-image }}
+          outputs: type=docker,dest=${{ env.docker-image }}_img
+          cache-from: type=local,src=/tmp/.buildx-cache-${{ env.docker-image }}
+          cache-to: type=local,dest=/tmp/.buildx-cache-${{ env.docker-image }}
+
+      - name: Upload image ${{ env.docker-image }} as artifact
         uses:  actions/upload-artifact@v2
         with:
-          name: ${{ matrix.component }}
-          path: ${{matrix.component}}_img
+          name: ${{ matrix.docker-image }}
+          path: ${{ matrix.docker-image }}_img
           retention-days: 1
 
   setting_minikube_cluster:
     name: Kubernetes Deployment
+
     runs-on: ubuntu-latest
-    needs: build_image
+
+    needs: build
+
     strategy:
       matrix:
-        databases: [pgsql,mysql]
-        brokers: [redis,rabbit]
+        databases: [pgsql, mysql]
+        brokers: [redis, rabbit]
+
     steps:
 #      - name: Login to DockerHub
 #        uses: docker/login-action@v1
@@ -100,36 +117,44 @@ jobs:
 #          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Checkout
         uses: actions/checkout@v2
+
       - name: Setup Minikube
         uses: manusa/[email protected]
         with:
           minikube version: 'v1.14.2'
           kubernetes version: 'v1.19.2'
           driver: docker
           start args: '--addons=ingress'
+
       - name: Status of minikube
-        run: minikube status
+        run: |-
+          minikube status
+
       - name: Load images from artifacts
         uses: actions/download-artifact@v2
+
       - name: Load docker images
         run: |-
              eval $(minikube docker-env)
              docker load -i nginx/nginx_img
              docker load -i django/django_img
              docker images
+
       - name: Configure HELM repos
         run: |-
              helm repo add stable https://charts.helm.sh/stable
              helm repo add bitnami https://charts.bitnami.com/bitnami
              helm dependency list ./helm/defectdojo
              helm dependency update ./helm/defectdojo
+
       - name: Set confings into Outputs
         id: set
         run: |-
               echo ::set-output name=pgsql:: "${{ env.HELM_PG_DATABASE_SETTINGS }}"
               echo ::set-output name=mysql:: "${{ env.HELM_MYSQL_DATABASE_SETTINGS }}"
               echo ::set-output name=redis:: "${{ env.HELM_REDIS_BROKER_SETTINGS }}"
               echo ::set-output name=rabbit:: "${{ env.HELM_RABBIT_BROKER_SETTINGS }}"
+
       # - name: Create image pull Secrets
       #   run: |-
       #         kubectl create secret docker-registry defectdojoregistrykey --docker-username=${{ secrets.DOCKERHUB_USERNAME }} --docker-password=${{ secrets.DOCKERHUB_TOKEN }}
@@ -145,11 +170,13 @@ jobs:
                ${{ steps.set.outputs[matrix.brokers] }} \
                --set createSecret=true \
               #  --set imagePullSecrets=defectdojoregistrykey
+
       - name: Check deployment status
         run: |-
              kubectl get pods
              kubectl get ingress
              kubectl get services
+
       - name: Check Application
         run: |-
              to_complete () {

.github/workflows/new-release-tag-docker.yml

@@ -3,6 +3,7 @@ name: "Release: 2. tag, release, docker push"
 env:
   GIT_USERNAME: "DefectDojo release bot"
   GIT_EMAIL: "[email protected]"
+  workflow_name: 'release 2 tag release docker push' # needed in cache key, which doesn't support comma's
 on:
   workflow_dispatch:
     inputs:
@@ -47,36 +48,56 @@ jobs:
           draft: true
           prerelease: false
 
+
   job-build-and-push:
     needs: tag-and-release
     runs-on: ubuntu-latest
     strategy:
-      matrix: 
+      matrix:
           docker-image: [django, nginx]
     steps:
       - name: Login to DockerHub
         uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
+
       - name: Checkout tag
         uses: actions/checkout@v2
         with:
           ref: ${{ github.event.inputs.release_number }}
+
       - id: set-repo-org
         run: echo ::set-output name=repoorg::${GITHUB_REPOSITORY%%/*} | tr '[:upper:]' '[:lower:]'
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+
       - name: Set up Docker Buildx
+        id: buildx
         uses: docker/setup-buildx-action@v1
+
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        env:
+          docker-image: ${{ matrix.docker-image }}
+        with:
+          path: /tmp/.buildx-cache-${{ env.docker-image }}
+          key: ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ env.workflow_name }}-${{ github.sha }}-${{ github.run_id }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ env.workflow_name}}-${{ github.sha }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-${{ env.workflow_name }}
+            ${{ runner.os }}-buildx-${{ env.docker-image }}-
+
       - name: Build and push images
+        uses: docker/build-push-action@v2
         env:
           REPO_ORG: ${{ steps.set-repo-org.outputs.repoorg }}
-        uses: docker/build-push-action@v2
+          docker-image: ${{ matrix.docker-image }}
         with:
           push: true
-          tags: ${{ env.REPO_ORG }}/defectdojo-${{ matrix.docker-image}}:${{ github.event.inputs.release_number }}, ${{ env.REPO_ORG }}/defectdojo-${{ matrix.docker-image}}:latest
-          file: ./Dockerfile.${{ matrix.docker-image }}
+          tags: ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:${{ github.event.inputs.release_number }}, ${{ env.REPO_ORG }}/defectdojo-${{ env.docker-image}}:latest
+          file: ./Dockerfile.${{ env.docker-image }}
           context: .
+          cache-from: type=local,src=/tmp/.buildx-cache-${{ env.docker-image }}
+          cache-to: type=local,dest=/tmp/.buildx-cache-${{ env.docker-image }}
+
       - name: Image digest
         run: echo ${{ steps.docker_build.outputs.digest }}