Feat/revise GitHub docker cache images (#584)

* feat: added gauge metrics for batch, blob, aggregation size * feat: upgrade actions/cache save and restore to 4.2.0 * feat: add secret inherit for transaction-exclusion-api * feat: revert to include cache-docker-images job in coordinator-testing.yml * feat: move cache-docker-images out from coordinator-testing and staterecovery-test * feat: text changes for testing * feat: set restore-keys as cached-images * feat: add step to create directory for cached docker images * feat: change cache dir from ~/docker-images to ./docker-images * feat: change actions cache version and keys * fix: cache-docker-images folder name * feat: temp disable cache-docker-images * feat: makeover with cache-docker-images and restore-cached-images * feat: put restore-cached-images in actions folder * feat: change runner for save cache step and revise hash * feat: testing with diff cache path * feat: testing with diff hash * feat: testing with original runner * feat: added composite action for Compute Docker Images Hash * feat: remove action permissions and use cache restore * Revert "feat: added gauge metrics for batch, blob, aggregation size" This reverts commit 4b1bfc4. * feat: removed internal built images from cache * feat: fix docker_images_unique command line * feat: fix docker_images_unique command line * feat: fix docker_images_unique command line * feat: add name for pull-and-cache-images * feat: update cache docker image list with those from traces-v1 * feat: move cache-docker-images to main workflow * feat: move cache-docker-images back to testing workflow * feat: remove leftover outputs in reuse-images-tags-and-push-workflow * feat: bump to exact version for actions/cache * feat: temp disable cache for docker images * feat: add more detailed comments * feat:remove unnecessary exception import
Consensys · Jan 27, 2025 · e4256a5 · e4256a5
1 parent 6995d99
commit e4256a5
Show file tree

Hide file tree

Showing 10 changed files with 124 additions and 51 deletions.
diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
@@ -0,0 +1,35 @@
+name: 'Compute Docker Images Hash'
+description: 'Compute Docker Images Hash for Cache Key'
+
+outputs:
+  docker_images_hash:
+    description: 'Docker Images Hash'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}
+  docker_images_trimmed:
+    description: 'Trimmed List of Docker Images'
+    value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      shell: bash
+      run: |
+        docker_images_traces_v1=$(docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_traces_v2=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images=$(echo "$docker_images_traces_v1 $docker_images_traces_v2")
+        docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u)
+        docker_images_trimmed=$(echo $docker_images_unique | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover" | xargs)
+        echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_UNIQUE=$(echo $docker_images_unique) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT
+        echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_trimmed | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT
+    - name: Show docker images hash
+      shell: bash
+      id: show-docker-images-hash
+      run: |
+        echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}"
+        echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}"
+        echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}"
+        echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}"
diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml
@@ -0,0 +1,25 @@
+name: 'Restore Cached Docker Images'
+description: 'Restore Cached Docker Images'
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Compute docker images hash
+      id: compute-docker-images-hash
+      uses: ./.github/actions/compute-docker-images-hash
+    - name: Create cached docker images folder
+      shell: bash
+      run: |
+        mkdir -p ~/docker-images-cached
+    - name: Cache common docker images
+      id: cache-common-docker-images
+      continue-on-error: true
+      uses: actions/cache/[email protected]
+      with:
+        path: ~/docker-images-cached
+        key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
+    - name: Load docker images
+      shell: bash
+      if: steps.cache-common-docker-images.outputs.cache-hit == 'true'
+      run: |
+        docker load -i ~/docker-images-cached/docker-images.tar
diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml
@@ -47,30 +47,35 @@ jobs:
     needs: [ check-dockerhub-secrets-present, changes ]
     if: ${{ always() && needs.check-dockerhub-secrets-present.outputs.secrets_present == 'true' && needs.changes.outputs.cache_images == 'true' }}
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
+    name: Pull and cache images
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Compute docker images hash
+        id: compute-docker-images-hash
+        uses: ./.github/actions/compute-docker-images-hash
+      - name: Create cached docker images folder
+        run: |
+          mkdir -p ~/docker-images-cached
+      - name: Cache common docker images
+        id: cache-common-docker-images
+        uses: actions/[email protected]
+        with:
+          path: ~/docker-images-cached
+          key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }}
+          lookup-only: true
       - name: Login to Docker Hub
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Pull docker images from compose
-        continue-on-error: true
+      - name: Pull docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
           docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull
-      - name: Save Docker images
+      - name: Cache docker images
+        if: steps.cache-common-docker-images.outputs.cache-hit != 'true'
         run: |
-          mkdir -p ~/docker-images
-          images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}')
-          for image in $images; do
-            imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g')
-            echo $image - ${imageFileName}
-            docker save $image > ~/docker-images/${imageFileName}.tar
-          done
-      - name: Cache common docker images
-        continue-on-error: true
-        uses: actions/cache/[email protected]
-        with:
-          path: ~/docker-images
-          key: cached-images
+          docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_trimmed }} -o  ~/docker-images-cached/docker-images.tar
+          echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar)
diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml
@@ -17,9 +17,6 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  cache-docker-images:
-    uses: ./.github/workflows/cache-docker-images.yml
-    secrets: inherit
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
@@ -40,14 +37,12 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: actions/cache/[email protected]
-        with:
-          path: ~/docker-images
-          key: cached-images
-          restore-keys: |
-            cached-images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       # Install pnpm to compile smart contracts
       - name: Setup nodejs environment
         uses: ./.github/actions/setup-nodejs

diff --git a/.github/workflows/reuse-check-images-tags-and-push.yml b/.github/workflows/reuse-check-images-tags-and-push.yml
@@ -53,15 +53,10 @@ jobs:
     name: Check image tags exist
     outputs:
       last_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.common_ancestor_commit_tag_exists }}
       last_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.last_commit_tag_exists }}
-      common_ancestor_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.common_ancestor_commit_tag_exists }}
     steps:
       - name: Checkout
         uses: actions/checkout@v4

diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
@@ -97,14 +97,12 @@ jobs:
         run: |
           mkdir -p tmp/local/traces/v2/conflated
           chmod -R a+w tmp/local/traces/v2/conflated
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: actions/cache/[email protected]
-        with:
-          path: ~/docker-images
-          key: cached-images
-          restore-keys: |
-            cached-
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       - name: Pull all images with retry
         uses: nick-fields/retry@v3
         with:

diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml
@@ -20,10 +20,9 @@ jobs:
   run-tests:
     env:
       COMMIT_TAG: ${{ inputs.commit_tag }}
-      GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
     # ~2.5 mins saved vs large
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xl
     name: Staterecovery tests
@@ -38,14 +37,12 @@ jobs:
         # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
         # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
         uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1
-      - name: Restore cached images
-        id: restore-cached-images
-        uses: actions/cache/[email protected]
-        with:
-          path: ~/docker-images
-          key: cached-images
-          restore-keys: |
-            cached-images
+      # Disable cache for pulling docker images as empirically found that this 
+      # (retrieving cache and loading docker images) actually increased test time-to-completion
+      # - name: Restore cached images
+      #   id: restore-cached-images
+      #   uses: ./.github/actions/restore-docker-images
+      #   continue-on-error: true
       - name: Staterecovery - Build and Unit tests
         run: |
           ./gradlew state-recovery:besu-plugin:buildNeeded

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -26,7 +26,14 @@ on:
         type: string
 
 jobs:
+  # Disable cache for pulling docker images as empirically found that this 
+  # (retrieving cache and loading docker images) actually increased test time-to-completion
+  # cache-docker-images:
+  #   uses: ./.github/workflows/cache-docker-images.yml
+  #   secrets: inherit
+
   coordinator:
+    # needs: [ cache-docker-images ]
     uses: ./.github/workflows/coordinator-testing.yml
     if: ${{ inputs.coordinator_changed == 'true' }}
     with:
@@ -50,8 +57,10 @@ jobs:
   transaction-exclusion-api:
     uses: ./.github/workflows/transaction-exclusion-api-testing.yml
     if: ${{ inputs.transaction_exclusion_api_changed == 'true' }}
+    secrets: inherit
 
   staterecovery:
+    # needs: [ cache-docker-images ]
     uses: ./.github/workflows/staterecovery-testing.yml
     if: ${{ inputs.staterecovery_changed == 'true' }}
     with:

diff --git a/.github/workflows/transaction-exclusion-api-testing.yml b/.github/workflows/transaction-exclusion-api-testing.yml
@@ -2,6 +2,11 @@ name: transaction-exclusion-api-testing
 
 on:
   workflow_call:
+    secrets:
+      DOCKERHUB_USERNAME:
+        required: false
+      DOCKERHUB_TOKEN:
+        required: false
   workflow_dispatch:
     inputs:
       coverage:
@@ -16,6 +21,9 @@ concurrency:
 
 jobs:
   run-tests:
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
     # ~1.5 mins saved vs small
     runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med
     name: Transaction exclusion api tests
@@ -36,6 +44,12 @@ jobs:
         if: ${{ !inputs.coverage }}
         run: |
           ./gradlew transaction-exclusion-api:app:buildNeeded
+      - name: Login to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
       - name: Run integration tests
         run: |
           ./gradlew transaction-exclusion-api:app:integrationTestAllNeeded
diff --git a/docker/compose.yml b/docker/compose.yml
@@ -16,7 +16,7 @@ networks:
       config:
         - subnet: 10.10.10.0/24
 
-# To debug inside the network and volumes
+# To debug inside the network and volumes:
 # docker run --rm -it --network=docker_linea -v=linea-local-dev:/data -v=linea-logs:/logs weibeld/ubuntu-networking bash
 
 services: