From e4256a5c595480bc9762c7eb2b41417f89e41fca Mon Sep 17 00:00:00 2001 From: jonesho <81145364+jonesho@users.noreply.github.com> Date: Tue, 28 Jan 2025 00:53:03 +0800 Subject: [PATCH] Feat/revise GitHub docker cache images (#584) * feat: added gauge metrics for batch, blob, aggregation size * feat: upgrade actions/cache save and restore to 4.2.0 * feat: add secret inherit for transaction-exclusion-api * feat: revert to include cache-docker-images job in coordinator-testing.yml * feat: move cache-docker-images out from coordinator-testing and staterecovery-test * feat: text changes for testing * feat: set restore-keys as cached-images * feat: add step to create directory for cached docker images * feat: change cache dir from ~/docker-images to ./docker-images * feat: change actions cache version and keys * fix: cache-docker-images folder name * feat: temp disable cache-docker-images * feat: makeover with cache-docker-images and restore-cached-images * feat: put restore-cached-images in actions folder * feat: change runner for save cache step and revise hash * feat: testing with diff cache path * feat: testing with diff hash * feat: testing with original runner * feat: added composite action for Compute Docker Images Hash * feat: remove action permissions and use cache restore * Revert "feat: added gauge metrics for batch, blob, aggregation size" This reverts commit 4b1bfc4b27f5c5ee80ed1554625600d10e771f49. * feat: removed internal built images from cache * feat: fix docker_images_unique command line * feat: fix docker_images_unique command line * feat: fix docker_images_unique command line * feat: add name for pull-and-cache-images * feat: update cache docker image list with those from traces-v1 * feat: move cache-docker-images to main workflow * feat: move cache-docker-images back to testing workflow * feat: remove leftover outputs in reuse-images-tags-and-push-workflow * feat: bump to exact version for actions/cache * feat: temp disable cache for docker images * feat: add more detailed comments * feat:remove unnecessary exception import --- .../compute-docker-images-hash/action.yml | 35 ++++++++++++++++++ .../actions/restore-docker-images/action.yml | 25 +++++++++++++ .github/workflows/cache-docker-images.yml | 37 +++++++++++-------- .github/workflows/coordinator-testing.yml | 17 +++------ .../reuse-check-images-tags-and-push.yml | 5 --- .github/workflows/reuse-run-e2e-tests.yml | 14 +++---- .github/workflows/staterecovery-testing.yml | 17 ++++----- .github/workflows/testing.yml | 9 +++++ .../transaction-exclusion-api-testing.yml | 14 +++++++ docker/compose.yml | 2 +- 10 files changed, 124 insertions(+), 51 deletions(-) create mode 100644 .github/actions/compute-docker-images-hash/action.yml create mode 100644 .github/actions/restore-docker-images/action.yml diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml new file mode 100644 index 000000000..7f195e68f --- /dev/null +++ b/.github/actions/compute-docker-images-hash/action.yml @@ -0,0 +1,35 @@ +name: 'Compute Docker Images Hash' +description: 'Compute Docker Images Hash for Cache Key' + +outputs: + docker_images_hash: + description: 'Docker Images Hash' + value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }} + docker_images_trimmed: + description: 'Trimmed List of Docker Images' + value: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }} + +runs: + using: 'composite' + steps: + - name: Compute docker images hash + id: compute-docker-images-hash + shell: bash + run: | + docker_images_traces_v1=$(docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}') + docker_images_traces_v2=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}') + docker_images=$(echo "$docker_images_traces_v1 $docker_images_traces_v2") + docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u) + docker_images_trimmed=$(echo $docker_images_unique | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover" | xargs) + echo DOCKER_IMAGES=$(echo $docker_images) >> $GITHUB_OUTPUT + echo DOCKER_IMAGES_UNIQUE=$(echo $docker_images_unique) >> $GITHUB_OUTPUT + echo DOCKER_IMAGES_TRIMMED=$(echo $docker_images_trimmed) >> $GITHUB_OUTPUT + echo DOCKER_IMAGES_HASH=$(echo -n $docker_images_trimmed | sha256sum | awk '{print $1}') >> $GITHUB_OUTPUT + - name: Show docker images hash + shell: bash + id: show-docker-images-hash + run: | + echo "DOCKER_IMAGES: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES }}" + echo "DOCKER_IMAGES_UNIQUE: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_UNIQUE }}" + echo "DOCKER_IMAGES_TRIMMED: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_TRIMMED }}" + echo "DOCKER_IMAGES_HASH: ${{ steps.compute-docker-images-hash.outputs.DOCKER_IMAGES_HASH }}" diff --git a/.github/actions/restore-docker-images/action.yml b/.github/actions/restore-docker-images/action.yml new file mode 100644 index 000000000..5139c23b3 --- /dev/null +++ b/.github/actions/restore-docker-images/action.yml @@ -0,0 +1,25 @@ +name: 'Restore Cached Docker Images' +description: 'Restore Cached Docker Images' + +runs: + using: 'composite' + steps: + - name: Compute docker images hash + id: compute-docker-images-hash + uses: ./.github/actions/compute-docker-images-hash + - name: Create cached docker images folder + shell: bash + run: | + mkdir -p ~/docker-images-cached + - name: Cache common docker images + id: cache-common-docker-images + continue-on-error: true + uses: actions/cache/restore@v4.2.0 + with: + path: ~/docker-images-cached + key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }} + - name: Load docker images + shell: bash + if: steps.cache-common-docker-images.outputs.cache-hit == 'true' + run: | + docker load -i ~/docker-images-cached/docker-images.tar diff --git a/.github/workflows/cache-docker-images.yml b/.github/workflows/cache-docker-images.yml index 584fd92f6..5614aab38 100644 --- a/.github/workflows/cache-docker-images.yml +++ b/.github/workflows/cache-docker-images.yml @@ -47,30 +47,35 @@ jobs: needs: [ check-dockerhub-secrets-present, changes ] if: ${{ always() && needs.check-dockerhub-secrets-present.outputs.secrets_present == 'true' && needs.changes.outputs.cache_images == 'true' }} runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med + name: Pull and cache images steps: - name: Checkout uses: actions/checkout@v4 + - name: Compute docker images hash + id: compute-docker-images-hash + uses: ./.github/actions/compute-docker-images-hash + - name: Create cached docker images folder + run: | + mkdir -p ~/docker-images-cached + - name: Cache common docker images + id: cache-common-docker-images + uses: actions/cache@v4.2.0 + with: + path: ~/docker-images-cached + key: docker-images-cache-${{ steps.compute-docker-images-hash.outputs.docker_images_hash }} + lookup-only: true - name: Login to Docker Hub + if: steps.cache-common-docker-images.outputs.cache-hit != 'true' uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Pull docker images from compose - continue-on-error: true + - name: Pull docker images + if: steps.cache-common-docker-images.outputs.cache-hit != 'true' run: | docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull - - name: Save Docker images + - name: Cache docker images + if: steps.cache-common-docker-images.outputs.cache-hit != 'true' run: | - mkdir -p ~/docker-images - images=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config | grep "image:" | awk '{print $2}') - for image in $images; do - imageFileName=$(echo $image | sed -e 's|.*/||' -e 's|:|-|' -e 's/\./_/g') - echo $image - ${imageFileName} - docker save $image > ~/docker-images/${imageFileName}.tar - done - - name: Cache common docker images - continue-on-error: true - uses: actions/cache/save@v4.2.0 - with: - path: ~/docker-images - key: cached-images + docker save ${{ steps.compute-docker-images-hash.outputs.docker_images_trimmed }} -o ~/docker-images-cached/docker-images.tar + echo docker-images.tar=$(ls -lh ~/docker-images-cached/docker-images.tar) diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml index 0df306ec5..064a90c38 100644 --- a/.github/workflows/coordinator-testing.yml +++ b/.github/workflows/coordinator-testing.yml @@ -17,9 +17,6 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - cache-docker-images: - uses: ./.github/workflows/cache-docker-images.yml - secrets: inherit run-tests: env: COMMIT_TAG: ${{ inputs.commit_tag }} @@ -40,14 +37,12 @@ jobs: # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 - - name: Restore cached images - id: restore-cached-images - uses: actions/cache/restore@v4.2.0 - with: - path: ~/docker-images - key: cached-images - restore-keys: | - cached-images + # Disable cache for pulling docker images as empirically found that this + # (retrieving cache and loading docker images) actually increased test time-to-completion + # - name: Restore cached images + # id: restore-cached-images + # uses: ./.github/actions/restore-docker-images + # continue-on-error: true # Install pnpm to compile smart contracts - name: Setup nodejs environment uses: ./.github/actions/setup-nodejs diff --git a/.github/workflows/reuse-check-images-tags-and-push.yml b/.github/workflows/reuse-check-images-tags-and-push.yml index 4d78b8448..9b2d82480 100644 --- a/.github/workflows/reuse-check-images-tags-and-push.yml +++ b/.github/workflows/reuse-check-images-tags-and-push.yml @@ -53,15 +53,10 @@ jobs: name: Check image tags exist outputs: last_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists_coordinator: ${{ steps.check_image_tags_exist_coordinator.outputs.common_ancestor_commit_tag_exists }} last_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists_postman: ${{ steps.check_image_tags_exist_postman.outputs.common_ancestor_commit_tag_exists }} last_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists_prover: ${{ steps.check_image_tags_exist_prover.outputs.common_ancestor_commit_tag_exists }} last_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists_traces_api_facade: ${{ steps.check_image_tags_exist_traces_api_facade.outputs.common_ancestor_commit_tag_exists }} last_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists_transaction_exclusion_api: ${{ steps.check_image_tags_exist_transaction_exclusion_api.outputs.common_ancestor_commit_tag_exists }} steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index 66fa2141f..3aa99b74e 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -97,14 +97,12 @@ jobs: run: | mkdir -p tmp/local/traces/v2/conflated chmod -R a+w tmp/local/traces/v2/conflated - - name: Restore cached images - id: restore-cached-images - uses: actions/cache/restore@v4.2.0 - with: - path: ~/docker-images - key: cached-images - restore-keys: | - cached- + # Disable cache for pulling docker images as empirically found that this + # (retrieving cache and loading docker images) actually increased test time-to-completion + # - name: Restore cached images + # id: restore-cached-images + # uses: ./.github/actions/restore-docker-images + # continue-on-error: true - name: Pull all images with retry uses: nick-fields/retry@v3 with: diff --git a/.github/workflows/staterecovery-testing.yml b/.github/workflows/staterecovery-testing.yml index c0b261543..bde8de16e 100644 --- a/.github/workflows/staterecovery-testing.yml +++ b/.github/workflows/staterecovery-testing.yml @@ -20,10 +20,9 @@ jobs: run-tests: env: COMMIT_TAG: ${{ inputs.commit_tag }} - GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} # ~2.5 mins saved vs large runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-xl name: Staterecovery tests @@ -38,14 +37,12 @@ jobs: # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies. # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md uses: gradle/actions/setup-gradle@cc4fc85e6b35bafd578d5ffbc76a5518407e1af0 #v4.2.1 - - name: Restore cached images - id: restore-cached-images - uses: actions/cache/restore@v4.2.0 - with: - path: ~/docker-images - key: cached-images - restore-keys: | - cached-images + # Disable cache for pulling docker images as empirically found that this + # (retrieving cache and loading docker images) actually increased test time-to-completion + # - name: Restore cached images + # id: restore-cached-images + # uses: ./.github/actions/restore-docker-images + # continue-on-error: true - name: Staterecovery - Build and Unit tests run: | ./gradlew state-recovery:besu-plugin:buildNeeded diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index f973c8ee4..104efa353 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -26,7 +26,14 @@ on: type: string jobs: + # Disable cache for pulling docker images as empirically found that this + # (retrieving cache and loading docker images) actually increased test time-to-completion + # cache-docker-images: + # uses: ./.github/workflows/cache-docker-images.yml + # secrets: inherit + coordinator: + # needs: [ cache-docker-images ] uses: ./.github/workflows/coordinator-testing.yml if: ${{ inputs.coordinator_changed == 'true' }} with: @@ -50,8 +57,10 @@ jobs: transaction-exclusion-api: uses: ./.github/workflows/transaction-exclusion-api-testing.yml if: ${{ inputs.transaction_exclusion_api_changed == 'true' }} + secrets: inherit staterecovery: + # needs: [ cache-docker-images ] uses: ./.github/workflows/staterecovery-testing.yml if: ${{ inputs.staterecovery_changed == 'true' }} with: diff --git a/.github/workflows/transaction-exclusion-api-testing.yml b/.github/workflows/transaction-exclusion-api-testing.yml index f76f57ca4..e479343d4 100644 --- a/.github/workflows/transaction-exclusion-api-testing.yml +++ b/.github/workflows/transaction-exclusion-api-testing.yml @@ -2,6 +2,11 @@ name: transaction-exclusion-api-testing on: workflow_call: + secrets: + DOCKERHUB_USERNAME: + required: false + DOCKERHUB_TOKEN: + required: false workflow_dispatch: inputs: coverage: @@ -16,6 +21,9 @@ concurrency: jobs: run-tests: + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} # ~1.5 mins saved vs small runs-on: gha-runner-scale-set-ubuntu-22.04-amd64-med name: Transaction exclusion api tests @@ -36,6 +44,12 @@ jobs: if: ${{ !inputs.coverage }} run: | ./gradlew transaction-exclusion-api:app:buildNeeded + - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Run integration tests run: | ./gradlew transaction-exclusion-api:app:integrationTestAllNeeded diff --git a/docker/compose.yml b/docker/compose.yml index 2b459c9d1..0a3120efb 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -16,7 +16,7 @@ networks: config: - subnet: 10.10.10.0/24 -# To debug inside the network and volumes +# To debug inside the network and volumes: # docker run --rm -it --network=docker_linea -v=linea-local-dev:/data -v=linea-logs:/logs weibeld/ubuntu-networking bash services: