docker: breakdown into smaller files and more efficient resources (#626)

* docker: breakdown into smaller files and more efficient resources
Consensys · Jan 30, 2025 · a63965e · a63965e
1 parent a8cdd28
commit a63965e
Show file tree

Hide file tree

Showing 18 changed files with 778 additions and 719 deletions.
diff --git a/.github/actions/compute-docker-images-hash/action.yml b/.github/actions/compute-docker-images-hash/action.yml
@@ -16,8 +16,8 @@ runs:
       id: compute-docker-images-hash
       shell: bash
       run: |
-        docker_images_traces_v1=$(docker compose -f docker/compose.yml -f docker/compose-local-dev.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
-        docker_images_traces_v2=$(docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_traces_v1=$(docker compose -f docker/compose-tracing-v1-ci-extension.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
+        docker_images_traces_v2=$(docker compose -f docker/compose-tracing-v2-ci-extension.yml --profile l1 --profile l2 config 2>/dev/null | grep "image:" | awk '{print $2}')
         docker_images=$(echo "$docker_images_traces_v1 $docker_images_traces_v2")
         docker_images_unique=$(echo $docker_images | xargs -n1 | sort -u)
         docker_images_trimmed=$(echo $docker_images_unique | xargs -n1 | grep -Ev "linea-postman|linea-coordinator|linea-transaction-exclusion-api|linea-traces-api-facade|linea-prover" | xargs)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -48,9 +48,7 @@ jobs:
               - 'config/common/traces-limits-v2.toml'
               - 'config/coordinator/**'
               - 'e2e/**'
-              - 'docker/compose.yml'
-              - 'docker/compose-local-dev.overrides.yml'
-              - 'docker/compose-local-dev-traces-v2.overrides.yml'
+              - 'docker/compose-*.yml'
             staterecovery:
               - 'state-recovery/**'
               - 'buildSrc/**'

diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml
@@ -96,8 +96,8 @@ jobs:
       - name: Create directory for conflated traces
         run: |
           mkdir -p tmp/local/traces/v2/conflated
-          chmod -R a+w tmp/local/traces/v2/conflated
-      # Disable cache for pulling docker images as empirically found that this 
+          chmod -R a+rw tmp/local/
+      # Disable cache for pulling docker images as empirically found that this
       # (retrieving cache and loading docker images) actually increased test time-to-completion
       # - name: Restore cached images
       #   id: restore-cached-images
@@ -111,7 +111,7 @@ jobs:
           retry_wait_seconds: 30
           timeout_minutes: 10
           command: |
-            make pull-images-external-to-monorepo
+            make docker-pull-images-external-to-monorepo
       - name: Download local docker image artifacts
         uses: actions/download-artifact@v4
         with:
@@ -134,7 +134,7 @@ jobs:
           retry_wait_seconds: 30
           timeout_minutes: 10
           command: |
-            make start-all
+            make start-env-with-tracing-v1-ci
           on_retry_command: |
             make clean-environment
       - name: Spin up fresh environment with besu tracing with retry
@@ -146,7 +146,7 @@ jobs:
           retry_wait_seconds: 30
           timeout_minutes: 10
           command: |
-            make start-all-traces-v2
+            make start-env-with-tracing-v2-ci
           on_retry_command: |
             make clean-environment
       - name: List docker containers/images
@@ -168,16 +168,17 @@ jobs:
         if: ${{ failure() && inputs.e2e-tests-logs-dump }}
         run: |
           mkdir -p docker_logs
+          find tmp/local/ >> docker_logs/files_in_shared_dir.txt
           docker ps -a >> docker_logs/docker_ps.txt
           docker logs coordinator --since 1h &>> docker_logs/coordinator.txt
-          docker logs prover --since 1h &>> docker_logs/prover.txt
-          docker logs prover-v3 --since 1h &>> docker_logs/prover-v3.txt;
+          docker logs prover --since 1h &>> docker_logs/prover.txt || true
+          docker logs prover-v3 --since 1h &>> docker_logs/prover-v3.txt; || true
           docker logs shomei --since 1h &>> docker_logs/shomei.txt
           docker logs zkbesu-shomei --since 1h &>> docker_logs/zkbesu-shomei.txt
           docker logs shomei-frontend --since 1h &>> docker_logs/shomei-frontend.txt
           docker logs postman --since 1h &>> docker_logs/postman.txt
-          docker logs traces-node --since 1h &>> docker_logs/traces-node.txt
-          docker logs traces-node-v2 --since 1h &>> docker_logs/traces-node-v2.txt;
+          docker logs traces-node --since 1h &>> docker_logs/traces-node.txt || true
+          docker logs traces-node-v2 --since 1h &>> docker_logs/traces-node-v2.txt; || true
           docker logs l2-node-besu --since 1h &>> docker_logs/l2-node-besu.txt;
           docker logs transaction-exclusion-api --since 1h &>> docker_logs/transaction-exclusion-api.txt
           docker logs sequencer --since 1h &>> docker_logs/sequencer.txt