chore(ci): recover from earthly bug with --no-cache, build images fro…

…m registry (#7462)

This tries to get a handle on the cache bug by both not reading build
images from inline cache, and restarting with `--no-cache` if there's a
graph edge bug

- rely on pushed build-images from registry instead of earthly inline
cache
- only notify on CI failure in slack if it master is failing on the
rerun
- pump up timeouts as now anything might run --no-cache
- set up ARM CI like normal CI but with just one e2e test (fix a blocker
for this with a permission kludge) including with reruns and notifying
on second run

.github/workflows/ci-arm.yml

@@ -22,58 +22,112 @@ env:
   # kludge until we move away from runners
   WAIT_FOR_RUNNERS: false
 jobs:
+  setup:
+    uses: ./.github/workflows/setup-runner.yml
+    with:
+      username: ${{ github.event.pull_request.user.login || github.actor }}
+      runner_type: builder-arm
+    secrets: inherit
+
+  changes:
+    runs-on: ubuntu-20.04
+    # Required permissions.
+    permissions:
+      pull-requests: read
+    # Set job outputs to values from filter step
+    outputs:
+      build-images: ${{ steps.filter.outputs.build-images }}
+    steps:
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
+        id: filter
+        with:
+          filters: |
+            build-images:
+              - 'build-images/**'
+
+  build-images:
+    needs: [setup, changes]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
+    steps:
+      # permission kludge before checkout, see https://github.com/actions/checkout/issues/211#issuecomment-611986243
+      - run: sudo chown -R $USER:$USER /home/ubuntu/
+      - uses: actions/checkout@v4
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-images-arm
+      - name: "Push Build Images If Changed"
+        if: ${{ needs.changes.outputs.build-images }}
+        timeout-minutes: 40
+        run: |
+          earthly-ci --push ./build-images/+build
+
   build:
-    runs-on: ubuntu-latest
+    needs: [build-images]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
     steps:
+      # permission kludge before checkout, see https://github.com/actions/checkout/issues/211#issuecomment-611986243
+      - run: sudo chown -R $USER:$USER /home/ubuntu/
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
+        with:
+          concurrency_key: build-arm
+      # prepare images locally, tagged by commit hash
       - name: "Build E2E Image"
         timeout-minutes: 40
-        uses: ./.github/ensure-builder
-        with:
-          runner_type: builder-arm
-          run: |
-            set -eux
-            git submodule update --init --recursive --recommend-shallow
-            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
-            scripts/earthly-ci \
-              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              ./yarn-project+export-e2e-test-images
+        run: |
+          earthly-ci ./yarn-project+export-e2e-test-images
 
-  # all the end-to-end integration tests for aztec
+  # all the non-bench end-to-end integration tests for aztec
   e2e:
-    needs: build
-    runs-on: ubuntu-latest
+    needs: [build]
+    runs-on: ${{ github.event.pull_request.user.login || github.actor }}-arm
     steps:
       - uses: actions/checkout@v4
-        with: { ref: "${{ github.event.pull_request.head.sha }}" }
-      - name: "Test"
-        timeout-minutes: 25
-        uses: ./.github/ensure-builder
+        with: { ref: "${{ env.GIT_COMMIT }}" }
+      - uses: ./.github/ci-setup-action
         with:
-          runner_type: builder-arm
-          run: |
-            sudo shutdown -P 25 # hack until core part of the scripts
-            set -eux
-            echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
-            scripts/earthly-ci \
-              --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
-              --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
-              --no-output ./yarn-project/end-to-end/+uniswap-trade-on-l1-from-l2
+          concurrency_key: e2e-arm
+      # prepare images locally, tagged by commit hash
+      - name: "Build E2E Image"
+        timeout-minutes: 40
+        run: |
+          earthly-ci ./yarn-project/end-to-end+uniswap-trade-on-l1-from-l2
 
-  # not notifying failures right now
-  # notify:
-  #   needs: [e2e]
-  #   runs-on: ubuntu-latest
-  #   if: ${{ github.ref == 'refs/heads/master' && failure() }}
-  #   steps:
-  #     - name: Send notification to aztec3-ci channel if workflow failed on master
-  #       uses: slackapi/[email protected]
-  #       with:
-  #         payload: |
-  #           {
-  #             "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-  #           }
-  #       env:
-  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}
+  rerun-check:
+    runs-on: ubuntu-20.04
+    permissions:
+      actions: write
+    needs: [setup, build-images, build, e2e]
+    if: ${{ !cancelled() }}
+    steps:
+      - name: Check for Rerun
+        env:
+          # We treat any skipped or failing jobs as a failure for the workflow as a whole.
+          HAD_FAILURE: ${{ contains(needs.*.result, 'failure') }}
+          GH_REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          if [[ $HAD_FAILURE == true ]] && [[ $RUN_ATTEMPT -lt 2 ]] ; then
+            echo "Retrying first workflow failure. This is a stop-gap until things are more stable."
+            gh workflow run rerun.yml -F run_id=${{ github.run_id }}
+          fi
+
+  # NOTE: we only notify failures after a rerun has occurred
+  notify:
+    needs: [e2e]
+    runs-on: ubuntu-latest
+    if: ${{ github.ref == 'refs/heads/master' && failure() && github.run_attempt >= 2 }}
+    steps:
+      - name: Send notification to aztec3-ci channel if workflow failed on master
+        uses: slackapi/[email protected]
+        with:
+          payload: |
+            {
+              "url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WORKFLOW_TRIGGER_URL }}