diff --git a/.github/ci-setup-action/action.yml b/.github/ci-setup-action/action.yml new file mode 100644 index 00000000000..15df4df1a28 --- /dev/null +++ b/.github/ci-setup-action/action.yml @@ -0,0 +1,35 @@ +# Reusable setup workflow for CI tasks +name: Setup Workflow +description: 'Reusable setup steps' + +inputs: + dockerhub_password: + required: true + description: 'DockerHub Password' + github_actor: + required: true + description: 'GitHub Actor' + +runs: + # define an action, runs in OS of caller + using: composite + steps: + - name: Cache Submodules + id: cache-submodules + uses: actions/cache@v3 + with: + path: .git + key: submodules-${{ hashFiles('.gitmodules') }} + + - name: Checkout Submodules + shell: bash + run: git submodule sync --recursive && git submodule update --init --recursive + + - name: Setup Earthly + uses: earthly/actions-setup@v1 + with: + version: 'v0.8.5' + + - name: Setup Env + shell: bash + run: ./scripts/setup_env.sh ${{ inputs.dockerhub_password }} ${{ inputs.github_actor }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fead02fb6f..d42849a778a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,24 +15,22 @@ jobs: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: build-${{ github.event.pull_request.number || github.ref_name }}-x86 - cancel-in-progress: ${{ github.ref_name != 'master' }} + group: build-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true steps: - - uses: earthly/actions-setup@v1 - with: - version: v0.8.5 - - name: Checkout uses: actions/checkout@v4 with: + # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - name: Setup - working-directory: ./scripts - run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }} + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - - name: Test + - name: Build working-directory: ./yarn-project run: | # push to registry @@ -58,29 +56,27 @@ jobs: - e2e-crowdfunding-and-claim # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-arm - cancel-in-progress: ${{ github.ref_name != 'master' }} + group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id|| github.ref_name }}-arm + cancel-in-progress: true steps: - - uses: earthly/actions-setup@v1 - with: - version: v0.8.5 - - name: Checkout uses: actions/checkout@v4 with: + # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - name: Setup - working-directory: ./scripts - run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }} + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - - name: Test + - name: Build working-directory: ./yarn-project/end-to-end/ # We don't do much on arm, just run it on their builder run: # Flags: - # - e2e_build=true to get images from a local build, not a dockerhub registry strategy + # - e2e_mode=build to get images from a local build, not a dockerhub registry strategy earthly-cloud build arm --no-output +${{ matrix.test }} --e2e_mode=build # all the end-to-end integration tests for aztec @@ -95,22 +91,20 @@ jobs: test: ${{ fromJson( needs.build-x86.outputs.e2e_list )}} # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: ${{ matrix.test }}-${{ github.event.pull_request.number || github.ref_name }}-x86 - cancel-in-progress: ${{ github.ref_name != 'master' }} + group: ${{ matrix.test }}-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true steps: - - uses: earthly/actions-setup@v1 - with: - version: v0.8.5 - - name: Checkout uses: actions/checkout@v4 with: + # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - name: Setup - working-directory: ./scripts - run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }} + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Pull E2E Images working-directory: ./barretenberg/cpp/ @@ -123,7 +117,8 @@ jobs: - name: Test working-directory: ./yarn-project/end-to-end/ run: | - earthly-cloud build x86 --no-output +${{ matrix.test }} --e2e_mode=cache + # we blank earthly token just to be sure this is a pure local run + EARTHLY_TOKEN="" earthly -P --no-output +${{ matrix.test }} --e2e_mode=cache # barretenberg (prover) native tests bb-native-tests: @@ -134,26 +129,24 @@ jobs: fail-fast: false matrix: environment: [x86] - # pending fix fot intermittent test + # pending fix for intermittent test # environment: [x86, arm] # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: bb-native-tests-${{ github.event.pull_request.number || github.ref_name }}-${{ matrix.environment }} - cancel-in-progress: ${{ github.ref_name != 'master' }} + group: bb-native-tests-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-${{ matrix.environment }} + cancel-in-progress: true steps: - - uses: earthly/actions-setup@v1 - with: - version: v0.8.5 - - name: Checkout uses: actions/checkout@v4 with: + # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - name: Setup - working-directory: ./scripts - run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }} + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Test working-directory: ./barretenberg/cpp/ @@ -170,28 +163,35 @@ jobs: EARTHLY_TOKEN: ${{ secrets.EARTHLY_TOKEN }} # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - # TODO this allows NO concurrency of bench jobs - # TODO if this takes too long, we need to divide up into different bench machines for each pipeline step - group: bench-x86 + # We use a looser concurrency limit here, but we place a lock + # in the actual step action so that only one bench takes place + group: bench-${{ github.ref_name == 'master' && github.run_id || github.ref_name }}-x86 + cancel-in-progress: true steps: - - uses: earthly/actions-setup@v1 - with: - version: v0.8.5 - - name: Checkout uses: actions/checkout@v4 with: + # we check out submodules in ci-setup-action ref: ${{ github.event.pull_request.head.sha }} - submodules: recursive - name: Setup - working-directory: ./scripts - run: ./setup_env.sh ${{ secrets.DOCKERHUB_PASSWORD }} ${{ github.actor }} + uses: ./.github/ci-setup-action + with: + dockerhub_password: ${{ secrets.DOCKERHUB_PASSWORD }} + github_actor: ${{ github.actor }} - name: Build and Push Binaries working-directory: ./barretenberg/cpp/ run: earthly-cloud build x86 --push +bench-base + # Only allow one person to use this runner + # The reason is that as detailed in https://github.com/ben-z/gh-action-mutex + # things do not become 'pending' in github actions, and instead just cancel one another + - name: Set up mutex + uses: ben-z/gh-action-mutex@v1.0.0-alpha.9 + with: + branch: gh-action-mutex-bench + # Use bench_mode=cache to read the pushed build above - name: Client IVC Bench working-directory: ./barretenberg/cpp/ diff --git a/.github/workflows/protocol-circuits-gate-diff.yml b/.github/workflows/protocol-circuits-gate-diff.yml index 3978acd93f0..8126a13193c 100644 --- a/.github/workflows/protocol-circuits-gate-diff.yml +++ b/.github/workflows/protocol-circuits-gate-diff.yml @@ -8,10 +8,9 @@ on: jobs: compare_protocol_circuits_gates: - # cancel if reran on same PR if exists, otherwise if on same commit concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.ref_name != 'master' }} + group: compare_protocol_circuits_gates-${{ github.ref_name == 'master' && github.run_id || github.ref_name }} + cancel-in-progress: true if: "!startsWith(github.head_ref, 'release-please--')" runs-on: ubuntu-20.04 steps: