ci: Optimize contracts build

.github/workflows/build-contracts.yml

@@ -0,0 +1,83 @@
+name: Build contracts
+on:
+  workflow_call:
+    inputs:
+      compilers:
+        description: 'JSON of required compilers and their versions'
+        type: string
+        required: false
+        default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18"] } , { "zkvyper": ["1.3.13"] }]'
+
+jobs:
+  build-images:
+    name: Build and upload contracts
+    runs-on: [matterlabs-ci-runner]
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo CI=1 >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo CI=1 >> .env
+          echo IN_DOCKER=1 >> .env
+
+      # TODO: Remove after when we can upgrade hardhat-plugins
+      - name: pre-download compiilers
+        run: |
+          # Download needed versions of vyper compiler
+          # Not sanitized due to unconventional path and tags
+          mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
+          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
+          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
+          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
+          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3
+
+          COMPILERS_JSON='${{ inputs.compilers }}'
+          echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
+            mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
+            wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
+            chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
+          done
+
+      - name: start-services
+        run: |
+          echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
+          mkdir -p ./volumes/postgres
+          docker compose up -d zk postgres
+          ci_run sccache --start-server
+
+      - name: build contracts
+        run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts
+          ci_run zk
+          ci_run zk clean --all
+          ci_run zk run yarn
+          ci_run cp etc/tokens/{test,localhost}.json
+          ci_run zk compiler all
+          ci_run zk contract build
+          ci_run zk f yarn run l2-contracts build
+
+      - name: upload contracts
+        uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392
+        with:
+          name: contracts
+          path: |
+            ./contracts/system-contracts/**/artifacts/
+            ./contracts/system-contracts/**/artifacts-zk/
+            ./contracts/l1-contracts/**/artifacts/
+            ./contracts/l1-contracts/**/artifacts-zk/
+            ./contracts/l2-contracts/**/artifacts/
+            ./contracts/l2-contracts/**/artifacts-zk/
+          compression-level: 0
+
+      - name: Show sccache stats
+        if: always()
+        run: |
+          ci_run sccache --show-stats
+          ci_run cat /tmp/sccache_log.txt

.github/workflows/build-core-template.yml

@@ -22,27 +22,32 @@ on:
         type: string
         default: "push"
         required: false
-      compilers:
-        description: 'JSON of required compilers and their versions'
-        type: string
-        required: false
-        default: '[{ "zksolc": ["1.3.14", "1.3.16", "1.3.17", "1.3.1", "1.3.7", "1.3.18"] } , { "zkvyper": ["1.3.13"] }]'
-
 jobs:
   build-images:
     name: Build and Push Docker Images
     env:
       image_tag: ${{ inputs.image_tag }}
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
-    runs-on: [matterlabs-ci-runner]
+    runs-on: ${{ matrix.component.runner }}
     strategy:
       matrix:
-        component:
-          - server-v2
-          - external-node
-          - contract-verifier
-          - cross-external-nodes-checker
-          - snapshots-creator
+          component:
+          - name: server-v2
+            platforms: linux/amd64
+            runner: matterlabs-ci-runner
+          - name: external-node
+            platforms: linux/amd64
+            runner: matterlabs-ci-runner
+          - name: contract-verifier
+            platforms: linux/amd64
+            runner: matterlabs-ci-runner
+          - name: cross-external-nodes-checker
+            platforms: linux/amd64
+            runner: matterlabs-ci-runner
+          - name: snapshots-creator
+            platforms: linux/amd64
+            runner: matterlabs-ci-runner
+
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:
@@ -56,23 +61,11 @@ jobs:
           echo CI=1 >> .env
           echo IN_DOCKER=1 >> .env
 
-      # TODO: Remove after when we can upgrade hardhat-plugins
-      - name: pre-download compiilers
-        run: |
-          # Download needed versions of vyper compiler
-          # Not sanitized due to unconventional path and tags
-          mkdir -p ./hardhat-nodejs/compilers-v2/vyper/linux
-          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10 https://github.com/vyperlang/vyper/releases/download/v0.3.10/vyper.0.3.10+commit.91361694.linux
-          wget -nv -O ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3 https://github.com/vyperlang/vyper/releases/download/v0.3.3/vyper.0.3.3+commit.48e326f0.linux
-          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.10
-          chmod +x  ./hardhat-nodejs/compilers-v2/vyper/linux/0.3.3
-
-          COMPILERS_JSON='${{ inputs.compilers }}'
-          echo "$COMPILERS_JSON" | jq -r '.[] | to_entries[] | .key as $compiler | .value[] | "\(.),\($compiler)"' | while IFS=, read -r version compiler; do
-            mkdir -p "./hardhat-nodejs/compilers-v2/$compiler"
-            wget -nv -O "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}" "https://github.com/matter-labs/${compiler}-bin/releases/download/v${version}/${compiler}-linux-amd64-musl-v${version}"
-            chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
-          done
+      - name: Download contracts
+        uses: actions/download-artifact@v4
+        with:
+          name: contracts
+          path: ./contracts/
 
       - name: start-services
         run: |
@@ -87,13 +80,8 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/sdk/binaryen
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
-          ci_run zk
-          ci_run zk clean --all
-          ci_run zk run yarn
-          ci_run cp etc/tokens/{test,localhost}.json
-          ci_run zk compiler all
-          ci_run zk contract build
-          ci_run zk f yarn run l2-contracts build
+          ci_run zk || true
+          ci_run yarn zk build
           ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
       - name: login to Docker registries
@@ -105,10 +93,11 @@ jobs:
       - name: update-images
         env:
           DOCKER_ACTION: ${{ inputs.action }}
-          COMPONENT: ${{ matrix.component }}
+          COMPONENT: ${{ matrix.component.name }}
+          PLATFORMS: ${{ matrix.component.platforms }}
         run: |
           ci_run rustup default nightly-2023-08-21
-          ci_run zk docker $DOCKER_ACTION $COMPONENT
+          ci_run zk docker $DOCKER_ACTION --platforms=${PLATFORMS} $COMPONENT
       - name: Show sccache stats
         if: always()
         run: |

.github/workflows/ci.yml

@@ -96,9 +96,15 @@ jobs:
     name: CI for Docs
     uses: ./.github/workflows/ci-docs-reusable.yml
 
+  build-contracts:
+    name: Build contracts
+    needs: changed_files
+    if: ${{ (needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }}
+    uses: ./.github/workflows/build-contracts.yml
+
   build-core-images:
     name: Build core images
-    needs: changed_files
+    needs: build-contracts
     if: ${{ (needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }}
     uses: ./.github/workflows/build-core-template.yml
     with:

infrastructure/zk/src/docker.ts

@@ -1,6 +1,5 @@
 import { Command } from 'commander';
 import * as utils from './utils';
-import * as contract from './contract';
 
 const IMAGES = [
     'server-v2',
@@ -20,12 +19,17 @@ const IMAGES = [
     'proof-fri-compressor',
     'snapshots-creator'
 ];
+
+const DOCKER_REGISTRIES = ['us-docker.pkg.dev/matterlabs-infra/matterlabs-docker', 'matterlabs'];
+
 const UNIX_TIMESTAMP = Date.now();
 
 async function dockerCommand(
     command: 'push' | 'build',
     image: string,
+    platforms: string[] = ['linux/amd64'],
     customTag?: string,
+    buildExtraArgs: string = '',
     dockerOrg: string = 'matterlabs'
 ) {
     // Generating all tags for containers. We need 2 tags here: SHA and SHA+TS
@@ -37,8 +41,8 @@ async function dockerCommand(
 
     // We want an alternative flow for Rust image
     if (image == 'rust') {
-        await dockerCommand(command, 'server-v2', customTag, dockerOrg);
-        await dockerCommand(command, 'prover', customTag, dockerOrg);
+        await dockerCommand(command, 'server-v2', platforms, customTag, dockerOrg);
+        await dockerCommand(command, 'prover', platforms, customTag, dockerOrg);
         return;
     }
     if (!IMAGES.includes(image)) {
@@ -54,10 +58,7 @@ async function dockerCommand(
     // Main build\push flow
     switch (command) {
         case 'build':
-            await _build(image, tagList, dockerOrg);
-            break;
-        case 'push':
-            await _push(image, tagList);
+            await _build(image, tagList, dockerOrg, platforms, buildExtraArgs);
             break;
         default:
             console.log(`Unknown command for docker ${command}.`);
@@ -88,16 +89,21 @@ function defaultTagList(image: string, imageTagSha: string, imageTagShaTS: strin
     return tagList;
 }
 
-async function _build(image: string, tagList: string[], dockerOrg: string) {
-    if (image === 'server-v2' || image === 'external-node' || image === 'prover') {
-        await contract.build();
-    }
+async function _build(
+    image: string,
+    tagList: string[],
+    dockerOrg: string,
+    platforms: string[],
+    extraArgs: string = ''
+) {
     let tagsToBuild = '';
 
-    // generate list of tags for image - we want 3 tags (latest, SHA, SHA+TimeStamp) for listed components and only "latest" for everything else
-    tagsToBuild = tagList.map((tag) => `-t ${dockerOrg}/${image}:${tag}`).join(' ');
+    for (const tag of tagList) {
+        for (const registry of DOCKER_REGISTRIES) {
+            tagsToBuild = tagsToBuild + `-t ${registry}/${image}:${tag} `;
+        }
+    }
 
-    // Conditionally add build argument if image is prover-v2
     let buildArgs = '';
     if (image === 'prover-v2') {
         const eraBellmanCudaRelease = process.env.ERA_BELLMAN_CUDA_RELEASE;
@@ -107,53 +113,29 @@ async function _build(image: string, tagList: string[], dockerOrg: string) {
         const cudaArch = process.env.CUDA_ARCH;
         buildArgs += `--build-arg CUDA_ARCH='${cudaArch}'`;
     }
+    buildArgs += extraArgs;
 
-    // HACK
-    // For prover-v2 which is not a prover, but should be built from the prover dockerfile. So here we go.
-    const imagePath = image == 'prover-v2' ? 'prover' : image;
+    const imagePath = image === 'prover-v2' ? 'prover' : image;
 
     const buildCommand =
-        `DOCKER_BUILDKIT=1 docker build ${tagsToBuild}` +
+        `DOCKER_BUILDKIT=1 docker buildx build ${tagsToBuild}` +
+        ` --platform=${platforms.join(',')}` +
         (buildArgs ? ` ${buildArgs}` : '') +
         ` -f ./docker/${imagePath}/Dockerfile .`;
 
     await utils.spawn(buildCommand);
 }
 
-async function _push(image: string, tagList: string[]) {
-    // For development purposes, we want to use `2.0` tags for 2.0 images, just to not interfere with 1.x
-
-    for (const tag of tagList) {
-        await utils.spawn(`docker push matterlabs/${image}:${tag}`);
-        await utils.spawn(
-            `docker tag matterlabs/${image}:${tag} us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`
-        );
-        await utils.spawn(`docker push us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`);
-
-        if (image == 'circuit-synthesizer') {
-            await utils.spawn(
-                `docker tag us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag} asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`
-            );
-            await utils.spawn(
-                `docker tag us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag} europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`
-            );
-            await utils.spawn(`docker push asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`);
-            await utils.spawn(`docker push europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${image}:${tag}`);
-        }
-    }
-}
-
 export async function build(image: string, cmd: Command) {
-    await dockerCommand('build', image, cmd.customTag);
+    await dockerCommand('build', image, cmd.platforms, cmd.customTag);
 }
 
 export async function customBuildForHyperchain(image: string, dockerOrg: string) {
-    await dockerCommand('build', image, '', dockerOrg);
+    await dockerCommand('build', image, ['linux/amd64'], dockerOrg);
 }
 
 export async function push(image: string, cmd: Command) {
-    await dockerCommand('build', image, cmd.customTag);
-    await dockerCommand('push', image, cmd.customTag);
+    await dockerCommand('build', image, cmd.platforms, cmd.customTag, '--push');
 }
 
 export async function restart(container: string) {
@@ -169,12 +151,13 @@ export const command = new Command('docker').description('docker management');
 command
     .command('build <image>')
     .option('--custom-tag <value>', 'Custom tag for image')
+    .option('--platforms <platforms>', 'Comma-separated list of platforms', (val) => val.split(','), ['linux/amd64'])
     .description('build docker image')
     .action(build);
 command
     .command('push <image>')
     .option('--custom-tag <value>', 'Custom tag for image')
-    .description('build and push docker image')
+    .option('--platforms <platforms>', 'Comma-separated list of platforms', (val) => val.split(','), ['linux/amd64'])
     .action(push);
 command.command('pull').description('pull all containers').action(pull);
 command.command('restart <container>').description('restart container in docker-compose.yml').action(restart);