feat: start multiple zkevm-chain provers (#94)

* feat: start multiple zkevm-chain provers * feat: update scripts
taikoxyz · Jul 25, 2023 · 2d12441 · 2d12441
1 parent 08a5b0d
commit 2d12441
Show file tree

Hide file tree

Showing 8 changed files with 26 additions and 14 deletions.
diff --git a/.env.sample b/.env.sample
@@ -7,7 +7,6 @@ PORT_L2_EXECTION_ENGINE_HTTP=8547
 PORT_L2_EXECTION_ENGINE_WS=8548
 PORT_L2_EXECTION_ENGINE_METRICS=6060
 PORT_L2_EXECTION_ENGINE_P2P=30306
-PORT_ZKEVM_CHAIN_PROVER_RPCD=9001
 PORT_PROMETHEUS=9091
 PORT_GRAFANA=3001
 
@@ -31,6 +30,8 @@ L1_ENDPOINT_WS=
 # If you want to be a prover who generates and submits zero knowledge proofs of proposed L2 blocks, you need to change
 # `ENABLE_PROVER` to true and set `L1_PROVER_PRIVATE_KEY`.
 ENABLE_PROVER=false
+# How many provers you want to run concurrently, note that each prover will cost ~8 cores / ~16 GB memory.
+ZKEVM_CHAIN_INSTANCES_NUM=1
 # A L1 account (with balance) private key which will send the TaikoL1.proveBlock transactions.
 # WARNING: only use a test account, pasting your private key in plain text here is not secure.
 L1_PROVER_PRIVATE_KEY=

diff --git a/.env.sample.l3 b/.env.sample.l3
@@ -7,7 +7,6 @@ PORT_L3_EXECTION_ENGINE_HTTP=8549
 PORT_L3_EXECTION_ENGINE_WS=8550
 PORT_L3_EXECTION_ENGINE_METRICS=6061
 PORT_L3_EXECTION_ENGINE_P2P=30307
-PORT_ZKEVM_CHAIN_PROVER_RPCD=9002
 PORT_PROMETHEUS=9092
 PORT_GRAFANA=3002
 
@@ -33,6 +32,8 @@ L2_ENDPOINT_WS=
 # `ENABLE_PROVER` to true and set `L2_PROVER_PRIVATE_KEY`.
 ENABLE_PROVER=false
 PROVE_UNASSIGNED_BLOCKS=true
+# How many provers you want to run concurrently, note that each prover will cost ~16 cores / ~32 GB memory.
+ZKEVM_CHAIN_INSTANCES_NUM=1
 # A L2 account (with balance) private key which will send the TaikoL1.proveBlock transactions.
 # WARNING: only use a test account, pasting your private key in plain text here is not secure.
 L2_PROVER_PRIVATE_KEY=

diff --git a/docker-compose.l3.yml b/docker-compose.l3.yml
@@ -112,8 +112,6 @@ services:
   l3_zkevm_chain_prover_rpcd:
     image: gcr.io/evmchain/eldfell-proverd:latest
     restart: unless-stopped
-    ports:
-      - "${PORT_ZKEVM_CHAIN_PROVER_RPCD}:9000"
     pull_policy: always
     env_file:
       - .env.l3
@@ -124,6 +122,11 @@ services:
       - /bin/sh
       - -c
       - "/script/start-zkevm-chain-rpcd.sh"
+    deploy:
+      replicas: ${ZKEVM_CHAIN_INSTANCES_NUM}
+      resources:
+        limits:
+          memory: 32G
     <<: *logging
 
   l3_prometheus:

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -110,8 +110,6 @@ services:
   zkevm_chain_prover_rpcd:
     image: gcr.io/evmchain/grimsvotn-proverd:latest
     restart: unless-stopped
-    ports:
-      - "${PORT_ZKEVM_CHAIN_PROVER_RPCD}:9000"
     pull_policy: always
     env_file:
       - .env
@@ -122,6 +120,11 @@ services:
       - /bin/sh
       - -c
       - "/script/start-zkevm-chain-rpcd.sh"
+    deploy:
+      replicas: ${ZKEVM_CHAIN_INSTANCES_NUM}
+      resources:
+        limits:
+          memory: 32G
     <<: *logging
 
   prometheus:

diff --git a/script/l2/start-prover-relayer.sh b/script/l2/start-prover-relayer.sh
@@ -20,7 +20,7 @@ if [ "$ENABLE_PROVER" == "true" ]; then
         --zkevmRpcdEndpoint http://zkevm_chain_prover_rpcd:9000 \
         --zkevmRpcdParamsPath /data \
         --l1.proverPrivKey ${L1_PROVER_PRIVATE_KEY} \
-        --maxConcurrentProvingJobs 1
+        --maxConcurrentProvingJobs ${ZKEVM_CHAIN_INSTANCES_NUM}
 else
     sleep infinity
 fi
diff --git a/script/l2/start-zkevm-chain-rpcd.sh b/script/l2/start-zkevm-chain-rpcd.sh
@@ -9,7 +9,9 @@ if [ "$ENABLE_PROVER" == "true" ]; then
         wget https://storage.googleapis.com/zkevm-circuits-keys/19.bin -P /data
     fi
 
-    /prover_rpcd --bind 0.0.0.0:9000
+    /prover_rpcd \
+        --bind 0.0.0.0:9000 \
+        --lookup zkevm_chain_prover_rpcd:9000
 else
     sleep infinity
 fi
diff --git a/script/l3/start-prover-relayer.sh b/script/l3/start-prover-relayer.sh
@@ -8,7 +8,7 @@ if [ "$ENABLE_PROVER" == "true" ]; then
         chmod +x ./wait
     fi
 
-    WAIT_HOSTS=l3_zkevm_chain_prover_rpcd:${PORT_ZKEVM_CHAIN_PROVER_RPCD} WAIT_TIMEOUT=3600 ./wait
+    WAIT_HOSTS=l3_zkevm_chain_prover_rpcd:9000 WAIT_TIMEOUT=3600 ./wait
 
     if [ "$PROVE_UNASSIGNED_BLOCKS" == "true" ]; then
         taiko-client prover \
@@ -19,10 +19,10 @@ if [ "$ENABLE_PROVER" == "true" ]; then
         --taikoL1 ${TAIKO_L1_ADDRESS} \
         --taikoL2 ${TAIKO_L2_ADDRESS} \
         --taikoProverPoolL1 ${PROVER_POOL_ADDRESS} \
-        --zkevmRpcdEndpoint http://l3_zkevm_chain_prover_rpcd:${PORT_ZKEVM_CHAIN_PROVER_RPCD} \
+        --zkevmRpcdEndpoint http://l3_zkevm_chain_prover_rpcd:9000 \
         --zkevmRpcdParamsPath /data \
         --l1.proverPrivKey ${L2_PROVER_PRIVATE_KEY} \
-        --maxConcurrentProvingJobs 1 \
+        --maxConcurrentProvingJobs ${ZKEVM_CHAIN_INSTANCES_NUM} \
         --prover.proveUnassignedBlocks
     else
         taiko-client prover \
@@ -33,10 +33,10 @@ if [ "$ENABLE_PROVER" == "true" ]; then
         --taikoL1 ${TAIKO_L1_ADDRESS} \
         --taikoL2 ${TAIKO_L2_ADDRESS} \
         --taikoProverPoolL1 ${PROVER_POOL_ADDRESS} \
-        --zkevmRpcdEndpoint http://l3_zkevm_chain_prover_rpcd:${PORT_ZKEVM_CHAIN_PROVER_RPCD} \
+        --zkevmRpcdEndpoint http://l3_zkevm_chain_prover_rpcd:9000 \
         --zkevmRpcdParamsPath /data \
         --l1.proverPrivKey ${L2_PROVER_PRIVATE_KEY} \
-        --maxConcurrentProvingJobs 1 \
+        --maxConcurrentProvingJobs ${ZKEVM_CHAIN_INSTANCES_NUM} \
         --prover.proveUnassignedBlocks=false
     fi
 else

diff --git a/script/l3/start-zkevm-chain-rpcd.sh b/script/l3/start-zkevm-chain-rpcd.sh
@@ -7,7 +7,9 @@ if [ "$ENABLE_PROVER" == "true" ]; then
 
     rm -rf /data/kzg_bn254_21.srs && wget https://storage.googleapis.com/zkevm-circuits-keys/kzg_bn254_21.srs -P /data
 
-    /prover_rpcd --bind 0.0.0.0:${PORT_ZKEVM_CHAIN_PROVER_RPCD}
+    /prover_rpcd \
+        --bind 0.0.0.0:9000 \
+        --lookup l3_zkevm_chain_prover_rpcd:9000
 else
     sleep infinity
 fi