fix: deploy l2 contracts fails on 48 validator (#9860)

This was sometimes erroring out on the first attempt while awaiting the transaction to settle, but then subsequent calls were attempting to reinitialize. Added better logging to help clarify. Also adjust the validator url in the template to be the top level service, to better distribute transactions across the network and not rely so heavily on the boot node for gossiping. --------- Co-authored-by: ludamad <[email protected]>
AztecProtocol · Nov 11, 2024 · 3cc2818 · 3cc2818
1 parent 6e28795
commit 3cc2818
Show file tree

Hide file tree

Showing 7 changed files with 96 additions and 69 deletions.
diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl
@@ -78,7 +78,7 @@ http://{{ include "aztec-network.fullname" . }}-boot-node-0.{{ include "aztec-ne
 {{- if .Values.validator.externalTcpHost -}}
 http://{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.nodePort }}
 {{- else -}}
-http://{{ include "aztec-network.fullname" . }}-validator-0.{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }}
+http://{{ include "aztec-network.fullname" . }}-validator.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.validator.service.nodePort }}
 {{- end -}}
 {{- end -}}
 

diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml
@@ -49,7 +49,7 @@ spec:
             - name: ETHEREUM_HOST
               value: {{ include "aztec-network.ethereumHost" . | quote }}
             - name: AZTEC_NODE_URL
-              value: {{ include "aztec-network.bootNodeUrl" . | quote }}
+              value: {{ include "aztec-network.validatorUrl" . | quote }}
             - name: LOG_JSON
               value: "1"
             - name: LOG_LEVEL

diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml
@@ -111,6 +111,7 @@ proverNode:
   storage: "8Gi"
 
 pxe:
+  proverEnabled: false
   externalHost: ""
   logLevel: "debug"
   debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database"

diff --git a/spartan/aztec-network/values/48-validators.yaml b/spartan/aztec-network/values/48-validators.yaml
@@ -1,15 +1,4 @@
-##########
-# BEWARE #
-##########
-# You need to deploy the metrics helm chart before using this values file.
-# head to spartan/metrics and run `./install.sh`
-# (then `./forward.sh` if you want to see it)
-telemetry:
-  enabled: true
-  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
-
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 48
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80

diff --git a/spartan/scripts/deploy_spartan.sh b/spartan/scripts/deploy_spartan.sh
@@ -6,6 +6,7 @@ TAG=$1
 VALUES=$2
 NAMESPACE=${3:-spartan}
 PROD=${4:-true}
+IMAGE_REPO=${5:-aztecprotocol}
 PROD_ARGS=""
 if [ "$PROD" = "true" ] ; then
   PROD_ARGS="--set network.public=true --set telemetry.enabled=true --set telemetry.otelCollectorEndpoint=http://metrics-opentelemetry-collector.metrics:4318"
@@ -45,8 +46,8 @@ log_stern &
 
 function upgrade() {
   # pull and resolve the image just to be absolutely sure k8s gets the latest image in the tag we want
-  docker pull --platform linux/amd64 aztecprotocol/aztec:$TAG
-  IMAGE=$(docker inspect --format='{{index .RepoDigests 0}}' aztecprotocol/aztec:$TAG)
+  docker pull --platform linux/amd64 $IMAGE_REPO/aztec:$TAG
+  IMAGE=$(docker inspect --format='{{index .RepoDigests 0}}' $IMAGE_REPO/aztec:$TAG)
   if ! [ -z "${PRINT_ONLY:-}" ] ; then
     helm template $NAMESPACE $SCRIPT_DIR/../aztec-network \
           --namespace $NAMESPACE \

diff --git a/yarn-project/cli/src/cmds/misc/setup_contracts.ts b/yarn-project/cli/src/cmds/misc/setup_contracts.ts
@@ -17,9 +17,23 @@ export async function setupCanonicalL2FeeJuice(
   const { FeeJuiceContract } = await import('@aztec/noir-contracts.js');
 
   const feeJuiceContract = await FeeJuiceContract.at(ProtocolContractAddress.FeeJuice, deployer);
+
   log('setupCanonicalL2FeeJuice: Calling initialize on fee juice contract...');
-  await feeJuiceContract.methods
-    .initialize(feeJuicePortalAddress)
-    .send({ fee: { paymentMethod: new NoFeePaymentMethod(), gasSettings: GasSettings.teardownless() } })
-    .wait(waitOpts);
+
+  try {
+    const provenTx = await feeJuiceContract.methods
+      .initialize(feeJuicePortalAddress)
+      .prove({ fee: { paymentMethod: new NoFeePaymentMethod(), gasSettings: GasSettings.teardownless() } });
+
+    await provenTx.send().wait(waitOpts);
+    log('setupCanonicalL2FeeJuice: Fee juice contract initialized');
+  } catch (e: any) {
+    // TODO: make this less brittle, e.g. using a 204 http code
+    if (e instanceof Error && e.message.includes('(method pxe_simulateTx) (code 400) Assertion failed')) {
+      log('setupCanonicalL2FeeJuice: Fee juice contract already initialized');
+    } else {
+      log('setupCanonicalL2FeeJuice: Error initializing fee juice contract', e);
+      throw e;
+    }
+  }
 }
diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts
@@ -522,10 +522,19 @@ export class PXEService implements PXE {
     txRequest: TxExecutionRequest,
     privateExecutionResult: PrivateExecutionResult,
   ): Promise<TxProvingResult> {
-    return this.jobQueue.put(async () => {
-      const { publicInputs, clientIvcProof } = await this.#prove(txRequest, this.proofCreator, privateExecutionResult);
-      return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!);
-    });
+    return this.jobQueue
+      .put(async () => {
+        const { publicInputs, clientIvcProof } = await this.#prove(
+          txRequest,
+          this.proofCreator,
+          privateExecutionResult,
+        );
+        return new TxProvingResult(privateExecutionResult, publicInputs, clientIvcProof!);
+      })
+      .catch(err => {
+        this.log.error(err);
+        throw err;
+      });
   }
 
   // TODO(#7456) Prevent msgSender being defined here for the first call
@@ -537,47 +546,52 @@ export class PXEService implements PXE {
     profile: boolean = false,
     scopes?: AztecAddress[],
   ): Promise<TxSimulationResult> {
-    return await this.jobQueue.put(async () => {
-      const privateExecutionResult = await this.#executePrivate(txRequest, msgSender, scopes);
-
-      let publicInputs: PrivateKernelTailCircuitPublicInputs;
-      let profileResult;
-      if (profile) {
-        ({ publicInputs, profileResult } = await this.#profileKernelProver(
-          txRequest,
-          this.proofCreator,
-          privateExecutionResult,
-        ));
-      } else {
-        publicInputs = await this.#simulateKernels(txRequest, privateExecutionResult);
-      }
+    return await this.jobQueue
+      .put(async () => {
+        const privateExecutionResult = await this.#executePrivate(txRequest, msgSender, scopes);
+
+        let publicInputs: PrivateKernelTailCircuitPublicInputs;
+        let profileResult;
+        if (profile) {
+          ({ publicInputs, profileResult } = await this.#profileKernelProver(
+            txRequest,
+            this.proofCreator,
+            privateExecutionResult,
+          ));
+        } else {
+          publicInputs = await this.#simulateKernels(txRequest, privateExecutionResult);
+        }
 
-      const privateSimulationResult = new PrivateSimulationResult(privateExecutionResult, publicInputs);
-      const simulatedTx = privateSimulationResult.toSimulatedTx();
-      let publicOutput: PublicSimulationOutput | undefined;
-      if (simulatePublic) {
-        publicOutput = await this.#simulatePublicCalls(simulatedTx);
-      }
+        const privateSimulationResult = new PrivateSimulationResult(privateExecutionResult, publicInputs);
+        const simulatedTx = privateSimulationResult.toSimulatedTx();
+        let publicOutput: PublicSimulationOutput | undefined;
+        if (simulatePublic) {
+          publicOutput = await this.#simulatePublicCalls(simulatedTx);
+        }
 
-      if (!skipTxValidation) {
-        if (!(await this.node.isValidTx(simulatedTx, true))) {
-          throw new Error('The simulated transaction is unable to be added to state and is invalid.');
+        if (!skipTxValidation) {
+          if (!(await this.node.isValidTx(simulatedTx, true))) {
+            throw new Error('The simulated transaction is unable to be added to state and is invalid.');
+          }
         }
-      }
 
-      // We log only if the msgSender is undefined, as simulating with a different msgSender
-      // is unlikely to be a real transaction, and likely to be only used to read data.
-      // Meaning that it will not necessarily have produced a nullifier (and thus have no TxHash)
-      // If we log, the `getTxHash` function will throw.
-      if (!msgSender) {
-        this.log.info(`Executed local simulation for ${simulatedTx.getTxHash()}`);
-      }
-      return TxSimulationResult.fromPrivateSimulationResultAndPublicOutput(
-        privateSimulationResult,
-        publicOutput,
-        profileResult,
-      );
-    });
+        // We log only if the msgSender is undefined, as simulating with a different msgSender
+        // is unlikely to be a real transaction, and likely to be only used to read data.
+        // Meaning that it will not necessarily have produced a nullifier (and thus have no TxHash)
+        // If we log, the `getTxHash` function will throw.
+        if (!msgSender) {
+          this.log.info(`Executed local simulation for ${simulatedTx.getTxHash()}`);
+        }
+        return TxSimulationResult.fromPrivateSimulationResultAndPublicOutput(
+          privateSimulationResult,
+          publicOutput,
+          profileResult,
+        );
+      })
+      .catch(err => {
+        this.log.error(err);
+        throw err;
+      });
   }
 
   public async sendTx(tx: Tx): Promise<TxHash> {
@@ -586,7 +600,10 @@ export class PXEService implements PXE {
       throw new Error(`A settled tx with equal hash ${txHash.toString()} exists.`);
     }
     this.log.info(`Sending transaction ${txHash}`);
-    await this.node.sendTx(tx);
+    await this.node.sendTx(tx).catch(err => {
+      this.log.error(err);
+      throw err;
+    });
     this.log.info(`Sent transaction ${txHash}`);
     return txHash;
   }
@@ -599,14 +616,19 @@ export class PXEService implements PXE {
     scopes?: AztecAddress[],
   ): Promise<AbiDecoded> {
     // all simulations must be serialized w.r.t. the synchronizer
-    return await this.jobQueue.put(async () => {
-      // TODO - Should check if `from` has the permission to call the view function.
-      const functionCall = await this.#getFunctionCall(functionName, args, to);
-      const executionResult = await this.#simulateUnconstrained(functionCall, scopes);
-
-      // TODO - Return typed result based on the function artifact.
-      return executionResult;
-    });
+    return await this.jobQueue
+      .put(async () => {
+        // TODO - Should check if `from` has the permission to call the view function.
+        const functionCall = await this.#getFunctionCall(functionName, args, to);
+        const executionResult = await this.#simulateUnconstrained(functionCall, scopes);
+
+        // TODO - Return typed result based on the function artifact.
+        return executionResult;
+      })
+      .catch(err => {
+        this.log.error(err);
+        throw err;
+      });
   }
 
   public getTxReceipt(txHash: TxHash): Promise<TxReceipt> {