fix: attempt to cancel timed-out L1 transactions (#11567)

Add utility to send cancellation requests in l1_tx_utils.
We call this utility when a TX has reached 'timed out' stage.
This is fired asynchronously as it shouldn't block proceeding processes
that come after

Fixes #11345

spartan/terraform/deploy-release/variables.tf

@@ -1,7 +1,7 @@
 variable "GKE_CLUSTER_CONTEXT" {
   description = "GKE cluster context"
   type        = string
-  default     = "gke_testnet-440309_us-west1-a_aztec-gke"
+  default     = "gke_testnet-440309_us-west1-a_aztec-gke-private"
 }
 
 variable "RELEASE_NAME" {

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -529,19 +529,18 @@ describe('L1Publisher integration', () => {
       await expect(publisher.proposeL2Block(block)).resolves.toEqual(false);
 
       // Test for both calls
-      // NOTE: First error is from the simulate fn, which isn't supported by anvil
-      expect(loggerErrorSpy).toHaveBeenCalledTimes(3);
+      expect(loggerErrorSpy).toHaveBeenCalledTimes(2);
 
       // Test first call
       expect(loggerErrorSpy).toHaveBeenNthCalledWith(
-        2,
+        1,
         expect.stringMatching(/^L1 transaction 0x[a-f0-9]{64} reverted$/i),
         expect.anything(),
       );
 
       // Test second call
       expect(loggerErrorSpy).toHaveBeenNthCalledWith(
-        3,
+        2,
         expect.stringMatching(
           /^Rollup process tx reverted\. The contract function "propose" reverted\. Error: Rollup__InvalidInHash/i,
         ),

yarn-project/ethereum/src/l1_tx_utils.test.ts

@@ -477,4 +477,125 @@ describe('GasUtils', () => {
     ).rejects.toThrow(/timed out/);
     expect(Date.now() - now).toBeGreaterThanOrEqual(990);
   }, 60_000);
+
+  it('attempts to cancel timed out transactions', async () => {
+    // Disable auto-mining to control block production
+    await cheatCodes.setIntervalMining(0);
+    await cheatCodes.setAutomine(false);
+
+    const request = {
+      to: '0x1234567890123456789012345678901234567890' as `0x${string}`,
+      data: '0x' as `0x${string}`,
+      value: 0n,
+    };
+
+    // Send initial transaction
+    const { txHash } = await gasUtils.sendTransaction(request);
+    const initialTx = await publicClient.getTransaction({ hash: txHash });
+
+    // Try to monitor with a short timeout
+    const monitorPromise = gasUtils.monitorTransaction(
+      request,
+      txHash,
+      { gasLimit: initialTx.gas! },
+      { txTimeoutMs: 100, checkIntervalMs: 10 }, // Short timeout to trigger cancellation quickly
+    );
+
+    // Wait for timeout and catch the error
+    await expect(monitorPromise).rejects.toThrow('timed out');
+
+    // Wait for cancellation tx to be sent
+    await sleep(100);
+
+    // Get the nonce that was used
+    const nonce = initialTx.nonce;
+
+    // Get pending transactions
+    const pendingBlock = await publicClient.getBlock({ blockTag: 'pending' });
+    const pendingTxHash = pendingBlock.transactions[0];
+    const cancelTx = await publicClient.getTransaction({ hash: pendingTxHash });
+
+    // // Verify cancellation tx
+    expect(cancelTx).toBeDefined();
+    expect(cancelTx!.nonce).toBe(nonce);
+    expect(cancelTx!.to!.toLowerCase()).toBe(walletClient.account.address.toLowerCase());
+    expect(cancelTx!.value).toBe(0n);
+    expect(cancelTx!.maxFeePerGas).toBeGreaterThan(initialTx.maxFeePerGas!);
+    expect(cancelTx!.maxPriorityFeePerGas).toBeGreaterThan(initialTx.maxPriorityFeePerGas!);
+    expect(cancelTx!.gas).toBe(21000n);
+
+    // Mine a block to process the cancellation
+    await cheatCodes.evmMine();
+
+    // Verify the original transaction is no longer present
+    await expect(publicClient.getTransaction({ hash: txHash })).rejects.toThrow();
+  }, 10_000);
+
+  it('attempts to cancel timed out blob transactions with correct parameters', async () => {
+    // Disable auto-mining to control block production
+    await cheatCodes.setAutomine(false);
+    await cheatCodes.setIntervalMining(0);
+
+    // Create blob data
+    const blobData = new Uint8Array(131072).fill(1);
+    const kzg = Blob.getViemKzgInstance();
+
+    const request = {
+      to: '0x1234567890123456789012345678901234567890' as `0x${string}`,
+      data: '0x' as `0x${string}`,
+      value: 0n,
+    };
+
+    // Send initial blob transaction
+    const { txHash } = await gasUtils.sendTransaction(request, undefined, {
+      blobs: [blobData],
+      kzg,
+      maxFeePerBlobGas: 100n * WEI_CONST, // 100 gwei
+    });
+    const initialTx = await publicClient.getTransaction({ hash: txHash });
+
+    // Try to monitor with a short timeout
+    const monitorPromise = gasUtils.monitorTransaction(
+      request,
+      txHash,
+      { gasLimit: initialTx.gas! },
+      { txTimeoutMs: 100, checkIntervalMs: 10 }, // Short timeout to trigger cancellation quickly
+      {
+        blobs: [blobData],
+        kzg,
+        maxFeePerBlobGas: 100n * WEI_CONST,
+      },
+    );
+
+    // Wait for timeout and catch the error
+    await expect(monitorPromise).rejects.toThrow('timed out');
+
+    // Wait for cancellation tx to be sent
+    await sleep(100);
+
+    // Get the nonce that was used
+    const nonce = initialTx.nonce;
+
+    // Get pending transactions
+    const pendingBlock = await publicClient.getBlock({ blockTag: 'pending' });
+    const pendingTxHash = pendingBlock.transactions[0];
+    const cancelTx = await publicClient.getTransaction({ hash: pendingTxHash });
+
+    // Verify cancellation tx
+    expect(cancelTx).toBeDefined();
+    expect(cancelTx!.nonce).toBe(nonce);
+    expect(cancelTx!.to!.toLowerCase()).toBe(walletClient.account.address.toLowerCase());
+    expect(cancelTx!.value).toBe(0n);
+    expect(cancelTx!.maxFeePerGas).toBeGreaterThan(initialTx.maxFeePerGas!);
+    expect(cancelTx!.maxPriorityFeePerGas).toBeGreaterThan(initialTx.maxPriorityFeePerGas!);
+    expect(cancelTx!.maxFeePerBlobGas).toBeGreaterThan(initialTx.maxFeePerBlobGas!);
+    expect(cancelTx!.blobVersionedHashes).toBeDefined();
+    expect(cancelTx!.blobVersionedHashes!.length).toBe(1);
+
+    // Mine a block to process the cancellation
+    await cheatCodes.evmMine();
+
+    // Verify the original transaction is no longer present
+    await expect(publicClient.getTransaction({ hash: txHash })).rejects.toThrow();
+  }, 10_000);
 });

yarn-project/ethereum/src/l1_tx_utils.ts

@@ -1,3 +1,4 @@
+import { Blob } from '@aztec/foundation/blob';
 import { times } from '@aztec/foundation/collection';
 import {
   type ConfigMappingsType,
@@ -167,7 +168,7 @@ export const defaultL1TxUtilsConfig = getDefaultConfig<L1TxUtilsConfig>(l1TxUtil
 
 export interface L1TxRequest {
   to: Address | null;
-  data: Hex;
+  data?: Hex;
   value?: bigint;
 }
 
@@ -272,7 +273,9 @@ export class L1TxUtils {
     params: { gasLimit: bigint },
     _gasConfig?: Partial<L1TxUtilsConfig> & { txTimeoutAt?: Date },
     _blobInputs?: L1BlobInputs,
+    isCancelTx: boolean = false,
   ): Promise<TransactionReceipt> {
+    const isBlobTx = !!_blobInputs;
     const gasConfig = { ...this.config, ..._gasConfig };
     const account = this.walletClient.account;
     const blobInputs = _blobInputs || {};
@@ -288,6 +291,10 @@ export class L1TxUtils {
       true,
     );
 
+    if (!tx) {
+      throw new Error(`Failed to get L1 transaction ${initialTxHash} to monitor`);
+    }
+
     if (tx?.nonce === undefined || tx?.nonce === null) {
       throw new Error(`Failed to get L1 transaction ${initialTxHash} nonce`);
     }
@@ -297,6 +304,11 @@ export class L1TxUtils {
     let currentTxHash = initialTxHash;
     let attempts = 0;
     let lastAttemptSent = Date.now();
+    let lastGasPrice: GasPrice = {
+      maxFeePerGas: tx.maxFeePerGas!,
+      maxPriorityFeePerGas: tx.maxPriorityFeePerGas!,
+      maxFeePerBlobGas: tx.maxFeePerBlobGas!,
+    };
     const initialTxTime = lastAttemptSent;
 
     let txTimedOut = false;
@@ -355,7 +367,7 @@ export class L1TxUtils {
           attempts++;
           const newGasPrice = await this.getGasPrice(
             gasConfig,
-            !!blobInputs,
+            isBlobTx,
             attempts,
             tx.maxFeePerGas && tx.maxPriorityFeePerGas
               ? {
@@ -365,6 +377,7 @@ export class L1TxUtils {
                 }
               : undefined,
           );
+          lastGasPrice = newGasPrice;
 
           this.logger?.debug(
             `L1 transaction ${currentTxHash} appears stuck. Attempting speed-up ${attempts}/${gasConfig.maxAttempts} ` +
@@ -400,10 +413,25 @@ export class L1TxUtils {
       // Check if tx has timed out.
       txTimedOut = isTimedOut();
     }
-    this.logger?.error(`L1 transaction ${currentTxHash} timed out`, {
-      txHash: currentTxHash,
-      ...tx,
-    });
+
+    if (!isCancelTx) {
+      // Fire cancellation without awaiting to avoid blocking the main thread
+      this.attemptTxCancellation(nonce, isBlobTx, lastGasPrice, attempts)
+        .then(cancelTxHash => {
+          this.logger?.debug(`Sent cancellation tx ${cancelTxHash} for timed out tx ${currentTxHash}`);
+        })
+        .catch(err => {
+          const viemError = formatViemError(err);
+          this.logger?.error(`Failed to send cancellation for timed out tx ${currentTxHash}:`, viemError.message, {
+            metaMessages: viemError.metaMessages,
+          });
+        });
+
+      this.logger?.error(`L1 transaction ${currentTxHash} timed out`, {
+        txHash: currentTxHash,
+        ...tx,
+      });
+    }
     throw new Error(`L1 transaction ${currentTxHash} timed out`);
   }
 
@@ -469,7 +497,6 @@ export class L1TxUtils {
       // same for blob gas fee
       maxFeePerBlobGas = (maxFeePerBlobGas * (1_000n + 125n)) / 1_000n;
     }
-
     if (attempt > 0) {
       const configBump =
         gasConfig.priorityFeeRetryBumpPercentage ?? defaultL1TxUtilsConfig.priorityFeeRetryBumpPercentage!;
@@ -478,7 +505,6 @@ export class L1TxUtils {
       const minBumpPercentage = isBlobTx ? MIN_BLOB_REPLACEMENT_BUMP_PERCENTAGE : MIN_REPLACEMENT_BUMP_PERCENTAGE;
 
       const bumpPercentage = configBump > minBumpPercentage ? configBump : minBumpPercentage;
-
       // Calculate minimum required fees based on previous attempt
       // multiply by 100 & divide by 100 to maintain some precision
       const minPriorityFee =
@@ -619,11 +645,13 @@ export class L1TxUtils {
       return result[0].gasUsed;
     } catch (err) {
       if (err instanceof MethodNotFoundRpcError || err instanceof MethodNotSupportedRpcError) {
-        this.logger?.error('Node does not support eth_simulateV1 API');
         if (gasConfig.fallbackGasEstimate) {
-          this.logger?.debug(`Using fallback gas estimate: ${gasConfig.fallbackGasEstimate}`);
+          this.logger?.warn(
+            `Node does not support eth_simulateV1 API. Using fallback gas estimate: ${gasConfig.fallbackGasEstimate}`,
+          );
           return gasConfig.fallbackGasEstimate;
         }
+        this.logger?.error('Node does not support eth_simulateV1 API');
       }
       throw err;
     }
@@ -633,4 +661,83 @@ export class L1TxUtils {
     const gasConfig = { ...this.config, ..._gasConfig };
     return gasLimit + (gasLimit * BigInt((gasConfig?.gasLimitBufferPercentage || 0) * 1_00)) / 100_00n;
   }
+
+  /**
+   * Attempts to cancel a transaction by sending a 0-value tx to self with same nonce but higher gas prices
+   * @param nonce - The nonce of the transaction to cancel
+   * @param previousGasPrice - The gas price of the previous transaction
+   * @param attempts - The number of attempts to cancel the transaction
+   * @returns The hash of the cancellation transaction
+   */
+  private async attemptTxCancellation(nonce: number, isBlobTx = false, previousGasPrice?: GasPrice, attempts = 0) {
+    const account = this.walletClient.account;
+
+    // Get gas price with higher priority fee for cancellation
+    const cancelGasPrice = await this.getGasPrice(
+      {
+        ...this.config,
+        // Use high bump for cancellation to ensure it replaces the original tx
+        priorityFeeRetryBumpPercentage: 150, // 150% bump should be enough to replace any tx
+      },
+      isBlobTx,
+      attempts + 1,
+      previousGasPrice,
+    );
+
+    this.logger?.debug(`Attempting to cancel transaction with nonce ${nonce}`, {
+      maxFeePerGas: formatGwei(cancelGasPrice.maxFeePerGas),
+      maxPriorityFeePerGas: formatGwei(cancelGasPrice.maxPriorityFeePerGas),
+    });
+    const request = {
+      to: account.address,
+      value: 0n,
+    };
+
+    // Send 0-value tx to self with higher gas price
+    if (!isBlobTx) {
+      const cancelTxHash = await this.walletClient.sendTransaction({
+        ...request,
+        nonce,
+        gas: 21_000n, // Standard ETH transfer gas
+        maxFeePerGas: cancelGasPrice.maxFeePerGas,
+        maxPriorityFeePerGas: cancelGasPrice.maxPriorityFeePerGas,
+      });
+      const receipt = await this.monitorTransaction(
+        request,
+        cancelTxHash,
+        { gasLimit: 21_000n },
+        undefined,
+        undefined,
+        true,
+      );
+
+      return receipt.transactionHash;
+    } else {
+      const blobData = new Uint8Array(131072).fill(0);
+      const kzg = Blob.getViemKzgInstance();
+      const blobInputs = {
+        blobs: [blobData],
+        kzg,
+        maxFeePerBlobGas: cancelGasPrice.maxFeePerBlobGas!,
+      };
+      const cancelTxHash = await this.walletClient.sendTransaction({
+        ...request,
+        ...blobInputs,
+        nonce,
+        gas: 21_000n,
+        maxFeePerGas: cancelGasPrice.maxFeePerGas,
+        maxPriorityFeePerGas: cancelGasPrice.maxPriorityFeePerGas,
+      });
+      const receipt = await this.monitorTransaction(
+        request,
+        cancelTxHash,
+        { gasLimit: 21_000n },
+        undefined,
+        blobInputs,
+        true,
+      );
+
+      return receipt.transactionHash;
+    }
+  }
 }