From 0cc74d5c1f975643338ee8faa0bc0b433cddf581 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 9 Apr 2024 16:13:39 +0200 Subject: [PATCH 1/5] Restructure web3.js plugin --- .../simulation/assertions/blobsAssertion.ts | 2 +- .../simulation/clients/execution/geth.ts | 2 +- .../clients/execution/nethermind.ts | 2 +- .../blobsEIP4844Transaction.ts} | 61 ------------------- .../utils/simulation/web3js/plugins/index.ts | 8 +++ .../web3js/plugins/web3AdminPlugin.ts | 37 +++++++++++ .../web3js/plugins/web3ExtendedEthPlugin.ts | 19 ++++++ 7 files changed, 67 insertions(+), 64 deletions(-) rename packages/cli/test/utils/simulation/{web3JsPlugins.ts => web3js/blobsEIP4844Transaction.ts} (77%) create mode 100644 packages/cli/test/utils/simulation/web3js/plugins/index.ts create mode 100644 packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts create mode 100644 packages/cli/test/utils/simulation/web3js/plugins/web3ExtendedEthPlugin.ts diff --git a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts index 9c32e00f52ff..473ac95f06a2 100644 --- a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts @@ -4,7 +4,7 @@ import {fromHex, toHex} from "@lodestar/utils"; import {SimulationAssertion, AssertionMatch, AssertionResult, NodePair} from "../interfaces.js"; import {EL_GENESIS_ACCOUNT, EL_GENESIS_SECRET_KEY, SIM_ENV_CHAIN_ID} from "../constants.js"; import {generateBlobsForTransaction} from "../utils/blobs.js"; -import {BlobsEIP4844Transaction} from "../web3JsPlugins.js"; +import {BlobsEIP4844Transaction} from "../web3js/blobsEIP4844Transaction.js"; const numberOfBlobs = 6; diff --git a/packages/cli/test/utils/simulation/clients/execution/geth.ts b/packages/cli/test/utils/simulation/clients/execution/geth.ts index f3f8d8f73a5f..7cd24a90da04 100644 --- a/packages/cli/test/utils/simulation/clients/execution/geth.ts +++ b/packages/cli/test/utils/simulation/clients/execution/geth.ts @@ -9,7 +9,7 @@ import { SHARED_JWT_SECRET, SIM_ENV_NETWORK_ID, } from "../../constants.js"; -import {registerWeb3JsPlugins} from "../../web3JsPlugins.js"; +import {registerWeb3JsPlugins} from "../../web3js/plugins/index.js"; import {ExecutionClient, ExecutionNodeGenerator, ExecutionStartMode, JobOptions, RunnerType} from "../../interfaces.js"; import {getNodeMountedPaths} from "../../utils/paths.js"; import {getNodePorts} from "../../utils/ports.js"; diff --git a/packages/cli/test/utils/simulation/clients/execution/nethermind.ts b/packages/cli/test/utils/simulation/clients/execution/nethermind.ts index 0abc7ad8cc75..53f190f3ec76 100644 --- a/packages/cli/test/utils/simulation/clients/execution/nethermind.ts +++ b/packages/cli/test/utils/simulation/clients/execution/nethermind.ts @@ -2,7 +2,7 @@ import {writeFile} from "node:fs/promises"; import path from "node:path"; import got from "got"; import {Web3} from "web3"; -import {registerWeb3JsPlugins} from "../../web3JsPlugins.js"; +import {registerWeb3JsPlugins} from "../../web3js/plugins/index.js"; import {ExecutionClient, ExecutionNodeGenerator, JobOptions, RunnerType} from "../../interfaces.js"; import {getNethermindChainSpec} from "../../utils/executionGenesis.js"; import {getNodeMountedPaths} from "../../utils/paths.js"; diff --git a/packages/cli/test/utils/simulation/web3JsPlugins.ts b/packages/cli/test/utils/simulation/web3js/blobsEIP4844Transaction.ts similarity index 77% rename from packages/cli/test/utils/simulation/web3JsPlugins.ts rename to packages/cli/test/utils/simulation/web3js/blobsEIP4844Transaction.ts index 4ed6bdc7ab1e..a5dbf9408b7d 100644 --- a/packages/cli/test/utils/simulation/web3JsPlugins.ts +++ b/packages/cli/test/utils/simulation/web3js/blobsEIP4844Transaction.ts @@ -1,4 +1,3 @@ -import {Web3PluginBase, Web3} from "web3"; import {RLP} from "@ethereumjs/rlp"; import {keccak256} from "ethereum-cryptography/keccak.js"; import { @@ -158,63 +157,3 @@ export class BlobsEIP4844Transaction extends FeeMarketEIP1559Transaction { ); } } - -class Web3AdminPlugin extends Web3PluginBase { - /** - * The admin plugin as available via the provider object - * like in the example below. - * - * await node.web3.admin.addPeer(elIdentity.enode); - */ - pluginNamespace = "admin"; - - async nodeInfo(): Promise<{ - enode: string; - id: string; - ip: string; - listenAddr: string; - name: string; - ports: { - discovery: number; - listener: number; - }; - protocols: { - eth: { - difficulty: number; - genesis: string; - head: string; - network: number; - }; - }; - }> { - return this.requestManager.send({method: "admin_nodeInfo", params: []}); - } - - async addPeer(enode: string): Promise { - return this.requestManager.send({method: "admin_addPeer", params: [enode]}); - } -} - -class Web3ExtendedEthPlugin extends Web3PluginBase { - pluginNamespace = "extended"; - - async sendRawTransaction(tx: string): Promise { - return this.requestManager.send({method: "eth_sendRawTransaction", params: [tx]}); - } - - async sendPlainTransaction(...params: unknown[]): Promise { - return this.requestManager.send({method: "eth_sendTransaction", params: [...params]}); - } -} - -declare module "web3" { - interface Web3Context { - admin: Web3AdminPlugin; - extended: Web3ExtendedEthPlugin; - } -} - -export function registerWeb3JsPlugins(web3: Web3): void { - web3.registerPlugin(new Web3AdminPlugin()); - web3.registerPlugin(new Web3ExtendedEthPlugin()); -} diff --git a/packages/cli/test/utils/simulation/web3js/plugins/index.ts b/packages/cli/test/utils/simulation/web3js/plugins/index.ts new file mode 100644 index 000000000000..ebeca2de6185 --- /dev/null +++ b/packages/cli/test/utils/simulation/web3js/plugins/index.ts @@ -0,0 +1,8 @@ +import {Web3} from "web3"; +import {Web3AdminPlugin} from "./web3AdminPlugin.js"; +import {Web3ExtendedEthPlugin} from "./web3ExtendedEthPlugin.js"; + +export function registerWeb3JsPlugins(web3: Web3): void { + web3.registerPlugin(new Web3AdminPlugin()); + web3.registerPlugin(new Web3ExtendedEthPlugin()); +} diff --git a/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts b/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts new file mode 100644 index 000000000000..e6ae3cb15d34 --- /dev/null +++ b/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts @@ -0,0 +1,37 @@ +import {Web3PluginBase} from "web3"; + +export class Web3AdminPlugin extends Web3PluginBase { + /** + * The admin plugin as available via the provider object + * like in the example below. + * + * await node.web3.admin.addPeer(elIdentity.enode); + */ + pluginNamespace = "admin"; + + async nodeInfo(): Promise<{ + enode: string; + id: string; + ip: string; + listenAddr: string; + name: string; + ports: { + discovery: number; + listener: number; + }; + protocols: { + eth: { + difficulty: number; + genesis: string; + head: string; + network: number; + }; + }; + }> { + return this.requestManager.send({method: "admin_nodeInfo", params: []}); + } + + async addPeer(enode: string): Promise { + return this.requestManager.send({method: "admin_addPeer", params: [enode]}); + } +} diff --git a/packages/cli/test/utils/simulation/web3js/plugins/web3ExtendedEthPlugin.ts b/packages/cli/test/utils/simulation/web3js/plugins/web3ExtendedEthPlugin.ts new file mode 100644 index 000000000000..d07f0de793d0 --- /dev/null +++ b/packages/cli/test/utils/simulation/web3js/plugins/web3ExtendedEthPlugin.ts @@ -0,0 +1,19 @@ +import {Web3PluginBase} from "web3"; + +export class Web3ExtendedEthPlugin extends Web3PluginBase { + pluginNamespace = "extended"; + + async sendRawTransaction(tx: string): Promise { + return this.requestManager.send({method: "eth_sendRawTransaction", params: [tx]}); + } + + async sendPlainTransaction(...params: unknown[]): Promise { + return this.requestManager.send({method: "eth_sendTransaction", params: [...params]}); + } +} + +declare module "web3" { + interface Web3Context { + extended: Web3ExtendedEthPlugin; + } +} From 602f58384c9d1133ff3f4464871434567af7510d Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 9 Apr 2024 16:21:37 +0200 Subject: [PATCH 2/5] Verify the blobs --- .../simulation/assertions/blobsAssertion.ts | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts index 473ac95f06a2..915c2ed7210b 100644 --- a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts @@ -7,11 +7,12 @@ import {generateBlobsForTransaction} from "../utils/blobs.js"; import {BlobsEIP4844Transaction} from "../web3js/blobsEIP4844Transaction.js"; const numberOfBlobs = 6; +const sentBlobs: string[] = []; export function createBlobsAssertion( nodes: NodePair[], {sendBlobsAtSlot, validateBlobsAt}: {sendBlobsAtSlot: number; validateBlobsAt: number} -): SimulationAssertion { +): SimulationAssertion { return { id: `blobs-${nodes.map((n) => n.id).join("-")}`, match: ({slot}) => { @@ -45,32 +46,45 @@ export function createBlobsAssertion( }); const signedTx = tx.sign(fromHex(`0x${EL_GENESIS_SECRET_KEY}`)); await node.execution.provider?.extended.sendRawTransaction(toHex(signedTx.serialize())); + sentBlobs.push(...blobs.map((b) => toHex(b))); } const blobSideCars = await node.beacon.api.beacon.getBlobSidecars(slot); ApiError.assert(blobSideCars); - return blobSideCars.response.data.length; + return blobSideCars.response.data.map((b) => toHex(b.blob)); }, assert: async ({store}) => { const errors: AssertionResult[] = []; - let eip4844Blobs = 0; + const blobs: string[] = []; + for (let slot = sendBlobsAtSlot; slot <= validateBlobsAt; slot++) { - eip4844Blobs += store[slot] ?? 0; + blobs.push(...(store[slot] ?? [])); } - if (eip4844Blobs !== numberOfBlobs) { + if (blobs.length !== numberOfBlobs) { errors.push([ "Node does not have right number of blobs", { expectedBlobs: numberOfBlobs, - currentBlobs: eip4844Blobs, + currentBlobs: blobs.length, }, ]); } + for (let i = 0; i < blobs.length; i++) { + if (blobs[i] !== sentBlobs[i]) { + errors.push([ + "Node does not have the right blobs", + { + expectedBlob: sentBlobs[i], + currentBlob: blobs[i], + }, + ]); + } + } return errors; }, }; From c3235465ee85d41a7cc12ec3815d208d7e6bf8d5 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 9 Apr 2024 16:32:04 +0200 Subject: [PATCH 3/5] Fix plugin cache --- .../test/utils/simulation/web3js/plugins/web3AdminPlugin.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts b/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts index e6ae3cb15d34..0b04ded549b6 100644 --- a/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts +++ b/packages/cli/test/utils/simulation/web3js/plugins/web3AdminPlugin.ts @@ -35,3 +35,9 @@ export class Web3AdminPlugin extends Web3PluginBase { return this.requestManager.send({method: "admin_addPeer", params: [enode]}); } } + +declare module "web3" { + interface Web3Context { + admin: Web3AdminPlugin; + } +} From d7b3d9b83cb44328becc641fd31da2044027d4e5 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 9 Apr 2024 17:15:50 +0200 Subject: [PATCH 4/5] Compare blobs as buffer not as strings --- .../test/utils/simulation/assertions/blobsAssertion.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts index 915c2ed7210b..dc6bb6656c7c 100644 --- a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts @@ -12,7 +12,7 @@ const sentBlobs: string[] = []; export function createBlobsAssertion( nodes: NodePair[], {sendBlobsAtSlot, validateBlobsAt}: {sendBlobsAtSlot: number; validateBlobsAt: number} -): SimulationAssertion { +): SimulationAssertion { return { id: `blobs-${nodes.map((n) => n.id).join("-")}`, match: ({slot}) => { @@ -52,13 +52,13 @@ export function createBlobsAssertion( const blobSideCars = await node.beacon.api.beacon.getBlobSidecars(slot); ApiError.assert(blobSideCars); - return blobSideCars.response.data.map((b) => toHex(b.blob)); + return blobSideCars.response.data.map((b) => b.blob); }, assert: async ({store}) => { const errors: AssertionResult[] = []; - const blobs: string[] = []; + const blobs: Uint8Array[] = []; for (let slot = sendBlobsAtSlot; slot <= validateBlobsAt; slot++) { blobs.push(...(store[slot] ?? [])); @@ -75,7 +75,7 @@ export function createBlobsAssertion( } for (let i = 0; i < blobs.length; i++) { - if (blobs[i] !== sentBlobs[i]) { + if (!Buffer.from(blobs[i]).equals(Buffer.from(sentBlobs[i]))) { errors.push([ "Node does not have the right blobs", { From f75c533837f3cc649e2b89f85c1694f8c5d718fb Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Fri, 12 Apr 2024 11:33:43 +0200 Subject: [PATCH 5/5] Fix the blobs assertion --- .../simulation/assertions/blobsAssertion.ts | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts index dc6bb6656c7c..e714f0db8d7b 100644 --- a/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts +++ b/packages/cli/test/utils/simulation/assertions/blobsAssertion.ts @@ -7,7 +7,7 @@ import {generateBlobsForTransaction} from "../utils/blobs.js"; import {BlobsEIP4844Transaction} from "../web3js/blobsEIP4844Transaction.js"; const numberOfBlobs = 6; -const sentBlobs: string[] = []; +const sentBlobs: Uint8Array[] = []; export function createBlobsAssertion( nodes: NodePair[], @@ -46,7 +46,8 @@ export function createBlobsAssertion( }); const signedTx = tx.sign(fromHex(`0x${EL_GENESIS_SECRET_KEY}`)); await node.execution.provider?.extended.sendRawTransaction(toHex(signedTx.serialize())); - sentBlobs.push(...blobs.map((b) => toHex(b))); + + sentBlobs.push(...blobs.map((b) => fromHex(b))); } const blobSideCars = await node.beacon.api.beacon.getBlobSidecars(slot); @@ -76,16 +77,29 @@ export function createBlobsAssertion( for (let i = 0; i < blobs.length; i++) { if (!Buffer.from(blobs[i]).equals(Buffer.from(sentBlobs[i]))) { - errors.push([ - "Node does not have the right blobs", - { - expectedBlob: sentBlobs[i], - currentBlob: blobs[i], - }, - ]); + errors.push(["Node does not have the correct blobs", {index: i}]); } } return errors; }, + + async dump({store, nodes}) { + const result: Record = { + "expectedBlobs.txt": sentBlobs.map(toHex).join("\n"), + }; + + for (const node of nodes) { + const blobs: Uint8Array[] = []; + for (let slot = sendBlobsAtSlot; slot <= validateBlobsAt; slot++) { + if (store[node.beacon.id] !== undefined) { + blobs.push(...(store[node.beacon.id][slot] ?? [])); + } + } + + result[`blobs-${node.beacon.id}.txt`] = blobs.map(toHex).join("\n"); + } + + return result; + }, }; }