From 4134889619cdcc449ffe90b30129d0e78c9fbe98 Mon Sep 17 00:00:00 2001 From: andri lim Date: Sun, 9 Mar 2025 16:45:16 +0700 Subject: [PATCH] Lazily load KZG trusted setup (#3130) --- execution_chain/core/eip4844.nim | 18 +- execution_chain/core/lazy_kzg.nim | 331 ++++++++++++++++++ execution_chain/nimbus_execution_client.nim | 10 +- .../nodocker/consensus/consensus_sim.nim | 6 - .../nodocker/engine/cancun/blobs.nim | 4 +- .../nodocker/engine/cancun/helpers.nim | 2 +- .../nodocker/engine/engine_sim.nim | 8 +- .../nodocker/pyspec/pyspec_sim.nim | 6 - nrpc/nrpc.nim | 4 +- tests/networking/test_ecies.nim | 2 +- tests/test_blockchain_json.nim | 11 +- tests/test_generalstate_json.nim | 12 - tests/test_txpool.nim | 1 - tools/evmstate/evmstate.nim | 5 - tools/t8n/transition.nim | 4 - 15 files changed, 343 insertions(+), 81 deletions(-) create mode 100644 execution_chain/core/lazy_kzg.nim diff --git a/execution_chain/core/eip4844.nim b/execution_chain/core/eip4844.nim index 6932d52aa..298140fe6 100644 --- a/execution_chain/core/eip4844.nim +++ b/execution_chain/core/eip4844.nim @@ -9,13 +9,12 @@ # according to those terms. import - std/[os, strutils], stew/arrayops, nimcrypto/sha2, - kzg4844/kzg, results, stint, ./eip7691, + ./lazy_kzg as kzg, ../constants, ../common/common @@ -217,18 +216,3 @@ proc validateBlobTransactionWrapper*(tx: PooledTransaction): return err("tx versioned hash not match commitments at index " & $i) ok() - -proc loadKzgTrustedSetup*(): Result[void, string] = - const - vendorDir = currentSourcePath.parentDir.replace('\\', '/') & "/../../vendor" - trustedSetupDir = vendorDir & "/nim-kzg4844/kzg4844/csources/src" - trustedSetup = staticRead trustedSetupDir & "/trusted_setup.txt" - - # If the baked-in trusted setup was loaded successfully, it's harmless to - # try again (which happpens in tests) - var loaded {.global.}: bool - if not loaded: - ?loadTrustedSetupFromString(trustedSetup, 0) - loaded = true - ok() - diff --git a/execution_chain/core/lazy_kzg.nim b/execution_chain/core/lazy_kzg.nim new file mode 100644 index 000000000..248f2059a --- /dev/null +++ b/execution_chain/core/lazy_kzg.nim @@ -0,0 +1,331 @@ +# nimbus-execution-client +# Copyright (c) 2025 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms + +############################################################ +# Main API, wrapper on top of C FFI +# But lazily load the trusted setup +############################################################ + +{.push gcsafe, raises: [].} + +import + std/[streams, strutils], + stew/[assign2, byteutils], + results, + kzg4844/kzg_abi + +export + results, + kzg_abi.KzgBlob, + kzg_abi.KzgBytes32, + kzg_abi.KzgBytes48, + kzg_abi.KzgCell, + kzg_abi.KzgCommitment, + kzg_abi.KzgProof + +const + TrustedSetupNotLoadedErr = "Trusted setup is not loaded." + TrustedSetupAlreadyLoadedErr = "Trusted setup is already loaded." + +type + KzgCtx = object + initialized: bool + settings: ptr KzgSettings + + KzgProofAndY* = object + proof*: KzgProof + y*: KzgBytes32 + + KzgCells* = array[CELLS_PER_EXT_BLOB, KzgCell] + KzgCellsAndKzgProofs* = object + cells*: array[CELLS_PER_EXT_BLOB, KzgCell] + proofs*: array[CELLS_PER_EXT_BLOB, KzgProof] + +############################################################## +# Global variables +############################################################## + +var gCtx: KzgCtx +gCtx.initialized = false +gCtx.settings = nil + +############################################################## +# Private helpers +############################################################## + +template getPtr(x: untyped): auto = + addr(x) + +template verify(res: KZG_RET, ret: untyped): untyped = + if res != KZG_OK: + return err($res) + ok(ret) + +############################################################## +# Public functions +############################################################## + +proc loadTrustedSetup*(input: File, precompute: Natural): Result[void, string] = + if gCtx.initialized: + return err(TrustedSetupAlreadyLoadedErr) + gCtx.settings = cast[ptr KzgSettings](alloc0(sizeof(KzgSettings))) + let res = load_trusted_setup_file(gCtx.settings, input, precompute.uint64) + if res != KZG_OK: + dealloc(gCtx.settings) + gCtx.settings = nil + return err($res) + gCtx.initialized = true + return ok() + +proc loadTrustedSetup*(fileName: string, precompute: Natural): Result[void, string] = + try: + let file = open(fileName) + result = file.loadTrustedSetup(precompute) + file.close() + except IOError as ex: + return err(ex.msg) + +proc loadTrustedSetup*(g1MonomialBytes: openArray[byte], + g1LagrangeBytes: openArray[byte], + g2MonomialBytes: openArray[byte], + precompute: Natural): + Result[void, string] = + if gCtx.initialized: + return err(TrustedSetupAlreadyLoadedErr) + if g1MonomialBytes.len == 0 or g1LagrangeBytes.len == 0 or g2MonomialBytes.len == 0: + return err($KZG_BADARGS) + + gCtx.settings = cast[ptr KzgSettings](alloc0(sizeof(KzgSettings))) + let res = load_trusted_setup(gCtx.settings, + g1MonomialBytes[0].getPtr, + g1MonomialBytes.len.uint64, + g1LagrangeBytes[0].getPtr, + g1LagrangeBytes.len.uint64, + g2MonomialBytes[0].getPtr, + g2MonomialBytes.len.uint64, + precompute.uint64) + if res != KZG_OK: + dealloc(gCtx.settings) + gCtx.settings = nil + return err($res) + gCtx.initialized = true + return ok() + +const + NumG1 = FIELD_ELEMENTS_PER_BLOB + NumG2 = 65 + G1Len = 48 + G2Len = 96 + +type + TrustedSetup = object + g1MonomialBytes: array[NumG1 * G1Len, byte] + g1LagrangeBytes: array[NumG1 * G1Len, byte] + g2MonomialBytes: array[NumG2 * G2Len, byte] + +proc parseTrustedSetup(input: string): Result[TrustedSetup, string] = + var + s = newStringStream(input) + ts: TrustedSetup + + try: + let numG1 = s.readLine().parseInt() + if numG1 != NumG1: + return err("invalid number of G1 points, expect $1, got $2" % [ + $NumG1, $numG1 + ]) + let numG2 = s.readLine().parseInt() + if numG2 != NumG2: + return err("invalid number of G2 points, expect $1, got $2" % [ + $NumG2, $numG2 + ]) + + for i in 0 ..< NumG1: + let p = hexToByteArray[G1Len](s.readLine()) + assign(ts.g1LagrangeBytes.toOpenArray(i * G1Len, ((i + 1) * G1Len) - 1), p) + + for i in 0 ..< NumG2: + let p = hexToByteArray[G2Len](s.readLine()) + assign(ts.g2MonomialBytes.toOpenArray(i * G2Len, ((i + 1) * G2Len) - 1), p) + + for i in 0 ..< NumG1: + let p = hexToByteArray[G1Len](s.readLine()) + assign(ts.g1MonomialBytes.toOpenArray(i * G1Len, ((i + 1) * G1Len) - 1), p) + + except ValueError as ex: + return err(ex.msg) + except OSError as ex: + return err(ex.msg) + except IOError as ex: + return err(ex.msg) + + ok(ts) + +proc loadTrustedSetupFromString*(input: string, precompute: Natural): Result[void, string] = + let ts = ?parseTrustedSetup(input) + loadTrustedSetup(ts.g1MonomialBytes, ts.g1LagrangeBytes, ts.g2MonomialBytes, precompute) + +proc lazyLoadTrustedSetup(): Result[void, string] = + const ts = parseTrustedSetup(kzg_abi.trustedSetup).expect("parseTrustedSetup no error") + loadTrustedSetup(ts.g1MonomialBytes, ts.g1LagrangeBytes, ts.g2MonomialBytes, 0) + +proc freeTrustedSetup*(): Result[void, string] = + if not gCtx.initialized: + return err(TrustedSetupNotLoadedErr) + free_trusted_setup(gCtx.settings) + gCtx.initialized = false + dealloc(gCtx.settings) + gCtx.settings = nil + return ok() + +proc blobToKzgCommitment*(blob: KzgBlob): Result[KzgCommitment, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var ret: KzgCommitment + let res = blob_to_kzg_commitment(ret, blob.getPtr, gCtx.settings) + verify(res, ret) + +proc computeKzgProof*(blob: KzgBlob, + z: KzgBytes32): Result[KzgProofAndY, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var ret: KzgProofAndY + let res = compute_kzg_proof( + ret.proof, + ret.y, + blob.getPtr, + z.getPtr, + gCtx.settings) + verify(res, ret) + +proc computeBlobKzgProof*(blob: KzgBlob, + commitmentBytes: KzgBytes48): Result[KzgProof, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var proof: KzgProof + let res = compute_blob_kzg_proof( + proof, + blob.getPtr, + commitmentBytes.getPtr, + gCtx.settings) + verify(res, proof) + +proc verifyKzgProof*(commitment: KzgBytes48, + z: KzgBytes32, # Input Point + y: KzgBytes32, # Claimed Value + proof: KzgBytes48): Result[bool, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var valid: bool + let res = verify_kzg_proof( + valid, + commitment.getPtr, + z.getPtr, + y.getPtr, + proof.getPtr, + gCtx.settings) + verify(res, valid) + +proc verifyBlobKzgProof*(blob: KzgBlob, + commitment: KzgBytes48, + proof: KzgBytes48): Result[bool, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var valid: bool + let res = verify_blob_kzg_proof( + valid, + blob.getPtr, + commitment.getPtr, + proof.getPtr, + gCtx.settings) + verify(res, valid) + +proc verifyBlobKzgProofBatch*(blobs: openArray[KzgBlob], + commitments: openArray[KzgBytes48], + proofs: openArray[KzgBytes48]): Result[bool, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + if blobs.len != commitments.len: + return err($KZG_BADARGS) + if blobs.len != proofs.len: + return err($KZG_BADARGS) + if blobs.len == 0: + return ok(true) + + var valid: bool + let res = verify_blob_kzg_proof_batch( + valid, + blobs[0].getPtr, + commitments[0].getPtr, + proofs[0].getPtr, + blobs.len.uint64, + gCtx.settings) + verify(res, valid) + +proc computeCellsAndKzgProofs*(blob: KzgBlob): Result[KzgCellsAndKzgProofs, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + var ret: KzgCellsAndKzgProofs + var cellsPtr: ptr KzgCell = ret.cells[0].getPtr + var proofsPtr: ptr KzgProof = ret.proofs[0].getPtr + let res = compute_cells_and_kzg_proofs( + cellsPtr, + proofsPtr, + blob.getPtr, + gCtx.settings) + verify(res, ret) + +proc recoverCellsAndKzgProofs*(cellIndices: openArray[uint64], + cells: openArray[KzgCell]): Result[KzgCellsAndKzgProofs, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + if cells.len != cellIndices.len: + return err($KZG_BADARGS) + if cells.len == 0: + return err($KZG_BADARGS) + + var ret: KzgCellsAndKzgProofs + var recoveredCellsPtr: ptr KzgCell = ret.cells[0].getPtr + var recoveredProofsPtr: ptr KzgProof = ret.proofs[0].getPtr + let res = recover_cells_and_kzg_proofs( + recoveredCellsPtr, + recoveredProofsPtr, + cellIndices[0].getPtr, + cells[0].getPtr, + cells.len.uint64, + gCtx.settings) + verify(res, ret) + +proc verifyCellKzgProofBatch*(commitments: openArray[KzgBytes48], + cellIndices: openArray[uint64], + cells: openArray[KzgCell], + proofs: openArray[KzgBytes48]): Result[bool, string] = + if not gCtx.initialized: + ?lazyLoadTrustedSetup() + if commitments.len != cells.len: + return err($KZG_BADARGS) + if cellIndices.len != cells.len: + return err($KZG_BADARGS) + if proofs.len != cells.len: + return err($KZG_BADARGS) + if cells.len == 0: + return ok(true) + + var valid: bool + let res = verify_cell_kzg_proof_batch( + valid, + commitments[0].getPtr, + cellIndices[0].getPtr, + cells[0].getPtr, + proofs[0].getPtr, + cells.len.uint64, + gCtx.settings) + verify(res, valid) + +{. pop .} diff --git a/execution_chain/nimbus_execution_client.nim b/execution_chain/nimbus_execution_client.nim index cd416b3d2..f9494aaa2 100644 --- a/execution_chain/nimbus_execution_client.nim +++ b/execution_chain/nimbus_execution_client.nim @@ -16,7 +16,6 @@ import eth/net/nat, metrics, metrics/chronicles_support, - kzg4844/kzg, stew/byteutils, ./rpc, ./version, @@ -24,7 +23,7 @@ import ./nimbus_desc, ./nimbus_import, ./core/block_import, - ./core/eip4844, + ./core/lazy_kzg, ./db/core_db/persistent, ./db/storage_types, ./sync/wire_protocol, @@ -190,17 +189,14 @@ proc run(nimbus: NimbusNode, conf: NimbusConf) = evmcSetLibraryPath(conf.evm) # Trusted setup is needed for processing Cancun+ blocks + # If user not specify the trusted setup, baked in + # trusted setup will be loaded, lazily. if conf.trustedSetupFile.isSome: let fileName = conf.trustedSetupFile.get() let res = loadTrustedSetup(fileName, 0) if res.isErr: fatal "Cannot load Kzg trusted setup from file", msg=res.error quit(QuitFailure) - else: - let res = loadKzgTrustedSetup() - if res.isErr: - fatal "Cannot load baked in Kzg trusted setup", msg=res.error - quit(QuitFailure) createDir(string conf.dataDir) let coreDB = diff --git a/hive_integration/nodocker/consensus/consensus_sim.nim b/hive_integration/nodocker/consensus/consensus_sim.nim index 4ecbdfc7b..0fea02f11 100644 --- a/hive_integration/nodocker/consensus/consensus_sim.nim +++ b/hive_integration/nodocker/consensus/consensus_sim.nim @@ -14,7 +14,6 @@ import ../../../execution_chain/core/chain, ../../../execution_chain/core/block_import, ../../../execution_chain/common, - ../../../execution_chain/core/eip4844, ../sim_utils, ./extract_consensus_data @@ -66,11 +65,6 @@ proc main() = let taskPool = Taskpool.new() let start = getTime() - let res = loadKzgTrustedSetup() - if res.isErr: - echo "FATAL: ", res.error - quit(QuitFailure) - for fileName in walkDirRec(basePath): if not fileName.endsWith(".json"): continue diff --git a/hive_integration/nodocker/engine/cancun/blobs.nim b/hive_integration/nodocker/engine/cancun/blobs.nim index c78e620f4..5c1e6186f 100644 --- a/hive_integration/nodocker/engine/cancun/blobs.nim +++ b/hive_integration/nodocker/engine/cancun/blobs.nim @@ -10,12 +10,12 @@ import eth/common/[base, hashes], - kzg4844/kzg, kzg4844/kzg_abi, stew/endians2, nimcrypto/sha2, results, - ../../../../execution_chain/core/eip4844 + ../../../../execution_chain/core/eip4844, + ../../../../execution_chain/core/lazy_kzg as kzg export base, hashes type diff --git a/hive_integration/nodocker/engine/cancun/helpers.nim b/hive_integration/nodocker/engine/cancun/helpers.nim index 9c94b67b4..1b5cceca3 100644 --- a/hive_integration/nodocker/engine/cancun/helpers.nim +++ b/hive_integration/nodocker/engine/cancun/helpers.nim @@ -15,11 +15,11 @@ import eth/common/eth_types_rlp, chronicles, stew/byteutils, - kzg4844/kzg, ../types, ../engine_client, ../../../../execution_chain/constants, ../../../../execution_chain/core/eip4844, + ../../../../execution_chain/core/lazy_kzg as kzg, ../../../../execution_chain/rpc/rpc_types, web3/execution_types, ../../../../execution_chain/beacon/web3_eth_conv, diff --git a/hive_integration/nodocker/engine/engine_sim.nim b/hive_integration/nodocker/engine/engine_sim.nim index 425d527fc..51a381e3c 100644 --- a/hive_integration/nodocker/engine/engine_sim.nim +++ b/hive_integration/nodocker/engine/engine_sim.nim @@ -13,8 +13,7 @@ import chronicles, results, ./types, - ../sim_utils, - ../../../execution_chain/core/eip4844 + ../sim_utils import ./engine_tests, @@ -37,11 +36,6 @@ proc main() = var stat: SimStat let start = getTime() - let res = loadKzgTrustedSetup() - if res.isErr: - fatal "Cannot load baked in Kzg trusted setup", msg=res.error - quit(QuitFailure) - for x in testList: let status = if x.run(x.spec): TestStatus.OK diff --git a/hive_integration/nodocker/pyspec/pyspec_sim.nim b/hive_integration/nodocker/pyspec/pyspec_sim.nim index 90a287b94..eaaaf69f0 100644 --- a/hive_integration/nodocker/pyspec/pyspec_sim.nim +++ b/hive_integration/nodocker/pyspec/pyspec_sim.nim @@ -19,7 +19,6 @@ import ../../../tools/evmstate/helpers as ehp, ../../../execution_chain/beacon/web3_eth_conv, ../../../execution_chain/beacon/payload_conv, - ../../../execution_chain/core/eip4844, ../engine/engine_client, ../engine/types, ./test_env @@ -202,11 +201,6 @@ proc main() = let taskPool = Taskpool.new() let start = getTime() - let res = loadKzgTrustedSetup() - if res.isErr: - echo "FATAL: ", res.error - quit(QuitFailure) - let testVectors = collectTestVectors() for fileName in testVectors: if not fileName.endsWith(".json"): diff --git a/nrpc/nrpc.nim b/nrpc/nrpc.nim index db8a76f79..076552633 100644 --- a/nrpc/nrpc.nim +++ b/nrpc/nrpc.nim @@ -12,10 +12,10 @@ import chronicles, ../execution_chain/constants, ../execution_chain/core/chain, + ../execution_chain/core/lazy_kzg, ./config, ../execution_chain/utils/era_helpers, - kzg4844/kzg, - web3, + web3, web3/[engine_api, primitives, conversions], beacon_chain/spec/digest, beacon_chain/el/el_conf, diff --git a/tests/networking/test_ecies.nim b/tests/networking/test_ecies.nim index 11b68058a..b6f0ec444 100644 --- a/tests/networking/test_ecies.nim +++ b/tests/networking/test_ecies.nim @@ -11,7 +11,7 @@ import unittest2, - nimcrypto/[utils, sha2, hmac, rijndael], + nimcrypto/[utils, sha2, hmac], eth/common/keys, ../../execution_chain/networking/rlpx/ecies diff --git a/tests/test_blockchain_json.nim b/tests/test_blockchain_json.nim index 816640ae1..d9ef14769 100644 --- a/tests/test_blockchain_json.nim +++ b/tests/test_blockchain_json.nim @@ -18,8 +18,7 @@ import ../execution_chain/core/chain/forked_chain, ../tools/common/helpers as chp, ../tools/evmstate/helpers, - ../execution_chain/common/common, - ../execution_chain/core/eip4844 + ../execution_chain/common/common const debugMode = false @@ -124,10 +123,6 @@ proc blockchainJsonMain*() = legacyFolder = "eth_tests/LegacyTests/Constantinople/BlockchainTests" newFolder = "eth_tests/BlockchainTests" - loadKzgTrustedSetup().isOkOr: - echo "FATAL: ", error - quit(QuitFailure) - if false: suite "block chain json tests": jsonTest(legacyFolder, "BlockchainTests", executeFile, skipBCTests) @@ -137,10 +132,6 @@ proc blockchainJsonMain*() = when debugMode: proc executeFile(name: string) = - loadKzgTrustedSetup().isOkOr: - echo "FATAL: ", error - quit(QuitFailure) - var testStatusIMPL: TestStatus let node = json.parseFile(name) executeFile(node, testStatusIMPL) diff --git a/tests/test_generalstate_json.nim b/tests/test_generalstate_json.nim index c42ac8c25..f07603edf 100644 --- a/tests/test_generalstate_json.nim +++ b/tests/test_generalstate_json.nim @@ -15,7 +15,6 @@ import ../execution_chain/common/common, ../execution_chain/utils/[utils, debug], ../execution_chain/evm/tracer/legacy_tracer, - ../execution_chain/core/eip4844, ../tools/common/helpers as chp, ../tools/evmstate/helpers, ../tools/common/state_clearing, @@ -40,9 +39,6 @@ type subFixture: int fork: string -var - trustedSetupLoaded = false - proc toBytes(x: string): seq[byte] = result = newSeq[byte](x.len) for i in 0.. 0: if not prepareAndRun(conf.inputFile, conf): quit(QuitFailure) diff --git a/tools/t8n/transition.nim b/tools/t8n/transition.nim index 794592326..16dc66176 100644 --- a/tools/t8n/transition.nim +++ b/tools/t8n/transition.nim @@ -498,10 +498,6 @@ proc transitionAction*(ctx: var TransContext, conf: T8NConf) = if com.isCancunOrLater(ctx.env.currentTimestamp): if ctx.env.parentBeaconBlockRoot.isNone: raise newError(ErrorConfig, "Cancun config but missing 'parentBeaconBlockRoot' in env section") - - let res = loadKzgTrustedSetup() - if res.isErr: - raise newError(ErrorConfig, res.error) else: # un-set it if it has been set too early ctx.env.parentBeaconBlockRoot = Opt.none(Hash32)