From 3cffdb2d5e144f2e3d8617fa22aacf6cce5998a2 Mon Sep 17 00:00:00 2001 From: Jacob Lindahl Date: Fri, 13 Dec 2024 03:14:21 +0900 Subject: [PATCH] feat: add support for custom genesis state (#3259) This PR introduces the `custom_genesis_export` tool, designed to export zkSync PostgreSQL database state in a format suitable for initializing a custom genesis state for a new chain. - Exports `initial_writes`, `storage_logs`, and `factory_deps` (excluding system context entries) into a binary file. - Updates the `genesis.yaml` file with: - `genesis_root_hash`, `rollup_last_leaf_index`, and `genesis_commitment` matching the exported data. - Adds `custom_genesis_state_path` pointing to the export file, which is recognised by the server during genesis. - Adds support for initialising from a custom genesis state to the server. --------- Co-authored-by: Ivan Schasny Co-authored-by: Ivan Schasny <31857042+ischasny@users.noreply.github.com> --- Cargo.lock | 19 ++ Cargo.toml | 1 + core/bin/custom_genesis_export/.gitignore | 1 + core/bin/custom_genesis_export/Cargo.toml | 30 +++ core/bin/custom_genesis_export/README.md | 60 ++++++ core/bin/custom_genesis_export/src/main.rs | 141 ++++++++++++ core/lib/config/src/configs/genesis.rs | 2 + core/lib/config/src/testonly.rs | 1 + ...aa3061382dfe4142c7fc46df1dfebec34ec92.json | 32 +++ ...11c5e74988422abd052a33ff1ec5db41f7d52.json | 26 +++ core/lib/dal/src/custom_genesis_export_dal.rs | 124 +++++++++++ core/lib/dal/src/lib.rs | 15 +- core/lib/env_config/src/genesis.rs | 15 ++ core/lib/protobuf_config/src/genesis.rs | 2 + .../src/proto/config/genesis.proto | 1 + .../node/api_server/src/web3/namespaces/en.rs | 2 + core/node/genesis/Cargo.toml | 1 + core/node/genesis/src/lib.rs | 203 +++++++++++++----- core/node/genesis/src/utils.rs | 37 ++-- .../src/external_node/genesis.rs | 4 + .../src/main_node/genesis.rs | 19 +- core/node/node_sync/src/genesis.rs | 5 +- 22 files changed, 664 insertions(+), 77 deletions(-) create mode 100644 core/bin/custom_genesis_export/.gitignore create mode 100644 core/bin/custom_genesis_export/Cargo.toml create mode 100644 core/bin/custom_genesis_export/README.md create mode 100644 core/bin/custom_genesis_export/src/main.rs create mode 100644 core/lib/dal/.sqlx/query-42f15afb71632bdfab7befb651eaa3061382dfe4142c7fc46df1dfebec34ec92.json create mode 100644 core/lib/dal/.sqlx/query-bd37db29c86a84ed09ed0633e8511c5e74988422abd052a33ff1ec5db41f7d52.json create mode 100644 core/lib/dal/src/custom_genesis_export_dal.rs diff --git a/Cargo.lock b/Cargo.lock index 07218f2310cb..442802faea3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2308,6 +2308,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "custom_genesis_export" +version = "0.1.0" +dependencies = [ + "anyhow", + "bincode", + "clap 4.5.23", + "futures 0.3.31", + "sqlx", + "tokio", + "zksync_contracts", + "zksync_core_leftovers", + "zksync_dal", + "zksync_node_genesis", + "zksync_protobuf_config", + "zksync_types", +] + [[package]] name = "darling" version = "0.13.4" @@ -12597,6 +12615,7 @@ name = "zksync_node_genesis" version = "0.1.0" dependencies = [ "anyhow", + "bincode", "itertools 0.10.5", "thiserror 1.0.69", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 80a1e4104265..074d27d6574d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ # Binaries "core/bin/block_reverter", "core/bin/contract-verifier", + "core/bin/custom_genesis_export", "core/bin/external_node", "core/bin/merkle_tree_consistency_checker", "core/bin/snapshots_creator", diff --git a/core/bin/custom_genesis_export/.gitignore b/core/bin/custom_genesis_export/.gitignore new file mode 100644 index 000000000000..a8a0dcec4472 --- /dev/null +++ b/core/bin/custom_genesis_export/.gitignore @@ -0,0 +1 @@ +*.bin diff --git a/core/bin/custom_genesis_export/Cargo.toml b/core/bin/custom_genesis_export/Cargo.toml new file mode 100644 index 000000000000..566f0a979297 --- /dev/null +++ b/core/bin/custom_genesis_export/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "custom_genesis_export" +version.workspace = true +edition.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +clap = { workspace = true, features = ["derive"] } +futures.workspace = true +sqlx = { workspace = true, features = [ + "runtime-tokio", + "tls-native-tls", + "macros", + "postgres", +] } +tokio = { workspace = true, features = ["full"] } + +zksync_types.workspace = true +zksync_node_genesis.workspace = true +zksync_contracts.workspace = true +zksync_core_leftovers.workspace = true +zksync_protobuf_config.workspace = true +zksync_dal.workspace = true +anyhow.workspace = true +bincode.workspace = true diff --git a/core/bin/custom_genesis_export/README.md b/core/bin/custom_genesis_export/README.md new file mode 100644 index 000000000000..e95bcc814f2a --- /dev/null +++ b/core/bin/custom_genesis_export/README.md @@ -0,0 +1,60 @@ +# Custom Genesis Export + +The `custom_genesis_export` tool allows exporting a state from a zkSync PostgreSQL database in a format that can be +included as a custom genesis state for a new chain. + +This is particularly useful in data migration scenarios where a large existing state needs to be applied to a newly +created chain. + +A typical workflow could be: + +- Run a chain locally, not connected to the real L1, and add all required data to it. +- Export the data using the `custom_genesis_export` tool. +- Create a new chain connected to the real ecosystem using the exported data. + +## How it works + +The tool exports all entries from `storage_logs`, and `factory_deps`, except those related to the system context. The +data is then written to a binary file using the Rust standard library following a simple serialisation format. + +`custom_genesis_export` can be built using the following command: + +```shell +cargo build --release -p custom_genesis_export +``` + +And then executed using the following command, where: + +- `database-url` is the URL of the PostgreSQL database. +- `genesis-config-path` is the path to the `genesis.yaml` configuration file, used to set up a new chain (located in the + `file_based` directory). +- `output-path` is the path to the generated binary output file. + +```shell +custom_genesis_export --database-url=postgres://postgres:notsecurepassword@localhost:5432/zksync_server_localhost_validium --genesis-config-path=/Users/ischasny/Dev/zksync-era/etc/env/file_based/genesis.yaml --output-path=export.bin +``` + +> Please make sure that the database is not written into before running data export. + +After the export is completed, the tool will make the following updates to the `genesis.yaml` file in-place: + +- Update `genesis_root_hash`, `rollup_last_leaf_index`, and `genesis_commitment` to match the contents of the export + file. +- Add a `custom_genesis_state_path` property pointing to the data export. + +The modified genesis file can be used to bootstrap an ecosystem or initialize new chains. The data export will be +automatically recognized by the server during the execution of `zkstack ecosystem init ...` and +`zkstack chain create ...` commands. + +### Running considerations + +- All chains within the same ecosystem must be bootstrapped from the same genesis state. This is enforced at the + protocol level. If two chains require different states, this can only be achieved by bringing the chain into the + ecosystem through governance voting. + - If a chain is added to the ecosystem via a vote, ensure no assets are minted on the old bridge, as this would create + discrepancies with the new one. One should set gas prices to zero when generating a state to account for that. +- To calculate genesis parameters, the tool must load all VM logs into RAM. This is due to implementation specifics. For + larger states, ensure the VM has sufficient RAM capacity. +- After the import, block numbers for all VM logs will be reset to zero - if the imported data has been indexed based on + block number, such indexes will break. +- External Nodes will have to be bootstrapped from data snapshot (i.e. genesis can't be generated locally). diff --git a/core/bin/custom_genesis_export/src/main.rs b/core/bin/custom_genesis_export/src/main.rs new file mode 100644 index 000000000000..9db715aad99f --- /dev/null +++ b/core/bin/custom_genesis_export/src/main.rs @@ -0,0 +1,141 @@ +extern crate core; + +use std::{fs, fs::File, io::BufWriter, path::PathBuf, str::FromStr}; + +use clap::Parser; +use zksync_contracts::BaseSystemContractsHashes; +use zksync_core_leftovers::temp_config_store::read_yaml_repr; +use zksync_dal::{custom_genesis_export_dal::GenesisState, ConnectionPool, Core, CoreDal}; +use zksync_node_genesis::{make_genesis_batch_params, utils::get_deduped_log_queries}; +use zksync_protobuf_config::encode_yaml_repr; +use zksync_types::{url::SensitiveUrl, StorageLog}; + +#[derive(Debug, Parser)] +#[command(name = "Custom genesis export tool", author = "Matter Labs")] +struct Args { + /// PostgreSQL connection string for the database to export. + #[arg(short, long)] + database_url: Option, + + /// Output file path. + #[arg(short, long, default_value = "genesis_export.bin")] + output_path: PathBuf, + + /// Path to the genesis.yaml + #[arg(short, long)] + genesis_config_path: PathBuf, +} + +/// The `custom_genesis_export` tool allows exporting storage logs and factory dependencies +/// from the ZKSync PostgreSQL database in a way that they can be used as a custom genesis state for a new chain. +/// +/// Inputs: +/// * `database_url` - URL to the PostgreSQL database. +/// * `output` - Path to the output file. +/// * `genesis_config_path` - Path to the `genesis.yaml` configuration file, which will be used to set up a new chain (located in the `file_based` directory). +/// +/// Given the inputs above, `custom_genesis_export` will perform the following: +/// * Read storage logs, and factory dependencies; filter out those related to the system context, +/// and save the remaining data to the output file. +/// * Calculate the new `genesis_root_hash`, `rollup_last_leaf_index`, and `genesis_commitment`, then update these +/// in-place in the provided `genesis.yaml`. Additionally, the tool will add a `custom_genesis_state_path` property +/// pointing to the genesis export. +/// +/// Note: To calculate the new genesis parameters, the current implementation requires loading all storage logs +/// into RAM. This is necessary due to the specific sorting and filtering that need to be applied. +/// For larger states, keep this in mind and ensure you have a machine with sufficient RAM. +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let args = Args::parse(); + + let mut out = BufWriter::new(File::create(&args.output_path)?); + + println!( + "Export file: {}", + args.output_path.canonicalize()?.display(), + ); + + println!("Connecting to source database..."); + + let db_url = args.database_url.or_else(|| std::env::var("DATABASE_URL").ok()).expect("Specify the database connection string in either a CLI argument or in the DATABASE_URL environment variable."); + // we need only 1 DB connection at most for data export + let connection_pool_builder = + ConnectionPool::::builder(SensitiveUrl::from_str(db_url.as_str())?, 1); + let connection_pool = connection_pool_builder.build().await?; + + let mut storage = connection_pool.connection().await?; + let mut transaction = storage.start_transaction().await?; + + println!("Connected to source database."); + + let storage_logs = transaction + .custom_genesis_export_dal() + .get_storage_logs() + .await?; + let factory_deps = transaction + .custom_genesis_export_dal() + .get_factory_deps() + .await?; + + transaction.commit().await?; + + println!( + "Loaded {} storage logs {} factory deps from source database.", + storage_logs.len(), + factory_deps.len() + ); + + let storage_logs_for_genesis: Vec = + storage_logs.iter().map(StorageLog::from).collect(); + + bincode::serialize_into( + &mut out, + &GenesisState { + storage_logs, + factory_deps, + }, + )?; + + println!( + "Saved genesis state into the file {}.", + args.output_path.display() + ); + println!("Calculating new genesis parameters"); + + let mut genesis_config = read_yaml_repr::( + &args.genesis_config_path, + )?; + + let base_system_contract_hashes = BaseSystemContractsHashes { + bootloader: genesis_config + .bootloader_hash + .ok_or(anyhow::anyhow!("No bootloader_hash specified"))?, + default_aa: genesis_config + .default_aa_hash + .ok_or(anyhow::anyhow!("No default_aa_hash specified"))?, + evm_emulator: genesis_config.evm_emulator_hash, + }; + + let (genesis_batch_params, _) = make_genesis_batch_params( + get_deduped_log_queries(&storage_logs_for_genesis), + base_system_contract_hashes, + genesis_config + .protocol_version + .ok_or(anyhow::anyhow!("No bootloader_hash specified"))? + .minor, + ); + + genesis_config.genesis_root_hash = Some(genesis_batch_params.root_hash); + genesis_config.rollup_last_leaf_index = Some(genesis_batch_params.rollup_last_leaf_index); + genesis_config.genesis_commitment = Some(genesis_batch_params.commitment); + genesis_config.custom_genesis_state_path = + args.output_path.canonicalize()?.to_str().map(String::from); + + let bytes = + encode_yaml_repr::(&genesis_config)?; + fs::write(&args.genesis_config_path, &bytes)?; + + println!("Done."); + + Ok(()) +} diff --git a/core/lib/config/src/configs/genesis.rs b/core/lib/config/src/configs/genesis.rs index 9e1ffbd87cb5..5d5e39309bb8 100644 --- a/core/lib/config/src/configs/genesis.rs +++ b/core/lib/config/src/configs/genesis.rs @@ -32,6 +32,7 @@ pub struct GenesisConfig { pub fee_account: Address, pub dummy_verifier: bool, pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + pub custom_genesis_state_path: Option, } impl GenesisConfig { @@ -60,6 +61,7 @@ impl GenesisConfig { l2_chain_id: L2ChainId::default(), dummy_verifier: false, l1_batch_commit_data_generator_mode: L1BatchCommitmentMode::Rollup, + custom_genesis_state_path: None, } } } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 1a3f63d9b278..b026aabc6111 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -746,6 +746,7 @@ impl Distribution for EncodeDist { 0 => L1BatchCommitmentMode::Rollup, _ => L1BatchCommitmentMode::Validium, }, + custom_genesis_state_path: None, } } } diff --git a/core/lib/dal/.sqlx/query-42f15afb71632bdfab7befb651eaa3061382dfe4142c7fc46df1dfebec34ec92.json b/core/lib/dal/.sqlx/query-42f15afb71632bdfab7befb651eaa3061382dfe4142c7fc46df1dfebec34ec92.json new file mode 100644 index 000000000000..aeff27a88300 --- /dev/null +++ b/core/lib/dal/.sqlx/query-42f15afb71632bdfab7befb651eaa3061382dfe4142c7fc46df1dfebec34ec92.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH latest_storage_logs AS (\n SELECT DISTINCT ON (hashed_key)\n hashed_key,\n address,\n key,\n value\n FROM storage_logs\n ORDER BY hashed_key, miniblock_number DESC, operation_number DESC\n )\n \n SELECT\n lsl.address,\n lsl.key,\n lsl.value\n FROM\n initial_writes iw\n JOIN\n latest_storage_logs lsl ON iw.hashed_key = lsl.hashed_key\n WHERE\n lsl.value\n <> '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea\n AND (\n lsl.address <> '\\x000000000000000000000000000000000000800b'::bytea OR\n lsl.key IN (\n '\\x0000000000000000000000000000000000000000000000000000000000000000'::bytea,\n '\\x0000000000000000000000000000000000000000000000000000000000000003'::bytea,\n '\\x0000000000000000000000000000000000000000000000000000000000000004'::bytea,\n '\\x0000000000000000000000000000000000000000000000000000000000000005'::bytea\n )\n );\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "address", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "key", + "type_info": "Bytea" + }, + { + "ordinal": 2, + "name": "value", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true, + false + ] + }, + "hash": "42f15afb71632bdfab7befb651eaa3061382dfe4142c7fc46df1dfebec34ec92" +} diff --git a/core/lib/dal/.sqlx/query-bd37db29c86a84ed09ed0633e8511c5e74988422abd052a33ff1ec5db41f7d52.json b/core/lib/dal/.sqlx/query-bd37db29c86a84ed09ed0633e8511c5e74988422abd052a33ff1ec5db41f7d52.json new file mode 100644 index 000000000000..f33238c1f0c8 --- /dev/null +++ b/core/lib/dal/.sqlx/query-bd37db29c86a84ed09ed0633e8511c5e74988422abd052a33ff1ec5db41f7d52.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n bytecode_hash AS \"bytecode_hash!\",\n bytecode AS \"bytecode!\"\n FROM factory_deps\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bytecode_hash!", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "bytecode!", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "bd37db29c86a84ed09ed0633e8511c5e74988422abd052a33ff1ec5db41f7d52" +} diff --git a/core/lib/dal/src/custom_genesis_export_dal.rs b/core/lib/dal/src/custom_genesis_export_dal.rs new file mode 100644 index 000000000000..da74061fb5a5 --- /dev/null +++ b/core/lib/dal/src/custom_genesis_export_dal.rs @@ -0,0 +1,124 @@ +use serde::{Deserialize, Serialize}; +use zksync_db_connection::{connection::Connection, error::DalResult, instrument::InstrumentExt}; +use zksync_types::{AccountTreeId, StorageKey, StorageLog, H160, H256}; + +use crate::Core; + +#[derive(Debug)] +pub struct CustomGenesisExportDal<'a, 'c> { + pub(crate) storage: &'a mut Connection<'c, Core>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenesisState { + pub storage_logs: Vec, + pub factory_deps: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageLogRow { + pub address: [u8; 20], + pub key: [u8; 32], + pub value: [u8; 32], +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FactoryDepRow { + pub bytecode_hash: [u8; 32], + pub bytecode: Vec, +} + +impl CustomGenesisExportDal<'_, '_> { + pub async fn get_storage_logs(&mut self) -> DalResult> { + // This method returns storage logs that are used for genesis export. + // + // The where clause with addresses filters out SystemContext related records + // 0x0 -- chainId, + // 0x3 -- blockGasLimit, + // 0x4 -- coinbase, + // 0x5 -- difficulty + let rows = sqlx::query!( + r#" + WITH latest_storage_logs AS ( + SELECT DISTINCT ON (hashed_key) + hashed_key, + address, + key, + value + FROM storage_logs + ORDER BY hashed_key, miniblock_number DESC, operation_number DESC + ) + + SELECT + lsl.address, + lsl.key, + lsl.value + FROM + initial_writes iw + JOIN + latest_storage_logs lsl ON iw.hashed_key = lsl.hashed_key + WHERE + lsl.value + <> '\x0000000000000000000000000000000000000000000000000000000000000000'::bytea + AND ( + lsl.address <> '\x000000000000000000000000000000000000800b'::bytea OR + lsl.key IN ( + '\x0000000000000000000000000000000000000000000000000000000000000000'::bytea, + '\x0000000000000000000000000000000000000000000000000000000000000003'::bytea, + '\x0000000000000000000000000000000000000000000000000000000000000004'::bytea, + '\x0000000000000000000000000000000000000000000000000000000000000005'::bytea + ) + ); + "#, + ) + .instrument("get_storage_logs") + .fetch_all(self.storage) + .await?; + + let storage_logs: Vec = rows + .into_iter() + .map(|row| StorageLogRow { + address: row.address.unwrap().try_into().unwrap(), + key: row.key.unwrap().try_into().unwrap(), + value: row.value.try_into().unwrap(), + }) + .collect(); + + Ok(storage_logs) + } + + pub async fn get_factory_deps(&mut self) -> DalResult> { + // 1. Fetch the rows from the database + let rows = sqlx::query!( + r#" + SELECT + bytecode_hash AS "bytecode_hash!", + bytecode AS "bytecode!" + FROM factory_deps + "# + ) + .instrument("get_factory_deps") + .fetch_all(self.storage) + .await?; + + // 2. Map the rows to FactoryDepRow structs + let factory_deps: Vec = rows + .into_iter() + .map(|row| FactoryDepRow { + bytecode_hash: row.bytecode_hash.try_into().unwrap(), + bytecode: row.bytecode, + }) + .collect(); + + Ok(factory_deps) + } +} + +impl From<&StorageLogRow> for StorageLog { + fn from(value: &StorageLogRow) -> Self { + StorageLog::new_write_log( + StorageKey::new(AccountTreeId::new(H160(value.address)), H256(value.key)), + H256(value.value), + ) + } +} diff --git a/core/lib/dal/src/lib.rs b/core/lib/dal/src/lib.rs index 20b428adec44..212ce4b50036 100644 --- a/core/lib/dal/src/lib.rs +++ b/core/lib/dal/src/lib.rs @@ -14,10 +14,10 @@ pub use zksync_db_connection::{ use crate::{ base_token_dal::BaseTokenDal, blocks_dal::BlocksDal, blocks_web3_dal::BlocksWeb3Dal, consensus_dal::ConsensusDal, contract_verification_dal::ContractVerificationDal, - data_availability_dal::DataAvailabilityDal, eth_sender_dal::EthSenderDal, - eth_watcher_dal::EthWatcherDal, events_dal::EventsDal, events_web3_dal::EventsWeb3Dal, - factory_deps_dal::FactoryDepsDal, proof_generation_dal::ProofGenerationDal, - protocol_versions_dal::ProtocolVersionsDal, + custom_genesis_export_dal::CustomGenesisExportDal, data_availability_dal::DataAvailabilityDal, + eth_sender_dal::EthSenderDal, eth_watcher_dal::EthWatcherDal, events_dal::EventsDal, + events_web3_dal::EventsWeb3Dal, factory_deps_dal::FactoryDepsDal, + proof_generation_dal::ProofGenerationDal, protocol_versions_dal::ProtocolVersionsDal, protocol_versions_web3_dal::ProtocolVersionsWeb3Dal, pruning_dal::PruningDal, snapshot_recovery_dal::SnapshotRecoveryDal, snapshots_creator_dal::SnapshotsCreatorDal, snapshots_dal::SnapshotsDal, storage_logs_dal::StorageLogsDal, @@ -33,6 +33,7 @@ pub mod blocks_web3_dal; pub mod consensus; pub mod consensus_dal; pub mod contract_verification_dal; +pub mod custom_genesis_export_dal; mod data_availability_dal; pub mod eth_sender_dal; pub mod eth_watcher_dal; @@ -132,6 +133,8 @@ where fn base_token_dal(&mut self) -> BaseTokenDal<'_, 'a>; fn eth_watcher_dal(&mut self) -> EthWatcherDal<'_, 'a>; + + fn custom_genesis_export_dal(&mut self) -> CustomGenesisExportDal<'_, 'a>; } #[derive(Clone, Debug)] @@ -258,4 +261,8 @@ impl<'a> CoreDal<'a> for Connection<'a, Core> { fn eth_watcher_dal(&mut self) -> EthWatcherDal<'_, 'a> { EthWatcherDal { storage: self } } + + fn custom_genesis_export_dal(&mut self) -> CustomGenesisExportDal<'_, 'a> { + CustomGenesisExportDal { storage: self } + } } diff --git a/core/lib/env_config/src/genesis.rs b/core/lib/env_config/src/genesis.rs index 55c79eceb502..db0228d91c13 100644 --- a/core/lib/env_config/src/genesis.rs +++ b/core/lib/env_config/src/genesis.rs @@ -37,6 +37,19 @@ impl FromEnv for ContractsForGenesis { } } +// For initializing genesis file from env it's required to have an additional struct, +// because these data is not present in any other structs +#[derive(Deserialize, Serialize, Debug, Clone)] +struct CustomGenesisState { + pub path: Option, +} + +impl FromEnv for CustomGenesisState { + fn from_env() -> anyhow::Result { + envy_load("custom_genesis_state", "CUSTOM_GENESIS_STATE_") + } +} + impl FromEnv for GenesisConfig { fn from_env() -> anyhow::Result { // Getting genesis from environmental variables is a temporary measure, that will be @@ -44,6 +57,7 @@ impl FromEnv for GenesisConfig { // #PLA-811 let network_config = &NetworkConfig::from_env()?; let contracts_config = &ContractsForGenesis::from_env()?; + let custom_genesis_state_config = CustomGenesisState::from_env()?; let state_keeper = StateKeeperConfig::from_env()?; // This is needed for backward compatibility, so if the new variable `genesis_protocol_semantic_version` @@ -79,6 +93,7 @@ impl FromEnv for GenesisConfig { .context("Fee account required for genesis")?, dummy_verifier: false, l1_batch_commit_data_generator_mode: state_keeper.l1_batch_commit_data_generator_mode, + custom_genesis_state_path: custom_genesis_state_config.path, }) } } diff --git a/core/lib/protobuf_config/src/genesis.rs b/core/lib/protobuf_config/src/genesis.rs index 7ecc768100fb..469140f7b0c2 100644 --- a/core/lib/protobuf_config/src/genesis.rs +++ b/core/lib/protobuf_config/src/genesis.rs @@ -99,6 +99,7 @@ impl ProtoRepr for proto::Genesis { .and_then(|x| Ok(proto::L1BatchCommitDataGeneratorMode::try_from(*x)?)) .context("l1_batch_commit_data_generator_mode")? .parse(), + custom_genesis_state_path: self.custom_genesis_state_path.clone(), }) } @@ -126,6 +127,7 @@ impl ProtoRepr for proto::Genesis { ) .into(), ), + custom_genesis_state_path: this.custom_genesis_state_path.clone(), } } } diff --git a/core/lib/protobuf_config/src/proto/config/genesis.proto b/core/lib/protobuf_config/src/proto/config/genesis.proto index e3a9a45366f9..2e9ebc82f25e 100644 --- a/core/lib/protobuf_config/src/proto/config/genesis.proto +++ b/core/lib/protobuf_config/src/proto/config/genesis.proto @@ -29,5 +29,6 @@ message Genesis { optional L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 29; // optional, default to rollup optional string genesis_protocol_semantic_version = 12; // optional; optional string evm_emulator_hash = 13; // optional; h256 + optional string custom_genesis_state_path = 14; // optional; reserved 11; reserved "shared_bridge"; } diff --git a/core/node/api_server/src/web3/namespaces/en.rs b/core/node/api_server/src/web3/namespaces/en.rs index 56a7e2df458c..f990144a4acd 100644 --- a/core/node/api_server/src/web3/namespaces/en.rs +++ b/core/node/api_server/src/web3/namespaces/en.rs @@ -222,6 +222,8 @@ impl EnNamespace { .state .api_config .l1_batch_commit_data_generator_mode, + // external node should initialise itself from a snapshot + custom_genesis_state_path: None, }; Ok(config) } diff --git a/core/node/genesis/Cargo.toml b/core/node/genesis/Cargo.toml index d625d7186bdf..4607db282728 100644 --- a/core/node/genesis/Cargo.toml +++ b/core/node/genesis/Cargo.toml @@ -26,3 +26,4 @@ anyhow.workspace = true itertools.workspace = true thiserror.workspace = true tracing.workspace = true +bincode.workspace = true diff --git a/core/node/genesis/src/lib.rs b/core/node/genesis/src/lib.rs index 38972389129e..d49c25bff03c 100644 --- a/core/node/genesis/src/lib.rs +++ b/core/node/genesis/src/lib.rs @@ -2,7 +2,7 @@ //! It initializes the Merkle tree with the basic setup (such as fields of special service accounts), //! setups the required databases, and outputs the data required to initialize a smart contract. -use std::fmt::Formatter; +use std::{collections::HashMap, fmt::Formatter}; use anyhow::Context as _; use zksync_config::GenesisConfig; @@ -10,7 +10,7 @@ use zksync_contracts::{ hyperchain_contract, verifier_contract, BaseSystemContracts, BaseSystemContractsHashes, SET_CHAIN_ID_EVENT, }; -use zksync_dal::{Connection, Core, CoreDal, DalError}; +use zksync_dal::{custom_genesis_export_dal::GenesisState, Connection, Core, CoreDal, DalError}; use zksync_eth_client::{CallFunctionArgs, EthInterface}; use zksync_merkle_tree::{domain::ZkSyncTree, TreeInstruction}; use zksync_multivm::utils::get_max_gas_per_pubdata_byte; @@ -25,19 +25,19 @@ use zksync_types::{ system_contracts::get_system_smart_contracts, u256_to_h256, web3::{BlockNumber, FilterBuilder}, + zk_evm_types::LogQuery, AccountTreeId, Address, Bloom, L1BatchNumber, L1ChainId, L2BlockNumber, L2ChainId, - ProtocolVersion, ProtocolVersionId, StorageKey, H256, U256, + ProtocolVersion, ProtocolVersionId, StorageKey, StorageLog, H256, U256, }; use crate::utils::{ add_eth_token, get_deduped_log_queries, get_storage_logs, - insert_base_system_contracts_to_factory_deps, insert_system_contracts, - save_genesis_l1_batch_metadata, + insert_base_system_contracts_to_factory_deps, insert_deduplicated_writes_and_protective_reads, + insert_factory_deps, insert_storage_logs, save_genesis_l1_batch_metadata, }; - #[cfg(test)] mod tests; -mod utils; +pub mod utils; #[derive(Debug, Clone)] pub struct BaseContractsHashError { @@ -185,53 +185,103 @@ pub fn mock_genesis_config() -> GenesisConfig { fee_account: Default::default(), dummy_verifier: false, l1_batch_commit_data_generator_mode: Default::default(), + custom_genesis_state_path: None, } } -// Insert genesis batch into the database -pub async fn insert_genesis_batch( +pub fn make_genesis_batch_params( + deduped_log_queries: Vec, + base_system_contract_hashes: BaseSystemContractsHashes, + protocol_version: ProtocolVersionId, +) -> (GenesisBatchParams, L1BatchCommitment) { + let storage_logs = deduped_log_queries + .into_iter() + .filter(|log_query| log_query.rw_flag) // only writes + .enumerate() + .map(|(index, log)| { + TreeInstruction::write( + StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)) + .hashed_key_u256(), + (index + 1) as u64, + u256_to_h256(log.written_value), + ) + }) + .collect::>(); + + let metadata = ZkSyncTree::process_genesis_batch(&storage_logs); + let root_hash = metadata.root_hash; + let rollup_last_leaf_index = metadata.leaf_count + 1; + + let commitment_input = CommitmentInput::for_genesis_batch( + root_hash, + rollup_last_leaf_index, + base_system_contract_hashes, + protocol_version, + ); + let block_commitment = L1BatchCommitment::new(commitment_input); + let commitment = block_commitment.hash().commitment; + + ( + GenesisBatchParams { + root_hash, + commitment, + rollup_last_leaf_index, + }, + block_commitment, + ) +} + +pub async fn insert_genesis_batch_with_custom_state( storage: &mut Connection<'_, Core>, genesis_params: &GenesisParams, + custom_genesis_state: Option, ) -> Result { let mut transaction = storage.start_transaction().await?; let verifier_config = L1VerifierConfig { snark_wrapper_vk_hash: genesis_params.config.snark_wrapper_vk_hash, }; - create_genesis_l1_batch( + // if a custom genesis state was provided, read storage logs and factory dependencies from there + let (storage_logs, factory_deps): (Vec, HashMap>) = + match custom_genesis_state { + Some(r) => ( + r.storage_logs + .into_iter() + .map(|x| StorageLog::from(&x)) + .collect(), + r.factory_deps + .into_iter() + .map(|f| (H256(f.bytecode_hash), f.bytecode)) + .collect(), + ), + None => ( + get_storage_logs(&genesis_params.system_contracts), + genesis_params + .system_contracts + .iter() + .map(|c| { + ( + BytecodeHash::for_bytecode(&c.bytecode).value(), + c.bytecode.clone(), + ) + }) + .collect(), + ), + }; + + // This action disregards how leaf indeces used to be ordered before, and it reorders them by + // sorting by , which is required for calculating genesis parameters. + let deduped_log_queries = create_genesis_l1_batch_from_storage_logs_and_factory_deps( &mut transaction, genesis_params.protocol_version(), genesis_params.base_system_contracts(), - genesis_params.system_contracts(), + &storage_logs, + factory_deps, verifier_config, ) .await?; tracing::info!("chain_schema_genesis is complete"); - let deduped_log_queries = - get_deduped_log_queries(&get_storage_logs(genesis_params.system_contracts())); - - let (deduplicated_writes, _): (Vec<_>, Vec<_>) = deduped_log_queries - .into_iter() - .partition(|log_query| log_query.rw_flag); - - let storage_logs: Vec = deduplicated_writes - .iter() - .enumerate() - .map(|(index, log)| { - TreeInstruction::write( - StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)) - .hashed_key_u256(), - (index + 1) as u64, - u256_to_h256(log.written_value), - ) - }) - .collect(); - - let metadata = ZkSyncTree::process_genesis_batch(&storage_logs); - let genesis_root_hash = metadata.root_hash; - let rollup_last_leaf_index = metadata.leaf_count + 1; - let base_system_contract_hashes = BaseSystemContractsHashes { bootloader: genesis_params .config @@ -243,27 +293,31 @@ pub async fn insert_genesis_batch( .ok_or(GenesisError::MalformedConfig("default_aa_hash"))?, evm_emulator: genesis_params.config.evm_emulator_hash, }; - let commitment_input = CommitmentInput::for_genesis_batch( - genesis_root_hash, - rollup_last_leaf_index, + + let (genesis_batch_params, block_commitment) = make_genesis_batch_params( + deduped_log_queries, base_system_contract_hashes, genesis_params.minor_protocol_version(), ); - let block_commitment = L1BatchCommitment::new(commitment_input); save_genesis_l1_batch_metadata( &mut transaction, block_commitment.clone(), - genesis_root_hash, - rollup_last_leaf_index, + genesis_batch_params.root_hash, + genesis_batch_params.rollup_last_leaf_index, ) .await?; transaction.commit().await?; - Ok(GenesisBatchParams { - root_hash: genesis_root_hash, - commitment: block_commitment.hash().commitment, - rollup_last_leaf_index, - }) + + Ok(genesis_batch_params) +} + +// Insert genesis batch into the database +pub async fn insert_genesis_batch( + storage: &mut Connection<'_, Core>, + genesis_params: &GenesisParams, +) -> Result { + insert_genesis_batch_with_custom_state(storage, genesis_params, None).await } pub async fn is_genesis_needed(storage: &mut Connection<'_, Core>) -> Result { @@ -316,6 +370,7 @@ pub async fn validate_genesis_params( pub async fn ensure_genesis_state( storage: &mut Connection<'_, Core>, genesis_params: &GenesisParams, + custom_genesis_state: Option, ) -> Result { let mut transaction = storage.start_transaction().await?; @@ -333,7 +388,12 @@ pub async fn ensure_genesis_state( root_hash, commitment, rollup_last_leaf_index, - } = insert_genesis_batch(&mut transaction, genesis_params).await?; + } = insert_genesis_batch_with_custom_state( + &mut transaction, + genesis_params, + custom_genesis_state, + ) + .await?; let expected_root_hash = genesis_params .config @@ -371,14 +431,14 @@ pub async fn ensure_genesis_state( Ok(root_hash) } -#[allow(clippy::too_many_arguments)] -pub async fn create_genesis_l1_batch( +pub(crate) async fn create_genesis_l1_batch_from_storage_logs_and_factory_deps( storage: &mut Connection<'_, Core>, protocol_version: ProtocolSemanticVersion, base_system_contracts: &BaseSystemContracts, - system_contracts: &[DeployedContract], + storage_logs: &[StorageLog], + factory_deps: HashMap>, l1_verifier_config: L1VerifierConfig, -) -> Result<(), GenesisError> { +) -> Result, GenesisError> { let version = ProtocolVersion { version: protocol_version, timestamp: 0, @@ -436,6 +496,33 @@ pub async fn create_genesis_l1_batch( .mark_l2_blocks_as_executed_in_l1_batch(L1BatchNumber(0)) .await?; + insert_base_system_contracts_to_factory_deps(&mut transaction, base_system_contracts).await?; + + let mut genesis_transaction = transaction.start_transaction().await?; + + insert_storage_logs(&mut genesis_transaction, storage_logs).await?; + let dedup_log_queries = get_deduped_log_queries(storage_logs); + insert_deduplicated_writes_and_protective_reads( + &mut genesis_transaction, + dedup_log_queries.as_slice(), + ) + .await?; + insert_factory_deps(&mut genesis_transaction, factory_deps).await?; + genesis_transaction.commit().await?; + add_eth_token(&mut transaction).await?; + + transaction.commit().await?; + Ok(dedup_log_queries) +} + +#[allow(clippy::too_many_arguments)] +pub async fn create_genesis_l1_batch( + storage: &mut Connection<'_, Core>, + protocol_version: ProtocolSemanticVersion, + base_system_contracts: &BaseSystemContracts, + system_contracts: &[DeployedContract], + l1_verifier_config: L1VerifierConfig, +) -> Result<(), GenesisError> { let storage_logs = get_storage_logs(system_contracts); let factory_deps = system_contracts @@ -448,11 +535,15 @@ pub async fn create_genesis_l1_batch( }) .collect(); - insert_base_system_contracts_to_factory_deps(&mut transaction, base_system_contracts).await?; - insert_system_contracts(&mut transaction, factory_deps, &storage_logs).await?; - add_eth_token(&mut transaction).await?; - - transaction.commit().await?; + create_genesis_l1_batch_from_storage_logs_and_factory_deps( + storage, + protocol_version, + base_system_contracts, + &storage_logs, + factory_deps, + l1_verifier_config, + ) + .await?; Ok(()) } diff --git a/core/node/genesis/src/utils.rs b/core/node/genesis/src/utils.rs index a51f49a166a2..a9b0831b31bc 100644 --- a/core/node/genesis/src/utils.rs +++ b/core/node/genesis/src/utils.rs @@ -74,7 +74,7 @@ pub(super) fn get_storage_logs(system_contracts: &[DeployedContract]) -> Vec Vec { +pub fn get_deduped_log_queries(storage_logs: &[StorageLog]) -> Vec { // we don't produce proof for the genesis block, // but we still need to populate the table // to have the correct initial state of the merkle tree @@ -172,29 +172,32 @@ pub(super) async fn save_genesis_l1_batch_metadata( Ok(()) } -pub(super) async fn insert_system_contracts( - storage: &mut Connection<'_, Core>, - factory_deps: HashMap>, +pub(super) async fn insert_storage_logs( + transaction: &mut Connection<'_, Core>, storage_logs: &[StorageLog], ) -> Result<(), GenesisError> { - let (deduplicated_writes, protective_reads): (Vec<_>, Vec<_>) = - get_deduped_log_queries(storage_logs) - .into_iter() - .partition(|log_query| log_query.rw_flag); - - let mut transaction = storage.start_transaction().await?; transaction .storage_logs_dal() .insert_storage_logs(L2BlockNumber(0), storage_logs) .await?; + Ok(()) +} + +pub(super) async fn insert_deduplicated_writes_and_protective_reads( + transaction: &mut Connection<'_, Core>, + deduped_log_queries: &[LogQuery], +) -> Result<(), GenesisError> { + let (deduplicated_writes, protective_reads): (Vec<_>, Vec<_>) = deduped_log_queries + .iter() + .partition(|log_query| log_query.rw_flag); transaction .storage_logs_dedup_dal() .insert_protective_reads( L1BatchNumber(0), &protective_reads - .into_iter() - .map(StorageLog::from) + .iter() + .map(|log_query: &LogQuery| StorageLog::from(*log_query)) // Pass the log_query to from() .collect::>(), ) .await?; @@ -205,16 +208,22 @@ pub(super) async fn insert_system_contracts( StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)).hashed_key() }) .collect(); + transaction .storage_logs_dedup_dal() .insert_initial_writes(L1BatchNumber(0), &written_storage_keys) .await?; + Ok(()) +} + +pub(super) async fn insert_factory_deps( + transaction: &mut Connection<'_, Core>, + factory_deps: HashMap>, +) -> Result<(), GenesisError> { transaction .factory_deps_dal() .insert_factory_deps(L2BlockNumber(0), &factory_deps) .await?; - - transaction.commit().await?; Ok(()) } diff --git a/core/node/node_storage_init/src/external_node/genesis.rs b/core/node/node_storage_init/src/external_node/genesis.rs index b7a7efa9cf53..d5c487535bf4 100644 --- a/core/node/node_storage_init/src/external_node/genesis.rs +++ b/core/node/node_storage_init/src/external_node/genesis.rs @@ -22,10 +22,14 @@ impl InitializeStorage for ExternalNodeGenesis { _stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { let mut storage = self.pool.connection_tagged("en").await?; + + // Custom genesis state for external nodes is not supported. If the main node has a custom genesis, + // its external nodes should be started from a snapshot instead. zksync_node_sync::genesis::perform_genesis_if_needed( &mut storage, self.l2_chain_id, &self.client.clone().for_component("genesis"), + None, ) .await .context("performing genesis failed") diff --git a/core/node/node_storage_init/src/main_node/genesis.rs b/core/node/node_storage_init/src/main_node/genesis.rs index e98473840370..a5d6c0e628ac 100644 --- a/core/node/node_storage_init/src/main_node/genesis.rs +++ b/core/node/node_storage_init/src/main_node/genesis.rs @@ -1,8 +1,11 @@ +use std::fs::File; + use anyhow::Context as _; use tokio::sync::watch; use zksync_config::{ContractsConfig, GenesisConfig}; use zksync_dal::{ConnectionPool, Core, CoreDal as _}; use zksync_node_genesis::GenesisParams; +use zksync_object_store::bincode; use zksync_web3_decl::client::{DynClient, L1}; use crate::traits::InitializeStorage; @@ -36,7 +39,21 @@ impl InitializeStorage for MainNodeGenesis { self.contracts.diamond_proxy_addr, ) .await?; - zksync_node_genesis::ensure_genesis_state(&mut storage, ¶ms).await?; + + let custom_genesis_state_reader = match &self.genesis.custom_genesis_state_path { + Some(path) => match File::open(path) { + Ok(file) => Some(bincode::deserialize_from(file)?), + Err(e) => return Err(e.into()), // Propagate other errors + }, + None => None, + }; + + zksync_node_genesis::ensure_genesis_state( + &mut storage, + ¶ms, + custom_genesis_state_reader, + ) + .await?; if let Some(ecosystem_contracts) = &self.contracts.ecosystem_contracts { zksync_node_genesis::save_set_chain_id_tx( diff --git a/core/node/node_sync/src/genesis.rs b/core/node/node_sync/src/genesis.rs index 7401bdd9c9d4..8d60599ff7ca 100644 --- a/core/node/node_sync/src/genesis.rs +++ b/core/node/node_sync/src/genesis.rs @@ -1,6 +1,6 @@ use anyhow::Context as _; use zksync_contracts::{BaseSystemContracts, BaseSystemContractsHashes, SystemContractCode}; -use zksync_dal::{Connection, Core, CoreDal}; +use zksync_dal::{custom_genesis_export_dal::GenesisState, Connection, Core, CoreDal}; use zksync_node_genesis::{ensure_genesis_state, GenesisParams}; use zksync_types::{ block::DeployedContract, system_contracts::get_system_smart_contracts, AccountTreeId, L2ChainId, @@ -16,13 +16,14 @@ pub async fn perform_genesis_if_needed( storage: &mut Connection<'_, Core>, zksync_chain_id: L2ChainId, client: &dyn MainNodeClient, + custom_genesis_state: Option, ) -> anyhow::Result<()> { let mut transaction = storage.start_transaction().await?; // We want to check whether the genesis is needed before we create genesis params to not // make the node startup slower. if transaction.blocks_dal().is_genesis_needed().await? { let genesis_params = create_genesis_params(client, zksync_chain_id).await?; - ensure_genesis_state(&mut transaction, &genesis_params) + ensure_genesis_state(&mut transaction, &genesis_params, custom_genesis_state) .await .context("ensure_genesis_state")?; }