From 7b6e59083cf0cafeaef5dd4b2dd39257ff91316d Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 27 Nov 2024 16:18:43 +0200 Subject: [PATCH] feat(pruning): Record L1 batch root hash in pruning logs (#3266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Records root hashes of the last hard-pruned L1 batch in pruning logs. ## Why ❔ This allows to check tree integrity after recovery after pruning. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zkstack dev fmt` and `zkstack dev lint`. --- ...f30cbc385e6fb6c8b8ae2c2d06b871a4cae72.json | 27 +++ ...b71ef6258e011f424c705d56c9827e8277c84.json | 44 +++++ ...dc766262b5bdc58be0444e164b1bd9feed02d.json | 38 ---- ...d_batch_root_hash_to_pruning_logs.down.sql | 2 + ...add_batch_root_hash_to_pruning_logs.up.sql | 3 + core/lib/dal/src/consensus_dal/mod.rs | 4 +- core/lib/dal/src/pruning_dal/mod.rs | 177 +++++++++++------- core/lib/dal/src/pruning_dal/tests.rs | 91 +++++---- core/lib/snapshots_applier/src/lib.rs | 5 +- .../api_server/src/execution_sandbox/mod.rs | 10 +- core/node/consensus/src/storage/testonly.rs | 20 +- core/node/db_pruner/src/lib.rs | 80 ++++---- core/node/db_pruner/src/tests.rs | 110 +++++------ core/node/metadata_calculator/src/pruning.rs | 14 +- .../metadata_calculator/src/recovery/mod.rs | 27 ++- .../metadata_calculator/src/recovery/tests.rs | 123 ++++++++---- core/node/metadata_calculator/src/tests.rs | 9 +- core/node/metadata_calculator/src/updater.rs | 2 +- core/node/test_utils/src/lib.rs | 10 +- 19 files changed, 473 insertions(+), 323 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-3785c01a8eb1eaeaf7baf0a8ba7f30cbc385e6fb6c8b8ae2c2d06b871a4cae72.json create mode 100644 core/lib/dal/.sqlx/query-8bab3c429fc5bbf4a91819e55cfb71ef6258e011f424c705d56c9827e8277c84.json delete mode 100644 core/lib/dal/.sqlx/query-c9a842d04e8b225e43f07f76541dc766262b5bdc58be0444e164b1bd9feed02d.json create mode 100644 core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.down.sql create mode 100644 core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.up.sql diff --git a/core/lib/dal/.sqlx/query-3785c01a8eb1eaeaf7baf0a8ba7f30cbc385e6fb6c8b8ae2c2d06b871a4cae72.json b/core/lib/dal/.sqlx/query-3785c01a8eb1eaeaf7baf0a8ba7f30cbc385e6fb6c8b8ae2c2d06b871a4cae72.json new file mode 100644 index 000000000000..dfaccfbc99d3 --- /dev/null +++ b/core/lib/dal/.sqlx/query-3785c01a8eb1eaeaf7baf0a8ba7f30cbc385e6fb6c8b8ae2c2d06b871a4cae72.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n pruning_log (\n pruned_l1_batch,\n pruned_miniblock,\n pruned_l1_batch_root_hash,\n type,\n created_at,\n updated_at\n )\n VALUES\n ($1, $2, $3, $4, NOW(), NOW())\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Bytea", + { + "Custom": { + "name": "prune_type", + "kind": { + "Enum": [ + "Soft", + "Hard" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "3785c01a8eb1eaeaf7baf0a8ba7f30cbc385e6fb6c8b8ae2c2d06b871a4cae72" +} diff --git a/core/lib/dal/.sqlx/query-8bab3c429fc5bbf4a91819e55cfb71ef6258e011f424c705d56c9827e8277c84.json b/core/lib/dal/.sqlx/query-8bab3c429fc5bbf4a91819e55cfb71ef6258e011f424c705d56c9827e8277c84.json new file mode 100644 index 000000000000..0132af8fbe6c --- /dev/null +++ b/core/lib/dal/.sqlx/query-8bab3c429fc5bbf4a91819e55cfb71ef6258e011f424c705d56c9827e8277c84.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH\n soft AS (\n SELECT\n pruned_l1_batch,\n pruned_miniblock\n FROM\n pruning_log\n WHERE\n type = 'Soft'\n ORDER BY\n pruned_l1_batch DESC\n LIMIT\n 1\n ),\n \n hard AS (\n SELECT\n pruned_l1_batch,\n pruned_miniblock,\n pruned_l1_batch_root_hash\n FROM\n pruning_log\n WHERE\n type = 'Hard'\n ORDER BY\n pruned_l1_batch DESC\n LIMIT\n 1\n )\n \n SELECT\n soft.pruned_l1_batch AS last_soft_pruned_l1_batch,\n soft.pruned_miniblock AS last_soft_pruned_l2_block,\n hard.pruned_l1_batch AS last_hard_pruned_l1_batch,\n hard.pruned_miniblock AS last_hard_pruned_l2_block,\n hard.pruned_l1_batch_root_hash AS last_hard_pruned_batch_root_hash\n FROM\n soft\n FULL JOIN hard ON TRUE\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_soft_pruned_l1_batch", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "last_soft_pruned_l2_block", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_hard_pruned_l1_batch", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "last_hard_pruned_l2_block", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "last_hard_pruned_batch_root_hash", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true, + true, + true, + true + ] + }, + "hash": "8bab3c429fc5bbf4a91819e55cfb71ef6258e011f424c705d56c9827e8277c84" +} diff --git a/core/lib/dal/.sqlx/query-c9a842d04e8b225e43f07f76541dc766262b5bdc58be0444e164b1bd9feed02d.json b/core/lib/dal/.sqlx/query-c9a842d04e8b225e43f07f76541dc766262b5bdc58be0444e164b1bd9feed02d.json deleted file mode 100644 index fb28539ccdf6..000000000000 --- a/core/lib/dal/.sqlx/query-c9a842d04e8b225e43f07f76541dc766262b5bdc58be0444e164b1bd9feed02d.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH\n soft AS (\n SELECT\n pruned_l1_batch,\n pruned_miniblock\n FROM\n pruning_log\n WHERE\n type = 'Soft'\n ORDER BY\n pruned_l1_batch DESC\n LIMIT\n 1\n ),\n \n hard AS (\n SELECT\n pruned_l1_batch,\n pruned_miniblock\n FROM\n pruning_log\n WHERE\n type = 'Hard'\n ORDER BY\n pruned_l1_batch DESC\n LIMIT\n 1\n )\n \n SELECT\n soft.pruned_l1_batch AS last_soft_pruned_l1_batch,\n soft.pruned_miniblock AS last_soft_pruned_miniblock,\n hard.pruned_l1_batch AS last_hard_pruned_l1_batch,\n hard.pruned_miniblock AS last_hard_pruned_miniblock\n FROM\n soft\n FULL JOIN hard ON TRUE\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "last_soft_pruned_l1_batch", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "last_soft_pruned_miniblock", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "last_hard_pruned_l1_batch", - "type_info": "Int8" - }, - { - "ordinal": 3, - "name": "last_hard_pruned_miniblock", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - true, - true, - true, - true - ] - }, - "hash": "c9a842d04e8b225e43f07f76541dc766262b5bdc58be0444e164b1bd9feed02d" -} diff --git a/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.down.sql b/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.down.sql new file mode 100644 index 000000000000..d6beeb80e8a5 --- /dev/null +++ b/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE pruning_log + DROP COLUMN pruned_l1_batch_root_hash; diff --git a/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.up.sql b/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.up.sql new file mode 100644 index 000000000000..6a990781f61c --- /dev/null +++ b/core/lib/dal/migrations/20241112120944_add_batch_root_hash_to_pruning_logs.up.sql @@ -0,0 +1,3 @@ +-- nullable for backward compatibility +ALTER TABLE pruning_log + ADD COLUMN pruned_l1_batch_root_hash BYTEA DEFAULT NULL; diff --git a/core/lib/dal/src/consensus_dal/mod.rs b/core/lib/dal/src/consensus_dal/mod.rs index a091421d857c..7f3bcd1166ad 100644 --- a/core/lib/dal/src/consensus_dal/mod.rs +++ b/core/lib/dal/src/consensus_dal/mod.rs @@ -301,10 +301,10 @@ impl ConsensusDal<'_, '_> { .get_pruning_info() .await .context("get_pruning_info()")?; - Ok(match info.last_soft_pruned_l2_block { + Ok(match info.last_soft_pruned { // It is guaranteed that pruning info values are set for storage recovered from // snapshot, even if pruning was not enabled. - Some(last_pruned) => validator::BlockNumber(last_pruned.0.into()) + 1, + Some(last_pruned) => validator::BlockNumber(last_pruned.l2_block.0.into()) + 1, // No snapshot and no pruning: None => validator::BlockNumber(0), }) diff --git a/core/lib/dal/src/pruning_dal/mod.rs b/core/lib/dal/src/pruning_dal/mod.rs index bcd9fdcfc3e1..85127ac7030b 100644 --- a/core/lib/dal/src/pruning_dal/mod.rs +++ b/core/lib/dal/src/pruning_dal/mod.rs @@ -1,25 +1,43 @@ use std::ops; use zksync_db_connection::{connection::Connection, error::DalResult, instrument::InstrumentExt}; -use zksync_types::{L1BatchNumber, L2BlockNumber}; +use zksync_types::{L1BatchNumber, L2BlockNumber, H256}; use crate::Core; #[cfg(test)] mod tests; -#[derive(Debug)] -pub struct PruningDal<'a, 'c> { - pub(crate) storage: &'a mut Connection<'c, Core>, +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SoftPruningInfo { + pub l1_batch: L1BatchNumber, + pub l2_block: L2BlockNumber, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct HardPruningInfo { + pub l1_batch: L1BatchNumber, + pub l2_block: L2BlockNumber, + /// May be set to `None` for old pruning logs. + pub l1_batch_root_hash: Option, } /// Information about Postgres pruning. #[derive(Debug, Clone, Copy, Default, PartialEq)] pub struct PruningInfo { - pub last_soft_pruned_l1_batch: Option, - pub last_soft_pruned_l2_block: Option, - pub last_hard_pruned_l1_batch: Option, - pub last_hard_pruned_l2_block: Option, + /// Information about last soft pruning. Soft pruning is expected to be ahead or equal to hard pruning. + pub last_soft_pruned: Option, + /// Information about last hard pruning. + pub last_hard_pruned: Option, +} + +impl PruningInfo { + /// Returns `true` iff pruning is caught up, i.e., all soft-pruned data is hard-pruned. + pub fn is_caught_up(&self) -> bool { + let soft_pruned_l1_batch = self.last_soft_pruned.map(|info| info.l1_batch); + let hard_pruned_l1_batch = self.last_hard_pruned.map(|info| info.l1_batch); + soft_pruned_l1_batch == hard_pruned_l1_batch + } } /// Statistics about a single hard pruning iteration. @@ -33,6 +51,44 @@ pub struct HardPruningStats { pub deleted_l2_to_l1_logs: u64, } +#[derive(Debug)] +struct StoragePruningInfo { + last_soft_pruned_l1_batch: Option, + last_soft_pruned_l2_block: Option, + last_hard_pruned_l1_batch: Option, + last_hard_pruned_l2_block: Option, + last_hard_pruned_batch_root_hash: Option>, +} + +impl StoragePruningInfo { + fn as_soft(&self) -> Option { + Some(SoftPruningInfo { + l1_batch: L1BatchNumber(self.last_soft_pruned_l1_batch? as u32), + l2_block: L2BlockNumber(self.last_soft_pruned_l2_block? as u32), + }) + } + + fn as_hard(&self) -> Option { + Some(HardPruningInfo { + l1_batch: L1BatchNumber(self.last_hard_pruned_l1_batch? as u32), + l2_block: L2BlockNumber(self.last_hard_pruned_l2_block? as u32), + l1_batch_root_hash: self + .last_hard_pruned_batch_root_hash + .as_deref() + .map(H256::from_slice), + }) + } +} + +impl From for PruningInfo { + fn from(row: StoragePruningInfo) -> Self { + Self { + last_soft_pruned: row.as_soft(), + last_hard_pruned: row.as_hard(), + } + } +} + #[derive(Debug, sqlx::Type)] #[sqlx(type_name = "prune_type")] enum PruneType { @@ -40,9 +96,15 @@ enum PruneType { Hard, } +#[derive(Debug)] +pub struct PruningDal<'a, 'c> { + pub(crate) storage: &'a mut Connection<'c, Core>, +} + impl PruningDal<'_, '_> { pub async fn get_pruning_info(&mut self) -> DalResult { - let pruning_info = sqlx::query!( + let row = sqlx::query_as!( + StoragePruningInfo, r#" WITH soft AS ( @@ -62,7 +124,8 @@ impl PruningDal<'_, '_> { hard AS ( SELECT pruned_l1_batch, - pruned_miniblock + pruned_miniblock, + pruned_l1_batch_root_hash FROM pruning_log WHERE @@ -75,36 +138,24 @@ impl PruningDal<'_, '_> { SELECT soft.pruned_l1_batch AS last_soft_pruned_l1_batch, - soft.pruned_miniblock AS last_soft_pruned_miniblock, + soft.pruned_miniblock AS last_soft_pruned_l2_block, hard.pruned_l1_batch AS last_hard_pruned_l1_batch, - hard.pruned_miniblock AS last_hard_pruned_miniblock + hard.pruned_miniblock AS last_hard_pruned_l2_block, + hard.pruned_l1_batch_root_hash AS last_hard_pruned_batch_root_hash FROM soft FULL JOIN hard ON TRUE "# ) - .map(|row| PruningInfo { - last_soft_pruned_l1_batch: row - .last_soft_pruned_l1_batch - .map(|num| L1BatchNumber(num as u32)), - last_soft_pruned_l2_block: row - .last_soft_pruned_miniblock - .map(|num| L2BlockNumber(num as u32)), - last_hard_pruned_l1_batch: row - .last_hard_pruned_l1_batch - .map(|num| L1BatchNumber(num as u32)), - last_hard_pruned_l2_block: row - .last_hard_pruned_miniblock - .map(|num| L2BlockNumber(num as u32)), - }) .instrument("get_last_soft_pruned_batch") .report_latency() .fetch_optional(self.storage) .await?; - Ok(pruning_info.unwrap_or_default()) + + Ok(row.map(PruningInfo::from).unwrap_or_default()) } - pub async fn soft_prune_batches_range( + pub async fn insert_soft_pruning_log( &mut self, last_l1_batch_to_prune: L1BatchNumber, last_l2_block_to_prune: L2BlockNumber, @@ -137,6 +188,7 @@ impl PruningDal<'_, '_> { Ok(()) } + /// Does not insert pruning logs; the caller is responsible to do this! pub async fn hard_prune_batches_range( &mut self, last_l1_batch_to_prune: L1BatchNumber, @@ -159,42 +211,38 @@ impl PruningDal<'_, '_> { .fetch_one(self.storage) .await?; - // We don't have any L2 blocks available when recovering from a snapshot - let stats = if let Some(first_l2_block_to_prune) = row.first_miniblock_to_prune { - let first_l2_block_to_prune = L2BlockNumber(first_l2_block_to_prune as u32); - - let deleted_events = self - .delete_events(first_l2_block_to_prune..=last_l2_block_to_prune) - .await?; - let deleted_l2_to_l1_logs = self - .delete_l2_to_l1_logs(first_l2_block_to_prune..=last_l2_block_to_prune) - .await?; - let deleted_call_traces = self - .delete_call_traces(first_l2_block_to_prune..=last_l2_block_to_prune) - .await?; - self.clear_transaction_fields(first_l2_block_to_prune..=last_l2_block_to_prune) - .await?; - - let deleted_storage_logs = self - .prune_storage_logs(first_l2_block_to_prune..=last_l2_block_to_prune) - .await?; - let deleted_l1_batches = self.delete_l1_batches(last_l1_batch_to_prune).await?; - let deleted_l2_blocks = self.delete_l2_blocks(last_l2_block_to_prune).await?; - - HardPruningStats { - deleted_l1_batches, - deleted_l2_blocks, - deleted_events, - deleted_l2_to_l1_logs, - deleted_call_traces, - deleted_storage_logs, - } - } else { - HardPruningStats::default() + let Some(first_l2_block_to_prune) = row.first_miniblock_to_prune else { + return Ok(HardPruningStats::default()); }; - self.insert_hard_pruning_log(last_l1_batch_to_prune, last_l2_block_to_prune) + let first_l2_block_to_prune = L2BlockNumber(first_l2_block_to_prune as u32); + + let deleted_events = self + .delete_events(first_l2_block_to_prune..=last_l2_block_to_prune) + .await?; + let deleted_l2_to_l1_logs = self + .delete_l2_to_l1_logs(first_l2_block_to_prune..=last_l2_block_to_prune) + .await?; + let deleted_call_traces = self + .delete_call_traces(first_l2_block_to_prune..=last_l2_block_to_prune) .await?; + self.clear_transaction_fields(first_l2_block_to_prune..=last_l2_block_to_prune) + .await?; + + let deleted_storage_logs = self + .prune_storage_logs(first_l2_block_to_prune..=last_l2_block_to_prune) + .await?; + let deleted_l1_batches = self.delete_l1_batches(last_l1_batch_to_prune).await?; + let deleted_l2_blocks = self.delete_l2_blocks(last_l2_block_to_prune).await?; + + let stats = HardPruningStats { + deleted_l1_batches, + deleted_l2_blocks, + deleted_events, + deleted_l2_to_l1_logs, + deleted_call_traces, + deleted_storage_logs, + }; Ok(stats) } @@ -389,10 +437,11 @@ impl PruningDal<'_, '_> { Ok(execution_result.rows_affected()) } - async fn insert_hard_pruning_log( + pub async fn insert_hard_pruning_log( &mut self, last_l1_batch_to_prune: L1BatchNumber, last_l2_block_to_prune: L2BlockNumber, + last_pruned_l1_batch_root_hash: H256, ) -> DalResult<()> { sqlx::query!( r#" @@ -400,15 +449,17 @@ impl PruningDal<'_, '_> { pruning_log ( pruned_l1_batch, pruned_miniblock, + pruned_l1_batch_root_hash, type, created_at, updated_at ) VALUES - ($1, $2, $3, NOW(), NOW()) + ($1, $2, $3, $4, NOW(), NOW()) "#, i64::from(last_l1_batch_to_prune.0), i64::from(last_l2_block_to_prune.0), + last_pruned_l1_batch_root_hash.as_bytes(), PruneType::Hard as PruneType ) .instrument("hard_prune_batches_range#insert_pruning_log") diff --git a/core/lib/dal/src/pruning_dal/tests.rs b/core/lib/dal/src/pruning_dal/tests.rs index 70dda48d8c82..14f664a401fe 100644 --- a/core/lib/dal/src/pruning_dal/tests.rs +++ b/core/lib/dal/src/pruning_dal/tests.rs @@ -96,6 +96,13 @@ async fn insert_l1_batch(conn: &mut Connection<'_, Core>, l1_batch_number: L1Bat .insert_mock_l1_batch(&header) .await .unwrap(); + conn.blocks_dal() + .set_l1_batch_hash( + l1_batch_number, + H256::from_low_u64_be(l1_batch_number.0.into()), + ) + .await + .unwrap(); } async fn insert_realistic_l1_batches(conn: &mut Connection<'_, Core>, l1_batches_count: u32) { @@ -121,11 +128,11 @@ async fn insert_realistic_l1_batches(conn: &mut Connection<'_, Core>, l1_batches } } -async fn assert_l1_batch_objects_exists( +async fn assert_l1_batches_exist( conn: &mut Connection<'_, Core>, l1_batches_range: ops::RangeInclusive, ) { - for l1_batch_number in l1_batches_range.start().0..l1_batches_range.end().0 { + for l1_batch_number in l1_batches_range.start().0..=l1_batches_range.end().0 { let l1_batch_number = L1BatchNumber(l1_batch_number); assert!(conn .blocks_dal() @@ -150,7 +157,7 @@ async fn assert_l1_batch_objects_exists( } } -async fn assert_l1_batch_objects_dont_exist( +async fn assert_l1_batches_not_exist( conn: &mut Connection<'_, Core>, l1_batches_range: ops::RangeInclusive, ) { @@ -159,7 +166,7 @@ async fn assert_l1_batch_objects_dont_exist( .dump_all_storage_logs_for_tests() .await; - for l1_batch_number in l1_batches_range.start().0..l1_batches_range.end().0 { + for l1_batch_number in l1_batches_range.start().0..=l1_batches_range.end().0 { let l1_batch_number = L1BatchNumber(l1_batch_number); let mut l2_block_number = L2BlockNumber(l1_batch_number.0 * 2); assert!(conn @@ -204,55 +211,60 @@ async fn soft_pruning_works() { assert_eq!( PruningInfo { - last_soft_pruned_l2_block: None, - last_soft_pruned_l1_batch: None, - last_hard_pruned_l2_block: None, - last_hard_pruned_l1_batch: None + last_soft_pruned: None, + last_hard_pruned: None, }, transaction.pruning_dal().get_pruning_info().await.unwrap() ); transaction .pruning_dal() - .soft_prune_batches_range(L1BatchNumber(5), L2BlockNumber(11)) + .insert_soft_pruning_log(L1BatchNumber(5), L2BlockNumber(11)) .await .unwrap(); assert_eq!( PruningInfo { - last_soft_pruned_l2_block: Some(L2BlockNumber(11)), - last_soft_pruned_l1_batch: Some(L1BatchNumber(5)), - last_hard_pruned_l2_block: None, - last_hard_pruned_l1_batch: None + last_soft_pruned: Some(SoftPruningInfo { + l2_block: L2BlockNumber(11), + l1_batch: L1BatchNumber(5), + }), + last_hard_pruned: None, }, transaction.pruning_dal().get_pruning_info().await.unwrap() ); transaction .pruning_dal() - .soft_prune_batches_range(L1BatchNumber(10), L2BlockNumber(21)) + .insert_soft_pruning_log(L1BatchNumber(10), L2BlockNumber(21)) .await .unwrap(); assert_eq!( PruningInfo { - last_soft_pruned_l2_block: Some(L2BlockNumber(21)), - last_soft_pruned_l1_batch: Some(L1BatchNumber(10)), - last_hard_pruned_l2_block: None, - last_hard_pruned_l1_batch: None + last_soft_pruned: Some(SoftPruningInfo { + l2_block: L2BlockNumber(21), + l1_batch: L1BatchNumber(10), + }), + last_hard_pruned: None, }, transaction.pruning_dal().get_pruning_info().await.unwrap() ); transaction .pruning_dal() - .hard_prune_batches_range(L1BatchNumber(10), L2BlockNumber(21)) + .insert_hard_pruning_log(L1BatchNumber(10), L2BlockNumber(21), H256::repeat_byte(23)) .await .unwrap(); assert_eq!( PruningInfo { - last_soft_pruned_l2_block: Some(L2BlockNumber(21)), - last_soft_pruned_l1_batch: Some(L1BatchNumber(10)), - last_hard_pruned_l2_block: Some(L2BlockNumber(21)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(10)) + last_soft_pruned: Some(SoftPruningInfo { + l2_block: L2BlockNumber(21), + l1_batch: L1BatchNumber(10), + }), + last_hard_pruned: Some(HardPruningInfo { + l2_block: L2BlockNumber(21), + l1_batch: L1BatchNumber(10), + l1_batch_root_hash: Some(H256::repeat_byte(23)), + }), }, transaction.pruning_dal().get_pruning_info().await.unwrap() ); @@ -362,7 +374,7 @@ async fn storage_logs_pruning_works_correctly() { let stats = transaction .pruning_dal() - .hard_prune_batches_range(L1BatchNumber(10), L2BlockNumber(21)) + .hard_prune_batches_range(L1BatchNumber(9), L2BlockNumber(19)) .await .unwrap(); let actual_logs = transaction @@ -393,13 +405,13 @@ async fn l1_batches_can_be_hard_pruned() { let mut transaction = conn.start_transaction().await.unwrap(); insert_realistic_l1_batches(&mut transaction, 10).await; - assert_l1_batch_objects_exists(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(10)).await; + assert_l1_batches_exist(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(9)).await; assert!(transaction .pruning_dal() .get_pruning_info() .await .unwrap() - .last_hard_pruned_l1_batch + .last_hard_pruned .is_none()); transaction @@ -408,21 +420,12 @@ async fn l1_batches_can_be_hard_pruned() { .await .unwrap(); - assert_l1_batch_objects_dont_exist(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(5)).await; - assert_l1_batch_objects_exists(&mut transaction, L1BatchNumber(6)..=L1BatchNumber(10)).await; - assert_eq!( - Some(L1BatchNumber(5)), - transaction - .pruning_dal() - .get_pruning_info() - .await - .unwrap() - .last_hard_pruned_l1_batch - ); + assert_l1_batches_not_exist(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(5)).await; + assert_l1_batches_exist(&mut transaction, L1BatchNumber(6)..=L1BatchNumber(9)).await; let stats = transaction .pruning_dal() - .hard_prune_batches_range(L1BatchNumber(10), L2BlockNumber(21)) + .hard_prune_batches_range(L1BatchNumber(9), L2BlockNumber(19)) .await .unwrap(); assert_eq!(stats.deleted_l1_batches, 4); @@ -430,17 +433,7 @@ async fn l1_batches_can_be_hard_pruned() { assert_eq!(stats.deleted_events, 40); assert_eq!(stats.deleted_l2_to_l1_logs, 40); - assert_l1_batch_objects_dont_exist(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(10)) - .await; - assert_eq!( - Some(L1BatchNumber(10)), - transaction - .pruning_dal() - .get_pruning_info() - .await - .unwrap() - .last_hard_pruned_l1_batch - ); + assert_l1_batches_not_exist(&mut transaction, L1BatchNumber(1)..=L1BatchNumber(9)).await; } #[tokio::test] diff --git a/core/lib/snapshots_applier/src/lib.rs b/core/lib/snapshots_applier/src/lib.rs index 2c68b56ca5c6..f75fb2917b8e 100644 --- a/core/lib/snapshots_applier/src/lib.rs +++ b/core/lib/snapshots_applier/src/lib.rs @@ -725,16 +725,17 @@ impl<'a> SnapshotsApplier<'a> { // This allows to not deal with the corner cases when a node was recovered from a snapshot, but its pruning log is empty. storage_transaction .pruning_dal() - .soft_prune_batches_range( + .insert_soft_pruning_log( this.applied_snapshot_status.l1_batch_number, this.applied_snapshot_status.l2_block_number, ) .await?; storage_transaction .pruning_dal() - .hard_prune_batches_range( + .insert_hard_pruning_log( this.applied_snapshot_status.l1_batch_number, this.applied_snapshot_status.l2_block_number, + this.applied_snapshot_status.l1_batch_root_hash, ) .await?; } diff --git a/core/node/api_server/src/execution_sandbox/mod.rs b/core/node/api_server/src/execution_sandbox/mod.rs index b560d161ab52..bcba200f5ebc 100644 --- a/core/node/api_server/src/execution_sandbox/mod.rs +++ b/core/node/api_server/src/execution_sandbox/mod.rs @@ -228,9 +228,8 @@ impl BlockStartInfo { storage: &mut Connection<'_, Core>, ) -> anyhow::Result { let cached_pruning_info = self.get_pruning_info(storage).await?; - let last_block = cached_pruning_info.last_soft_pruned_l2_block; - if let Some(L2BlockNumber(last_block)) = last_block { - return Ok(L2BlockNumber(last_block + 1)); + if let Some(pruned) = cached_pruning_info.last_soft_pruned { + return Ok(pruned.l2_block + 1); } Ok(L2BlockNumber(0)) } @@ -240,9 +239,8 @@ impl BlockStartInfo { storage: &mut Connection<'_, Core>, ) -> anyhow::Result { let cached_pruning_info = self.get_pruning_info(storage).await?; - let last_batch = cached_pruning_info.last_soft_pruned_l1_batch; - if let Some(L1BatchNumber(last_block)) = last_batch { - return Ok(L1BatchNumber(last_block + 1)); + if let Some(pruned) = cached_pruning_info.last_soft_pruned { + return Ok(pruned.l1_batch + 1); } Ok(L1BatchNumber(0)) } diff --git a/core/node/consensus/src/storage/testonly.rs b/core/node/consensus/src/storage/testonly.rs index 0f29e2468267..295ae4fc1790 100644 --- a/core/node/consensus/src/storage/testonly.rs +++ b/core/node/consensus/src/storage/testonly.rs @@ -247,15 +247,20 @@ impl ConnectionPool { .await .wrap("get_l2_block_range_of_l1_batch()")? .context("batch not found")?; - let last_batch = L1BatchNumber(last_batch.0.try_into().context("oveflow")?); - let last_block = L2BlockNumber(last_block.0.try_into().context("oveflow")?); + let last_batch = L1BatchNumber(last_batch.0.try_into().context("overflow")?); + let last_batch_root_hash = ctx + .wait(conn.0.blocks_dal().get_l1_batch_state_root(last_batch)) + .await? + .context("get_l1_batch_state_root()")? + .unwrap_or_default(); + let last_block = L2BlockNumber(last_block.0.try_into().context("overflow")?); ctx.wait( conn.0 .pruning_dal() - .soft_prune_batches_range(last_batch, last_block), + .insert_soft_pruning_log(last_batch, last_block), ) .await? - .context("soft_prune_batches_range()")?; + .context("insert_soft_pruning_log()")?; ctx.wait( conn.0 .pruning_dal() @@ -263,6 +268,13 @@ impl ConnectionPool { ) .await? .context("hard_prune_batches_range()")?; + ctx.wait(conn.0.pruning_dal().insert_hard_pruning_log( + last_batch, + last_block, + last_batch_root_hash, + )) + .await? + .context("insert_hard_pruning_log()")?; Ok(()) } } diff --git a/core/node/db_pruner/src/lib.rs b/core/node/db_pruner/src/lib.rs index 4b4a53c68aa0..6ec8b2ce4602 100644 --- a/core/node/db_pruner/src/lib.rs +++ b/core/node/db_pruner/src/lib.rs @@ -8,7 +8,10 @@ use std::{ use anyhow::Context as _; use serde::{Deserialize, Serialize}; use tokio::sync::watch; -use zksync_dal::{pruning_dal::PruningInfo, Connection, ConnectionPool, Core, CoreDal}; +use zksync_dal::{ + pruning_dal::{HardPruningInfo, PruningInfo, SoftPruningInfo}, + Connection, ConnectionPool, Core, CoreDal, +}; use zksync_health_check::{Health, HealthStatus, HealthUpdater, ReactiveHealthCheck}; use zksync_types::{L1BatchNumber, L2BlockNumber}; @@ -53,10 +56,10 @@ struct DbPrunerHealth { impl From for DbPrunerHealth { fn from(info: PruningInfo) -> Self { Self { - last_soft_pruned_l1_batch: info.last_soft_pruned_l1_batch, - last_soft_pruned_l2_block: info.last_soft_pruned_l2_block, - last_hard_pruned_l1_batch: info.last_hard_pruned_l1_batch, - last_hard_pruned_l2_block: info.last_hard_pruned_l2_block, + last_soft_pruned_l1_batch: info.last_soft_pruned.map(|info| info.l1_batch), + last_soft_pruned_l2_block: info.last_soft_pruned.map(|info| info.l2_block), + last_hard_pruned_l1_batch: info.last_hard_pruned.map(|info| info.l1_batch), + last_hard_pruned_l2_block: info.last_hard_pruned.map(|info| info.l2_block), } } } @@ -188,13 +191,10 @@ impl DbPruner { let mut transaction = storage.start_transaction().await?; let mut current_pruning_info = transaction.pruning_dal().get_pruning_info().await?; - let next_l1_batch_to_prune = L1BatchNumber( - current_pruning_info - .last_soft_pruned_l1_batch - .unwrap_or(L1BatchNumber(0)) - .0 - + self.config.pruned_batch_chunk_size, - ); + let next_l1_batch_to_prune = current_pruning_info + .last_soft_pruned + .map_or(L1BatchNumber(0), |info| info.l1_batch) + + self.config.pruned_batch_chunk_size; if !self.is_l1_batch_prunable(next_l1_batch_to_prune).await { METRICS.pruning_chunk_duration[&PruneType::NoOp].observe(start.elapsed()); return Ok(false); @@ -207,7 +207,7 @@ impl DbPruner { .with_context(|| format!("L1 batch #{next_l1_batch_to_prune} is ready to be pruned, but has no L2 blocks"))?; transaction .pruning_dal() - .soft_prune_batches_range(next_l1_batch_to_prune, next_l2_block_to_prune) + .insert_soft_pruning_log(next_l1_batch_to_prune, next_l2_block_to_prune) .await?; transaction.commit().await?; @@ -218,8 +218,10 @@ impl DbPruner { "Soft pruned db l1_batches up to {next_l1_batch_to_prune} and L2 blocks up to {next_l2_block_to_prune}, operation took {latency:?}", ); - current_pruning_info.last_soft_pruned_l1_batch = Some(next_l1_batch_to_prune); - current_pruning_info.last_soft_pruned_l2_block = Some(next_l2_block_to_prune); + current_pruning_info.last_soft_pruned = Some(SoftPruningInfo { + l1_batch: next_l1_batch_to_prune, + l2_block: next_l2_block_to_prune, + }); self.update_health(current_pruning_info); Ok(true) } @@ -233,20 +235,26 @@ impl DbPruner { let mut transaction = storage.start_transaction().await?; let mut current_pruning_info = transaction.pruning_dal().get_pruning_info().await?; - let last_soft_pruned_l1_batch = - current_pruning_info.last_soft_pruned_l1_batch.with_context(|| { - format!("bogus pruning info {current_pruning_info:?}: trying to hard-prune data, but there is no soft-pruned L1 batch") - })?; - let last_soft_pruned_l2_block = - current_pruning_info.last_soft_pruned_l2_block.with_context(|| { - format!("bogus pruning info {current_pruning_info:?}: trying to hard-prune data, but there is no soft-pruned L2 block") + let soft_pruned = current_pruning_info.last_soft_pruned.with_context(|| { + format!("bogus pruning info {current_pruning_info:?}: trying to hard-prune data, but there is no soft-pruned data") + })?; + + let last_pruned_l1_batch_root_hash = transaction + .blocks_dal() + .get_l1_batch_state_root(soft_pruned.l1_batch) + .await? + .with_context(|| { + format!( + "hard-pruned L1 batch #{} does not have root hash", + soft_pruned.l1_batch + ) })?; let mut dal = transaction.pruning_dal(); let stats = tokio::select! { result = dal.hard_prune_batches_range( - last_soft_pruned_l1_batch, - last_soft_pruned_l2_block, + soft_pruned.l1_batch, + soft_pruned.l2_block, ) => result?, _ = stop_receiver.changed() => { @@ -258,15 +266,23 @@ impl DbPruner { } }; METRICS.observe_hard_pruning(stats); + + dal.insert_hard_pruning_log( + soft_pruned.l1_batch, + soft_pruned.l2_block, + last_pruned_l1_batch_root_hash, + ) + .await?; transaction.commit().await?; let latency = latency.observe(); - tracing::info!( - "Hard pruned db l1_batches up to {last_soft_pruned_l1_batch} and L2 blocks up to {last_soft_pruned_l2_block}, \ - operation took {latency:?}" - ); - current_pruning_info.last_hard_pruned_l1_batch = Some(last_soft_pruned_l1_batch); - current_pruning_info.last_hard_pruned_l2_block = Some(last_soft_pruned_l2_block); + let hard_pruning_info = HardPruningInfo { + l1_batch: soft_pruned.l1_batch, + l2_block: soft_pruned.l2_block, + l1_batch_root_hash: Some(last_pruned_l1_batch_root_hash), + }; + tracing::info!("Hard pruned data up to {hard_pruning_info:?}, operation took {latency:?}"); + current_pruning_info.last_hard_pruned = Some(hard_pruning_info); self.update_health(current_pruning_info); Ok(PruningIterationOutcome::Pruned) } @@ -280,9 +296,7 @@ impl DbPruner { self.update_health(current_pruning_info); // If this `if` is not entered, it means that the node has restarted after soft pruning - if current_pruning_info.last_soft_pruned_l1_batch - == current_pruning_info.last_hard_pruned_l1_batch - { + if current_pruning_info.is_caught_up() { let pruning_done = self.soft_prune(&mut storage).await?; if !pruning_done { return Ok(PruningIterationOutcome::NoOp); diff --git a/core/node/db_pruner/src/tests.rs b/core/node/db_pruner/src/tests.rs index 99fbada423dc..099914541fee 100644 --- a/core/node/db_pruner/src/tests.rs +++ b/core/node/db_pruner/src/tests.rs @@ -12,8 +12,7 @@ use zksync_node_test_utils::{ l1_batch_metadata_to_commitment_artifacts, }; use zksync_types::{ - aggregated_operations::AggregatedActionType, block::L2BlockHeader, Address, L2BlockNumber, - ProtocolVersion, H256, + aggregated_operations::AggregatedActionType, L2BlockNumber, ProtocolVersion, H256, }; use super::*; @@ -95,8 +94,8 @@ async fn is_l1_batch_prunable_works() { async fn insert_l2_blocks( conn: &mut Connection<'_, Core>, - l1_batches_count: u64, - l2_blocks_per_batch: u64, + l1_batches_count: u32, + l2_blocks_per_batch: u32, ) { conn.protocol_versions_dal() .save_protocol_version_with_tx(&ProtocolVersion::default()) @@ -104,36 +103,31 @@ async fn insert_l2_blocks( .unwrap(); for l1_batch_number in 0..l1_batches_count { + let l1_batch_number = L1BatchNumber(l1_batch_number); for l2_block_index in 0..l2_blocks_per_batch { - let l2_block_number = - L2BlockNumber((l1_batch_number * l2_blocks_per_batch + l2_block_index) as u32); - let l2_block_header = L2BlockHeader { - number: l2_block_number, - timestamp: 0, - hash: H256::from_low_u64_be(u64::from(l2_block_number.0)), - l1_tx_count: 0, - l2_tx_count: 0, - fee_account_address: Address::repeat_byte(1), - base_fee_per_gas: 0, - gas_per_pubdata_limit: 0, - batch_fee_input: Default::default(), - base_system_contracts_hashes: Default::default(), - protocol_version: Some(Default::default()), - virtual_blocks: 0, - gas_limit: 0, - logs_bloom: Default::default(), - pubdata_params: Default::default(), - }; + let l2_block_number = l1_batch_number.0 * l2_blocks_per_batch + l2_block_index; + let l2_block_header = create_l2_block(l2_block_number); conn.blocks_dal() .insert_l2_block(&l2_block_header) .await .unwrap(); conn.blocks_dal() - .mark_l2_blocks_as_executed_in_l1_batch(L1BatchNumber(l1_batch_number as u32)) + .mark_l2_blocks_as_executed_in_l1_batch(l1_batch_number) .await .unwrap(); } + + let l1_batch_header = create_l1_batch(l1_batch_number.0); + conn.blocks_dal() + .insert_mock_l1_batch(&l1_batch_header) + .await + .unwrap(); + let root_hash = H256::from_low_u64_be(l1_batch_number.0.into()); + conn.blocks_dal() + .set_l1_batch_hash(l1_batch_number, root_hash) + .await + .unwrap(); } } @@ -144,7 +138,7 @@ async fn hard_pruning_ignores_conditions_checks() { insert_l2_blocks(&mut conn, 10, 2).await; conn.pruning_dal() - .soft_prune_batches_range(L1BatchNumber(2), L2BlockNumber(5)) + .insert_soft_pruning_log(L1BatchNumber(2), L2BlockNumber(5)) .await .unwrap(); @@ -167,24 +161,34 @@ async fn hard_pruning_ignores_conditions_checks() { .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(2)), - last_soft_pruned_l2_block: Some(L2BlockNumber(5)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(2)), - last_hard_pruned_l2_block: Some(L2BlockNumber(5)), - }, + test_pruning_info(2, 5), conn.pruning_dal().get_pruning_info().await.unwrap() ); let health = health_check.check_health().await; assert_matches!(health.status(), HealthStatus::Ready); } + +fn test_pruning_info(l1_batch: u32, l2_block: u32) -> PruningInfo { + PruningInfo { + last_soft_pruned: Some(SoftPruningInfo { + l1_batch: L1BatchNumber(l1_batch), + l2_block: L2BlockNumber(l2_block), + }), + last_hard_pruned: Some(HardPruningInfo { + l1_batch: L1BatchNumber(l1_batch), + l2_block: L2BlockNumber(l2_block), + l1_batch_root_hash: Some(H256::from_low_u64_be(l1_batch.into())), + }), + } +} + #[test(tokio::test)] async fn pruner_catches_up_with_hard_pruning_up_to_soft_pruning_boundary_ignoring_chunk_size() { let pool = ConnectionPool::::test_pool().await; let mut conn = pool.connection().await.unwrap(); insert_l2_blocks(&mut conn, 10, 2).await; conn.pruning_dal() - .soft_prune_batches_range(L1BatchNumber(2), L2BlockNumber(5)) + .insert_soft_pruning_log(L1BatchNumber(2), L2BlockNumber(5)) .await .unwrap(); @@ -205,12 +209,7 @@ async fn pruner_catches_up_with_hard_pruning_up_to_soft_pruning_boundary_ignorin .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(2)), - last_soft_pruned_l2_block: Some(L2BlockNumber(5)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(2)), - last_hard_pruned_l2_block: Some(L2BlockNumber(5)), - }, + test_pruning_info(2, 5), conn.pruning_dal().get_pruning_info().await.unwrap() ); @@ -219,12 +218,7 @@ async fn pruner_catches_up_with_hard_pruning_up_to_soft_pruning_boundary_ignorin .await .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(7)), - last_soft_pruned_l2_block: Some(L2BlockNumber(15)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(7)), - last_hard_pruned_l2_block: Some(L2BlockNumber(15)), - }, + test_pruning_info(7, 15), conn.pruning_dal().get_pruning_info().await.unwrap() ); } @@ -253,12 +247,7 @@ async fn unconstrained_pruner_with_fresh_database() { .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(3)), - last_soft_pruned_l2_block: Some(L2BlockNumber(7)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(3)), - last_hard_pruned_l2_block: Some(L2BlockNumber(7)), - }, + test_pruning_info(3, 7), conn.pruning_dal().get_pruning_info().await.unwrap() ); @@ -267,12 +256,7 @@ async fn unconstrained_pruner_with_fresh_database() { .await .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(6)), - last_soft_pruned_l2_block: Some(L2BlockNumber(13)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(6)), - last_hard_pruned_l2_block: Some(L2BlockNumber(13)), - }, + test_pruning_info(6, 13), conn.pruning_dal().get_pruning_info().await.unwrap() ); } @@ -302,12 +286,7 @@ async fn pruning_blocked_after_first_chunk() { .unwrap(); assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(3)), - last_soft_pruned_l2_block: Some(L2BlockNumber(7)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(3)), - last_hard_pruned_l2_block: Some(L2BlockNumber(7)), - }, + test_pruning_info(3, 7), conn.pruning_dal().get_pruning_info().await.unwrap() ); @@ -318,12 +297,7 @@ async fn pruning_blocked_after_first_chunk() { assert_matches!(outcome, PruningIterationOutcome::NoOp); // pruning shouldn't have progressed as chunk 6 cannot be pruned assert_eq!( - PruningInfo { - last_soft_pruned_l1_batch: Some(L1BatchNumber(3)), - last_soft_pruned_l2_block: Some(L2BlockNumber(7)), - last_hard_pruned_l1_batch: Some(L1BatchNumber(3)), - last_hard_pruned_l2_block: Some(L2BlockNumber(7)), - }, + test_pruning_info(3, 7), conn.pruning_dal().get_pruning_info().await.unwrap() ); } diff --git a/core/node/metadata_calculator/src/pruning.rs b/core/node/metadata_calculator/src/pruning.rs index 4ac05e55c302..77e4c30dc1d1 100644 --- a/core/node/metadata_calculator/src/pruning.rs +++ b/core/node/metadata_calculator/src/pruning.rs @@ -101,8 +101,8 @@ impl MerkleTreePruningTask { let pruning_info = storage.pruning_dal().get_pruning_info().await?; drop(storage); - if let Some(l1_batch_number) = pruning_info.last_hard_pruned_l1_batch { - let target_retained_l1_batch_number = l1_batch_number + 1; + if let Some(pruned) = pruning_info.last_hard_pruned { + let target_retained_l1_batch_number = pruned.l1_batch + 1; let target_retained_version = u64::from(target_retained_l1_batch_number.0); let Ok(prev_target_version) = pruner_handle.set_target_retained_version(target_retained_version) @@ -148,7 +148,7 @@ mod tests { use test_casing::test_casing; use zksync_node_genesis::{insert_genesis_batch, GenesisParams}; use zksync_node_test_utils::prepare_recovery_snapshot; - use zksync_types::{L1BatchNumber, L2BlockNumber}; + use zksync_types::{L1BatchNumber, L2BlockNumber, H256}; use super::*; use crate::{ @@ -194,6 +194,11 @@ mod tests { .hard_prune_batches_range(L1BatchNumber(3), L2BlockNumber(3)) .await .unwrap(); + storage + .pruning_dal() + .insert_hard_pruning_log(L1BatchNumber(3), L2BlockNumber(3), H256::zero()) + .await + .unwrap(); while reader.clone().info().await.min_l1_batch_number.unwrap() <= L1BatchNumber(3) { tokio::time::sleep(POLL_INTERVAL).await; @@ -322,9 +327,10 @@ mod tests { // Prune first 3 created batches in Postgres. storage .pruning_dal() - .hard_prune_batches_range( + .insert_hard_pruning_log( snapshot_recovery.l1_batch_number + 3, snapshot_recovery.l2_block_number + 3, + H256::zero(), // not used ) .await .unwrap(); diff --git a/core/node/metadata_calculator/src/recovery/mod.rs b/core/node/metadata_calculator/src/recovery/mod.rs index ce7207471791..372cbc96a6df 100644 --- a/core/node/metadata_calculator/src/recovery/mod.rs +++ b/core/node/metadata_calculator/src/recovery/mod.rs @@ -132,7 +132,7 @@ impl InitParameters { let (l1_batch, l2_block); let mut expected_root_hash = None; - match (recovery_status, pruning_info.last_hard_pruned_l2_block) { + match (recovery_status, pruning_info.last_hard_pruned) { (Some(recovery), None) => { tracing::warn!( "Snapshot recovery {recovery:?} is present on the node, but pruning info is empty; assuming no pruning happened" @@ -141,21 +141,20 @@ impl InitParameters { l2_block = recovery.l2_block_number; expected_root_hash = Some(recovery.l1_batch_root_hash); } - (Some(recovery), Some(pruned_l2_block)) => { + (Some(recovery), Some(pruned)) => { // We have both recovery and some pruning on top of it. - l2_block = pruned_l2_block.max(recovery.l2_block_number); - l1_batch = pruning_info - .last_hard_pruned_l1_batch - .with_context(|| format!("malformed pruning info: {pruning_info:?}"))?; - if l1_batch == recovery.l1_batch_number { + l2_block = pruned.l2_block.max(recovery.l2_block_number); + l1_batch = pruned.l1_batch; + if let Some(root_hash) = pruned.l1_batch_root_hash { + expected_root_hash = Some(root_hash); + } else if l1_batch == recovery.l1_batch_number { expected_root_hash = Some(recovery.l1_batch_root_hash); } } - (None, Some(pruned_l2_block)) => { - l2_block = pruned_l2_block; - l1_batch = pruning_info - .last_hard_pruned_l1_batch - .with_context(|| format!("malformed pruning info: {pruning_info:?}"))?; + (None, Some(pruned)) => { + l2_block = pruned.l2_block; + l1_batch = pruned.l1_batch; + expected_root_hash = pruned.l1_batch_root_hash; } (None, None) => return Ok(None), }; @@ -384,9 +383,9 @@ impl AsyncTreeRecovery { snapshot_l2_block: L2BlockNumber, ) -> anyhow::Result<()> { let pruning_info = storage.pruning_dal().get_pruning_info().await?; - if let Some(last_hard_pruned_l2_block) = pruning_info.last_hard_pruned_l2_block { + if let Some(pruned) = pruning_info.last_hard_pruned { anyhow::ensure!( - last_hard_pruned_l2_block == snapshot_l2_block, + pruned.l2_block == snapshot_l2_block, "Additional data was pruned compared to tree recovery L2 block #{snapshot_l2_block}: {pruning_info:?}. \ Continuing recovery is impossible; to recover the tree, drop its RocksDB directory, stop pruning and restart recovery" ); diff --git a/core/node/metadata_calculator/src/recovery/tests.rs b/core/node/metadata_calculator/src/recovery/tests.rs index 4b2ba578a5b6..8b5371441162 100644 --- a/core/node/metadata_calculator/src/recovery/tests.rs +++ b/core/node/metadata_calculator/src/recovery/tests.rs @@ -1,6 +1,6 @@ //! Tests for metadata calculator snapshot recovery. -use std::{collections::HashMap, path::Path, sync::Mutex}; +use std::{path::Path, sync::Mutex}; use assert_matches::assert_matches; use tempfile::TempDir; @@ -16,7 +16,7 @@ use zksync_merkle_tree::{domain::ZkSyncTree, recovery::PersistenceThreadHandle, use zksync_node_genesis::{insert_genesis_batch, GenesisParams}; use zksync_node_test_utils::prepare_recovery_snapshot; use zksync_storage::RocksDB; -use zksync_types::{L1BatchNumber, U256}; +use zksync_types::L1BatchNumber; use super::*; use crate::{ @@ -28,6 +28,8 @@ use crate::{ MetadataCalculator, MetadataCalculatorConfig, }; +impl HandleRecoveryEvent for () {} + #[test] fn calculating_chunk_count() { let mut snapshot = InitParameters { @@ -116,9 +118,16 @@ async fn prune_storage(pool: &ConnectionPool, pruned_l1_batch: L1BatchNumb .await .unwrap() .expect("L1 batch not present in Postgres"); + let root_hash = storage + .blocks_dal() + .get_l1_batch_state_root(pruned_l1_batch) + .await + .unwrap() + .expect("L1 batch does not have root hash"); + storage .pruning_dal() - .soft_prune_batches_range(pruned_l1_batch, pruned_l2_block) + .insert_soft_pruning_log(pruned_l1_batch, pruned_l2_block) .await .unwrap(); let pruning_stats = storage @@ -130,6 +139,11 @@ async fn prune_storage(pool: &ConnectionPool, pruned_l1_batch: L1BatchNumb pruning_stats.deleted_l1_batches > 0 && pruning_stats.deleted_l2_blocks > 0, "{pruning_stats:?}" ); + storage + .pruning_dal() + .insert_hard_pruning_log(pruned_l1_batch, pruned_l2_block, root_hash) + .await + .unwrap(); } #[tokio::test] @@ -425,8 +439,7 @@ async fn entire_recovery_workflow(case: RecoveryWorkflowCase) { calculator_task.await.expect("calculator panicked").unwrap(); } -/// `pruned_batches == 0` is a sanity check. -#[test_casing(4, [0, 1, 2, 4])] +#[test_casing(3, [1, 2, 4])] #[tokio::test] async fn recovery_with_further_pruning(pruned_batches: u32) { const NEW_BATCH_COUNT: usize = 5; @@ -459,41 +472,80 @@ async fn recovery_with_further_pruning(pruned_batches: u32) { .await; db_transaction.commit().await.unwrap(); - let all_logs = storage - .storage_logs_dal() - .dump_all_storage_logs_for_tests() - .await; - assert_eq!(all_logs.len(), 400); - let initial_writes = storage - .storage_logs_dedup_dal() - .dump_all_initial_writes_for_tests() - .await; - let initial_writes: HashMap<_, _> = initial_writes - .into_iter() - .map(|write| (write.hashed_key, write.index)) - .collect(); - drop(storage); + // Run the first tree instance to compute root hashes for all batches. + let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); + let (calculator, _) = + setup_calculator(&temp_dir.path().join("first"), pool.clone(), true).await; + let expected_root_hash = run_calculator(calculator).await; - let instructions: Vec<_> = all_logs - .iter() - .map(|log| { - let leaf_index = initial_writes[&log.hashed_key]; - let key = U256::from_little_endian(log.hashed_key.as_bytes()); - TreeInstruction::write(key, leaf_index, log.value) - }) - .collect(); - let expected_root_hash = ZkSyncTree::process_genesis_batch(&instructions).root_hash; - - if pruned_batches > 0 { - prune_storage(&pool, snapshot_recovery.l1_batch_number + pruned_batches).await; - } + prune_storage(&pool, snapshot_recovery.l1_batch_number + pruned_batches).await; // Create a new tree instance. It should recover and process the remaining batches. let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); - let (calculator, _) = setup_calculator(temp_dir.path(), pool, true).await; + let (calculator, _) = setup_calculator(&temp_dir.path().join("new"), pool, true).await; assert_eq!(run_calculator(calculator).await, expected_root_hash); } +#[tokio::test] +async fn detecting_root_hash_mismatch_after_pruning() { + let pool = ConnectionPool::::test_pool().await; + let snapshot_logs = gen_storage_logs(100..300, 1).pop().unwrap(); + let mut storage = pool.connection().await.unwrap(); + let mut db_transaction = storage.start_transaction().await.unwrap(); + let snapshot_recovery = prepare_recovery_snapshot( + &mut db_transaction, + L1BatchNumber(23), + L2BlockNumber(42), + &snapshot_logs, + ) + .await; + + let logs = gen_storage_logs(200..400, 5); + extend_db_state_from_l1_batch( + &mut db_transaction, + snapshot_recovery.l1_batch_number + 1, + snapshot_recovery.l2_block_number + 1, + logs, + ) + .await; + // Intentionally add an incorrect root has of the batch to be pruned. + db_transaction + .blocks_dal() + .set_l1_batch_hash(snapshot_recovery.l1_batch_number + 1, H256::repeat_byte(42)) + .await + .unwrap(); + db_transaction.commit().await.unwrap(); + + prune_storage(&pool, snapshot_recovery.l1_batch_number + 1).await; + + let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); + let config = MetadataCalculatorRecoveryConfig::default(); + let (tree, _) = create_tree_recovery(temp_dir.path(), L1BatchNumber(1), &config).await; + let (_stop_sender, stop_receiver) = watch::channel(false); + let recovery_options = RecoveryOptions { + chunk_count: 5, + concurrency_limit: 1, + events: Box::new(()), + }; + let init_params = InitParameters::new(&pool, &config) + .await + .unwrap() + .expect("no init params"); + assert_eq!(init_params.expected_root_hash, Some(H256::repeat_byte(42))); + + let err = tree + .recover(init_params, recovery_options, &pool, &stop_receiver) + .await + .unwrap_err(); + let err = format!("{err:#}").to_lowercase(); + assert!(err.contains("root hash"), "{err}"); + + // Because of an abrupt error, terminating a RocksDB instance needs to be handled explicitly. + tokio::task::spawn_blocking(RocksDB::await_rocksdb_termination) + .await + .unwrap(); +} + #[derive(Debug)] struct PruningEventListener { pool: ConnectionPool, @@ -519,6 +571,11 @@ async fn pruning_during_recovery_is_detected() { let logs = gen_storage_logs(200..400, 5); extend_db_state(&mut storage, logs).await; drop(storage); + + // Set root hashes for all L1 batches in Postgres. + let (calculator, _) = + setup_calculator(&temp_dir.path().join("first"), pool.clone(), true).await; + run_calculator(calculator).await; prune_storage(&pool, L1BatchNumber(1)).await; let tree_path = temp_dir.path().join("recovery"); diff --git a/core/node/metadata_calculator/src/tests.rs b/core/node/metadata_calculator/src/tests.rs index 9717ce5682ce..ff939d1ae582 100644 --- a/core/node/metadata_calculator/src/tests.rs +++ b/core/node/metadata_calculator/src/tests.rs @@ -402,7 +402,7 @@ async fn error_on_pruned_next_l1_batch(sealed_protective_reads: bool) { extend_db_state(&mut storage, new_logs).await; storage .pruning_dal() - .soft_prune_batches_range(L1BatchNumber(5), L2BlockNumber(5)) + .insert_soft_pruning_log(L1BatchNumber(5), L2BlockNumber(5)) .await .unwrap(); storage @@ -410,6 +410,11 @@ async fn error_on_pruned_next_l1_batch(sealed_protective_reads: bool) { .hard_prune_batches_range(L1BatchNumber(5), L2BlockNumber(5)) .await .unwrap(); + storage + .pruning_dal() + .insert_hard_pruning_log(L1BatchNumber(5), L2BlockNumber(5), H256::zero()) + .await + .unwrap(); // Sanity check: there should be no pruned batch headers. let next_l1_batch_header = storage .blocks_dal() @@ -696,7 +701,7 @@ async fn setup_calculator_with_options( ) -> MetadataCalculator { let mut storage = pool.connection().await.unwrap(); let pruning_info = storage.pruning_dal().get_pruning_info().await.unwrap(); - let has_pruning_logs = pruning_info.last_hard_pruned_l1_batch.is_some(); + let has_pruning_logs = pruning_info.last_hard_pruned.is_some(); if !has_pruning_logs && storage.blocks_dal().is_genesis_needed().await.unwrap() { insert_genesis_batch(&mut storage, &GenesisParams::mock()) .await diff --git a/core/node/metadata_calculator/src/updater.rs b/core/node/metadata_calculator/src/updater.rs index 17fd5d900eab..2758bd1501c0 100644 --- a/core/node/metadata_calculator/src/updater.rs +++ b/core/node/metadata_calculator/src/updater.rs @@ -186,7 +186,7 @@ impl TreeUpdater { ) -> anyhow::Result<()> { let pruning_info = storage.pruning_dal().get_pruning_info().await?; anyhow::ensure!( - Some(l1_batch_number) > pruning_info.last_soft_pruned_l1_batch, + pruning_info.last_soft_pruned.map_or(true, |info| info.l1_batch < l1_batch_number), "L1 batch #{l1_batch_number}, next to be processed by the tree, is pruned; the tree cannot continue operating" ); Ok(()) diff --git a/core/node/test_utils/src/lib.rs b/core/node/test_utils/src/lib.rs index 9a02c18cd235..ac900e72bb6b 100644 --- a/core/node/test_utils/src/lib.rs +++ b/core/node/test_utils/src/lib.rs @@ -382,16 +382,18 @@ pub async fn recover( storage .pruning_dal() - .soft_prune_batches_range(snapshot.l1_batch.number, snapshot.l2_block.number) + .insert_soft_pruning_log(snapshot.l1_batch.number, snapshot.l2_block.number) .await .unwrap(); - storage .pruning_dal() - .hard_prune_batches_range(snapshot.l1_batch.number, snapshot.l2_block.number) + .insert_hard_pruning_log( + snapshot.l1_batch.number, + snapshot.l2_block.number, + snapshot_recovery.l1_batch_root_hash, + ) .await .unwrap(); - storage.commit().await.unwrap(); snapshot_recovery }