diff --git a/.changelog/unreleased/features/503-lazy-vec-and-map.md b/.changelog/unreleased/features/503-lazy-vec-and-map.md new file mode 100644 index 0000000000..d29ee5fd9c --- /dev/null +++ b/.changelog/unreleased/features/503-lazy-vec-and-map.md @@ -0,0 +1,2 @@ +- Added lazy vector and map data structures for ledger storage + ([#503](https://github.com/anoma/namada/pull/503)) \ No newline at end of file diff --git a/shared/src/ledger/storage/mod.rs b/shared/src/ledger/storage/mod.rs index 25f5ece03e..422bbe4d5e 100644 --- a/shared/src/ledger/storage/mod.rs +++ b/shared/src/ledger/storage/mod.rs @@ -809,6 +809,44 @@ where } } +impl StorageWrite for &mut Storage +where + D: DB + for<'iter> DBIter<'iter>, + H: StorageHasher, +{ + fn write( + &mut self, + key: &crate::types::storage::Key, + val: T, + ) -> storage_api::Result<()> { + let val = val.try_to_vec().unwrap(); + self.write_bytes(key, val) + } + + fn write_bytes( + &mut self, + key: &crate::types::storage::Key, + val: impl AsRef<[u8]>, + ) -> storage_api::Result<()> { + let _ = self + .db + .write_subspace_val(self.block.height, key, val) + .into_storage_result()?; + Ok(()) + } + + fn delete( + &mut self, + key: &crate::types::storage::Key, + ) -> storage_api::Result<()> { + let _ = self + .db + .delete_subspace_val(self.block.height, key) + .into_storage_result()?; + Ok(()) + } +} + impl From for Error { fn from(error: MerkleTreeError) -> Self { Self::MerkleTreeError(error) diff --git a/shared/src/ledger/storage_api/collections/lazy_map.rs b/shared/src/ledger/storage_api/collections/lazy_map.rs new file mode 100644 index 0000000000..34a0f7d891 --- /dev/null +++ b/shared/src/ledger/storage_api/collections/lazy_map.rs @@ -0,0 +1,563 @@ +//! Lazy map. + +use std::collections::HashMap; +use std::fmt::Debug; +use std::hash::Hash; +use std::marker::PhantomData; + +use borsh::{BorshDeserialize, BorshSerialize}; +use thiserror::Error; + +use super::super::Result; +use super::{LazyCollection, ReadError}; +use crate::ledger::storage_api::validation::{self, Data}; +use crate::ledger::storage_api::{self, ResultExt, StorageRead, StorageWrite}; +use crate::ledger::vp_env::VpEnv; +use crate::types::storage::{self, DbKeySeg, KeySeg}; + +/// Subkey corresponding to the data elements of the LazyMap +pub const DATA_SUBKEY: &str = "data"; + +/// Lazy map. +/// +/// This can be used as an alternative to `std::collections::HashMap` and +/// `BTreeMap`. In the lazy map, the elements do not reside in memory but are +/// instead read and written to storage sub-keys of the storage `key` used to +/// construct the map. +/// +/// In the [`LazyMap`], the type of key `K` can be anything that implements +/// [`storage::KeySeg`] and this trait is used to turn the keys into key +/// segments. +#[derive(Debug)] +pub struct LazyMap { + key: storage::Key, + phantom_k: PhantomData, + phantom_v: PhantomData, + phantom_son: PhantomData, +} + +/// A `LazyMap` with another `LazyCollection` inside it's value `V` +pub type NestedMap = LazyMap; + +/// Possible sub-keys of a [`LazyMap`] +#[derive(Clone, Debug)] +pub enum SubKey { + /// Data sub-key, further sub-keyed by its literal map key + Data(K), +} + +/// Possible sub-keys of a [`LazyMap`], together with their [`validation::Data`] +/// that contains prior and posterior state. +#[derive(Clone, Debug)] +pub enum SubKeyWithData { + /// Data sub-key, further sub-keyed by its literal map key + Data(K, Data), +} + +/// Possible actions that can modify a simple (not nested) [`LazyMap`]. This +/// roughly corresponds to the methods that have `StorageWrite` access. +#[derive(Clone, Debug)] +pub enum Action { + /// Insert or update a value `V` at key `K` in a [`LazyMap`]. + Insert(K, V), + /// Remove a value `V` at key `K` from a [`LazyMap`]. + Remove(K, V), + /// Update a value `V` at key `K` in a [`LazyMap`]. + Update { + /// key at which the value is updated + key: K, + /// value before the update + pre: V, + /// value after the update + post: V, + }, +} + +/// Possible actions that can modify a nested [`LazyMap`]. +#[derive(Clone, Debug)] +pub enum NestedAction { + /// Nested collection action `A` at key `K` + At(K, A), +} + +/// Possible sub-keys of a nested [`LazyMap`] +#[derive(Clone, Debug)] +pub enum NestedSubKey { + /// Data sub-key + Data { + /// Literal map key + key: K, + /// Sub-key in the nested collection + nested_sub_key: S, + }, +} + +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum ValidationError { + #[error("Invalid storage key {0}")] + InvalidSubKey(storage::Key), + #[error("Invalid nested storage key {0}")] + InvalidNestedSubKey(storage::Key), +} + +/// [`LazyMap`] validation result +pub type ValidationResult = std::result::Result; + +impl LazyCollection for LazyMap +where + K: storage::KeySeg + Clone + Hash + Eq + Debug, + V: LazyCollection + Debug, +{ + type Action = NestedAction::Action>; + type SubKey = NestedSubKey::SubKey>; + type SubKeyWithData = + NestedSubKey::SubKeyWithData>; + type Value = ::Value; + + fn open(key: storage::Key) -> Self { + Self { + key, + phantom_k: PhantomData, + phantom_v: PhantomData, + phantom_son: PhantomData, + } + } + + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result> { + let suffix = match key.split_prefix(&self.key) { + None => { + // not matching prefix, irrelevant + return Ok(None); + } + Some(None) => { + // no suffix, invalid + return Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(); + } + Some(Some(suffix)) => suffix, + }; + + // Match the suffix against expected sub-keys + match &suffix.segments[..2] { + [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] + if sub_a == DATA_SUBKEY => + { + if let Ok(key_in_kv) = storage::KeySeg::parse(sub_b.clone()) { + let nested = self.at(&key_in_kv).is_valid_sub_key(key)?; + match nested { + Some(nested_sub_key) => Ok(Some(NestedSubKey::Data { + key: key_in_kv, + nested_sub_key, + })), + None => Err(ValidationError::InvalidNestedSubKey( + key.clone(), + )) + .into_storage_result(), + } + } else { + Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result() + } + } + _ => Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(), + } + } + + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>, + { + let NestedSubKey::Data { + key, + // In here, we just have a nested sub-key without data + nested_sub_key, + } = sub_key; + // Try to read data from the nested collection + let nested_data = ::read_sub_key_data( + env, + storage_key, + nested_sub_key, + )?; + // If found, transform it back into a `NestedSubKey`, but with + // `nested_sub_key` replaced with the one we read + Ok(nested_data.map(|nested_sub_key| NestedSubKey::Data { + key, + nested_sub_key, + })) + } + + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { + // We have to group the nested sub-keys by the key from this map + let mut grouped_by_key: HashMap< + K, + Vec<::SubKeyWithData>, + > = HashMap::new(); + for NestedSubKey::Data { + key, + nested_sub_key, + } in keys + { + grouped_by_key + .entry(key) + .or_insert_with(Vec::new) + .push(nested_sub_key); + } + + // Recurse for each sub-keys group + let mut actions = vec![]; + for (key, sub_keys) in grouped_by_key { + let nested_actions = + ::validate_changed_sub_keys(sub_keys)?; + actions.extend( + nested_actions + .into_iter() + .map(|action| NestedAction::At(key.clone(), action)), + ); + } + Ok(actions) + } +} + +impl LazyCollection for LazyMap +where + K: storage::KeySeg + Debug, + V: BorshDeserialize + BorshSerialize + 'static + Debug, +{ + type Action = Action; + type SubKey = SubKey; + type SubKeyWithData = SubKeyWithData; + type Value = V; + + /// Create or use an existing map with the given storage `key`. + fn open(key: storage::Key) -> Self { + Self { + key, + phantom_k: PhantomData, + phantom_v: PhantomData, + phantom_son: PhantomData, + } + } + + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result> { + let suffix = match key.split_prefix(&self.key) { + None => { + // not matching prefix, irrelevant + return Ok(None); + } + Some(None) => { + // no suffix, invalid + return Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(); + } + Some(Some(suffix)) => suffix, + }; + + // Match the suffix against expected sub-keys + match &suffix.segments[..] { + [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] + if sub_a == DATA_SUBKEY => + { + if let Ok(key_in_kv) = storage::KeySeg::parse(sub_b.clone()) { + Ok(Some(SubKey::Data(key_in_kv))) + } else { + Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result() + } + } + _ => Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(), + } + } + + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>, + { + let SubKey::Data(key) = sub_key; + let data = validation::read_data(env, storage_key)?; + Ok(data.map(|data| SubKeyWithData::Data(key, data))) + } + + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { + Ok(keys + .into_iter() + .map(|change| { + let SubKeyWithData::Data(key, data) = change; + match data { + Data::Add { post } => Action::Insert(key, post), + Data::Update { pre, post } => { + Action::Update { key, pre, post } + } + Data::Delete { pre } => Action::Remove(key, pre), + } + }) + .collect()) + } +} + +// Generic `LazyMap` methods that require no bounds on values `V` +impl LazyMap +where + K: storage::KeySeg, +{ + /// Returns whether the set contains a value. + pub fn contains(&self, storage: &S, key: &K) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + storage.has_key(&self.get_data_key(key)) + } + + /// Get the prefix of set's elements storage + fn get_data_prefix(&self) -> storage::Key { + self.key.push(&DATA_SUBKEY.to_owned()).unwrap() + } + + /// Get the sub-key of a given element + fn get_data_key(&self, key: &K) -> storage::Key { + let key_str = key.to_db_key(); + self.get_data_prefix().push(&key_str).unwrap() + } +} + +// `LazyMap` methods with nested `LazyCollection`s `V` +impl LazyMap +where + K: storage::KeySeg + Clone + Hash + Eq + Debug, + V: LazyCollection + Debug, +{ + /// Get a nested collection at given key `key`. If there is no nested + /// collection at the given key, a new empty one will be provided. The + /// nested collection may be manipulated through its methods. + pub fn at(&self, key: &K) -> V { + V::open(self.get_data_key(key)) + } + + /// An iterator visiting all key-value elements, where the values are from + /// the inner-most collection. The iterator element type is `Result<_>`, + /// because iterator's call to `next` may fail with e.g. out of gas or + /// data decoding error. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded maps to avoid gas usage increasing with the length of the + /// map. + pub fn iter<'iter>( + &'iter self, + storage: &'iter impl StorageRead<'iter>, + ) -> Result< + impl Iterator< + Item = Result<( + ::SubKey, + ::Value, + )>, + > + 'iter, + > { + let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; + Ok(iter.map(|key_val_res| { + let (key, val) = key_val_res?; + let sub_key = LazyCollection::is_valid_sub_key(self, &key)? + .ok_or(ReadError::UnexpectedlyEmptyStorageKey) + .into_storage_result()?; + Ok((sub_key, val)) + })) + } +} + +// `LazyMap` methods with borsh encoded values `V` +impl LazyMap +where + K: storage::KeySeg, + V: BorshDeserialize + BorshSerialize + 'static, +{ + /// Inserts a key-value pair into the map. + /// + /// The full storage key identifies the key in the pair, while the value is + /// held within the storage key. + /// + /// If the map did not have this key present, `None` is returned. + /// If the map did have this key present, the value is updated, and the old + /// value is returned. Unlike in `std::collection::HashMap`, the key is also + /// updated; this matters for types that can be `==` without being + /// identical. + pub fn insert( + &self, + storage: &mut S, + key: K, + val: V, + ) -> Result> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let previous = self.get(storage, &key)?; + + let data_key = self.get_data_key(&key); + Self::write_key_val(storage, &data_key, val)?; + + Ok(previous) + } + + /// Removes a key from the map, returning the value at the key if the key + /// was previously in the map. + pub fn remove(&self, storage: &mut S, key: &K) -> Result> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let value = self.get(storage, key)?; + + let data_key = self.get_data_key(key); + storage.delete(&data_key)?; + + Ok(value) + } + + /// Returns the value corresponding to the key, if any. + pub fn get(&self, storage: &S, key: &K) -> Result> + where + S: for<'iter> StorageRead<'iter>, + { + let data_key = self.get_data_key(key); + Self::read_key_val(storage, &data_key) + } + + /// Returns whether the map contains no elements. + pub fn is_empty(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + let mut iter = + storage_api::iter_prefix_bytes(storage, &self.get_data_prefix())?; + Ok(iter.next().is_none()) + } + + /// Reads the number of elements in the map. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded maps to avoid gas usage increasing with the length of the + /// set. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + let iter = + storage_api::iter_prefix_bytes(storage, &self.get_data_prefix())?; + iter.count().try_into().into_storage_result() + } + + /// An iterator visiting all key-value elements. The iterator element type + /// is `Result<(K, V)>`, because iterator's call to `next` may fail with + /// e.g. out of gas or data decoding error. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded maps to avoid gas usage increasing with the length of the + /// map. + pub fn iter<'iter>( + &self, + storage: &'iter impl StorageRead<'iter>, + ) -> Result> + 'iter> { + let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; + Ok(iter.map(|key_val_res| { + let (key, val) = key_val_res?; + let last_key_seg = key + .last() + .ok_or(ReadError::UnexpectedlyEmptyStorageKey) + .into_storage_result()?; + let key = K::parse(last_key_seg.raw()).into_storage_result()?; + Ok((key, val)) + })) + } + + /// Reads a value from storage + fn read_key_val( + storage: &S, + storage_key: &storage::Key, + ) -> Result> + where + S: for<'iter> StorageRead<'iter>, + { + let res = storage.read(storage_key)?; + Ok(res) + } + + /// Write a value into storage + fn write_key_val( + storage: &mut impl StorageWrite, + storage_key: &storage::Key, + val: V, + ) -> Result<()> { + storage.write(storage_key, val) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::ledger::storage::testing::TestStorage; + + #[test] + fn test_lazy_map_basics() -> storage_api::Result<()> { + let mut storage = TestStorage::default(); + + let key = storage::Key::parse("test").unwrap(); + let lazy_map = LazyMap::::open(key); + + // The map should be empty at first + assert!(lazy_map.is_empty(&storage)?); + assert!(lazy_map.len(&storage)? == 0); + assert!(!lazy_map.contains(&storage, &0)?); + assert!(!lazy_map.contains(&storage, &1)?); + assert!(lazy_map.iter(&storage)?.next().is_none()); + assert!(lazy_map.get(&storage, &0)?.is_none()); + assert!(lazy_map.get(&storage, &1)?.is_none()); + assert!(lazy_map.remove(&mut storage, &0)?.is_none()); + assert!(lazy_map.remove(&mut storage, &1)?.is_none()); + + // Insert a new value and check that it's added + let (key, val) = (123, "Test".to_string()); + lazy_map.insert(&mut storage, key, val.clone())?; + assert!(!lazy_map.contains(&storage, &0)?); + assert!(lazy_map.contains(&storage, &key)?); + assert!(!lazy_map.is_empty(&storage)?); + assert!(lazy_map.len(&storage)? == 1); + assert_eq!( + lazy_map.iter(&storage)?.next().unwrap()?, + (key, val.clone()) + ); + assert!(lazy_map.get(&storage, &0)?.is_none()); + assert_eq!(lazy_map.get(&storage, &key)?.unwrap(), val); + + // Remove the last value and check that the map is empty again + let removed = lazy_map.remove(&mut storage, &key)?.unwrap(); + assert_eq!(removed, val); + assert!(lazy_map.is_empty(&storage)?); + assert!(lazy_map.len(&storage)? == 0); + assert!(!lazy_map.contains(&storage, &0)?); + assert!(!lazy_map.contains(&storage, &1)?); + assert!(lazy_map.get(&storage, &0)?.is_none()); + assert!(lazy_map.get(&storage, &key)?.is_none()); + assert!(lazy_map.iter(&storage)?.next().is_none()); + assert!(lazy_map.remove(&mut storage, &key)?.is_none()); + + Ok(()) + } +} diff --git a/shared/src/ledger/storage_api/collections/lazy_vec.rs b/shared/src/ledger/storage_api/collections/lazy_vec.rs new file mode 100644 index 0000000000..59eaa225e5 --- /dev/null +++ b/shared/src/ledger/storage_api/collections/lazy_vec.rs @@ -0,0 +1,516 @@ +//! Lazy dynamically-sized vector. + +use std::collections::BTreeSet; +use std::fmt::Debug; +use std::marker::PhantomData; + +use borsh::{BorshDeserialize, BorshSerialize}; +use thiserror::Error; + +use super::super::Result; +use super::LazyCollection; +use crate::ledger::storage_api::validation::{self, Data}; +use crate::ledger::storage_api::{self, ResultExt, StorageRead, StorageWrite}; +use crate::ledger::vp_env::VpEnv; +use crate::types::storage::{self, DbKeySeg}; + +/// Subkey pointing to the length of the LazyVec +pub const LEN_SUBKEY: &str = "len"; +/// Subkey corresponding to the data elements of the LazyVec +pub const DATA_SUBKEY: &str = "data"; + +/// Using `u64` for vector's indices +pub type Index = u64; + +/// Lazy dynamically-sized vector. +/// +/// This can be used as an alternative to `std::collections::Vec`. In the lazy +/// vector, the elements do not reside in memory but are instead read and +/// written to storage sub-keys of the storage `key` used to construct the +/// vector. +#[derive(Clone, Debug)] +pub struct LazyVec { + key: storage::Key, + phantom: PhantomData, +} + +/// Possible sub-keys of a [`LazyVec`] +#[derive(Debug)] +pub enum SubKey { + /// Length sub-key + Len, + /// Data sub-key, further sub-keyed by its index + Data(Index), +} + +/// Possible sub-keys of a [`LazyVec`], together with their [`validation::Data`] +/// that contains prior and posterior state. +#[derive(Debug)] +pub enum SubKeyWithData { + /// Length sub-key + Len(Data), + /// Data sub-key, further sub-keyed by its index + Data(Index, Data), +} + +/// Possible actions that can modify a [`LazyVec`]. This roughly corresponds to +/// the methods that have `StorageWrite` access. +#[derive(Clone, Debug)] +pub enum Action { + /// Push a value `T` into a [`LazyVec`] + Push(T), + /// Pop a value `T` from a [`LazyVec`] + Pop(T), + /// Update a value `T` at index from pre to post state in a [`LazyVec`] + Update { + /// index at which the value is updated + index: Index, + /// value before the update + pre: T, + /// value after the update + post: T, + }, +} + +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum ValidationError { + #[error("Incorrect difference in LazyVec's length")] + InvalidLenDiff, + #[error("An empty LazyVec must be deleted from storage")] + EmptyVecShouldBeDeleted, + #[error("Push at a wrong index. Got {got}, expected {expected}.")] + UnexpectedPushIndex { got: Index, expected: Index }, + #[error("Pop at a wrong index. Got {got}, expected {expected}.")] + UnexpectedPopIndex { got: Index, expected: Index }, + #[error( + "Update (or a combination of pop and push) at a wrong index. Got \ + {got}, expected maximum {max}." + )] + UnexpectedUpdateIndex { got: Index, max: Index }, + #[error("An index has overflown its representation: {0}")] + IndexOverflow(>::Error), + #[error("Unexpected underflow in `{0} - {0}`")] + UnexpectedUnderflow(Index, Index), + #[error("Invalid storage key {0}")] + InvalidSubKey(storage::Key), +} + +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum UpdateError { + #[error( + "Invalid index into a LazyVec. Got {index}, but the length is {len}" + )] + InvalidIndex { index: Index, len: u64 }, +} + +/// [`LazyVec`] validation result +pub type ValidationResult = std::result::Result; + +impl LazyCollection for LazyVec +where + T: BorshSerialize + BorshDeserialize + 'static + Debug, +{ + type Action = Action; + type SubKey = SubKey; + type SubKeyWithData = SubKeyWithData; + type Value = T; + + /// Create or use an existing vector with the given storage `key`. + fn open(key: storage::Key) -> Self { + Self { + key, + phantom: PhantomData, + } + } + + /// Check if the given storage key is a valid LazyVec sub-key and if so + /// return which one + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result> { + let suffix = match key.split_prefix(&self.key) { + None => { + // not matching prefix, irrelevant + return Ok(None); + } + Some(None) => { + // no suffix, invalid + return Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(); + } + Some(Some(suffix)) => suffix, + }; + + // Match the suffix against expected sub-keys + match &suffix.segments[..] { + [DbKeySeg::StringSeg(sub)] if sub == LEN_SUBKEY => { + Ok(Some(SubKey::Len)) + } + [DbKeySeg::StringSeg(sub_a), DbKeySeg::StringSeg(sub_b)] + if sub_a == DATA_SUBKEY => + { + if let Ok(index) = storage::KeySeg::parse(sub_b.clone()) { + Ok(Some(SubKey::Data(index))) + } else { + Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result() + } + } + _ => Err(ValidationError::InvalidSubKey(key.clone())) + .into_storage_result(), + } + } + + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>, + { + let change = match sub_key { + SubKey::Len => { + let data = validation::read_data(env, storage_key)?; + data.map(SubKeyWithData::Len) + } + SubKey::Data(index) => { + let data = validation::read_data(env, storage_key)?; + data.map(|data| SubKeyWithData::Data(index, data)) + } + }; + Ok(change) + } + + /// The validation rules for a [`LazyVec`] are: + /// - A difference in the vector's length must correspond to the + /// difference in how many elements were pushed versus how many elements + /// were popped. + /// - An empty vector must be deleted from storage + /// - In addition, we check that indices of any changes are within an + /// expected range (i.e. the vectors indices should always be + /// monotonically increasing from zero) + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result> { + let mut actions = vec![]; + + // We need to accumulate some values for what's changed + let mut post_gt_pre = false; + let mut len_diff: u64 = 0; + let mut len_pre: u64 = 0; + let mut added = BTreeSet::::default(); + let mut updated = BTreeSet::::default(); + let mut deleted = BTreeSet::::default(); + + for key in keys { + match key { + SubKeyWithData::Len(data) => match data { + Data::Add { post } => { + if post == 0 { + return Err( + ValidationError::EmptyVecShouldBeDeleted, + ) + .into_storage_result(); + } + post_gt_pre = true; + len_diff = post; + } + Data::Update { pre, post } => { + if post == 0 { + return Err( + ValidationError::EmptyVecShouldBeDeleted, + ) + .into_storage_result(); + } + if post > pre { + post_gt_pre = true; + len_diff = post - pre; + } else { + len_diff = pre - post; + } + len_pre = pre; + } + Data::Delete { pre } => { + len_diff = pre; + len_pre = pre; + } + }, + SubKeyWithData::Data(index, data) => match data { + Data::Add { post } => { + actions.push(Action::Push(post)); + added.insert(index); + } + Data::Update { pre, post } => { + actions.push(Action::Update { index, pre, post }); + updated.insert(index); + } + Data::Delete { pre } => { + actions.push(Action::Pop(pre)); + deleted.insert(index); + } + }, + } + } + let added_len: u64 = added + .len() + .try_into() + .map_err(ValidationError::IndexOverflow) + .into_storage_result()?; + let deleted_len: u64 = deleted + .len() + .try_into() + .map_err(ValidationError::IndexOverflow) + .into_storage_result()?; + + if len_diff != 0 + && !(if post_gt_pre { + deleted_len + len_diff == added_len + } else { + added_len + len_diff == deleted_len + }) + { + return Err(ValidationError::InvalidLenDiff).into_storage_result(); + } + + let mut last_added = Option::None; + // Iterate additions in increasing order of indices + for index in added { + if let Some(last_added) = last_added { + // Following additions should be at monotonically increasing + // indices + let expected = last_added + 1; + if expected != index { + return Err(ValidationError::UnexpectedPushIndex { + got: index, + expected, + }) + .into_storage_result(); + } + } else if index != len_pre { + // The first addition must be at the pre length value. + // If something is deleted and a new value is added + // in its place, it will go through `Data::Update` + // instead. + return Err(ValidationError::UnexpectedPushIndex { + got: index, + expected: len_pre, + }) + .into_storage_result(); + } + last_added = Some(index); + } + + let mut last_deleted = Option::None; + // Also iterate deletions in increasing order of indices + for index in deleted { + if let Some(last_added) = last_deleted { + // Following deletions should be at monotonically increasing + // indices + let expected = last_added + 1; + if expected != index { + return Err(ValidationError::UnexpectedPopIndex { + got: index, + expected, + }) + .into_storage_result(); + } + } + last_deleted = Some(index); + } + if let Some(index) = last_deleted { + if len_pre > 0 { + let expected = len_pre - 1; + if index != expected { + // The last deletion must be at the pre length value minus 1 + return Err(ValidationError::UnexpectedPopIndex { + got: index, + expected: len_pre, + }) + .into_storage_result(); + } + } + } + + // And finally iterate updates + for index in updated { + // Update index has to be within the length bounds + let max = len_pre + len_diff; + if index >= max { + return Err(ValidationError::UnexpectedUpdateIndex { + got: index, + max, + }) + .into_storage_result(); + } + } + + Ok(actions) + } +} + +// Generic `LazyVec` methods that require no bounds on values `T` +impl LazyVec { + /// Reads the number of elements in the vector. + #[allow(clippy::len_without_is_empty)] + pub fn len(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + let len = storage.read(&self.get_len_key())?; + Ok(len.unwrap_or_default()) + } + + /// Returns `true` if the vector contains no elements. + pub fn is_empty(&self, storage: &S) -> Result + where + S: for<'iter> StorageRead<'iter>, + { + Ok(self.len(storage)? == 0) + } + + /// Get the prefix of set's elements storage + fn get_data_prefix(&self) -> storage::Key { + self.key.push(&DATA_SUBKEY.to_owned()).unwrap() + } + + /// Get the sub-key of vector's elements storage + fn get_data_key(&self, index: Index) -> storage::Key { + self.get_data_prefix().push(&index).unwrap() + } + + /// Get the sub-key of vector's length storage + fn get_len_key(&self) -> storage::Key { + self.key.push(&LEN_SUBKEY.to_owned()).unwrap() + } +} + +// `LazyVec` methods with borsh encoded values `T` +impl LazyVec +where + T: BorshSerialize + BorshDeserialize + 'static, +{ + /// Appends an element to the back of a collection. + pub fn push(&self, storage: &mut S, val: T) -> Result<()> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + let data_key = self.get_data_key(len); + storage.write(&data_key, val)?; + storage.write(&self.get_len_key(), len + 1) + } + + /// Removes the last element from a vector and returns it, or `Ok(None)` if + /// it is empty. + /// + /// Note that an empty vector is completely removed from storage. + pub fn pop(&self, storage: &mut S) -> Result> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + if len == 0 { + Ok(None) + } else { + let index = len - 1; + let data_key = self.get_data_key(index); + if len == 1 { + storage.delete(&self.get_len_key())?; + } else { + storage.write(&self.get_len_key(), index)?; + } + let popped_val = storage.read(&data_key)?; + storage.delete(&data_key)?; + Ok(popped_val) + } + } + + /// Update an element at the given index. + /// + /// The index must be smaller than the length of the vector, otherwise this + /// will fail with `UpdateError::InvalidIndex`. + pub fn update(&self, storage: &mut S, index: Index, val: T) -> Result<()> + where + S: StorageWrite + for<'iter> StorageRead<'iter>, + { + let len = self.len(storage)?; + if index >= len { + return Err(UpdateError::InvalidIndex { index, len }) + .into_storage_result(); + } + let data_key = self.get_data_key(index); + storage.write(&data_key, val) + } + + /// Read an element at the index or `Ok(None)` if out of bounds. + pub fn get(&self, storage: &S, index: Index) -> Result> + where + S: for<'iter> StorageRead<'iter>, + { + storage.read(&self.get_data_key(index)) + } + + /// An iterator visiting all elements. The iterator element type is + /// `Result`, because iterator's call to `next` may fail with e.g. out of + /// gas or data decoding error. + /// + /// Note that this function shouldn't be used in transactions and VPs code + /// on unbounded sets to avoid gas usage increasing with the length of the + /// set. + pub fn iter<'iter>( + &self, + storage: &'iter impl StorageRead<'iter>, + ) -> Result> + 'iter> { + let iter = storage_api::iter_prefix(storage, &self.get_data_prefix())?; + Ok(iter.map(|key_val_res| { + let (_key, val) = key_val_res?; + Ok(val) + })) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::ledger::storage::testing::TestStorage; + + #[test] + fn test_lazy_vec_basics() -> storage_api::Result<()> { + let mut storage = TestStorage::default(); + + let key = storage::Key::parse("test").unwrap(); + let lazy_vec = LazyVec::::open(key); + + // The vec should be empty at first + assert!(lazy_vec.is_empty(&storage)?); + assert!(lazy_vec.len(&storage)? == 0); + assert!(lazy_vec.iter(&storage)?.next().is_none()); + assert!(lazy_vec.pop(&mut storage)?.is_none()); + assert!(lazy_vec.get(&storage, 0)?.is_none()); + assert!(lazy_vec.get(&storage, 1)?.is_none()); + + // Push a new value and check that it's added + lazy_vec.push(&mut storage, 15_u32)?; + assert!(!lazy_vec.is_empty(&storage)?); + assert!(lazy_vec.len(&storage)? == 1); + assert_eq!(lazy_vec.iter(&storage)?.next().unwrap()?, 15_u32); + assert_eq!(lazy_vec.get(&storage, 0)?.unwrap(), 15_u32); + assert!(lazy_vec.get(&storage, 1)?.is_none()); + + // Pop the last value and check that the vec is empty again + let popped = lazy_vec.pop(&mut storage)?.unwrap(); + assert_eq!(popped, 15_u32); + assert!(lazy_vec.is_empty(&storage)?); + assert!(lazy_vec.len(&storage)? == 0); + assert!(lazy_vec.iter(&storage)?.next().is_none()); + assert!(lazy_vec.pop(&mut storage)?.is_none()); + assert!(lazy_vec.get(&storage, 0)?.is_none()); + assert!(lazy_vec.get(&storage, 1)?.is_none()); + + Ok(()) + } +} diff --git a/shared/src/ledger/storage_api/collections/mod.rs b/shared/src/ledger/storage_api/collections/mod.rs new file mode 100644 index 0000000000..688b76bd49 --- /dev/null +++ b/shared/src/ledger/storage_api/collections/mod.rs @@ -0,0 +1,143 @@ +//! Lazy data structures for storage access where elements are not all loaded +//! into memory. This serves to minimize gas costs, avoid unbounded iteration +//! in some cases, and ease the validation of storage changes in VPs. +//! +//! Rather than finding the diff of the state before and after (which requires +//! iteration over both of the states that also have to be decoded), VPs will +//! just receive the storage sub-keys that have experienced changes without +//! having to check any of the unchanged elements. + +use std::fmt::Debug; + +use borsh::BorshDeserialize; +use derivative::Derivative; +use thiserror::Error; + +pub mod lazy_map; +pub mod lazy_vec; + +pub use lazy_map::LazyMap; +pub use lazy_vec::LazyVec; + +use crate::ledger::storage_api; +use crate::ledger::vp_env::VpEnv; +use crate::types::storage; + +#[allow(missing_docs)] +#[derive(Error, Debug)] +pub enum ReadError { + #[error("A storage key was unexpectedly empty")] + UnexpectedlyEmptyStorageKey, +} + +/// Simple lazy collection with borsh deserializable elements +#[derive(Debug)] +pub struct Simple; + +/// Lazy collection with a nested lazy collection +#[derive(Debug)] +pub struct Nested; + +/// A lazy collection of storage values is a handler with some storage prefix +/// that is given to its `fn new()`. The values are typically nested under this +/// prefix and they can be changed individually (e.g. without reading in the +/// whole collection) and their changes directly indicated to the validity +/// predicates, which do not need to iterate the whole collection pre/post to +/// find diffs. +/// +/// An empty collection must be deleted from storage. +pub trait LazyCollection { + /// Actions on the collection determined from changed storage keys by + /// `Self::validate` + type Action; + + /// Possible sub-keys in the collection + type SubKey: Debug; + + /// Possible sub-keys together with the data read from storage + type SubKeyWithData: Debug; + + /// A type of a value in the inner-most collection + type Value: BorshDeserialize; + + /// Create or use an existing vector with the given storage `key`. + fn open(key: storage::Key) -> Self; + + /// Check if the given storage key is a valid LazyVec sub-key and if so + /// return which one. Returns: + /// - `Ok(Some(_))` if it's a valid sub-key + /// - `Ok(None)` if it's not a sub-key + /// - `Err(_)` if it's an invalid sub-key + fn is_valid_sub_key( + &self, + key: &storage::Key, + ) -> storage_api::Result>; + + /// Try to read and decode the data for each change storage key in prior and + /// posterior state. If there is no value in neither prior or posterior + /// state (which is a possible state when transaction e.g. writes and then + /// deletes one storage key, but it is treated as a no-op as it doesn't + /// affect result of validation), returns `Ok(None)`. + fn read_sub_key_data( + env: &ENV, + storage_key: &storage::Key, + sub_key: Self::SubKey, + ) -> storage_api::Result> + where + ENV: for<'a> VpEnv<'a>; + + /// Validate changed sub-keys associated with their data and return back + /// a vector of `Self::Action`s, if the changes are valid + fn validate_changed_sub_keys( + keys: Vec, + ) -> storage_api::Result>; + + /// Accumulate storage changes inside a `ValidationBuilder`. This is + /// typically done by the validity predicate while looping through the + /// changed keys. If the resulting `builder` is not `None`, one must + /// call `fn build()` on it to get the validation result. + /// This function will return `Ok(true)` if the storage key is a valid + /// sub-key of this collection, `Ok(false)` if the storage key doesn't match + /// the prefix of this collection, or error if the prefix matches this + /// collection, but the key itself is not recognized. + fn accumulate( + &self, + env: &ENV, + builder: &mut Option>, + key_changed: &storage::Key, + ) -> storage_api::Result + where + ENV: for<'a> VpEnv<'a>, + { + if let Some(sub) = self.is_valid_sub_key(key_changed)? { + let change = Self::read_sub_key_data(env, key_changed, sub)?; + if let Some(change) = change { + let builder = + builder.get_or_insert(ValidationBuilder::default()); + builder.changes.push(change); + } + return Ok(true); + } + Ok(false) + } + + /// Execute validation on the validation builder, to be called when + /// `accumulate` instantiates the builder to `Some(_)`, after all the + /// changes storage keys have been processed. + fn validate( + builder: ValidationBuilder, + ) -> storage_api::Result> { + Self::validate_changed_sub_keys(builder.changes) + } +} + +/// Validation builder from storage changes. The changes can +/// be accumulated with `LazyCollection::accumulate()` and then turned into a +/// list of valid actions on the collection with `LazyCollection::validate()`. +#[derive(Debug, Derivative)] +// https://mcarton.github.io/rust-derivative/latest/Default.html#custom-bound +#[derivative(Default(bound = ""))] +pub struct ValidationBuilder { + /// The accumulator of found changes under the vector + pub changes: Vec, +} diff --git a/shared/src/ledger/storage_api/mod.rs b/shared/src/ledger/storage_api/mod.rs index 06ec8361f5..b806f35801 100644 --- a/shared/src/ledger/storage_api/mod.rs +++ b/shared/src/ledger/storage_api/mod.rs @@ -1,7 +1,9 @@ //! The common storage read trait is implemented in the storage, client RPC, tx //! and VPs (both native and WASM). +pub mod collections; mod error; +pub mod validation; use borsh::{BorshDeserialize, BorshSerialize}; pub use error::{CustomError, Error, OptionExt, Result, ResultExt}; diff --git a/shared/src/ledger/storage_api/validation/mod.rs b/shared/src/ledger/storage_api/validation/mod.rs new file mode 100644 index 0000000000..ca0e779a75 --- /dev/null +++ b/shared/src/ledger/storage_api/validation/mod.rs @@ -0,0 +1,54 @@ +//! Storage change validation helpers + +use std::fmt::Debug; + +use borsh::BorshDeserialize; + +use crate::ledger::storage_api; +use crate::ledger::vp_env::VpEnv; +use crate::types::storage; + +/// Data update with prior and posterior state. +#[derive(Clone, Debug)] +pub enum Data { + /// Newly added value + Add { + /// Posterior state + post: T, + }, + /// Updated value prior and posterior state + Update { + /// Prior state + pre: T, + /// Posterior state + post: T, + }, + /// Deleted value + Delete { + /// Prior state + pre: T, + }, +} + +/// Read the prior and posterior state for the given key. +pub fn read_data( + env: &ENV, + key: &storage::Key, +) -> Result>, storage_api::Error> +where + T: BorshDeserialize, + ENV: for<'a> VpEnv<'a>, +{ + let pre = env.read_pre(key)?; + let post = env.read_post(key)?; + Ok(match (pre, post) { + (None, None) => { + // If the key was inserted and then deleted in the same tx, we don't + // need to validate it as it's not visible to any VPs + None + } + (None, Some(post)) => Some(Data::Add { post }), + (Some(pre), None) => Some(Data::Delete { pre }), + (Some(pre), Some(post)) => Some(Data::Update { pre, post }), + }) +} diff --git a/shared/src/types/storage.rs b/shared/src/types/storage.rs index 6cc86f099e..c6b7f9dfbf 100644 --- a/shared/src/types/storage.rs +++ b/shared/src/types/storage.rs @@ -214,6 +214,13 @@ pub struct Key { pub segments: Vec, } +/// A [`Key`] made of borrowed key segments [`DbKeySeg`]. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub struct KeyRef<'a> { + /// Reference of key segments + pub segments: &'a [DbKeySeg], +} + impl From for Key { fn from(seg: DbKeySeg) -> Self { Self { @@ -371,7 +378,24 @@ impl Key { self.len() == 0 } - /// Returns a key of the validity predicate of the given address. + /// Returns the first segment of the key, or `None` if it is empty. + pub fn first(&self) -> Option<&DbKeySeg> { + self.segments.first() + } + + /// Returns the last segment of the key, or `None` if it is empty. + pub fn last(&self) -> Option<&DbKeySeg> { + self.segments.last() + } + + /// Returns the prefix before the last segment and last segment of the key, + /// or `None` if it is empty. + pub fn split_last(&self) -> Option<(KeyRef<'_>, &DbKeySeg)> { + let (last, prefix) = self.segments.split_last()?; + Some((KeyRef { segments: prefix }, last)) + } + + /// Returns a key of the validity predicate of the given address /// Only this function can push "?" segment for validity predicate pub fn validity_predicate(addr: &Address) -> Self { let mut segments = Self::from(addr.to_db_key()).segments; @@ -414,8 +438,11 @@ impl Key { .split_off(2) .join(&KEY_SEGMENT_SEPARATOR.to_string()), ) - .map_err(|e| Error::Temporary { - error: format!("Cannot parse key segments {}: {}", db_key, e), + .map_err(|e| { + Error::ParseKeySeg(format!( + "Cannot parse key segments {}: {}", + db_key, e + )) })?, }; Ok(key) @@ -443,6 +470,28 @@ impl Key { }), } } + + /// Check if the key begins with the given prefix and returns: + /// - `Some(Some(suffix))` the suffix after the match with, if any, or + /// - `Some(None)` if the prefix is matched, but it has no suffix, or + /// - `None` if it doesn't match + pub fn split_prefix(&self, prefix: &Self) -> Option> { + if self.segments.len() < prefix.segments.len() { + return None; + } else if self == prefix { + return Some(None); + } + // This is safe, because we check that the length of segments in self >= + // in prefix above + let (self_prefix, rest) = self.segments.split_at(prefix.segments.len()); + if self_prefix == prefix.segments { + Some(Some(Key { + segments: rest.to_vec(), + })) + } else { + None + } + } } impl Display for Key { @@ -457,6 +506,20 @@ impl Display for Key { } } +impl KeyRef<'_> { + /// Check if [`KeyRef`] is equal to a [`Key`]. + pub fn eq_owned(&self, other: &Key) -> bool { + self.segments == other.segments + } + + /// Returns the prefix before the last segment and last segment of the key, + /// or `None` if it is empty. + pub fn split_last(&self) -> Option<(KeyRef<'_>, &DbKeySeg)> { + let (last, prefix) = self.segments.split_last()?; + Some((KeyRef { segments: prefix }, last)) + } +} + // TODO use std::convert::{TryFrom, Into}? /// Represents a segment in a path that may be used as a database key pub trait KeySeg { @@ -543,7 +606,12 @@ impl KeySeg for String { impl KeySeg for BlockHeight { fn parse(string: String) -> Result { - let h: u64 = KeySeg::parse(string)?; + let h = string.parse::().map_err(|e| { + Error::ParseKeySeg(format!( + "Unexpected height value {}, {}", + string, e + )) + })?; Ok(BlockHeight(h)) } diff --git a/tests/proptest-regressions/storage_api/collections/lazy_map.txt b/tests/proptest-regressions/storage_api/collections/lazy_map.txt new file mode 100644 index 0000000000..2de7510923 --- /dev/null +++ b/tests/proptest-regressions/storage_api/collections/lazy_map.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 59b8eaaf5d8e03e58b346ef229a2487f68fea488197420f150682f7275ce2b83 # shrinks to (initial_state, transitions) = (AbstractLazyMapState { valid_transitions: [], committed_transitions: [] }, [Insert(11178241982156558453, TestVal { x: 9618691367534591266, y: true }), CommitTx, Update(11178241982156558453, TestVal { x: 2635377083098935189, y: false }), Update(11178241982156558453, TestVal { x: 11485387163946255361, y: false }), Insert(4380901092919801530, TestVal { x: 17235291421018840542, y: false }), Update(11178241982156558453, TestVal { x: 1936190700145956620, y: false }), Update(11178241982156558453, TestVal { x: 6934621224353358508, y: false }), Update(11178241982156558453, TestVal { x: 16175036327810390362, y: true }), Remove(5606457884982633480), Insert(7124206407862523505, TestVal { x: 5513772825695605555, y: true }), CommitTxAndBlock, CommitTx, Insert(13347045100814804679, TestVal { x: 5157295776286367034, y: false }), Update(7124206407862523505, TestVal { x: 1989909525753197955, y: false }), Update(4380901092919801530, TestVal { x: 13085578877588425331, y: false }), Update(7124206407862523505, TestVal { x: 1620781139263176467, y: true }), Insert(5806457332157050619, TestVal { x: 14632354209749334932, y: true }), Remove(1613213961397167063), Update(7124206407862523505, TestVal { x: 3848976302483310370, y: true }), Update(4380901092919801530, TestVal { x: 15281186775251770467, y: false }), Remove(5303306623647571548), Insert(5905425607805327902, TestVal { x: 1274794101048822414, y: false }), Insert(2305446651611241243, TestVal { x: 7872403441503057017, y: true }), Insert(2843165193114615911, TestVal { x: 13698490566286768452, y: false }), Insert(3364298091459048760, TestVal { x: 8891279000465212397, y: true }), CommitTx, Insert(17278527568142155478, TestVal { x: 8166151895050476136, y: false }), Remove(9206713523174765253), Remove(1148985045479283759), Insert(13346103305566843535, TestVal { x: 13148026974798633058, y: true }), Remove(17185699086139524651), CommitTx, Update(7124206407862523505, TestVal { x: 3047872255943216792, y: false }), CommitTxAndBlock, CommitTxAndBlock, Remove(4672009405538026945), Update(5905425607805327902, TestVal { x: 6635343936644805461, y: false }), Insert(14100441716981493843, TestVal { x: 8068697312326956479, y: true }), Insert(8370580326875672309, TestVal { x: 18416630552728813406, y: false }), Update(2305446651611241243, TestVal { x: 3777718192999015176, y: false }), Remove(1532142753559370584), Remove(10097030807802775125), Insert(10080356901530935857, TestVal { x: 17171047520093964037, y: false }), Update(3364298091459048760, TestVal { x: 702372485798608773, y: true }), Insert(5504969092734638033, TestVal { x: 314752460808087203, y: true }), Remove(5486040497128339175), Insert(7884678026881625058, TestVal { x: 4313610278903495077, y: true }), CommitTx, Insert(11228024342874184864, TestVal { x: 428512502841968552, y: false }), Insert(4684666745142518471, TestVal { x: 13122515680485564107, y: true }), Remove(14243063045921130600), Remove(4530767959521683042), Insert(10236349778753659715, TestVal { x: 3138294567956031715, y: true }), Update(2305446651611241243, TestVal { x: 8133236604817109805, y: false }), Update(2843165193114615911, TestVal { x: 12001998927296899868, y: false }), CommitTxAndBlock, CommitTx, CommitTxAndBlock]) diff --git a/tests/proptest-regressions/storage_api/collections/lazy_vec.txt b/tests/proptest-regressions/storage_api/collections/lazy_vec.txt new file mode 100644 index 0000000000..97a16dcbeb --- /dev/null +++ b/tests/proptest-regressions/storage_api/collections/lazy_vec.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 4330a283e32b5ff3f38d0af2298e1e98c30b1901c1027b572070a1af3356688e # shrinks to (initial_state, transitions) = (AbstractLazyVecState { valid_transitions: [], committed_transitions: [] }, [Push(TestVecItem { x: 15352583996758053781, y: true }), Pop, CommitTx, Push(TestVecItem { x: 6904067244182623445, y: false }), CommitTx, Pop, Push(TestVecItem { x: 759762287021483883, y: true }), Push(TestVecItem { x: 7885704082671389345, y: true }), Pop, Pop, Push(TestVecItem { x: 2762344561419437403, y: false }), Push(TestVecItem { x: 11448034977049028254, y: false }), Update { index: 0, value: TestVecItem { x: 7097339541298715775, y: false } }, Pop, Pop, Push(TestVecItem { x: 457884036257686887, y: true }), CommitTx, Push(TestVecItem { x: 17719281119971095810, y: true }), CommitTx, Push(TestVecItem { x: 4612681906563857058, y: false }), CommitTx, CommitTx, Pop, CommitTx, Pop, Push(TestVecItem { x: 4269537158299505726, y: false }), CommitTx, Pop, Pop, CommitTx, CommitTx, CommitTx, CommitTx, Push(TestVecItem { x: 9020889554694833528, y: true }), Push(TestVecItem { x: 4022797489860699620, y: false }), Update { index: 0, value: TestVecItem { x: 6485081152860611495, y: true } }, Pop, CommitTx, Push(TestVecItem { x: 14470031031894733310, y: false }), Push(TestVecItem { x: 1113274973965556867, y: true }), Push(TestVecItem { x: 4122902042678339346, y: false }), Push(TestVecItem { x: 9672639635189564637, y: true }), Pop, Pop, Pop, CommitTx, Update { index: 0, value: TestVecItem { x: 6372193991838429158, y: false } }, Push(TestVecItem { x: 15140852824102579010, y: false }), Pop, Pop, Pop, Push(TestVecItem { x: 4012218522073776592, y: false }), Push(TestVecItem { x: 10637893847792386454, y: true }), Push(TestVecItem { x: 3357788278949652885, y: false }), CommitTx, CommitTx, Pop, Pop, CommitTx, Pop, Push(TestVecItem { x: 11768518086398350214, y: true }), Push(TestVecItem { x: 4361685178396183644, y: true }), Pop, CommitTx, Push(TestVecItem { x: 2450907664540456425, y: false }), Push(TestVecItem { x: 18184919885943118586, y: true }), Update { index: 1, value: TestVecItem { x: 10611906658537706503, y: false } }, Push(TestVecItem { x: 4887827541279511396, y: false }), Update { index: 0, value: TestVecItem { x: 13021774003761931172, y: false } }, Push(TestVecItem { x: 3644118228573898014, y: false }), CommitTx, Update { index: 0, value: TestVecItem { x: 1276840798381751183, y: false } }, Pop, Pop]) diff --git a/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt b/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt new file mode 100644 index 0000000000..d587a9680e --- /dev/null +++ b/tests/proptest-regressions/storage_api/collections/nested_lazy_map.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc b5ce7502439712f95a4b50de0d5455e0a6788cc95dbd535e749d5717da0ee8e1 # shrinks to (initial_state, transitions) = (AbstractLazyMapState { valid_transitions: [], committed_transitions: [] }, [Insert((22253647846329582, -2060910714, -85), TestVal { x: 16862967849328560500, y: true })]) diff --git a/tests/src/lib.rs b/tests/src/lib.rs index f1d2b812bc..c993b4b72c 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -11,6 +11,8 @@ pub use vm_host_env::{ibc, tx, vp}; mod e2e; pub mod native_vp; pub mod storage; +#[cfg(test)] +mod storage_api; /// Using this import requires `tracing` and `tracing-subscriber` dependencies. /// Set env var `RUST_LOG=info` to see the logs from a test run (and diff --git a/tests/src/storage_api/collections/lazy_map.rs b/tests/src/storage_api/collections/lazy_map.rs new file mode 100644 index 0000000000..afff09bbf1 --- /dev/null +++ b/tests/src/storage_api/collections/lazy_map.rs @@ -0,0 +1,613 @@ +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::convert::TryInto; + + use borsh::{BorshDeserialize, BorshSerialize}; + use namada::types::address::{self, Address}; + use namada::types::storage; + use namada_tx_prelude::storage::KeySeg; + use namada_tx_prelude::storage_api::collections::{ + lazy_map, LazyCollection, LazyMap, + }; + use proptest::prelude::*; + use proptest::prop_state_machine; + use proptest::state_machine::{AbstractStateMachine, StateMachineTest}; + use proptest::test_runner::Config; + use test_log::test; + + use crate::tx::tx_host_env; + use crate::vp::vp_host_env; + + prop_state_machine! { + #![proptest_config(Config { + // Instead of the default 256, we only run 5 because otherwise it + // takes too long and it's preferable to crank up the number of + // transitions instead, to allow each case to run for more epochs as + // some issues only manifest once the model progresses further. + // Additionally, more cases will be explored every time this test is + // executed in the CI. + cases: 5, + .. Config::default() + })] + #[test] + fn lazy_map_api_state_machine_test(sequential 1..100 => ConcreteLazyMapState); + } + + /// Type of key used in the map + type TestKey = u64; + + /// Some borsh-serializable type with arbitrary fields to be used inside + /// LazyMap state machine test + #[derive( + Clone, + Debug, + BorshSerialize, + BorshDeserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + struct TestVal { + x: u64, + y: bool, + } + + /// A `StateMachineTest` implemented on this struct manipulates it with + /// `Transition`s, which are also being accumulated into + /// `current_transitions`. It then: + /// + /// - checks its state against an in-memory `std::collections::HashMap` + /// - runs validation and checks that the `LazyMap::Action`s reported from + /// validation match with transitions that were applied + /// + /// Additionally, one of the transitions is to commit a block and/or + /// transaction, during which the currently accumulated state changes are + /// persisted, or promoted from transaction write log to block's write log. + #[derive(Debug)] + struct ConcreteLazyMapState { + /// Address is used to prefix the storage key of the `lazy_map` in + /// order to simulate a transaction and a validity predicate + /// check from changes on the `lazy_map` + address: Address, + /// In the test, we apply the same transitions on the `lazy_map` as on + /// `eager_map` to check that `lazy_map`'s state is consistent with + /// `eager_map`. + eager_map: BTreeMap, + /// Handle to a lazy map + lazy_map: LazyMap, + /// Valid LazyMap changes in the current transaction + current_transitions: Vec, + } + + #[derive(Clone, Debug, Default)] + struct AbstractLazyMapState { + /// Valid LazyMap changes in the current transaction + valid_transitions: Vec, + /// Valid LazyMap changes committed to storage + committed_transitions: Vec, + } + + /// Possible transitions that can modify a [`LazyMap`]. + /// This roughly corresponds to the methods that have `StorageWrite` + /// access and is very similar to [`Action`] + #[derive(Clone, Debug)] + enum Transition { + /// Commit all valid transitions in the current transaction + CommitTx, + /// Commit all valid transitions in the current transaction and also + /// commit the current block + CommitTxAndBlock, + /// Insert a key-val into a [`LazyMap`] + Insert(TestKey, TestVal), + /// Remove a key-val from a [`LazyMap`] + Remove(TestKey), + /// Update a value at key from pre to post state in a + /// [`LazyMap`] + Update(TestKey, TestVal), + } + + impl AbstractStateMachine for AbstractLazyMapState { + type State = Self; + type Transition = Transition; + + fn init_state() -> BoxedStrategy { + Just(Self::default()).boxed() + } + + // Apply a random transition to the state + fn transitions(state: &Self::State) -> BoxedStrategy { + let length = state.len(); + if length == 0 { + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_map_key(), arb_map_val()).prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } else { + let keys = state.find_existing_keys(); + let arb_existing_map_key = + || proptest::sample::select(keys.clone()); + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_existing_map_key(), arb_map_val()).prop_map(|(key, val)| + Transition::Update(key, val) + ), + 3 => arb_existing_map_key().prop_map(Transition::Remove), + 5 => (arb_map_key().prop_filter("insert on non-existing keys only", + move |key| !keys.contains(key)), arb_map_val()) + .prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } + } + + fn apply_abstract( + mut state: Self::State, + transition: &Self::Transition, + ) -> Self::State { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + let valid_actions_to_commit = + std::mem::take(&mut state.valid_transitions); + state + .committed_transitions + .extend(valid_actions_to_commit.into_iter()); + } + _ => state.valid_transitions.push(transition.clone()), + } + state + } + + fn preconditions( + state: &Self::State, + transition: &Self::Transition, + ) -> bool { + let length = state.len(); + // Ensure that the remove or update transitions are not applied + // to an empty state + if length == 0 + && matches!( + transition, + Transition::Remove(_) | Transition::Update(_, _) + ) + { + return false; + } + match transition { + Transition::Update(key, _) | Transition::Remove(key) => { + let keys = state.find_existing_keys(); + // Ensure that the update/remove key is an existing one + keys.contains(key) + } + Transition::Insert(key, _) => { + let keys = state.find_existing_keys(); + // Ensure that the insert key is not an existing one + !keys.contains(key) + } + _ => true, + } + } + } + + impl StateMachineTest for ConcreteLazyMapState { + type Abstract = AbstractLazyMapState; + type ConcreteState = Self; + + fn init_test( + _initial_state: ::State, + ) -> Self::ConcreteState { + // Init transaction env in which we'll be applying the transitions + tx_host_env::init(); + + // The lazy_map's path must be prefixed by the address to be able + // to trigger a validity predicate on it + let address = address::testing::established_address_1(); + tx_host_env::with(|env| env.spawn_accounts([&address])); + let lazy_map_prefix: storage::Key = address.to_db_key().into(); + + Self { + address, + eager_map: BTreeMap::new(), + lazy_map: LazyMap::open( + lazy_map_prefix.push(&"arbitrary".to_string()).unwrap(), + ), + current_transitions: vec![], + } + } + + fn apply_concrete( + mut state: Self::ConcreteState, + transition: ::Transition, + ) -> Self::ConcreteState { + // Apply transitions in transaction env + let ctx = tx_host_env::ctx(); + + // Persist the transitions in the current tx, or clear previous ones + // if we're committing a tx + match &transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + state.current_transitions = vec![]; + } + _ => { + state.current_transitions.push(transition.clone()); + } + } + + // Transition application on lazy map and post-conditions: + match &transition { + Transition::CommitTx => { + // commit the tx without committing the block + tx_host_env::with(|env| env.write_log.commit_tx()); + } + Transition::CommitTxAndBlock => { + // commit the tx and the block + tx_host_env::commit_tx_and_block(); + } + Transition::Insert(key, value) => { + state.lazy_map.insert(ctx, *key, value.clone()).unwrap(); + + // Post-conditions: + let stored_value = + state.lazy_map.get(ctx, key).unwrap().unwrap(); + assert_eq!( + &stored_value, value, + "the new item must be added to the back" + ); + + state.assert_validation_accepted(); + } + Transition::Remove(key) => { + let removed = + state.lazy_map.remove(ctx, key).unwrap().unwrap(); + + // Post-conditions: + assert_eq!( + &removed, + state.eager_map.get(key).unwrap(), + "removed element matches the value in eager map \ + before it's updated" + ); + + state.assert_validation_accepted(); + } + Transition::Update(key, value) => { + let old_val = + state.lazy_map.get(ctx, key).unwrap().unwrap(); + + state.lazy_map.insert(ctx, *key, value.clone()).unwrap(); + + // Post-conditions: + let new_val = + state.lazy_map.get(ctx, key).unwrap().unwrap(); + assert_eq!( + &old_val, + state.eager_map.get(key).unwrap(), + "old value must match the value at the same key in \ + the eager map before it's updated" + ); + assert_eq!( + &new_val, value, + "new value must match that which was passed into the \ + Transition::Update" + ); + + state.assert_validation_accepted(); + } + } + + // Apply transition in the eager map for comparison + apply_transition_on_eager_map(&mut state.eager_map, &transition); + + // Global post-conditions: + + // All items in eager map must be present in lazy map + for (key, expected_item) in state.eager_map.iter() { + let got = + state.lazy_map.get(ctx, key).unwrap().expect( + "The expected item must be present in lazy map", + ); + assert_eq!(expected_item, &got, "at key {key}"); + } + + // All items in lazy map must be present in eager map + for key_val in state.lazy_map.iter(ctx).unwrap() { + let (key, expected_val) = key_val.unwrap(); + let got = state + .eager_map + .get(&key) + .expect("The expected item must be present in eager map"); + assert_eq!(&expected_val, got, "at key {key}"); + } + + state + } + } + + impl AbstractLazyMapState { + /// Find the length of the map from the applied transitions + fn len(&self) -> u64 { + (map_len_diff_from_transitions(self.committed_transitions.iter()) + + map_len_diff_from_transitions(self.valid_transitions.iter())) + .try_into() + .expect( + "It shouldn't be possible to underflow length from all \ + transactions applied in abstract state", + ) + } + + /// Build an eager map from the committed and current transitions + fn eager_map(&self) -> BTreeMap { + let mut eager_map = BTreeMap::new(); + for transition in &self.committed_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + for transition in &self.valid_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + eager_map + } + + /// Find the keys currently present in the map + fn find_existing_keys(&self) -> Vec { + self.eager_map().keys().cloned().collect() + } + } + + /// Find the difference in length of the map from the applied transitions + fn map_len_diff_from_transitions<'a>( + transitions: impl Iterator, + ) -> i64 { + let mut insert_count: i64 = 0; + let mut remove_count: i64 = 0; + + for trans in transitions { + match trans { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Update(_, _) => {} + Transition::Insert(_, _) => insert_count += 1, + Transition::Remove(_) => remove_count += 1, + } + } + insert_count - remove_count + } + + impl ConcreteLazyMapState { + fn assert_validation_accepted(&self) { + // Init the VP env from tx env in which we applied the map + // transitions + let tx_env = tx_host_env::take(); + vp_host_env::init_from_tx(self.address.clone(), tx_env, |_| {}); + + // Simulate a validity predicate run using the lazy map's validation + // helpers + let changed_keys = + vp_host_env::with(|env| env.all_touched_storage_keys()); + + let mut validation_builder = None; + + // Push followed by pop is a no-op, in which case we'd still see the + // changed keys for these actions, but they wouldn't affect the + // validation result and they never get persisted, but we'd still + // them as changed key here. To guard against this case, + // we check that `map_len_from_transitions` is not empty. + let map_len_diff = + map_len_diff_from_transitions(self.current_transitions.iter()); + + // To help debug validation issues... + dbg!( + &self.current_transitions, + &changed_keys + .iter() + .map(storage::Key::to_string) + .collect::>() + ); + + for key in &changed_keys { + let is_sub_key = self + .lazy_map + .accumulate( + vp_host_env::ctx(), + &mut validation_builder, + key, + ) + .unwrap(); + + assert!( + is_sub_key, + "We're only modifying the lazy_map's keys here. Key: \ + \"{key}\", map length diff {map_len_diff}" + ); + } + if !changed_keys.is_empty() && map_len_diff != 0 { + assert!( + validation_builder.is_some(), + "If some keys were changed, the builder must get filled in" + ); + let actions = LazyMap::::validate( + validation_builder.unwrap(), + ) + .unwrap(); + let mut actions_to_check = actions.clone(); + + // Check that every transition has a corresponding action from + // validation. We drop the found actions to check that all + // actions are matched too. + let current_transitions = + normalize_transitions(&self.current_transitions); + for transition in ¤t_transitions { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + } + Transition::Insert(expected_key, expected_val) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let lazy_map::Action::Insert(key, val) = + action + { + if expected_key == key + && expected_val == val + { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Remove(expected_key) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let lazy_map::Action::Remove(key, _val) = + action + { + if expected_key == key { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Update(expected_key, value) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let lazy_map::Action::Update { + key, + pre: _, + post, + } = action + { + if expected_key == key && post == value { + actions_to_check.remove(ix); + break; + } + } + } + } + } + } + + assert!( + actions_to_check.is_empty(), + "All the actions reported from validation {actions:#?} \ + should have been matched with SM transitions \ + {current_transitions:#?}, but these actions didn't \ + match: {actions_to_check:#?}", + ) + } + + // Put the tx_env back before checking the result + tx_host_env::set_from_vp_env(vp_host_env::take()); + } + } + + /// Generate an arbitrary `TestKey` + fn arb_map_key() -> impl Strategy { + any::() + } + + /// Generate an arbitrary `TestVal` + fn arb_map_val() -> impl Strategy { + (any::(), any::()).prop_map(|(x, y)| TestVal { x, y }) + } + + /// Apply `Transition` on an eager `Map`. + fn apply_transition_on_eager_map( + map: &mut BTreeMap, + transition: &Transition, + ) { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => {} + Transition::Insert(key, value) => { + map.insert(*key, value.clone()); + } + Transition::Remove(key) => { + let _popped = map.remove(key); + } + Transition::Update(key, value) => { + let entry = map.get_mut(key).unwrap(); + *entry = value.clone(); + } + } + } + + /// Normalize transitions: + /// - remove(key) + insert(key, val) -> update(key, val) + /// - insert(key, val) + update(key, new_val) -> insert(key, new_val) + /// - update(key, val) + update(key, new_val) -> update(key, new_val) + /// + /// Note that the normalizable transitions pairs do not have to be directly + /// next to each other, but their order does matter. + fn normalize_transitions(transitions: &[Transition]) -> Vec { + let mut collapsed = vec![]; + 'outer: for transition in transitions { + match transition { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Remove(_) => collapsed.push(transition.clone()), + Transition::Insert(key, val) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Remove(remove_key) = + collapsed_transition + { + if key == remove_key { + // remove(key) + insert(key, val) -> update(key, + // val) + + // Replace the Remove with an Update instead of + // inserting the Insert + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, val.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + Transition::Update(key, value) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Insert(insert_key, _) = + collapsed_transition + { + if key == insert_key { + // insert(key, val) + update(key, new_val) -> + // insert(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Insert(*key, value.clone()); + continue 'outer; + } + } else if let Transition::Update(update_key, _) = + collapsed_transition + { + if key == update_key { + // update(key, val) + update(key, new_val) -> + // update(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, value.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + } + } + collapsed + } +} diff --git a/tests/src/storage_api/collections/lazy_vec.rs b/tests/src/storage_api/collections/lazy_vec.rs new file mode 100644 index 0000000000..65e08b4ca7 --- /dev/null +++ b/tests/src/storage_api/collections/lazy_vec.rs @@ -0,0 +1,634 @@ +#[cfg(test)] +mod tests { + use std::convert::TryInto; + + use borsh::{BorshDeserialize, BorshSerialize}; + use namada::types::address::{self, Address}; + use namada::types::storage; + use namada_tx_prelude::storage::KeySeg; + use namada_tx_prelude::storage_api::collections::{ + lazy_vec, LazyCollection, LazyVec, + }; + use proptest::prelude::*; + use proptest::prop_state_machine; + use proptest::state_machine::{AbstractStateMachine, StateMachineTest}; + use proptest::test_runner::Config; + use test_log::test; + + use crate::tx::tx_host_env; + use crate::vp::vp_host_env; + + prop_state_machine! { + #![proptest_config(Config { + // Instead of the default 256, we only run 5 because otherwise it + // takes too long and it's preferable to crank up the number of + // transitions instead, to allow each case to run for more epochs as + // some issues only manifest once the model progresses further. + // Additionally, more cases will be explored every time this test is + // executed in the CI. + cases: 5, + .. Config::default() + })] + #[test] + fn lazy_vec_api_state_machine_test(sequential 1..100 => ConcreteLazyVecState); + } + + /// Some borsh-serializable type with arbitrary fields to be used inside + /// LazyVec state machine test + #[derive( + Clone, + Debug, + BorshSerialize, + BorshDeserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + struct TestVecItem { + x: u64, + y: bool, + } + + /// A `StateMachineTest` implemented on this struct manipulates it with + /// `Transition`s, which are also being accumulated into + /// `current_transitions`. It then: + /// + /// - checks its state against an in-memory `std::collections::Vec` + /// - runs validation and checks that the `LazyVec::Action`s reported from + /// validation match with transitions that were applied + /// + /// Additionally, one of the transitions is to commit a block and/or + /// transaction, during which the currently accumulated state changes are + /// persisted, or promoted from transaction write log to block's write log. + #[derive(Debug)] + struct ConcreteLazyVecState { + /// Address is used to prefix the storage key of the `lazy_vec` in + /// order to simulate a transaction and a validity predicate + /// check from changes on the `lazy_vec` + address: Address, + /// In the test, we apply the same transitions on the `lazy_vec` as on + /// `eager_vec` to check that `lazy_vec`'s state is consistent with + /// `eager_vec`. + eager_vec: Vec, + /// Handle to a lazy vec + lazy_vec: LazyVec, + /// Valid LazyVec changes in the current transaction + current_transitions: Vec>, + } + + #[derive(Clone, Debug)] + struct AbstractLazyVecState { + /// Valid LazyVec changes in the current transaction + valid_transitions: Vec>, + /// Valid LazyVec changes committed to storage + committed_transitions: Vec>, + } + + /// Possible transitions that can modify a [`LazyVec`]. This roughly + /// corresponds to the methods that have `StorageWrite` access and is very + /// similar to [`Action`] + #[derive(Clone, Debug)] + pub enum Transition { + /// Commit all valid transitions in the current transaction + CommitTx, + /// Commit all valid transitions in the current transaction and also + /// commit the current block + CommitTxAndBlock, + /// Push a value `T` into a [`LazyVec`] + Push(T), + /// Pop a value from a [`LazyVec`] + Pop, + /// Update a value `T` at index from pre to post state in a + /// [`LazyVec`] + Update { + /// index at which the value is updated + index: lazy_vec::Index, + /// value to update the element to + value: T, + }, + } + + impl AbstractStateMachine for AbstractLazyVecState { + type State = Self; + type Transition = Transition; + + fn init_state() -> BoxedStrategy { + Just(Self { + valid_transitions: vec![], + committed_transitions: vec![], + }) + .boxed() + } + + // Apply a random transition to the state + fn transitions(state: &Self::State) -> BoxedStrategy { + let length = state.len(); + if length == 0 { + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => arb_test_vec_item().prop_map(Transition::Push) + ] + .boxed() + } else { + let arb_index = || { + let indices: Vec = (0..length).collect(); + proptest::sample::select(indices) + }; + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_index(), arb_test_vec_item()).prop_map( + |(index, value)| Transition::Update { index, value } + ), + 3 => Just(Transition::Pop), + 5 => arb_test_vec_item().prop_map(Transition::Push), + ] + .boxed() + } + } + + fn apply_abstract( + mut state: Self::State, + transition: &Self::Transition, + ) -> Self::State { + match transition { + Transition::CommitTx => { + let valid_actions_to_commit = + std::mem::take(&mut state.valid_transitions); + state + .committed_transitions + .extend(valid_actions_to_commit.into_iter()); + } + _ => state.valid_transitions.push(transition.clone()), + } + state + } + + fn preconditions( + state: &Self::State, + transition: &Self::Transition, + ) -> bool { + let length = state.len(); + if length == 0 { + // Ensure that the pop or update transitions are not applied to + // an empty state + !matches!( + transition, + Transition::Pop | Transition::Update { .. } + ) + } else if let Transition::Update { index, .. } = transition { + // Ensure that the update index is a valid one + *index < (length - 1) + } else { + true + } + } + } + + impl StateMachineTest for ConcreteLazyVecState { + type Abstract = AbstractLazyVecState; + type ConcreteState = Self; + + fn init_test( + _initial_state: ::State, + ) -> Self::ConcreteState { + // Init transaction env in which we'll be applying the transitions + tx_host_env::init(); + + // The lazy_vec's path must be prefixed by the address to be able + // to trigger a validity predicate on it + let address = address::testing::established_address_1(); + tx_host_env::with(|env| env.spawn_accounts([&address])); + let lazy_vec_prefix: storage::Key = address.to_db_key().into(); + + Self { + address, + eager_vec: vec![], + lazy_vec: LazyVec::open( + lazy_vec_prefix.push(&"arbitrary".to_string()).unwrap(), + ), + current_transitions: vec![], + } + } + + fn apply_concrete( + mut state: Self::ConcreteState, + transition: ::Transition, + ) -> Self::ConcreteState { + // Apply transitions in transaction env + let ctx = tx_host_env::ctx(); + + // Persist the transitions in the current tx, or clear previous ones + // if we're committing a tx + match &transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + state.current_transitions = vec![]; + } + _ => { + state.current_transitions.push(transition.clone()); + } + } + + // Transition application on lazy vec and post-conditions: + match &transition { + Transition::CommitTx => { + // commit the tx without committing the block + tx_host_env::with(|env| env.write_log.commit_tx()); + } + Transition::CommitTxAndBlock => { + // commit the tx and the block + tx_host_env::commit_tx_and_block(); + } + Transition::Push(value) => { + let old_len = state.lazy_vec.len(ctx).unwrap(); + + state.lazy_vec.push(ctx, value.clone()).unwrap(); + + // Post-conditions: + let new_len = state.lazy_vec.len(ctx).unwrap(); + let stored_value = + state.lazy_vec.get(ctx, new_len - 1).unwrap().unwrap(); + assert_eq!( + &stored_value, value, + "the new item must be added to the back" + ); + assert_eq!(old_len + 1, new_len, "length must increment"); + + state.assert_validation_accepted(new_len); + } + Transition::Pop => { + let old_len = state.lazy_vec.len(ctx).unwrap(); + + let popped = state.lazy_vec.pop(ctx).unwrap().unwrap(); + + // Post-conditions: + let new_len = state.lazy_vec.len(ctx).unwrap(); + assert_eq!(old_len, new_len + 1, "length must decrement"); + assert_eq!( + &popped, + state.eager_vec.last().unwrap(), + "popped element matches the last element in eager vec \ + before it's updated" + ); + + state.assert_validation_accepted(new_len); + } + Transition::Update { index, value } => { + let old_len = state.lazy_vec.len(ctx).unwrap(); + let old_val = + state.lazy_vec.get(ctx, *index).unwrap().unwrap(); + + state.lazy_vec.update(ctx, *index, value.clone()).unwrap(); + + // Post-conditions: + let new_len = state.lazy_vec.len(ctx).unwrap(); + let new_val = + state.lazy_vec.get(ctx, *index).unwrap().unwrap(); + assert_eq!(old_len, new_len, "length must not change"); + assert_eq!( + &old_val, + state.eager_vec.get(*index as usize).unwrap(), + "old value must match the value at the same index in \ + the eager vec before it's updated" + ); + assert_eq!( + &new_val, value, + "new value must match that which was passed into the \ + Transition::Update" + ); + + state.assert_validation_accepted(new_len); + } + } + + // Apply transition in the eager vec for comparison + apply_transition_on_eager_vec(&mut state.eager_vec, &transition); + + // Global post-conditions: + + // All items in eager vec must be present in lazy vec + for (ix, expected_item) in state.eager_vec.iter().enumerate() { + let got = state + .lazy_vec + .get(ctx, ix as lazy_vec::Index) + .unwrap() + .expect("The expected item must be present in lazy vec"); + assert_eq!(expected_item, &got, "at index {ix}"); + } + + // All items in lazy vec must be present in eager vec + for (ix, expected_item) in + state.lazy_vec.iter(ctx).unwrap().enumerate() + { + let expected_item = expected_item.unwrap(); + let got = state + .eager_vec + .get(ix) + .expect("The expected item must be present in eager vec"); + assert_eq!(&expected_item, got, "at index {ix}"); + } + + state + } + } + + impl AbstractLazyVecState { + /// Find the length of the vector from the applied transitions + fn len(&self) -> u64 { + (vec_len_diff_from_transitions(self.committed_transitions.iter()) + + vec_len_diff_from_transitions(self.valid_transitions.iter())) + .try_into() + .expect( + "It shouldn't be possible to underflow length from all \ + transactions applied in abstract state", + ) + } + } + + /// Find the difference in length of the vector from the applied transitions + fn vec_len_diff_from_transitions<'a>( + all_transitions: impl Iterator>, + ) -> i64 { + let mut push_count: i64 = 0; + let mut pop_count: i64 = 0; + + for trans in all_transitions { + match trans { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Update { .. } => {} + Transition::Push(_) => push_count += 1, + Transition::Pop => pop_count += 1, + } + } + push_count - pop_count + } + + impl ConcreteLazyVecState { + fn assert_validation_accepted(&self, new_vec_len: u64) { + // Init the VP env from tx env in which we applied the vec + // transitions + let tx_env = tx_host_env::take(); + vp_host_env::init_from_tx(self.address.clone(), tx_env, |_| {}); + + // Simulate a validity predicate run using the lazy vec's validation + // helpers + let changed_keys = + vp_host_env::with(|env| env.all_touched_storage_keys()); + + let mut validation_builder = None; + + // Push followed by pop is a no-op, in which case we'd still see the + // changed keys for these actions, but they wouldn't affect the + // validation result and they never get persisted, but we'd still + // them as changed key here. To guard against this case, + // we check that `vec_len_from_transitions` is not empty. + let vec_len_diff = + vec_len_diff_from_transitions(self.current_transitions.iter()); + + // To help debug validation issues... + dbg!( + &self.current_transitions, + &changed_keys + .iter() + .map(storage::Key::to_string) + .collect::>() + ); + + for key in &changed_keys { + let is_sub_key = self + .lazy_vec + .accumulate( + vp_host_env::ctx(), + &mut validation_builder, + key, + ) + .unwrap(); + + assert!( + is_sub_key, + "We're only modifying the lazy_vec's keys here. Key: \ + \"{key}\", vec length diff {vec_len_diff}" + ); + } + if !changed_keys.is_empty() && vec_len_diff != 0 { + assert!( + validation_builder.is_some(), + "If some keys were changed, the builder must get filled in" + ); + let actions = LazyVec::::validate( + validation_builder.unwrap(), + ) + .expect( + "With valid transitions only, validation should always \ + pass", + ); + let mut actions_to_check = actions.clone(); + + // Check that every transition has a corresponding action from + // validation. We drop the found actions to check that all + // actions are matched too. + let current_transitions = normalize_transitions( + &self.current_transitions, + new_vec_len, + ); + for transition in ¤t_transitions { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + } + Transition::Push(expected_val) => { + let mut ix = 0; + while ix < actions_to_check.len() { + if let lazy_vec::Action::Push(val) = + &actions_to_check[ix] + { + if expected_val == val { + actions_to_check.remove(ix); + break; + } + } + ix += 1; + } + } + Transition::Pop => { + let mut ix = 0; + while ix < actions_to_check.len() { + if let lazy_vec::Action::Pop(_val) = + &actions_to_check[ix] + { + actions_to_check.remove(ix); + break; + } + ix += 1; + } + } + Transition::Update { + index: expected_index, + value, + } => { + let mut ix = 0; + while ix < actions_to_check.len() { + if let lazy_vec::Action::Update { + index, + pre: _, + post, + } = &actions_to_check[ix] + { + if expected_index == index && post == value + { + actions_to_check.remove(ix); + break; + } + } + ix += 1; + } + } + } + } + + assert!( + actions_to_check.is_empty(), + "All the actions reported from validation {actions:#?} \ + should have been matched with SM transitions \ + {current_transitions:#?}, but these actions didn't \ + match: {actions_to_check:#?}", + ) + } + + // Put the tx_env back before checking the result + tx_host_env::set_from_vp_env(vp_host_env::take()); + } + } + + /// Generate an arbitrary `TestVecItem` + fn arb_test_vec_item() -> impl Strategy { + (any::(), any::()).prop_map(|(x, y)| TestVecItem { x, y }) + } + + /// Apply `Transition` on an eager `Vec`. + fn apply_transition_on_eager_vec( + vec: &mut Vec, + transition: &Transition, + ) { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => {} + Transition::Push(value) => vec.push(value.clone()), + Transition::Pop => { + let _popped = vec.pop(); + } + Transition::Update { index, value } => { + let entry = vec.get_mut(*index as usize).unwrap(); + *entry = value.clone(); + } + } + } + + /// Normalize transitions: + /// - pop at ix + push(val) at ix -> update(ix, val) + /// - push(val) at ix + update(ix, new_val) -> push(new_val) at ix + /// - update(ix, val) + update(ix, new_val) -> update(ix, new_val) + /// + /// Note that the normalizable transitions pairs do not have to be directly + /// next to each other, but their order does matter. + fn normalize_transitions( + transitions: &[Transition], + new_vec_len: u64, + ) -> Vec> { + let stack_start_pos = ((new_vec_len as i64) + - vec_len_diff_from_transitions(transitions.iter())) + as u64; + let mut stack_pos = stack_start_pos; + let mut collapsed = vec![]; + 'outer: for transition in transitions { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + collapsed.push(transition.clone()) + } + Transition::Push(value) => { + // If there are some pops, the last one can be collapsed + // with this push + if stack_pos < stack_start_pos { + // Find the pop from the back + let mut found_ix = None; + for (ix, transition) in + collapsed.iter().enumerate().rev() + { + if let Transition::Pop = transition { + found_ix = Some(ix); + break; + } + } + let ix = found_ix.expect("Pop must be found"); + // pop at ix + push(val) at ix -> update(ix, val) + + // Replace the Pop with an Update and don't insert the + // Push + *collapsed.get_mut(ix).unwrap() = Transition::Update { + index: stack_pos, + value: value.clone(), + }; + } else { + collapsed.push(transition.clone()); + } + stack_pos += 1; + } + Transition::Pop => { + collapsed.push(transition.clone()); + stack_pos -= 1; + } + Transition::Update { index, value } => { + // If there are some pushes, check if one of them is at the + // same index as this update + if stack_pos > stack_start_pos { + let mut current_pos = stack_start_pos; + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + match collapsed_transition { + Transition::CommitTx + | Transition::CommitTxAndBlock => {} + Transition::Push(_) => { + if ¤t_pos == index { + // push(val) at `ix` + update(ix, + // new_val) -> + // push(new_val) at `ix` + + // Replace the Push with the new Push of + // Update's + // value and don't insert the Update + *collapsed.get_mut(ix).unwrap() = + Transition::Push(value.clone()); + continue 'outer; + } + current_pos += 1; + } + Transition::Pop => { + current_pos -= 1; + } + Transition::Update { + index: prev_update_index, + value: _, + } => { + if index == prev_update_index { + // update(ix, val) + update(ix, new_val) + // -> update(ix, new_val) + + // Replace the Update with the new + // Update instead of inserting it + *collapsed.get_mut(ix).unwrap() = + transition.clone(); + continue 'outer; + } + } + } + } + } + collapsed.push(transition.clone()) + } + } + } + collapsed + } +} diff --git a/tests/src/storage_api/collections/mod.rs b/tests/src/storage_api/collections/mod.rs new file mode 100644 index 0000000000..f39b880c09 --- /dev/null +++ b/tests/src/storage_api/collections/mod.rs @@ -0,0 +1,3 @@ +mod lazy_map; +mod lazy_vec; +mod nested_lazy_map; diff --git a/tests/src/storage_api/collections/nested_lazy_map.rs b/tests/src/storage_api/collections/nested_lazy_map.rs new file mode 100644 index 0000000000..037decce46 --- /dev/null +++ b/tests/src/storage_api/collections/nested_lazy_map.rs @@ -0,0 +1,723 @@ +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::convert::TryInto; + + use borsh::{BorshDeserialize, BorshSerialize}; + use namada::types::address::{self, Address}; + use namada::types::storage; + use namada_tx_prelude::storage::KeySeg; + use namada_tx_prelude::storage_api::collections::lazy_map::{ + NestedMap, NestedSubKey, SubKey, + }; + use namada_tx_prelude::storage_api::collections::{ + lazy_map, LazyCollection, LazyMap, + }; + use proptest::prelude::*; + use proptest::prop_state_machine; + use proptest::state_machine::{AbstractStateMachine, StateMachineTest}; + use proptest::test_runner::Config; + use test_log::test; + + use crate::tx::tx_host_env; + use crate::vp::vp_host_env; + + prop_state_machine! { + #![proptest_config(Config { + // Instead of the default 256, we only run 5 because otherwise it + // takes too long and it's preferable to crank up the number of + // transitions instead, to allow each case to run for more epochs as + // some issues only manifest once the model progresses further. + // Additionally, more cases will be explored every time this test is + // executed in the CI. + cases: 5, + .. Config::default() + })] + #[test] + fn nested_lazy_map_api_state_machine_test(sequential 1..100 => ConcreteLazyMapState); + } + + /// Some borsh-serializable type with arbitrary fields to be used inside + /// LazyMap state machine test + #[derive( + Clone, + Debug, + BorshSerialize, + BorshDeserialize, + PartialEq, + Eq, + PartialOrd, + Ord, + )] + struct TestVal { + x: u64, + y: bool, + } + + type KeyOuter = u64; + type KeyMiddle = i32; + type KeyInner = i8; + + type NestedTestMap = + NestedMap>>; + + type NestedEagerMap = + BTreeMap>>; + + /// A `StateMachineTest` implemented on this struct manipulates it with + /// `Transition`s, which are also being accumulated into + /// `current_transitions`. It then: + /// + /// - checks its state against an in-memory `std::collections::HashMap` + /// - runs validation and checks that the `LazyMap::Action`s reported from + /// validation match with transitions that were applied + /// + /// Additionally, one of the transitions is to commit a block and/or + /// transaction, during which the currently accumulated state changes are + /// persisted, or promoted from transaction write log to block's write log. + #[derive(Debug)] + struct ConcreteLazyMapState { + /// Address is used to prefix the storage key of the `lazy_map` in + /// order to simulate a transaction and a validity predicate + /// check from changes on the `lazy_map` + address: Address, + /// In the test, we apply the same transitions on the `lazy_map` as on + /// `eager_map` to check that `lazy_map`'s state is consistent with + /// `eager_map`. + eager_map: NestedEagerMap, + /// Handle to a lazy map with nested lazy collections + lazy_map: NestedTestMap, + /// Valid LazyMap changes in the current transaction + current_transitions: Vec, + } + + #[derive(Clone, Debug, Default)] + struct AbstractLazyMapState { + /// Valid LazyMap changes in the current transaction + valid_transitions: Vec, + /// Valid LazyMap changes committed to storage + committed_transitions: Vec, + } + + /// Possible transitions that can modify a [`NestedTestMap`]. + /// This roughly corresponds to the methods that have `StorageWrite` + /// access and is very similar to [`Action`] + #[derive(Clone, Debug)] + enum Transition { + /// Commit all valid transitions in the current transaction + CommitTx, + /// Commit all valid transitions in the current transaction and also + /// commit the current block + CommitTxAndBlock, + /// Insert a key-val into a [`LazyMap`] + Insert(Key, TestVal), + /// Remove a key-val from a [`LazyMap`] + Remove(Key), + /// Update a value at key from pre to post state in a + /// [`LazyMap`] + Update(Key, TestVal), + } + + /// A key for transition + type Key = (KeyOuter, KeyMiddle, KeyInner); + + impl AbstractStateMachine for AbstractLazyMapState { + type State = Self; + type Transition = Transition; + + fn init_state() -> BoxedStrategy { + Just(Self::default()).boxed() + } + + // Apply a random transition to the state + fn transitions(state: &Self::State) -> BoxedStrategy { + let length = state.len(); + if length == 0 { + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_map_key(), arb_map_val()).prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } else { + let keys = state.find_existing_keys(); + let arb_existing_map_key = + || proptest::sample::select(keys.clone()); + prop_oneof![ + 1 => Just(Transition::CommitTx), + 1 => Just(Transition::CommitTxAndBlock), + 3 => (arb_existing_map_key(), arb_map_val()).prop_map(|(key, val)| + Transition::Update(key, val)), + 3 => arb_existing_map_key().prop_map(Transition::Remove), + 5 => (arb_map_key().prop_filter( + "insert on non-existing keys only", + move |key| !keys.contains(key)), arb_map_val()) + .prop_map(|(key, val)| Transition::Insert(key, val)) + ] + .boxed() + } + } + + fn apply_abstract( + mut state: Self::State, + transition: &Self::Transition, + ) -> Self::State { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + let valid_actions_to_commit = + std::mem::take(&mut state.valid_transitions); + state + .committed_transitions + .extend(valid_actions_to_commit.into_iter()); + } + _ => state.valid_transitions.push(transition.clone()), + } + state + } + + fn preconditions( + state: &Self::State, + transition: &Self::Transition, + ) -> bool { + let length = state.len(); + // Ensure that the remove or update transitions are not applied + // to an empty state + if length == 0 + && matches!( + transition, + Transition::Remove(_) | Transition::Update(_, _) + ) + { + return false; + } + match transition { + Transition::Update(key, _) | Transition::Remove(key) => { + let keys = state.find_existing_keys(); + // Ensure that the update/remove key is an existing one + keys.contains(key) + } + Transition::Insert(key, _) => { + let keys = state.find_existing_keys(); + // Ensure that the insert key is not an existing one + !keys.contains(key) + } + _ => true, + } + } + } + + impl StateMachineTest for ConcreteLazyMapState { + type Abstract = AbstractLazyMapState; + type ConcreteState = Self; + + fn init_test( + _initial_state: ::State, + ) -> Self::ConcreteState { + // Init transaction env in which we'll be applying the transitions + tx_host_env::init(); + + // The lazy_map's path must be prefixed by the address to be able + // to trigger a validity predicate on it + let address = address::testing::established_address_1(); + tx_host_env::with(|env| env.spawn_accounts([&address])); + let lazy_map_prefix: storage::Key = address.to_db_key().into(); + + Self { + address, + eager_map: BTreeMap::new(), + lazy_map: NestedTestMap::open( + lazy_map_prefix.push(&"arbitrary".to_string()).unwrap(), + ), + current_transitions: vec![], + } + } + + fn apply_concrete( + mut state: Self::ConcreteState, + transition: ::Transition, + ) -> Self::ConcreteState { + // Apply transitions in transaction env + let ctx = tx_host_env::ctx(); + + // Persist the transitions in the current tx, or clear previous ones + // if we're committing a tx + match &transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + state.current_transitions = vec![]; + } + _ => { + state.current_transitions.push(transition.clone()); + } + } + + // Transition application on lazy map and post-conditions: + match &transition { + Transition::CommitTx => { + // commit the tx without committing the block + tx_host_env::with(|env| env.write_log.commit_tx()); + } + Transition::CommitTxAndBlock => { + // commit the tx and the block + tx_host_env::commit_tx_and_block(); + } + Transition::Insert( + (key_outer, key_middle, key_inner), + value, + ) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + inner.insert(ctx, *key_inner, value.clone()).unwrap(); + + // Post-conditions: + let stored_value = + inner.get(ctx, key_inner).unwrap().unwrap(); + assert_eq!( + &stored_value, value, + "the new item must be added to the back" + ); + + state.assert_validation_accepted(); + } + Transition::Remove((key_outer, key_middle, key_inner)) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + let removed = + inner.remove(ctx, key_inner).unwrap().unwrap(); + + // Post-conditions: + assert_eq!( + &removed, + state + .eager_map + .get(key_outer) + .unwrap() + .get(key_middle) + .unwrap() + .get(key_inner) + .unwrap(), + "removed element matches the value in eager map \ + before it's updated" + ); + + state.assert_validation_accepted(); + } + Transition::Update( + (key_outer, key_middle, key_inner), + value, + ) => { + let inner = state.lazy_map.at(key_outer).at(key_middle); + + let old_val = inner.get(ctx, key_inner).unwrap().unwrap(); + + inner.insert(ctx, *key_inner, value.clone()).unwrap(); + + // Post-conditions: + let new_val = inner.get(ctx, key_inner).unwrap().unwrap(); + assert_eq!( + &old_val, + state + .eager_map + .get(key_outer) + .unwrap() + .get(key_middle) + .unwrap() + .get(key_inner) + .unwrap(), + "old value must match the value at the same key in \ + the eager map before it's updated" + ); + assert_eq!( + &new_val, value, + "new value must match that which was passed into the \ + Transition::Update" + ); + + state.assert_validation_accepted(); + } + } + + // Apply transition in the eager map for comparison + apply_transition_on_eager_map(&mut state.eager_map, &transition); + + // Global post-conditions: + + // All items in eager map must be present in lazy map + for (key_outer, middle) in state.eager_map.iter() { + for (key_middle, inner) in middle { + for (key_inner, expected_item) in inner { + let got = state + .lazy_map + .at(key_outer) + .at(key_middle) + .get(ctx, key_inner) + .unwrap() + .expect( + "The expected item must be present in lazy map", + ); + assert_eq!( + expected_item, &got, + "at key {key_outer}, {key_middle} {key_inner}" + ); + } + } + } + + // All items in lazy map must be present in eager map + for key_val in state.lazy_map.iter(ctx).unwrap() { + let ( + NestedSubKey::Data { + key: key_outer, + nested_sub_key: + NestedSubKey::Data { + key: key_middle, + nested_sub_key: SubKey::Data(key_inner), + }, + }, + expected_val, + ) = key_val.unwrap(); + let got = state + .eager_map + .get(&key_outer) + .unwrap() + .get(&key_middle) + .unwrap() + .get(&key_inner) + .expect("The expected item must be present in eager map"); + assert_eq!( + &expected_val, got, + "at key {key_outer}, {key_middle} {key_inner})" + ); + } + + state + } + } + + impl AbstractLazyMapState { + /// Find the length of the map from the applied transitions + fn len(&self) -> u64 { + (map_len_diff_from_transitions(self.committed_transitions.iter()) + + map_len_diff_from_transitions(self.valid_transitions.iter())) + .try_into() + .expect( + "It shouldn't be possible to underflow length from all \ + transactions applied in abstract state", + ) + } + + /// Build an eager map from the committed and current transitions + fn eager_map(&self) -> NestedEagerMap { + let mut eager_map = BTreeMap::new(); + for transition in &self.committed_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + for transition in &self.valid_transitions { + apply_transition_on_eager_map(&mut eager_map, transition); + } + eager_map + } + + /// Find the keys currently present in the map + fn find_existing_keys(&self) -> Vec { + let outer_map = self.eager_map(); + outer_map + .into_iter() + .fold(vec![], |acc, (outer, middle_map)| { + middle_map.into_iter().fold( + acc, + |mut acc, (middle, inner_map)| { + acc.extend( + inner_map + .into_iter() + .map(|(inner, _)| (outer, middle, inner)), + ); + acc + }, + ) + }) + } + } + + /// Find the difference in length of the map from the applied transitions + fn map_len_diff_from_transitions<'a>( + transitions: impl Iterator, + ) -> i64 { + let mut insert_count: i64 = 0; + let mut remove_count: i64 = 0; + + for trans in transitions { + match trans { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Update(_, _) => {} + Transition::Insert(_, _) => insert_count += 1, + Transition::Remove(_) => remove_count += 1, + } + } + insert_count - remove_count + } + + impl ConcreteLazyMapState { + fn assert_validation_accepted(&self) { + // Init the VP env from tx env in which we applied the map + // transitions + let tx_env = tx_host_env::take(); + vp_host_env::init_from_tx(self.address.clone(), tx_env, |_| {}); + + // Simulate a validity predicate run using the lazy map's validation + // helpers + let changed_keys = + vp_host_env::with(|env| env.all_touched_storage_keys()); + + let mut validation_builder = None; + + // Push followed by pop is a no-op, in which case we'd still see the + // changed keys for these actions, but they wouldn't affect the + // validation result and they never get persisted, but we'd still + // them as changed key here. To guard against this case, + // we check that `map_len_from_transitions` is not empty. + let map_len_diff = + map_len_diff_from_transitions(self.current_transitions.iter()); + + // To help debug validation issues... + dbg!( + &self.current_transitions, + &changed_keys + .iter() + .map(storage::Key::to_string) + .collect::>() + ); + + for key in &changed_keys { + let is_sub_key = self + .lazy_map + .accumulate( + vp_host_env::ctx(), + &mut validation_builder, + key, + ) + .unwrap(); + + assert!( + is_sub_key, + "We're only modifying the lazy_map's keys here. Key: \ + \"{key}\", map length diff {map_len_diff}" + ); + } + if !changed_keys.is_empty() && map_len_diff != 0 { + assert!( + validation_builder.is_some(), + "If some keys were changed, the builder must get filled in" + ); + let actions = + NestedTestMap::validate(validation_builder.unwrap()) + .unwrap(); + let mut actions_to_check = actions.clone(); + + // Check that every transition has a corresponding action from + // validation. We drop the found actions to check that all + // actions are matched too. + let current_transitions = + normalize_transitions(&self.current_transitions); + for transition in ¤t_transitions { + use lazy_map::Action; + use lazy_map::NestedAction::At; + + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => { + } + Transition::Insert(expected_key, expected_val) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Insert(key_inner, val), + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key + && expected_val == val + { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Remove(expected_key) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Remove(key_inner, _val), + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key { + actions_to_check.remove(ix); + break; + } + } + } + } + Transition::Update(expected_key, value) => { + for (ix, action) in + actions_to_check.iter().enumerate() + { + if let At( + key_outer, + At( + key_middle, + Action::Update { + key: key_inner, + pre: _, + post, + }, + ), + ) = action + { + let key = + (*key_outer, *key_middle, *key_inner); + if expected_key == &key && post == value { + actions_to_check.remove(ix); + break; + } + } + } + } + } + } + + assert!( + actions_to_check.is_empty(), + "All the actions reported from validation {actions:#?} \ + should have been matched with SM transitions \ + {current_transitions:#?}, but these actions didn't \ + match: {actions_to_check:#?}", + ) + } + + // Put the tx_env back before checking the result + tx_host_env::set_from_vp_env(vp_host_env::take()); + } + } + + /// Generate an arbitrary `TestKey` + fn arb_map_key() -> impl Strategy { + (any::(), any::(), any::()) + } + + /// Generate an arbitrary `TestVal` + fn arb_map_val() -> impl Strategy { + (any::(), any::()).prop_map(|(x, y)| TestVal { x, y }) + } + + /// Apply `Transition` on an eager `Map`. + fn apply_transition_on_eager_map( + map: &mut NestedEagerMap, + transition: &Transition, + ) { + match transition { + Transition::CommitTx | Transition::CommitTxAndBlock => {} + Transition::Insert((key_outer, key_middle, key_inner), value) + | Transition::Update((key_outer, key_middle, key_inner), value) => { + let middle = + map.entry(*key_outer).or_insert_with(Default::default); + let inner = + middle.entry(*key_middle).or_insert_with(Default::default); + inner.insert(*key_inner, value.clone()); + } + Transition::Remove((key_outer, key_middle, key_inner)) => { + let middle = + map.entry(*key_outer).or_insert_with(Default::default); + let inner = + middle.entry(*key_middle).or_insert_with(Default::default); + let _popped = inner.remove(key_inner); + } + } + } + + /// Normalize transitions: + /// - remove(key) + insert(key, val) -> update(key, val) + /// - insert(key, val) + update(key, new_val) -> insert(key, new_val) + /// - update(key, val) + update(key, new_val) -> update(key, new_val) + /// + /// Note that the normalizable transitions pairs do not have to be directly + /// next to each other, but their order does matter. + fn normalize_transitions(transitions: &[Transition]) -> Vec { + let mut collapsed = vec![]; + 'outer: for transition in transitions { + match transition { + Transition::CommitTx + | Transition::CommitTxAndBlock + | Transition::Remove(_) => collapsed.push(transition.clone()), + Transition::Insert(key, val) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Remove(remove_key) = + collapsed_transition + { + if key == remove_key { + // remove(key) + insert(key, val) -> update(key, + // val) + + // Replace the Remove with an Update instead of + // inserting the Insert + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, val.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + Transition::Update(key, value) => { + for (ix, collapsed_transition) in + collapsed.iter().enumerate() + { + if let Transition::Insert(insert_key, _) = + collapsed_transition + { + if key == insert_key { + // insert(key, val) + update(key, new_val) -> + // insert(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Insert(*key, value.clone()); + continue 'outer; + } + } else if let Transition::Update(update_key, _) = + collapsed_transition + { + if key == update_key { + // update(key, val) + update(key, new_val) -> + // update(key, new_val) + + // Replace the insert with the new update's + // value instead of inserting it + *collapsed.get_mut(ix).unwrap() = + Transition::Update(*key, value.clone()); + continue 'outer; + } + } + } + collapsed.push(transition.clone()); + } + } + } + collapsed + } +} diff --git a/tests/src/storage_api/mod.rs b/tests/src/storage_api/mod.rs new file mode 100644 index 0000000000..bc487bd59e --- /dev/null +++ b/tests/src/storage_api/mod.rs @@ -0,0 +1 @@ +mod collections; diff --git a/tests/src/vm_host_env/tx.rs b/tests/src/vm_host_env/tx.rs index a27a78b09b..6a3ef96084 100644 --- a/tests/src/vm_host_env/tx.rs +++ b/tests/src/vm_host_env/tx.rs @@ -18,6 +18,8 @@ use namada::vm::{self, WasmCacheRwAccess}; use namada_tx_prelude::{BorshSerialize, Ctx}; use tempfile::TempDir; +use crate::vp::TestVpEnv; + /// Tx execution context provides access to host env functions static mut CTX: Ctx = unsafe { Ctx::new() }; @@ -235,6 +237,29 @@ mod native_tx_host_env { with(|env| env.commit_tx_and_block()) } + /// Set the [`TestTxEnv`] back from a [`TestVpEnv`]. This is useful when + /// testing validation with multiple transactions that accumulate some state + /// changes. + pub fn set_from_vp_env(vp_env: TestVpEnv) { + let TestVpEnv { + storage, + write_log, + tx, + vp_wasm_cache, + vp_cache_dir, + .. + } = vp_env; + let tx_env = TestTxEnv { + storage, + write_log, + vp_wasm_cache, + vp_cache_dir, + tx, + ..Default::default() + }; + set(tx_env); + } + /// A helper macro to create implementations of the host environment /// functions exported to wasm, which uses the environment from the /// `ENV` variable. diff --git a/tx_prelude/src/lib.rs b/tx_prelude/src/lib.rs index ac7da770fd..730adb3155 100644 --- a/tx_prelude/src/lib.rs +++ b/tx_prelude/src/lib.rs @@ -182,7 +182,7 @@ impl StorageRead<'_> for Ctx { fn rev_iter_prefix( &self, prefix: &storage::Key, - ) -> storage_api::Result { + ) -> Result { let prefix = prefix.to_string(); let iter_id = unsafe { anoma_tx_rev_iter_prefix(prefix.as_ptr() as _, prefix.len() as _) diff --git a/vp_prelude/src/lib.rs b/vp_prelude/src/lib.rs index 3e745d7641..e6618bc5de 100644 --- a/vp_prelude/src/lib.rs +++ b/vp_prelude/src/lib.rs @@ -335,14 +335,14 @@ impl StorageRead<'_> for CtxPreStorageRead<'_> { fn iter_prefix( &self, prefix: &storage::Key, - ) -> Result { + ) -> Result { iter_prefix_impl(prefix) } fn rev_iter_prefix( &self, prefix: &storage::Key, - ) -> storage_api::Result { + ) -> Result { rev_iter_prefix_impl(prefix) } @@ -396,7 +396,7 @@ impl StorageRead<'_> for CtxPostStorageRead<'_> { fn iter_prefix( &self, prefix: &storage::Key, - ) -> Result { + ) -> Result { iter_prefix_impl(prefix) } @@ -436,7 +436,7 @@ fn iter_prefix_impl( fn rev_iter_prefix_impl( prefix: &storage::Key, -) -> Result)>, storage_api::Error> { +) -> Result)>, Error> { let prefix = prefix.to_string(); let iter_id = unsafe { anoma_vp_rev_iter_prefix(prefix.as_ptr() as _, prefix.len() as _) @@ -444,7 +444,7 @@ fn rev_iter_prefix_impl( Ok(KeyValIterator(iter_id, PhantomData)) } -fn get_chain_id() -> Result { +fn get_chain_id() -> Result { let result = Vec::with_capacity(CHAIN_ID_LENGTH); unsafe { anoma_vp_get_chain_id(result.as_ptr() as _); diff --git a/wasm/checksums.json b/wasm/checksums.json index 429ecb24c0..91a99dbe22 100644 --- a/wasm/checksums.json +++ b/wasm/checksums.json @@ -1,19 +1,19 @@ { - "tx_bond.wasm": "tx_bond.9008e4363607a4bbb7ff17e537c0895b7abf6b7a06955ebc251931937dd3c920.wasm", + "tx_bond.wasm": "tx_bond.8ef02cded8d2cf293a5af5923e6cf9fce3936c192377b2971cfff733badd4326.wasm", "tx_from_intent.wasm": "tx_from_intent.e21563260c03cfdab1f195878f49bf93722027ad26fcd097cfebbc5c4d279082.wasm", - "tx_ibc.wasm": "tx_ibc.15820a047cb543c4f684733c4b63eb598c6300300ef71b3b49413d934710e4f6.wasm", - "tx_init_account.wasm": "tx_init_account.042bed0039e4ccc0c8add03b60811d8ba9aef88ad92976388ea67ba945875f53.wasm", - "tx_init_nft.wasm": "tx_init_nft.9c59f6e6d89d54836929824eb6dea8fcac1c490610cc25b6bfb5c3db69599567.wasm", - "tx_init_proposal.wasm": "tx_init_proposal.fb8b9f98ee249b70cba583e5cf745fe8566c933a7506ffc7fd7cb4389c9437c0.wasm", - "tx_init_validator.wasm": "tx_init_validator.62cfb6bc0e3f2b74041a87343cd15afd0c8c208839c20127248df9df0bb698cc.wasm", - "tx_mint_nft.wasm": "tx_mint_nft.c65d3c80364e13e06eed6be6d76f427e0a69262542860eacabb7a5c8b3ca3403.wasm", - "tx_transfer.wasm": "tx_transfer.a83024bd2ac2d3968fa86c77a1d8cfcd127c385da3d66b561d2e4b5803bb990c.wasm", - "tx_unbond.wasm": "tx_unbond.f178aa50f6eef1a467100c927f60a8d61c30b245e66188a7786ec0654e273a26.wasm", - "tx_update_vp.wasm": "tx_update_vp.4873920039cf237469b283bf91ae189538e66b138a0b201e67a85e4e9a1f13b6.wasm", - "tx_vote_proposal.wasm": "tx_vote_proposal.eb019532916387369a9c6e11757715ac40b1831a7b9c069b8c7635bfe0618da8.wasm", - "tx_withdraw.wasm": "tx_withdraw.b7c7c8e6300803539f3c6fd089e8726d0b1a6ff5d8606f66931c54a6d836f718.wasm", - "vp_nft.wasm": "vp_nft.80dd20e9012b033a4974a6eec35d3a88a8d5f9b56a2865d21ab7ab42b87f1dd1.wasm", - "vp_testnet_faucet.wasm": "vp_testnet_faucet.84ff286cde2d8d3d6ac62cd61af8de047cbc9a99b3a39caaacf5c0da107ce205.wasm", - "vp_token.wasm": "vp_token.648909b1ec5540aa9cc6bc7096cb42b3720c2368f9e9dfeb2bf4646b6e6c2ba6.wasm", - "vp_user.wasm": "vp_user.3ab5e6d1b1746ff08f9cc658b3723f2202d78ea97da7706cad07819efcff85b8.wasm" + "tx_ibc.wasm": "tx_ibc.5aabb78e23847eca23a7d27e29556bf4675e4a631a4deb97e7c1ae3a6f0d0aca.wasm", + "tx_init_account.wasm": "tx_init_account.20e6f3506f578e3ad98423ca63c64ce201b3c549538364aa87b9f1b6a60d78da.wasm", + "tx_init_nft.wasm": "tx_init_nft.9aa0a1633aa24d7ca17e12afc5cd561339378e9fb45895bd1822d441e1e84c62.wasm", + "tx_init_proposal.wasm": "tx_init_proposal.b36e774f02f26dd2fe95131532f45e72d8067e556dfbd296b279080e1f8c6d23.wasm", + "tx_init_validator.wasm": "tx_init_validator.3dc817af916c5c7fdcd9e23d9fac7bfa96325ec80d592ec687b4e70706952c40.wasm", + "tx_mint_nft.wasm": "tx_mint_nft.afdaada80a6d144318aa94e5d15c3fabcb0fd3e62b0b8ef0ea09187a2489de1a.wasm", + "tx_transfer.wasm": "tx_transfer.9293e4a1f8a6522b87b369bacfbf4784216c6bc28ef259d4891b3154b6ad4c1f.wasm", + "tx_unbond.wasm": "tx_unbond.7d22424d49491133ef27108f06d8a8b0a8337223ddc45e206ed8ba6b5baf1316.wasm", + "tx_update_vp.wasm": "tx_update_vp.cbf9af660eebf1ac49a04305a1080c21e025cccf388348c590ffbd19eb3a46b2.wasm", + "tx_vote_proposal.wasm": "tx_vote_proposal.af3e24dee3e4f27cfa5145f8f8a7e145fcaeeb8c0e183bf8614b1f8144f42e88.wasm", + "tx_withdraw.wasm": "tx_withdraw.8734c9344ab57555bf62863cf701301bab8624abb720e1f2a7139eec5241ebcd.wasm", + "vp_nft.wasm": "vp_nft.d486d6d31a31cc70a2dffbac13cbe341fcc049759d8be3fc7a035dcb6fe59bb3.wasm", + "vp_testnet_faucet.wasm": "vp_testnet_faucet.f1fcbeae203923f4fca37699d1f4da890f880f3585cc8b625697b47e9b74ebe0.wasm", + "vp_token.wasm": "vp_token.c3e586db9e0b106a6f1a348d7eafc93ce42d747dc0efdff9f382830ec5a754e4.wasm", + "vp_user.wasm": "vp_user.1bb0432d936d6949a99fccd74a4c7a970303e6719a45593c23e630b61cf5df02.wasm" } \ No newline at end of file