Merge branch 'namada/tomas/sorted-prefix-iter' (#458)

* namada/tomas/sorted-prefix-iter:
  [ci skip] wasm checksums update
  changelog: add #458
  tests: extend prefix iter tests for reverse order
  add support for rev_iter_prefix in storage and VP and tx envs
  test/vm_host_env: check prefix iter order in tx and VP
  shared/storage/key: add support for int/uint keys that maintain order
  deps: replace hex with data-encoding
  changelog: add #335
  wasm checksums update
  test/vm_host_env: refactor prefix iter tests with new `iter_prefix` fn
  storage_api: build a nicer `iter_prefix` function on top of StorageRead
  changelog: add #331
  update wasm checksums
  fix missing StorageWrite for Storage merkle tree update
  storage: remove unnecessary clone
  add more comments for StorageRead lifetime with an example usage
  add <'iter> lifetime to StorageRead trait to get around lack of GATs
  ledger: impl StorageWrite for Storage
  ledger: factor out TxEnv write methods into a new StorageWrite trait
  update wasm checksums
  ledger/pos: implement PosReadOnly using StorageRead trait
  ledger/tx: inherit StorageRead in TxEnv
  ledger/storage: impl StorageRead for Storage
  ledger/vp: impl StorageRead for WASM VPs pre and post state
  ledger/native_vp: implement StorageRead for pre and post state
  ledger: add StorageRead trait with extensible error type
  [ci skip] wasm checksums update
  update wasm checksums
  wasm_for_tests: make
  Update shared/src/ledger/vp_env.rs
  changelog: add #1093
  wasm: improve error handling
  macros: add error handling to transaction and validity_predicate macros
  wasm: update for VM API changes
  tests: update for VM API changes
  VM: move vm_env sub-mod into tx/vp_prelude and add Tx/VpEnv and Ctx
  shared: update native_vp implementations for VpEnv methods
  shared/ledger/native_vp: implement VpEnv
  shared/vm: rename host_env TxEnv/VpEnv to TxVmEnv/VpVmEnv
  ledger: add tx and VP traits with host env functions

.changelog/v0.7.1/improvements/1093-unify-native-and-wasm-vp.md

@@ -0,0 +1,3 @@
+- Added WASM transaction and validity predicate `Ctx` with methods for host
+  environment functions to unify the interface of native VPs and WASM VPs under
+  `trait VpEnv` ([#1093](https://github.com/anoma/anoma/pull/1093))

.changelog/v0.7.1/improvements/331-common-write-storage-trait.md

@@ -0,0 +1,2 @@
+- Added a StorageWrite trait for a common interface for transactions and direct
+  storage access for protocol ([#331](https://github.com/anoma/namada/pull/331))

.changelog/v0.7.1/improvements/335-refactor-storage-prefix-iter.md

@@ -0,0 +1,3 @@
+- Added a simpler prefix iterator API that returns `std::iter::Iterator` with
+  the storage keys parsed and a variant that also decodes stored values with
+  Borsh ([#335](https://github.com/anoma/namada/pull/335))

.changelog/v0.7.1/improvements/409-sorted-prefix-iter.md

@@ -0,0 +1,3 @@
+- Fix order of prefix iterator to be sorted by storage
+  keys and add support for a reverse order prefix iterator.
+  ([#409](https://github.com/anoma/namada/issues/409))

apps/Cargo.toml

@@ -62,6 +62,7 @@ clap = {git = "https://github.com/clap-rs/clap/", tag = "v3.0.0-beta.2", default
 color-eyre = "0.5.10"
 config = "0.11.0"
 curl = "0.4.43"
+data-encoding = "2.3.2"
 derivative = "2.2.0"
 directories = "4.0.1"
 ed25519-consensus = "1.2.0"
@@ -71,7 +72,6 @@ eyre = "0.6.5"
 flate2 = "1.0.22"
 file-lock = "2.0.2"
 futures = "0.3"
-hex = "0.4.3"
 itertools = "0.10.1"
 jsonpath_lib = "0.3.0"
 libc = "0.2.97"

apps/src/lib/client/rpc.rs

@@ -11,6 +11,7 @@ use async_std::fs::{self};
 use async_std::path::PathBuf;
 use async_std::prelude::*;
 use borsh::BorshDeserialize;
+use data_encoding::HEXLOWER;
 use itertools::Itertools;
 use namada::ledger::governance::storage as gov_storage;
 use namada::ledger::governance::utils::Votes;
@@ -81,7 +82,7 @@ pub async fn query_raw_bytes(_ctx: Context, args: args::QueryRawBytes) {
         .unwrap();
     match response.code {
         Code::Ok => {
-            println!("{}", hex::encode(&response.value));
+            println!("{}", HEXLOWER.encode(&response.value));
         }
         Code::Err(err) => {
             eprintln!(

apps/src/lib/config/genesis.rs

@@ -26,7 +26,7 @@ pub mod genesis_config {
     use std::path::Path;
     use std::str::FromStr;
 
-    use hex;
+    use data_encoding::HEXLOWER;
     use namada::ledger::governance::parameters::GovParams;
     use namada::ledger::parameters::{EpochDuration, Parameters};
     use namada::ledger::pos::types::BasisPoints;
@@ -50,12 +50,12 @@ pub mod genesis_config {
 
     impl HexString {
         pub fn to_bytes(&self) -> Result<Vec<u8>, HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             Ok(bytes)
         }
 
         pub fn to_sha256_bytes(&self) -> Result<[u8; 32], HexKeyError> {
-            let bytes = hex::decode(&self.0)?;
+            let bytes = HEXLOWER.decode(self.0.as_ref())?;
             let slice = bytes.as_slice();
             let array: [u8; 32] = slice.try_into()?;
             Ok(array)
@@ -76,15 +76,15 @@ pub mod genesis_config {
     #[derive(Error, Debug)]
     pub enum HexKeyError {
         #[error("Invalid hex string: {0:?}")]
-        InvalidHexString(hex::FromHexError),
+        InvalidHexString(data_encoding::DecodeError),
         #[error("Invalid sha256 checksum: {0}")]
         InvalidSha256(TryFromSliceError),
         #[error("Invalid public key: {0}")]
         InvalidPublicKey(ParsePublicKeyError),
     }
 
-    impl From<hex::FromHexError> for HexKeyError {
-        fn from(err: hex::FromHexError) -> Self {
+    impl From<data_encoding::DecodeError> for HexKeyError {
+        fn from(err: data_encoding::DecodeError) -> Self {
             Self::InvalidHexString(err)
         }
     }

apps/src/lib/node/gossip/p2p/identity.rs

@@ -21,6 +21,7 @@ pub struct Identity {
 // TODO this is needed because libp2p does not export ed255519 serde
 // feature maybe a MR for libp2p to export theses functions ?
 mod keypair_serde {
+    use data_encoding::HEXLOWER;
     use libp2p::identity::ed25519::Keypair;
     use serde::de::Error;
     use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -33,15 +34,16 @@ mod keypair_serde {
         S: Serializer,
     {
         let bytes = value.encode();
-        let string = hex::encode(&bytes[..]);
+        let string = HEXLOWER.encode(&bytes[..]);
         string.serialize(serializer)
     }
     pub fn deserialize<'d, D>(deserializer: D) -> Result<Keypair, D::Error>
     where
         D: Deserializer<'d>,
     {
         let string = String::deserialize(deserializer)?;
-        let mut bytes = hex::decode(&string).map_err(Error::custom)?;
+        let mut bytes =
+            HEXLOWER.decode(string.as_ref()).map_err(Error::custom)?;
         Keypair::decode(bytes.as_mut()).map_err(Error::custom)
     }
 }

apps/src/lib/node/ledger/protocol/mod.rs

@@ -249,10 +249,13 @@ where
                 }
                 Address::Internal(internal_addr) => {
                     let ctx = native_vp::Ctx::new(
+                        addr,
                         storage,
                         write_log,
                         tx,
                         gas_meter,
+                        &keys_changed,
+                        &verifiers,
                         vp_wasm_cache.clone(),
                     );
                     let tx_data = match tx.data.as_ref() {

apps/src/lib/node/ledger/storage/rocksdb.rs

@@ -806,24 +806,36 @@ impl<'iter> DBIter<'iter> for RocksDB {
         &'iter self,
         prefix: &Key,
     ) -> PersistentPrefixIterator<'iter> {
-        let db_prefix = "subspace/".to_owned();
-        let prefix = format!("{}{}", db_prefix, prefix);
+        iter_prefix(self, prefix, Direction::Forward)
+    }
 
-        let mut read_opts = ReadOptions::default();
-        // don't use the prefix bloom filter
-        read_opts.set_total_order_seek(true);
-        let mut upper_prefix = prefix.clone().into_bytes();
-        if let Some(last) = upper_prefix.pop() {
-            upper_prefix.push(last + 1);
-        }
-        read_opts.set_iterate_upper_bound(upper_prefix);
+    fn rev_iter_prefix(&'iter self, prefix: &Key) -> Self::PrefixIter {
+        iter_prefix(self, prefix, Direction::Reverse)
+    }
+}
 
-        let iter = self.0.iterator_opt(
-            IteratorMode::From(prefix.as_bytes(), Direction::Forward),
-            read_opts,
-        );
-        PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
+fn iter_prefix<'iter>(
+    db: &'iter RocksDB,
+    prefix: &Key,
+    direction: Direction,
+) -> PersistentPrefixIterator<'iter> {
+    let db_prefix = "subspace/".to_owned();
+    let prefix = format!("{}{}", db_prefix, prefix);
+
+    let mut read_opts = ReadOptions::default();
+    // don't use the prefix bloom filter
+    read_opts.set_total_order_seek(true);
+    let mut upper_prefix = prefix.clone().into_bytes();
+    if let Some(last) = upper_prefix.pop() {
+        upper_prefix.push(last + 1);
     }
+    read_opts.set_iterate_upper_bound(upper_prefix);
+
+    let iter = db.0.iterator_opt(
+        IteratorMode::From(prefix.as_bytes(), direction),
+        read_opts,
+    );
+    PersistentPrefixIterator(PrefixIterator::new(iter, db_prefix))
 }
 
 #[derive(Debug)]

apps/src/lib/wallet/keys.rs

@@ -5,6 +5,7 @@ use std::rc::Rc;
 use std::str::FromStr;
 
 use borsh::{BorshDeserialize, BorshSerialize};
+use data_encoding::HEXLOWER;
 use namada::types::key::*;
 use orion::{aead, kdf};
 use serde::{Deserialize, Serialize};
@@ -108,15 +109,15 @@ pub struct EncryptedKeypair(Vec<u8>);
 
 impl Display for EncryptedKeypair {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", hex::encode(&self.0))
+        write!(f, "{}", HEXLOWER.encode(self.0.as_ref()))
     }
 }
 
 impl FromStr for EncryptedKeypair {
-    type Err = hex::FromHexError;
+    type Err = data_encoding::DecodeError;
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
-        hex::decode(s).map(Self)
+        HEXLOWER.decode(s.as_ref()).map(Self)
     }
 }
 

apps/src/lib/wasm_loader/mod.rs

@@ -4,8 +4,8 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
+use data_encoding::HEXLOWER;
 use futures::future::join_all;
-use hex;
 use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 use thiserror::Error;
@@ -144,7 +144,7 @@ pub async fn pre_fetch_wasm(wasm_directory: impl AsRef<Path>) {
                 Ok(bytes) => {
                     let mut hasher = Sha256::new();
                     hasher.update(bytes);
-                    let result = hex::encode(hasher.finalize());
+                    let result = HEXLOWER.encode(&hasher.finalize());
                     let derived_name = format!(
                         "{}.{}.wasm",
                         &name.split('.').collect::<Vec<&str>>()[0],