Merge branch 'master' into romac/upgrade-tokio

.gitignore

@@ -11,3 +11,6 @@ Cargo.lock
 
 # These are log files emitted by model-based tests
 **/*.log
+
+# RPC probe results
+/rpc-probe/probe-results/

CHANGELOG.md

@@ -1,22 +1,42 @@
 ## Unreleased
 
-### BREAKING CHANGES:
+### BREAKING CHANGES
 
-- `[tendermint-proto]` The `DomainType` trait has now been renamed to
-  `Protobuf` to clarify its purpose throughout the codebase. ([#672])
 - `[tendermint-rpc, tendermint-light-client]` Upgrade Tokio to version 0.3.0 ([#683])
   - Upgrade `hyper` to `v0.14-dev`
   - Upgrade `async-tungstenite` to `v0.10`
+
+[#683]: https://github.com/informalsystems/tendermint-rs/issues/683
+
+## v0.17.0-rc3
+
+*Nov 18, 2020*
+
+This release candidate focuses primarily on small but important fixes to our
+serialization infrastructure.
+
+### BREAKING CHANGES:
+
+- `[tendermint-proto]` The `DomainType` trait has now been renamed to
+  `Protobuf` to clarify its purpose throughout the codebase. ([#672])
 
 ### BUG FIXES:
 
 - `[tendermint]` (Since v0.17.0-rc2) Fix abci::Data serialization to base64-encoded string. ([#667])
 - `[tendermint]` (Since v0.17.0-rc2) Simplify abci::Log serialization ([#667])
+- `[tendermint]` (Since v0.17.0-rc2) consensus::State backwards compatibility for deserialization ([#679])
 
+### IMPROVEMENTS:
+
+- `[rpc]` A new RPC probe (in the `rpc-probe` directory) has been added to
+  facilitate quick, pre-scripted interactions with a Tendermint node (via its
+  WebSocket endpoint). This aims to help improve testing and compatibility
+  between Tendermint in Go and Rust. ([#653])
+
+[#653]: https://github.com/informalsystems/tendermint-rs/pull/653
 [#667]: https://github.com/informalsystems/tendermint-rs/issues/667
 [#672]: https://github.com/informalsystems/tendermint-rs/pull/672
-[#683]: https://github.com/informalsystems/tendermint-rs/issues/683
-
+[#679]: https://github.com/informalsystems/tendermint-rs/issues/679
 
 ## v0.17.0-rc2
 

Cargo.toml

@@ -5,6 +5,7 @@ members = [
     "light-node",
     "proto",
     "rpc",
+    "rpc-probe",
     "tendermint",
     "testgen"
 ]

light-client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name       = "tendermint-light-client"
-version    = "0.17.0-rc2"
+version    = "0.17.0-rc3"
 edition    = "2018"
 license    = "Apache-2.0"
 readme     = "README.md"
@@ -27,8 +27,8 @@ rpc-client = ["tendermint-rpc/http-client"]
 secp256k1 = ["tendermint/secp256k1", "tendermint-rpc/secp256k1"]
 
 [dependencies]
-tendermint = { version = "0.17.0-rc2", path = "../tendermint" }
-tendermint-rpc = { version = "0.17.0-rc2", path = "../rpc", default-features = false }
+tendermint = { version = "0.17.0-rc3", path = "../tendermint" }
+tendermint-rpc = { version = "0.17.0-rc3", path = "../rpc", default-features = false }
 
 anomaly = { version = "0.2.0", features = ["serializer"] }
 contracts = "0.4.0"

light-client/src/lib.rs

@@ -10,7 +10,7 @@
     nonstandard_style
 )]
 #![doc(
-    html_root_url = "https://docs.rs/tendermint-light-client/0.17.0-rc2",
+    html_root_url = "https://docs.rs/tendermint-light-client/0.17.0-rc3",
     html_logo_url = "https://raw.githubusercontent.com/informalsystems/tendermint-rs/master/img/logo-tendermint-rs_3961x4001.png"
 )]
 

light-node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name       = "tendermint-light-node"
-version    = "0.17.0-rc2"
+version    = "0.17.0-rc3"
 edition    = "2018"
 license    = "Apache-2.0"
 repository = "https://github.com/informalsystems/tendermint-rs"
@@ -37,9 +37,9 @@ jsonrpc-derive = "14.2"
 serde = { version = "1", features = ["serde_derive"] }
 serde_json = "1.0"
 sled = "0.34.3"
-tendermint = { version = "0.17.0-rc2", path = "../tendermint" }
-tendermint-light-client = { version = "0.17.0-rc2", path = "../light-client" }
-tendermint-rpc = { version = "0.17.0-rc2", path = "../rpc", features = [ "http-client" ] }
+tendermint = { version = "0.17.0-rc3", path = "../tendermint" }
+tendermint-light-client = { version = "0.17.0-rc3", path = "../light-client" }
+tendermint-rpc = { version = "0.17.0-rc3", path = "../rpc", features = [ "http-client" ] }
 thiserror = "1.0"
 tokio = { version = "0.3", features = ["full"] }
 

light-node/src/lib.rs

@@ -15,7 +15,7 @@
     unused_qualifications
 )]
 #![doc(
-    html_root_url = "https://docs.rs/tendermint-light-node/0.17.0-rc2",
+    html_root_url = "https://docs.rs/tendermint-light-node/0.17.0-rc3",
     html_logo_url = "https://raw.githubusercontent.com/informalsystems/tendermint-rs/master/img/logo-tendermint-rs_3961x4001.png"
 )]
 

proto-compiler/src/constants.rs

@@ -25,6 +25,7 @@ const NULLABLEVECARRAY: &str = r#"#[serde(with = "crate::serializers::txs")]"#;
 const NULLABLE: &str = r#"#[serde(with = "crate::serializers::nullable")]"#;
 const ALIAS_POWER_QUOTED: &str =
     r#"#[serde(alias = "power", with = "crate::serializers::from_str")]"#;
+const PART_SET_HEADER_TOTAL: &str = r#"#[serde(with = "crate::serializers::part_set_header_total")]"#;
 const RENAME_PUBKEY: &str = r#"#[serde(rename = "tendermint/PubKeyEd25519", with = "crate::serializers::bytes::base64string")]"#;
 const RENAME_DUPLICATEVOTE: &str = r#"#[serde(rename = "tendermint/DuplicateVoteEvidence")]"#;
 const RENAME_LIGHTCLIENTATTACK: &str =
@@ -97,6 +98,7 @@ pub static CUSTOM_FIELD_ATTRIBUTES: &[(&str, &str)] = &[
         ".tendermint.types.CanonicalBlockID.part_set_header",
         ALIAS_PARTS,
     ),
+    (".tendermint.types.PartSetHeader.total", PART_SET_HEADER_TOTAL),
     (".tendermint.types.PartSetHeader.hash", HEXSTRING),
     (".tendermint.types.Header.height", QUOTED),
     (".tendermint.types.Header.time", OPTIONAL),

proto/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tendermint-proto"
-version = "0.17.0-rc2"
+version = "0.17.0-rc3"
 authors = ["Greg Szabo <[email protected]>"]
 edition = "2018"
 license = "Apache-2.0"

proto/src/lib.rs

@@ -3,7 +3,7 @@
 #![deny(warnings, trivial_casts, trivial_numeric_casts, unused_import_braces)]
 #![allow(clippy::large_enum_variant)]
 #![forbid(unsafe_code)]
-#![doc(html_root_url = "https://docs.rs/tendermint-proto/0.17.0-rc2")]
+#![doc(html_root_url = "https://docs.rs/tendermint-proto/0.17.0-rc3")]
 
 // Built-in prost_types with slight customization to enable JSON-encoding
 pub mod google {

proto/src/prost/tendermint.types.rs

@@ -35,6 +35,7 @@ pub struct SimpleValidator {
 #[derive(::serde::Deserialize, ::serde::Serialize)]
 pub struct PartSetHeader {
     #[prost(uint32, tag="1")]
+    #[serde(with = "crate::serializers::part_set_header_total")]
     pub total: u32,
     #[prost(bytes, tag="2")]
     #[serde(with = "crate::serializers::bytes::hexstring")]

proto/src/serializers.rs

@@ -49,6 +49,7 @@ pub mod evidence;
 pub mod from_str;
 pub mod nullable;
 pub mod optional;
+pub mod part_set_header_total;
 pub mod time_duration;
 pub mod timestamp;
 pub mod txs;

proto/src/serializers/from_str.rs

@@ -1,6 +1,6 @@
 //! Serialize and deserialize any `T` that implements [[std::str::FromStr]]
-//! and [[std::fmt::Display]] from or into string. Note this be used for
-//! all primitive data types (e.g. .
+//! and [[std::fmt::Display]] from or into string. Note this can be used for
+//! all primitive data types.
 use serde::{de::Error as _, Deserialize, Deserializer, Serialize, Serializer};
 
 /// Deserialize string into T

proto/src/serializers/optional.rs

@@ -1,4 +1,6 @@
 //! Serialize/deserialize Option<T> type where T has a serializer/deserializer.
+//! Use `null` if the received value equals the Default implementation.
+// Todo: Rename this serializer to something like "default_eq_none" to mirror its purpose better.
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 
 /// Deserialize Option<T>
@@ -7,7 +9,7 @@ where
     D: Deserializer<'de>,
     T: Deserialize<'de> + Default + PartialEq,
 {
-    Ok(Some(T::deserialize(deserializer)?).filter(|t| t != &T::default()))
+    Ok(Option::<T>::deserialize(deserializer)?.filter(|t| t != &T::default()))
 }
 
 /// Serialize Option<T>

proto/src/serializers/part_set_header_total.rs

@@ -0,0 +1,51 @@
+//! Serialize and deserialize part_set_header.total (from string or u32), (into u32 in
+//! part_set_header.total).
+//!
+//! The deserializer is created for backwards compatibility: `total` was changed from a
+//! string-quoted integer value into an integer value without quotes in Tendermint Core v0.34.0.
+//! This deserializer allows backwards-compatibility by deserializing both ways.
+//! See also: https://github.com/informalsystems/tendermint-rs/issues/679
+use serde::{de::Error, de::Visitor, Deserializer, Serialize, Serializer};
+use std::convert::TryFrom;
+use std::fmt::Formatter;
+
+struct PartSetHeaderTotalStringOrU32;
+
+/// Deserialize (string or u32) into u32(part_set_header.total)
+pub fn deserialize<'de, D>(deserializer: D) -> Result<u32, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    deserializer.deserialize_any(PartSetHeaderTotalStringOrU32)
+}
+
+/// Serialize from u32(part_set_header.total) into u32
+pub fn serialize<S>(value: &u32, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    value.serialize(serializer)
+}
+
+impl<'de> Visitor<'de> for PartSetHeaderTotalStringOrU32 {
+    type Value = u32;
+
+    fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
+        formatter.write_str("an u32 integer or string between 0 and 2^32")
+    }
+
+    fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+    where
+        E: Error,
+    {
+        u32::try_from(v).map_err(|e| E::custom(format!("part_set_header.total {}", e)))
+    }
+
+    fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+    where
+        E: Error,
+    {
+        v.parse::<u32>()
+            .map_err(|e| E::custom(format!("part_set_header.total {}", e)))
+    }
+}

rpc-probe/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "tendermint-rpc-probe"
+version = "0.17.0-rc3"
+authors = ["Thane Thomson <[email protected]>"]
+edition = "2018"
+license    = "Apache-2.0"
+homepage   = "https://www.tendermint.com/"
+repository = "https://github.com/informalsystems/tendermint-rs"
+readme     = "README.md"
+
+description = """
+    tendermint-rpc-probe allows us to probe a running Tendermint instance with
+    a given sequence of requests, capturing those requests and/or responses.
+    """
+
+[package.metadata.docs.rs]
+all-features = true
+
+[dependencies]
+async-tungstenite = { version = "0.10", features = [ "tokio-runtime" ] }
+futures = "0.3"
+getrandom = "0.1"
+log = "0.4"
+serde = { version = "1", features = [ "derive" ] }
+serde_json = "1"
+simple_logger = "1.11"
+structopt = "0.3"
+subtle-encoding = "0.5.1"
+thiserror = "1.0"
+tokio = { version = "0.3", features = [ "full" ] }
+uuid = "0.8"

rpc-probe/README.md

@@ -0,0 +1,95 @@
+# Tendermint RPC Probe
+
+The Tendermint RPC probe is an application that assists in testing the various
+crates in this repository. It currently allows you to execute a quick probe of
+a running [Tendermint] node, where a quick probe executes requests against all
+of the [Tendermint RPC] endpoints (including subscriptions for different event
+types), and saves all of the responses it gets as JSON files. These JSON files
+can be used in testing in other crates.
+
+## Requirements
+
+To run this probe locally, you will need:
+
+* The Rust toolchain (latest stable)
+* Docker
+
+## Usage (with Docker)
+
+From the root of the tendermint.rs repository:
+
+```bash
+cd rpc-probe
+./run-with-docker.sh
+```
+
+This will:
+
+1. Build the `tendermint-rpc-probe` executable
+2. Pull the latest version of the Tendermint Docker image
+3. Initialize and run a Tendermint node with the `kvstore` app in the
+   background. (This node exposes a WebSocket endpoint at
+   `ws://127.0.0.1:26657/websocket`)
+4. Run `tendermint-rpc-probe` against the running Tendermint node.
+5. Terminate the Docker image.
+
+To run a specific version of Tendermint, simply:
+
+```bash
+TENDERMINT_TAG="v0.33.8" ./run-with-docker.sh
+```
+
+## Usage (without Docker)
+
+Simply run:
+
+```bash
+cargo run -- --help
+```
+
+to see what options are available to run the probe.
+
+For example:
+
+```bash
+# Executes the probe with all default options (i.e. against a Tendermint node
+# listening on 127.0.0.1:26657)
+cargo run
+
+# Customize the address
+cargo run -- --addr ws://192.168.1.15:26657/websocket
+
+# Customize how long to wait before each request (in milliseconds)
+# Defaults to 1000ms
+cargo run -- --request-wait 100
+```
+
+## Output
+
+By default, all request and response JSON-RPC messages will be written into a
+folder called `probe-results` in the `rpc-probe` directory.
+
+For example, the `probe-results/incoming/abci_info.json` file (returned by the
+[`abci_info`] RPC request) could look something like:
+
+```json
+{
+  "id": "8944f639-7da0-4595-ac5e-3e432079f510",
+  "jsonrpc": "2.0",
+  "result": {
+    "response": {
+      "app_version": "1",
+      "data": "{\"size\":0}",
+      "last_block_app_hash": "AAAAAAAAAAA=",
+      "last_block_height": "13",
+      "version": "0.17.0"
+    }
+  }
+}
+```
+
+The full JSON-RPC wrapper is saved to disk.
+
+[Tendermint]: https://github.com/tendermint/tendermint
+[Tendermint RPC]: https://docs.tendermint.com/master/rpc/
+[`abci_info`]: https://docs.tendermint.com/master/rpc/#/ABCI/abci_info