[Merged by Bors] - add quoted serialization util for FixedVector and VariableList

consensus/serde_utils/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2018"
 [dependencies]
 serde = { version = "1.0.116", features = ["derive"] }
 serde_derive = "1.0.116"
+eth2_ssz_types = {path = "../ssz_types"}
 hex = "0.4.2"
 
 [dev-dependencies]

consensus/serde_utils/src/lib.rs

@@ -2,6 +2,8 @@ mod quoted_int;
 
 pub mod bytes_4_hex;
 pub mod hex;
+pub mod quoted_u64_fixed_vec;
+pub mod quoted_u64_var_list;
 pub mod quoted_u64_vec;
 pub mod u32_hex;
 pub mod u8_hex;

consensus/serde_utils/src/quoted_u64_fixed_vec.rs

@@ -0,0 +1,129 @@
+//! Formats `FixedVector<u64,N>` using quotes.
+//!
+//! E.g., `FixedVector::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding. If `N` does not equal the length of the `Vec`, the `Vec` is truncated.
+
+use serde::ser::SerializeSeq;
+use serde::{Deserializer, Serializer};
+use serde_derive::{Deserialize, Serialize};
+use ssz_types::typenum::Unsigned;
+use ssz_types::FixedVector;
+use std::marker::PhantomData;
+
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct QuotedIntWrapper {
+    #[serde(with = "crate::quoted_u64")]
+    pub int: u64,
+}
+
+pub struct QuotedIntFixedVecVisitor<N> {
+    _phantom: PhantomData<N>,
+}
+
+impl<'a, N> serde::de::Visitor<'a> for QuotedIntFixedVecVisitor<N>
+where
+    N: Unsigned,
+{
+    type Value = FixedVector<u64, N>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of quoted or unquoted integers")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let mut vec = vec![];
+
+        while let Some(val) = seq.next_element()? {
+            let val: QuotedIntWrapper = val;
+            vec.push(val.int);
+        }
+        let fix: FixedVector<u64, N> = FixedVector::from(vec);
+        Ok(fix)
+    }
+}
+
+pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for &int in value {
+        seq.serialize_element(&QuotedIntWrapper { int })?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<FixedVector<u64, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    deserializer.deserialize_any(QuotedIntFixedVecVisitor {
+        _phantom: PhantomData,
+    })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use ssz_types::typenum::U4;
+
+    #[derive(Debug, Serialize, Deserialize)]
+    struct Obj {
+        #[serde(with = "crate::quoted_u64_fixed_vec")]
+        values: FixedVector<u64, U4>,
+    }
+
+    #[test]
+    fn quoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn unquoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn mixed_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn empty_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![0, 0, 0, 0]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn short_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2] }"#).unwrap();
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 0, 0]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn long_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap();
+        dbg!(&obj.values);
+        let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn whole_list_quoted_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
+    }
+}

consensus/serde_utils/src/quoted_u64_var_list.rs

@@ -0,0 +1,128 @@
+//! Formats `VariableList<u64,N>` using quotes.
+//!
+//! E.g., `VariableList::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
+//!
+//! Quotes can be optional during decoding. If `N` is greater than the length of the `Vec`, the `Vec` is truncated.
+
+use serde::ser::SerializeSeq;
+use serde::{Deserializer, Serializer};
+use serde_derive::{Deserialize, Serialize};
+use ssz_types::typenum::Unsigned;
+use ssz_types::VariableList;
+use std::marker::PhantomData;
+
+#[derive(Serialize, Deserialize)]
+#[serde(transparent)]
+pub struct QuotedIntWrapper {
+    #[serde(with = "crate::quoted_u64")]
+    pub int: u64,
+}
+
+pub struct QuotedIntVarListVisitor<N> {
+    _phantom: PhantomData<N>,
+}
+
+impl<'a, N> serde::de::Visitor<'a> for QuotedIntVarListVisitor<N>
+where
+    N: Unsigned,
+{
+    type Value = VariableList<u64, N>;
+
+    fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(formatter, "a list of quoted or unquoted integers")
+    }
+
+    fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
+    where
+        A: serde::de::SeqAccess<'a>,
+    {
+        let mut vec = vec![];
+
+        while let Some(val) = seq.next_element()? {
+            let val: QuotedIntWrapper = val;
+            vec.push(val.int);
+        }
+        let fix: VariableList<u64, N> = VariableList::from(vec);
+        Ok(fix)
+    }
+}
+
+pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut seq = serializer.serialize_seq(Some(value.len()))?;
+    for &int in value {
+        seq.serialize_element(&QuotedIntWrapper { int })?;
+    }
+    seq.end()
+}
+
+pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u64, N>, D::Error>
+where
+    D: Deserializer<'de>,
+    N: Unsigned,
+{
+    deserializer.deserialize_any(QuotedIntVarListVisitor {
+        _phantom: PhantomData,
+    })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use ssz_types::typenum::U4;
+
+    #[derive(Debug, Serialize, Deserialize)]
+    struct Obj {
+        #[serde(with = "crate::quoted_u64_var_list")]
+        values: VariableList<u64, U4>,
+    }
+
+    #[test]
+    fn quoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn unquoted_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn mixed_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn empty_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [] }"#).unwrap();
+        assert!(obj.values.is_empty());
+    }
+
+    #[test]
+    fn short_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2] }"#).unwrap();
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn long_list_success() {
+        let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap();
+        dbg!(&obj.values);
+        let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
+        assert_eq!(obj.values, expected);
+    }
+
+    #[test]
+    fn whole_list_quoted_err() {
+        serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
+    }
+}

consensus/ssz_types/Cargo.toml

@@ -9,12 +9,12 @@ name = "ssz_types"
 
 [dependencies]
 tree_hash = "0.1.1"
-serde = "1.0.116"
+serde = { version = "1.0.116", features = ["derive"] }
 serde_derive = "1.0.116"
-serde_utils = { path = "../serde_utils" }
 eth2_ssz = "0.1.2"
 typenum = "1.12.0"
 arbitrary = { version = "0.4.6", features = ["derive"], optional = true }
+hex = "0.4.2"
 
 [dev-dependencies]
 tree_hash_derive = "0.2.0"

consensus/ssz_types/src/bitfield.rs

@@ -1,10 +1,10 @@
 use crate::tree_hash::bitfield_bytes_tree_hash_root;
 use crate::Error;
 use core::marker::PhantomData;
-use serde::de::{Deserialize, Deserializer};
+use serde::de::{self, Deserialize, Deserializer, Visitor};
 use serde::ser::{Serialize, Serializer};
-use serde_utils::hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{Decode, Encode};
+use std::fmt;
 use tree_hash::Hash256;
 use typenum::Unsigned;
 
@@ -617,6 +617,36 @@ impl<N: Unsigned + Clone> tree_hash::TreeHash for Bitfield<Fixed<N>> {
     }
 }
 
+/// Encode `data` as a 0x-prefixed hex string.
+fn hex_encode<T: AsRef<[u8]>>(data: T) -> String {
+    let hex = hex::encode(data);
+    let mut s = "0x".to_string();
+    s.push_str(hex.as_str());
+    s
+}
+
+struct PrefixedHexVisitor;
+
+impl<'de> Visitor<'de> for PrefixedHexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string with 0x prefix")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.starts_with("0x") {
+            Ok(hex::decode(&value[2..])
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+        } else {
+            Err(de::Error::custom("missing 0x prefix"))
+        }
+    }
+}
+
 #[cfg(feature = "arbitrary")]
 impl<N: 'static + Unsigned> arbitrary::Arbitrary for Bitfield<Fixed<N>> {
     fn arbitrary(u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {

consensus/types/src/beacon_state.rs

@@ -181,12 +181,14 @@ where
     #[compare_fields(as_slice)]
     pub validators: VariableList<Validator, T::ValidatorRegistryLimit>,
     #[compare_fields(as_slice)]
+    #[serde(with = "serde_utils::quoted_u64_var_list")]
     pub balances: VariableList<u64, T::ValidatorRegistryLimit>,
 
     // Randomness
     pub randao_mixes: FixedVector<Hash256, T::EpochsPerHistoricalVector>,
 
     // Slashings
+    #[serde(with = "serde_utils::quoted_u64_fixed_vec")]
     pub slashings: FixedVector<u64, T::EpochsPerSlashingsVector>,
 
     // Attestations