Marge fixes to test_harness, add serdehex crate

eth2/utils/bls/Cargo.toml

@@ -10,4 +10,5 @@ hashing = { path = "../hashing" }
 hex = "0.3"
 serde = "1.0"
 serde_derive = "1.0"
+serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }

eth2/utils/bls/src/aggregate_signature.rs

@@ -2,7 +2,9 @@ use super::{AggregatePublicKey, Signature};
 use bls_aggregates::{
     AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
 };
+use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{
     decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
 };
@@ -82,7 +84,19 @@ impl Serialize for AggregateSignature {
     where
         S: Serializer,
     {
-        serializer.serialize_bytes(&ssz_encode(self))
+        serializer.serialize_str(&hex_encode(ssz_encode(self)))
+    }
+}
+
+impl<'de> Deserialize<'de> for AggregateSignature {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let (obj, _) = <_>::ssz_decode(&bytes[..], 0)
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+        Ok(obj)
     }
 }
 

eth2/utils/bls/src/public_key.rs

@@ -1,9 +1,8 @@
-use super::serde_vistors::HexVisitor;
 use super::SecretKey;
 use bls_aggregates::PublicKey as RawPublicKey;
-use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
 use ssz::{
     decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
 };
@@ -81,7 +80,7 @@ impl Serialize for PublicKey {
     where
         S: Serializer,
     {
-        serializer.serialize_str(&hex_encode(ssz_encode(self)))
+        serializer.serialize_str(&hex_encode(self.as_raw().as_bytes()))
     }
 }
 
@@ -90,10 +89,10 @@ impl<'de> Deserialize<'de> for PublicKey {
     where
         D: Deserializer<'de>,
     {
-        let bytes = deserializer.deserialize_str(HexVisitor)?;
-        let (pubkey, _) = <_>::ssz_decode(&bytes[..], 0)
-            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
-        Ok(pubkey)
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let obj = PublicKey::from_bytes(&bytes[..])
+            .map_err(|e| serde::de::Error::custom(format!("invalid pubkey ({:?})", e)))?;
+        Ok(obj)
     }
 }
 

eth2/utils/boolean-bitfield/Cargo.toml

@@ -5,6 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
+serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"
 serde = "1.0"

eth2/utils/boolean-bitfield/src/lib.rs

@@ -3,7 +3,10 @@ extern crate ssz;
 
 use bit_vec::BitVec;
 
+use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode, PrefixedHexVisitor};
+use ssz::Decodable;
 use std::cmp;
 use std::default;
 
@@ -178,11 +181,25 @@ impl ssz::Decodable for BooleanBitfield {
 }
 
 impl Serialize for BooleanBitfield {
+    /// Serde serialization is compliant the Ethereum YAML test format.
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
     where
         S: Serializer,
     {
-        serializer.serialize_bytes(&ssz::ssz_encode(self))
+        serializer.serialize_str(&encode(&ssz::ssz_encode(self)))
+    }
+}
+
+impl<'de> Deserialize<'de> for BooleanBitfield {
+    /// Serde serialization is compliant the Ethereum YAML test format.
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
+        let (bitfield, _) = <_>::ssz_decode(&bytes[..], 0)
+            .map_err(|e| serde::de::Error::custom(format!("invalid ssz ({:?})", e)))?;
+        Ok(bitfield)
     }
 }
 

eth2/utils/serde_hex/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "serde_hex"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+serde = "1.0"
+hex = "0.3"

eth2/utils/serde_hex/src/lib.rs

@@ -0,0 +1,59 @@
+use hex;
+use hex::ToHex;
+use serde::de::{self, Visitor};
+use std::fmt;
+
+pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
+    let mut hex = String::with_capacity(data.as_ref().len() * 2);
+
+    // Writing to a string never errors, so we can unwrap here.
+    data.write_hex(&mut hex).unwrap();
+
+    let mut s = "0x".to_string();
+
+    s.push_str(hex.as_str());
+
+    s
+}
+
+pub struct PrefixedHexVisitor;
+
+impl<'de> Visitor<'de> for PrefixedHexVisitor {
+    type Value = Vec<u8>;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a hex string with 0x prefix")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+    where
+        E: de::Error,
+    {
+        if value.starts_with("0x") {
+            Ok(hex::decode(&value[2..])
+                .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?)
+        } else {
+            Err(de::Error::custom("missing 0x prefix"))
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn encoding() {
+        let bytes = vec![0, 255];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x00ff");
+
+        let bytes = vec![];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x");
+
+        let bytes = vec![1, 2, 3];
+        let hex = encode(&bytes);
+        assert_eq!(hex.as_str(), "0x010203");
+    }
+}

eth2/utils/ssz/src/impl_decode.rs

@@ -24,11 +24,30 @@ macro_rules! impl_decodable_for_uint {
     };
 }
 
+macro_rules! impl_decodable_for_u8_array {
+    ($len: expr) => {
+        impl Decodable for [u8; $len] {
+            fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+                if index + $len > bytes.len() {
+                    Err(DecodeError::TooShort)
+                } else {
+                    let mut array: [u8; $len] = [0; $len];
+                    array.copy_from_slice(&bytes[index..index + $len]);
+
+                    Ok((array, index + $len))
+                }
+            }
+        }
+    };
+}
+
 impl_decodable_for_uint!(u16, 16);
 impl_decodable_for_uint!(u32, 32);
 impl_decodable_for_uint!(u64, 64);
 impl_decodable_for_uint!(usize, 64);
 
+impl_decodable_for_u8_array!(4);
+
 impl Decodable for u8 {
     fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
         if index >= bytes.len() {
@@ -246,4 +265,12 @@ mod tests {
         let result: Result<(bool, usize), DecodeError> = decode_ssz(&ssz, 0);
         assert_eq!(result, Err(DecodeError::Invalid));
     }
+
+    #[test]
+    fn test_decode_u8_array() {
+        let ssz = vec![0, 1, 2, 3];
+        let (result, index): ([u8; 4], usize) = decode_ssz(&ssz, 0).unwrap();
+        assert_eq!(index, 4);
+        assert_eq!(result, [0, 1, 2, 3]);
+    }
 }

eth2/utils/ssz/src/impl_encode.rs

@@ -40,12 +40,25 @@ macro_rules! impl_encodable_for_uint {
     };
 }
 
+macro_rules! impl_encodable_for_u8_array {
+    ($len: expr) => {
+        impl Encodable for [u8; $len] {
+            fn ssz_append(&self, s: &mut SszStream) {
+                let bytes: Vec<u8> = self.iter().cloned().collect();
+                s.append_encoded_raw(&bytes);
+            }
+        }
+    };
+}
+
 impl_encodable_for_uint!(u8, 8);
 impl_encodable_for_uint!(u16, 16);
 impl_encodable_for_uint!(u32, 32);
 impl_encodable_for_uint!(u64, 64);
 impl_encodable_for_uint!(usize, 64);
 
+impl_encodable_for_u8_array!(4);
+
 impl Encodable for bool {
     fn ssz_append(&self, s: &mut SszStream) {
         let byte = if *self { 0b1000_0000 } else { 0b0000_0000 };
@@ -77,6 +90,7 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::ssz_encode;
 
     #[test]
     fn test_ssz_encode_h256() {
@@ -226,4 +240,15 @@ mod tests {
         ssz.append(&x);
         assert_eq!(ssz.drain(), vec![0b1000_0000]);
     }
+
+    #[test]
+    fn test_ssz_encode_u8_array() {
+        let x: [u8; 4] = [0, 1, 7, 8];
+        let ssz = ssz_encode(&x);
+        assert_eq!(ssz, vec![0, 1, 7, 8]);
+
+        let x: [u8; 4] = [255, 255, 255, 255];
+        let ssz = ssz_encode(&x);
+        assert_eq!(ssz, vec![255, 255, 255, 255]);
+    }
 }