Merge branch 'v0.6.1' into docker-env

eth2/utils/bls/Cargo.toml

@@ -5,10 +5,11 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
+milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.9.0" }
 cached_tree_hash = { path = "../cached_tree_hash" }
 hashing = { path = "../hashing" }
 hex = "0.3"
+rand = "^0.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_hex = { path = "../serde_hex" }

eth2/utils/bls/src/aggregate_public_key.rs

@@ -1,5 +1,5 @@
 use super::PublicKey;
-use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey;
+use milagro_bls::AggregatePublicKey as RawAggregatePublicKey;
 
 /// A BLS aggregate public key.
 ///

eth2/utils/bls/src/aggregate_signature.rs

@@ -1,8 +1,8 @@
 use super::*;
-use bls_aggregates::{
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
+use milagro_bls::{
     AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
 };
-use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};

eth2/utils/bls/src/fake_aggregate_public_key.rs

@@ -0,0 +1,36 @@
+use super::{PublicKey, BLS_PUBLIC_KEY_BYTE_SIZE};
+use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey;
+
+/// A BLS aggregate public key.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, Clone, Default)]
+pub struct FakeAggregatePublicKey {
+    bytes: Vec<u8>,
+}
+
+impl FakeAggregatePublicKey {
+    pub fn new() -> Self {
+        Self::zero()
+    }
+
+    /// Creates a new all-zero's aggregate public key
+    pub fn zero() -> Self {
+        Self {
+            bytes: vec![0; BLS_PUBLIC_KEY_BYTE_SIZE],
+        }
+    }
+
+    pub fn add(&mut self, _public_key: &PublicKey) {
+        // No nothing.
+    }
+
+    pub fn as_raw(&self) -> &FakeAggregatePublicKey {
+        &self
+    }
+
+    pub fn as_bytes(&self) -> Vec<u8> {
+        self.bytes.clone()
+    }
+}

eth2/utils/bls/src/fake_aggregate_signature.rs

@@ -1,4 +1,7 @@
-use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_SIZE};
+use super::{
+    fake_aggregate_public_key::FakeAggregatePublicKey, fake_signature::FakeSignature,
+    BLS_AGG_SIG_BYTE_SIZE,
+};
 use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -43,7 +46,7 @@ impl FakeAggregateSignature {
         &self,
         _msg: &[u8],
         _domain: u64,
-        _aggregate_public_key: &AggregatePublicKey,
+        _aggregate_public_key: &FakeAggregatePublicKey,
     ) -> bool {
         true
     }
@@ -53,7 +56,7 @@ impl FakeAggregateSignature {
         &self,
         _messages: &[&[u8]],
         _domain: u64,
-        _aggregate_public_keys: &[&AggregatePublicKey],
+        _aggregate_public_keys: &[&FakeAggregatePublicKey],
     ) -> bool {
         true
     }

eth2/utils/bls/src/fake_public_key.rs

@@ -0,0 +1,169 @@
+use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE};
+use bls_aggregates::PublicKey as RawPublicKey;
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
+use serde::de::{Deserialize, Deserializer};
+use serde::ser::{Serialize, Serializer};
+use serde_hex::{encode as hex_encode, HexVisitor};
+use ssz::{ssz_encode, Decode, DecodeError};
+use std::default;
+use std::fmt;
+use std::hash::{Hash, Hasher};
+use tree_hash::tree_hash_ssz_encoding_as_vector;
+
+/// A single BLS signature.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, Clone, Eq)]
+pub struct FakePublicKey {
+    bytes: Vec<u8>,
+}
+
+impl FakePublicKey {
+    pub fn from_secret_key(_secret_key: &SecretKey) -> Self {
+        Self::zero()
+    }
+
+    /// Creates a new all-zero's public key
+    pub fn zero() -> Self {
+        Self {
+            bytes: vec![0; BLS_PUBLIC_KEY_BYTE_SIZE],
+        }
+    }
+
+    /// Returns the underlying point as compressed bytes.
+    ///
+    /// Identical to `self.as_uncompressed_bytes()`.
+    pub fn as_bytes(&self) -> Vec<u8> {
+        self.bytes.clone()
+    }
+
+    /// Converts compressed bytes to FakePublicKey
+    pub fn from_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        Ok(Self {
+            bytes: bytes.to_vec(),
+        })
+    }
+
+    /// Returns the FakePublicKey as (x, y) bytes
+    pub fn as_uncompressed_bytes(&self) -> Vec<u8> {
+        self.as_bytes()
+    }
+
+    /// Converts (x, y) bytes to FakePublicKey
+    pub fn from_uncompressed_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        Self::from_bytes(bytes)
+    }
+
+    /// Returns the last 6 bytes of the SSZ encoding of the public key, as a hex string.
+    ///
+    /// Useful for providing a short identifier to the user.
+    pub fn concatenated_hex_id(&self) -> String {
+        let bytes = ssz_encode(self);
+        let end_bytes = &bytes[bytes.len().saturating_sub(6)..bytes.len()];
+        hex_encode(end_bytes)
+    }
+
+    // Returns itself
+    pub fn as_raw(&self) -> &Self {
+        self
+    }
+}
+
+impl fmt::Display for FakePublicKey {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.concatenated_hex_id())
+    }
+}
+
+impl default::Default for FakePublicKey {
+    fn default() -> Self {
+        let secret_key = SecretKey::random();
+        FakePublicKey::from_secret_key(&secret_key)
+    }
+}
+
+impl_ssz!(FakePublicKey, BLS_PUBLIC_KEY_BYTE_SIZE, "FakePublicKey");
+
+impl Serialize for FakePublicKey {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.serialize_str(&hex_encode(self.as_bytes()))
+    }
+}
+
+impl<'de> Deserialize<'de> for FakePublicKey {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let bytes = deserializer.deserialize_str(HexVisitor)?;
+        let pubkey = Self::from_ssz_bytes(&bytes[..])
+            .map_err(|e| serde::de::Error::custom(format!("invalid pubkey ({:?})", e)))?;
+        Ok(pubkey)
+    }
+}
+
+tree_hash_ssz_encoding_as_vector!(FakePublicKey);
+cached_tree_hash_ssz_encoding_as_vector!(FakePublicKey, 48);
+
+impl PartialEq for FakePublicKey {
+    fn eq(&self, other: &FakePublicKey) -> bool {
+        ssz_encode(self) == ssz_encode(other)
+    }
+}
+
+impl Hash for FakePublicKey {
+    /// Note: this is distinct from consensus serialization, it will produce a different hash.
+    ///
+    /// This method uses the uncompressed bytes, which are much faster to obtain than the
+    /// compressed bytes required for consensus serialization.
+    ///
+    /// Use `ssz::Encode` to obtain the bytes required for consensus hashing.
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.as_uncompressed_bytes().hash(state)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use ssz::ssz_encode;
+    use tree_hash::TreeHash;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let sk = SecretKey::random();
+        let original = FakePublicKey::from_secret_key(&sk);
+
+        let bytes = ssz_encode(&original);
+        let decoded = FakePublicKey::from_ssz_bytes(&bytes).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let sk = SecretKey::random();
+        let original = FakePublicKey::from_secret_key(&sk);
+
+        let mut cache = cached_tree_hash::TreeHashCache::new(&original).unwrap();
+
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            original.tree_hash_root()
+        );
+
+        let sk = SecretKey::random();
+        let modified = FakePublicKey::from_secret_key(&sk);
+
+        cache.update(&modified).unwrap();
+
+        assert_eq!(
+            cache.tree_hash_root().unwrap().to_vec(),
+            modified.tree_hash_root()
+        );
+    }
+}

eth2/utils/bls/src/lib.rs

@@ -1,35 +1,52 @@
-extern crate bls_aggregates;
+extern crate milagro_bls;
 extern crate ssz;
 
 #[macro_use]
 mod macros;
-mod aggregate_public_key;
 mod keypair;
-mod public_key;
 mod secret_key;
 
-#[cfg(not(feature = "fake_crypto"))]
-mod aggregate_signature;
-#[cfg(not(feature = "fake_crypto"))]
-mod signature;
-#[cfg(not(feature = "fake_crypto"))]
-pub use crate::aggregate_signature::AggregateSignature;
-#[cfg(not(feature = "fake_crypto"))]
-pub use crate::signature::Signature;
+pub use crate::keypair::Keypair;
+pub use crate::secret_key::SecretKey;
+pub use milagro_bls::{compress_g2, hash_on_g2};
 
+#[cfg(feature = "fake_crypto")]
+mod fake_aggregate_public_key;
 #[cfg(feature = "fake_crypto")]
 mod fake_aggregate_signature;
 #[cfg(feature = "fake_crypto")]
+mod fake_public_key;
+#[cfg(feature = "fake_crypto")]
 mod fake_signature;
-#[cfg(feature = "fake_crypto")]
-pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature;
-#[cfg(feature = "fake_crypto")]
-pub use crate::fake_signature::FakeSignature as Signature;
 
-pub use crate::aggregate_public_key::AggregatePublicKey;
-pub use crate::keypair::Keypair;
-pub use crate::public_key::PublicKey;
-pub use crate::secret_key::SecretKey;
+#[cfg(not(feature = "fake_crypto"))]
+mod aggregate_public_key;
+#[cfg(not(feature = "fake_crypto"))]
+mod aggregate_signature;
+#[cfg(not(feature = "fake_crypto"))]
+mod public_key;
+#[cfg(not(feature = "fake_crypto"))]
+mod signature;
+
+#[cfg(feature = "fake_crypto")]
+pub use fakes::*;
+#[cfg(feature = "fake_crypto")]
+mod fakes {
+    pub use crate::fake_aggregate_public_key::FakeAggregatePublicKey as AggregatePublicKey;
+    pub use crate::fake_aggregate_signature::FakeAggregateSignature as AggregateSignature;
+    pub use crate::fake_public_key::FakePublicKey as PublicKey;
+    pub use crate::fake_signature::FakeSignature as Signature;
+}
+
+#[cfg(not(feature = "fake_crypto"))]
+pub use reals::*;
+#[cfg(not(feature = "fake_crypto"))]
+mod reals {
+    pub use crate::aggregate_public_key::AggregatePublicKey;
+    pub use crate::aggregate_signature::AggregateSignature;
+    pub use crate::public_key::PublicKey;
+    pub use crate::signature::Signature;
+}
 
 pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96;
 pub const BLS_SIG_BYTE_SIZE: usize = 96;

eth2/utils/bls/src/public_key.rs

@@ -1,10 +1,10 @@
 use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE};
-use bls_aggregates::PublicKey as RawPublicKey;
 use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
+use milagro_bls::PublicKey as RawPublicKey;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
-use ssz::{ssz_encode, Decode, DecodeError};
+use ssz::{Decode, DecodeError, Encode};
 use std::default;
 use std::fmt;
 use std::hash::{Hash, Hasher};
@@ -14,7 +14,7 @@ use tree_hash::tree_hash_ssz_encoding_as_vector;
 ///
 /// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
 /// serialization).
-#[derive(Debug, Clone, Eq)]
+#[derive(Clone, Eq)]
 pub struct PublicKey(RawPublicKey);
 
 impl PublicKey {
@@ -60,9 +60,14 @@ impl PublicKey {
     ///
     /// Useful for providing a short identifier to the user.
     pub fn concatenated_hex_id(&self) -> String {
-        let bytes = ssz_encode(self);
-        let end_bytes = &bytes[bytes.len().saturating_sub(6)..bytes.len()];
-        hex_encode(end_bytes)
+        self.as_hex_string()[0..6].to_string()
+    }
+
+    /// Returns the point as a hex string of the SSZ encoding.
+    ///
+    /// Note: the string is prefixed with `0x`.
+    pub fn as_hex_string(&self) -> String {
+        hex_encode(self.as_ssz_bytes())
     }
 }
 
@@ -72,6 +77,12 @@ impl fmt::Display for PublicKey {
     }
 }
 
+impl fmt::Debug for PublicKey {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "{}", self.as_hex_string())
+    }
+}
+
 impl default::Default for PublicKey {
     fn default() -> Self {
         let secret_key = SecretKey::random();
@@ -107,7 +118,7 @@ cached_tree_hash_ssz_encoding_as_vector!(PublicKey, 48);
 
 impl PartialEq for PublicKey {
     fn eq(&self, other: &PublicKey) -> bool {
-        ssz_encode(self) == ssz_encode(other)
+        self.as_ssz_bytes() == other.as_ssz_bytes()
     }
 }
 

eth2/utils/bls/src/secret_key.rs

@@ -1,6 +1,8 @@
+extern crate rand;
+
 use super::BLS_SECRET_KEY_BYTE_SIZE;
-use bls_aggregates::SecretKey as RawSecretKey;
 use hex::encode as hex_encode;
+use milagro_bls::SecretKey as RawSecretKey;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;
@@ -16,7 +18,7 @@ pub struct SecretKey(RawSecretKey);
 
 impl SecretKey {
     pub fn random() -> Self {
-        SecretKey(RawSecretKey::random())
+        SecretKey(RawSecretKey::random(&mut rand::thread_rng()))
     }
 
     /// Returns the underlying point as compressed bytes.

eth2/utils/bls/src/signature.rs

@@ -1,7 +1,7 @@
 use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE};
-use bls_aggregates::Signature as RawSignature;
 use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use hex::encode as hex_encode;
+use milagro_bls::Signature as RawSignature;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::HexVisitor;

eth2/utils/cached_tree_hash/src/impls.rs

@@ -60,29 +60,36 @@ impl CachedTreeHash for bool {
     }
 }
 
-impl CachedTreeHash for [u8; 4] {
-    fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
-        Ok(TreeHashCache::from_bytes(
-            merkleize(self.to_vec()),
-            false,
-            None,
-        )?)
-    }
+macro_rules! impl_for_u8_array {
+    ($len: expr) => {
+        impl CachedTreeHash for [u8; $len] {
+            fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+                Ok(TreeHashCache::from_bytes(
+                    merkleize(self.to_vec()),
+                    false,
+                    None,
+                )?)
+            }
 
-    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
-        BTreeSchema::from_lengths(depth, vec![1])
-    }
+            fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+                BTreeSchema::from_lengths(depth, vec![1])
+            }
 
-    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.to_vec());
-        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+                let leaf = merkleize(self.to_vec());
+                cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
-        cache.chunk_index += 1;
+                cache.chunk_index += 1;
 
-        Ok(())
-    }
+                Ok(())
+            }
+        }
+    };
 }
 
+impl_for_u8_array!(4);
+impl_for_u8_array!(32);
+
 impl CachedTreeHash for H256 {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(

eth2/utils/compare_fields/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "compare_fields"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dev-dependencies]
+compare_fields_derive = { path = "../compare_fields_derive" }
+
+[dependencies]

eth2/utils/compare_fields/src/lib.rs

@@ -0,0 +1,179 @@
+//! Provides field-by-field comparisons for structs and vecs.
+//!
+//! Returns comparisons as data, without making assumptions about the desired equality (e.g.,
+//! does not `panic!` on inequality).
+//!
+//! Note: `compare_fields_derive` requires `PartialEq` and `Debug` implementations.
+//!
+//! ## Example
+//!
+//! ```rust
+//! use compare_fields::{CompareFields, Comparison, FieldComparison};
+//! use compare_fields_derive::CompareFields;
+//!
+//! #[derive(PartialEq, Debug, CompareFields)]
+//! pub struct Bar {
+//!     a: u64,
+//!     b: u16,
+//!     #[compare_fields(as_slice)]
+//!     c: Vec<Foo>
+//! }
+//!
+//! #[derive(Clone, PartialEq, Debug, CompareFields)]
+//! pub struct Foo {
+//!     d: String
+//! }
+//!
+//! let cat = Foo {d: "cat".to_string()};
+//! let dog = Foo {d: "dog".to_string()};
+//! let chicken = Foo {d: "chicken".to_string()};
+//!
+//! let mut bar_a = Bar {
+//!     a: 42,
+//!     b: 12,
+//!     c: vec![ cat.clone(), dog.clone() ],
+//! };
+//!
+//! let mut bar_b = Bar {
+//!     a: 42,
+//!     b: 99,
+//!     c: vec![ chicken.clone(), dog.clone()]
+//! };
+//!
+//! let cat_dog = Comparison::Child(FieldComparison {
+//!     field_name: "d".to_string(),
+//!     equal: false,
+//!     a: "\"cat\"".to_string(),
+//!     b: "\"dog\"".to_string(),
+//! });
+//! assert_eq!(cat.compare_fields(&dog), vec![cat_dog]);
+//!
+//! let bar_a_b = vec![
+//!     Comparison::Child(FieldComparison {
+//!         field_name: "a".to_string(),
+//!         equal: true,
+//!         a: "42".to_string(),
+//!         b: "42".to_string(),
+//!     }),
+//!     Comparison::Child(FieldComparison {
+//!         field_name: "b".to_string(),
+//!         equal: false,
+//!         a: "12".to_string(),
+//!         b: "99".to_string(),
+//!     }),
+//!     Comparison::Parent{
+//!         field_name: "c".to_string(),
+//!         equal: false,
+//!         children: vec![
+//!             FieldComparison {
+//!                 field_name: "0".to_string(),
+//!                 equal: false,
+//!                 a: "Some(Foo { d: \"cat\" })".to_string(),
+//!                 b: "Some(Foo { d: \"chicken\" })".to_string(),
+//!             },
+//!             FieldComparison {
+//!                 field_name: "1".to_string(),
+//!                 equal: true,
+//!                 a: "Some(Foo { d: \"dog\" })".to_string(),
+//!                 b: "Some(Foo { d: \"dog\" })".to_string(),
+//!             }
+//!         ]
+//!     }
+//! ];
+//! assert_eq!(bar_a.compare_fields(&bar_b), bar_a_b);
+//!
+//!
+//!
+//! // TODO:
+//! ```
+use std::fmt::Debug;
+
+#[derive(Debug, PartialEq, Clone)]
+pub enum Comparison {
+    Child(FieldComparison),
+    Parent {
+        field_name: String,
+        equal: bool,
+        children: Vec<FieldComparison>,
+    },
+}
+
+impl Comparison {
+    pub fn child<T: Debug + PartialEq<T>>(field_name: String, a: &T, b: &T) -> Self {
+        Comparison::Child(FieldComparison::new(field_name, a, b))
+    }
+
+    pub fn parent(field_name: String, equal: bool, children: Vec<FieldComparison>) -> Self {
+        Comparison::Parent {
+            field_name,
+            equal,
+            children,
+        }
+    }
+
+    pub fn from_slice<T: Debug + PartialEq<T>>(field_name: String, a: &[T], b: &[T]) -> Self {
+        let mut children = vec![];
+
+        for i in 0..std::cmp::max(a.len(), b.len()) {
+            children.push(FieldComparison::new(
+                format!("{:}", i),
+                &a.get(i),
+                &b.get(i),
+            ));
+        }
+
+        Self::parent(field_name, a == b, children)
+    }
+
+    pub fn retain_children<F>(&mut self, f: F)
+    where
+        F: FnMut(&FieldComparison) -> bool,
+    {
+        match self {
+            Comparison::Child(_) => (),
+            Comparison::Parent { children, .. } => children.retain(f),
+        }
+    }
+
+    pub fn equal(&self) -> bool {
+        match self {
+            Comparison::Child(fc) => fc.equal,
+            Comparison::Parent { equal, .. } => *equal,
+        }
+    }
+
+    pub fn not_equal(&self) -> bool {
+        !self.equal()
+    }
+}
+
+#[derive(Debug, PartialEq, Clone)]
+pub struct FieldComparison {
+    pub field_name: String,
+    pub equal: bool,
+    pub a: String,
+    pub b: String,
+}
+
+pub trait CompareFields {
+    fn compare_fields(&self, b: &Self) -> Vec<Comparison>;
+}
+
+impl FieldComparison {
+    pub fn new<T: Debug + PartialEq<T>>(field_name: String, a: &T, b: &T) -> Self {
+        Self {
+            field_name,
+            equal: a == b,
+            a: format!("{:?}", a),
+            b: format!("{:?}", b),
+        }
+    }
+
+    pub fn equal(&self) -> bool {
+        self.equal
+    }
+
+    pub fn not_equal(&self) -> bool {
+        !self.equal()
+    }
+}

eth2/utils/compare_fields_derive/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "compare_fields_derive"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+syn = "0.15"
+quote = "0.6"

eth2/utils/compare_fields_derive/src/lib.rs

@@ -0,0 +1,77 @@
+#![recursion_limit = "256"]
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use quote::quote;
+use syn::{parse_macro_input, DeriveInput};
+
+fn is_slice(field: &syn::Field) -> bool {
+    for attr in &field.attrs {
+        if attr.tts.to_string() == "( as_slice )" {
+            return true;
+        }
+    }
+    false
+}
+
+#[proc_macro_derive(CompareFields, attributes(compare_fields))]
+pub fn compare_fields_derive(input: TokenStream) -> TokenStream {
+    let item = parse_macro_input!(input as DeriveInput);
+
+    let name = &item.ident;
+    let (impl_generics, ty_generics, where_clause) = &item.generics.split_for_impl();
+
+    let struct_data = match &item.data {
+        syn::Data::Struct(s) => s,
+        _ => panic!("compare_fields_derive only supports structs."),
+    };
+
+    let mut quotes = vec![];
+
+    for field in struct_data.fields.iter() {
+        let ident_a = match &field.ident {
+            Some(ref ident) => ident,
+            _ => panic!("compare_fields_derive only supports named struct fields."),
+        };
+
+        let field_name = format!("{:}", ident_a);
+        let ident_b = ident_a.clone();
+
+        let quote = if is_slice(field) {
+            quote! {
+                comparisons.push(compare_fields::Comparison::from_slice(
+                        #field_name.to_string(),
+                        &self.#ident_a,
+                        &b.#ident_b)
+                );
+            }
+        } else {
+            quote! {
+                comparisons.push(
+                    compare_fields::Comparison::child(
+                        #field_name.to_string(),
+                        &self.#ident_a,
+                        &b.#ident_b
+                    )
+                );
+            }
+        };
+
+        quotes.push(quote);
+    }
+
+    let output = quote! {
+        impl #impl_generics compare_fields::CompareFields for #name #ty_generics #where_clause {
+            fn compare_fields(&self, b: &Self) -> Vec<compare_fields::Comparison> {
+                let mut comparisons = vec![];
+
+                #(
+                    #quotes
+                )*
+
+                comparisons
+            }
+        }
+    };
+    output.into()
+}

eth2/utils/fixed_len_vec/src/lib.rs

@@ -1,6 +1,6 @@
 use serde_derive::{Deserialize, Serialize};
 use std::marker::PhantomData;
-use std::ops::{Index, IndexMut};
+use std::ops::{Deref, Index, IndexMut};
 use std::slice::SliceIndex;
 use typenum::Unsigned;
 
@@ -9,6 +9,7 @@ pub use typenum;
 mod impls;
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
+#[serde(transparent)]
 pub struct FixedLenVec<T, N> {
     vec: Vec<T>,
     _phantom: PhantomData<N>,
@@ -70,6 +71,14 @@ impl<T, N: Unsigned, I: SliceIndex<[T]>> IndexMut<I> for FixedLenVec<T, N> {
     }
 }
 
+impl<T, N: Unsigned> Deref for FixedLenVec<T, N> {
+    type Target = [T];
+
+    fn deref(&self) -> &[T] {
+        &self.vec[..]
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -104,6 +113,16 @@ mod test {
         let fixed: FixedLenVec<u64, U4> = FixedLenVec::from(vec.clone());
         assert_eq!(&fixed[..], &vec![0, 0, 0, 0][..]);
     }
+
+    #[test]
+    fn deref() {
+        let vec = vec![0, 2, 4, 6];
+        let fixed: FixedLenVec<u64, U4> = FixedLenVec::from(vec);
+
+        assert_eq!(fixed.get(0), Some(&0));
+        assert_eq!(fixed.get(3), Some(&6));
+        assert_eq!(fixed.get(4), None);
+    }
 }
 
 #[cfg(test)]

eth2/utils/hashing/Cargo.toml

@@ -5,4 +5,4 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
-tiny-keccak = "1.4.2"
+ring = "0.14.6"

eth2/utils/hashing/src/lib.rs

@@ -1,11 +1,7 @@
-use tiny_keccak::Keccak;
+use ring::digest::{digest, SHA256};
 
 pub fn hash(input: &[u8]) -> Vec<u8> {
-    let mut keccak = Keccak::new_keccak256();
-    keccak.update(input);
-    let mut result = vec![0; 32];
-    keccak.finalize(result.as_mut_slice());
-    result
+    digest(&SHA256, input).as_ref().into()
 }
 
 /// Get merkle root of some hashed values - the input leaf nodes is expected to already be hashed
@@ -41,19 +37,16 @@ pub fn merkle_root(values: &[Vec<u8>]) -> Option<Vec<u8>> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use std::convert::From;
+    use ring::test;
 
     #[test]
     fn test_hashing() {
-        let input: Vec<u8> = From::from("hello");
+        let input: Vec<u8> = b"hello world".as_ref().into();
 
         let output = hash(input.as_ref());
-        let expected = &[
-            0x1c, 0x8a, 0xff, 0x95, 0x06, 0x85, 0xc2, 0xed, 0x4b, 0xc3, 0x17, 0x4f, 0x34, 0x72,
-            0x28, 0x7b, 0x56, 0xd9, 0x51, 0x7b, 0x9c, 0x94, 0x81, 0x27, 0x31, 0x9a, 0x09, 0xa7,
-            0xa3, 0x6d, 0xea, 0xc8,
-        ];
-        assert_eq!(expected, output.as_slice());
+        let expected_hex = "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9";
+        let expected: Vec<u8> = test::from_hex(expected_hex).unwrap();
+        assert_eq!(expected, output);
     }
 
     #[test]

eth2/utils/honey-badger-split/src/lib.rs

@@ -52,6 +52,38 @@ impl<T> SplitExt<T> for [T] {
 mod tests {
     use super::*;
 
+    fn alternative_split_at_index<T>(indices: &[T], index: usize, count: usize) -> &[T] {
+        let start = (indices.len() * index) / count;
+        let end = (indices.len() * (index + 1)) / count;
+
+        &indices[start..end]
+    }
+
+    fn alternative_split<T: Clone>(input: &[T], n: usize) -> Vec<&[T]> {
+        (0..n)
+            .into_iter()
+            .map(|i| alternative_split_at_index(&input, i, n))
+            .collect()
+    }
+
+    fn honey_badger_vs_alternative_fn(num_items: usize, num_chunks: usize) {
+        let input: Vec<usize> = (0..num_items).collect();
+
+        let hb: Vec<&[usize]> = input.honey_badger_split(num_chunks).collect();
+        let spec: Vec<&[usize]> = alternative_split(&input, num_chunks);
+
+        assert_eq!(hb, spec);
+    }
+
+    #[test]
+    fn vs_eth_spec_fn() {
+        for i in 0..10 {
+            for j in 0..10 {
+                honey_badger_vs_alternative_fn(i, j);
+            }
+        }
+    }
+
     #[test]
     fn test_honey_badger_split() {
         /*

eth2/utils/ssz/src/decode/impls.rs

@@ -1,5 +1,5 @@
 use super::*;
-use ethereum_types::H256;
+use ethereum_types::{H256, U128, U256};
 
 macro_rules! impl_decodable_for_uint {
     ($type: ident, $bit_size: expr) => {
@@ -83,6 +83,48 @@ impl Decode for H256 {
     }
 }
 
+impl Decode for U256 {
+    fn is_ssz_fixed_len() -> bool {
+        true
+    }
+
+    fn ssz_fixed_len() -> usize {
+        32
+    }
+
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        let len = bytes.len();
+        let expected = <Self as Decode>::ssz_fixed_len();
+
+        if len != expected {
+            Err(DecodeError::InvalidByteLength { len, expected })
+        } else {
+            Ok(U256::from_little_endian(bytes))
+        }
+    }
+}
+
+impl Decode for U128 {
+    fn is_ssz_fixed_len() -> bool {
+        true
+    }
+
+    fn ssz_fixed_len() -> usize {
+        16
+    }
+
+    fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
+        let len = bytes.len();
+        let expected = <Self as Decode>::ssz_fixed_len();
+
+        if len != expected {
+            Err(DecodeError::InvalidByteLength { len, expected })
+        } else {
+            Ok(U128::from_little_endian(bytes))
+        }
+    }
+}
+
 macro_rules! impl_decodable_for_u8_array {
     ($len: expr) => {
         impl Decode for [u8; $len] {
@@ -112,6 +154,7 @@ macro_rules! impl_decodable_for_u8_array {
 }
 
 impl_decodable_for_u8_array!(4);
+impl_decodable_for_u8_array!(32);
 
 impl<T: Decode> Decode for Vec<T> {
     fn is_ssz_fixed_len() -> bool {

eth2/utils/ssz/src/encode/impls.rs

@@ -1,5 +1,5 @@
 use super::*;
-use ethereum_types::H256;
+use ethereum_types::{H256, U128, U256};
 
 macro_rules! impl_encodable_for_uint {
     ($type: ident, $bit_size: expr) => {
@@ -77,6 +77,42 @@ impl Encode for H256 {
     }
 }
 
+impl Encode for U256 {
+    fn is_ssz_fixed_len() -> bool {
+        true
+    }
+
+    fn ssz_fixed_len() -> usize {
+        32
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        let n = <Self as Encode>::ssz_fixed_len();
+        let s = buf.len();
+
+        buf.resize(s + n, 0);
+        self.to_little_endian(&mut buf[s..]);
+    }
+}
+
+impl Encode for U128 {
+    fn is_ssz_fixed_len() -> bool {
+        true
+    }
+
+    fn ssz_fixed_len() -> usize {
+        16
+    }
+
+    fn ssz_append(&self, buf: &mut Vec<u8>) {
+        let n = <Self as Encode>::ssz_fixed_len();
+        let s = buf.len();
+
+        buf.resize(s + n, 0);
+        self.to_little_endian(&mut buf[s..]);
+    }
+}
+
 macro_rules! impl_encodable_for_u8_array {
     ($len: expr) => {
         impl Encode for [u8; $len] {
@@ -96,6 +132,7 @@ macro_rules! impl_encodable_for_u8_array {
 }
 
 impl_encodable_for_u8_array!(4);
+impl_encodable_for_u8_array!(32);
 
 #[cfg(test)]
 mod tests {

eth2/utils/swap_or_not_shuffle/Cargo.toml

@@ -15,6 +15,5 @@ hex = "0.3"
 ethereum-types = "0.5"
 
 [dependencies]
-bytes = "0.4"
 hashing = { path = "../hashing" }
 int_to_bytes = { path = "../int_to_bytes" }

eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs

@@ -1,8 +1,6 @@
-use bytes::Buf;
 use hashing::hash;
 use int_to_bytes::{int_to_bytes1, int_to_bytes4};
 use std::cmp::max;
-use std::io::Cursor;
 
 /// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
 ///
@@ -43,7 +41,7 @@ pub fn get_permutated_index(
 }
 
 fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option<usize> {
-    let flip = (pivot + list_size - index) % list_size;
+    let flip = (pivot + (list_size - index)) % list_size;
     let position = max(index, flip);
     let source = hash_with_round_and_position(seed, round, position)?;
     let byte = source[(position % 256) / 8];
@@ -68,9 +66,10 @@ fn hash_with_round(seed: &[u8], round: u8) -> Vec<u8> {
     hash(&seed[..])
 }
 
-fn bytes_to_int64(bytes: &[u8]) -> u64 {
-    let mut cursor = Cursor::new(bytes);
-    cursor.get_u64_le()
+fn bytes_to_int64(slice: &[u8]) -> u64 {
+    let mut bytes = [0; 8];
+    bytes.copy_from_slice(&slice[0..8]);
+    u64::from_le_bytes(bytes)
 }
 
 #[cfg(test)]

eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs

@@ -1,7 +1,5 @@
-use bytes::Buf;
 use hashing::hash;
 use int_to_bytes::int_to_bytes4;
-use std::io::Cursor;
 
 const SEED_SIZE: usize = 32;
 const ROUND_SIZE: usize = 1;
@@ -117,9 +115,10 @@ pub fn shuffle_list(
     Some(input)
 }
 
-fn bytes_to_int64(bytes: &[u8]) -> u64 {
-    let mut cursor = Cursor::new(bytes);
-    cursor.get_u64_le()
+fn bytes_to_int64(slice: &[u8]) -> u64 {
+    let mut bytes = [0; 8];
+    bytes.copy_from_slice(&slice[0..8]);
+    u64::from_le_bytes(bytes)
 }
 
 #[cfg(test)]

eth2/utils/tree_hash/src/impls.rs

@@ -51,24 +51,31 @@ impl TreeHash for bool {
     }
 }
 
-impl TreeHash for [u8; 4] {
-    fn tree_hash_type() -> TreeHashType {
-        TreeHashType::Vector
-    }
+macro_rules! impl_for_u8_array {
+    ($len: expr) => {
+        impl TreeHash for [u8; $len] {
+            fn tree_hash_type() -> TreeHashType {
+                TreeHashType::Vector
+            }
 
-    fn tree_hash_packed_encoding(&self) -> Vec<u8> {
-        unreachable!("bytesN should never be packed.")
-    }
+            fn tree_hash_packed_encoding(&self) -> Vec<u8> {
+                unreachable!("bytesN should never be packed.")
+            }
 
-    fn tree_hash_packing_factor() -> usize {
-        unreachable!("bytesN should never be packed.")
-    }
+            fn tree_hash_packing_factor() -> usize {
+                unreachable!("bytesN should never be packed.")
+            }
 
-    fn tree_hash_root(&self) -> Vec<u8> {
-        merkle_root(&self[..])
-    }
+            fn tree_hash_root(&self) -> Vec<u8> {
+                merkle_root(&self[..])
+            }
+        }
+    };
 }
 
+impl_for_u8_array!(4);
+impl_for_u8_array!(32);
+
 impl TreeHash for H256 {
     fn tree_hash_type() -> TreeHashType {
         TreeHashType::Vector