Begin implementing cached hashing in types

eth2/utils/bls/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2018"
 
 [dependencies]
 bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.6.1" }
+cached_tree_hash = { path = "../cached_tree_hash" }
 hashing = { path = "../hashing" }
 hex = "0.3"
 serde = "1.0"

eth2/utils/bls/src/aggregate_signature.rs

@@ -2,6 +2,7 @@ use super::{AggregatePublicKey, Signature, BLS_AGG_SIG_BYTE_SIZE};
 use bls_aggregates::{
     AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
 };
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
@@ -167,6 +168,7 @@ impl<'de> Deserialize<'de> for AggregateSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(AggregateSignature);
+cached_tree_hash_ssz_encoding_as_vector!(AggregateSignature, 96);
 
 #[cfg(test)]
 mod tests {

eth2/utils/bls/src/fake_aggregate_signature.rs

@@ -1,4 +1,5 @@
 use super::{fake_signature::FakeSignature, AggregatePublicKey, BLS_AGG_SIG_BYTE_SIZE};
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, PrefixedHexVisitor};
@@ -100,6 +101,7 @@ impl<'de> Deserialize<'de> for FakeAggregateSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature);
+cached_tree_hash_ssz_encoding_as_vector!(FakeAggregateSignature, 96);
 
 #[cfg(test)]
 mod tests {

eth2/utils/bls/src/fake_signature.rs

@@ -1,4 +1,5 @@
 use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE};
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -75,6 +76,7 @@ impl Decodable for FakeSignature {
 }
 
 tree_hash_ssz_encoding_as_vector!(FakeSignature);
+cached_tree_hash_ssz_encoding_as_vector!(FakeSignature, 96);
 
 impl Serialize for FakeSignature {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>

eth2/utils/bls/src/public_key.rs

@@ -1,5 +1,6 @@
 use super::{SecretKey, BLS_PUBLIC_KEY_BYTE_SIZE};
 use bls_aggregates::PublicKey as RawPublicKey;
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode as hex_encode, HexVisitor};
@@ -106,6 +107,7 @@ impl<'de> Deserialize<'de> for PublicKey {
 }
 
 tree_hash_ssz_encoding_as_vector!(PublicKey);
+cached_tree_hash_ssz_encoding_as_vector!(PublicKey, 48);
 
 impl PartialEq for PublicKey {
     fn eq(&self, other: &PublicKey) -> bool {
@@ -129,6 +131,7 @@ impl Hash for PublicKey {
 mod tests {
     use super::*;
     use ssz::ssz_encode;
+    use tree_hash::TreeHash;
 
     #[test]
     pub fn test_ssz_round_trip() {
@@ -140,4 +143,21 @@ mod tests {
 
         assert_eq!(original, decoded);
     }
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let sk = SecretKey::random();
+        let original = PublicKey::from_secret_key(&sk);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+        let sk = SecretKey::random();
+        let modified = PublicKey::from_secret_key(&sk);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+    }
 }

eth2/utils/bls/src/signature.rs

@@ -1,5 +1,6 @@
 use super::{PublicKey, SecretKey, BLS_SIG_BYTE_SIZE};
 use bls_aggregates::Signature as RawSignature;
+use cached_tree_hash::cached_tree_hash_ssz_encoding_as_vector;
 use hex::encode as hex_encode;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
@@ -116,6 +117,7 @@ impl Decodable for Signature {
 }
 
 tree_hash_ssz_encoding_as_vector!(Signature);
+cached_tree_hash_ssz_encoding_as_vector!(Signature, 96);
 
 impl Serialize for Signature {
     /// Serde serialization is compliant the Ethereum YAML test format.
@@ -145,6 +147,7 @@ mod tests {
     use super::super::Keypair;
     use super::*;
     use ssz::ssz_encode;
+    use tree_hash::TreeHash;
 
     #[test]
     pub fn test_ssz_round_trip() {
@@ -158,6 +161,22 @@ mod tests {
         assert_eq!(original, decoded);
     }
 
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let keypair = Keypair::random();
+        let original = Signature::new(&[42, 42], 0, &keypair.sk);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+
+        let modified = Signature::new(&[99, 99], 0, &keypair.sk);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+    }
+
     #[test]
     pub fn test_empty_signature() {
         let sig = Signature::empty_signature();

eth2/utils/boolean-bitfield/Cargo.toml

@@ -5,6 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
 edition = "2018"
 
 [dependencies]
+cached_tree_hash = { path = "../cached_tree_hash" }
 serde_hex = { path = "../serde_hex" }
 ssz = { path = "../ssz" }
 bit-vec = "0.5.0"

eth2/utils/boolean-bitfield/src/lib.rs

@@ -3,6 +3,7 @@ extern crate ssz;
 
 use bit_reverse::LookupReverse;
 use bit_vec::BitVec;
+use cached_tree_hash::cached_tree_hash_bytes_as_list;
 use serde::de::{Deserialize, Deserializer};
 use serde::ser::{Serialize, Serializer};
 use serde_hex::{encode, PrefixedHexVisitor};
@@ -270,11 +271,31 @@ impl tree_hash::TreeHash for BooleanBitfield {
     }
 }
 
+cached_tree_hash_bytes_as_list!(BooleanBitfield);
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use serde_yaml;
     use ssz::{decode, ssz_encode, SszStream};
+    use tree_hash::TreeHash;
+
+    #[test]
+    pub fn test_cached_tree_hash() {
+        let original = BooleanBitfield::from_bytes(&vec![18; 12][..]);
+
+        let mut hasher = cached_tree_hash::CachedTreeHasher::new(&original).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), original.tree_hash_root());
+        /*
+
+        let modified = BooleanBitfield::from_bytes(&vec![2; 1][..]);
+
+        hasher.update(&modified).unwrap();
+
+        assert_eq!(hasher.tree_hash_root().unwrap(), modified.tree_hash_root());
+        */
+    }
 
     #[test]
     fn test_new_bitfield() {

eth2/utils/cached_tree_hash/src/impls.rs

@@ -1,12 +1,46 @@
 use super::*;
 use crate::merkleize::merkleize;
+use ethereum_types::H256;
 
 pub mod vec;
 
-impl CachedTreeHash<u64> for u64 {
+macro_rules! impl_for_single_leaf_int {
+    ($type: ident) => {
+        impl CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+                Ok(TreeHashCache::from_bytes(
+                    merkleize(self.to_le_bytes().to_vec()),
+                    false,
+                    None,
+                )?)
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+                BTreeSchema::from_lengths(depth, vec![1])
+            }
+
+            fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+                let leaf = merkleize(self.to_le_bytes().to_vec());
+                cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+
+                cache.chunk_index += 1;
+
+                Ok(())
+            }
+        }
+    };
+}
+
+impl_for_single_leaf_int!(u8);
+impl_for_single_leaf_int!(u16);
+impl_for_single_leaf_int!(u32);
+impl_for_single_leaf_int!(u64);
+impl_for_single_leaf_int!(usize);
+
+impl CachedTreeHash<bool> for bool {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(self.to_le_bytes().to_vec()),
+            merkleize((*self as u8).to_le_bytes().to_vec()),
             false,
             None,
         )?)
@@ -17,20 +51,19 @@ impl CachedTreeHash<u64> for u64 {
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.to_le_bytes().to_vec());
+        let leaf = merkleize((*self as u8).to_le_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
         cache.chunk_index += 1;
-        // cache.overlay_index += 1;
 
         Ok(())
     }
 }
 
-impl CachedTreeHash<usize> for usize {
+impl CachedTreeHash<[u8; 4]> for [u8; 4] {
     fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
         Ok(TreeHashCache::from_bytes(
-            merkleize(self.to_le_bytes().to_vec()),
+            merkleize(self.to_vec()),
             false,
             None,
         )?)
@@ -41,11 +74,33 @@ impl CachedTreeHash<usize> for usize {
     }
 
     fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
-        let leaf = merkleize(self.to_le_bytes().to_vec());
+        let leaf = merkleize(self.to_vec());
+        cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
+
+        cache.chunk_index += 1;
+
+        Ok(())
+    }
+}
+
+impl CachedTreeHash<H256> for H256 {
+    fn new_tree_hash_cache(&self, _depth: usize) -> Result<TreeHashCache, Error> {
+        Ok(TreeHashCache::from_bytes(
+            merkleize(self.as_bytes().to_vec()),
+            false,
+            None,
+        )?)
+    }
+
+    fn tree_hash_cache_schema(&self, depth: usize) -> BTreeSchema {
+        BTreeSchema::from_lengths(depth, vec![1])
+    }
+
+    fn update_tree_hash_cache(&self, cache: &mut TreeHashCache) -> Result<(), Error> {
+        let leaf = merkleize(self.as_bytes().to_vec());
         cache.maybe_update_chunk(cache.chunk_index, &leaf)?;
 
         cache.chunk_index += 1;
-        // cache.overlay_index += 1;
 
         Ok(())
     }

eth2/utils/cached_tree_hash/src/impls/vec.rs

@@ -95,6 +95,11 @@ pub fn update_tree_hash_cache<T: CachedTreeHash<T>>(
     let old_overlay = cache.get_overlay(cache.schema_index, cache.chunk_index)?;
     let new_overlay = BTreeOverlay::new(vec, cache.chunk_index, old_overlay.depth);
 
+    dbg!(cache.schema_index);
+    dbg!(cache.schemas.len());
+    dbg!(&old_overlay);
+    dbg!(&new_overlay);
+
     cache.replace_overlay(cache.schema_index, cache.chunk_index, new_overlay.clone())?;
 
     cache.schema_index += 1;

eth2/utils/cached_tree_hash/src/lib.rs

@@ -1,4 +1,5 @@
 use hashing::hash;
+use merkleize::num_unsanitized_leaves;
 use std::ops::Range;
 use tree_hash::{TreeHash, TreeHashType, BYTES_PER_CHUNK, HASHSIZE};
 
@@ -62,3 +63,92 @@ impl CachedTreeHasher {
         Ok(self.cache.root()?.to_vec())
     }
 }
+
+#[macro_export]
+macro_rules! cached_tree_hash_ssz_encoding_as_vector {
+    ($type: ident, $num_bytes: expr) => {
+        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(
+                &self,
+                depth: usize,
+            ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                let (cache, _schema) = cached_tree_hash::impls::vec::new_tree_hash_cache(
+                    &ssz::ssz_encode(self),
+                    depth,
+                )?;
+
+                Ok(cache)
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+                let lengths =
+                    vec![1; cached_tree_hash::merkleize::num_unsanitized_leaves($num_bytes)];
+                cached_tree_hash::BTreeSchema::from_lengths(depth, lengths)
+            }
+
+            fn update_tree_hash_cache(
+                &self,
+                cache: &mut cached_tree_hash::TreeHashCache,
+            ) -> Result<(), cached_tree_hash::Error> {
+                cached_tree_hash::impls::vec::update_tree_hash_cache(
+                    &ssz::ssz_encode(self),
+                    cache,
+                )?;
+
+                Ok(())
+            }
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! cached_tree_hash_bytes_as_list {
+    ($type: ident) => {
+        impl cached_tree_hash::CachedTreeHash<$type> for $type {
+            fn new_tree_hash_cache(
+                &self,
+                depth: usize,
+            ) -> Result<cached_tree_hash::TreeHashCache, cached_tree_hash::Error> {
+                let bytes = self.to_bytes();
+
+                let (mut cache, schema) =
+                    cached_tree_hash::impls::vec::new_tree_hash_cache(&bytes, depth)?;
+
+                cache.add_length_nodes(schema.into_overlay(0).chunk_range(), bytes.len())?;
+
+                Ok(cache)
+            }
+
+            fn num_tree_hash_cache_chunks(&self) -> usize {
+                // Add two extra nodes to cater for the node before and after to allow mixing-in length.
+                cached_tree_hash::BTreeOverlay::new(self, 0, 0).num_chunks() + 2
+            }
+
+            fn tree_hash_cache_schema(&self, depth: usize) -> cached_tree_hash::BTreeSchema {
+                cached_tree_hash::impls::vec::produce_schema(&ssz::ssz_encode(self), depth)
+            }
+
+            fn update_tree_hash_cache(
+                &self,
+                cache: &mut cached_tree_hash::TreeHashCache,
+            ) -> Result<(), cached_tree_hash::Error> {
+                let bytes = self.to_bytes();
+
+                // Skip the length-mixed-in root node.
+                cache.chunk_index += 1;
+
+                // Update the cache, returning the new overlay.
+                let new_overlay =
+                    cached_tree_hash::impls::vec::update_tree_hash_cache(&bytes, cache)?;
+
+                // Mix in length
+                cache.mix_in_length(new_overlay.chunk_range(), bytes.len())?;
+
+                // Skip an extra node to clear the length node.
+                cache.chunk_index = new_overlay.next_node() + 1;
+
+                Ok(())
+            }
+        }
+    };
+}

eth2/utils/cached_tree_hash/src/merkleize.rs

@@ -60,7 +60,7 @@ fn last_leaf_needs_padding(num_bytes: usize) -> bool {
 }
 
 /// Rounds up
-fn num_unsanitized_leaves(num_bytes: usize) -> usize {
+pub fn num_unsanitized_leaves(num_bytes: usize) -> usize {
     (num_bytes + HASHSIZE - 1) / HASHSIZE
 }
 

eth2/utils/cached_tree_hash/tests/tests.rs

@@ -231,6 +231,41 @@ pub struct StructWithVecOfStructs {
     pub c: Vec<Inner>,
 }
 
+fn get_inners() -> Vec<Inner> {
+    vec![
+        Inner {
+            a: 12,
+            b: 13,
+            c: 14,
+            d: 15,
+        },
+        Inner {
+            a: 99,
+            b: 100,
+            c: 101,
+            d: 102,
+        },
+        Inner {
+            a: 255,
+            b: 256,
+            c: 257,
+            d: 0,
+        },
+        Inner {
+            a: 1000,
+            b: 2000,
+            c: 3000,
+            d: 0,
+        },
+        Inner {
+            a: 0,
+            b: 0,
+            c: 0,
+            d: 0,
+        },
+    ]
+}
+
 fn get_struct_with_vec_of_structs() -> Vec<StructWithVecOfStructs> {
     let inner_a = Inner {
         a: 12,
@@ -342,6 +377,56 @@ fn test_struct_with_vec_of_struct_with_vec_of_structs() {
     }
 }
 
+#[derive(Clone, Debug, TreeHash, CachedTreeHash)]
+pub struct StructWithTwoVecs {
+    pub a: Vec<Inner>,
+    pub b: Vec<Inner>,
+}
+
+#[test]
+fn test_struct_with_two_vecs() {
+    let inners = get_inners();
+
+    let variants = vec![
+        StructWithTwoVecs {
+            a: inners[..].to_vec(),
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..1].to_vec(),
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..1].to_vec(),
+            b: inners[0..2].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[0..4].to_vec(),
+            b: inners[0..2].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: vec![],
+            b: inners[..].to_vec(),
+        },
+        StructWithTwoVecs {
+            a: inners[..].to_vec(),
+            b: vec![],
+        },
+        StructWithTwoVecs {
+            a: inners[0..3].to_vec(),
+            b: inners[0..1].to_vec(),
+        },
+    ];
+
+    test_routine(variants[0].clone(), variants[6..7].to_vec());
+
+    /*
+    for v in &variants {
+        test_routine(v.clone(), variants.clone());
+    }
+    */
+}
+
 #[derive(Clone, Debug, TreeHash, CachedTreeHash)]
 pub struct Inner {
     pub a: u64,

eth2/utils/tree_hash/src/impls.rs

@@ -52,7 +52,7 @@ impl TreeHash for bool {
 
 impl TreeHash for [u8; 4] {
     fn tree_hash_type() -> TreeHashType {
-        TreeHashType::List
+        TreeHashType::Vector
     }
 
     fn tree_hash_packed_encoding(&self) -> Vec<u8> {