From e5783d43a9c3bb8d389737b9ee3b676be70062b6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 10 Apr 2019 16:59:14 +1000 Subject: [PATCH] First passing vec modified cache test --- eth2/utils/ssz/src/cached_tree_hash.rs | 29 ++++- eth2/utils/ssz/src/cached_tree_hash/impls.rs | 110 +++++++++---------- eth2/utils/ssz/src/cached_tree_hash/tests.rs | 18 ++- 3 files changed, 90 insertions(+), 67 deletions(-) diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs index ba55fbf1b4..e7f2114e4c 100644 --- a/eth2/utils/ssz/src/cached_tree_hash.rs +++ b/eth2/utils/ssz/src/cached_tree_hash.rs @@ -46,6 +46,8 @@ pub trait CachedTreeHash { fn packed_encoding(&self) -> Vec; + fn packing_factor() -> usize; + fn cached_hash_tree_root( &self, other: &Self::Item, @@ -165,16 +167,19 @@ impl TreeHashCache { self.cache.splice(byte_start..byte_end, replace_with) } - pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Option<()> { + pub fn maybe_update_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { let start = chunk * BYTES_PER_CHUNK; let end = start + BYTES_PER_CHUNK; if !self.chunk_equals(chunk, to)? { - self.cache.get_mut(start..end)?.copy_from_slice(to); + self.cache + .get_mut(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))? + .copy_from_slice(to); self.chunk_modified[chunk] = true; } - Some(()) + Ok(()) } pub fn modify_chunk(&mut self, chunk: usize, to: &[u8]) -> Result<(), Error> { @@ -191,11 +196,25 @@ impl TreeHashCache { Ok(()) } - pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Option { + pub fn chunk_equals(&mut self, chunk: usize, other: &[u8]) -> Result { let start = chunk * BYTES_PER_CHUNK; let end = start + BYTES_PER_CHUNK; - Some(self.cache.get(start..end)? == other) + Ok(self + .cache + .get(start..end) + .ok_or_else(|| Error::NoModifiedFieldForChunk(chunk))? + == other) + } + + pub fn set_changed(&mut self, chunk: usize, to: bool) -> Result<(), Error> { + if chunk < self.chunk_modified.len() { + self.chunk_modified[chunk] = to; + + Ok(()) + } else { + Err(Error::NoModifiedFieldForChunk(chunk)) + } } pub fn changed(&self, chunk: usize) -> Result { diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs index e088d481de..621c5d02bd 100644 --- a/eth2/utils/ssz/src/cached_tree_hash/impls.rs +++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs @@ -28,6 +28,10 @@ impl CachedTreeHash for u64 { ssz_encode(self) } + fn packing_factor() -> usize { + 32 / 8 + } + fn cached_hash_tree_root( &self, other: &Self, @@ -55,20 +59,7 @@ where fn build_tree_hash_cache(&self) -> Result { match T::item_type() { - ItemType::Basic => { - let num_packed_bytes = self.num_bytes(); - let num_leaves = num_sanitized_leaves(num_packed_bytes); - - let mut packed = Vec::with_capacity(num_leaves * HASHSIZE); - - for item in self { - packed.append(&mut item.packed_encoding()); - } - - let packed = sanitise_bytes(packed); - - TreeHashCache::from_bytes(merkleize(packed)) - } + ItemType::Basic => TreeHashCache::from_bytes(merkleize(get_packed_leaves(self))), ItemType::Composite | ItemType::List => { let subtrees = self .iter() @@ -81,11 +72,18 @@ where } fn offsets(&self) -> Result, Error> { - let mut offsets = vec![]; + let offsets = match T::item_type() { + ItemType::Basic => vec![1; self.len() / T::packing_factor()], + ItemType::Composite | ItemType::List => { + let mut offsets = vec![]; - for item in self { - offsets.push(item.offsets()?.iter().sum()) - } + for item in self { + offsets.push(item.offsets()?.iter().sum()) + } + + offsets + } + }; Ok(offsets) } @@ -103,60 +101,58 @@ where panic!("List should never be packed") } + fn packing_factor() -> usize { + 1 + } + fn cached_hash_tree_root( &self, other: &Self::Item, cache: &mut TreeHashCache, chunk: usize, ) -> Result { - /* - let num_packed_bytes = self.num_bytes(); - let num_leaves = num_sanitized_leaves(num_packed_bytes); + let offset_handler = OffsetHandler::new(self, chunk)?; - if num_leaves != num_sanitized_leaves(other.num_bytes()) { - panic!("Need to handle a change in leaf count"); + match T::item_type() { + ItemType::Basic => { + let leaves = get_packed_leaves(self); + + for (i, chunk) in offset_handler.iter_leaf_nodes().enumerate() { + if let Some(latest) = leaves.get(i * HASHSIZE..(i + 1) * HASHSIZE) { + if !cache.chunk_equals(*chunk, latest)? { + dbg!(chunk); + cache.set_changed(*chunk, true)?; + } + } + } + let first_leaf_chunk = offset_handler.first_leaf_node()?; + cache.chunk_splice(first_leaf_chunk..offset_handler.next_node, leaves); + } + _ => panic!("not implemented"), } - let mut packed = Vec::with_capacity(num_leaves * HASHSIZE); - - // TODO: try and avoid fully encoding the whole list - for item in self { - packed.append(&mut ssz_encode(item)); - } - - let packed = sanitise_bytes(packed); - - let num_nodes = num_nodes(num_leaves); - let num_internal_nodes = num_nodes - num_leaves; - - { - let mut chunk = chunk + num_internal_nodes; - for new_chunk_bytes in packed.chunks(HASHSIZE) { - cache.maybe_update_chunk(chunk, new_chunk_bytes)?; - chunk += 1; + for (&parent, children) in offset_handler.iter_internal_nodes().rev() { + if cache.either_modified(children)? { + cache.modify_chunk(parent, &cache.hash_children(children)?)?; } } - // Iterate backwards through the internal nodes, rehashing any node where it's children - // have changed. - for chunk in (chunk..chunk + num_internal_nodes).into_iter().rev() { - if cache.children_modified(chunk)? { - cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?; - } - } - - Some(chunk + num_nodes) - */ - // TODO - Ok(42) + Ok(offset_handler.next_node()) } } -/* -fn get_packed_leaves(vec: Vec) -> Vec +fn get_packed_leaves(vec: &Vec) -> Vec where - T: Encodable, + T: CachedTreeHash, { - // + let num_packed_bytes = vec.num_bytes(); + let num_leaves = num_sanitized_leaves(num_packed_bytes); + + let mut packed = Vec::with_capacity(num_leaves * HASHSIZE); + + for item in vec { + packed.append(&mut item.packed_encoding()); + } + + sanitise_bytes(packed) } -*/ diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs index 8124a8dd8e..e65c87bbdc 100644 --- a/eth2/utils/ssz/src/cached_tree_hash/tests.rs +++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs @@ -69,6 +69,10 @@ impl CachedTreeHash for Inner { panic!("Struct should never be packed") } + fn packing_factor() -> usize { + 1 + } + fn cached_hash_tree_root( &self, other: &Self, @@ -156,6 +160,10 @@ impl CachedTreeHash for Outer { panic!("Struct should never be packed") } + fn packing_factor() -> usize { + 1 + } + fn cached_hash_tree_root( &self, other: &Self, @@ -339,7 +347,6 @@ fn outer_builds() { assert_eq!(merkle, cache); } -/* #[test] fn partial_modification_u64_vec() { let n: u64 = 50; @@ -347,7 +354,7 @@ fn partial_modification_u64_vec() { let original_vec: Vec = (0..n).collect(); // Generate initial cache. - let original_cache = original_vec.build_cache_bytes(); + let original_cache: Vec = TreeHashCache::new(&original_vec).unwrap().into(); // Modify the vec let mut modified_vec = original_vec.clone(); @@ -355,7 +362,9 @@ fn partial_modification_u64_vec() { // Perform a differential hash let mut cache_struct = TreeHashCache::from_bytes(original_cache.clone()).unwrap(); - modified_vec.cached_hash_tree_root(&original_vec, &mut cache_struct, 0); + modified_vec + .cached_hash_tree_root(&original_vec, &mut cache_struct, 0) + .unwrap(); let modified_cache: Vec = cache_struct.into(); // Generate reference data. @@ -376,7 +385,7 @@ fn large_vec_of_u64_builds() { let my_vec: Vec = (0..n).collect(); // Generate function output. - let cache = my_vec.build_cache_bytes(); + let cache: Vec = TreeHashCache::new(&my_vec).unwrap().into(); // Generate reference data. let mut data = vec![]; @@ -388,7 +397,6 @@ fn large_vec_of_u64_builds() { assert_eq!(expected, cache); } -*/ #[test] fn vec_of_inner_builds() {