From 1285f1e9f8b6efd4fee83e6a9acbc3cc46c86a48 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 28 Mar 2019 11:11:20 +1100 Subject: [PATCH] Restructure cached tree hash files, breaks tests --- eth2/utils/ssz/src/cached_tree_hash.rs | 335 +++---------------- eth2/utils/ssz/src/cached_tree_hash/impls.rs | 30 ++ eth2/utils/ssz/src/cached_tree_hash/tests.rs | 227 +++++++++++++ 3 files changed, 299 insertions(+), 293 deletions(-) create mode 100644 eth2/utils/ssz/src/cached_tree_hash/impls.rs create mode 100644 eth2/utils/ssz/src/cached_tree_hash/tests.rs diff --git a/eth2/utils/ssz/src/cached_tree_hash.rs b/eth2/utils/ssz/src/cached_tree_hash.rs index 525f35db34..6535e5cdaa 100644 --- a/eth2/utils/ssz/src/cached_tree_hash.rs +++ b/eth2/utils/ssz/src/cached_tree_hash.rs @@ -1,11 +1,27 @@ -use crate::ssz_encode; use hashing::hash; -use int_to_bytes::int_to_bytes32; + +mod impls; +mod tests; const BYTES_PER_CHUNK: usize = 32; const HASHSIZE: usize = 32; const MERKLE_HASH_CHUNCK: usize = 2 * BYTES_PER_CHUNK; +pub trait CachedTreeHash { + fn build_cache_bytes(&self) -> Vec; + + fn num_bytes(&self) -> usize; + + fn max_num_leaves(&self) -> usize; + + fn cached_hash_tree_root( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Option; +} + pub struct TreeHashCache { cache: Vec, chunk_modified: Vec, @@ -68,142 +84,13 @@ fn num_nodes(num_leaves: usize) -> usize { 2 * num_leaves - 1 } -pub trait CachedTreeHash { - fn build_cache_bytes(&self) -> Vec; +/// Split `values` into a power-of-two, identical-length chunks (padding with `0`) and merkleize +/// them, returning the entire merkle tree. +/// +/// The root hash is `merkleize(values)[0..BYTES_PER_CHUNK]`. +pub fn merkleize(values: Vec) -> Vec { + let values = sanitise_bytes(values); - fn num_bytes(&self) -> usize; - - fn max_num_leaves(&self) -> usize; - - fn cached_hash_tree_root( - &self, - other: &Self, - cache: &mut TreeHashCache, - chunk: usize, - ) -> Option; -} - -impl CachedTreeHash for u64 { - fn build_cache_bytes(&self) -> Vec { - merkleize(&int_to_bytes32(*self)) - } - - fn num_bytes(&self) -> usize { - 8 - } - - fn max_num_leaves(&self) -> usize { - 1 - } - - fn cached_hash_tree_root( - &self, - other: &Self, - cache: &mut TreeHashCache, - chunk: usize, - ) -> Option { - if self != other { - cache.modify_chunk(chunk, &merkleize(&int_to_bytes32(*self)))?; - } - - Some(chunk + 1) - } -} - -#[derive(Clone)] -pub struct Inner { - pub a: u64, - pub b: u64, - pub c: u64, - pub d: u64, -} - -impl CachedTreeHash for Inner { - fn build_cache_bytes(&self) -> Vec { - let mut leaves = vec![]; - - leaves.append(&mut self.a.build_cache_bytes()); - leaves.append(&mut self.b.build_cache_bytes()); - leaves.append(&mut self.c.build_cache_bytes()); - leaves.append(&mut self.d.build_cache_bytes()); - - merkleize(&leaves) - } - - fn max_num_leaves(&self) -> usize { - let mut leaves = 0; - leaves += self.a.max_num_leaves(); - leaves += self.b.max_num_leaves(); - leaves += self.c.max_num_leaves(); - leaves += self.d.max_num_leaves(); - leaves - } - - fn num_bytes(&self) -> usize { - let mut bytes = 0; - bytes += self.a.num_bytes(); - bytes += self.b.num_bytes(); - bytes += self.c.num_bytes(); - bytes += self.d.num_bytes(); - bytes - } - - fn cached_hash_tree_root( - &self, - other: &Self, - cache: &mut TreeHashCache, - chunk: usize, - ) -> Option { - let num_leaves = self.max_num_leaves(); - let num_nodes = num_nodes(num_leaves); - let num_internal_nodes = num_nodes - num_leaves; - - // Skip past the internal nodes and update any changed leaf nodes. - { - let chunk = chunk + num_internal_nodes; - let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?; - let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?; - let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?; - let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?; - } - - // Iterate backwards through the internal nodes, rehashing any node where it's children - // have changed. - for chunk in (0..num_internal_nodes).into_iter().rev() { - if cache.children_modified(chunk)? { - cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?; - } - } - - Some(chunk + num_nodes) - } -} - -fn last_leaf_needs_padding(num_bytes: usize) -> bool { - num_bytes % HASHSIZE != 0 -} - -fn num_leaves(num_bytes: usize) -> usize { - num_bytes / HASHSIZE -} - -fn num_bytes(num_leaves: usize) -> usize { - num_leaves * HASHSIZE -} - -pub fn sanitise_bytes(mut bytes: Vec) -> Vec { - let present_leaves = num_leaves(bytes.len()); - let required_leaves = present_leaves.next_power_of_two(); - - if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) { - bytes.resize(num_bytes(required_leaves), 0); - } - - bytes -} - -/// A reference function to test against. -pub fn merkleize(values: &[u8]) -> Vec { let leaves = values.len() / HASHSIZE; if leaves == 0 { @@ -231,163 +118,25 @@ pub fn merkleize(values: &[u8]) -> Vec { o } -#[cfg(test)] -mod tests { - use super::*; +pub fn sanitise_bytes(mut bytes: Vec) -> Vec { + let present_leaves = num_leaves(bytes.len()); + let required_leaves = present_leaves.next_power_of_two(); - fn join(many: Vec>) -> Vec { - let mut all = vec![]; - for one in many { - all.extend_from_slice(&mut one.clone()) - } - all + if (present_leaves != required_leaves) | last_leaf_needs_padding(bytes.len()) { + bytes.resize(num_bytes(required_leaves), 0); } - #[test] - fn merkleize_odd() { - let data = join(vec![ - int_to_bytes32(1), - int_to_bytes32(2), - int_to_bytes32(3), - int_to_bytes32(4), - int_to_bytes32(5), - ]); - - merkleize(&sanitise_bytes(data)); - } - - fn generic_test(index: usize) { - let inner = Inner { - a: 1, - b: 2, - c: 3, - d: 4, - }; - - let cache = inner.build_cache_bytes(); - - let changed_inner = match index { - 0 => Inner { - a: 42, - ..inner.clone() - }, - 1 => Inner { - b: 42, - ..inner.clone() - }, - 2 => Inner { - c: 42, - ..inner.clone() - }, - 3 => Inner { - d: 42, - ..inner.clone() - }, - _ => panic!("bad index"), - }; - - let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap(); - - changed_inner - .cached_hash_tree_root(&inner, &mut cache_struct, 0) - .unwrap(); - - // assert_eq!(*cache_struct.hash_count, 3); - - let new_cache: Vec = cache_struct.into(); - - let data1 = int_to_bytes32(1); - let data2 = int_to_bytes32(2); - let data3 = int_to_bytes32(3); - let data4 = int_to_bytes32(4); - - let mut data = vec![data1, data2, data3, data4]; - - data[index] = int_to_bytes32(42); - - let expected = merkleize(&join(data)); - - assert_eq!(expected, new_cache); - } - - #[test] - fn cached_hash_on_inner() { - generic_test(0); - generic_test(1); - generic_test(2); - generic_test(3); - } - - #[test] - fn build_cache_matches_merkelize() { - let data1 = int_to_bytes32(1); - let data2 = int_to_bytes32(2); - let data3 = int_to_bytes32(3); - let data4 = int_to_bytes32(4); - - let data = join(vec![data1, data2, data3, data4]); - let expected = merkleize(&data); - - let inner = Inner { - a: 1, - b: 2, - c: 3, - d: 4, - }; - - let cache = inner.build_cache_bytes(); - - assert_eq!(expected, cache); - } - - #[test] - fn merkleize_4_leaves() { - let data1 = hash(&int_to_bytes32(1)); - let data2 = hash(&int_to_bytes32(2)); - let data3 = hash(&int_to_bytes32(3)); - let data4 = hash(&int_to_bytes32(4)); - - let data = join(vec![ - data1.clone(), - data2.clone(), - data3.clone(), - data4.clone(), - ]); - - let cache = merkleize(&data); - - let hash_12 = { - let mut joined = vec![]; - joined.append(&mut data1.clone()); - joined.append(&mut data2.clone()); - hash(&joined) - }; - let hash_34 = { - let mut joined = vec![]; - joined.append(&mut data3.clone()); - joined.append(&mut data4.clone()); - hash(&joined) - }; - let hash_hash12_hash_34 = { - let mut joined = vec![]; - joined.append(&mut hash_12.clone()); - joined.append(&mut hash_34.clone()); - hash(&joined) - }; - - for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() { - let expected = match i { - 0 => hash_hash12_hash_34.clone(), - 1 => hash_12.clone(), - 2 => hash_34.clone(), - 3 => data1.clone(), - 4 => data2.clone(), - 5 => data3.clone(), - 6 => data4.clone(), - _ => vec![], - }; - - assert_eq!(chunk, &expected[..], "failed at {}", i); - } - } + bytes +} + +fn last_leaf_needs_padding(num_bytes: usize) -> bool { + num_bytes % HASHSIZE != 0 +} + +fn num_leaves(num_bytes: usize) -> usize { + num_bytes / HASHSIZE +} + +fn num_bytes(num_leaves: usize) -> usize { + num_leaves * HASHSIZE } diff --git a/eth2/utils/ssz/src/cached_tree_hash/impls.rs b/eth2/utils/ssz/src/cached_tree_hash/impls.rs new file mode 100644 index 0000000000..b6b0d463aa --- /dev/null +++ b/eth2/utils/ssz/src/cached_tree_hash/impls.rs @@ -0,0 +1,30 @@ +use super::*; +use crate::ssz_encode; + +impl CachedTreeHash for u64 { + fn build_cache_bytes(&self) -> Vec { + merkleize(ssz_encode(self)) + } + + fn num_bytes(&self) -> usize { + 8 + } + + fn max_num_leaves(&self) -> usize { + 1 + } + + fn cached_hash_tree_root( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Option { + if self != other { + let leaf = merkleize(ssz_encode(self)); + cache.modify_chunk(chunk, &leaf)?; + } + + Some(chunk + 1) + } +} diff --git a/eth2/utils/ssz/src/cached_tree_hash/tests.rs b/eth2/utils/ssz/src/cached_tree_hash/tests.rs new file mode 100644 index 0000000000..79665f89de --- /dev/null +++ b/eth2/utils/ssz/src/cached_tree_hash/tests.rs @@ -0,0 +1,227 @@ +use super::*; +use int_to_bytes::int_to_bytes32; + +#[derive(Clone)] +pub struct Inner { + pub a: u64, + pub b: u64, + pub c: u64, + pub d: u64, +} + +impl CachedTreeHash for Inner { + fn build_cache_bytes(&self) -> Vec { + let mut leaves = vec![]; + + leaves.append(&mut self.a.build_cache_bytes()); + leaves.append(&mut self.b.build_cache_bytes()); + leaves.append(&mut self.c.build_cache_bytes()); + leaves.append(&mut self.d.build_cache_bytes()); + + merkleize(leaves) + } + + fn max_num_leaves(&self) -> usize { + let mut leaves = 0; + leaves += self.a.max_num_leaves(); + leaves += self.b.max_num_leaves(); + leaves += self.c.max_num_leaves(); + leaves += self.d.max_num_leaves(); + leaves + } + + fn num_bytes(&self) -> usize { + let mut bytes = 0; + bytes += self.a.num_bytes(); + bytes += self.b.num_bytes(); + bytes += self.c.num_bytes(); + bytes += self.d.num_bytes(); + bytes + } + + fn cached_hash_tree_root( + &self, + other: &Self, + cache: &mut TreeHashCache, + chunk: usize, + ) -> Option { + let num_leaves = self.max_num_leaves(); + let num_nodes = num_nodes(num_leaves); + let num_internal_nodes = num_nodes - num_leaves; + + // Skip past the internal nodes and update any changed leaf nodes. + { + let chunk = chunk + num_internal_nodes; + let chunk = self.a.cached_hash_tree_root(&other.a, cache, chunk)?; + let chunk = self.b.cached_hash_tree_root(&other.b, cache, chunk)?; + let chunk = self.c.cached_hash_tree_root(&other.c, cache, chunk)?; + let _chunk = self.d.cached_hash_tree_root(&other.d, cache, chunk)?; + } + + // Iterate backwards through the internal nodes, rehashing any node where it's children + // have changed. + for chunk in (0..num_internal_nodes).into_iter().rev() { + if cache.children_modified(chunk)? { + cache.modify_chunk(chunk, &cache.hash_children(chunk)?)?; + } + } + + Some(chunk + num_nodes) + } +} + +fn join(many: Vec>) -> Vec { + let mut all = vec![]; + for one in many { + all.extend_from_slice(&mut one.clone()) + } + all +} + +#[test] +fn merkleize_odd() { + let data = join(vec![ + int_to_bytes32(1), + int_to_bytes32(2), + int_to_bytes32(3), + int_to_bytes32(4), + int_to_bytes32(5), + ]); + + merkleize(sanitise_bytes(data)); +} + +fn generic_test(index: usize) { + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let cache = inner.build_cache_bytes(); + + let changed_inner = match index { + 0 => Inner { + a: 42, + ..inner.clone() + }, + 1 => Inner { + b: 42, + ..inner.clone() + }, + 2 => Inner { + c: 42, + ..inner.clone() + }, + 3 => Inner { + d: 42, + ..inner.clone() + }, + _ => panic!("bad index"), + }; + + let mut cache_struct = TreeHashCache::from_bytes(cache.clone()).unwrap(); + + changed_inner + .cached_hash_tree_root(&inner, &mut cache_struct, 0) + .unwrap(); + + // assert_eq!(*cache_struct.hash_count, 3); + + let new_cache: Vec = cache_struct.into(); + + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let mut data = vec![data1, data2, data3, data4]; + + data[index] = int_to_bytes32(42); + + let expected = merkleize(join(data)); + + assert_eq!(expected, new_cache); +} + +#[test] +fn cached_hash_on_inner() { + generic_test(0); + generic_test(1); + generic_test(2); + generic_test(3); +} + +#[test] +fn build_cache_matches_merkelize() { + let data1 = int_to_bytes32(1); + let data2 = int_to_bytes32(2); + let data3 = int_to_bytes32(3); + let data4 = int_to_bytes32(4); + + let data = join(vec![data1, data2, data3, data4]); + let expected = merkleize(data); + + let inner = Inner { + a: 1, + b: 2, + c: 3, + d: 4, + }; + + let cache = inner.build_cache_bytes(); + + assert_eq!(expected, cache); +} + +#[test] +fn merkleize_4_leaves() { + let data1 = hash(&int_to_bytes32(1)); + let data2 = hash(&int_to_bytes32(2)); + let data3 = hash(&int_to_bytes32(3)); + let data4 = hash(&int_to_bytes32(4)); + + let data = join(vec![ + data1.clone(), + data2.clone(), + data3.clone(), + data4.clone(), + ]); + + let cache = merkleize(data); + + let hash_12 = { + let mut joined = vec![]; + joined.append(&mut data1.clone()); + joined.append(&mut data2.clone()); + hash(&joined) + }; + let hash_34 = { + let mut joined = vec![]; + joined.append(&mut data3.clone()); + joined.append(&mut data4.clone()); + hash(&joined) + }; + let hash_hash12_hash_34 = { + let mut joined = vec![]; + joined.append(&mut hash_12.clone()); + joined.append(&mut hash_34.clone()); + hash(&joined) + }; + + for (i, chunk) in cache.chunks(HASHSIZE).enumerate().rev() { + let expected = match i { + 0 => hash_hash12_hash_34.clone(), + 1 => hash_12.clone(), + 2 => hash_34.clone(), + 3 => data1.clone(), + 4 => data2.clone(), + 5 => data3.clone(), + 6 => data4.clone(), + _ => vec![], + }; + + assert_eq!(chunk, &expected[..], "failed at {}", i); + } +}