gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 10:22:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tree hash cache arena (#836)

Browse Source 
* Start adding interop genesis state to lcli

* Use more efficient method to generate genesis state

* Remove duplicate int_to_bytes32

* Add lcli command to change state genesis time

* Add option to allow VC to start with unsynced BN

* Set VC to do parallel key loading

* Don't default to dummy eth1 backend

* Add endpoint to dump operation pool

* Add metrics for op pool

* Remove state clone for slot notifier

* Add mem size approximation for tree hash cache

* Avoid cloning tree hash when getting head

* Avoid cloning tree hash when getting head

* Add working arena-based cached tree hash

* Add another benchmark

* Add pre-allocation for caches

* Make cache nullable

* Fix bugs in cache tree hash

* Add validator tree hash optimization

* Optimize hash_concat

* Make hash32_concat return fixed-len array

* Fix failing API tests

* Add new beacon state cache struct

* Add validator-specific cache

* Separate list and values arenas

* Add parallel validator registry hashing

* Remove MultiTreeHashCache

* Remove cached tree hash macro

* Fix failing tree hash test

* Address Michael's comments

* Add CachedTreeHash impl for ef tests

* Fix messy merge conflict

* Rename cache struct, add comments

* Rename cache struct, add comments

* Remove unnecessary mutability

* Wrap iter in result

* Tidy cached tree hash

* Address Michael comments

* Address more comments

* Use ring::Context
This commit is contained in:
Paul Hauner
2020-02-07 12:42:49 +11:00
committed by GitHub
parent f267bf2afe
commit c3182e3c1c
20 changed files with 1341 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
eth2/utils/cached_tree_hash/src/impls.rs
View File

@@ -1,4 +1,4 @@
use crate::{CachedTreeHash, Error, Hash256, TreeHashCache};
use crate::{CacheArena, CachedTreeHash, Error, Hash256, TreeHashCache};
use ssz_types::{typenum::Unsigned, FixedVector, VariableList};
use std::mem::size_of;
use tree_hash::{mix_in_length, BYTES_PER_CHUNK};
@@ -13,6 +13,17 @@ pub fn int_log(n: usize) -> usize {
}
}
pub fn hash256_leaf_count(len: usize) -> usize {
len
}
pub fn u64_leaf_count(len: usize) -> usize {
let type_size = size_of::<u64>();
let vals_per_chunk = BYTES_PER_CHUNK / type_size;
(len + vals_per_chunk - 1) / vals_per_chunk
}
pub fn hash256_iter<'a>(
values: &'a [Hash256],
) -> impl Iterator<Item = [u8; BYTES_PER_CHUNK]> + ExactSizeIterator + 'a {
@@ -36,35 +47,59 @@ pub fn u64_iter<'a>(
}
impl<N: Unsigned> CachedTreeHash<TreeHashCache> for FixedVector<Hash256, N> {
fn new_tree_hash_cache() -> TreeHashCache {
TreeHashCache::new(int_log(N::to_usize()))
fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
TreeHashCache::new(
arena,
int_log(N::to_usize()),
hash256_leaf_count(self.len()),
)
}
fn recalculate_tree_hash_root(&self, cache: &mut TreeHashCache) -> Result<Hash256, Error> {
cache.recalculate_merkle_root(hash256_iter(&self))
fn recalculate_tree_hash_root(
&self,
arena: &mut CacheArena,
cache: &mut TreeHashCache,
) -> Result<Hash256, Error> {
cache.recalculate_merkle_root(arena, hash256_iter(&self))
}
}
impl<N: Unsigned> CachedTreeHash<TreeHashCache> for FixedVector<u64, N> {
fn new_tree_hash_cache() -> TreeHashCache {
fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
let vals_per_chunk = BYTES_PER_CHUNK / size_of::<u64>();
TreeHashCache::new(int_log(N::to_usize() / vals_per_chunk))
TreeHashCache::new(
arena,
int_log(N::to_usize() / vals_per_chunk),
u64_leaf_count(self.len()),
)
}
fn recalculate_tree_hash_root(&self, cache: &mut TreeHashCache) -> Result<Hash256, Error> {
cache.recalculate_merkle_root(u64_iter(&self))
fn recalculate_tree_hash_root(
&self,
arena: &mut CacheArena,
cache: &mut TreeHashCache,
) -> Result<Hash256, Error> {
cache.recalculate_merkle_root(arena, u64_iter(&self))
}
}
impl<N: Unsigned> CachedTreeHash<TreeHashCache> for VariableList<Hash256, N> {
fn new_tree_hash_cache() -> TreeHashCache {
TreeHashCache::new(int_log(N::to_usize()))
fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
TreeHashCache::new(
arena,
int_log(N::to_usize()),
hash256_leaf_count(self.len()),
)
}
fn recalculate_tree_hash_root(&self, cache: &mut TreeHashCache) -> Result<Hash256, Error> {
fn recalculate_tree_hash_root(
&self,
arena: &mut CacheArena,
cache: &mut TreeHashCache,
) -> Result<Hash256, Error> {
Ok(Hash256::from_slice(&mix_in_length(
cache
.recalculate_merkle_root(hash256_iter(&self))?
.recalculate_merkle_root(arena, hash256_iter(&self))?
.as_bytes(),
self.len(),
)))
@@ -72,14 +107,24 @@ impl<N: Unsigned> CachedTreeHash<TreeHashCache> for VariableList<Hash256, N> {
}
impl<N: Unsigned> CachedTreeHash<TreeHashCache> for VariableList<u64, N> {
fn new_tree_hash_cache() -> TreeHashCache {
fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
let vals_per_chunk = BYTES_PER_CHUNK / size_of::<u64>();
TreeHashCache::new(int_log(N::to_usize() / vals_per_chunk))
TreeHashCache::new(
arena,
int_log(N::to_usize() / vals_per_chunk),
u64_leaf_count(self.len()),
)
}
fn recalculate_tree_hash_root(&self, cache: &mut TreeHashCache) -> Result<Hash256, Error> {
fn recalculate_tree_hash_root(
&self,
arena: &mut CacheArena,
cache: &mut TreeHashCache,
) -> Result<Hash256, Error> {
Ok(Hash256::from_slice(&mix_in_length(
cache.recalculate_merkle_root(u64_iter(&self))?.as_bytes(),
cache
.recalculate_merkle_root(arena, u64_iter(&self))?
.as_bytes(),
self.len(),
)))
}