gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-21 22:04:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add no-copy block processing cache (#863)

Browse Source 
* Add state cache, remove store cache

* Only build the head committee cache

* Fix compile error

* Fix compile error from merge

* Rename state_cache -> checkpoint_cache

* Rename Checkpoint -> Snapshot

* Tidy, add comments

* Tidy up find_head function

* Change some checkpoint -> snapshot

* Add tests

* Expose max_len

* Remove dead code

* Tidy

* Fix bug
This commit is contained in:
Paul Hauner
2020-04-06 10:53:33 +10:00
committed by GitHub
parent 93bcee147d
commit 2fb6b7c793
13 changed files with 494 additions and 254 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
beacon_node/store/src/hot_cold_store.rs
View File

@@ -22,7 +22,6 @@ use std::convert::TryInto;
use std::marker::PhantomData;
use std::path::Path;
use std::sync::Arc;
use types::beacon_state::CloneConfig;
use types::*;
/// 32-byte key for accessing the `split` of the freezer DB.
@@ -47,8 +46,6 @@ pub struct HotColdDB<E: EthSpec> {
pub(crate) hot_db: LevelDB<E>,
/// LRU cache of deserialized blocks. Updated whenever a block is loaded.
block_cache: Mutex<LruCache<Hash256, SignedBeaconBlock<E>>>,
/// LRU cache of deserialized states. Updated whenever a state is loaded.
state_cache: Mutex<LruCache<Hash256, BeaconState<E>>>,
/// Chain spec.
spec: ChainSpec,
/// Logger.
@@ -145,7 +142,7 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
}
/// Store a state in the store.
fn put_state(&self, state_root: &Hash256, state: BeaconState<E>) -> Result<(), Error> {
fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error> {
if state.slot < self.get_split_slot() {
self.store_cold_state(state_root, &state)
} else {
@@ -159,7 +156,7 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
state_root: &Hash256,
slot: Option<Slot>,
) -> Result<Option<BeaconState<E>>, Error> {
self.get_state_with(state_root, slot, CloneConfig::all())
self.get_state_with(state_root, slot)
}
/// Get a state from the store.
@@ -169,7 +166,6 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
&self,
state_root: &Hash256,
slot: Option<Slot>,
clone_config: CloneConfig,
) -> Result<Option<BeaconState<E>>, Error> {
metrics::inc_counter(&metrics::BEACON_STATE_GET_COUNT);
@@ -177,10 +173,10 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
if slot < self.get_split_slot() {
self.load_cold_state_by_slot(slot).map(Some)
} else {
self.load_hot_state(state_root, clone_config)
self.load_hot_state(state_root)
}
} else {
match self.load_hot_state(state_root, clone_config)? {
match self.load_hot_state(state_root)? {
Some(state) => Ok(Some(state)),
None => self.load_cold_state(state_root),
}
@@ -204,9 +200,6 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
.key_delete(DBColumn::BeaconState.into(), state_root.as_bytes())?;
}
// Delete from the cache.
self.state_cache.lock().pop(state_root);
Ok(())
}
@@ -309,10 +302,7 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
{
// NOTE: minor inefficiency here because we load an unnecessary hot state summary
let state = self
.load_hot_state(
&epoch_boundary_state_root,
CloneConfig::committee_caches_only(),
)?
.load_hot_state(&epoch_boundary_state_root)?
.ok_or_else(|| {
HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root)
})?;
@@ -349,7 +339,6 @@ impl<E: EthSpec> HotColdDB<E> {
cold_db: LevelDB::open(cold_path)?,
hot_db: LevelDB::open(hot_path)?,
block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
state_cache: Mutex::new(LruCache::new(config.state_cache_size)),
config,
spec,
log,
@@ -371,7 +360,7 @@ impl<E: EthSpec> HotColdDB<E> {
pub fn store_hot_state(
&self,
state_root: &Hash256,
state: BeaconState<E>,
state: &BeaconState<E>,
) -> Result<(), Error> {
// On the epoch boundary, store the full state.
if state.slot % E::slots_per_epoch() == 0 {
@@ -387,10 +376,7 @@ impl<E: EthSpec> HotColdDB<E> {
// Store a summary of the state.
// We store one even for the epoch boundary states, as we may need their slots
// when doing a look up by state root.
self.put_state_summary(state_root, HotStateSummary::new(state_root, &state)?)?;
// Store the state in the cache.
self.state_cache.lock().put(*state_root, state);
self.put_state_summary(state_root, HotStateSummary::new(state_root, state)?)?;
Ok(())
}
@@ -398,24 +384,9 @@ impl<E: EthSpec> HotColdDB<E> {
/// Load a post-finalization state from the hot database.
///
/// Will replay blocks from the nearest epoch boundary.
pub fn load_hot_state(
&self,
state_root: &Hash256,
clone_config: CloneConfig,
) -> Result<Option<BeaconState<E>>, Error> {
pub fn load_hot_state(&self, state_root: &Hash256) -> Result<Option<BeaconState<E>>, Error> {
metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT);
// Check the cache.
if let Some(state) = self.state_cache.lock().get(state_root) {
metrics::inc_counter(&metrics::BEACON_STATE_CACHE_HIT_COUNT);
let timer = metrics::start_timer(&metrics::BEACON_STATE_CACHE_CLONE_TIME);
let state = state.clone_with(clone_config);
metrics::stop_timer(timer);
return Ok(Some(state));
}
if let Some(HotStateSummary {
slot,
latest_block_root,
@@ -439,9 +410,6 @@ impl<E: EthSpec> HotColdDB<E> {
self.replay_blocks(boundary_state, blocks, slot)?
};
// Update the LRU cache.
self.state_cache.lock().put(*state_root, state.clone());
Ok(Some(state))
} else {
Ok(None)

6
beacon_node/store/src/iter.rs
View File

@@ -345,7 +345,7 @@ mod test {
let state_a_root = hashes.next().unwrap();
state_b.state_roots[0] = state_a_root;
store.put_state(&state_a_root, state_a).unwrap();
store.put_state(&state_a_root, &state_a).unwrap();
let iter = BlockRootsIterator::new(store, &state_b);
@@ -393,8 +393,8 @@ mod test {
let state_a_root = Hash256::from_low_u64_be(slots_per_historical_root as u64);
let state_b_root = Hash256::from_low_u64_be(slots_per_historical_root as u64 * 2);
store.put_state(&state_a_root, state_a).unwrap();
store.put_state(&state_b_root, state_b.clone()).unwrap();
store.put_state(&state_a_root, &state_a).unwrap();
store.put_state(&state_b_root, &state_b.clone()).unwrap();
let iter = StateRootsIterator::new(store, &state_b);

2
beacon_node/store/src/leveldb_store.rs
View File

@@ -123,7 +123,7 @@ impl<E: EthSpec> Store<E> for LevelDB<E> {
}
/// Store a state in the store.
fn put_state(&self, state_root: &Hash256, state: BeaconState<E>) -> Result<(), Error> {
fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error> {
store_full_state(self, state_root, &state)
}

4
beacon_node/store/src/lib.rs
View File

@@ -38,7 +38,6 @@ pub use errors::Error;
pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer;
pub use metrics::scrape_for_metrics;
pub use state_batch::StateBatch;
pub use types::beacon_state::CloneConfig;
pub use types::*;
/// An object capable of storing and retrieving objects implementing `StoreItem`.
@@ -97,7 +96,7 @@ pub trait Store<E: EthSpec>: Sync + Send + Sized + 'static {
}
/// Store a state in the store.
fn put_state(&self, state_root: &Hash256, state: BeaconState<E>) -> Result<(), Error>;
fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error>;
/// Store a state summary in the store.
// NOTE: this is a hack for the HotColdDb, we could consider splitting this
@@ -122,7 +121,6 @@ pub trait Store<E: EthSpec>: Sync + Send + Sized + 'static {
&self,
state_root: &Hash256,
slot: Option<Slot>,
_clone_config: CloneConfig,
) -> Result<Option<BeaconState<E>>, Error> {
// Default impl ignores config. Overriden in `HotColdDb`.
self.get_state(state_root, slot)

2
beacon_node/store/src/memory_store.rs
View File

@@ -76,7 +76,7 @@ impl<E: EthSpec> Store<E> for MemoryStore<E> {
}
/// Store a state in the store.
fn put_state(&self, state_root: &Hash256, state: BeaconState<E>) -> Result<(), Error> {
fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error> {
store_full_state(self, state_root, &state)
}

2
beacon_node/store/src/state_batch.rs
View File

@@ -38,7 +38,7 @@ impl<E: EthSpec> StateBatch<E> {
/// May fail to write the full batch if any of the items error (i.e. not atomic!)
pub fn commit<S: Store<E>>(self, store: &S) -> Result<(), Error> {
self.items.into_iter().try_for_each(|item| match item {
BatchItem::Full(state_root, state) => store.put_state(&state_root, state),
BatchItem::Full(state_root, state) => store.put_state(&state_root, &state),
BatchItem::Summary(state_root, summary) => {
store.put_state_summary(&state_root, summary)
}