gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 18:32:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add LRU cache to database (#837)

Browse Source 
* Add LRU caches to store

* Improvements to LRU caches

* Take state by value in `Store::put_state`

* Store blocks by value, configurable cache sizes

* Use a StateBatch to efficiently store skip states

* Fix store tests

* Add CloneConfig test, remove unused metrics

* Use Mutexes instead of RwLocks for LRU caches
This commit is contained in:
Michael Sproul
2020-02-10 11:30:21 +11:00
committed by GitHub
parent c3182e3c1c
commit e0b9fa599f
29 changed files with 514 additions and 385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -1,5 +1,4 @@
use crate::checkpoint::CheckPoint;
use crate::checkpoint_cache::CheckPointCache;
use crate::errors::{BeaconChainError as Error, BlockProductionError};
use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend};
use crate::events::{EventHandler, EventKind};
@@ -22,7 +21,6 @@ use state_processing::per_block_processing::{
use state_processing::{
per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy,
};
use std::borrow::Cow;
use std::cmp::Ordering;
use std::fs;
use std::io::prelude::*;
@@ -31,7 +29,7 @@ use std::time::{Duration, Instant};
use store::iter::{
BlockRootsIterator, ReverseBlockRootIterator, ReverseStateRootIterator, StateRootsIterator,
};
use store::{Error as DBError, Migrate, Store};
use store::{Error as DBError, Migrate, StateBatch, Store};
use tree_hash::TreeHash;
use types::*;
@@ -149,8 +147,6 @@ pub struct BeaconChain<T: BeaconChainTypes> {
pub event_handler: T::EventHandler,
/// Used to track the heads of the beacon chain.
pub(crate) head_tracker: HeadTracker,
/// Provides a small cache of `BeaconState` and `BeaconBlock`.
pub(crate) checkpoint_cache: CheckPointCache<T::EthSpec>,
/// Logging to CLI, etc.
pub(crate) log: Logger,
}
@@ -168,11 +164,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let beacon_block_root = canonical_head.beacon_state.finalized_checkpoint.root;
let beacon_block = self
.store
.get::<BeaconBlock<_>>(&beacon_block_root)?
.get_block(&beacon_block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(beacon_block_root))?;
let beacon_state_root = beacon_block.state_root;
let beacon_state = self
.get_state_caching(&beacon_state_root, Some(beacon_block.slot))?
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(beacon_state_root))?;
CheckPoint {
@@ -306,10 +302,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
block_root: Hash256,
) -> Result<ReverseBlockRootIterator<T::EthSpec, T::Store>, Error> {
let block = self
.get_block_caching(&block_root)?
.get_block(&block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(block_root))?;
let state = self
.get_state_caching(&block.state_root, Some(block.slot))?
.get_state(&block.state_root, Some(block.slot))?
.ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
let iter = BlockRootsIterator::owned(self.store.clone(), state);
Ok(ReverseBlockRootIterator::new(
@@ -392,7 +388,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
block_root: &Hash256,
) -> Result<Option<BeaconBlock<T::EthSpec>>, Error> {
Ok(self.store.get(block_root)?)
Ok(self.store.get_block(block_root)?)
}
/// Returns the state at the given root, if any.
@@ -408,44 +404,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self.store.get_state(state_root, slot)?)
}
/// Returns the block at the given root, if any.
///
/// ## Errors
///
/// May return a database error.
pub fn get_block_caching(
&self,
block_root: &Hash256,
) -> Result<Option<BeaconBlock<T::EthSpec>>, Error> {
if let Some(block) = self.checkpoint_cache.get_block(block_root) {
Ok(Some(block))
} else {
Ok(self.store.get(block_root)?)
}
}
/// Returns the state at the given root, if any.
///
/// ## Errors
///
/// May return a database error.
pub fn get_state_caching(
&self,
state_root: &Hash256,
slot: Option<Slot>,
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
if let Some(state) = self.checkpoint_cache.get_state(state_root) {
Ok(Some(state))
} else {
Ok(self.store.get_state(state_root, slot)?)
}
}
/// Returns the state at the given root, if any.
///
/// The return state does not contain any caches other than the committee caches. This method
/// is much faster than `Self::get_state_caching` because it does not clone the tree hash cache
/// when the state is found in the checkpoint cache.
/// is much faster than `Self::get_state` because it does not clone the tree hash cache
/// when the state is found in the cache.
///
/// ## Errors
///
@@ -455,14 +418,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
state_root: &Hash256,
slot: Option<Slot>,
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
if let Some(state) = self
.checkpoint_cache
.get_state_only_with_committee_cache(state_root)
{
Ok(Some(state))
} else {
Ok(self.store.get_state(state_root, slot)?)
}
Ok(self.store.get_state_with(
state_root,
slot,
types::beacon_state::CloneConfig::committee_caches_only(),
)?)
}
/// Returns a `Checkpoint` representing the head block and state. Contains the "best block";
@@ -568,7 +528,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.ok_or_else(|| Error::NoStateForSlot(slot))?;
Ok(self
.get_state_caching(&state_root, Some(slot))?
.get_state(&state_root, Some(slot))?
.ok_or_else(|| Error::NoStateForSlot(slot))?)
}
}
@@ -890,7 +850,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// An honest validator would have set this block to be the head of the chain (i.e., the
// result of running fork choice).
let result = if let Some(attestation_head_block) =
self.get_block_caching(&attestation.data.beacon_block_root)?
self.get_block(&attestation.data.beacon_block_root)?
{
// If the attestation points to a block in the same epoch in which it was made,
// then it is sufficient to load the state from that epoch's boundary, because
@@ -1274,22 +1234,21 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// Load the blocks parent block from the database, returning invalid if that block is not
// found.
let parent_block: BeaconBlock<T::EthSpec> =
match self.get_block_caching(&block.parent_root)? {
Some(block) => block,
None => {
return Ok(BlockProcessingOutcome::ParentUnknown {
parent: block.parent_root,
reference_location: "database",
});
}
};
let parent_block = match self.get_block(&block.parent_root)? {
Some(block) => block,
None => {
return Ok(BlockProcessingOutcome::ParentUnknown {
parent: block.parent_root,
reference_location: "database",
});
}
};
// Load the parent blocks state from the database, returning an error if it is not found.
// It is an error because if we know the parent block we should also know the parent state.
let parent_state_root = parent_block.state_root;
let parent_state = self
.get_state_caching(&parent_state_root, Some(parent_block.slot))?
.get_state(&parent_state_root, Some(parent_block.slot))?
.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {:?}", parent_state_root))
})?;
@@ -1300,25 +1259,26 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE);
// Keep a list of any states that were "skipped" (block-less) in between the parent state
// slot and the block slot. These will need to be stored in the database.
let mut intermediate_states = vec![];
// Keep a batch of any states that were "skipped" (block-less) in between the parent state
// slot and the block slot. These will be stored in the database.
let mut intermediate_states = StateBatch::new();
// Transition the parent state to the block slot.
let mut state: BeaconState<T::EthSpec> = parent_state;
let distance = block.slot.as_u64().saturating_sub(state.slot.as_u64());
for i in 0..distance {
if i > 0 {
intermediate_states.push(state.clone());
}
let state_root = if i == 0 {
Some(parent_block.state_root)
parent_block.state_root
} else {
None
// This is a new state we've reached, so stage it for storage in the DB.
// Computing the state root here is time-equivalent to computing it during slot
// processing, but we get early access to it.
let state_root = state.update_tree_hash_cache()?;
intermediate_states.add_state(state_root, &state)?;
state_root
};
per_slot_processing(&mut state, state_root, &self.spec)?;
per_slot_processing(&mut state, Some(state_root), &self.spec)?;
}
metrics::stop_timer(catchup_timer);
@@ -1393,23 +1353,17 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
metrics::stop_timer(fork_choice_register_timer);
self.head_tracker.register_block(block_root, &block);
metrics::observe(
&metrics::OPERATIONS_PER_BLOCK_ATTESTATION,
block.body.attestations.len() as f64,
);
let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE);
// Store all the states between the parent block state and this blocks slot before storing
// Store all the states between the parent block state and this block's slot before storing
// the final state.
for (i, intermediate_state) in intermediate_states.iter().enumerate() {
// To avoid doing an unnecessary tree hash, use the following (slot + 1) state's
// state_roots field to find the root.
let following_state = match intermediate_states.get(i + 1) {
Some(following_state) => following_state,
None => &state,
};
let intermediate_state_root =
following_state.get_state_root(intermediate_state.slot)?;
self.store
.put_state(&intermediate_state_root, intermediate_state)?;
}
intermediate_states.commit(&*self.store)?;
// Store the block and state.
// NOTE: we store the block *after* the state to guard against inconsistency in the event of
@@ -1417,29 +1371,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// solution would be to use a database transaction (once our choice of database and API
// settles down).
// See: https://github.com/sigp/lighthouse/issues/692
self.store.put_state(&state_root, &state)?;
self.store.put(&block_root, &block)?;
self.store.put_state(&state_root, state)?;
self.store.put_block(&block_root, block)?;
metrics::stop_timer(db_write_timer);
self.head_tracker.register_block(block_root, &block);
metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES);
metrics::observe(
&metrics::OPERATIONS_PER_BLOCK_ATTESTATION,
block.body.attestations.len() as f64,
);
// Store the block in the checkpoint cache.
//
// A block that was just imported is likely to be referenced by the next block that we
// import.
self.checkpoint_cache.insert(Cow::Owned(CheckPoint {
beacon_block_root: block_root,
beacon_block: block,
beacon_state_root: state_root,
beacon_state: state,
}));
metrics::stop_timer(full_timer);
@@ -1575,13 +1512,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let result = if beacon_block_root != self.head_info()?.block_root {
metrics::inc_counter(&metrics::FORK_CHOICE_CHANGED_HEAD);
let beacon_block: BeaconBlock<T::EthSpec> = self
.get_block_caching(&beacon_block_root)?
let beacon_block = self
.get_block(&beacon_block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(beacon_block_root))?;
let beacon_state_root = beacon_block.state_root;
let beacon_state: BeaconState<T::EthSpec> = self
.get_state_caching(&beacon_state_root, Some(beacon_block.slot))?
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(beacon_state_root))?;
let previous_slot = self.head_info()?.slot;
@@ -1650,11 +1587,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let timer = metrics::start_timer(&metrics::UPDATE_HEAD_TIMES);
// Store the head in the checkpoint cache.
//
// The head block is likely to be referenced by the next imported block.
self.checkpoint_cache.insert(Cow::Borrowed(&new_head));
// Update the checkpoint that stores the head of the chain at the time it received the
// block.
*self
@@ -1703,7 +1635,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
) -> Result<(), Error> {
let finalized_block = self
.store
.get::<BeaconBlock<T::EthSpec>>(&finalized_block_root)?
.get_block(&finalized_block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(finalized_block_root))?;
let new_finalized_epoch = finalized_block.slot.epoch(T::EthSpec::slots_per_epoch());

4
beacon_node/beacon_chain/src/builder.rs
View File

@@ -1,4 +1,3 @@
use crate::checkpoint_cache::CheckPointCache;
use crate::eth1_chain::CachingEth1Backend;
use crate::events::NullEventHandler;
use crate::head_tracker::HeadTracker;
@@ -219,7 +218,7 @@ where
self.genesis_block_root = Some(beacon_block_root);
store
.put_state(&beacon_state_root, &beacon_state)
.put_state(&beacon_state_root, beacon_state.clone())
.map_err(|e| format!("Failed to store genesis state: {:?}", e))?;
store
.put(&beacon_block_root, &beacon_block)
@@ -334,7 +333,6 @@ where
.event_handler
.ok_or_else(|| "Cannot build without an event handler".to_string())?,
head_tracker: self.head_tracker.unwrap_or_default(),
checkpoint_cache: CheckPointCache::default(),
log: log.clone(),
};

125
beacon_node/beacon_chain/src/checkpoint_cache.rs
View File

@@ -1,125 +0,0 @@
use crate::checkpoint::CheckPoint;
use crate::metrics;
use parking_lot::RwLock;
use std::borrow::Cow;
use types::{BeaconBlock, BeaconState, EthSpec, Hash256};
const CACHE_SIZE: usize = 4;
struct Inner<T: EthSpec> {
oldest: usize,
limit: usize,
checkpoints: Vec<CheckPoint<T>>,
}
impl<T: EthSpec> Default for Inner<T> {
fn default() -> Self {
Self {
oldest: 0,
limit: CACHE_SIZE,
checkpoints: vec![],
}
}
}
pub struct CheckPointCache<T: EthSpec> {
inner: RwLock<Inner<T>>,
}
impl<T: EthSpec> Default for CheckPointCache<T> {
fn default() -> Self {
Self {
inner: RwLock::new(Inner::default()),
}
}
}
impl<T: EthSpec> CheckPointCache<T> {
pub fn insert(&self, checkpoint: Cow<CheckPoint<T>>) {
if self
.inner
.read()
.checkpoints
.iter()
// This is `O(n)` but whilst `n == 4` it ain't no thing.
.any(|local| local.beacon_state_root == checkpoint.beacon_state_root)
{
// Adding a known checkpoint to the cache should be a no-op.
return;
}
let mut inner = self.inner.write();
if inner.checkpoints.len() < inner.limit {
inner.checkpoints.push(checkpoint.into_owned())
} else {
let i = inner.oldest; // to satisfy the borrow checker.
inner.checkpoints[i] = checkpoint.into_owned();
inner.oldest += 1;
inner.oldest %= inner.limit;
}
}
pub fn get_state(&self, state_root: &Hash256) -> Option<BeaconState<T>> {
self.inner
.read()
.checkpoints
.iter()
// Also `O(n)`.
.find(|checkpoint| checkpoint.beacon_state_root == *state_root)
.map(|checkpoint| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
checkpoint.beacon_state.clone()
})
.or_else(|| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
None
})
}
pub fn get_state_only_with_committee_cache(
&self,
state_root: &Hash256,
) -> Option<BeaconState<T>> {
self.inner
.read()
.checkpoints
.iter()
// Also `O(n)`.
.find(|checkpoint| checkpoint.beacon_state_root == *state_root)
.map(|checkpoint| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
let mut state = checkpoint.beacon_state.clone_without_caches();
state.committee_caches = checkpoint.beacon_state.committee_caches.clone();
state
})
.or_else(|| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
None
})
}
pub fn get_block(&self, block_root: &Hash256) -> Option<BeaconBlock<T>> {
self.inner
.read()
.checkpoints
.iter()
// Also `O(n)`.
.find(|checkpoint| checkpoint.beacon_block_root == *block_root)
.map(|checkpoint| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_HITS);
checkpoint.beacon_block.clone()
})
.or_else(|| {
metrics::inc_counter(&metrics::CHECKPOINT_CACHE_MISSES);
None
})
}
}

4
beacon_node/beacon_chain/src/eth1_chain.rs
View File

@@ -883,7 +883,7 @@ mod test {
&state
.get_state_root(prev_state.slot)
.expect("should find state root"),
&prev_state,
prev_state,
)
.expect("should store state");
@@ -953,7 +953,7 @@ mod test {
&state
.get_state_root(Slot::new(0))
.expect("should find state root"),
&prev_state,
prev_state,
)
.expect("should store state");

2
beacon_node/beacon_chain/src/fork_choice/checkpoint_manager.rs
View File

@@ -302,7 +302,7 @@ impl CheckpointManager {
metrics::inc_counter(&metrics::BALANCES_CACHE_MISSES);
let block = chain
.get_block_caching(&block_root)?
.get_block(&block_root)?
.ok_or_else(|| Error::UnknownJustifiedBlock(block_root))?;
let state = chain

1
beacon_node/beacon_chain/src/lib.rs
View File

@@ -5,7 +5,6 @@ extern crate lazy_static;
mod beacon_chain;
pub mod builder;
mod checkpoint;
mod checkpoint_cache;
mod errors;
pub mod eth1_chain;
pub mod events;

8
beacon_node/beacon_chain/src/metrics.rs
View File

@@ -149,14 +149,6 @@ lazy_static! {
pub static ref PERSIST_CHAIN: Result<Histogram> =
try_create_histogram("beacon_persist_chain", "Time taken to update the canonical head");
/*
* Checkpoint cache
*/
pub static ref CHECKPOINT_CACHE_HITS: Result<IntCounter> =
try_create_int_counter("beacon_checkpoint_cache_hits_total", "Count of times checkpoint cache fulfils request");
pub static ref CHECKPOINT_CACHE_MISSES: Result<IntCounter> =
try_create_int_counter("beacon_checkpoint_cache_misses_total", "Count of times checkpoint cache fulfils request");
/*
* Eth1
*/