gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-20 21:34:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement freezer database (#508)

Browse Source 
* Implement freezer database for state vectors

* Improve BeaconState safe accessors

And fix a bug in the compact committees accessor.

* Banish dodgy type bounds back to gRPC

* Clean up

* Switch to exclusive end points in chunked vec

* Cleaning up and start of tests

* Randao fix, more tests

* Fix unsightly hack

* Resolve test FIXMEs

* Config file support

* More clean-ups, migrator beginnings

* Finish migrator, integrate into BeaconChain

* Fixups

* Fix store tests

* Fix BeaconChain tests

* Fix LMD GHOST tests

* Address review comments, delete 'static bounds

* Cargo format

* Address review comments

* Fix LMD ghost tests

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

* Update for new builder paradigm, spec changes

* Freezer DB integration into BeaconNode

* Cleaning up

* This works, clean it up

* Cleanups

* Fix and improve store tests

* Refine store test

* Delete unused beacon_chain_builder.rs

* Fix CLI

* Store state at split slot in hot database

* Make fork choice lookup fast again

* Store freezer DB split slot in the database

* Handle potential div by 0 in chunked_vector

* Exclude committee caches from freezer DB

* Remove FIXME about long-running test
This commit is contained in:
Michael Sproul
2019-11-27 10:54:46 +11:00
committed by Paul Hauner
parent a514968155
commit bf2eeae3f2
36 changed files with 2486 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -3,7 +3,6 @@ use crate::errors::{BeaconChainError as Error, BlockProductionError};
use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend};
use crate::events::{EventHandler, EventKind};
use crate::fork_choice::{Error as ForkChoiceError, ForkChoice};
use crate::iter::{ReverseBlockRootIterator, ReverseStateRootIterator};
use crate::metrics;
use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY};
use lmd_ghost::LmdGhost;
@@ -26,8 +25,10 @@ use state_processing::{
use std::fs;
use std::io::prelude::*;
use std::sync::Arc;
use store::iter::{BlockRootsIterator, StateRootsIterator};
use store::{Error as DBError, Store};
use store::iter::{
BlockRootsIterator, ReverseBlockRootIterator, ReverseStateRootIterator, StateRootsIterator,
};
use store::{Error as DBError, Migrate, Store};
use tree_hash::TreeHash;
use types::*;
@@ -95,6 +96,7 @@ pub enum AttestationProcessingOutcome {
pub trait BeaconChainTypes: Send + Sync + 'static {
type Store: store::Store;
type StoreMigrator: store::Migrate<Self::Store, Self::EthSpec>;
type SlotClock: slot_clock::SlotClock;
type LmdGhost: LmdGhost<Self::Store, Self::EthSpec>;
type Eth1Chain: Eth1ChainBackend<Self::EthSpec>;
@@ -108,6 +110,8 @@ pub struct BeaconChain<T: BeaconChainTypes> {
pub spec: ChainSpec,
/// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB.
pub store: Arc<T::Store>,
/// Database migrator for running background maintenance on the store.
pub store_migrator: T::StoreMigrator,
/// Reports the current slot, typically based upon the system clock.
pub slot_clock: T::SlotClock,
/// Stores all operations (e.g., `Attestation`, `Deposit`, etc) that are candidates for
@@ -237,7 +241,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.get_block(&block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(block_root))?;
let state = self
.get_state(&block.state_root)?
.store
.get_state(&block.state_root, Some(block.slot))?
.ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
let iter = BlockRootsIterator::owned(self.store.clone(), state);
Ok(ReverseBlockRootIterator::new(
@@ -318,18 +323,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self.store.get(block_root)?)
}
/// Returns the state at the given root, if any.
///
/// ## Errors
///
/// May return a database error.
pub fn get_state(
&self,
state_root: &Hash256,
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
Ok(self.store.get(state_root)?)
}
/// Returns a `Checkpoint` representing the head block and state. Contains the "best block";
/// the head of the canonical `BeaconChain`.
///
@@ -403,7 +396,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self
.store
.get(&state_root)?
.get_state(&state_root, Some(slot))?
.ok_or_else(|| Error::NoStateForSlot(slot))?)
}
}
@@ -762,7 +755,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// not guaranteed to be from the same slot or epoch as the attestation.
let mut state: BeaconState<T::EthSpec> = self
.store
.get(&attestation_head_block.state_root)?
.get_state(
&attestation_head_block.state_root,
Some(attestation_head_block.slot),
)?
.ok_or_else(|| Error::MissingBeaconState(attestation_head_block.state_root))?;
// Ensure the state loaded from the database matches the state of the attestation
@@ -1123,8 +1119,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let parent_state_root = parent_block.state_root;
let parent_state = self
.store
.get(&parent_state_root)?
.ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?;
.get_state(&parent_state_root, Some(parent_block.slot))?
.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {:?}", parent_state_root))
})?;
metrics::stop_timer(db_read_timer);
@@ -1214,12 +1212,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
following_state.get_state_root(intermediate_state.slot)?;
self.store
.put(&intermediate_state_root, intermediate_state)?;
.put_state(&intermediate_state_root, intermediate_state)?;
}
// Store the block and state.
self.store.put(&block_root, &block)?;
self.store.put(&state_root, &state)?;
self.store.put_state(&state_root, &state)?;
metrics::stop_timer(db_write_timer);
@@ -1282,7 +1280,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.state_at_slot(slot - 1)
.map_err(|_| BlockProductionError::UnableToProduceAtSlot(slot))?;
self.produce_block_on_state(state.clone(), slot, randao_reveal)
self.produce_block_on_state(state, slot, randao_reveal)
}
/// Produce a block for some `slot` upon the given `state`.
@@ -1396,7 +1394,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let beacon_state_root = beacon_block.state_root;
let beacon_state: BeaconState<T::EthSpec> = self
.store
.get(&beacon_state_root)?
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(beacon_state_root))?;
let previous_slot = self.head().beacon_block.slot;
@@ -1519,11 +1517,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let finalized_state = self
.store
.get::<BeaconState<T::EthSpec>>(&finalized_block.state_root)?
.get_state(&finalized_block.state_root, Some(finalized_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(finalized_block.state_root))?;
self.op_pool.prune_all(&finalized_state, &self.spec);
// TODO: configurable max finality distance
let max_finality_distance = 0;
self.store_migrator.freeze_to_state(
finalized_block.state_root,
finalized_state,
max_finality_distance,
);
let _ = self.event_handler.register(EventKind::BeaconFinalization {
epoch: new_finalized_epoch,
root: finalized_block_root,
@@ -1568,9 +1574,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Error::DBInconsistent(format!("Missing block {}", beacon_block_root))
})?;
let beacon_state_root = beacon_block.state_root;
let beacon_state = self.store.get(&beacon_state_root)?.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {}", beacon_state_root))
})?;
let beacon_state = self
.store
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {:?}", beacon_state_root))
})?;
let slot = CheckPoint {
beacon_block,