gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 18:32:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into spec-v0.12

Browse Source
This commit is contained in:
Michael Sproul
2020-06-17 12:34:11 +10:00
parent 764cb2d32a 9db0c28051
commit e6f97bf466
60 changed files with 877 additions and 1242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -25,6 +25,7 @@ use crate::validator_pubkey_cache::ValidatorPubkeyCache;
use crate::BeaconForkChoiceStore;
use crate::BeaconSnapshot;
use fork_choice::ForkChoice;
use itertools::process_results;
use operation_pool::{OperationPool, PersistedOperationPool};
use parking_lot::RwLock;
use slog::{crit, debug, error, info, trace, warn, Logger};
@@ -45,7 +46,7 @@ use std::io::prelude::*;
use std::sync::Arc;
use std::time::{Duration, Instant};
use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator};
use store::{Error as DBError, Store};
use store::{Error as DBError, HotColdDB};
use types::*;
pub type ForkChoiceError = fork_choice::Error<crate::ForkChoiceStoreError>;
@@ -155,10 +156,11 @@ pub struct HeadInfo {
}
pub trait BeaconChainTypes: Send + Sync + 'static {
type Store: store::Store<Self::EthSpec>;
type StoreMigrator: Migrate<Self::EthSpec>;
type HotStore: store::ItemStore<Self::EthSpec>;
type ColdStore: store::ItemStore<Self::EthSpec>;
type StoreMigrator: Migrate<Self::EthSpec, Self::HotStore, Self::ColdStore>;
type SlotClock: slot_clock::SlotClock;
type Eth1Chain: Eth1ChainBackend<Self::EthSpec, Self::Store>;
type Eth1Chain: Eth1ChainBackend<Self::EthSpec>;
type EthSpec: types::EthSpec;
type EventHandler: EventHandler<Self::EthSpec>;
}
@@ -168,7 +170,7 @@ pub trait BeaconChainTypes: Send + Sync + 'static {
pub struct BeaconChain<T: BeaconChainTypes> {
pub spec: ChainSpec,
/// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB.
pub store: Arc<T::Store>,
pub store: Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>,
/// Database migrator for running background maintenance on the store.
pub store_migrator: T::StoreMigrator,
/// Reports the current slot, typically based upon the system clock.
@@ -192,7 +194,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
/// Maintains a record of which validators have proposed blocks for each slot.
pub observed_block_producers: ObservedBlockProducers<T::EthSpec>,
/// Provides information from the Ethereum 1 (PoW) chain.
pub eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec, T::Store>>,
pub eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
/// Stores a "snapshot" of the chain at the time the head-of-the-chain block was received.
pub(crate) canonical_head: TimeoutRwLock<BeaconSnapshot<T::EthSpec>>,
/// The root of the genesis block.
@@ -330,27 +332,33 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// - Iterator returns `(Hash256, Slot)`.
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
/// returned may be earlier than the wall-clock slot.
pub fn rev_iter_block_roots(&self) -> Result<impl Iterator<Item = (Hash256, Slot)>, Error> {
pub fn rev_iter_block_roots(
&self,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> {
let head = self.head()?;
let iter = BlockRootsIterator::owned(self.store.clone(), head.beacon_state);
Ok(std::iter::once((head.beacon_block_root, head.beacon_block.slot())).chain(iter))
Ok(
std::iter::once(Ok((head.beacon_block_root, head.beacon_block.slot())))
.chain(iter)
.map(|result| result.map_err(|e| e.into())),
)
}
pub fn forwards_iter_block_roots(
&self,
start_slot: Slot,
) -> Result<<T::Store as Store<T::EthSpec>>::ForwardsBlockRootsIterator, Error> {
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> {
let local_head = self.head()?;
Ok(T::Store::forwards_block_roots_iterator(
let iter = HotColdDB::forwards_block_roots_iterator(
self.store.clone(),
start_slot,
local_head.beacon_state,
local_head.beacon_block_root,
&self.spec,
))
)?;
Ok(iter.map(|result| result.map_err(Into::into)))
}
/// Traverse backwards from `block_root` to find the block roots of its ancestors.
@@ -365,7 +373,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
pub fn rev_iter_block_roots_from(
&self,
block_root: Hash256,
) -> Result<impl Iterator<Item = (Hash256, Slot)>, Error> {
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> {
let block = self
.get_block(&block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(block_root))?;
@@ -373,7 +381,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.get_state(&block.state_root(), Some(block.slot()))?
.ok_or_else(|| Error::MissingBeaconState(block.state_root()))?;
let iter = BlockRootsIterator::owned(self.store.clone(), state);
Ok(std::iter::once((block_root, block.slot())).chain(iter))
Ok(std::iter::once(Ok((block_root, block.slot())))
.chain(iter)
.map(|result| result.map_err(|e| e.into())))
}
/// Traverse backwards from `block_root` to find the root of the ancestor block at `slot`.
@@ -382,10 +392,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
block_root: Hash256,
slot: Slot,
) -> Result<Option<Hash256>, Error> {
Ok(self
.rev_iter_block_roots_from(block_root)?
.find(|(_, ancestor_slot)| *ancestor_slot == slot)
.map(|(ancestor_block_root, _)| ancestor_block_root))
process_results(self.rev_iter_block_roots_from(block_root)?, |mut iter| {
iter.find(|(_, ancestor_slot)| *ancestor_slot == slot)
.map(|(ancestor_block_root, _)| ancestor_block_root)
})
}
/// Iterates across all `(state_root, slot)` pairs from the head of the chain (inclusive) to
@@ -397,13 +407,16 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// - Iterator returns `(Hash256, Slot)`.
/// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot
/// returned may be earlier than the wall-clock slot.
pub fn rev_iter_state_roots(&self) -> Result<impl Iterator<Item = (Hash256, Slot)>, Error> {
pub fn rev_iter_state_roots(
&self,
) -> Result<impl Iterator<Item = Result<(Hash256, Slot), Error>>, Error> {
let head = self.head()?;
let slot = head.beacon_state.slot;
let iter = StateRootsIterator::owned(self.store.clone(), head.beacon_state);
Ok(std::iter::once((head.beacon_state_root, slot)).chain(iter))
let iter = std::iter::once(Ok((head.beacon_state_root, slot)))
.chain(iter)
.map(|result| result.map_err(Into::into));
Ok(iter)
}
/// Returns the block at the given slot, if any. Only returns blocks in the canonical chain.
@@ -415,10 +428,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
slot: Slot,
) -> Result<Option<SignedBeaconBlock<T::EthSpec>>, Error> {
let root = self
.rev_iter_block_roots()?
.find(|(_, this_slot)| *this_slot == slot)
.map(|(root, _)| root);
let root = process_results(self.rev_iter_block_roots()?, |mut iter| {
iter.find(|(_, this_slot)| *this_slot == slot)
.map(|(root, _)| root)
})?;
if let Some(block_root) = root {
Ok(self.store.get_item(&block_root)?)
@@ -564,12 +577,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(state)
}
Ordering::Less => {
let state_root = self
.rev_iter_state_roots()?
.take_while(|(_root, current_slot)| *current_slot >= slot)
.find(|(_root, current_slot)| *current_slot == slot)
.map(|(root, _slot)| root)
.ok_or_else(|| Error::NoStateForSlot(slot))?;
let state_root = process_results(self.rev_iter_state_roots()?, |iter| {
iter.take_while(|(_, current_slot)| *current_slot >= slot)
.find(|(_, current_slot)| *current_slot == slot)
.map(|(root, _slot)| root)
})?
.ok_or_else(|| Error::NoStateForSlot(slot))?;
Ok(self
.get_state(&state_root, Some(slot))?
@@ -644,10 +657,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// Returns None if a block doesn't exist at the slot.
pub fn root_at_slot(&self, target_slot: Slot) -> Result<Option<Hash256>, Error> {
Ok(self
.rev_iter_block_roots()?
.find(|(_root, slot)| *slot == target_slot)
.map(|(root, _slot)| root))
process_results(self.rev_iter_block_roots()?, |mut iter| {
iter.find(|(_, slot)| *slot == target_slot)
.map(|(root, _)| root)
})
}
/// Returns the block proposer for a given slot.

135
beacon_node/beacon_chain/src/builder.rs
View File

@@ -25,7 +25,7 @@ use std::marker::PhantomData;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use store::Store;
use store::{HotColdDB, ItemStore};
use types::{
BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Signature, SignedBeaconBlock, Slot,
};
@@ -34,28 +34,48 @@ pub const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
/// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
/// functionality and only exists to satisfy the type system.
pub struct Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>(
pub struct Witness<
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>(
PhantomData<(
TStore,
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
)>,
);
impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler> BeaconChainTypes
for Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
BeaconChainTypes
for Witness<
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>
where
TStore: Store<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec> + 'static,
THotStore: ItemStore<TEthSpec> + 'static,
TColdStore: ItemStore<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
TSlotClock: SlotClock + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
type Store = TStore;
type HotStore = THotStore;
type ColdStore = TColdStore;
type StoreMigrator = TStoreMigrator;
type SlotClock = TSlotClock;
type Eth1Chain = TEth1Backend;
@@ -72,7 +92,7 @@ where
///
/// See the tests for an example of a complete working example.
pub struct BeaconChainBuilder<T: BeaconChainTypes> {
store: Option<Arc<T::Store>>,
store: Option<Arc<HotColdDB<T::EthSpec, T::HotStore, T::ColdStore>>>,
store_migrator: Option<T::StoreMigrator>,
canonical_head: Option<BeaconSnapshot<T::EthSpec>>,
/// The finalized checkpoint to anchor the chain. May be genesis or a higher
@@ -80,7 +100,7 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
pub finalized_snapshot: Option<BeaconSnapshot<T::EthSpec>>,
genesis_block_root: Option<Hash256>,
op_pool: Option<OperationPool<T::EthSpec>>,
eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec, T::Store>>,
eth1_chain: Option<Eth1Chain<T::Eth1Chain, T::EthSpec>>,
event_handler: Option<T::EventHandler>,
slot_clock: Option<T::SlotClock>,
head_tracker: Option<HeadTracker>,
@@ -92,15 +112,24 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
log: Option<Logger>,
}
impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
BeaconChainBuilder<
Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>,
>
where
TStore: Store<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec> + 'static,
THotStore: ItemStore<TEthSpec> + 'static,
TColdStore: ItemStore<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
TSlotClock: SlotClock + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
@@ -141,7 +170,7 @@ where
/// Sets the store (database).
///
/// Should generally be called early in the build chain.
pub fn store(mut self, store: Arc<TStore>) -> Self {
pub fn store(mut self, store: Arc<HotColdDB<TEthSpec, THotStore, TColdStore>>) -> Self {
self.store = Some(store);
self
}
@@ -378,7 +407,15 @@ where
self,
) -> Result<
BeaconChain<
Witness<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>,
>,
String,
> {
@@ -500,20 +537,22 @@ where
}
}
impl<TStore, TStoreMigrator, TSlotClock, TEthSpec, TEventHandler>
impl<TStoreMigrator, TSlotClock, TEthSpec, TEventHandler, THotStore, TColdStore>
BeaconChainBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
CachingEth1Backend<TEthSpec, TStore>,
CachingEth1Backend<TEthSpec>,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>,
>
where
TStore: Store<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec> + 'static,
THotStore: ItemStore<TEthSpec> + 'static,
TColdStore: ItemStore<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
TSlotClock: SlotClock + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
@@ -529,12 +568,8 @@ where
.log
.as_ref()
.ok_or_else(|| "dummy_eth1_backend requires a log".to_string())?;
let store = self
.store
.clone()
.ok_or_else(|| "dummy_eth1_backend requires a store.".to_string())?;
let backend = CachingEth1Backend::new(Eth1Config::default(), log.clone(), store);
let backend = CachingEth1Backend::new(Eth1Config::default(), log.clone());
let mut eth1_chain = Eth1Chain::new(backend);
eth1_chain.use_dummy_backend = true;
@@ -545,14 +580,23 @@ where
}
}
impl<TStore, TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler>
impl<TStoreMigrator, TEth1Backend, TEthSpec, TEventHandler, THotStore, TColdStore>
BeaconChainBuilder<
Witness<TStore, TStoreMigrator, TestingSlotClock, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStoreMigrator,
TestingSlotClock,
TEth1Backend,
TEthSpec,
TEventHandler,
THotStore,
TColdStore,
>,
>
where
TStore: Store<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
THotStore: ItemStore<TEthSpec> + 'static,
TColdStore: ItemStore<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
@@ -577,22 +621,24 @@ where
}
}
impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec>
impl<TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, THotStore, TColdStore>
BeaconChainBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TEth1Backend,
TEthSpec,
NullEventHandler<TEthSpec>,
THotStore,
TColdStore,
>,
>
where
TStore: Store<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec> + 'static,
THotStore: ItemStore<TEthSpec> + 'static,
TColdStore: ItemStore<TEthSpec> + 'static,
TStoreMigrator: Migrate<TEthSpec, THotStore, TColdStore> + 'static,
TSlotClock: SlotClock + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec, TStore> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
{
/// Sets the `BeaconChain` event handler to `NullEventHandler`.
@@ -622,12 +668,14 @@ fn genesis_block<T: EthSpec>(
#[cfg(test)]
mod test {
use super::*;
use crate::migrate::{MemoryStore, NullMigrator};
use crate::migrate::NullMigrator;
use eth2_hashing::hash;
use genesis::{generate_deterministic_keypairs, interop_genesis_state};
use sloggers::{null::NullLoggerBuilder, Build};
use ssz::Encode;
use std::time::Duration;
use store::config::StoreConfig;
use store::{HotColdDB, MemoryStore};
use tempfile::tempdir;
use types::{EthSpec, MinimalEthSpec, Slot};
@@ -644,7 +692,12 @@ mod test {
let genesis_time = 13_371_337;
let log = get_logger();
let store = Arc::new(MemoryStore::open());
let store: HotColdDB<
MinimalEthSpec,
MemoryStore<MinimalEthSpec>,
MemoryStore<MinimalEthSpec>,
> = HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal(), log.clone())
.unwrap();
let spec = MinimalEthSpec::default_spec();
let data_dir = tempdir().expect("should create temporary data_dir");
@@ -657,7 +710,7 @@ mod test {
let chain = BeaconChainBuilder::new(MinimalEthSpec)
.logger(log.clone())
.store(store)
.store(Arc::new(store))
.store_migrator(NullMigrator)
.data_dir(data_dir.path().to_path_buf())
.genesis_state(genesis_state)

71
beacon_node/beacon_chain/src/eth1_chain.rs
View File

@@ -10,8 +10,7 @@ use std::cmp::Ordering;
use std::collections::HashMap;
use std::iter::DoubleEndedIterator;
use std::marker::PhantomData;
use std::sync::Arc;
use store::{DBColumn, Error as StoreError, Store, StoreItem};
use store::{DBColumn, Error as StoreError, StoreItem};
use types::{
BeaconState, BeaconStateError, ChainSpec, Deposit, Eth1Data, EthSpec, Hash256, Slot, Unsigned,
DEPOSIT_TREE_DEPTH,
@@ -75,24 +74,22 @@ impl StoreItem for SszEth1 {
}
/// Holds an `Eth1ChainBackend` and serves requests from the `BeaconChain`.
pub struct Eth1Chain<T, E, S>
pub struct Eth1Chain<T, E>
where
T: Eth1ChainBackend<E, S>,
T: Eth1ChainBackend<E>,
E: EthSpec,
S: Store<E>,
{
backend: T,
/// When `true`, the backend will be ignored and dummy data from the 2019 Canada interop method
/// will be used instead.
pub use_dummy_backend: bool,
_phantom: PhantomData<(E, S)>,
_phantom: PhantomData<E>,
}
impl<T, E, S> Eth1Chain<T, E, S>
impl<T, E> Eth1Chain<T, E>
where
T: Eth1ChainBackend<E, S>,
T: Eth1ChainBackend<E>,
E: EthSpec,
S: Store<E>,
{
pub fn new(backend: T) -> Self {
Self {
@@ -110,7 +107,7 @@ where
spec: &ChainSpec,
) -> Result<Eth1Data, Error> {
if self.use_dummy_backend {
let dummy_backend: DummyEth1ChainBackend<E, S> = DummyEth1ChainBackend::default();
let dummy_backend: DummyEth1ChainBackend<E> = DummyEth1ChainBackend::default();
dummy_backend.eth1_data(state, spec)
} else {
self.backend.eth1_data(state, spec)
@@ -132,7 +129,7 @@ where
spec: &ChainSpec,
) -> Result<Vec<Deposit>, Error> {
if self.use_dummy_backend {
let dummy_backend: DummyEth1ChainBackend<E, S> = DummyEth1ChainBackend::default();
let dummy_backend: DummyEth1ChainBackend<E> = DummyEth1ChainBackend::default();
dummy_backend.queued_deposits(state, eth1_data_vote, spec)
} else {
self.backend.queued_deposits(state, eth1_data_vote, spec)
@@ -145,11 +142,10 @@ where
pub fn from_ssz_container(
ssz_container: &SszEth1,
config: Eth1Config,
store: Arc<S>,
log: &Logger,
) -> Result<Self, String> {
let backend =
Eth1ChainBackend::from_bytes(&ssz_container.backend_bytes, config, store, log.clone())?;
Eth1ChainBackend::from_bytes(&ssz_container.backend_bytes, config, log.clone())?;
Ok(Self {
use_dummy_backend: ssz_container.use_dummy_backend,
backend,
@@ -171,7 +167,7 @@ where
}
}
pub trait Eth1ChainBackend<T: EthSpec, S: Store<T>>: Sized + Send + Sync {
pub trait Eth1ChainBackend<T: EthSpec>: Sized + Send + Sync {
/// Returns the `Eth1Data` that should be included in a block being produced for the given
/// `state`.
fn eth1_data(&self, beacon_state: &BeaconState<T>, spec: &ChainSpec)
@@ -195,12 +191,7 @@ pub trait Eth1ChainBackend<T: EthSpec, S: Store<T>>: Sized + Send + Sync {
fn as_bytes(&self) -> Vec<u8>;
/// Create a `Eth1ChainBackend` instance given encoded bytes.
fn from_bytes(
bytes: &[u8],
config: Eth1Config,
store: Arc<S>,
log: Logger,
) -> Result<Self, String>;
fn from_bytes(bytes: &[u8], config: Eth1Config, log: Logger) -> Result<Self, String>;
}
/// Provides a simple, testing-only backend that generates deterministic, meaningless eth1 data.
@@ -208,9 +199,9 @@ pub trait Eth1ChainBackend<T: EthSpec, S: Store<T>>: Sized + Send + Sync {
/// Never creates deposits, therefore the validator set is static.
///
/// This was used in the 2019 Canada interop workshops.
pub struct DummyEth1ChainBackend<T: EthSpec, S: Store<T>>(PhantomData<(T, S)>);
pub struct DummyEth1ChainBackend<T: EthSpec>(PhantomData<T>);
impl<T: EthSpec, S: Store<T>> Eth1ChainBackend<T, S> for DummyEth1ChainBackend<T, S> {
impl<T: EthSpec> Eth1ChainBackend<T> for DummyEth1ChainBackend<T> {
/// Produce some deterministic junk based upon the current epoch.
fn eth1_data(&self, state: &BeaconState<T>, _spec: &ChainSpec) -> Result<Eth1Data, Error> {
let current_epoch = state.current_epoch();
@@ -243,17 +234,12 @@ impl<T: EthSpec, S: Store<T>> Eth1ChainBackend<T, S> for DummyEth1ChainBackend<T
}
/// Create dummy eth1 backend.
fn from_bytes(
_bytes: &[u8],
_config: Eth1Config,
_store: Arc<S>,
_log: Logger,
) -> Result<Self, String> {
fn from_bytes(_bytes: &[u8], _config: Eth1Config, _log: Logger) -> Result<Self, String> {
Ok(Self(PhantomData))
}
}
impl<T: EthSpec, S: Store<T>> Default for DummyEth1ChainBackend<T, S> {
impl<T: EthSpec> Default for DummyEth1ChainBackend<T> {
fn default() -> Self {
Self(PhantomData)
}
@@ -265,21 +251,19 @@ impl<T: EthSpec, S: Store<T>> Default for DummyEth1ChainBackend<T, S> {
/// The `core` connects to some external eth1 client (e.g., Parity/Geth) and polls it for
/// information.
#[derive(Clone)]
pub struct CachingEth1Backend<T: EthSpec, S> {
pub struct CachingEth1Backend<T: EthSpec> {
pub core: HttpService,
store: Arc<S>,
log: Logger,
_phantom: PhantomData<T>,
}
impl<T: EthSpec, S: Store<T>> CachingEth1Backend<T, S> {
impl<T: EthSpec> CachingEth1Backend<T> {
/// Instantiates `self` with empty caches.
///
/// Does not connect to the eth1 node or start any tasks to keep the cache updated.
pub fn new(config: Eth1Config, log: Logger, store: Arc<S>) -> Self {
pub fn new(config: Eth1Config, log: Logger) -> Self {
Self {
core: HttpService::new(config, log.clone()),
store,
log,
_phantom: PhantomData,
}
@@ -291,17 +275,16 @@ impl<T: EthSpec, S: Store<T>> CachingEth1Backend<T, S> {
}
/// Instantiates `self` from an existing service.
pub fn from_service(service: HttpService, store: Arc<S>) -> Self {
pub fn from_service(service: HttpService) -> Self {
Self {
log: service.log.clone(),
core: service,
store,
_phantom: PhantomData,
}
}
}
impl<T: EthSpec, S: Store<T>> Eth1ChainBackend<T, S> for CachingEth1Backend<T, S> {
impl<T: EthSpec> Eth1ChainBackend<T> for CachingEth1Backend<T> {
fn eth1_data(&self, state: &BeaconState<T>, spec: &ChainSpec) -> Result<Eth1Data, Error> {
let period = T::SlotsPerEth1VotingPeriod::to_u64();
let voting_period_start_slot = (state.slot / period) * period;
@@ -406,16 +389,10 @@ impl<T: EthSpec, S: Store<T>> Eth1ChainBackend<T, S> for CachingEth1Backend<T, S
}
/// Recover the cached backend from encoded bytes.
fn from_bytes(
bytes: &[u8],
config: Eth1Config,
store: Arc<S>,
log: Logger,
) -> Result<Self, String> {
fn from_bytes(bytes: &[u8], config: Eth1Config, log: Logger) -> Result<Self, String> {
let inner = HttpService::from_bytes(bytes, config, log.clone())?;
Ok(Self {
core: inner,
store,
log,
_phantom: PhantomData,
})
@@ -572,17 +549,15 @@ mod test {
mod eth1_chain_json_backend {
use super::*;
use eth1::DepositLog;
use store::MemoryStore;
use types::test_utils::{generate_deterministic_keypair, TestingDepositBuilder};
fn get_eth1_chain() -> Eth1Chain<CachingEth1Backend<E, MemoryStore<E>>, E, MemoryStore<E>> {
fn get_eth1_chain() -> Eth1Chain<CachingEth1Backend<E>, E> {
let eth1_config = Eth1Config {
..Eth1Config::default()
};
let log = null_logger().unwrap();
let store = Arc::new(MemoryStore::open());
Eth1Chain::new(CachingEth1Backend::new(eth1_config, log, store))
Eth1Chain::new(CachingEth1Backend::new(eth1_config, log))
}
fn get_deposit_log(i: u64, spec: &ChainSpec) -> DepositLog {

50
beacon_node/beacon_chain/src/migrate.rs
View File

@@ -9,14 +9,16 @@ use std::sync::Arc;
use std::thread;
use store::hot_cold_store::{process_finalization, HotColdDBError};
use store::iter::{ParentRootBlockIterator, RootsIterator};
use store::{Error, Store, StoreOp};
use store::{Error, ItemStore, StoreOp};
pub use store::{HotColdDB, MemoryStore};
use types::*;
use types::{BeaconState, EthSpec, Hash256, Slot};
/// Trait for migration processes that update the database upon finalization.
pub trait Migrate<E: EthSpec>: Send + Sync + 'static {
fn new(db: Arc<HotColdDB<E>>, log: Logger) -> Self;
pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
Send + Sync + 'static
{
fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self;
fn process_finalization(
&self,
@@ -30,18 +32,23 @@ pub trait Migrate<E: EthSpec>: Send + Sync + 'static {
}
/// Traverses live heads and prunes blocks and states of chains that we know can't be built
/// upon because finalization would prohibit it. This is a optimisation intended to save disk
/// upon because finalization would prohibit it. This is an optimisation intended to save disk
/// space.
///
/// Assumptions:
/// * It is called after every finalization.
fn prune_abandoned_forks(
store: Arc<HotColdDB<E>>,
store: Arc<HotColdDB<E, Hot, Cold>>,
head_tracker: Arc<HeadTracker>,
old_finalized_block_hash: SignedBeaconBlockHash,
new_finalized_block_hash: SignedBeaconBlockHash,
new_finalized_slot: Slot,
) -> Result<(), BeaconChainError> {
// There will never be any blocks to prune if there is only a single head in the chain.
if head_tracker.heads().len() == 1 {
return Ok(());
}
let old_finalized_slot = store
.get_block(&old_finalized_block_hash.into())?
.ok_or_else(|| BeaconChainError::MissingBeaconBlock(old_finalized_block_hash.into()))?
@@ -84,9 +91,10 @@ pub trait Migrate<E: EthSpec>: Send + Sync + 'static {
.ok_or_else(|| BeaconStateError::MissingBeaconBlock(head_hash.into()))?
.state_root();
let iterator = std::iter::once((head_hash, head_state_hash, head_slot))
let iter = std::iter::once(Ok((head_hash, head_state_hash, head_slot)))
.chain(RootsIterator::from_block(Arc::clone(&store), head_hash)?);
for (block_hash, state_hash, slot) in iterator {
for maybe_tuple in iter {
let (block_hash, state_hash, slot) = maybe_tuple?;
if slot < old_finalized_slot {
// We must assume here any candidate chains include old_finalized_block_hash,
// i.e. there aren't any forks starting at a block that is a strict ancestor of
@@ -165,8 +173,8 @@ pub trait Migrate<E: EthSpec>: Send + Sync + 'static {
/// Migrator that does nothing, for stores that don't need migration.
pub struct NullMigrator;
impl<E: EthSpec> Migrate<E> for NullMigrator {
fn new(_: Arc<HotColdDB<E>>, _: Logger) -> Self {
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
NullMigrator
}
}
@@ -174,12 +182,14 @@ impl<E: EthSpec> Migrate<E> for NullMigrator {
/// Migrator that immediately calls the store's migration function, blocking the current execution.
///
/// Mostly useful for tests.
pub struct BlockingMigrator<E: EthSpec> {
db: Arc<HotColdDB<E>>,
pub struct BlockingMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
db: Arc<HotColdDB<E, Hot, Cold>>,
}
impl<E: EthSpec> Migrate<E> for BlockingMigrator<E> {
fn new(db: Arc<HotColdDB<E>>, _: Logger) -> Self {
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
for BlockingMigrator<E, Hot, Cold>
{
fn new(db: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
BlockingMigrator { db }
}
@@ -219,14 +229,16 @@ type MpscSender<E> = mpsc::Sender<(
)>;
/// Migrator that runs a background thread to migrate state from the hot to the cold database.
pub struct BackgroundMigrator<E: EthSpec> {
db: Arc<HotColdDB<E>>,
pub struct BackgroundMigrator<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
db: Arc<HotColdDB<E, Hot, Cold>>,
tx_thread: Mutex<(MpscSender<E>, thread::JoinHandle<()>)>,
log: Logger,
}
impl<E: EthSpec> Migrate<E> for BackgroundMigrator<E> {
fn new(db: Arc<HotColdDB<E>>, log: Logger) -> Self {
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
for BackgroundMigrator<E, Hot, Cold>
{
fn new(db: Arc<HotColdDB<E, Hot, Cold>>, log: Logger) -> Self {
let tx_thread = Mutex::new(Self::spawn_thread(db.clone(), log.clone()));
Self { db, tx_thread, log }
}
@@ -277,7 +289,7 @@ impl<E: EthSpec> Migrate<E> for BackgroundMigrator<E> {
}
}
impl<E: EthSpec> BackgroundMigrator<E> {
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Hot, Cold> {
/// Return true if a migration needs to be performed, given a new `finalized_slot`.
fn needs_migration(&self, finalized_slot: Slot, max_finality_distance: u64) -> bool {
let finality_distance = finalized_slot - self.db.get_split_slot();
@@ -288,7 +300,7 @@ impl<E: EthSpec> BackgroundMigrator<E> {
///
/// Return a channel handle for sending new finalized states to the thread.
fn spawn_thread(
db: Arc<HotColdDB<E>>,
db: Arc<HotColdDB<E, Hot, Cold>>,
log: Logger,
) -> (
mpsc::Sender<(

41
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -18,7 +18,7 @@ use std::borrow::Cow;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use store::{HotColdDB, MemoryStore, Store};
use store::{config::StoreConfig, HotColdDB, ItemStore, LevelDB, MemoryStore};
use tempfile::{tempdir, TempDir};
use tree_hash::TreeHash;
use types::{
@@ -34,17 +34,19 @@ pub const HARNESS_GENESIS_TIME: u64 = 1_567_552_690;
// This parameter is required by a builder but not used because we use the `TestingSlotClock`.
pub const HARNESS_SLOT_TIME: Duration = Duration::from_secs(1);
pub type BaseHarnessType<TStore, TStoreMigrator, TEthSpec> = Witness<
TStore,
pub type BaseHarnessType<TStoreMigrator, TEthSpec, THotStore, TColdStore> = Witness<
TStoreMigrator,
TestingSlotClock,
CachingEth1Backend<TEthSpec, TStore>,
CachingEth1Backend<TEthSpec>,
TEthSpec,
NullEventHandler<TEthSpec>,
THotStore,
TColdStore,
>;
pub type HarnessType<E> = BaseHarnessType<MemoryStore<E>, NullMigrator, E>;
pub type DiskHarnessType<E> = BaseHarnessType<HotColdDB<E>, BlockingMigrator<E>, E>;
pub type HarnessType<E> = BaseHarnessType<NullMigrator, E, MemoryStore<E>, MemoryStore<E>>;
pub type DiskHarnessType<E> =
BaseHarnessType<BlockingMigrator<E, LevelDB<E>, LevelDB<E>>, E, LevelDB<E>, LevelDB<E>>;
/// Indicates how the `BeaconChainHarness` should produce blocks.
#[derive(Clone, Copy, Debug)]
@@ -84,12 +86,12 @@ pub struct BeaconChainHarness<T: BeaconChainTypes> {
impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
/// Instantiate a new harness with `validator_count` initial validators.
pub fn new(eth_spec_instance: E, keypairs: Vec<Keypair>) -> Self {
pub fn new(eth_spec_instance: E, keypairs: Vec<Keypair>, config: StoreConfig) -> Self {
// Setting the target aggregators to really high means that _all_ validators in the
// committee are required to produce an aggregate. This is overkill, however with small
// validator counts it's the only way to be certain there is _at least one_ aggregator per
// committee.
Self::new_with_target_aggregators(eth_spec_instance, keypairs, 1 << 32)
Self::new_with_target_aggregators(eth_spec_instance, keypairs, 1 << 32, config)
}
/// Instantiate a new harness with `validator_count` initial validators and a custom
@@ -98,6 +100,7 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
eth_spec_instance: E,
keypairs: Vec<Keypair>,
target_aggregators_per_committee: u64,
config: StoreConfig,
) -> Self {
let data_dir = tempdir().expect("should create temporary data_dir");
let mut spec = E::default_spec();
@@ -105,11 +108,11 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
spec.target_aggregators_per_committee = target_aggregators_per_committee;
let log = NullLoggerBuilder.build().expect("logger should build");
let store = HotColdDB::open_ephemeral(config, spec.clone(), log.clone()).unwrap();
let chain = BeaconChainBuilder::new(eth_spec_instance)
.logger(log.clone())
.logger(log)
.custom_spec(spec.clone())
.store(Arc::new(MemoryStore::open()))
.store(Arc::new(store))
.store_migrator(NullMigrator)
.data_dir(data_dir.path().to_path_buf())
.genesis_state(
@@ -138,7 +141,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
/// Instantiate a new harness with `validator_count` initial validators.
pub fn new_with_disk_store(
eth_spec_instance: E,
store: Arc<HotColdDB<E>>,
store: Arc<HotColdDB<E, LevelDB<E>, LevelDB<E>>>,
keypairs: Vec<Keypair>,
) -> Self {
let data_dir = tempdir().expect("should create temporary data_dir");
@@ -176,7 +179,7 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
/// Instantiate a new harness with `validator_count` initial validators.
pub fn resume_from_disk_store(
eth_spec_instance: E,
store: Arc<HotColdDB<E>>,
store: Arc<HotColdDB<E, LevelDB<E>, LevelDB<E>>>,
keypairs: Vec<Keypair>,
data_dir: TempDir,
) -> Self {
@@ -188,7 +191,10 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
.logger(log.clone())
.custom_spec(spec)
.store(store.clone())
.store_migrator(<BlockingMigrator<_> as Migrate<E>>::new(store, log.clone()))
.store_migrator(<BlockingMigrator<_, _, _> as Migrate<E, _, _>>::new(
store,
log.clone(),
))
.data_dir(data_dir.path().to_path_buf())
.resume_from_db()
.expect("should resume beacon chain from db")
@@ -209,11 +215,12 @@ impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
}
}
impl<S, M, E> BeaconChainHarness<BaseHarnessType<S, M, E>>
impl<M, E, Hot, Cold> BeaconChainHarness<BaseHarnessType<M, E, Hot, Cold>>
where
S: Store<E>,
M: Migrate<E>,
M: Migrate<E, Hot, Cold>,
E: EthSpec,
Hot: ItemStore<E>,
Cold: ItemStore<E>,
{
/// Advance the slot of the `BeaconChain`.
///