gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement freezer database (#508)

Browse Source 
* Implement freezer database for state vectors

* Improve BeaconState safe accessors

And fix a bug in the compact committees accessor.

* Banish dodgy type bounds back to gRPC

* Clean up

* Switch to exclusive end points in chunked vec

* Cleaning up and start of tests

* Randao fix, more tests

* Fix unsightly hack

* Resolve test FIXMEs

* Config file support

* More clean-ups, migrator beginnings

* Finish migrator, integrate into BeaconChain

* Fixups

* Fix store tests

* Fix BeaconChain tests

* Fix LMD GHOST tests

* Address review comments, delete 'static bounds

* Cargo format

* Address review comments

* Fix LMD ghost tests

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

* Update for new builder paradigm, spec changes

* Freezer DB integration into BeaconNode

* Cleaning up

* This works, clean it up

* Cleanups

* Fix and improve store tests

* Refine store test

* Delete unused beacon_chain_builder.rs

* Fix CLI

* Store state at split slot in hot database

* Make fork choice lookup fast again

* Store freezer DB split slot in the database

* Handle potential div by 0 in chunked_vector

* Exclude committee caches from freezer DB

* Remove FIXME about long-running test
This commit is contained in:
Michael Sproul
2019-11-27 10:54:46 +11:00
committed by Paul Hauner
parent a514968155
commit bf2eeae3f2
36 changed files with 2486 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -3,7 +3,6 @@ use crate::errors::{BeaconChainError as Error, BlockProductionError};
use crate::eth1_chain::{Eth1Chain, Eth1ChainBackend};
use crate::events::{EventHandler, EventKind};
use crate::fork_choice::{Error as ForkChoiceError, ForkChoice};
use crate::iter::{ReverseBlockRootIterator, ReverseStateRootIterator};
use crate::metrics;
use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY};
use lmd_ghost::LmdGhost;
@@ -26,8 +25,10 @@ use state_processing::{
use std::fs;
use std::io::prelude::*;
use std::sync::Arc;
use store::iter::{BlockRootsIterator, StateRootsIterator};
use store::{Error as DBError, Store};
use store::iter::{
BlockRootsIterator, ReverseBlockRootIterator, ReverseStateRootIterator, StateRootsIterator,
};
use store::{Error as DBError, Migrate, Store};
use tree_hash::TreeHash;
use types::*;
@@ -95,6 +96,7 @@ pub enum AttestationProcessingOutcome {
pub trait BeaconChainTypes: Send + Sync + 'static {
type Store: store::Store;
type StoreMigrator: store::Migrate<Self::Store, Self::EthSpec>;
type SlotClock: slot_clock::SlotClock;
type LmdGhost: LmdGhost<Self::Store, Self::EthSpec>;
type Eth1Chain: Eth1ChainBackend<Self::EthSpec>;
@@ -108,6 +110,8 @@ pub struct BeaconChain<T: BeaconChainTypes> {
pub spec: ChainSpec,
/// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB.
pub store: Arc<T::Store>,
/// Database migrator for running background maintenance on the store.
pub store_migrator: T::StoreMigrator,
/// Reports the current slot, typically based upon the system clock.
pub slot_clock: T::SlotClock,
/// Stores all operations (e.g., `Attestation`, `Deposit`, etc) that are candidates for
@@ -237,7 +241,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.get_block(&block_root)?
.ok_or_else(|| Error::MissingBeaconBlock(block_root))?;
let state = self
.get_state(&block.state_root)?
.store
.get_state(&block.state_root, Some(block.slot))?
.ok_or_else(|| Error::MissingBeaconState(block.state_root))?;
let iter = BlockRootsIterator::owned(self.store.clone(), state);
Ok(ReverseBlockRootIterator::new(
@@ -318,18 +323,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self.store.get(block_root)?)
}
/// Returns the state at the given root, if any.
///
/// ## Errors
///
/// May return a database error.
pub fn get_state(
&self,
state_root: &Hash256,
) -> Result<Option<BeaconState<T::EthSpec>>, Error> {
Ok(self.store.get(state_root)?)
}
/// Returns a `Checkpoint` representing the head block and state. Contains the "best block";
/// the head of the canonical `BeaconChain`.
///
@@ -403,7 +396,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Ok(self
.store
.get(&state_root)?
.get_state(&state_root, Some(slot))?
.ok_or_else(|| Error::NoStateForSlot(slot))?)
}
}
@@ -762,7 +755,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// not guaranteed to be from the same slot or epoch as the attestation.
let mut state: BeaconState<T::EthSpec> = self
.store
.get(&attestation_head_block.state_root)?
.get_state(
&attestation_head_block.state_root,
Some(attestation_head_block.slot),
)?
.ok_or_else(|| Error::MissingBeaconState(attestation_head_block.state_root))?;
// Ensure the state loaded from the database matches the state of the attestation
@@ -1123,8 +1119,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let parent_state_root = parent_block.state_root;
let parent_state = self
.store
.get(&parent_state_root)?
.ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?;
.get_state(&parent_state_root, Some(parent_block.slot))?
.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {:?}", parent_state_root))
})?;
metrics::stop_timer(db_read_timer);
@@ -1214,12 +1212,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
following_state.get_state_root(intermediate_state.slot)?;
self.store
.put(&intermediate_state_root, intermediate_state)?;
.put_state(&intermediate_state_root, intermediate_state)?;
}
// Store the block and state.
self.store.put(&block_root, &block)?;
self.store.put(&state_root, &state)?;
self.store.put_state(&state_root, &state)?;
metrics::stop_timer(db_write_timer);
@@ -1282,7 +1280,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.state_at_slot(slot - 1)
.map_err(|_| BlockProductionError::UnableToProduceAtSlot(slot))?;
self.produce_block_on_state(state.clone(), slot, randao_reveal)
self.produce_block_on_state(state, slot, randao_reveal)
}
/// Produce a block for some `slot` upon the given `state`.
@@ -1396,7 +1394,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let beacon_state_root = beacon_block.state_root;
let beacon_state: BeaconState<T::EthSpec> = self
.store
.get(&beacon_state_root)?
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(beacon_state_root))?;
let previous_slot = self.head().beacon_block.slot;
@@ -1519,11 +1517,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let finalized_state = self
.store
.get::<BeaconState<T::EthSpec>>(&finalized_block.state_root)?
.get_state(&finalized_block.state_root, Some(finalized_block.slot))?
.ok_or_else(|| Error::MissingBeaconState(finalized_block.state_root))?;
self.op_pool.prune_all(&finalized_state, &self.spec);
// TODO: configurable max finality distance
let max_finality_distance = 0;
self.store_migrator.freeze_to_state(
finalized_block.state_root,
finalized_state,
max_finality_distance,
);
let _ = self.event_handler.register(EventKind::BeaconFinalization {
epoch: new_finalized_epoch,
root: finalized_block_root,
@@ -1568,9 +1574,12 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
Error::DBInconsistent(format!("Missing block {}", beacon_block_root))
})?;
let beacon_state_root = beacon_block.state_root;
let beacon_state = self.store.get(&beacon_state_root)?.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {}", beacon_state_root))
})?;
let beacon_state = self
.store
.get_state(&beacon_state_root, Some(beacon_block.slot))?
.ok_or_else(|| {
Error::DBInconsistent(format!("Missing state {:?}", beacon_state_root))
})?;
let slot = CheckPoint {
beacon_block,

331
beacon_node/beacon_chain/src/beacon_chain_builder.rs
View File

@@ -1,331 +0,0 @@
use crate::{BeaconChain, BeaconChainTypes};
use eth2_hashing::hash;
use lighthouse_bootstrap::Bootstrapper;
use merkle_proof::MerkleTree;
use rayon::prelude::*;
use slog::Logger;
use ssz::{Decode, Encode};
use state_processing::initialize_beacon_state_from_eth1;
use std::fs::File;
use std::io::prelude::*;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::SystemTime;
use tree_hash::{SignedRoot, TreeHash};
use types::{
BeaconBlock, BeaconState, ChainSpec, Deposit, DepositData, Domain, EthSpec, Fork, Hash256,
Keypair, PublicKey, Signature,
};
enum BuildStrategy<T: BeaconChainTypes> {
FromGenesis {
genesis_state: Box<BeaconState<T::EthSpec>>,
genesis_block: Box<BeaconBlock<T::EthSpec>>,
},
LoadFromStore,
}
pub struct BeaconChainBuilder<T: BeaconChainTypes> {
build_strategy: BuildStrategy<T>,
spec: ChainSpec,
log: Logger,
}
impl<T: BeaconChainTypes> BeaconChainBuilder<T> {
pub fn recent_genesis(
keypairs: &[Keypair],
minutes: u64,
spec: ChainSpec,
log: Logger,
) -> Result<Self, String> {
Self::quick_start(recent_genesis_time(minutes), keypairs, spec, log)
}
pub fn quick_start(
genesis_time: u64,
keypairs: &[Keypair],
spec: ChainSpec,
log: Logger,
) -> Result<Self, String> {
let genesis_state = interop_genesis_state(keypairs, genesis_time, &spec)?;
Ok(Self::from_genesis_state(genesis_state, spec, log))
}
pub fn yaml_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
let file = File::open(file.clone())
.map_err(|e| format!("Unable to open YAML genesis state file {:?}: {:?}", file, e))?;
let genesis_state = serde_yaml::from_reader(file)
.map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))?;
Ok(Self::from_genesis_state(genesis_state, spec, log))
}
pub fn ssz_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
let mut file = File::open(file.clone())
.map_err(|e| format!("Unable to open SSZ genesis state file {:?}: {:?}", file, e))?;
let mut bytes = vec![];
file.read_to_end(&mut bytes)
.map_err(|e| format!("Failed to read SSZ file: {:?}", e))?;
let genesis_state = BeaconState::from_ssz_bytes(&bytes)
.map_err(|e| format!("Unable to parse SSZ genesis state file: {:?}", e))?;
Ok(Self::from_genesis_state(genesis_state, spec, log))
}
pub fn json_state(file: &PathBuf, spec: ChainSpec, log: Logger) -> Result<Self, String> {
let file = File::open(file.clone())
.map_err(|e| format!("Unable to open JSON genesis state file {:?}: {:?}", file, e))?;
let genesis_state = serde_json::from_reader(file)
.map_err(|e| format!("Unable to parse JSON genesis state file: {:?}", e))?;
Ok(Self::from_genesis_state(genesis_state, spec, log))
}
pub fn http_bootstrap(server: &str, spec: ChainSpec, log: Logger) -> Result<Self, String> {
let bootstrapper = Bootstrapper::connect(server.to_string(), &log)
.map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?;
let (genesis_state, genesis_block) = bootstrapper
.genesis()
.map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?;
Ok(Self {
build_strategy: BuildStrategy::FromGenesis {
genesis_block: Box::new(genesis_block),
genesis_state: Box::new(genesis_state),
},
spec,
log,
})
}
fn from_genesis_state(
genesis_state: BeaconState<T::EthSpec>,
spec: ChainSpec,
log: Logger,
) -> Self {
Self {
build_strategy: BuildStrategy::FromGenesis {
genesis_block: Box::new(genesis_block(&genesis_state, &spec)),
genesis_state: Box::new(genesis_state),
},
spec,
log,
}
}
pub fn from_store(spec: ChainSpec, log: Logger) -> Self {
Self {
build_strategy: BuildStrategy::LoadFromStore,
spec,
log,
}
}
pub fn build(
self,
store: Arc<T::Store>,
eth1_backend: T::Eth1Chain,
event_handler: T::EventHandler,
) -> Result<BeaconChain<T>, String> {
Ok(match self.build_strategy {
BuildStrategy::LoadFromStore => {
BeaconChain::from_store(store, eth1_backend, event_handler, self.spec, self.log)
.map_err(|e| format!("Error loading BeaconChain from database: {:?}", e))?
.ok_or_else(|| "Unable to find exising BeaconChain in database.".to_string())?
}
BuildStrategy::FromGenesis {
genesis_block,
genesis_state,
} => BeaconChain::from_genesis(
store,
eth1_backend,
event_handler,
genesis_state.as_ref().clone(),
genesis_block.as_ref().clone(),
self.spec,
self.log,
)
.map_err(|e| format!("Failed to initialize new beacon chain: {:?}", e))?,
})
}
}
fn genesis_block<T: EthSpec>(genesis_state: &BeaconState<T>, spec: &ChainSpec) -> BeaconBlock<T> {
let mut genesis_block = BeaconBlock::empty(&spec);
genesis_block.state_root = genesis_state.canonical_root();
genesis_block
}
/// Builds a genesis state as defined by the Eth2 interop procedure (see below).
///
/// Reference:
/// https://github.com/ethereum/eth2.0-pm/tree/6e41fcf383ebeb5125938850d8e9b4e9888389b4/interop/mocked_start
fn interop_genesis_state<T: EthSpec>(
keypairs: &[Keypair],
genesis_time: u64,
spec: &ChainSpec,
) -> Result<BeaconState<T>, String> {
let eth1_block_hash = Hash256::from_slice(&[0x42; 32]);
let eth1_timestamp = 2_u64.pow(40);
let amount = spec.max_effective_balance;
let withdrawal_credentials = |pubkey: &PublicKey| {
let mut credentials = hash(&pubkey.as_ssz_bytes());
credentials[0] = spec.bls_withdrawal_prefix_byte;
Hash256::from_slice(&credentials)
};
let datas = keypairs
.into_par_iter()
.map(|keypair| {
let mut data = DepositData {
withdrawal_credentials: withdrawal_credentials(&keypair.pk),
pubkey: keypair.pk.clone().into(),
amount,
signature: Signature::empty_signature().into(),
};
let domain = spec.get_domain(
spec.genesis_slot.epoch(T::slots_per_epoch()),
Domain::Deposit,
&Fork::default(),
);
data.signature = Signature::new(&data.signed_root()[..], domain, &keypair.sk).into();
data
})
.collect::<Vec<_>>();
let deposit_root_leaves = datas
.par_iter()
.map(|data| Hash256::from_slice(&data.tree_hash_root()))
.collect::<Vec<_>>();
let mut proofs = vec![];
let depth = spec.deposit_contract_tree_depth as usize;
let mut tree = MerkleTree::create(&[], depth);
for (i, deposit_leaf) in deposit_root_leaves.iter().enumerate() {
if let Err(_) = tree.push_leaf(*deposit_leaf, depth) {
return Err(String::from("Failed to push leaf"));
}
let (_, mut proof) = tree.generate_proof(i, depth);
proof.push(Hash256::from_slice(&int_to_bytes32(i + 1)));
assert_eq!(
proof.len(),
depth + 1,
"Deposit proof should be correct len"
);
proofs.push(proof);
}
let deposits = datas
.into_par_iter()
.zip(proofs.into_par_iter())
.map(|(data, proof)| (data, proof.into()))
.map(|(data, proof)| Deposit { proof, data })
.collect::<Vec<_>>();
let mut state =
initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits, spec)
.map_err(|e| format!("Unable to initialize genesis state: {:?}", e))?;
state.genesis_time = genesis_time;
// Invalid all the caches after all the manual state surgery.
state.drop_all_caches();
Ok(state)
}
/// Returns `int` as little-endian bytes with a length of 32.
fn int_to_bytes32(int: usize) -> Vec<u8> {
let mut vec = int.to_le_bytes().to_vec();
vec.resize(32, 0);
vec
}
/// Returns the system time, mod 30 minutes.
///
/// Used for easily creating testnets.
fn recent_genesis_time(minutes: u64) -> u64 {
let now = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_secs();
let secs_after_last_period = now.checked_rem(minutes * 60).unwrap_or(0);
now - secs_after_last_period
}
#[cfg(test)]
mod test {
use super::*;
use types::{test_utils::generate_deterministic_keypairs, EthSpec, MinimalEthSpec};
type TestEthSpec = MinimalEthSpec;
#[test]
fn interop_state() {
let validator_count = 16;
let genesis_time = 42;
let spec = &TestEthSpec::default_spec();
let keypairs = generate_deterministic_keypairs(validator_count);
let state = interop_genesis_state::<TestEthSpec>(&keypairs, genesis_time, spec)
.expect("should build state");
assert_eq!(
state.eth1_data.block_hash,
Hash256::from_slice(&[0x42; 32]),
"eth1 block hash should be co-ordinated junk"
);
assert_eq!(
state.genesis_time, genesis_time,
"genesis time should be as specified"
);
for b in &state.balances {
assert_eq!(
*b, spec.max_effective_balance,
"validator balances should be max effective balance"
);
}
for v in &state.validators {
let creds = v.withdrawal_credentials.as_bytes();
assert_eq!(
creds[0], spec.bls_withdrawal_prefix_byte,
"first byte of withdrawal creds should be bls prefix"
);
assert_eq!(
&creds[1..],
&hash(&v.pubkey.as_ssz_bytes())[1..],
"rest of withdrawal creds should be pubkey hash"
)
}
assert_eq!(
state.balances.len(),
validator_count,
"validator balances len should be correct"
);
assert_eq!(
state.validators.len(),
validator_count,
"validator count should be correct"
);
}
}

111
beacon_node/beacon_chain/src/builder.rs
View File

@@ -19,9 +19,18 @@ use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot};
/// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing
/// functionality and only exists to satisfy the type system.
pub struct Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>(
pub struct Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>(
PhantomData<(
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
@@ -30,10 +39,20 @@ pub struct Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEvent
)>,
);
impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler> BeaconChainTypes
for Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
BeaconChainTypes
for Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -41,6 +60,7 @@ where
TEventHandler: EventHandler<TEthSpec> + 'static,
{
type Store = TStore;
type StoreMigrator = TStoreMigrator;
type SlotClock = TSlotClock;
type LmdGhost = TLmdGhost;
type Eth1Chain = TEth1Backend;
@@ -58,6 +78,7 @@ where
/// See the tests for an example of a complete working example.
pub struct BeaconChainBuilder<T: BeaconChainTypes> {
store: Option<Arc<T::Store>>,
store_migrator: Option<T::StoreMigrator>,
/// The finalized checkpoint to anchor the chain. May be genesis or a higher
/// checkpoint.
pub finalized_checkpoint: Option<CheckPoint<T::EthSpec>>,
@@ -71,12 +92,21 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
log: Option<Logger>,
}
impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
BeaconChainBuilder<
Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -90,6 +120,7 @@ where
pub fn new(_eth_spec_instance: TEthSpec) -> Self {
Self {
store: None,
store_migrator: None,
finalized_checkpoint: None,
genesis_block_root: None,
op_pool: None,
@@ -119,6 +150,12 @@ where
self
}
/// Sets the store migrator.
pub fn store_migrator(mut self, store_migrator: TStoreMigrator) -> Self {
self.store_migrator = Some(store_migrator);
self
}
/// Sets the logger.
///
/// Should generally be called early in the build chain.
@@ -149,7 +186,15 @@ where
let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes());
let p: PersistedBeaconChain<
Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
> = match store.get(&key) {
Err(e) => {
return Err(format!(
@@ -195,7 +240,7 @@ where
self.genesis_block_root = Some(beacon_block_root);
store
.put(&beacon_state_root, &beacon_state)
.put_state(&beacon_state_root, &beacon_state)
.map_err(|e| format!("Failed to store genesis state: {:?}", e))?;
store
.put(&beacon_block_root, &beacon_block)
@@ -279,7 +324,17 @@ where
pub fn build(
self,
) -> Result<
BeaconChain<Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>>,
BeaconChain<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>,
String,
> {
let mut canonical_head = self
@@ -304,6 +359,9 @@ where
store: self
.store
.ok_or_else(|| "Cannot build without store".to_string())?,
store_migrator: self
.store_migrator
.ok_or_else(|| "Cannot build without store migrator".to_string())?,
slot_clock: self
.slot_clock
.ok_or_else(|| "Cannot build without slot clock".to_string())?,
@@ -336,10 +394,11 @@ where
}
}
impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
BeaconChainBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
ThreadSafeReducedTree<TStore, TEthSpec>,
TEth1Backend,
@@ -349,6 +408,7 @@ impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TSlotClock: SlotClock + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
@@ -378,10 +438,11 @@ where
}
}
impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
BeaconChainBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
CachingEth1Backend<TEthSpec, TStore>,
@@ -391,6 +452,7 @@ impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
@@ -428,12 +490,21 @@ where
}
}
impl<TStore, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
BeaconChainBuilder<
Witness<TStore, TestingSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStore,
TStoreMigrator,
TestingSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
@@ -460,12 +531,21 @@ where
}
}
impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
BeaconChainBuilder<
Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, NullEventHandler<TEthSpec>>,
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
NullEventHandler<TEthSpec>,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec> + 'static,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -494,7 +574,7 @@ mod test {
use sloggers::{null::NullLoggerBuilder, Build};
use ssz::Encode;
use std::time::Duration;
use store::MemoryStore;
use store::{migrate::NullMigrator, MemoryStore};
use types::{EthSpec, MinimalEthSpec, Slot};
type TestEthSpec = MinimalEthSpec;
@@ -523,6 +603,7 @@ mod test {
let chain = BeaconChainBuilder::new(MinimalEthSpec)
.logger(log.clone())
.store(store.clone())
.store_migrator(NullMigrator)
.genesis_state(genesis_state)
.expect("should build state using recent genesis")
.dummy_eth1_backend()

6
beacon_node/beacon_chain/src/eth1_chain.rs
View File

@@ -293,7 +293,7 @@ fn eth1_block_hash_at_start_of_voting_period<T: EthSpec, S: Store>(
.map_err(|e| Error::UnableToGetPreviousStateRoot(e))?;
store
.get::<BeaconState<T>>(&prev_state_root)
.get_state::<T>(&prev_state_root, Some(slot))
.map_err(|e| Error::StoreError(e))?
.map(|state| state.eth1_data.block_hash)
.ok_or_else(|| Error::PreviousStateNotInDB)
@@ -676,7 +676,7 @@ mod test {
);
store
.put(
.put_state(
&state
.get_state_root(prev_state.slot)
.expect("should find state root"),
@@ -738,7 +738,7 @@ mod test {
);
store
.put(
.put_state(
&state
.get_state_root(Slot::new(0))
.expect("should find state root"),

5
beacon_node/beacon_chain/src/fork_choice.rs
View File

@@ -137,8 +137,9 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
block_root
};
let mut state = chain
.get_state(&block.state_root)?
let mut state: BeaconState<T::EthSpec> = chain
.store
.get_state(&block.state_root, Some(block.slot))?
.ok_or_else(|| Error::MissingState(block.state_root))?;
// Fast-forward the state to the start slot of the epoch where it was justified.

48
beacon_node/beacon_chain/src/iter.rs
View File

@@ -1,48 +0,0 @@
use store::iter::{BlockRootsIterator, StateRootsIterator};
use types::{Hash256, Slot};
pub type ReverseBlockRootIterator<'a, E, S> =
ReverseHashAndSlotIterator<BlockRootsIterator<'a, E, S>>;
pub type ReverseStateRootIterator<'a, E, S> =
ReverseHashAndSlotIterator<StateRootsIterator<'a, E, S>>;
pub type ReverseHashAndSlotIterator<I> = ReverseChainIterator<(Hash256, Slot), I>;
/// Provides a wrapper for an iterator that returns a given `T` before it starts returning results of
/// the `Iterator`.
pub struct ReverseChainIterator<T, I> {
first_value_used: bool,
first_value: T,
iter: I,
}
impl<T, I> ReverseChainIterator<T, I>
where
T: Sized,
I: Iterator<Item = T> + Sized,
{
pub fn new(first_value: T, iter: I) -> Self {
Self {
first_value_used: false,
first_value,
iter,
}
}
}
impl<T, I> Iterator for ReverseChainIterator<T, I>
where
T: Clone,
I: Iterator<Item = T>,
{
type Item = T;
fn next(&mut self) -> Option<Self::Item> {
if self.first_value_used {
self.iter.next()
} else {
self.first_value_used = true;
Some(self.first_value.clone())
}
}
}

1
beacon_node/beacon_chain/src/lib.rs
View File

@@ -9,7 +9,6 @@ mod errors;
pub mod eth1_chain;
pub mod events;
mod fork_choice;
mod iter;
mod metrics;
mod persisted_beacon_chain;
pub mod test_utils;

6
beacon_node/beacon_chain/src/persisted_beacon_chain.rs
View File

@@ -2,7 +2,7 @@ use crate::{BeaconChainTypes, CheckPoint};
use operation_pool::PersistedOperationPool;
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use store::{DBColumn, Error as StoreError, StoreItem};
use store::{DBColumn, Error as StoreError, SimpleStoreItem};
use types::Hash256;
/// 32-byte key for accessing the `PersistedBeaconChain`.
@@ -15,7 +15,7 @@ pub struct PersistedBeaconChain<T: BeaconChainTypes> {
pub genesis_block_root: Hash256,
}
impl<T: BeaconChainTypes> StoreItem for PersistedBeaconChain<T> {
impl<T: BeaconChainTypes> SimpleStoreItem for PersistedBeaconChain<T> {
fn db_column() -> DBColumn {
DBColumn::BeaconChain
}
@@ -24,7 +24,7 @@ impl<T: BeaconChainTypes> StoreItem for PersistedBeaconChain<T> {
self.as_ssz_bytes()
}
fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, StoreError> {
fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
Self::from_ssz_bytes(bytes).map_err(Into::into)
}
}

71
beacon_node/beacon_chain/src/test_utils.rs
View File

@@ -12,7 +12,10 @@ use slot_clock::TestingSlotClock;
use state_processing::per_slot_processing;
use std::sync::Arc;
use std::time::Duration;
use store::MemoryStore;
use store::{
migrate::{BlockingMigrator, NullMigrator},
DiskStore, MemoryStore, Migrate, Store,
};
use tree_hash::{SignedRoot, TreeHash};
use types::{
AggregateSignature, Attestation, BeaconBlock, BeaconState, BitList, ChainSpec, Domain, EthSpec,
@@ -24,15 +27,19 @@ pub use types::test_utils::generate_deterministic_keypairs;
pub const HARNESS_GENESIS_TIME: u64 = 1_567_552_690; // 4th September 2019
pub type HarnessType<E> = Witness<
MemoryStore,
pub type BaseHarnessType<TStore, TStoreMigrator, TEthSpec> = Witness<
TStore,
TStoreMigrator,
TestingSlotClock,
ThreadSafeReducedTree<MemoryStore, E>,
CachingEth1Backend<E, MemoryStore>,
E,
NullEventHandler<E>,
ThreadSafeReducedTree<TStore, TEthSpec>,
CachingEth1Backend<TEthSpec, TStore>,
TEthSpec,
NullEventHandler<TEthSpec>,
>;
pub type HarnessType<E> = BaseHarnessType<MemoryStore, NullMigrator, E>;
pub type DiskHarnessType<E> = BaseHarnessType<DiskStore, BlockingMigrator<DiskStore>, E>;
/// Indicates how the `BeaconChainHarness` should produce blocks.
#[derive(Clone, Copy, Debug)]
pub enum BlockStrategy {
@@ -82,6 +89,7 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
.logger(log.clone())
.custom_spec(spec.clone())
.store(Arc::new(MemoryStore::open()))
.store_migrator(NullMigrator)
.genesis_state(
interop_genesis_state::<E>(&keypairs, HARNESS_GENESIS_TIME, &spec)
.expect("should generate interop state"),
@@ -103,7 +111,56 @@ impl<E: EthSpec> BeaconChainHarness<HarnessType<E>> {
keypairs,
}
}
}
impl<E: EthSpec> BeaconChainHarness<DiskHarnessType<E>> {
/// Instantiate a new harness with `validator_count` initial validators.
pub fn with_disk_store(
eth_spec_instance: E,
store: Arc<DiskStore>,
keypairs: Vec<Keypair>,
) -> Self {
let spec = E::default_spec();
let log = TerminalLoggerBuilder::new()
.level(Severity::Warning)
.build()
.expect("logger should build");
let chain = BeaconChainBuilder::new(eth_spec_instance)
.logger(log.clone())
.custom_spec(spec.clone())
.store(store.clone())
.store_migrator(<BlockingMigrator<_> as Migrate<_, E>>::new(store))
.genesis_state(
interop_genesis_state::<E>(&keypairs, HARNESS_GENESIS_TIME, &spec)
.expect("should generate interop state"),
)
.expect("should build state using recent genesis")
.dummy_eth1_backend()
.expect("should build dummy backend")
.null_event_handler()
.testing_slot_clock(Duration::from_secs(1))
.expect("should configure testing slot clock")
.empty_reduced_tree_fork_choice()
.expect("should add fork choice to builder")
.build()
.expect("should build");
Self {
spec: chain.spec.clone(),
chain,
keypairs,
}
}
}
impl<S, M, E> BeaconChainHarness<BaseHarnessType<S, M, E>>
where
S: Store,
M: Migrate<S, E>,
E: EthSpec,
{
/// Advance the slot of the `BeaconChain`.
///
/// Does not produce blocks or attestations.