From 76602a65fc1a4eeb032d10579d0dc1a8c1423e92 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 15:12:51 +1000 Subject: [PATCH 01/76] Add `new` fns to `ForkChoice` and `SlotClock` --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- beacon_node/client/src/beacon_chain_types.rs | 20 +++++------ eth2/fork_choice/src/bitwise_lmd_ghost.rs | 26 +++++++------- eth2/fork_choice/src/lib.rs | 24 +++---------- eth2/fork_choice/src/longest_chain.rs | 6 ++-- eth2/fork_choice/src/optimized_lmd_ghost.rs | 26 +++++++------- eth2/fork_choice/src/slow_lmd_ghost.rs | 20 +++++------ eth2/utils/slot_clock/src/lib.rs | 7 +++- .../slot_clock/src/system_time_slot_clock.rs | 36 ++++++++----------- .../slot_clock/src/testing_slot_clock.rs | 26 +++++++------- validator_client/src/service.rs | 3 +- 11 files changed, 88 insertions(+), 108 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 9f08b6f644..a614698c45 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -82,7 +82,7 @@ impl BlockProcessingOutcome { pub trait BeaconChainTypes { type Store: store::Store; type SlotClock: slot_clock::SlotClock; - type ForkChoice: fork_choice::ForkChoice; + type ForkChoice: fork_choice::ForkChoice; type EthSpec: types::EthSpec; } diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index b8236c679f..1971f56039 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -5,6 +5,8 @@ use beacon_chain::{ store::{DiskStore, MemoryStore, Store}, BeaconChain, BeaconChainTypes, }; +use fork_choice::ForkChoice; +use slot_clock::SlotClock; use std::sync::Arc; use tree_hash::TreeHash; use types::{ @@ -36,7 +38,7 @@ where >, { fn initialise_beacon_chain(_config: &ClientConfig) -> BeaconChain { - initialize_chain(MemoryStore::open()) + initialize_chain::<_, _, FewValidatorsEthSpec>(MemoryStore::open()) } } @@ -62,18 +64,15 @@ where fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain { let store = DiskStore::open(&config.db_name).expect("Unable to open DB."); - initialize_chain(store) + initialize_chain::<_, _, FewValidatorsEthSpec>(store) } } /// Produces a `BeaconChain` given some pre-initialized `Store`. fn initialize_chain(store: U) -> BeaconChain where - T: BeaconChainTypes< - Store = U, - SlotClock = SystemTimeSlotClock, - ForkChoice = BitwiseLMDGhost, - >, + T: BeaconChainTypes, + T::ForkChoice: ForkChoice, { let spec = T::EthSpec::spec(); @@ -86,14 +85,13 @@ where genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); // Slot clock - let slot_clock = SystemTimeSlotClock::new( + let slot_clock = T::SlotClock::new( spec.genesis_slot, genesis_state.genesis_time, spec.seconds_per_slot, - ) - .expect("Unable to load SystemTimeSlotClock"); + ); // Choose the fork choice - let fork_choice = BitwiseLMDGhost::new(store.clone()); + let fork_choice = T::ForkChoice::new(store.clone()); // Genesis chain //TODO: Handle error correctly diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 0e579c0b9e..a26a94b8ac 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -48,18 +48,6 @@ pub struct BitwiseLMDGhost { } impl BitwiseLMDGhost { - pub fn new(store: Arc) -> Self { - BitwiseLMDGhost { - cache: HashMap::new(), - ancestors: vec![HashMap::new(); 16], - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - max_known_height: SlotHeight::new(0), - store, - _phantom: PhantomData, - } - } - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to /// weighted votes. pub fn get_latest_votes( @@ -229,7 +217,19 @@ impl BitwiseLMDGhost { } } -impl ForkChoice for BitwiseLMDGhost { +impl ForkChoice for BitwiseLMDGhost { + fn new(store: Arc) -> Self { + BitwiseLMDGhost { + cache: HashMap::new(), + ancestors: vec![HashMap::new(); 16], + latest_attestation_targets: HashMap::new(), + children: HashMap::new(), + max_known_height: SlotHeight::new(0), + store, + _phantom: PhantomData, + } + } + fn add_block( &mut self, block: &BeaconBlock, diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index ffc40e6c6b..ce53c1051c 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -21,8 +21,7 @@ pub mod longest_chain; pub mod optimized_lmd_ghost; pub mod slow_lmd_ghost; -// use store::stores::BeaconBlockAtSlotError; -// use store::DBError; +use std::sync::Arc; use store::Error as DBError; use types::{BeaconBlock, ChainSpec, Hash256}; @@ -34,7 +33,10 @@ pub use slow_lmd_ghost::SlowLMDGhost; /// Defines the interface for Fork Choices. Each Fork choice will define their own data structures /// which can be built in block processing through the `add_block` and `add_attestation` functions. /// The main fork choice algorithm is specified in `find_head -pub trait ForkChoice: Send + Sync { +pub trait ForkChoice: Send + Sync { + /// Create a new `ForkChoice` which reads from `store`. + fn new(store: Arc) -> Self; + /// Called when a block has been added. Allows generic block-level data structures to be /// built for a given fork-choice. fn add_block( @@ -78,22 +80,6 @@ impl From for ForkChoiceError { } } -/* -impl From for ForkChoiceError { - fn from(e: BeaconBlockAtSlotError) -> ForkChoiceError { - match e { - BeaconBlockAtSlotError::UnknownBeaconBlock(hash) => { - ForkChoiceError::MissingBeaconBlock(hash) - } - BeaconBlockAtSlotError::InvalidBeaconBlock(hash) => { - ForkChoiceError::MissingBeaconBlock(hash) - } - BeaconBlockAtSlotError::DBError(string) => ForkChoiceError::StorageError(string), - } - } -} -*/ - /// Fork choice options that are currently implemented. #[derive(Debug, Clone)] pub enum ForkChoiceAlgorithm { diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs index 11453cf493..08e47cf393 100644 --- a/eth2/fork_choice/src/longest_chain.rs +++ b/eth2/fork_choice/src/longest_chain.rs @@ -10,16 +10,14 @@ pub struct LongestChain { store: Arc, } -impl LongestChain { - pub fn new(store: Arc) -> Self { +impl ForkChoice for LongestChain { + fn new(store: Arc) -> Self { LongestChain { head_block_hashes: Vec::new(), store, } } -} -impl ForkChoice for LongestChain { fn add_block( &mut self, block: &BeaconBlock, diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index dba6e60da1..2c1063a2bc 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -48,18 +48,6 @@ pub struct OptimizedLMDGhost { } impl OptimizedLMDGhost { - pub fn new(store: Arc) -> Self { - OptimizedLMDGhost { - cache: HashMap::new(), - ancestors: vec![HashMap::new(); 16], - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - max_known_height: SlotHeight::new(0), - store, - _phantom: PhantomData, - } - } - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to /// weighted votes. pub fn get_latest_votes( @@ -200,7 +188,19 @@ impl OptimizedLMDGhost { } } -impl ForkChoice for OptimizedLMDGhost { +impl ForkChoice for OptimizedLMDGhost { + fn new(store: Arc) -> Self { + OptimizedLMDGhost { + cache: HashMap::new(), + ancestors: vec![HashMap::new(); 16], + latest_attestation_targets: HashMap::new(), + children: HashMap::new(), + max_known_height: SlotHeight::new(0), + store, + _phantom: PhantomData, + } + } + fn add_block( &mut self, block: &BeaconBlock, diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 8883564179..38b1e8dabb 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -20,15 +20,6 @@ pub struct SlowLMDGhost { } impl SlowLMDGhost { - pub fn new(store: Arc) -> Self { - SlowLMDGhost { - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - store, - _phantom: PhantomData, - } - } - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to /// weighted votes. pub fn get_latest_votes( @@ -94,7 +85,16 @@ impl SlowLMDGhost { } } -impl ForkChoice for SlowLMDGhost { +impl ForkChoice for SlowLMDGhost { + fn new(store: Arc) -> Self { + SlowLMDGhost { + latest_attestation_targets: HashMap::new(), + children: HashMap::new(), + store, + _phantom: PhantomData, + } + } + /// Process when a block is added fn add_block( &mut self, diff --git a/eth2/utils/slot_clock/src/lib.rs b/eth2/utils/slot_clock/src/lib.rs index fd5a2d1d7d..7b86684fa4 100644 --- a/eth2/utils/slot_clock/src/lib.rs +++ b/eth2/utils/slot_clock/src/lib.rs @@ -6,9 +6,14 @@ pub use crate::testing_slot_clock::{Error as TestingSlotClockError, TestingSlotC use std::time::Duration; pub use types::Slot; -pub trait SlotClock: Send + Sync { +pub trait SlotClock: Send + Sync + Sized { type Error; + /// Create a new `SlotClock`. + /// + /// Returns an Error if `slot_duration_seconds == 0`. + fn new(genesis_slot: Slot, genesis_seconds: u64, slot_duration_seconds: u64) -> Self; + fn present_slot(&self) -> Result, Self::Error>; fn duration_to_next_slot(&self) -> Result, Self::Error>; diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 4dfc6b37da..7c184b02bf 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -18,31 +18,25 @@ pub struct SystemTimeSlotClock { slot_duration_seconds: u64, } -impl SystemTimeSlotClock { - /// Create a new `SystemTimeSlotClock`. - /// - /// Returns an Error if `slot_duration_seconds == 0`. - pub fn new( - genesis_slot: Slot, - genesis_seconds: u64, - slot_duration_seconds: u64, - ) -> Result { - if slot_duration_seconds == 0 { - Err(Error::SlotDurationIsZero) - } else { - Ok(Self { - genesis_slot, - genesis_seconds, - slot_duration_seconds, - }) - } - } -} - impl SlotClock for SystemTimeSlotClock { type Error = Error; + /// Create a new `SystemTimeSlotClock`. + /// + /// Returns an Error if `slot_duration_seconds == 0`. + fn new(genesis_slot: Slot, genesis_seconds: u64, slot_duration_seconds: u64) -> Self { + Self { + genesis_slot, + genesis_seconds, + slot_duration_seconds, + } + } + fn present_slot(&self) -> Result, Error> { + if self.slot_duration_seconds == 0 { + return Err(Error::SlotDurationIsZero); + } + let syslot_time = SystemTime::now(); let duration_since_epoch = syslot_time.duration_since(SystemTime::UNIX_EPOCH)?; let duration_since_genesis = diff --git a/eth2/utils/slot_clock/src/testing_slot_clock.rs b/eth2/utils/slot_clock/src/testing_slot_clock.rs index b5c36dfa0a..fc9b7201bb 100644 --- a/eth2/utils/slot_clock/src/testing_slot_clock.rs +++ b/eth2/utils/slot_clock/src/testing_slot_clock.rs @@ -8,30 +8,28 @@ pub enum Error {} /// Determines the present slot based upon the present system time. pub struct TestingSlotClock { - slot: RwLock, + slot: RwLock, } impl TestingSlotClock { - /// Create a new `TestingSlotClock`. - /// - /// Returns an Error if `slot_duration_seconds == 0`. - pub fn new(slot: u64) -> TestingSlotClock { - TestingSlotClock { - slot: RwLock::new(slot), - } - } - pub fn set_slot(&self, slot: u64) { - *self.slot.write().expect("TestingSlotClock poisoned.") = slot; + *self.slot.write().expect("TestingSlotClock poisoned.") = Slot::from(slot); } } impl SlotClock for TestingSlotClock { type Error = Error; + /// Create a new `TestingSlotClock` at `genesis_slot`. + fn new(genesis_slot: Slot, _genesis_seconds: u64, _slot_duration_seconds: u64) -> Self { + TestingSlotClock { + slot: RwLock::new(genesis_slot), + } + } + fn present_slot(&self) -> Result, Error> { let slot = *self.slot.read().expect("TestingSlotClock poisoned."); - Ok(Some(Slot::new(slot))) + Ok(Some(slot)) } /// Always returns a duration of 1 second. @@ -46,7 +44,9 @@ mod tests { #[test] fn test_slot_now() { - let clock = TestingSlotClock::new(10); + let null = 0; + + let clock = TestingSlotClock::new(Slot::new(10), null, null); assert_eq!(clock.present_slot(), Ok(Some(Slot::new(10)))); clock.set_slot(123); assert_eq!(clock.present_slot(), Ok(Some(Slot::new(123)))); diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index a340f99fc2..033394a0d3 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -155,8 +155,7 @@ impl Service { // build the validator slot clock let slot_clock = - SystemTimeSlotClock::new(genesis_slot, genesis_time, config.spec.seconds_per_slot) - .expect("Unable to instantiate SystemTimeSlotClock."); + SystemTimeSlotClock::new(genesis_slot, genesis_time, config.spec.seconds_per_slot); let current_slot = slot_clock .present_slot() From 9ed8a4d3806e3bb9ba778cdb8e00ec03ada65998 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 16:13:32 +1000 Subject: [PATCH 02/76] Implement basic `BeaconChain` persistence. --- beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 46 ++++++++++ beacon_node/beacon_chain/src/checkpoint.rs | 3 +- beacon_node/beacon_chain/src/lib.rs | 1 + .../src/persisted_beacon_chain.rs | 30 +++++++ beacon_node/client/src/beacon_chain_types.rs | 90 ++++++++----------- beacon_node/client/src/error.rs | 6 +- beacon_node/client/src/lib.rs | 7 +- beacon_node/eth2-libp2p/src/error.rs | 5 +- beacon_node/network/src/beacon_chain.rs | 4 +- beacon_node/network/src/error.rs | 5 +- beacon_node/rpc/src/beacon_chain.rs | 2 +- beacon_node/src/run.rs | 11 ++- 13 files changed, 137 insertions(+), 74 deletions(-) create mode 100644 beacon_node/beacon_chain/src/persisted_beacon_chain.rs diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 3a84256a7f..bf19a56a55 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -21,6 +21,7 @@ serde_derive = "1.0" serde_json = "1.0" slot_clock = { path = "../../eth2/utils/slot_clock" } ssz = { path = "../../eth2/utils/ssz" } +ssz_derive = { path = "../../eth2/utils/ssz_derive" } state_processing = { path = "../../eth2/state_processing" } tree_hash = { path = "../../eth2/utils/tree_hash" } types = { path = "../../eth2/types" } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a614698c45..bf807188f0 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,5 +1,6 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; +use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; use operation_pool::DepositInsertStatus; @@ -140,6 +141,51 @@ impl BeaconChain { }) } + /// Attempt to load an existing instance from the given `store`. + pub fn from_store(store: Arc) -> Result>, Error> { + let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes()); + let p: PersistedBeaconChain = match store.get(&key) { + Err(e) => return Err(e.into()), + Ok(None) => return Ok(None), + Ok(Some(p)) => p, + }; + + let spec = T::EthSpec::spec(); + + let slot_clock = T::SlotClock::new( + spec.genesis_slot, + p.state.genesis_time, + spec.seconds_per_slot, + ); + + let fork_choice = T::ForkChoice::new(store.clone()); + + Ok(Some(BeaconChain { + store, + slot_clock, + op_pool: OperationPool::default(), + canonical_head: RwLock::new(p.canonical_head), + finalized_head: RwLock::new(p.finalized_head), + state: RwLock::new(p.state), + spec, + fork_choice: RwLock::new(fork_choice), + })) + } + + /// Attempt to save this instance to `self.store`. + pub fn persist(&self) -> Result<(), Error> { + let p: PersistedBeaconChain = PersistedBeaconChain { + canonical_head: self.canonical_head.read().clone(), + finalized_head: self.finalized_head.read().clone(), + state: self.state.read().clone(), + }; + + let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes()); + self.store.put(&key, &p)?; + + Ok(()) + } + /// Returns the beacon block body for each beacon block root in `roots`. /// /// Fails if any root in `roots` does not have a corresponding block. diff --git a/beacon_node/beacon_chain/src/checkpoint.rs b/beacon_node/beacon_chain/src/checkpoint.rs index c069ac1042..c25e75a85d 100644 --- a/beacon_node/beacon_chain/src/checkpoint.rs +++ b/beacon_node/beacon_chain/src/checkpoint.rs @@ -1,9 +1,10 @@ use serde_derive::Serialize; +use ssz_derive::{Decode, Encode}; use types::{BeaconBlock, BeaconState, EthSpec, Hash256}; /// Represents some block and it's associated state. Generally, this will be used for tracking the /// head, justified head and finalized head. -#[derive(Clone, Serialize, PartialEq, Debug)] +#[derive(Clone, Serialize, PartialEq, Debug, Encode, Decode)] pub struct CheckPoint { pub beacon_block: BeaconBlock, pub beacon_block_root: Hash256, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 9f3058d0bc..0e3e01a4bd 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,6 +1,7 @@ mod beacon_chain; mod checkpoint; mod errors; +mod persisted_beacon_chain; pub use self::beacon_chain::{ BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, ValidBlock, diff --git a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs new file mode 100644 index 0000000000..cb34e995cd --- /dev/null +++ b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs @@ -0,0 +1,30 @@ +use crate::{BeaconChainTypes, CheckPoint}; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use store::{DBColumn, Error as StoreError, StoreItem}; +use types::BeaconState; + +/// 32-byte key for accessing the `PersistedBeaconChain`. +pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA"; + +#[derive(Encode, Decode)] +pub struct PersistedBeaconChain { + pub canonical_head: CheckPoint, + pub finalized_head: CheckPoint, + // TODO: operations pool. + pub state: BeaconState, +} + +impl StoreItem for PersistedBeaconChain { + fn db_column() -> DBColumn { + DBColumn::BeaconChain + } + + fn as_store_bytes(&self) -> Vec { + self.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &mut [u8]) -> Result { + Self::from_ssz_bytes(bytes).map_err(Into::into) + } +} diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 1971f56039..7ffb26b8b4 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -1,4 +1,3 @@ -use crate::ClientConfig; use beacon_chain::{ fork_choice::BitwiseLMDGhost, slot_clock::SystemTimeSlotClock, @@ -15,7 +14,7 @@ use types::{ /// Provides a new, initialized `BeaconChain` pub trait InitialiseBeaconChain { - fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain; + fn initialise_beacon_chain(store: Arc) -> BeaconChain; } /// A testnet-suitable BeaconChainType, using `MemoryStore`. @@ -29,16 +28,9 @@ impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { type EthSpec = FewValidatorsEthSpec; } -impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes -where - T: BeaconChainTypes< - Store = MemoryStore, - SlotClock = SystemTimeSlotClock, - ForkChoice = BitwiseLMDGhost, - >, -{ - fn initialise_beacon_chain(_config: &ClientConfig) -> BeaconChain { - initialize_chain::<_, _, FewValidatorsEthSpec>(MemoryStore::open()) +impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes { + fn initialise_beacon_chain(store: Arc) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store) } } @@ -53,55 +45,49 @@ impl BeaconChainTypes for TestnetDiskBeaconChainTypes { type EthSpec = FewValidatorsEthSpec; } -impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes -where - T: BeaconChainTypes< - Store = DiskStore, - SlotClock = SystemTimeSlotClock, - ForkChoice = BitwiseLMDGhost, - >, -{ - fn initialise_beacon_chain(config: &ClientConfig) -> BeaconChain { - let store = DiskStore::open(&config.db_name).expect("Unable to open DB."); - - initialize_chain::<_, _, FewValidatorsEthSpec>(store) +impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes { + fn initialise_beacon_chain(store: Arc) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store) } } -/// Produces a `BeaconChain` given some pre-initialized `Store`. -fn initialize_chain(store: U) -> BeaconChain +/// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. +fn maybe_load_from_store_for_testnet(store: Arc) -> BeaconChain where T: BeaconChainTypes, T::ForkChoice: ForkChoice, { - let spec = T::EthSpec::spec(); + if let Ok(Some(beacon_chain)) = BeaconChain::from_store(store.clone()) { + beacon_chain + } else { + let spec = T::EthSpec::spec(); - let store = Arc::new(store); + let state_builder = + TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); + let (genesis_state, _keypairs) = state_builder.build(); - let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); - let (genesis_state, _keypairs) = state_builder.build(); + let mut genesis_block = BeaconBlock::empty(&spec); + genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); - let mut genesis_block = BeaconBlock::empty(&spec); - genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); + // Slot clock + let slot_clock = T::SlotClock::new( + spec.genesis_slot, + genesis_state.genesis_time, + spec.seconds_per_slot, + ); + // Choose the fork choice + let fork_choice = T::ForkChoice::new(store.clone()); - // Slot clock - let slot_clock = T::SlotClock::new( - spec.genesis_slot, - genesis_state.genesis_time, - spec.seconds_per_slot, - ); - // Choose the fork choice - let fork_choice = T::ForkChoice::new(store.clone()); - - // Genesis chain - //TODO: Handle error correctly - BeaconChain::from_genesis( - store, - slot_clock, - genesis_state, - genesis_block, - spec.clone(), - fork_choice, - ) - .expect("Terminate if beacon chain generation fails") + // Genesis chain + //TODO: Handle error correctly + BeaconChain::from_genesis( + store, + slot_clock, + genesis_state, + genesis_block, + spec.clone(), + fork_choice, + ) + .expect("Terminate if beacon chain generation fails") + } } diff --git a/beacon_node/client/src/error.rs b/beacon_node/client/src/error.rs index 618813826b..b0272400ce 100644 --- a/beacon_node/client/src/error.rs +++ b/beacon_node/client/src/error.rs @@ -1,10 +1,6 @@ -// generates error types use network; -use error_chain::{ - error_chain, error_chain_processing, impl_error_chain_kind, impl_error_chain_processed, - impl_extract_backtrace, -}; +use error_chain::error_chain; error_chain! { links { diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 40be9b7b8e..df9eb86469 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -50,11 +50,14 @@ where /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( config: ClientConfig, + store: T::Store, log: slog::Logger, executor: &TaskExecutor, ) -> error::Result { - // generate a beacon chain - let beacon_chain = Arc::new(T::initialise_beacon_chain(&config)); + let store = Arc::new(store); + + // Load a `BeaconChain` from the store, or create a new one if it does not exist. + let beacon_chain = Arc::new(T::initialise_beacon_chain(store)); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") diff --git a/beacon_node/eth2-libp2p/src/error.rs b/beacon_node/eth2-libp2p/src/error.rs index 163fe575d2..a291e8fec5 100644 --- a/beacon_node/eth2-libp2p/src/error.rs +++ b/beacon_node/eth2-libp2p/src/error.rs @@ -1,8 +1,5 @@ // generates error types -use error_chain::{ - error_chain, error_chain_processing, impl_error_chain_kind, impl_error_chain_processed, - impl_extract_backtrace, -}; +use error_chain::error_chain; error_chain! {} diff --git a/beacon_node/network/src/beacon_chain.rs b/beacon_node/network/src/beacon_chain.rs index 6324e3a940..e38acbb728 100644 --- a/beacon_node/network/src/beacon_chain.rs +++ b/beacon_node/network/src/beacon_chain.rs @@ -5,9 +5,7 @@ use beacon_chain::{ AttestationValidationError, CheckPoint, }; use eth2_libp2p::rpc::HelloMessage; -use types::{ - Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot, -}; +use types::{Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, Hash256, Slot}; pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; diff --git a/beacon_node/network/src/error.rs b/beacon_node/network/src/error.rs index cdd6b62094..fc061ff442 100644 --- a/beacon_node/network/src/error.rs +++ b/beacon_node/network/src/error.rs @@ -1,10 +1,7 @@ // generates error types use eth2_libp2p; -use error_chain::{ - error_chain, error_chain_processing, impl_error_chain_kind, impl_error_chain_processed, - impl_extract_backtrace, -}; +use error_chain::error_chain; error_chain! { links { diff --git a/beacon_node/rpc/src/beacon_chain.rs b/beacon_node/rpc/src/beacon_chain.rs index b0a490137c..a37c219f67 100644 --- a/beacon_node/rpc/src/beacon_chain.rs +++ b/beacon_node/rpc/src/beacon_chain.rs @@ -5,7 +5,7 @@ use beacon_chain::{ AttestationValidationError, BlockProductionError, }; pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome}; -use types::{Attestation, AttestationData, BeaconBlock, EthSpec}; +use types::{Attestation, AttestationData, BeaconBlock}; /// The RPC's API to the beacon chain. pub trait BeaconChain: Send + Sync { diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 6ec65a92d4..d8ff202bf2 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -6,6 +6,7 @@ use futures::sync::oneshot; use futures::Future; use slog::info; use std::cell::RefCell; +use store::{DiskStore, MemoryStore}; use tokio::runtime::Builder; use tokio::runtime::Runtime; use tokio::runtime::TaskExecutor; @@ -32,8 +33,11 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul "BeaconNode starting"; "type" => "TestnetDiskBeaconChainTypes" ); + + let store = DiskStore::open(&config.db_name).expect("Unable to open DB."); + let client: Client = - Client::new(config, log.clone(), &executor)?; + Client::new(config, store, log.clone(), &executor)?; run(client, executor, runtime, log) } @@ -43,8 +47,11 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul "BeaconNode starting"; "type" => "TestnetMemoryBeaconChainTypes" ); + + let store = MemoryStore::open(); + let client: Client = - Client::new(config, log.clone(), &executor)?; + Client::new(config, store, log.clone(), &executor)?; run(client, executor, runtime, log) } From faa682a9b585b97cff61ad3693e2f736488a7115 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 16:32:46 +1000 Subject: [PATCH 03/76] Implement saving `BeaconChain` on client drop --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 ++++ beacon_node/client/src/beacon_chain_types.rs | 24 +++++++++++++++----- beacon_node/client/src/lib.rs | 14 +++++++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index bf807188f0..ca089789d5 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -336,6 +336,11 @@ impl BeaconChain { self.canonical_head.read() } + /// Returns the slot of the highest block in the canonical chain. + pub fn best_slot(&self) -> Slot { + self.canonical_head.read().beacon_block.slot + } + /// Updates the canonical `BeaconState` with the supplied state. /// /// Advances the chain forward to the present slot. This method is better than just setting diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 7ffb26b8b4..f923042995 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -5,6 +5,7 @@ use beacon_chain::{ BeaconChain, BeaconChainTypes, }; use fork_choice::ForkChoice; +use slog::{info, Logger}; use slot_clock::SlotClock; use std::sync::Arc; use tree_hash::TreeHash; @@ -14,7 +15,7 @@ use types::{ /// Provides a new, initialized `BeaconChain` pub trait InitialiseBeaconChain { - fn initialise_beacon_chain(store: Arc) -> BeaconChain; + fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain; } /// A testnet-suitable BeaconChainType, using `MemoryStore`. @@ -29,8 +30,8 @@ impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { } impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes { - fn initialise_beacon_chain(store: Arc) -> BeaconChain { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store) + fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) } } @@ -46,20 +47,31 @@ impl BeaconChainTypes for TestnetDiskBeaconChainTypes { } impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes { - fn initialise_beacon_chain(store: Arc) -> BeaconChain { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store) + fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) } } /// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. -fn maybe_load_from_store_for_testnet(store: Arc) -> BeaconChain +fn maybe_load_from_store_for_testnet( + store: Arc, + log: Logger, +) -> BeaconChain where T: BeaconChainTypes, T::ForkChoice: ForkChoice, { if let Ok(Some(beacon_chain)) = BeaconChain::from_store(store.clone()) { + info!( + log, + "Loaded BeaconChain from store"; + "slot" => beacon_chain.state.read().slot, + "best_slot" => beacon_chain.best_slot(), + ); + beacon_chain } else { + info!(log, "Initializing new BeaconChain from genesis"); let spec = T::EthSpec::spec(); let state_builder = diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index df9eb86469..734de47275 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -28,7 +28,7 @@ pub struct Client { /// Configuration for the lighthouse client. _config: ClientConfig, /// The beacon chain for the running client. - _beacon_chain: Arc>, + beacon_chain: Arc>, /// Reference to the network service. pub network: Arc>, /// Signal to terminate the RPC server. @@ -57,7 +57,7 @@ where let store = Arc::new(store); // Load a `BeaconChain` from the store, or create a new one if it does not exist. - let beacon_chain = Arc::new(T::initialise_beacon_chain(store)); + let beacon_chain = Arc::new(T::initialise_beacon_chain(store, log.clone())); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") @@ -151,7 +151,7 @@ where Ok(Client { _config: config, - _beacon_chain: beacon_chain, + beacon_chain, http_exit_signal, rpc_exit_signal, slot_timer_exit_signal: Some(slot_timer_exit_signal), @@ -162,6 +162,14 @@ where } } +impl Drop for Client { + fn drop(&mut self) { + // Save the beacon chain to it's store before dropping. + let _result = self.beacon_chain.persist(); + dbg!("Saved BeaconChain to store"); + } +} + fn do_state_catchup(chain: &Arc>, log: &slog::Logger) { if let Some(genesis_height) = chain.slots_since_genesis() { let result = chain.catchup_state(); From b28fa3d20b7d826e23edd8da570601fe3b940f7e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 17:22:27 +1000 Subject: [PATCH 04/76] Save the `BeaconChain` to store on state update --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index ca089789d5..eb79da1f53 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -347,6 +347,9 @@ impl BeaconChain { /// state and calling `catchup_state` as it will not result in an old state being installed and /// then having it iteratively updated -- in such a case it's possible for another thread to /// find the state at an old slot. + /// + /// Also persists the `BeaconChain` to the store, in the case the client does not exit + /// gracefully. pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, @@ -362,6 +365,8 @@ impl BeaconChain { *self.state.write() = state; + self.persist()?; + Ok(()) } From 9e6503c3260766a585ecad37fcf12c2aa2839d88 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 27 May 2019 17:54:32 +1000 Subject: [PATCH 05/76] Fix `fork_choice` tests --- eth2/fork_choice/tests/tests.rs | 60 +++++++++------------------------ 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 0327e8cb3f..7b23329b1e 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -1,14 +1,11 @@ #![cfg(not(debug_assertions))] -// Tests the available fork-choice algorithms - +/// Tests the available fork-choice algorithms pub use beacon_chain::BeaconChain; use bls::Signature; use store::MemoryStore; use store::Store; // use env_logger::{Builder, Env}; -use fork_choice::{ - BitwiseLMDGhost, ForkChoice, ForkChoiceAlgorithm, LongestChain, OptimizedLMDGhost, SlowLMDGhost, -}; +use fork_choice::{BitwiseLMDGhost, ForkChoice, LongestChain, OptimizedLMDGhost, SlowLMDGhost}; use std::collections::HashMap; use std::sync::Arc; use std::{fs::File, io::prelude::*, path::PathBuf}; @@ -25,8 +22,7 @@ fn test_optimized_lmd_ghost() { // set up logging // Builder::from_env(Env::default().default_filter_or("trace")).init(); - test_yaml_vectors( - ForkChoiceAlgorithm::OptimizedLMDGhost, + test_yaml_vectors::>( "tests/lmd_ghost_test_vectors.yaml", 100, ); @@ -37,8 +33,7 @@ fn test_bitwise_lmd_ghost() { // set up logging //Builder::from_env(Env::default().default_filter_or("trace")).init(); - test_yaml_vectors( - ForkChoiceAlgorithm::BitwiseLMDGhost, + test_yaml_vectors::>( "tests/bitwise_lmd_ghost_test_vectors.yaml", 100, ); @@ -46,8 +41,7 @@ fn test_bitwise_lmd_ghost() { #[test] fn test_slow_lmd_ghost() { - test_yaml_vectors( - ForkChoiceAlgorithm::SlowLMDGhost, + test_yaml_vectors::>( "tests/lmd_ghost_test_vectors.yaml", 100, ); @@ -55,16 +49,11 @@ fn test_slow_lmd_ghost() { #[test] fn test_longest_chain() { - test_yaml_vectors( - ForkChoiceAlgorithm::LongestChain, - "tests/longest_chain_test_vectors.yaml", - 100, - ); + test_yaml_vectors::>("tests/longest_chain_test_vectors.yaml", 100); } // run a generic test over given YAML test vectors -fn test_yaml_vectors( - fork_choice_algo: ForkChoiceAlgorithm, +fn test_yaml_vectors>( yaml_file_path: &str, emulated_validators: usize, // the number of validators used to give weights. ) { @@ -94,8 +83,7 @@ fn test_yaml_vectors( // process the tests for test_case in test_cases { // setup a fresh test - let (mut fork_choice, store, state_root) = - setup_inital_state(&fork_choice_algo, emulated_validators); + let (mut fork_choice, store, state_root) = setup_inital_state::(emulated_validators); // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) //let mut block_id_map: HashMap = HashMap::new(); @@ -204,32 +192,16 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { doc["test_cases"].as_vec().unwrap().clone() } -// initialise a single validator and state. All blocks will reference this state root. -fn setup_inital_state( - fork_choice_algo: &ForkChoiceAlgorithm, - num_validators: usize, -) -> (Box, Arc, Hash256) { +fn setup_inital_state( + // fork_choice_algo: &ForkChoiceAlgorithm, + num_validators: usize +) -> (T, Arc, Hash256) +where + T: ForkChoice, +{ let store = Arc::new(MemoryStore::open()); - // the fork choice instantiation - let fork_choice: Box = match fork_choice_algo { - ForkChoiceAlgorithm::OptimizedLMDGhost => { - let f: OptimizedLMDGhost = - OptimizedLMDGhost::new(store.clone()); - Box::new(f) - } - ForkChoiceAlgorithm::BitwiseLMDGhost => { - let f: BitwiseLMDGhost = - BitwiseLMDGhost::new(store.clone()); - Box::new(f) - } - ForkChoiceAlgorithm::SlowLMDGhost => { - let f: SlowLMDGhost = SlowLMDGhost::new(store.clone()); - Box::new(f) - } - ForkChoiceAlgorithm::LongestChain => Box::new(LongestChain::new(store.clone())), - }; - + let fork_choice = ForkChoice::new(store.clone()); let spec = FoundationEthSpec::spec(); let mut state_builder: TestingBeaconStateBuilder = From d72400cc9d9691b9558694cee74c6dd807b1674e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 10:30:20 +1000 Subject: [PATCH 06/76] Run rustfmt --- eth2/fork_choice/tests/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 7b23329b1e..2063ccc672 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -194,7 +194,7 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { fn setup_inital_state( // fork_choice_algo: &ForkChoiceAlgorithm, - num_validators: usize + num_validators: usize, ) -> (T, Arc, Hash256) where T: ForkChoice, From 6e5e1721f7445c34d8ca2c4050622ccf7825324f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 10:46:01 +1000 Subject: [PATCH 07/76] Fix lints introduced in Rust 1.35 --- beacon_node/client/src/client_config.rs | 2 +- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 7 ++++--- eth2/fork_choice/src/optimized_lmd_ghost.rs | 7 ++++--- validator_client/src/error.rs | 5 +---- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 243848e9f0..d8d3f2c4aa 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -98,7 +98,7 @@ impl ClientConfig { // Custom bootnodes if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - let mut boot_addresses_split = boot_addresses_str.split(","); + let boot_addresses_split = boot_addresses_str.split(","); for boot_address in boot_addresses_split { if let Ok(boot_address) = boot_address.parse::() { config.net_conf.boot_nodes.append(&mut vec![boot_address]); diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index a26a94b8ac..129dca9853 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -124,8 +124,9 @@ impl BitwiseLMDGhost { [log2_int((block_height - target_height - 1u64).as_u64()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(*ancestor_lookup, target_height, &spec) + .expect("All blocks should be added to the ancestor log lookup table") + .clone(); + self.get_ancestor(ancestor_lookup, target_height, &spec) } { // add the result to the cache self.cache.insert(cache_key, ancestor); @@ -151,7 +152,7 @@ impl BitwiseLMDGhost { // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0); + let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0).clone(); current_votes.insert(ancestor, current_vote_value + *votes); total_vote_count += votes; } diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index 2c1063a2bc..351c4decd4 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -124,8 +124,9 @@ impl OptimizedLMDGhost { [log2_int((block_height - target_height - 1u64).as_u64()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(*ancestor_lookup, target_height, &spec) + .expect("All blocks should be added to the ancestor log lookup table") + .clone(); + self.get_ancestor(ancestor_lookup, target_height, &spec) } { // add the result to the cache self.cache.insert(cache_key, ancestor); @@ -151,7 +152,7 @@ impl OptimizedLMDGhost { // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0); + let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0).clone(); current_votes.insert(ancestor, current_vote_value + *votes); total_vote_count += votes; } diff --git a/validator_client/src/error.rs b/validator_client/src/error.rs index 29d7ba8829..97500f900b 100644 --- a/validator_client/src/error.rs +++ b/validator_client/src/error.rs @@ -1,9 +1,6 @@ use slot_clock; -use error_chain::{ - error_chain, error_chain_processing, impl_error_chain_kind, impl_error_chain_processed, - impl_extract_backtrace, -}; +use error_chain::error_chain; error_chain! { links { } From 21ecaddac1c15a75d4172335cf1e0bac26761075 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 10:56:05 +1000 Subject: [PATCH 08/76] Fix various clippy lints --- beacon_node/client/src/client_config.rs | 2 +- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 7 +++---- eth2/fork_choice/src/optimized_lmd_ghost.rs | 7 +++---- eth2/utils/fixed_len_vec/src/impls.rs | 2 +- eth2/utils/ssz/src/decode.rs | 4 +--- eth2/utils/ssz/src/decode/impls.rs | 10 ++++------ 6 files changed, 13 insertions(+), 19 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index d8d3f2c4aa..15512342c7 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -98,7 +98,7 @@ impl ClientConfig { // Custom bootnodes if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - let boot_addresses_split = boot_addresses_str.split(","); + let boot_addresses_split = boot_addresses_str.split(','); for boot_address in boot_addresses_split { if let Ok(boot_address) = boot_address.parse::() { config.net_conf.boot_nodes.append(&mut vec![boot_address]); diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index 129dca9853..2d1b4e508a 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -120,12 +120,11 @@ impl BitwiseLMDGhost { // not in the cache recursively search for ancestors using a log-lookup if let Some(ancestor) = { - let ancestor_lookup = self.ancestors + let ancestor_lookup = *self.ancestors [log2_int((block_height - target_height - 1u64).as_u64()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table") - .clone(); + .expect("All blocks should be added to the ancestor log lookup table"); self.get_ancestor(ancestor_lookup, target_height, &spec) } { // add the result to the cache @@ -152,7 +151,7 @@ impl BitwiseLMDGhost { // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0).clone(); + let current_vote_value = *current_votes.get(&ancestor).unwrap_or_else(|| &0); current_votes.insert(ancestor, current_vote_value + *votes); total_vote_count += votes; } diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index 351c4decd4..ada8ce9cb8 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -120,12 +120,11 @@ impl OptimizedLMDGhost { // not in the cache recursively search for ancestors using a log-lookup if let Some(ancestor) = { - let ancestor_lookup = self.ancestors + let ancestor_lookup = *self.ancestors [log2_int((block_height - target_height - 1u64).as_u64()) as usize] .get(&block_hash) //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table") - .clone(); + .expect("All blocks should be added to the ancestor log lookup table"); self.get_ancestor(ancestor_lookup, target_height, &spec) } { // add the result to the cache @@ -152,7 +151,7 @@ impl OptimizedLMDGhost { // these have already been weighted by balance for (hash, votes) in latest_votes.iter() { if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = current_votes.get(&ancestor).unwrap_or_else(|| &0).clone(); + let current_vote_value = *current_votes.get(&ancestor).unwrap_or_else(|| &0); current_votes.insert(ancestor, current_vote_value + *votes); total_vote_count += votes; } diff --git a/eth2/utils/fixed_len_vec/src/impls.rs b/eth2/utils/fixed_len_vec/src/impls.rs index e1c54c1f78..691c8ee89a 100644 --- a/eth2/utils/fixed_len_vec/src/impls.rs +++ b/eth2/utils/fixed_len_vec/src/impls.rs @@ -100,7 +100,7 @@ where } fn from_ssz_bytes(bytes: &[u8]) -> Result { - if bytes.len() == 0 { + if bytes.is_empty() { Ok(FixedLenVec::from(vec![])) } else if T::is_ssz_fixed_len() { bytes diff --git a/eth2/utils/ssz/src/decode.rs b/eth2/utils/ssz/src/decode.rs index 891104733c..6934f1708c 100644 --- a/eth2/utils/ssz/src/decode.rs +++ b/eth2/utils/ssz/src/decode.rs @@ -102,9 +102,7 @@ impl<'a> SszDecoderBuilder<'a> { .and_then(|o| Some(o.offset)) .unwrap_or_else(|| BYTES_PER_LENGTH_OFFSET); - if previous_offset > offset { - return Err(DecodeError::OutOfBoundsByte { i: offset }); - } else if offset > self.bytes.len() { + if (previous_offset > offset) || (offset > self.bytes.len()) { return Err(DecodeError::OutOfBoundsByte { i: offset }); } diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 8a5a36780e..213f19bf5d 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -54,11 +54,9 @@ impl Decode for bool { match bytes[0] { 0b0000_0000 => Ok(false), 0b0000_0001 => Ok(true), - _ => { - return Err(DecodeError::BytesInvalid( - format!("Out-of-range for boolean: {}", bytes[0]).to_string(), - )) - } + _ => Err(DecodeError::BytesInvalid( + format!("Out-of-range for boolean: {}", bytes[0]).to_string(), + )), } } } @@ -121,7 +119,7 @@ impl Decode for Vec { } fn from_ssz_bytes(bytes: &[u8]) -> Result { - if bytes.len() == 0 { + if bytes.is_empty() { Ok(vec![]) } else if T::is_ssz_fixed_len() { bytes From 2a04da8bf765604ad1e361c02c24bfbfcb339971 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 12:45:48 +1000 Subject: [PATCH 09/76] Switch HTTP listen port to 5052 This is a quick-fix to allow gRPC and HTTP to co-exist. In the future I think we should swap this back to 5051. --- beacon_node/http_server/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 486badaff2..f8f5b67410 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -21,7 +21,7 @@ impl Default for HttpServerConfig { fn default() -> Self { Self { enabled: false, - listen_address: "127.0.0.1:5051".to_string(), + listen_address: "127.0.0.1:5052".to_string(), } } } From 3f27fd4edfa2954055f12aba337e9f5b6b3d47ea Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 13:50:51 +1000 Subject: [PATCH 10/76] Parse http CLI args for HTTP server --- beacon_node/client/src/client_config.rs | 13 +++++++++++++ beacon_node/client/src/lib.rs | 18 +++++++++++------- beacon_node/http_server/src/lib.rs | 14 +++++++------- beacon_node/src/main.rs | 22 ++++++++++++++++++++++ 4 files changed, 53 insertions(+), 14 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 15512342c7..a34b83253d 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -140,6 +140,19 @@ impl ClientConfig { } } + /* HTTP related arguments */ + + if args.is_present("http") { + config.http_conf.enabled = true; + } + + if let Some(listen_address) = args.value_of("http-address") { + config.http_conf.listen_address = listen_address.to_string(); + } + if let Some(listen_port) = args.value_of("http-port") { + config.http_conf.listen_port = listen_port.to_string(); + } + match args.value_of("db") { Some("disk") => config.db_type = DBType::Disk, Some("memory") => config.db_type = DBType::Memory, diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 734de47275..9bcae66ef7 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -115,13 +115,17 @@ where // Start the `http_server` service. // // Note: presently we are ignoring the config and _always_ starting a HTTP server. - let http_exit_signal = Some(http_server::start_service( - &config.http_conf, - executor, - network_send, - beacon_chain.clone(), - &log, - )); + let http_exit_signal = if config.http_conf.enabled { + Some(http_server::start_service( + &config.http_conf, + executor, + network_send, + beacon_chain.clone(), + &log, + )) + } else { + None + }; let (slot_timer_exit_signal, exit) = exit_future::signal(); if let Ok(Some(duration_to_next_slot)) = beacon_chain.slot_clock.duration_to_next_slot() { diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index f8f5b67410..02629f725c 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -15,13 +15,15 @@ use tokio::runtime::TaskExecutor; pub struct HttpServerConfig { pub enabled: bool, pub listen_address: String, + pub listen_port: String, } impl Default for HttpServerConfig { fn default() -> Self { Self { enabled: false, - listen_address: "127.0.0.1:5052".to_string(), + listen_address: "127.0.0.1".to_string(), + listen_port: "5052".to_string(), } } } @@ -69,16 +71,14 @@ pub fn start_service( // 2. Build an exit future that will shutdown the server when requested. // 3. Return the exit future, so the caller may shutdown the service when desired. let http_service = { + let listen_address = format!("{}:{}", config.listen_address, config.listen_port); // Start the HTTP server - let server_start_result = iron.http(config.listen_address.clone()); + let server_start_result = iron.http(listen_address.clone()); if server_start_result.is_ok() { - info!(log, "HTTP server running on {}", config.listen_address); + info!(log, "HTTP server running on {}", listen_address); } else { - warn!( - log, - "HTTP server failed to start on {}", config.listen_address - ); + warn!(log, "HTTP server failed to start on {}", listen_address); } // Build a future that will shutdown the HTTP server when the `shutdown_trigger` is diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index ef21218827..65f1899a01 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -68,6 +68,28 @@ fn main() { .help("Listen port for RPC endpoint.") .takes_value(true), ) + // HTTP related arguments + .arg( + Arg::with_name("http") + .long("http") + .value_name("HTTP") + .help("Enable the HTTP server.") + .takes_value(false), + ) + .arg( + Arg::with_name("http-address") + .long("http-address") + .value_name("HTTPADDRESS") + .help("Listen address for the HTTP server.") + .takes_value(true), + ) + .arg( + Arg::with_name("http-port") + .long("http-port") + .value_name("HTTPPORT") + .help("Listen port for the HTTP server.") + .takes_value(true), + ) .arg( Arg::with_name("db") .long("db") From 5e435e782137334f32bfd53b37d4538277dda7de Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 14:32:32 +1000 Subject: [PATCH 11/76] Add feat for `account_manager` to gen key range --- account_manager/src/main.rs | 87 +++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 28 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index c30b5b103d..28fe6defdd 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -43,6 +43,14 @@ fn main() { .help("The index of the validator, for which the test key is generated") .takes_value(true) .required(true), + ) + .arg( + Arg::with_name("end validator index") + .long("end_index") + .short("j") + .value_name("end_index") + .help("If supplied along with `index`, generates a range of keys.") + .takes_value(true), ), ) .get_matches(); @@ -55,37 +63,60 @@ fn main() { "data_dir" => &config.data_dir.to_str()); match matches.subcommand() { - ("generate", Some(_gen_m)) => { - let keypair = Keypair::random(); - let key_path: PathBuf = config - .save_key(&keypair) - .expect("Unable to save newly generated private key."); - debug!( - log, - "Keypair generated {:?}, saved to: {:?}", - keypair.identifier(), - key_path.to_string_lossy() - ); - } - ("generate_deterministic", Some(gen_d_matches)) => { - let validator_index = gen_d_matches - .value_of("validator index") - .expect("Validator index required.") - .parse::() - .expect("Invalid validator index.") as usize; - let keypair = generate_deterministic_keypair(validator_index); - let key_path: PathBuf = config - .save_key(&keypair) - .expect("Unable to save newly generated deterministic private key."); - debug!( - log, - "Deterministic Keypair generated {:?}, saved to: {:?}", - keypair.identifier(), - key_path.to_string_lossy() - ); + ("generate", Some(_)) => generate_random(&config, &log), + ("generate_deterministic", Some(m)) => { + if let Some(string) = m.value_of("validator index") { + let i: usize = string.parse().expect("Invalid validator index"); + if let Some(string) = m.value_of("end validator index") { + let j: usize = string.parse().expect("Invalid end validator index"); + + let indices: Vec = (i..j).collect(); + generate_deterministic_multiple(&indices, &config, &log) + } else { + generate_deterministic(i, &config, &log) + } + } } _ => panic!( "The account manager must be run with a subcommand. See help for more information." ), } } + +fn generate_random(config: &ValidatorClientConfig, log: &slog::Logger) { + save_key(&Keypair::random(), config, log) +} + +fn generate_deterministic_multiple( + validator_indices: &[usize], + config: &ValidatorClientConfig, + log: &slog::Logger, +) { + for validator_index in validator_indices { + generate_deterministic(*validator_index, config, log) + } +} + +fn generate_deterministic( + validator_index: usize, + config: &ValidatorClientConfig, + log: &slog::Logger, +) { + save_key( + &generate_deterministic_keypair(validator_index), + config, + log, + ) +} + +fn save_key(keypair: &Keypair, config: &ValidatorClientConfig, log: &slog::Logger) { + let key_path: PathBuf = config + .save_key(&keypair) + .expect("Unable to save newly generated private key."); + debug!( + log, + "Keypair generated {:?}, saved to: {:?}", + keypair.identifier(), + key_path.to_string_lossy() + ); +} From cb11656e12b9e835f26c78ee4cd4a887aa5af8e1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 14:33:13 +1000 Subject: [PATCH 12/76] Use `LighthouseTestnet` params, not `FewValidators` --- beacon_node/client/src/beacon_chain_types.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index f923042995..968630069c 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -10,7 +10,7 @@ use slot_clock::SlotClock; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, FewValidatorsEthSpec, Hash256, + test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, Hash256, LighthouseTestnetEthSpec, }; /// Provides a new, initialized `BeaconChain` @@ -26,7 +26,7 @@ impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { type Store = MemoryStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; - type EthSpec = FewValidatorsEthSpec; + type EthSpec = LighthouseTestnetEthSpec; } impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes { @@ -43,7 +43,7 @@ impl BeaconChainTypes for TestnetDiskBeaconChainTypes { type Store = DiskStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = BitwiseLMDGhost; - type EthSpec = FewValidatorsEthSpec; + type EthSpec = LighthouseTestnetEthSpec; } impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes { From 3e73a008eb5e0ece61a8d14157eabed85752174e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 16:10:52 +1000 Subject: [PATCH 13/76] Add constant for testnet validator count --- beacon_node/client/src/beacon_chain_types.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 968630069c..8990e842db 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -13,6 +13,9 @@ use types::{ test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, Hash256, LighthouseTestnetEthSpec, }; +/// The number initial validators when starting the `LighthouseTestnet`. +const TESTNET_VALIDATOR_COUNT: usize = 16; + /// Provides a new, initialized `BeaconChain` pub trait InitialiseBeaconChain { fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain; @@ -74,8 +77,10 @@ where info!(log, "Initializing new BeaconChain from genesis"); let spec = T::EthSpec::spec(); - let state_builder = - TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(8, &spec); + let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists( + TESTNET_VALIDATOR_COUNT, + &spec, + ); let (genesis_state, _keypairs) = state_builder.build(); let mut genesis_block = BeaconBlock::empty(&spec); From e756a0aaa400ef9d96fcd0b4649088b7c479526b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 16:11:16 +1000 Subject: [PATCH 14/76] Add extra metrics for prom --- beacon_node/http_server/src/metrics.rs | 31 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index eb7816d0eb..366d642b5d 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -29,16 +29,22 @@ fn handle_metrics(req: &mut Request) -> IronResul let r = Registry::new(); - let present_slot = if let Ok(Some(slot)) = beacon_chain.slot_clock.present_slot() { - slot - } else { - Slot::new(0) - }; - register_and_set_slot( + let present_slot = beacon_chain + .slot_clock + .present_slot() + .unwrap_or_else(|_| None) + .unwrap_or_else(|| Slot::new(0)); + register_and_set_slot(&r, "present_slot", "slock_clock_reading", present_slot); + + let best_slot = beacon_chain.head().beacon_block.slot; + register_and_set_slot(&r, "best_slot", "slot_of_block_at_head_of_chain", best_slot); + + let validator_count = beacon_chain.head().beacon_state.validator_registry.len(); + register_and_set( &r, - "present_slot", - "direct_slock_clock_reading", - present_slot, + "validator_count", + "total_number_of_validators", + validator_count as i64, ); // Gather the metrics. @@ -52,6 +58,13 @@ fn handle_metrics(req: &mut Request) -> IronResul Ok(Response::with((Status::Ok, prom_string))) } +fn register_and_set(registry: &Registry, name: &str, help: &str, value: i64) { + let counter_opts = Opts::new(name, help); + let counter = IntCounter::with_opts(counter_opts).unwrap(); + registry.register(Box::new(counter.clone())).unwrap(); + counter.inc_by(value); +} + fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) { let counter_opts = Opts::new(name, help); let counter = IntCounter::with_opts(counter_opts).unwrap(); From 345f7d5f18181e9c58d59ce8d5d60c20ee671251 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 17:30:09 +1000 Subject: [PATCH 15/76] Add global metrics registry, pass to `BeaconState` --- beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 10 ++++ beacon_node/beacon_chain/src/errors.rs | 8 +++ beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/beacon_chain/src/metrics.rs | 59 ++++++++++++++++++++ beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/lib.rs | 8 +++ beacon_node/http_server/src/key.rs | 7 +++ beacon_node/http_server/src/lib.rs | 7 ++- beacon_node/http_server/src/metrics.rs | 11 +++- 10 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 beacon_node/beacon_chain/src/metrics.rs diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index bf19a56a55..4f007cbb75 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -13,6 +13,7 @@ failure_derive = "0.1" hashing = { path = "../../eth2/utils/hashing" } fork_choice = { path = "../../eth2/fork_choice" } parking_lot = "0.7" +prometheus = "^0.6" log = "0.4" operation_pool = { path = "../../eth2/operation_pool" } env_logger = "0.6" diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index eb79da1f53..ce09095105 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,5 +1,6 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; +use crate::metrics::Metrics; use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; @@ -96,6 +97,7 @@ pub struct BeaconChain { pub state: RwLock>, pub spec: ChainSpec, pub fork_choice: RwLock, + pub metrics: Metrics, } impl BeaconChain { @@ -138,6 +140,7 @@ impl BeaconChain { canonical_head, spec, fork_choice: RwLock::new(fork_choice), + metrics: Metrics::new()?, }) } @@ -169,6 +172,7 @@ impl BeaconChain { state: RwLock::new(p.state), spec, fork_choice: RwLock::new(fork_choice), + metrics: Metrics::new()?, })) } @@ -621,6 +625,7 @@ impl BeaconChain { /// Will accept blocks from prior slots, however it will reject any block from a future slot. pub fn process_block(&self, block: BeaconBlock) -> Result { debug!("Processing block with slot {}...", block.slot); + self.metrics.blocks_processed.inc(); let block_root = block.block_header().canonical_root(); @@ -704,6 +709,8 @@ impl BeaconChain { self.update_state(state)?; } + self.metrics.valid_blocks_processed.inc(); + Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) } @@ -716,6 +723,7 @@ impl BeaconChain { randao_reveal: Signature, ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { debug!("Producing block at slot {}...", self.state.read().slot); + self.metrics.block_production_requests.inc(); let mut state = self.state.read().clone(); @@ -766,6 +774,8 @@ impl BeaconChain { block.state_root = state_root; + self.metrics.block_production_successes.inc(); + Ok((block, state)) } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 73884916aa..75f2fd84dd 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -1,3 +1,4 @@ +use crate::metrics::Error as MetricsError; use fork_choice::ForkChoiceError; use state_processing::BlockProcessingError; use state_processing::SlotProcessingError; @@ -25,10 +26,17 @@ pub enum BeaconChainError { MissingBeaconBlock(Hash256), MissingBeaconState(Hash256), SlotProcessingError(SlotProcessingError), + MetricsError(String), } easy_from_to!(SlotProcessingError, BeaconChainError); +impl From for BeaconChainError { + fn from(e: MetricsError) -> BeaconChainError { + BeaconChainError::MetricsError(format!("{:?}", e)) + } +} + #[derive(Debug, PartialEq)] pub enum BlockProductionError { UnableToGetBlockRootFromState, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 0e3e01a4bd..c80dc4715b 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,6 +1,7 @@ mod beacon_chain; mod checkpoint; mod errors; +mod metrics; mod persisted_beacon_chain; pub use self::beacon_chain::{ diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs new file mode 100644 index 0000000000..f73db5e13e --- /dev/null +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -0,0 +1,59 @@ +pub use prometheus::Error; +use prometheus::{IntCounter, Opts, Registry}; + +pub struct Metrics { + pub blocks_processed: IntCounter, + pub valid_blocks_processed: IntCounter, + pub block_production_requests: IntCounter, + pub block_production_successes: IntCounter, + pub attestation_production_requests: IntCounter, + pub attestation_production_successes: IntCounter, +} + +impl Metrics { + pub fn new() -> Result { + Ok(Self { + blocks_processed: { + let opts = Opts::new("blocks_processed", "total_blocks_processed"); + IntCounter::with_opts(opts)? + }, + valid_blocks_processed: { + let opts = Opts::new("valid_blocks_processed", "total_valid_blocks_processed"); + IntCounter::with_opts(opts)? + }, + block_production_requests: { + let opts = Opts::new("block_production_requests", "attempts_to_produce_new_block"); + IntCounter::with_opts(opts)? + }, + block_production_successes: { + let opts = Opts::new("block_production_successes", "blocks_successfully_produced"); + IntCounter::with_opts(opts)? + }, + attestation_production_requests: { + let opts = Opts::new( + "attestation_production_requests", + "total_attestation_production_requests", + ); + IntCounter::with_opts(opts)? + }, + attestation_production_successes: { + let opts = Opts::new( + "attestation_production_successes", + "total_attestation_production_successes", + ); + IntCounter::with_opts(opts)? + }, + }) + } + + pub fn register(&self, registry: &Registry) -> Result<(), Error> { + registry.register(Box::new(self.blocks_processed.clone()))?; + registry.register(Box::new(self.valid_blocks_processed.clone()))?; + registry.register(Box::new(self.block_production_requests.clone()))?; + registry.register(Box::new(self.block_production_successes.clone()))?; + registry.register(Box::new(self.attestation_production_requests.clone()))?; + registry.register(Box::new(self.attestation_production_successes.clone()))?; + + Ok(()) + } +} diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 387bf16757..afff86bccc 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -11,6 +11,7 @@ store = { path = "../store" } http_server = { path = "../http_server" } rpc = { path = "../rpc" } fork_choice = { path = "../../eth2/fork_choice" } +prometheus = "^0.6" types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } slot_clock = { path = "../../eth2/utils/slot_clock" } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 9bcae66ef7..b67cc6a0dc 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -10,6 +10,7 @@ use beacon_chain_types::InitialiseBeaconChain; use exit_future::Signal; use futures::{future::Future, Stream}; use network::Service as NetworkService; +use prometheus::Registry; use slog::{error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; @@ -54,10 +55,16 @@ where log: slog::Logger, executor: &TaskExecutor, ) -> error::Result { + let metrics_registry = Registry::new(); let store = Arc::new(store); // Load a `BeaconChain` from the store, or create a new one if it does not exist. let beacon_chain = Arc::new(T::initialise_beacon_chain(store, log.clone())); + // Registry all beacon chain metrics with the global registry. + beacon_chain + .metrics + .register(&metrics_registry) + .expect("Failed to registry metrics"); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") @@ -121,6 +128,7 @@ where executor, network_send, beacon_chain.clone(), + metrics_registry, &log, )) } else { diff --git a/beacon_node/http_server/src/key.rs b/beacon_node/http_server/src/key.rs index 2d27ce9f06..2cbe68cd0c 100644 --- a/beacon_node/http_server/src/key.rs +++ b/beacon_node/http_server/src/key.rs @@ -1,5 +1,6 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::typemap::Key; +use prometheus::Registry; use std::marker::PhantomData; use std::sync::Arc; @@ -10,3 +11,9 @@ pub struct BeaconChainKey { impl Key for BeaconChainKey { type Value = Arc>; } + +pub struct MetricsRegistryKey; + +impl Key for MetricsRegistryKey { + type Value = Registry; +} diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index 02629f725c..cc54b6e17b 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -6,6 +6,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use iron::prelude::*; use network::NetworkMessage; +use prometheus::Registry; use router::Router; use slog::{info, o, warn}; use std::sync::Arc; @@ -31,13 +32,14 @@ impl Default for HttpServerConfig { /// Build the `iron` HTTP server, defining the core routes. pub fn create_iron_http_server( beacon_chain: Arc>, + metrics_registry: Registry, ) -> Iron { let mut router = Router::new(); // A `GET` request to `/metrics` is handled by the `metrics` module. router.get( "/metrics", - metrics::build_handler(beacon_chain.clone()), + metrics::build_handler(beacon_chain.clone(), metrics_registry), "metrics", ); @@ -53,6 +55,7 @@ pub fn start_service( executor: &TaskExecutor, _network_chan: crossbeam_channel::Sender, beacon_chain: Arc>, + metrics_registry: Registry, log: &slog::Logger, ) -> exit_future::Signal { let log = log.new(o!("Service"=>"HTTP")); @@ -63,7 +66,7 @@ pub fn start_service( let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); // Create an `iron` http, without starting it yet. - let iron = create_iron_http_server(beacon_chain); + let iron = create_iron_http_server(beacon_chain, metrics_registry); // Create a HTTP server future. // diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 366d642b5d..608f38efdc 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -1,4 +1,7 @@ -use crate::{key::BeaconChainKey, map_persistent_err_to_500}; +use crate::{ + key::{BeaconChainKey, MetricsRegistryKey}, + map_persistent_err_to_500, +}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; @@ -11,10 +14,12 @@ use types::Slot; /// Yields a handler for the metrics endpoint. pub fn build_handler( beacon_chain: Arc>, + metrics_registry: Registry, ) -> impl Handler { let mut chain = Chain::new(handle_metrics::); chain.link(Read::>::both(beacon_chain)); + chain.link(Read::::both(metrics_registry)); chain } @@ -27,7 +32,9 @@ fn handle_metrics(req: &mut Request) -> IronResul .get::>>() .map_err(map_persistent_err_to_500)?; - let r = Registry::new(); + let r = req + .get::>() + .map_err(map_persistent_err_to_500)?; let present_slot = beacon_chain .slot_clock From 6d27c4366619001722b6bc878d37c2f3c3ee5b25 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 28 May 2019 18:05:52 +1000 Subject: [PATCH 16/76] Fix bug with metrics scraping --- beacon_node/http_server/src/key.rs | 7 +++ beacon_node/http_server/src/metrics.rs | 64 +++++++++++++++----------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/beacon_node/http_server/src/key.rs b/beacon_node/http_server/src/key.rs index 2cbe68cd0c..b84c5f6857 100644 --- a/beacon_node/http_server/src/key.rs +++ b/beacon_node/http_server/src/key.rs @@ -1,3 +1,4 @@ +use crate::metrics::LocalMetrics; use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::typemap::Key; use prometheus::Registry; @@ -17,3 +18,9 @@ pub struct MetricsRegistryKey; impl Key for MetricsRegistryKey { type Value = Registry; } + +pub struct LocalMetricsKey; + +impl Key for LocalMetricsKey { + type Value = LocalMetrics; +} diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 608f38efdc..4c243ce715 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -1,12 +1,12 @@ use crate::{ - key::{BeaconChainKey, MetricsRegistryKey}, + key::{BeaconChainKey, LocalMetricsKey, MetricsRegistryKey}, map_persistent_err_to_500, }; use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; use persistent::Read; -use prometheus::{Encoder, IntCounter, Opts, Registry, TextEncoder}; +use prometheus::{Encoder, IntGauge, Opts, Registry, TextEncoder}; use slot_clock::SlotClock; use std::sync::Arc; use types::Slot; @@ -18,12 +18,42 @@ pub fn build_handler( ) -> impl Handler { let mut chain = Chain::new(handle_metrics::); + let local_metrics = LocalMetrics::new().unwrap(); + local_metrics.register(&metrics_registry).unwrap(); + chain.link(Read::>::both(beacon_chain)); chain.link(Read::::both(metrics_registry)); + chain.link(Read::::both(local_metrics)); chain } +pub struct LocalMetrics { + present_slot: IntGauge, + validator_count: IntGauge, +} + +impl LocalMetrics { + pub fn new() -> Result { + Ok(Self { + present_slot: { + let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); + IntGauge::with_opts(opts)? + }, + validator_count: { + let opts = Opts::new("validator_count", "number_of_validators"); + IntGauge::with_opts(opts)? + }, + }) + } + + pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { + registry.register(Box::new(self.present_slot.clone()))?; + + Ok(()) + } +} + /// Handle a request for Prometheus metrics. /// /// Returns a text string containing all metrics. @@ -36,23 +66,19 @@ fn handle_metrics(req: &mut Request) -> IronResul .get::>() .map_err(map_persistent_err_to_500)?; + let local_metrics = req + .get::>() + .map_err(map_persistent_err_to_500)?; + let present_slot = beacon_chain .slot_clock .present_slot() .unwrap_or_else(|_| None) .unwrap_or_else(|| Slot::new(0)); - register_and_set_slot(&r, "present_slot", "slock_clock_reading", present_slot); - - let best_slot = beacon_chain.head().beacon_block.slot; - register_and_set_slot(&r, "best_slot", "slot_of_block_at_head_of_chain", best_slot); + local_metrics.present_slot.set(present_slot.as_u64() as i64); let validator_count = beacon_chain.head().beacon_state.validator_registry.len(); - register_and_set( - &r, - "validator_count", - "total_number_of_validators", - validator_count as i64, - ); + local_metrics.validator_count.set(validator_count as i64); // Gather the metrics. let mut buffer = vec![]; @@ -64,17 +90,3 @@ fn handle_metrics(req: &mut Request) -> IronResul Ok(Response::with((Status::Ok, prom_string))) } - -fn register_and_set(registry: &Registry, name: &str, help: &str, value: i64) { - let counter_opts = Opts::new(name, help); - let counter = IntCounter::with_opts(counter_opts).unwrap(); - registry.register(Box::new(counter.clone())).unwrap(); - counter.inc_by(value); -} - -fn register_and_set_slot(registry: &Registry, name: &str, help: &str, slot: Slot) { - let counter_opts = Opts::new(name, help); - let counter = IntCounter::with_opts(counter_opts).unwrap(); - registry.register(Box::new(counter.clone())).unwrap(); - counter.inc_by(slot.as_u64() as i64); -} From f89cb65360da05847e34ffcaedfcd47b4e9875b8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 13:55:17 +1000 Subject: [PATCH 17/76] Add `best_slot` metric --- beacon_node/http_server/src/metrics.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 4c243ce715..ed5eeb79ab 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -30,6 +30,7 @@ pub fn build_handler( pub struct LocalMetrics { present_slot: IntGauge, + best_slot: IntGauge, validator_count: IntGauge, } @@ -40,6 +41,10 @@ impl LocalMetrics { let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); IntGauge::with_opts(opts)? }, + best_slot: { + let opts = Opts::new("present_slot", "slot_of_block_at_chain_head"); + IntGauge::with_opts(opts)? + }, validator_count: { let opts = Opts::new("validator_count", "number_of_validators"); IntGauge::with_opts(opts)? @@ -49,6 +54,8 @@ impl LocalMetrics { pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { registry.register(Box::new(self.present_slot.clone()))?; + registry.register(Box::new(self.best_slot.clone()))?; + registry.register(Box::new(self.validator_count.clone()))?; Ok(()) } @@ -77,6 +84,9 @@ fn handle_metrics(req: &mut Request) -> IronResul .unwrap_or_else(|| Slot::new(0)); local_metrics.present_slot.set(present_slot.as_u64() as i64); + let best_slot = beacon_chain.head().beacon_block.slot; + local_metrics.best_slot.set(best_slot.as_u64() as i64); + let validator_count = beacon_chain.head().beacon_state.validator_registry.len(); local_metrics.validator_count.set(validator_count as i64); From 7c2ca85e34d7583a39d8874328f61d55bfc45855 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 14:06:38 +1000 Subject: [PATCH 18/76] Change account_manager CLI options --- account_manager/src/main.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index 28fe6defdd..aa691d31af 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -45,12 +45,13 @@ fn main() { .required(true), ) .arg( - Arg::with_name("end validator index") - .long("end_index") - .short("j") - .value_name("end_index") - .help("If supplied along with `index`, generates a range of keys.") - .takes_value(true), + Arg::with_name("validator count") + .long("validator_count") + .short("n") + .value_name("validator_count") + .help("If supplied along with `index`, generates keys `i..i + n`.") + .takes_value(true) + .default_value("1"), ), ) .get_matches(); @@ -67,10 +68,10 @@ fn main() { ("generate_deterministic", Some(m)) => { if let Some(string) = m.value_of("validator index") { let i: usize = string.parse().expect("Invalid validator index"); - if let Some(string) = m.value_of("end validator index") { - let j: usize = string.parse().expect("Invalid end validator index"); + if let Some(string) = m.value_of("validator count") { + let n: usize = string.parse().expect("Invalid end validator count"); - let indices: Vec = (i..j).collect(); + let indices: Vec = (i..i + n).collect(); generate_deterministic_multiple(&indices, &config, &log) } else { generate_deterministic(i, &config, &log) From a153f24bfad3fa76801e103353b65df726dfcb5e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 14:21:34 +1000 Subject: [PATCH 19/76] Fix duplicate metric label --- beacon_node/http_server/src/metrics.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index ed5eeb79ab..2c6cda0ead 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -42,7 +42,7 @@ impl LocalMetrics { IntGauge::with_opts(opts)? }, best_slot: { - let opts = Opts::new("present_slot", "slot_of_block_at_chain_head"); + let opts = Opts::new("best_slot", "slot_of_block_at_chain_head"); IntGauge::with_opts(opts)? }, validator_count: { From 64fbc6bf3c51ce0c3c4b3be6ebde8cf7909cebb6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 15:45:09 +1000 Subject: [PATCH 20/76] Add additional metrics to `BeaconChain` --- beacon_node/beacon_chain/src/beacon_chain.rs | 14 ++++++-- beacon_node/beacon_chain/src/metrics.rs | 37 ++++++++++++++++---- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index ce09095105..09bfebb9b8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -530,6 +530,9 @@ impl BeaconChain { /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { trace!("BeaconChain::produce_attestation: shard: {}", shard); + self.metrics.attestation_production_requests.inc(); + let timer = self.metrics.attestation_production_histogram.start_timer(); + let state = self.state.read(); let current_epoch_start_slot = self @@ -557,6 +560,9 @@ impl BeaconChain { *self.state.read().get_block_root(current_epoch_start_slot)? }; + self.metrics.attestation_production_successes.inc(); + timer.observe_duration(); + Ok(AttestationData { slot: self.state.read().slot, shard, @@ -625,7 +631,8 @@ impl BeaconChain { /// Will accept blocks from prior slots, however it will reject any block from a future slot. pub fn process_block(&self, block: BeaconBlock) -> Result { debug!("Processing block with slot {}...", block.slot); - self.metrics.blocks_processed.inc(); + self.metrics.block_processing_requests.inc(); + let timer = self.metrics.block_processing_historgram.start_timer(); let block_root = block.block_header().canonical_root(); @@ -709,7 +716,8 @@ impl BeaconChain { self.update_state(state)?; } - self.metrics.valid_blocks_processed.inc(); + self.metrics.block_processing_successes.inc(); + timer.observe_duration(); Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) } @@ -724,6 +732,7 @@ impl BeaconChain { ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { debug!("Producing block at slot {}...", self.state.read().slot); self.metrics.block_production_requests.inc(); + let timer = self.metrics.block_production_historgram.start_timer(); let mut state = self.state.read().clone(); @@ -775,6 +784,7 @@ impl BeaconChain { block.state_root = state_root; self.metrics.block_production_successes.inc(); + timer.observe_duration(); Ok((block, state)) } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index f73db5e13e..7c068119e2 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1,26 +1,34 @@ pub use prometheus::Error; -use prometheus::{IntCounter, Opts, Registry}; +use prometheus::{Histogram, HistogramOpts, IntCounter, Opts, Registry}; pub struct Metrics { - pub blocks_processed: IntCounter, - pub valid_blocks_processed: IntCounter, + pub block_processing_requests: IntCounter, + pub block_processing_successes: IntCounter, + pub block_processing_historgram: Histogram, pub block_production_requests: IntCounter, pub block_production_successes: IntCounter, + pub block_production_historgram: Histogram, pub attestation_production_requests: IntCounter, pub attestation_production_successes: IntCounter, + pub attestation_production_histogram: Histogram, } impl Metrics { pub fn new() -> Result { Ok(Self { - blocks_processed: { + block_processing_requests: { let opts = Opts::new("blocks_processed", "total_blocks_processed"); IntCounter::with_opts(opts)? }, - valid_blocks_processed: { + block_processing_successes: { let opts = Opts::new("valid_blocks_processed", "total_valid_blocks_processed"); IntCounter::with_opts(opts)? }, + block_processing_historgram: { + let opts = + HistogramOpts::new("block_processing_historgram", "block_processing_time"); + Histogram::with_opts(opts)? + }, block_production_requests: { let opts = Opts::new("block_production_requests", "attempts_to_produce_new_block"); IntCounter::with_opts(opts)? @@ -29,6 +37,11 @@ impl Metrics { let opts = Opts::new("block_production_successes", "blocks_successfully_produced"); IntCounter::with_opts(opts)? }, + block_production_historgram: { + let opts = + HistogramOpts::new("block_production_historgram", "block_production_time"); + Histogram::with_opts(opts)? + }, attestation_production_requests: { let opts = Opts::new( "attestation_production_requests", @@ -43,16 +56,26 @@ impl Metrics { ); IntCounter::with_opts(opts)? }, + attestation_production_histogram: { + let opts = HistogramOpts::new( + "attestation_production_histogram", + "attestation_production_time", + ); + Histogram::with_opts(opts)? + }, }) } pub fn register(&self, registry: &Registry) -> Result<(), Error> { - registry.register(Box::new(self.blocks_processed.clone()))?; - registry.register(Box::new(self.valid_blocks_processed.clone()))?; + registry.register(Box::new(self.block_processing_requests.clone()))?; + registry.register(Box::new(self.block_processing_successes.clone()))?; + registry.register(Box::new(self.block_processing_historgram.clone()))?; registry.register(Box::new(self.block_production_requests.clone()))?; registry.register(Box::new(self.block_production_successes.clone()))?; + registry.register(Box::new(self.block_production_historgram.clone()))?; registry.register(Box::new(self.attestation_production_requests.clone()))?; registry.register(Box::new(self.attestation_production_successes.clone()))?; + registry.register(Box::new(self.attestation_production_histogram.clone()))?; Ok(()) } From 9f1039a35007713832a9c7e447996cbcab7bd391 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 17:52:43 +1000 Subject: [PATCH 21/76] Add metrics to `BeaconChain`, tidy fork choice --- beacon_node/beacon_chain/src/beacon_chain.rs | 56 ++++++++++---- beacon_node/beacon_chain/src/metrics.rs | 80 ++++++++++++++++---- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 09bfebb9b8..a7421b922f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -530,8 +530,9 @@ impl BeaconChain { /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { trace!("BeaconChain::produce_attestation: shard: {}", shard); + self.metrics.attestation_production_requests.inc(); - let timer = self.metrics.attestation_production_histogram.start_timer(); + let timer = self.metrics.attestation_production_times.start_timer(); let state = self.state.read(); @@ -583,8 +584,20 @@ impl BeaconChain { &self, attestation: Attestation, ) -> Result<(), AttestationValidationError> { - self.op_pool - .insert_attestation(attestation, &*self.state.read(), &self.spec) + self.metrics.attestation_processing_requests.inc(); + let timer = self.metrics.attestation_processing_times.start_timer(); + + let result = self + .op_pool + .insert_attestation(attestation, &*self.state.read(), &self.spec); + + if result.is_ok() { + self.metrics.attestation_production_successes.inc(); + } + + timer.observe_duration(); + + result } /// Accept some deposit and queue it for inclusion in an appropriate block. @@ -632,7 +645,7 @@ impl BeaconChain { pub fn process_block(&self, block: BeaconBlock) -> Result { debug!("Processing block with slot {}...", block.slot); self.metrics.block_processing_requests.inc(); - let timer = self.metrics.block_processing_historgram.start_timer(); + let timer = self.metrics.block_processing_times.start_timer(); let block_root = block.block_header().canonical_root(); @@ -732,7 +745,7 @@ impl BeaconChain { ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { debug!("Producing block at slot {}...", self.state.read().slot); self.metrics.block_production_requests.inc(); - let timer = self.metrics.block_production_historgram.start_timer(); + let timer = self.metrics.block_production_times.start_timer(); let mut state = self.state.read().clone(); @@ -789,29 +802,40 @@ impl BeaconChain { Ok((block, state)) } - // TODO: Left this as is, modify later + /// Execute the fork choice algorithm and enthrone the result as the canonical head. pub fn fork_choice(&self) -> Result<(), Error> { - let present_head = self.finalized_head().beacon_block_root; + self.metrics.fork_choice_requests.inc(); - let new_head = self + let present_head_root = self.finalized_head().beacon_block_root; + + let timer = self.metrics.fork_choice_times.start_timer(); + + let new_head_root = self .fork_choice .write() - .find_head(&present_head, &self.spec)?; + .find_head(&present_head_root, &self.spec)?; + + timer.observe_duration(); + + if new_head_root != present_head_root { + self.metrics.fork_choice_changed_head.inc(); - if new_head != present_head { let block: BeaconBlock = self .store - .get(&new_head)? - .ok_or_else(|| Error::MissingBeaconBlock(new_head))?; - let block_root = block.canonical_root(); - + .get(&new_head_root)? + .ok_or_else(|| Error::MissingBeaconBlock(new_head_root))?; let state: BeaconState = self .store .get(&block.state_root)? .ok_or_else(|| Error::MissingBeaconState(block.state_root))?; - let state_root = state.canonical_root(); - self.update_canonical_head(block, block_root, state.clone(), state_root); + // Log if we switched to a new chain. + if present_head_root != block.previous_block_root { + self.metrics.fork_choice_reorg_count.inc(); + }; + + let state_root = block.state_root; + self.update_canonical_head(block, new_head_root, state.clone(), state_root); // Update the canonical `BeaconState`. self.update_state(state)?; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 7c068119e2..0fe738a8c3 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -4,13 +4,20 @@ use prometheus::{Histogram, HistogramOpts, IntCounter, Opts, Registry}; pub struct Metrics { pub block_processing_requests: IntCounter, pub block_processing_successes: IntCounter, - pub block_processing_historgram: Histogram, + pub block_processing_times: Histogram, pub block_production_requests: IntCounter, pub block_production_successes: IntCounter, - pub block_production_historgram: Histogram, + pub block_production_times: Histogram, pub attestation_production_requests: IntCounter, pub attestation_production_successes: IntCounter, - pub attestation_production_histogram: Histogram, + pub attestation_production_times: Histogram, + pub attestation_processing_requests: IntCounter, + pub attestation_processing_successes: IntCounter, + pub attestation_processing_times: Histogram, + pub fork_choice_requests: IntCounter, + pub fork_choice_changed_head: IntCounter, + pub fork_choice_reorg_count: IntCounter, + pub fork_choice_times: Histogram, } impl Metrics { @@ -24,9 +31,8 @@ impl Metrics { let opts = Opts::new("valid_blocks_processed", "total_valid_blocks_processed"); IntCounter::with_opts(opts)? }, - block_processing_historgram: { - let opts = - HistogramOpts::new("block_processing_historgram", "block_processing_time"); + block_processing_times: { + let opts = HistogramOpts::new("block_processing_times", "block_processing_time"); Histogram::with_opts(opts)? }, block_production_requests: { @@ -37,9 +43,8 @@ impl Metrics { let opts = Opts::new("block_production_successes", "blocks_successfully_produced"); IntCounter::with_opts(opts)? }, - block_production_historgram: { - let opts = - HistogramOpts::new("block_production_historgram", "block_production_time"); + block_production_times: { + let opts = HistogramOpts::new("block_production_times", "block_production_time"); Histogram::with_opts(opts)? }, attestation_production_requests: { @@ -56,26 +61,73 @@ impl Metrics { ); IntCounter::with_opts(opts)? }, - attestation_production_histogram: { + attestation_production_times: { let opts = HistogramOpts::new( - "attestation_production_histogram", + "attestation_production_times", "attestation_production_time", ); Histogram::with_opts(opts)? }, + attestation_processing_requests: { + let opts = Opts::new( + "attestation_processing_requests", + "total_attestation_processing_requests", + ); + IntCounter::with_opts(opts)? + }, + attestation_processing_successes: { + let opts = Opts::new( + "attestation_processing_successes", + "total_attestation_processing_successes", + ); + IntCounter::with_opts(opts)? + }, + attestation_processing_times: { + let opts = HistogramOpts::new( + "attestation_processing_times", + "attestation_processing_time", + ); + Histogram::with_opts(opts)? + }, + fork_choice_requests: { + let opts = Opts::new("fork_choice_requests", "total_times_fork_choice_called"); + IntCounter::with_opts(opts)? + }, + fork_choice_changed_head: { + let opts = Opts::new( + "fork_choice_changed_head", + "total_times_fork_choice_chose_a_new_head", + ); + IntCounter::with_opts(opts)? + }, + fork_choice_reorg_count: { + let opts = Opts::new("fork_choice_reorg_depth", "depth_of_reorg"); + IntCounter::with_opts(opts)? + }, + fork_choice_times: { + let opts = HistogramOpts::new("fork_choice_time", "total_time_to_run_fork_choice"); + Histogram::with_opts(opts)? + }, }) } pub fn register(&self, registry: &Registry) -> Result<(), Error> { registry.register(Box::new(self.block_processing_requests.clone()))?; registry.register(Box::new(self.block_processing_successes.clone()))?; - registry.register(Box::new(self.block_processing_historgram.clone()))?; + registry.register(Box::new(self.block_processing_times.clone()))?; registry.register(Box::new(self.block_production_requests.clone()))?; registry.register(Box::new(self.block_production_successes.clone()))?; - registry.register(Box::new(self.block_production_historgram.clone()))?; + registry.register(Box::new(self.block_production_times.clone()))?; registry.register(Box::new(self.attestation_production_requests.clone()))?; registry.register(Box::new(self.attestation_production_successes.clone()))?; - registry.register(Box::new(self.attestation_production_histogram.clone()))?; + registry.register(Box::new(self.attestation_production_times.clone()))?; + registry.register(Box::new(self.attestation_processing_requests.clone()))?; + registry.register(Box::new(self.attestation_processing_successes.clone()))?; + registry.register(Box::new(self.attestation_processing_times.clone()))?; + registry.register(Box::new(self.fork_choice_requests.clone()))?; + registry.register(Box::new(self.fork_choice_changed_head.clone()))?; + registry.register(Box::new(self.fork_choice_reorg_count.clone()))?; + registry.register(Box::new(self.fork_choice_times.clone()))?; Ok(()) } From 0b719e152350a9917ffa586a19fe82704b88ea16 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 17:53:13 +1000 Subject: [PATCH 22/76] Break `store` fns into smaller pieces --- beacon_node/store/src/block_at_slot.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/beacon_node/store/src/block_at_slot.rs b/beacon_node/store/src/block_at_slot.rs index 4a8abaefda..260a351147 100644 --- a/beacon_node/store/src/block_at_slot.rs +++ b/beacon_node/store/src/block_at_slot.rs @@ -25,15 +25,23 @@ pub fn get_block_at_preceeding_slot( slot: Slot, start_root: Hash256, ) -> Result, Error> { - let mut root = start_root; + Ok(match get_at_preceeding_slot(store, slot, start_root)? { + Some((hash, bytes)) => Some((hash, BeaconBlock::from_ssz_bytes(&bytes)?)), + None => None, + }) +} +fn get_at_preceeding_slot( + store: &T, + slot: Slot, + mut root: Hash256, +) -> Result)>, Error> { loop { if let Some(bytes) = get_block_bytes(store, root)? { let this_slot = read_slot_from_block_bytes(&bytes)?; if this_slot == slot { - let block = BeaconBlock::from_ssz_bytes(&bytes)?; - break Ok(Some((root, block))); + break Ok(Some((root, bytes))); } else if this_slot < slot { break Ok(None); } else { From 42b6e0c8a9634c742248248fba12b954b5ef62c1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 17:55:38 +1000 Subject: [PATCH 23/76] Run fork-choice after every block import --- beacon_node/beacon_chain/src/beacon_chain.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a7421b922f..d362f7fab1 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -713,21 +713,16 @@ impl BeaconChain { self.store.put(&block_root, &block)?; self.store.put(&state_root, &state)?; - // run the fork_choice add_block logic + // Register the new block with the fork choice service. self.fork_choice .write() .add_block(&block, &block_root, &self.spec)?; - // If the parent block was the parent_block, automatically update the canonical head. + // Execute the fork choice algorithm, enthroning a new head if discovered. // - // TODO: this is a first-in-best-dressed scenario that is not ideal; fork_choice should be - // run instead. - if self.head().beacon_block_root == parent_block_root { - self.update_canonical_head(block.clone(), block_root, state.clone(), state_root); - - // Update the canonical `BeaconState`. - self.update_state(state)?; - } + // Note: in the future we may choose to run fork-choice less often, potentially based upon + // some heuristic around number of attestations seen for the block. + self.fork_choice()?; self.metrics.block_processing_successes.inc(); timer.observe_duration(); From 4851d8be039073cfd5fdc30bbd1a6673d4ba92c6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 29 May 2019 18:09:51 +1000 Subject: [PATCH 24/76] Fix some metrics labels --- beacon_node/beacon_chain/src/metrics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 0fe738a8c3..a3c8553a19 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -24,11 +24,11 @@ impl Metrics { pub fn new() -> Result { Ok(Self { block_processing_requests: { - let opts = Opts::new("blocks_processed", "total_blocks_processed"); + let opts = Opts::new("block_processing_requests", "total_blocks_processed"); IntCounter::with_opts(opts)? }, block_processing_successes: { - let opts = Opts::new("valid_blocks_processed", "total_valid_blocks_processed"); + let opts = Opts::new("block_processing_successes", "total_valid_blocks_processed"); IntCounter::with_opts(opts)? }, block_processing_times: { @@ -101,7 +101,7 @@ impl Metrics { IntCounter::with_opts(opts)? }, fork_choice_reorg_count: { - let opts = Opts::new("fork_choice_reorg_depth", "depth_of_reorg"); + let opts = Opts::new("fork_choice_reorg_count", "number_of_reorgs"); IntCounter::with_opts(opts)? }, fork_choice_times: { From f44170cab1d653d733ef6855557150fc007b05f0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 30 May 2019 12:36:27 +1000 Subject: [PATCH 25/76] Add process metrics to `http-server` --- beacon_node/http_server/Cargo.toml | 2 +- beacon_node/http_server/src/metrics.rs | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index fb8bf9f4b7..098c3e1c9d 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -20,7 +20,7 @@ fork_choice = { path = "../../eth2/fork_choice" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } persistent = "^0.4" protobuf = "2.0.2" -prometheus = "^0.6" +prometheus = { version = "^0.6", features = ["process"] } clap = "2.32.0" store = { path = "../store" } dirs = "1.0.3" diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 2c6cda0ead..bfa80ec557 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -90,9 +90,13 @@ fn handle_metrics(req: &mut Request) -> IronResul let validator_count = beacon_chain.head().beacon_state.validator_registry.len(); local_metrics.validator_count.set(validator_count as i64); - // Gather the metrics. let mut buffer = vec![]; let encoder = TextEncoder::new(); + + // Gather `DEFAULT_REGISTRY` metrics. + encoder.encode(&prometheus::gather(), &mut buffer).unwrap(); + + // Gather metrics from our registry. let metric_families = r.gather(); encoder.encode(&metric_families, &mut buffer).unwrap(); From 0590504261de51bdc733231fce173cb1572cdde3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 30 May 2019 18:27:19 +1000 Subject: [PATCH 26/76] Remove `BeaconChain` wrapper trait from `network` --- beacon_node/network/src/beacon_chain.rs | 155 ------------------- beacon_node/network/src/lib.rs | 1 - beacon_node/network/src/message_handler.rs | 2 +- beacon_node/network/src/service.rs | 2 +- beacon_node/network/src/sync/import_queue.rs | 2 +- beacon_node/network/src/sync/simple_sync.rs | 55 +++++-- 6 files changed, 44 insertions(+), 173 deletions(-) delete mode 100644 beacon_node/network/src/beacon_chain.rs diff --git a/beacon_node/network/src/beacon_chain.rs b/beacon_node/network/src/beacon_chain.rs deleted file mode 100644 index e38acbb728..0000000000 --- a/beacon_node/network/src/beacon_chain.rs +++ /dev/null @@ -1,155 +0,0 @@ -use beacon_chain::BeaconChain as RawBeaconChain; -use beacon_chain::{ - parking_lot::RwLockReadGuard, - types::{BeaconState, ChainSpec}, - AttestationValidationError, CheckPoint, -}; -use eth2_libp2p::rpc::HelloMessage; -use types::{Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, Hash256, Slot}; - -pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; - -/// The network's API to the beacon chain. -pub trait BeaconChain: Send + Sync { - fn get_spec(&self) -> &ChainSpec; - - fn get_state(&self) -> RwLockReadGuard>; - - fn slot(&self) -> Slot; - - fn head(&self) -> RwLockReadGuard>; - - fn get_block(&self, block_root: &Hash256) -> Result, BeaconChainError>; - - fn best_slot(&self) -> Slot; - - fn best_block_root(&self) -> Hash256; - - fn finalized_head(&self) -> RwLockReadGuard>; - - fn finalized_epoch(&self) -> Epoch; - - fn hello_message(&self) -> HelloMessage; - - fn process_block(&self, block: BeaconBlock) - -> Result; - - fn process_attestation( - &self, - attestation: Attestation, - ) -> Result<(), AttestationValidationError>; - - fn get_block_roots( - &self, - start_slot: Slot, - count: usize, - skip: usize, - ) -> Result, BeaconChainError>; - - fn get_block_headers( - &self, - start_slot: Slot, - count: usize, - skip: usize, - ) -> Result, BeaconChainError>; - - fn get_block_bodies(&self, roots: &[Hash256]) - -> Result, BeaconChainError>; - - fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result; -} - -impl BeaconChain for RawBeaconChain { - fn get_spec(&self) -> &ChainSpec { - &self.spec - } - - fn get_state(&self) -> RwLockReadGuard> { - self.state.read() - } - - fn slot(&self) -> Slot { - self.get_state().slot - } - - fn head(&self) -> RwLockReadGuard> { - self.head() - } - - fn get_block(&self, block_root: &Hash256) -> Result, BeaconChainError> { - self.get_block(block_root) - } - - fn finalized_epoch(&self) -> Epoch { - self.get_state().finalized_epoch - } - - fn finalized_head(&self) -> RwLockReadGuard> { - self.finalized_head() - } - - fn best_slot(&self) -> Slot { - self.head().beacon_block.slot - } - - fn best_block_root(&self) -> Hash256 { - self.head().beacon_block_root - } - - fn hello_message(&self) -> HelloMessage { - let spec = self.get_spec(); - let state = self.get_state(); - - HelloMessage { - network_id: spec.chain_id, - latest_finalized_root: state.finalized_root, - latest_finalized_epoch: state.finalized_epoch, - best_root: self.best_block_root(), - best_slot: self.best_slot(), - } - } - - fn process_block( - &self, - block: BeaconBlock, - ) -> Result { - self.process_block(block) - } - - fn process_attestation( - &self, - attestation: Attestation, - ) -> Result<(), AttestationValidationError> { - self.process_attestation(attestation) - } - - fn get_block_roots( - &self, - start_slot: Slot, - count: usize, - skip: usize, - ) -> Result, BeaconChainError> { - self.get_block_roots(start_slot, count, skip) - } - - fn get_block_headers( - &self, - start_slot: Slot, - count: usize, - skip: usize, - ) -> Result, BeaconChainError> { - let roots = self.get_block_roots(start_slot, count, skip)?; - self.get_block_headers(&roots) - } - - fn get_block_bodies( - &self, - roots: &[Hash256], - ) -> Result, BeaconChainError> { - self.get_block_bodies(roots) - } - - fn is_new_block_root(&self, beacon_block_root: &Hash256) -> Result { - self.is_new_block_root(beacon_block_root) - } -} diff --git a/beacon_node/network/src/lib.rs b/beacon_node/network/src/lib.rs index c298e31b4e..b805c1d755 100644 --- a/beacon_node/network/src/lib.rs +++ b/beacon_node/network/src/lib.rs @@ -1,5 +1,4 @@ /// This crate provides the network server for Lighthouse. -pub mod beacon_chain; pub mod error; pub mod message_handler; pub mod service; diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index f6a27ad600..adafae145d 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -1,7 +1,7 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crate::error; use crate::service::{NetworkMessage, OutgoingMessage}; use crate::sync::SimpleSync; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use crossbeam_channel::{unbounded as channel, Sender}; use eth2_libp2p::{ behaviour::PubsubMessage, diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index d87b9e5a9d..9c71a60f7e 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,7 +1,7 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crate::error; use crate::message_handler::{HandlerMessage, MessageHandler}; use crate::NetworkConfig; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use crossbeam_channel::{unbounded as channel, Sender, TryRecvError}; use eth2_libp2p::Service as LibP2PService; use eth2_libp2p::{Libp2pEvent, PeerId}; diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 793f4c395e..5b03f58dff 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::PeerId; use slog::{debug, error}; diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 6ab8ea7d9a..0ca4f60115 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,6 +1,8 @@ use super::import_queue::ImportQueue; -use crate::beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; use crate::message_handler::NetworkContext; +use beacon_chain::{ + BeaconChain, BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, +}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; @@ -9,7 +11,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use tree_hash::TreeHash; -use types::{Attestation, BeaconBlock, Epoch, Hash256, Slot}; +use types::{Attestation, BeaconBlock, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot}; /// The number of slots that we can import blocks ahead of us, before going into full Sync mode. const SLOT_IMPORT_TOLERANCE: u64 = 100; @@ -90,7 +92,7 @@ impl From for PeerSyncInfo { impl From<&Arc>> for PeerSyncInfo { fn from(chain: &Arc>) -> PeerSyncInfo { - Self::from(chain.hello_message()) + Self::from(hello_message(chain)) } } @@ -153,7 +155,7 @@ impl SimpleSync { pub fn on_connect(&self, peer_id: PeerId, network: &mut NetworkContext) { info!(self.log, "PeerConnect"; "peer" => format!("{:?}", peer_id)); - network.send_rpc_request(peer_id, RPCRequest::Hello(self.chain.hello_message())); + network.send_rpc_request(peer_id, RPCRequest::Hello(hello_message(&self.chain))); } /// Handle a `Hello` request. @@ -172,7 +174,7 @@ impl SimpleSync { network.send_rpc_response( peer_id.clone(), request_id, - RPCResponse::Hello(self.chain.hello_message()), + RPCResponse::Hello(hello_message(&self.chain)), ); self.process_hello(peer_id, hello, network); @@ -202,7 +204,7 @@ impl SimpleSync { if local.has_higher_finalized_epoch_than(peer) { let peer_finalized_slot = peer .latest_finalized_epoch - .start_slot(self.chain.get_spec().slots_per_epoch); + .start_slot(T::EthSpec::spec().slots_per_epoch); let local_roots = self.chain.get_block_roots(peer_finalized_slot, 1, 0); @@ -245,7 +247,7 @@ impl SimpleSync { hello: HelloMessage, network: &mut NetworkContext, ) { - let spec = self.chain.get_spec(); + let spec = T::EthSpec::spec(); let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); @@ -424,7 +426,8 @@ impl SimpleSync { "count" => req.max_headers, ); - let headers = match self.chain.get_block_headers( + let headers = match get_block_headers( + &self.chain, req.start_slot, req.max_headers as usize, req.skip_slots as usize, @@ -596,7 +599,7 @@ impl SimpleSync { // parent(s). network.send_rpc_request( peer_id.clone(), - RPCRequest::Hello(self.chain.hello_message()), + RPCRequest::Hello(hello_message(&self.chain)), ); // Forward the block onto our peers. // @@ -835,15 +838,39 @@ impl SimpleSync { /// Returns `true` if the given slot is finalized in our chain. fn slot_is_finalized(&self, slot: Slot) -> bool { - slot <= self - .chain - .hello_message() + slot <= hello_message(&self.chain) .latest_finalized_epoch - .start_slot(self.chain.get_spec().slots_per_epoch) + .start_slot(T::EthSpec::spec().slots_per_epoch) } /// Generates our current state in the form of a HELLO RPC message. pub fn generate_hello(&self) -> HelloMessage { - self.chain.hello_message() + hello_message(&self.chain) } } + +/// Build a `HelloMessage` representing the state of the given `beacon_chain`. +fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { + let spec = T::EthSpec::spec(); + let state = &beacon_chain.head().beacon_state; + + HelloMessage { + network_id: spec.chain_id, + latest_finalized_root: state.finalized_root, + latest_finalized_epoch: state.finalized_epoch, + best_root: beacon_chain.head().beacon_block_root, + best_slot: beacon_chain.head().beacon_block.slot, + } +} + +/// Return a list of `BeaconBlockHeader` from the given `BeaconChain`, starting at `start_slot` and +/// returning `count` headers with a gap of `skip` slots between each. +fn get_block_headers( + beacon_chain: &BeaconChain, + start_slot: Slot, + count: usize, + skip: usize, +) -> Result, BeaconChainError> { + let roots = beacon_chain.get_block_roots(start_slot, count, skip)?; + beacon_chain.get_block_headers(&roots) +} From 5a5eebca06289ad36cdaa027b06e7e1762009e76 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 30 May 2019 18:35:27 +1000 Subject: [PATCH 27/76] Remove `BeaconChain` wrapper trait from `rpc` --- beacon_node/rpc/src/attestation.rs | 4 +- beacon_node/rpc/src/beacon_block.rs | 2 +- beacon_node/rpc/src/beacon_chain.rs | 71 ----------------------------- beacon_node/rpc/src/beacon_node.rs | 11 +++-- beacon_node/rpc/src/lib.rs | 3 +- beacon_node/rpc/src/validator.rs | 8 ++-- 6 files changed, 15 insertions(+), 84 deletions(-) delete mode 100644 beacon_node/rpc/src/beacon_chain.rs diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index 6048e42b1b..e764e1b1d1 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; use protos::services::{ @@ -34,7 +34,7 @@ impl AttestationService for AttestationServiceInstance { // verify the slot, drop lock on state afterwards { let slot_requested = req.get_slot(); - let state = self.chain.get_state(); + let state = &self.chain.head().beacon_state; // Start by performing some checks // Check that the AttestionData is for the current slot (otherwise it will not be valid) diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index e553b79e7f..c28c4f111b 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use crossbeam_channel; use eth2_libp2p::PubsubMessage; use futures::Future; diff --git a/beacon_node/rpc/src/beacon_chain.rs b/beacon_node/rpc/src/beacon_chain.rs deleted file mode 100644 index a37c219f67..0000000000 --- a/beacon_node/rpc/src/beacon_chain.rs +++ /dev/null @@ -1,71 +0,0 @@ -use beacon_chain::BeaconChain as RawBeaconChain; -use beacon_chain::{ - parking_lot::{RwLockReadGuard, RwLockWriteGuard}, - types::{BeaconState, ChainSpec, Signature}, - AttestationValidationError, BlockProductionError, -}; -pub use beacon_chain::{BeaconChainError, BeaconChainTypes, BlockProcessingOutcome}; -use types::{Attestation, AttestationData, BeaconBlock}; - -/// The RPC's API to the beacon chain. -pub trait BeaconChain: Send + Sync { - fn get_spec(&self) -> &ChainSpec; - - fn get_state(&self) -> RwLockReadGuard>; - - fn get_mut_state(&self) -> RwLockWriteGuard>; - - fn process_block(&self, block: BeaconBlock) - -> Result; - - fn produce_block( - &self, - randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), BlockProductionError>; - - fn produce_attestation_data(&self, shard: u64) -> Result; - - fn process_attestation( - &self, - attestation: Attestation, - ) -> Result<(), AttestationValidationError>; -} - -impl BeaconChain for RawBeaconChain { - fn get_spec(&self) -> &ChainSpec { - &self.spec - } - - fn get_state(&self) -> RwLockReadGuard> { - self.state.read() - } - - fn get_mut_state(&self) -> RwLockWriteGuard> { - self.state.write() - } - - fn process_block( - &self, - block: BeaconBlock, - ) -> Result { - self.process_block(block) - } - - fn produce_block( - &self, - randao_reveal: Signature, - ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { - self.produce_block(randao_reveal) - } - - fn produce_attestation_data(&self, shard: u64) -> Result { - self.produce_attestation_data(shard) - } - - fn process_attestation( - &self, - attestation: Attestation, - ) -> Result<(), AttestationValidationError> { - self.process_attestation(attestation) - } -} diff --git a/beacon_node/rpc/src/beacon_node.rs b/beacon_node/rpc/src/beacon_node.rs index a923bbb356..8b49b193e8 100644 --- a/beacon_node/rpc/src/beacon_node.rs +++ b/beacon_node/rpc/src/beacon_node.rs @@ -1,10 +1,11 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use futures::Future; use grpcio::{RpcContext, UnarySink}; use protos::services::{Empty, Fork, NodeInfoResponse}; use protos::services_grpc::BeaconNodeService; use slog::{trace, warn}; use std::sync::Arc; +use types::EthSpec; #[derive(Clone)] pub struct BeaconNodeServiceInstance { @@ -22,7 +23,7 @@ impl BeaconNodeService for BeaconNodeServiceInstance { node_info.set_version(version::version()); // get the chain state - let state = self.chain.get_state(); + let state = &self.chain.head().beacon_state; let state_fork = state.fork.clone(); let genesis_time = state.genesis_time; @@ -32,10 +33,12 @@ impl BeaconNodeService for BeaconNodeServiceInstance { fork.set_current_version(state_fork.current_version.to_vec()); fork.set_epoch(state_fork.epoch.into()); + let spec = T::EthSpec::spec(); + node_info.set_fork(fork); node_info.set_genesis_time(genesis_time); - node_info.set_genesis_slot(self.chain.get_spec().genesis_slot.as_u64()); - node_info.set_chain_id(u32::from(self.chain.get_spec().chain_id)); + node_info.set_genesis_slot(spec.genesis_slot.as_u64()); + node_info.set_chain_id(u32::from(spec.chain_id)); // send the node_info the requester let error_log = self.log.clone(); diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index 9646135b68..95c2e29163 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -1,15 +1,14 @@ mod attestation; mod beacon_block; -pub mod beacon_chain; mod beacon_node; pub mod config; mod validator; use self::attestation::AttestationServiceInstance; use self::beacon_block::BeaconBlockServiceInstance; -use self::beacon_chain::{BeaconChain, BeaconChainTypes}; use self::beacon_node::BeaconNodeServiceInstance; use self::validator::ValidatorServiceInstance; +use beacon_chain::{BeaconChain, BeaconChainTypes}; pub use config::Config as RPCConfig; use futures::Future; use grpcio::{Environment, ServerBuilder}; diff --git a/beacon_node/rpc/src/validator.rs b/beacon_node/rpc/src/validator.rs index e58c202d66..4ab9588c4d 100644 --- a/beacon_node/rpc/src/validator.rs +++ b/beacon_node/rpc/src/validator.rs @@ -1,4 +1,4 @@ -use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; use bls::PublicKey; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; @@ -7,7 +7,7 @@ use protos::services_grpc::ValidatorService; use slog::{trace, warn}; use ssz::Decode; use std::sync::Arc; -use types::{Epoch, RelativeEpoch}; +use types::{Epoch, EthSpec, RelativeEpoch}; #[derive(Clone)] pub struct ValidatorServiceInstance { @@ -29,8 +29,8 @@ impl ValidatorService for ValidatorServiceInstance { let validators = req.get_validators(); trace!(self.log, "RPC request"; "endpoint" => "GetValidatorDuties", "epoch" => req.get_epoch()); - let spec = self.chain.get_spec(); - let state = self.chain.get_state(); + let spec = T::EthSpec::spec(); + let state = &self.chain.head().beacon_state; let epoch = Epoch::from(req.get_epoch()); let mut resp = GetDutiesResponse::new(); let resp_validators = resp.mut_active_validators(); From 8acffcc0db35dc911c6150f5180dae46014479d3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 30 May 2019 18:38:41 +1000 Subject: [PATCH 28/76] Make some `BeaconChain` functions private --- beacon_node/beacon_chain/src/beacon_chain.rs | 8 ++++---- beacon_node/client/src/beacon_chain_types.rs | 2 +- beacon_node/client/src/lib.rs | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d362f7fab1..a04f6b1c86 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -94,7 +94,7 @@ pub struct BeaconChain { pub op_pool: OperationPool, canonical_head: RwLock>, finalized_head: RwLock>, - pub state: RwLock>, + state: RwLock>, pub spec: ChainSpec, pub fork_choice: RwLock, pub metrics: Metrics, @@ -310,7 +310,7 @@ impl BeaconChain { } /// Update the canonical head to some new values. - pub fn update_canonical_head( + fn update_canonical_head( &self, new_beacon_block: BeaconBlock, new_beacon_block_root: Hash256, @@ -354,7 +354,7 @@ impl BeaconChain { /// /// Also persists the `BeaconChain` to the store, in the case the client does not exit /// gracefully. - pub fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { + fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, _ => return Err(Error::UnableToReadSlot), @@ -407,7 +407,7 @@ impl BeaconChain { } /// Update the justified head to some new values. - pub fn update_finalized_head( + fn update_finalized_head( &self, new_beacon_block: BeaconBlock, new_beacon_block_root: Hash256, diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 8990e842db..24ad93ad44 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -68,7 +68,7 @@ where info!( log, "Loaded BeaconChain from store"; - "slot" => beacon_chain.state.read().slot, + "slot" => beacon_chain.head().beacon_state.slot, "best_slot" => beacon_chain.best_slot(), ); diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index b67cc6a0dc..4824d40ad2 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -75,7 +75,7 @@ where // If we don't block here we create an initial scenario where we're unable to process any // blocks and we're basically useless. { - let state_slot = beacon_chain.state.read().slot; + let state_slot = beacon_chain.head().beacon_state.slot; let wall_clock_slot = beacon_chain.read_slot_clock().unwrap(); let slots_since_genesis = beacon_chain.slots_since_genesis().unwrap(); info!( @@ -91,7 +91,7 @@ where info!( log, "State initialized"; - "state_slot" => beacon_chain.state.read().slot, + "state_slot" => beacon_chain.head().beacon_state.slot, "wall_clock_slot" => beacon_chain.read_slot_clock().unwrap(), ); @@ -190,7 +190,7 @@ fn do_state_catchup(chain: &Arc>, log: &slog "best_slot" => chain.head().beacon_block.slot, "latest_block_root" => format!("{}", chain.head().beacon_block_root), "wall_clock_slot" => chain.read_slot_clock().unwrap(), - "state_slot" => chain.state.read().slot, + "state_slot" => chain.head().beacon_state.slot, "slots_since_genesis" => genesis_height, ); From 2f9f8bf772b6c03f4a15f587f9b84ea90cde6a1f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 31 May 2019 17:32:20 +1000 Subject: [PATCH 29/76] Tidy `BeaconChain`; del `spec`, unify head updates --- beacon_node/beacon_chain/src/beacon_chain.rs | 299 +++++++++++-------- beacon_node/http_server/src/api.rs | 3 +- beacon_node/network/src/sync/simple_sync.rs | 2 +- validator_client/src/error.rs | 4 +- 4 files changed, 176 insertions(+), 132 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a04f6b1c86..a9c8791334 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -88,15 +88,28 @@ pub trait BeaconChainTypes { type EthSpec: types::EthSpec; } +/// Represents the "Beacon Chain" component of Ethereum 2.0. Allows import of blocks and block +/// operations and chooses a canonical head. pub struct BeaconChain { + /// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB. pub store: Arc, + /// Reports the current slot, typically based upon the system clock. pub slot_clock: T::SlotClock, + /// Stores all operations (e.g., `Attestation`, `Deposit`, etc) that are candidates for + /// inclusion in a block. pub op_pool: OperationPool, + /// Stores a "snapshot" of the chain at the time the head-of-the-chain block was recieved. canonical_head: RwLock>, + /// Stores a "snapshot" of the chain at the latest finalized point. finalized_head: RwLock>, + /// The same state from `self.canonical_head`, but updated at the start of each slot with a + /// skip slot if no block is recieved. This is effectively a cache that avoids repeating calls + /// to `per_slot_processing`. state: RwLock>, - pub spec: ChainSpec, + /// A state-machine that is updated with information from the network and chooses a canonical + /// head block. pub fork_choice: RwLock, + /// Stores metrics about this `BeaconChain`. pub metrics: Metrics, } @@ -138,7 +151,6 @@ impl BeaconChain { state: RwLock::new(genesis_state), finalized_head, canonical_head, - spec, fork_choice: RwLock::new(fork_choice), metrics: Metrics::new()?, }) @@ -170,7 +182,6 @@ impl BeaconChain { canonical_head: RwLock::new(p.canonical_head), finalized_head: RwLock::new(p.finalized_head), state: RwLock::new(p.state), - spec, fork_choice: RwLock::new(fork_choice), metrics: Metrics::new()?, })) @@ -309,103 +320,37 @@ impl BeaconChain { Ok(self.store.get(block_root)?) } - /// Update the canonical head to some new values. - fn update_canonical_head( - &self, - new_beacon_block: BeaconBlock, - new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, - new_beacon_state_root: Hash256, - ) { - debug!( - "Updating canonical head with block at slot: {}", - new_beacon_block.slot - ); - let mut head = self.canonical_head.write(); - head.update( - new_beacon_block, - new_beacon_block_root, - new_beacon_state, - new_beacon_state_root, - ); - } + /// Update the canonical head to `new_head`. + fn update_canonical_head(&self, new_head: CheckPoint) -> Result<(), Error> { + // Update the checkpoint that stores the head of the chain at the time it received the + // block. + *self.canonical_head.write() = new_head; - /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the - /// fork-choice rule). - /// - /// It is important to note that the `beacon_state` returned may not match the present slot. It - /// is the state as it was when the head block was received, which could be some slots prior to - /// now. - pub fn head(&self) -> RwLockReadGuard> { - self.canonical_head.read() - } + // Update the always-at-the-present-slot state we keep around for performance gains. + *self.state.write() = { + let mut state = self.canonical_head.read().beacon_state.clone(); - /// Returns the slot of the highest block in the canonical chain. - pub fn best_slot(&self) -> Slot { - self.canonical_head.read().beacon_block.slot - } + let present_slot = match self.slot_clock.present_slot() { + Ok(Some(slot)) => slot, + _ => return Err(Error::UnableToReadSlot), + }; - /// Updates the canonical `BeaconState` with the supplied state. - /// - /// Advances the chain forward to the present slot. This method is better than just setting - /// state and calling `catchup_state` as it will not result in an old state being installed and - /// then having it iteratively updated -- in such a case it's possible for another thread to - /// find the state at an old slot. - /// - /// Also persists the `BeaconChain` to the store, in the case the client does not exit - /// gracefully. - fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { - let present_slot = match self.slot_clock.present_slot() { - Ok(Some(slot)) => slot, - _ => return Err(Error::UnableToReadSlot), + // If required, transition the new state to the present slot. + for _ in state.slot.as_u64()..present_slot.as_u64() { + per_slot_processing(&mut state, &T::EthSpec::spec())?; + } + + state.build_all_caches(&T::EthSpec::spec())?; + + state }; - // If required, transition the new state to the present slot. - for _ in state.slot.as_u64()..present_slot.as_u64() { - per_slot_processing(&mut state, &self.spec)?; - } - - state.build_all_caches(&self.spec)?; - - *self.state.write() = state; - + // Save `self` to `self.store`. self.persist()?; Ok(()) } - /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`. - pub fn catchup_state(&self) -> Result<(), Error> { - let present_slot = match self.slot_clock.present_slot() { - Ok(Some(slot)) => slot, - _ => return Err(Error::UnableToReadSlot), - }; - - let mut state = self.state.write(); - - // If required, transition the new state to the present slot. - for _ in state.slot.as_u64()..present_slot.as_u64() { - // Ensure the next epoch state caches are built in case of an epoch transition. - state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, &self.spec)?; - state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, &self.spec)?; - - per_slot_processing(&mut *state, &self.spec)?; - } - - state.build_all_caches(&self.spec)?; - - Ok(()) - } - - /// Build all of the caches on the current state. - /// - /// Ideally this shouldn't be required, however we leave it here for testing. - pub fn ensure_state_caches_are_built(&self) -> Result<(), Error> { - self.state.write().build_all_caches(&self.spec)?; - - Ok(()) - } - /// Update the justified head to some new values. fn update_finalized_head( &self, @@ -423,6 +368,86 @@ impl BeaconChain { ); } + /* + /// Updates the canonical `BeaconState` with the supplied state. + /// + /// Advances the chain forward to the present slot. This method is better than just setting + /// state and calling `catchup_state` as it will not result in an old state being installed and + /// then having it iteratively updated -- in such a case it's possible for another thread to + /// find the state at an old slot. + /// + /// Also persists the `BeaconChain` to the store, in the case the client does not exit + /// gracefully. + fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { + let present_slot = match self.slot_clock.present_slot() { + Ok(Some(slot)) => slot, + _ => return Err(Error::UnableToReadSlot), + }; + + // If required, transition the new state to the present slot. + for _ in state.slot.as_u64()..present_slot.as_u64() { + per_slot_processing(&mut state, &T::EthSpec::spec())?; + } + + state.build_all_caches(&T::EthSpec::spec())?; + + *self.state.write() = state; + + self.persist()?; + + Ok(()) + } + */ + + /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the + /// fork-choice rule). + /// + /// It is important to note that the `beacon_state` returned may not match the present slot. It + /// is the state as it was when the head block was received, which could be some slots prior to + /// now. + pub fn head(&self) -> RwLockReadGuard> { + self.canonical_head.read() + } + + /// Returns the slot of the highest block in the canonical chain. + pub fn best_slot(&self) -> Slot { + self.canonical_head.read().beacon_block.slot + } + + /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`. + pub fn catchup_state(&self) -> Result<(), Error> { + let spec = &T::EthSpec::spec(); + + let present_slot = match self.slot_clock.present_slot() { + Ok(Some(slot)) => slot, + _ => return Err(Error::UnableToReadSlot), + }; + + let mut state = self.state.write(); + + // If required, transition the new state to the present slot. + for _ in state.slot.as_u64()..present_slot.as_u64() { + // Ensure the next epoch state caches are built in case of an epoch transition. + state.build_epoch_cache(RelativeEpoch::NextWithoutRegistryChange, spec)?; + state.build_epoch_cache(RelativeEpoch::NextWithRegistryChange, spec)?; + + per_slot_processing(&mut *state, spec)?; + } + + state.build_all_caches(spec)?; + + Ok(()) + } + + /// Build all of the caches on the current state. + /// + /// Ideally this shouldn't be required, however we leave it here for testing. + pub fn ensure_state_caches_are_built(&self) -> Result<(), Error> { + self.state.write().build_all_caches(&T::EthSpec::spec())?; + + Ok(()) + } + /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, /// indirectly, by the fork-choice rule). pub fn finalized_head(&self) -> RwLockReadGuard> { @@ -467,13 +492,12 @@ impl BeaconChain { /// genesis. pub fn slots_since_genesis(&self) -> Option { let now = self.read_slot_clock()?; + let genesis_slot = T::EthSpec::spec().genesis_slot; - if now < self.spec.genesis_slot { + if now < genesis_slot { None } else { - Some(SlotHeight::from( - now.as_u64() - self.spec.genesis_slot.as_u64(), - )) + Some(SlotHeight::from(now.as_u64() - genesis_slot.as_u64())) } } @@ -493,12 +517,12 @@ impl BeaconChain { pub fn block_proposer(&self, slot: Slot) -> Result { self.state .write() - .build_epoch_cache(RelativeEpoch::Current, &self.spec)?; + .build_epoch_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; let index = self.state.read().get_beacon_proposer_index( slot, RelativeEpoch::Current, - &self.spec, + &T::EthSpec::spec(), )?; Ok(index) @@ -519,7 +543,7 @@ impl BeaconChain { if let Some(attestation_duty) = self .state .read() - .get_attestation_duties(validator_index, &self.spec)? + .get_attestation_duties(validator_index, &T::EthSpec::spec())? { Ok(Some((attestation_duty.slot, attestation_duty.shard))) } else { @@ -529,7 +553,7 @@ impl BeaconChain { /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { - trace!("BeaconChain::produce_attestation: shard: {}", shard); + let slots_per_epoch = T::EthSpec::spec().slots_per_epoch; self.metrics.attestation_production_requests.inc(); let timer = self.metrics.attestation_production_times.start_timer(); @@ -540,8 +564,8 @@ impl BeaconChain { .state .read() .slot - .epoch(self.spec.slots_per_epoch) - .start_slot(self.spec.slots_per_epoch); + .epoch(slots_per_epoch) + .start_slot(slots_per_epoch); let target_root = if state.slot == current_epoch_start_slot { // If we're on the first slot of the state's epoch. @@ -554,7 +578,7 @@ impl BeaconChain { *self .state .read() - .get_block_root(current_epoch_start_slot - self.spec.slots_per_epoch)? + .get_block_root(current_epoch_start_slot - slots_per_epoch)? } } else { // If we're not on the first slot of the epoch. @@ -587,9 +611,9 @@ impl BeaconChain { self.metrics.attestation_processing_requests.inc(); let timer = self.metrics.attestation_processing_times.start_timer(); - let result = self - .op_pool - .insert_attestation(attestation, &*self.state.read(), &self.spec); + let result = + self.op_pool + .insert_attestation(attestation, &*self.state.read(), &T::EthSpec::spec()); if result.is_ok() { self.metrics.attestation_production_successes.inc(); @@ -606,19 +630,19 @@ impl BeaconChain { deposit: Deposit, ) -> Result { self.op_pool - .insert_deposit(deposit, &*self.state.read(), &self.spec) + .insert_deposit(deposit, &*self.state.read(), &T::EthSpec::spec()) } /// Accept some exit and queue it for inclusion in an appropriate block. pub fn process_voluntary_exit(&self, exit: VoluntaryExit) -> Result<(), ExitValidationError> { self.op_pool - .insert_voluntary_exit(exit, &*self.state.read(), &self.spec) + .insert_voluntary_exit(exit, &*self.state.read(), &T::EthSpec::spec()) } /// Accept some transfer and queue it for inclusion in an appropriate block. pub fn process_transfer(&self, transfer: Transfer) -> Result<(), TransferValidationError> { self.op_pool - .insert_transfer(transfer, &*self.state.read(), &self.spec) + .insert_transfer(transfer, &*self.state.read(), &T::EthSpec::spec()) } /// Accept some proposer slashing and queue it for inclusion in an appropriate block. @@ -626,8 +650,11 @@ impl BeaconChain { &self, proposer_slashing: ProposerSlashing, ) -> Result<(), ProposerSlashingValidationError> { - self.op_pool - .insert_proposer_slashing(proposer_slashing, &*self.state.read(), &self.spec) + self.op_pool.insert_proposer_slashing( + proposer_slashing, + &*self.state.read(), + &T::EthSpec::spec(), + ) } /// Accept some attester slashing and queue it for inclusion in an appropriate block. @@ -635,8 +662,11 @@ impl BeaconChain { &self, attester_slashing: AttesterSlashing, ) -> Result<(), AttesterSlashingValidationError> { - self.op_pool - .insert_attester_slashing(attester_slashing, &*self.state.read(), &self.spec) + self.op_pool.insert_attester_slashing( + attester_slashing, + &*self.state.read(), + &T::EthSpec::spec(), + ) } /// Accept some block and attempt to add it to block DAG. @@ -686,7 +716,7 @@ impl BeaconChain { // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, &self.spec) { + if let Err(e) = per_slot_processing(&mut state, &T::EthSpec::spec()) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); @@ -695,7 +725,7 @@ impl BeaconChain { // Apply the received block to its parent state (which has been transitioned into this // slot). - if let Err(e) = per_block_processing(&mut state, &block, &self.spec) { + if let Err(e) = per_block_processing(&mut state, &block, &T::EthSpec::spec()) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::PerBlockProcessingError(e), )); @@ -716,7 +746,7 @@ impl BeaconChain { // Register the new block with the fork choice service. self.fork_choice .write() - .add_block(&block, &block_root, &self.spec)?; + .add_block(&block, &block_root, &T::EthSpec::spec())?; // Execute the fork choice algorithm, enthroning a new head if discovered. // @@ -744,7 +774,7 @@ impl BeaconChain { let mut state = self.state.read().clone(); - state.build_epoch_cache(RelativeEpoch::Current, &self.spec)?; + state.build_epoch_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; trace!("Finding attestations for new block..."); @@ -752,14 +782,15 @@ impl BeaconChain { .get_block_root(state.slot - 1) .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)?; - let (proposer_slashings, attester_slashings) = - self.op_pool.get_slashings(&*self.state.read(), &self.spec); + let (proposer_slashings, attester_slashings) = self + .op_pool + .get_slashings(&*self.state.read(), &T::EthSpec::spec()); let mut block = BeaconBlock { slot: state.slot, previous_block_root, state_root: Hash256::zero(), // Updated after the state is calculated. - signature: self.spec.empty_signature.clone(), // To be completed by a validator. + signature: T::EthSpec::spec().empty_signature.clone(), // To be completed by a validator. body: BeaconBlockBody { randao_reveal, eth1_data: Eth1Data { @@ -771,12 +802,16 @@ impl BeaconChain { attester_slashings, attestations: self .op_pool - .get_attestations(&*self.state.read(), &self.spec), - deposits: self.op_pool.get_deposits(&*self.state.read(), &self.spec), + .get_attestations(&*self.state.read(), &T::EthSpec::spec()), + deposits: self + .op_pool + .get_deposits(&*self.state.read(), &T::EthSpec::spec()), voluntary_exits: self .op_pool - .get_voluntary_exits(&*self.state.read(), &self.spec), - transfers: self.op_pool.get_transfers(&*self.state.read(), &self.spec), + .get_voluntary_exits(&*self.state.read(), &T::EthSpec::spec()), + transfers: self + .op_pool + .get_transfers(&*self.state.read(), &T::EthSpec::spec()), }, }; @@ -785,7 +820,11 @@ impl BeaconChain { block.body.attestations.len() ); - per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec)?; + per_block_processing_without_verifying_block_signature( + &mut state, + &block, + &T::EthSpec::spec(), + )?; let state_root = state.canonical_root(); @@ -808,7 +847,7 @@ impl BeaconChain { let new_head_root = self .fork_choice .write() - .find_head(&present_head_root, &self.spec)?; + .find_head(&present_head_root, &T::EthSpec::spec())?; timer.observe_duration(); @@ -830,10 +869,12 @@ impl BeaconChain { }; let state_root = block.state_root; - self.update_canonical_head(block, new_head_root, state.clone(), state_root); - - // Update the canonical `BeaconState`. - self.update_state(state)?; + self.update_canonical_head(CheckPoint { + beacon_block: block, + beacon_block_root: new_head_root, + beacon_state: state.clone(), + beacon_state_root: state_root, + })?; } Ok(()) @@ -863,7 +904,7 @@ impl BeaconChain { loop { let beacon_block_root = last_slot.beacon_block.previous_block_root; - if beacon_block_root == self.spec.zero_hash { + if beacon_block_root == T::EthSpec::spec().zero_hash { break; // Genesis has been reached. } diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs index a910808998..2594f9c288 100644 --- a/beacon_node/http_server/src/api.rs +++ b/beacon_node/http_server/src/api.rs @@ -10,6 +10,7 @@ use persistent::Read; use router::Router; use serde_json::json; use std::sync::Arc; +use types::EthSpec; /// Yields a handler for the HTTP API. pub fn build_handler( @@ -64,7 +65,7 @@ fn handle_fork(req: &mut Request) -> IronResult(beacon_chain: &BeaconChain) -> HelloMes latest_finalized_root: state.finalized_root, latest_finalized_epoch: state.finalized_epoch, best_root: beacon_chain.head().beacon_block_root, - best_slot: beacon_chain.head().beacon_block.slot, + best_slot: state.slot, } } diff --git a/validator_client/src/error.rs b/validator_client/src/error.rs index 97500f900b..7740c4f2b6 100644 --- a/validator_client/src/error.rs +++ b/validator_client/src/error.rs @@ -1,6 +1,8 @@ use slot_clock; -use error_chain::error_chain; +use error_chain::{ + error_chain +}; error_chain! { links { } From 08bf5817c9770fc717e6fa6cd7d88ab3039b6f21 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 31 May 2019 17:51:32 +1000 Subject: [PATCH 30/76] Update fork choice to take just. head --- beacon_node/beacon_chain/src/beacon_chain.rs | 75 ++++++------------- .../src/persisted_beacon_chain.rs | 1 - 2 files changed, 22 insertions(+), 54 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a9c8791334..b790db7a22 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -100,8 +100,6 @@ pub struct BeaconChain { pub op_pool: OperationPool, /// Stores a "snapshot" of the chain at the time the head-of-the-chain block was recieved. canonical_head: RwLock>, - /// Stores a "snapshot" of the chain at the latest finalized point. - finalized_head: RwLock>, /// The same state from `self.canonical_head`, but updated at the start of each slot with a /// skip slot if no block is recieved. This is effectively a cache that avoids repeating calls /// to `per_slot_processing`. @@ -129,12 +127,6 @@ impl BeaconChain { let block_root = genesis_block.block_header().canonical_root(); store.put(&block_root, &genesis_block)?; - let finalized_head = RwLock::new(CheckPoint::new( - genesis_block.clone(), - block_root, - genesis_state.clone(), - state_root, - )); let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, @@ -149,7 +141,6 @@ impl BeaconChain { slot_clock, op_pool: OperationPool::new(), state: RwLock::new(genesis_state), - finalized_head, canonical_head, fork_choice: RwLock::new(fork_choice), metrics: Metrics::new()?, @@ -180,7 +171,6 @@ impl BeaconChain { slot_clock, op_pool: OperationPool::default(), canonical_head: RwLock::new(p.canonical_head), - finalized_head: RwLock::new(p.finalized_head), state: RwLock::new(p.state), fork_choice: RwLock::new(fork_choice), metrics: Metrics::new()?, @@ -191,7 +181,6 @@ impl BeaconChain { pub fn persist(&self) -> Result<(), Error> { let p: PersistedBeaconChain = PersistedBeaconChain { canonical_head: self.canonical_head.read().clone(), - finalized_head: self.finalized_head.read().clone(), state: self.state.read().clone(), }; @@ -351,23 +340,6 @@ impl BeaconChain { Ok(()) } - /// Update the justified head to some new values. - fn update_finalized_head( - &self, - new_beacon_block: BeaconBlock, - new_beacon_block_root: Hash256, - new_beacon_state: BeaconState, - new_beacon_state_root: Hash256, - ) { - let mut finalized_head = self.finalized_head.write(); - finalized_head.update( - new_beacon_block, - new_beacon_block_root, - new_beacon_state, - new_beacon_state_root, - ); - } - /* /// Updates the canonical `BeaconState` with the supplied state. /// @@ -448,12 +420,6 @@ impl BeaconChain { Ok(()) } - /// Returns a read-lock guarded `CheckPoint` struct for reading the justified head (as chosen, - /// indirectly, by the fork-choice rule). - pub fn finalized_head(&self) -> RwLockReadGuard> { - self.finalized_head.read() - } - /// Returns the validator index (if any) for the given public key. /// /// Information is retrieved from the present `beacon_state.validator_registry`. @@ -840,40 +806,43 @@ impl BeaconChain { pub fn fork_choice(&self) -> Result<(), Error> { self.metrics.fork_choice_requests.inc(); - let present_head_root = self.finalized_head().beacon_block_root; - + // Start fork choice metrics timer. let timer = self.metrics.fork_choice_times.start_timer(); - let new_head_root = self + // Determine the root of the block that is the head of the chain. + let beacon_block_root = self .fork_choice .write() - .find_head(&present_head_root, &T::EthSpec::spec())?; + .find_head(&self.head().beacon_state.current_justified_root, &T::EthSpec::spec())?; + // End fork choice metrics timer. timer.observe_duration(); - if new_head_root != present_head_root { + // If a new head was chosen. + if beacon_block_root != self.head().beacon_block_root { self.metrics.fork_choice_changed_head.inc(); - let block: BeaconBlock = self + let beacon_block: BeaconBlock = self .store - .get(&new_head_root)? - .ok_or_else(|| Error::MissingBeaconBlock(new_head_root))?; - let state: BeaconState = self - .store - .get(&block.state_root)? - .ok_or_else(|| Error::MissingBeaconState(block.state_root))?; + .get(&beacon_block_root)? + .ok_or_else(|| Error::MissingBeaconBlock(beacon_block_root))?; - // Log if we switched to a new chain. - if present_head_root != block.previous_block_root { + let beacon_state_root = beacon_block.state_root; + let beacon_state: BeaconState = self + .store + .get(&beacon_state_root)? + .ok_or_else(|| Error::MissingBeaconState(beacon_state_root))?; + + // If we switched to a new chain (instead of building atop the present chain). + if self.head().beacon_block_root != beacon_block.previous_block_root { self.metrics.fork_choice_reorg_count.inc(); }; - let state_root = block.state_root; self.update_canonical_head(CheckPoint { - beacon_block: block, - beacon_block_root: new_head_root, - beacon_state: state.clone(), - beacon_state_root: state_root, + beacon_block, + beacon_block_root, + beacon_state, + beacon_state_root, })?; } diff --git a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs index cb34e995cd..2a18451b63 100644 --- a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs +++ b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs @@ -10,7 +10,6 @@ pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA"; #[derive(Encode, Decode)] pub struct PersistedBeaconChain { pub canonical_head: CheckPoint, - pub finalized_head: CheckPoint, // TODO: operations pool. pub state: BeaconState, } From 5b425c9bf3f4f6c2d1d1626f451096e13e571718 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 31 May 2019 17:53:52 +1000 Subject: [PATCH 31/76] Switch runtime fork choice to optimized --- beacon_node/client/src/beacon_chain_types.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 24ad93ad44..f84a7fdf65 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -1,5 +1,5 @@ use beacon_chain::{ - fork_choice::BitwiseLMDGhost, + fork_choice::OptimizedLMDGhost, slot_clock::SystemTimeSlotClock, store::{DiskStore, MemoryStore, Store}, BeaconChain, BeaconChainTypes, @@ -28,7 +28,7 @@ pub struct TestnetMemoryBeaconChainTypes; impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { type Store = MemoryStore; type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; + type ForkChoice = OptimizedLMDGhost; type EthSpec = LighthouseTestnetEthSpec; } @@ -45,7 +45,7 @@ pub struct TestnetDiskBeaconChainTypes; impl BeaconChainTypes for TestnetDiskBeaconChainTypes { type Store = DiskStore; type SlotClock = SystemTimeSlotClock; - type ForkChoice = BitwiseLMDGhost; + type ForkChoice = OptimizedLMDGhost; type EthSpec = LighthouseTestnetEthSpec; } From b33b5c28b4675c7328e86cf433a9649e5d5c5be2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 31 May 2019 18:13:35 +1000 Subject: [PATCH 32/76] Add additional metrics for `BeaconChain` --- beacon_node/http_server/src/metrics.rs | 55 ++-------------- .../http_server/src/metrics/local_metrics.rs | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+), 48 deletions(-) create mode 100644 beacon_node/http_server/src/metrics/local_metrics.rs diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index bfa80ec557..7e716d5d7c 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -6,10 +6,12 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; use persistent::Read; -use prometheus::{Encoder, IntGauge, Opts, Registry, TextEncoder}; -use slot_clock::SlotClock; +use prometheus::{Encoder, Registry, TextEncoder}; use std::sync::Arc; -use types::Slot; + +pub use local_metrics::LocalMetrics; + +mod local_metrics; /// Yields a handler for the metrics endpoint. pub fn build_handler( @@ -28,39 +30,6 @@ pub fn build_handler( chain } -pub struct LocalMetrics { - present_slot: IntGauge, - best_slot: IntGauge, - validator_count: IntGauge, -} - -impl LocalMetrics { - pub fn new() -> Result { - Ok(Self { - present_slot: { - let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); - IntGauge::with_opts(opts)? - }, - best_slot: { - let opts = Opts::new("best_slot", "slot_of_block_at_chain_head"); - IntGauge::with_opts(opts)? - }, - validator_count: { - let opts = Opts::new("validator_count", "number_of_validators"); - IntGauge::with_opts(opts)? - }, - }) - } - - pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { - registry.register(Box::new(self.present_slot.clone()))?; - registry.register(Box::new(self.best_slot.clone()))?; - registry.register(Box::new(self.validator_count.clone()))?; - - Ok(()) - } -} - /// Handle a request for Prometheus metrics. /// /// Returns a text string containing all metrics. @@ -77,18 +46,8 @@ fn handle_metrics(req: &mut Request) -> IronResul .get::>() .map_err(map_persistent_err_to_500)?; - let present_slot = beacon_chain - .slot_clock - .present_slot() - .unwrap_or_else(|_| None) - .unwrap_or_else(|| Slot::new(0)); - local_metrics.present_slot.set(present_slot.as_u64() as i64); - - let best_slot = beacon_chain.head().beacon_block.slot; - local_metrics.best_slot.set(best_slot.as_u64() as i64); - - let validator_count = beacon_chain.head().beacon_state.validator_registry.len(); - local_metrics.validator_count.set(validator_count as i64); + // Update metrics that are calculated on each scrape. + local_metrics.update(&beacon_chain); let mut buffer = vec![]; let encoder = TextEncoder::new(); diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs new file mode 100644 index 0000000000..8b9b1fdad8 --- /dev/null +++ b/beacon_node/http_server/src/metrics/local_metrics.rs @@ -0,0 +1,66 @@ +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use prometheus::{IntGauge, Opts, Registry}; +use slot_clock::SlotClock; +use types::Slot; + +pub struct LocalMetrics { + present_slot: IntGauge, + best_slot: IntGauge, + validator_count: IntGauge, + justified_epoch: IntGauge, + finalized_epoch: IntGauge, +} + +impl LocalMetrics { + /// Create a new instance. + pub fn new() -> Result { + Ok(Self { + present_slot: { + let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); + IntGauge::with_opts(opts)? + }, + best_slot: { + let opts = Opts::new("best_slot", "slot_of_block_at_chain_head"); + IntGauge::with_opts(opts)? + }, + validator_count: { + let opts = Opts::new("validator_count", "number_of_validators"); + IntGauge::with_opts(opts)? + }, + justified_epoch: { + let opts = Opts::new("justified_epoch", "state_justified_epoch"); + IntGauge::with_opts(opts)? + }, + finalized_epoch: { + let opts = Opts::new("finalized_epoch", "state_finalized_epoch"); + IntGauge::with_opts(opts)? + }, + }) + } + + /// Registry this instance with the `registry`. + pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { + registry.register(Box::new(self.present_slot.clone()))?; + registry.register(Box::new(self.best_slot.clone()))?; + registry.register(Box::new(self.validator_count.clone()))?; + + Ok(()) + } + + /// Update the metrics in `self` to the latest values. + pub fn update(&self, beacon_chain: &BeaconChain) { + let state = &beacon_chain.head().beacon_state; + + let present_slot = beacon_chain + .slot_clock + .present_slot() + .unwrap_or_else(|_| None) + .unwrap_or_else(|| Slot::new(0)); + self.present_slot.set(present_slot.as_u64() as i64); + + self.best_slot.set(state.slot.as_u64() as i64); + self.validator_count.set(state.validator_registry.len() as i64); + self.justified_epoch.set(state.current_justified_epoch.as_u64() as i64); + self.finalized_epoch.set(state.finalized_epoch.as_u64() as i64); + } +} From 7058f62b5077a725edb1653c424c48a62ffcf099 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 08:26:25 +1000 Subject: [PATCH 33/76] Add extra validator balances metric --- .../http_server/src/metrics/local_metrics.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs index 8b9b1fdad8..68e02b0a11 100644 --- a/beacon_node/http_server/src/metrics/local_metrics.rs +++ b/beacon_node/http_server/src/metrics/local_metrics.rs @@ -3,12 +3,17 @@ use prometheus::{IntGauge, Opts, Registry}; use slot_clock::SlotClock; use types::Slot; +// If set to `true` will iterate and sum the balances of all validators in the state for each +// scrape. +const SHOULD_SUM_VALIDATOR_BALANCES: bool = true; + pub struct LocalMetrics { present_slot: IntGauge, best_slot: IntGauge, validator_count: IntGauge, justified_epoch: IntGauge, finalized_epoch: IntGauge, + validator_balances_sum: IntGauge, } impl LocalMetrics { @@ -35,6 +40,10 @@ impl LocalMetrics { let opts = Opts::new("finalized_epoch", "state_finalized_epoch"); IntGauge::with_opts(opts)? }, + validator_balances_sum: { + let opts = Opts::new("validator_balances_sum", "sum_of_all_validator_balances"); + IntGauge::with_opts(opts)? + }, }) } @@ -43,6 +52,9 @@ impl LocalMetrics { registry.register(Box::new(self.present_slot.clone()))?; registry.register(Box::new(self.best_slot.clone()))?; registry.register(Box::new(self.validator_count.clone()))?; + registry.register(Box::new(self.finalized_epoch.clone()))?; + registry.register(Box::new(self.justified_epoch.clone()))?; + registry.register(Box::new(self.validator_balances_sum.clone()))?; Ok(()) } @@ -62,5 +74,8 @@ impl LocalMetrics { self.validator_count.set(state.validator_registry.len() as i64); self.justified_epoch.set(state.current_justified_epoch.as_u64() as i64); self.finalized_epoch.set(state.finalized_epoch.as_u64() as i64); + if SHOULD_SUM_VALIDATOR_BALANCES { + self.validator_balances_sum.set(state.validator_balances.iter().sum::() as i64); + } } } From 8831db1e0fdfe3592f1ad2933ccdbaf41ce53a40 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 12:36:10 +1000 Subject: [PATCH 34/76] Fix recently introduced errors for gRPC block prod --- beacon_node/beacon_chain/src/beacon_chain.rs | 6 ++++++ beacon_node/rpc/src/attestation.rs | 2 +- beacon_node/rpc/src/validator.rs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b790db7a22..bac8f74a7d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -371,6 +371,12 @@ impl BeaconChain { } */ + /// Returns a read-lock guarded `BeaconState` which is the `canonical_head` that has been + /// updated to match the current slot clock. + pub fn current_state(&self) -> RwLockReadGuard> { + self.state.read() + } + /// Returns a read-lock guarded `CheckPoint` struct for reading the head (as chosen by the /// fork-choice rule). /// diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index e764e1b1d1..27fcad7cda 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -34,7 +34,7 @@ impl AttestationService for AttestationServiceInstance { // verify the slot, drop lock on state afterwards { let slot_requested = req.get_slot(); - let state = &self.chain.head().beacon_state; + let state = &self.chain.current_state(); // Start by performing some checks // Check that the AttestionData is for the current slot (otherwise it will not be valid) diff --git a/beacon_node/rpc/src/validator.rs b/beacon_node/rpc/src/validator.rs index 4ab9588c4d..68ce60b969 100644 --- a/beacon_node/rpc/src/validator.rs +++ b/beacon_node/rpc/src/validator.rs @@ -30,7 +30,7 @@ impl ValidatorService for ValidatorServiceInstance { trace!(self.log, "RPC request"; "endpoint" => "GetValidatorDuties", "epoch" => req.get_epoch()); let spec = T::EthSpec::spec(); - let state = &self.chain.head().beacon_state; + let state = &self.chain.current_state(); let epoch = Epoch::from(req.get_epoch()); let mut resp = GetDutiesResponse::new(); let resp_validators = resp.mut_active_validators(); From c8ba44b0b5566f60190e232941b8161d74a9e4de Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 12:56:35 +1000 Subject: [PATCH 35/76] Create db-level alias for genesis block --- beacon_node/beacon_chain/src/beacon_chain.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index bac8f74a7d..e0e3ffd4d8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -127,6 +127,14 @@ impl BeaconChain { let block_root = genesis_block.block_header().canonical_root(); store.put(&block_root, &genesis_block)?; + // Store the genesis block under the `0x00..00` hash too. + // + // The spec declares that for fork choice, the `ZERO_HASH` should alias to the genesis + // block. See: + // + // github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_fork-choice.md#implementation-notes + store.put(&spec.zero_hash, &genesis_block)?; + let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), block_root, From 29c5f297a66dab92bc138c13fd7410cf40d70af1 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 14:43:08 +1000 Subject: [PATCH 36/76] Add database size metric --- beacon_node/client/src/lib.rs | 1 + beacon_node/http_server/src/key.rs | 7 +++++++ beacon_node/http_server/src/lib.rs | 7 +++++-- beacon_node/http_server/src/metrics.rs | 11 +++++++++-- .../http_server/src/metrics/local_metrics.rs | 15 ++++++++++++++- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 4824d40ad2..d29d00ad49 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -128,6 +128,7 @@ where executor, network_send, beacon_chain.clone(), + config.db_name.clone(), metrics_registry, &log, )) diff --git a/beacon_node/http_server/src/key.rs b/beacon_node/http_server/src/key.rs index b84c5f6857..a69da6747f 100644 --- a/beacon_node/http_server/src/key.rs +++ b/beacon_node/http_server/src/key.rs @@ -3,6 +3,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use iron::typemap::Key; use prometheus::Registry; use std::marker::PhantomData; +use std::path::PathBuf; use std::sync::Arc; pub struct BeaconChainKey { @@ -24,3 +25,9 @@ pub struct LocalMetricsKey; impl Key for LocalMetricsKey { type Value = LocalMetrics; } + +pub struct DBPathKey; + +impl Key for DBPathKey { + type Value = PathBuf; +} diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index cc54b6e17b..fb94348265 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -9,6 +9,7 @@ use network::NetworkMessage; use prometheus::Registry; use router::Router; use slog::{info, o, warn}; +use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::TaskExecutor; @@ -32,6 +33,7 @@ impl Default for HttpServerConfig { /// Build the `iron` HTTP server, defining the core routes. pub fn create_iron_http_server( beacon_chain: Arc>, + db_path: PathBuf, metrics_registry: Registry, ) -> Iron { let mut router = Router::new(); @@ -39,7 +41,7 @@ pub fn create_iron_http_server( // A `GET` request to `/metrics` is handled by the `metrics` module. router.get( "/metrics", - metrics::build_handler(beacon_chain.clone(), metrics_registry), + metrics::build_handler(beacon_chain.clone(), db_path, metrics_registry), "metrics", ); @@ -55,6 +57,7 @@ pub fn start_service( executor: &TaskExecutor, _network_chan: crossbeam_channel::Sender, beacon_chain: Arc>, + db_path: PathBuf, metrics_registry: Registry, log: &slog::Logger, ) -> exit_future::Signal { @@ -66,7 +69,7 @@ pub fn start_service( let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); // Create an `iron` http, without starting it yet. - let iron = create_iron_http_server(beacon_chain, metrics_registry); + let iron = create_iron_http_server(beacon_chain, db_path, metrics_registry); // Create a HTTP server future. // diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs index 7e716d5d7c..1b1ed1f3d4 100644 --- a/beacon_node/http_server/src/metrics.rs +++ b/beacon_node/http_server/src/metrics.rs @@ -1,5 +1,5 @@ use crate::{ - key::{BeaconChainKey, LocalMetricsKey, MetricsRegistryKey}, + key::{BeaconChainKey, DBPathKey, LocalMetricsKey, MetricsRegistryKey}, map_persistent_err_to_500, }; use beacon_chain::{BeaconChain, BeaconChainTypes}; @@ -7,6 +7,7 @@ use iron::prelude::*; use iron::{status::Status, Handler, IronResult, Request, Response}; use persistent::Read; use prometheus::{Encoder, Registry, TextEncoder}; +use std::path::PathBuf; use std::sync::Arc; pub use local_metrics::LocalMetrics; @@ -16,6 +17,7 @@ mod local_metrics; /// Yields a handler for the metrics endpoint. pub fn build_handler( beacon_chain: Arc>, + db_path: PathBuf, metrics_registry: Registry, ) -> impl Handler { let mut chain = Chain::new(handle_metrics::); @@ -26,6 +28,7 @@ pub fn build_handler( chain.link(Read::>::both(beacon_chain)); chain.link(Read::::both(metrics_registry)); chain.link(Read::::both(local_metrics)); + chain.link(Read::::both(db_path)); chain } @@ -46,8 +49,12 @@ fn handle_metrics(req: &mut Request) -> IronResul .get::>() .map_err(map_persistent_err_to_500)?; + let db_path = req + .get::>() + .map_err(map_persistent_err_to_500)?; + // Update metrics that are calculated on each scrape. - local_metrics.update(&beacon_chain); + local_metrics.update(&beacon_chain, &db_path); let mut buffer = vec![]; let encoder = TextEncoder::new(); diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs index 68e02b0a11..e2b0e6513b 100644 --- a/beacon_node/http_server/src/metrics/local_metrics.rs +++ b/beacon_node/http_server/src/metrics/local_metrics.rs @@ -1,6 +1,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use prometheus::{IntGauge, Opts, Registry}; use slot_clock::SlotClock; +use std::path::PathBuf; +use std::fs::File; use types::Slot; // If set to `true` will iterate and sum the balances of all validators in the state for each @@ -14,6 +16,7 @@ pub struct LocalMetrics { justified_epoch: IntGauge, finalized_epoch: IntGauge, validator_balances_sum: IntGauge, + database_size: IntGauge, } impl LocalMetrics { @@ -44,6 +47,10 @@ impl LocalMetrics { let opts = Opts::new("validator_balances_sum", "sum_of_all_validator_balances"); IntGauge::with_opts(opts)? }, + database_size: { + let opts = Opts::new("database_size", "size_of_on_disk_db_in_mb"); + IntGauge::with_opts(opts)? + }, }) } @@ -55,12 +62,13 @@ impl LocalMetrics { registry.register(Box::new(self.finalized_epoch.clone()))?; registry.register(Box::new(self.justified_epoch.clone()))?; registry.register(Box::new(self.validator_balances_sum.clone()))?; + registry.register(Box::new(self.database_size.clone()))?; Ok(()) } /// Update the metrics in `self` to the latest values. - pub fn update(&self, beacon_chain: &BeaconChain) { + pub fn update(&self, beacon_chain: &BeaconChain, db_path: &PathBuf) { let state = &beacon_chain.head().beacon_state; let present_slot = beacon_chain @@ -77,5 +85,10 @@ impl LocalMetrics { if SHOULD_SUM_VALIDATOR_BALANCES { self.validator_balances_sum.set(state.validator_balances.iter().sum::() as i64); } + let db_size = File::open(db_path) + .and_then(|f| f.metadata()) + .and_then(|m| Ok(m.len())) + .unwrap_or(0); + self.database_size.set(db_size as i64); } } From 244ffbc6048f1eeaef243611aae5fc756d451a53 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 15:02:19 +1000 Subject: [PATCH 37/76] Store genesis root in beacon chain, fix fork bug --- beacon_node/beacon_chain/src/beacon_chain.rs | 30 +++++++++++-------- .../src/persisted_beacon_chain.rs | 3 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e0e3ffd4d8..64cd8ddfc8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -104,6 +104,8 @@ pub struct BeaconChain { /// skip slot if no block is recieved. This is effectively a cache that avoids repeating calls /// to `per_slot_processing`. state: RwLock>, + /// The root of the genesis block. + genesis_block_root: Hash256, /// A state-machine that is updated with information from the network and chooses a canonical /// head block. pub fork_choice: RwLock, @@ -124,20 +126,12 @@ impl BeaconChain { let state_root = genesis_state.canonical_root(); store.put(&state_root, &genesis_state)?; - let block_root = genesis_block.block_header().canonical_root(); - store.put(&block_root, &genesis_block)?; - - // Store the genesis block under the `0x00..00` hash too. - // - // The spec declares that for fork choice, the `ZERO_HASH` should alias to the genesis - // block. See: - // - // github.com/ethereum/eth2.0-specs/blob/dev/specs/core/0_fork-choice.md#implementation-notes - store.put(&spec.zero_hash, &genesis_block)?; + let genesis_block_root = genesis_block.block_header().canonical_root(); + store.put(&genesis_block_root, &genesis_block)?; let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), - block_root, + genesis_block_root, genesis_state.clone(), state_root, )); @@ -150,6 +144,7 @@ impl BeaconChain { op_pool: OperationPool::new(), state: RwLock::new(genesis_state), canonical_head, + genesis_block_root, fork_choice: RwLock::new(fork_choice), metrics: Metrics::new()?, }) @@ -181,6 +176,7 @@ impl BeaconChain { canonical_head: RwLock::new(p.canonical_head), state: RwLock::new(p.state), fork_choice: RwLock::new(fork_choice), + genesis_block_root: p.genesis_block_root, metrics: Metrics::new()?, })) } @@ -189,6 +185,7 @@ impl BeaconChain { pub fn persist(&self) -> Result<(), Error> { let p: PersistedBeaconChain = PersistedBeaconChain { canonical_head: self.canonical_head.read().clone(), + genesis_block_root: self.genesis_block_root, state: self.state.read().clone(), }; @@ -823,11 +820,20 @@ impl BeaconChain { // Start fork choice metrics timer. let timer = self.metrics.fork_choice_times.start_timer(); + let justified_root = { + let root = self.head().beacon_state.current_justified_root; + if root == T::EthSpec::spec().zero_hash { + self.genesis_block_root + } else { + root + } + }; + // Determine the root of the block that is the head of the chain. let beacon_block_root = self .fork_choice .write() - .find_head(&self.head().beacon_state.current_justified_root, &T::EthSpec::spec())?; + .find_head(&justified_root, &T::EthSpec::spec())?; // End fork choice metrics timer. timer.observe_duration(); diff --git a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs index 2a18451b63..f5bdfdee15 100644 --- a/beacon_node/beacon_chain/src/persisted_beacon_chain.rs +++ b/beacon_node/beacon_chain/src/persisted_beacon_chain.rs @@ -2,7 +2,7 @@ use crate::{BeaconChainTypes, CheckPoint}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use store::{DBColumn, Error as StoreError, StoreItem}; -use types::BeaconState; +use types::{BeaconState, Hash256}; /// 32-byte key for accessing the `PersistedBeaconChain`. pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA"; @@ -11,6 +11,7 @@ pub const BEACON_CHAIN_DB_KEY: &str = "PERSISTEDBEACONCHAINPERSISTEDBEA"; pub struct PersistedBeaconChain { pub canonical_head: CheckPoint, // TODO: operations pool. + pub genesis_block_root: Hash256, pub state: BeaconState, } From 997095fc434cc447400c22e4791bee85e66fd28c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 1 Jun 2019 15:29:13 +1000 Subject: [PATCH 38/76] Add attestations per block metric --- beacon_node/beacon_chain/src/beacon_chain.rs | 32 +------------------- beacon_node/beacon_chain/src/metrics.rs | 9 ++++++ 2 files changed, 10 insertions(+), 31 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 64cd8ddfc8..63a00747ed 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -345,37 +345,6 @@ impl BeaconChain { Ok(()) } - /* - /// Updates the canonical `BeaconState` with the supplied state. - /// - /// Advances the chain forward to the present slot. This method is better than just setting - /// state and calling `catchup_state` as it will not result in an old state being installed and - /// then having it iteratively updated -- in such a case it's possible for another thread to - /// find the state at an old slot. - /// - /// Also persists the `BeaconChain` to the store, in the case the client does not exit - /// gracefully. - fn update_state(&self, mut state: BeaconState) -> Result<(), Error> { - let present_slot = match self.slot_clock.present_slot() { - Ok(Some(slot)) => slot, - _ => return Err(Error::UnableToReadSlot), - }; - - // If required, transition the new state to the present slot. - for _ in state.slot.as_u64()..present_slot.as_u64() { - per_slot_processing(&mut state, &T::EthSpec::spec())?; - } - - state.build_all_caches(&T::EthSpec::spec())?; - - *self.state.write() = state; - - self.persist()?; - - Ok(()) - } - */ - /// Returns a read-lock guarded `BeaconState` which is the `canonical_head` that has been /// updated to match the current slot clock. pub fn current_state(&self) -> RwLockReadGuard> { @@ -732,6 +701,7 @@ impl BeaconChain { self.fork_choice()?; self.metrics.block_processing_successes.inc(); + self.metrics.operations_per_block_attestation.observe(block.body.attestations.len() as f64); timer.observe_duration(); Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index a3c8553a19..fa1718ebfb 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -18,6 +18,7 @@ pub struct Metrics { pub fork_choice_changed_head: IntCounter, pub fork_choice_reorg_count: IntCounter, pub fork_choice_times: Histogram, + pub operations_per_block_attestation: Histogram, } impl Metrics { @@ -108,6 +109,13 @@ impl Metrics { let opts = HistogramOpts::new("fork_choice_time", "total_time_to_run_fork_choice"); Histogram::with_opts(opts)? }, + operations_per_block_attestation: { + let opts = HistogramOpts::new( + "operations_per_block_attestation", + "count_of_attestations_per_block", + ); + Histogram::with_opts(opts)? + }, }) } @@ -128,6 +136,7 @@ impl Metrics { registry.register(Box::new(self.fork_choice_changed_head.clone()))?; registry.register(Box::new(self.fork_choice_reorg_count.clone()))?; registry.register(Box::new(self.fork_choice_times.clone()))?; + registry.register(Box::new(self.operations_per_block_attestation.clone()))?; Ok(()) } From c25ede42ebe147d12b59095cf08059226837f5aa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 3 Jun 2019 17:26:40 +1000 Subject: [PATCH 39/76] Add benches, examples to `fork_choice` --- eth2/fork_choice/Cargo.toml | 5 ++ eth2/fork_choice/benches/benches.rs | 75 ++++++++++++++++ eth2/fork_choice/examples/example.rs | 40 +++++++++ eth2/fork_choice/src/lib.rs | 1 + eth2/fork_choice/src/optimized_lmd_ghost.rs | 7 +- eth2/fork_choice/src/test_utils.rs | 94 +++++++++++++++++++++ 6 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 eth2/fork_choice/benches/benches.rs create mode 100644 eth2/fork_choice/examples/example.rs create mode 100644 eth2/fork_choice/src/test_utils.rs diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml index f2e6825ed9..e37e415e49 100644 --- a/eth2/fork_choice/Cargo.toml +++ b/eth2/fork_choice/Cargo.toml @@ -4,6 +4,10 @@ version = "0.1.0" authors = ["Age Manning "] edition = "2018" +[[bench]] +name = "benches" +harness = false + [dependencies] store = { path = "../../beacon_node/store" } ssz = { path = "../utils/ssz" } @@ -12,6 +16,7 @@ log = "0.4.6" bit-vec = "0.5.0" [dev-dependencies] +criterion = "0.2" hex = "0.3.2" yaml-rust = "0.4.2" bls = { path = "../utils/bls" } diff --git a/eth2/fork_choice/benches/benches.rs b/eth2/fork_choice/benches/benches.rs new file mode 100644 index 0000000000..065cde6552 --- /dev/null +++ b/eth2/fork_choice/benches/benches.rs @@ -0,0 +1,75 @@ +use criterion::Criterion; +use criterion::{criterion_group, criterion_main, Benchmark}; +use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; +use std::sync::Arc; +use store::MemoryStore; +use types::{ChainSpec, EthSpec, FoundationEthSpec}; + +pub type TestedForkChoice = OptimizedLMDGhost; +pub type TestedEthSpec = FoundationEthSpec; + +/// Helper function to setup a builder and spec. +fn setup( + validator_count: usize, + chain_length: usize, +) -> ( + TestingForkChoiceBuilder, + ChainSpec, +) { + let store = MemoryStore::open(); + let builder: TestingForkChoiceBuilder = + TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); + let spec = TestedEthSpec::spec(); + + (builder, spec) +} + +/// Benches adding blocks to fork_choice. +fn add_block(c: &mut Criterion) { + let validator_count = 16; + let chain_length = 100; + + let (builder, spec) = setup(validator_count, chain_length); + + c.bench( + &format!("{}_blocks", chain_length), + Benchmark::new("add_blocks", move |b| { + b.iter(|| { + let mut fc = builder.build::>(); + for (root, block) in builder.chain.iter().skip(1) { + fc.add_block(block, root, &spec).unwrap(); + } + }) + }) + .sample_size(10), + ); +} + +/// Benches fork choice head finding. +fn find_head(c: &mut Criterion) { + let validator_count = 16; + let chain_length = 64 * 2; + + let (builder, spec) = setup(validator_count, chain_length); + + let mut fc = builder.build::>(); + for (root, block) in builder.chain.iter().skip(1) { + fc.add_block(block, root, &spec).unwrap(); + } + + let head_root = builder.chain.last().unwrap().0; + for i in 0..validator_count { + fc.add_attestation(i as u64, &head_root, &spec).unwrap(); + } + + c.bench( + &format!("{}_blocks", chain_length), + Benchmark::new("find_head", move |b| { + b.iter(|| fc.find_head(&builder.genesis_root(), &spec).unwrap()) + }) + .sample_size(10), + ); +} + +criterion_group!(benches, add_block, find_head); +criterion_main!(benches); diff --git a/eth2/fork_choice/examples/example.rs b/eth2/fork_choice/examples/example.rs new file mode 100644 index 0000000000..927cf23b76 --- /dev/null +++ b/eth2/fork_choice/examples/example.rs @@ -0,0 +1,40 @@ +use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; +use std::sync::Arc; +use store::{MemoryStore, Store}; +use types::{BeaconBlock, ChainSpec, EthSpec, FoundationEthSpec, Hash256}; + +fn main() { + let validator_count = 16; + let chain_length = 100; + let repetitions = 50; + + let store = MemoryStore::open(); + let builder: TestingForkChoiceBuilder = + TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); + + let fork_choosers: Vec> = (0..repetitions) + .into_iter() + .map(|_| builder.build()) + .collect(); + + let spec = &FoundationEthSpec::spec(); + + println!("Running {} times...", repetitions); + for fc in fork_choosers { + do_thing(fc, &builder.chain, builder.genesis_root(), spec); + } +} + +#[inline(never)] +fn do_thing, S: Store>( + mut fc: F, + chain: &[(Hash256, BeaconBlock)], + genesis_root: Hash256, + spec: &ChainSpec, +) { + for (root, block) in chain.iter().skip(1) { + fc.add_block(block, root, spec).unwrap(); + } + + let _head = fc.find_head(&genesis_root, spec).unwrap(); +} diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs index ce53c1051c..f4a1fa5cb6 100644 --- a/eth2/fork_choice/src/lib.rs +++ b/eth2/fork_choice/src/lib.rs @@ -20,6 +20,7 @@ pub mod bitwise_lmd_ghost; pub mod longest_chain; pub mod optimized_lmd_ghost; pub mod slow_lmd_ghost; +pub mod test_utils; use std::sync::Arc; use store::Error as DBError; diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index ada8ce9cb8..d3f1598766 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -69,12 +69,11 @@ impl OptimizedLMDGhost { let active_validator_indices = current_state.get_active_validator_indices(block_slot.epoch(spec.slots_per_epoch)); + let validator_balances = ¤t_state.validator_balances; for index in active_validator_indices { - let balance = std::cmp::min( - current_state.validator_balances[index], - spec.max_deposit_amount, - ) / spec.fork_choice_balance_increment; + let balance = std::cmp::min(validator_balances[index], spec.max_deposit_amount) + / spec.fork_choice_balance_increment; if balance > 0 { if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { *latest_votes.entry(*target).or_insert_with(|| 0) += balance; diff --git a/eth2/fork_choice/src/test_utils.rs b/eth2/fork_choice/src/test_utils.rs new file mode 100644 index 0000000000..76264fcb63 --- /dev/null +++ b/eth2/fork_choice/src/test_utils.rs @@ -0,0 +1,94 @@ +use crate::ForkChoice; +use std::marker::PhantomData; +use std::sync::Arc; +use store::Store; +use types::{ + test_utils::{SeedableRng, TestRandom, TestingBeaconStateBuilder, XorShiftRng}, + BeaconBlock, BeaconState, EthSpec, FoundationEthSpec, Hash256, Keypair, +}; + +/// Creates a chain of blocks and produces `ForkChoice` instances with pre-filled stores. +pub struct TestingForkChoiceBuilder { + store: Arc, + pub chain: Vec<(Hash256, BeaconBlock)>, + _phantom: PhantomData, +} + +impl TestingForkChoiceBuilder { + pub fn new(validator_count: usize, chain_length: usize, store: Arc) -> Self { + let chain = get_chain_of_blocks::( + chain_length, + validator_count, + store.clone(), + ); + + Self { + store, + chain, + _phantom: PhantomData, + } + } + + pub fn genesis_root(&self) -> Hash256 { + self.chain[0].0 + } + + /// Return a new `ForkChoice` instance with a chain stored in it's `Store`. + pub fn build>(&self) -> F { + F::new(self.store.clone()) + } +} + +fn get_state(validator_count: usize) -> BeaconState { + let spec = &T::spec(); + + let builder: TestingBeaconStateBuilder = + TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec); + let (state, _keypairs) = builder.build(); + state +} + +/// Generates a chain of blocks of length `len`. +/// +/// Creates a `BeaconState` for the block and stores it in `Store`, along with the block. +/// +/// Returns the chain of blocks. +fn get_chain_of_blocks( + len: usize, + validator_count: usize, + store: Arc, +) -> Vec<(Hash256, BeaconBlock)> { + let spec = T::spec(); + let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![]; + let mut unique_hashes = (0..).into_iter().map(|i| Hash256::from(i)); + let mut random_block = BeaconBlock::random_for_test(&mut XorShiftRng::from_seed([42; 16])); + random_block.previous_block_root = Hash256::zero(); + let beacon_state = get_state::(validator_count); + + for i in 0..len { + let slot = spec.genesis_slot + i as u64; + + // Generate and store the state. + let mut state = beacon_state.clone(); + state.slot = slot; + let state_root = unique_hashes.next().unwrap(); + store.put(&state_root, &state).unwrap(); + + // Generate the block. + let mut block = random_block.clone(); + block.slot = slot; + block.state_root = state_root; + + // Chain all the blocks to their parents. + if i > 0 { + block.previous_block_root = blocks_and_roots[i - 1].0; + } + + // Store the block. + let block_root = unique_hashes.next().unwrap(); + store.put(&block_root, &block).unwrap(); + blocks_and_roots.push((block_root, block)); + } + + blocks_and_roots +} From 82202a7765091be89032749906a3951ab51e6d1c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 09:37:40 +1000 Subject: [PATCH 40/76] Update components to suit v0.6.1 API --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 ++-- .../http_server/src/metrics/local_metrics.rs | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 76de2d431f..77011a6d59 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -726,7 +726,7 @@ impl BeaconChain { let mut state = self.state.read().clone(); - state.build_epoch_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; + state.build_committee_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; trace!("Finding attestations for new block..."); @@ -742,7 +742,7 @@ impl BeaconChain { slot: state.slot, previous_block_root, state_root: Hash256::zero(), // Updated after the state is calculated. - signature: T::EthSpec::spec().empty_signature.clone(), // To be completed by a validator. + signature: Signature::empty_signature(), // To be completed by a validator. body: BeaconBlockBody { randao_reveal, eth1_data: Eth1Data { diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs index e2b0e6513b..cd7de90f6f 100644 --- a/beacon_node/http_server/src/metrics/local_metrics.rs +++ b/beacon_node/http_server/src/metrics/local_metrics.rs @@ -1,8 +1,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use prometheus::{IntGauge, Opts, Registry}; use slot_clock::SlotClock; -use std::path::PathBuf; use std::fs::File; +use std::path::PathBuf; use types::Slot; // If set to `true` will iterate and sum the balances of all validators in the state for each @@ -79,11 +79,15 @@ impl LocalMetrics { self.present_slot.set(present_slot.as_u64() as i64); self.best_slot.set(state.slot.as_u64() as i64); - self.validator_count.set(state.validator_registry.len() as i64); - self.justified_epoch.set(state.current_justified_epoch.as_u64() as i64); - self.finalized_epoch.set(state.finalized_epoch.as_u64() as i64); + self.validator_count + .set(state.validator_registry.len() as i64); + self.justified_epoch + .set(state.current_justified_epoch.as_u64() as i64); + self.finalized_epoch + .set(state.finalized_epoch.as_u64() as i64); if SHOULD_SUM_VALIDATOR_BALANCES { - self.validator_balances_sum.set(state.validator_balances.iter().sum::() as i64); + self.validator_balances_sum + .set(state.balances.iter().sum::() as i64); } let db_size = File::open(db_path) .and_then(|f| f.metadata()) From 7005234fd1f4b1c15637623286269521c4098460 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 09:38:13 +1000 Subject: [PATCH 41/76] Run rust fmt --- eth2/fork_choice/tests/tests.rs | 2 +- validator_client/src/error.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index a7d2ec7aae..9270571ca3 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -196,7 +196,7 @@ fn load_test_cases_from_yaml(file_path: &str) -> Vec { fn setup_inital_state( // fork_choice_algo: &ForkChoiceAlgorithm, - num_validators: usize, + num_validators: usize ) -> (T, Arc, Hash256) where T: ForkChoice, diff --git a/validator_client/src/error.rs b/validator_client/src/error.rs index 7740c4f2b6..97500f900b 100644 --- a/validator_client/src/error.rs +++ b/validator_client/src/error.rs @@ -1,8 +1,6 @@ use slot_clock; -use error_chain::{ - error_chain -}; +use error_chain::error_chain; error_chain! { links { } From 7a2ab2e9aa931ebad5db9af30b392ecdebdc9708 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 12:03:54 +1000 Subject: [PATCH 42/76] Add support for SSZ union type via Option --- eth2/utils/ssz/src/decode.rs | 6 +++ eth2/utils/ssz/src/decode/impls.rs | 30 ++++++++++++++ eth2/utils/ssz/src/encode.rs | 7 ++++ eth2/utils/ssz/src/encode/impls.rs | 36 +++++++++++++++++ eth2/utils/ssz/tests/tests.rs | 63 +++++++++++++++++++++++++++++- 5 files changed, 141 insertions(+), 1 deletion(-) diff --git a/eth2/utils/ssz/src/decode.rs b/eth2/utils/ssz/src/decode.rs index 6934f1708c..77144092b1 100644 --- a/eth2/utils/ssz/src/decode.rs +++ b/eth2/utils/ssz/src/decode.rs @@ -218,6 +218,12 @@ impl<'a> SszDecoder<'a> { } } +/// Reads a `BYTES_PER_LENGTH_OFFSET`-byte union index from `bytes`, where `bytes.len() >= +/// BYTES_PER_LENGTH_OFFSET`. +pub fn read_union_index(bytes: &[u8]) -> Result { + read_offset(bytes) +} + /// Reads a `BYTES_PER_LENGTH_OFFSET`-byte length from `bytes`, where `bytes.len() >= /// BYTES_PER_LENGTH_OFFSET`. fn read_offset(bytes: &[u8]) -> Result { diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 3e567b9674..7f1ad26ee4 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -62,6 +62,36 @@ impl Decode for bool { } } +/// The SSZ union type. +impl Decode for Option { + fn is_ssz_fixed_len() -> bool { + false + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + if bytes.len() < BYTES_PER_LENGTH_OFFSET { + return Err(DecodeError::InvalidByteLength { + len: bytes.len(), + expected: BYTES_PER_LENGTH_OFFSET, + }); + } + + let (index_bytes, value_bytes) = bytes.split_at(BYTES_PER_LENGTH_OFFSET); + + let index = read_union_index(index_bytes)?; + if index == 0 { + Ok(None) + } else if index == 1 { + Ok(Some(T::from_ssz_bytes(value_bytes)?)) + } else { + Err(DecodeError::BytesInvalid(format!( + "{} is not a valid union index for Option", + index + ))) + } + } +} + impl Decode for H256 { fn is_ssz_fixed_len() -> bool { true diff --git a/eth2/utils/ssz/src/encode.rs b/eth2/utils/ssz/src/encode.rs index 257ece2a25..6ceb08debc 100644 --- a/eth2/utils/ssz/src/encode.rs +++ b/eth2/utils/ssz/src/encode.rs @@ -126,6 +126,13 @@ impl<'a> SszEncoder<'a> { } } +/// Encode `index` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length. +/// +/// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised. +pub fn encode_union_index(index: usize) -> Vec { + encode_length(index) +} + /// Encode `len` as a little-endian byte vec of `BYTES_PER_LENGTH_OFFSET` length. /// /// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised. diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 0d6891c5e2..8dbcf17074 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -25,6 +25,23 @@ impl_encodable_for_uint!(u32, 32); impl_encodable_for_uint!(u64, 64); impl_encodable_for_uint!(usize, 64); +/// The SSZ "union" type. +impl Encode for Option { + fn is_ssz_fixed_len() -> bool { + false + } + + fn ssz_append(&self, buf: &mut Vec) { + match self { + None => buf.append(&mut encode_union_index(0)), + Some(t) => { + buf.append(&mut encode_union_index(1)); + t.ssz_append(buf); + } + } + } +} + impl Encode for Vec { fn is_ssz_fixed_len() -> bool { false @@ -168,6 +185,25 @@ mod tests { ); } + #[test] + fn ssz_encode_option_u16() { + assert_eq!(Some(65535_u16).as_ssz_bytes(), vec![1, 0, 0, 0, 255, 255]); + + let none: Option = None; + assert_eq!(none.as_ssz_bytes(), vec![0, 0, 0, 0]); + } + + #[test] + fn ssz_encode_option_vec_u16() { + assert_eq!( + Some(vec![0_u16, 1]).as_ssz_bytes(), + vec![1, 0, 0, 0, 0, 0, 1, 0] + ); + + let none: Option> = None; + assert_eq!(none.as_ssz_bytes(), vec![0, 0, 0, 0]); + } + #[test] fn ssz_encode_u8() { assert_eq!(0_u8.as_ssz_bytes(), vec![0]); diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index ed318d9244..06c6898171 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -276,7 +276,7 @@ mod round_trip { fn offsets_decreasing() { let bytes = vec![ // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 - // | offset | ofset | offset | variable + // | offset | offset | offset | variable 01, 00, 14, 00, 00, 00, 15, 00, 00, 00, 14, 00, 00, 00, 00, 00, ]; @@ -285,4 +285,65 @@ mod round_trip { Err(DecodeError::OutOfBoundsByte { i: 14 }) ); } + + #[derive(Debug, PartialEq, Encode, Decode)] + struct TwoVariableLenOptions { + a: u16, + b: Option, + c: Option>, + d: Option>, + } + + #[test] + fn two_variable_len_options_encoding() { + let s = TwoVariableLenOptions { + a: 42, + b: None, + c: Some(vec![0]), + d: None, + }; + + let bytes = vec![ + // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 + // | option | offset | offset | option = vec![ + TwoVariableLenOptions { + a: 42, + b: Some(12), + c: Some(vec![0]), + d: Some(vec![1]), + }, + TwoVariableLenOptions { + a: 42, + b: Some(12), + c: Some(vec![0]), + d: None, + }, + TwoVariableLenOptions { + a: 42, + b: None, + c: Some(vec![0]), + d: None, + }, + TwoVariableLenOptions { + a: 42, + b: None, + c: None, + d: None, + }, + ]; + + round_trip(vec); + } } From 45fb11b20894abd8a50e758c9a723478a76eb5dc Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 12:24:13 +1000 Subject: [PATCH 43/76] Impl ssz enc/dec for `NonZeroUsize` --- eth2/utils/ssz/src/decode/impls.rs | 23 +++++++++++++++++++++++ eth2/utils/ssz/src/encode/impls.rs | 15 +++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 7f1ad26ee4..7dd7fac3c1 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -1,5 +1,6 @@ use super::*; use ethereum_types::{H256, U128, U256}; +use core::num::NonZeroUsize; macro_rules! impl_decodable_for_uint { ($type: ident, $bit_size: expr) => { @@ -62,6 +63,28 @@ impl Decode for bool { } } +impl Decode for NonZeroUsize { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + let x = usize::from_ssz_bytes(bytes)?; + + if x == 0 { + Err(DecodeError::BytesInvalid("NonZeroUsize cannot be zero.".to_string())) + } else { + // `unwrap` is safe here as `NonZeroUsize::new()` succeeds if `x > 0` and this path + // never executes when `x == 0`. + Ok(NonZeroUsize::new(x).unwrap()) + } + } +} + /// The SSZ union type. impl Decode for Option { fn is_ssz_fixed_len() -> bool { diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 8dbcf17074..04492a1f2d 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -1,4 +1,5 @@ use super::*; +use core::num::NonZeroUsize; use ethereum_types::{H256, U128, U256}; macro_rules! impl_encodable_for_uint { @@ -80,6 +81,20 @@ impl Encode for bool { } } +impl Encode for NonZeroUsize { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + self.get().ssz_append(buf) + } +} + impl Encode for H256 { fn is_ssz_fixed_len() -> bool { true From f530f5a848c539c4658f8e717f5800c310c0ab69 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 13:13:38 +1000 Subject: [PATCH 44/76] Ensure committees are built for block processing --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 77011a6d59..944079aa01 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -672,6 +672,8 @@ impl BeaconChain { } } + state.build_committee_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; + // Apply the received block to its parent state (which has been transitioned into this // slot). if let Err(e) = per_block_processing(&mut state, &block, &T::EthSpec::spec()) { From 67fdb4a7fb42f210acb89d96c9d5f9b3d0213ec9 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 13:13:58 +1000 Subject: [PATCH 45/76] Store beacon state committee cache in DB --- beacon_node/store/src/impls.rs | 16 +---- beacon_node/store/src/impls/beacon_state.rs | 64 +++++++++++++++++++ eth2/types/src/beacon_state.rs | 3 +- .../types/src/beacon_state/committee_cache.rs | 3 +- 4 files changed, 70 insertions(+), 16 deletions(-) create mode 100644 beacon_node/store/src/impls/beacon_state.rs diff --git a/beacon_node/store/src/impls.rs b/beacon_node/store/src/impls.rs index 91f8d52de6..418fcade1e 100644 --- a/beacon_node/store/src/impls.rs +++ b/beacon_node/store/src/impls.rs @@ -1,6 +1,8 @@ use crate::*; use ssz::{Decode, Encode}; +mod beacon_state; + impl StoreItem for BeaconBlock { fn db_column() -> DBColumn { DBColumn::BeaconBlock @@ -14,17 +16,3 @@ impl StoreItem for BeaconBlock { Self::from_ssz_bytes(bytes).map_err(Into::into) } } - -impl StoreItem for BeaconState { - fn db_column() -> DBColumn { - DBColumn::BeaconState - } - - fn as_store_bytes(&self) -> Vec { - self.as_ssz_bytes() - } - - fn from_store_bytes(bytes: &mut [u8]) -> Result { - Self::from_ssz_bytes(bytes).map_err(Into::into) - } -} diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs new file mode 100644 index 0000000000..40d2cacb80 --- /dev/null +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -0,0 +1,64 @@ +use crate::*; +use ssz::{Decode, DecodeError, Encode}; +use ssz_derive::{Decode, Encode}; +use std::convert::TryInto; +use types::beacon_state::{CACHED_EPOCHS, CommitteeCache}; + +/// A container for storing `BeaconState` components. +#[derive(Encode, Decode)] +struct StorageContainer { + state_bytes: Vec, + committee_caches_bytes: Vec>, +} + +impl StorageContainer { + /// Create a new instance for storing a `BeaconState`. + pub fn new(state: &BeaconState) -> Self { + let mut committee_caches_bytes = vec![]; + + for cache in state.committee_caches[..].iter() { + committee_caches_bytes.push(cache.as_ssz_bytes()); + } + + Self { + state_bytes: state.as_ssz_bytes(), + committee_caches_bytes, + } + } +} + +impl TryInto> for StorageContainer { + type Error = Error; + + fn try_into(self) -> Result, Error> { + let mut state: BeaconState = BeaconState::from_ssz_bytes(&self.state_bytes)?; + + for i in 0..CACHED_EPOCHS { + let bytes = &self.committee_caches_bytes.get(i).ok_or_else(|| { + Error::SszDecodeError(DecodeError::BytesInvalid( + "Insufficient committees for BeaconState".to_string(), + )) + })?; + + state.committee_caches[i] = CommitteeCache::from_ssz_bytes(bytes)?; + } + + Ok(state) + } +} + +impl StoreItem for BeaconState { + fn db_column() -> DBColumn { + DBColumn::BeaconState + } + + fn as_store_bytes(&self) -> Vec { + let container = StorageContainer::new(self); + container.as_ssz_bytes() + } + + fn from_store_bytes(bytes: &mut [u8]) -> Result { + let container = StorageContainer::from_ssz_bytes(bytes)?; + container.try_into() + } +} diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 4e510940f0..471b82b20b 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -1,4 +1,4 @@ -use self::committee_cache::{get_active_validator_indices, CommitteeCache}; +use self::committee_cache::get_active_validator_indices; use self::exit_cache::ExitCache; use crate::test_utils::TestRandom; use crate::*; @@ -15,6 +15,7 @@ use test_random_derive::TestRandom; use tree_hash::TreeHash; use tree_hash_derive::{CachedTreeHash, TreeHash}; +pub use self::committee_cache::CommitteeCache; pub use beacon_state_types::*; mod beacon_state_types; diff --git a/eth2/types/src/beacon_state/committee_cache.rs b/eth2/types/src/beacon_state/committee_cache.rs index 27374c3391..d1e0e70307 100644 --- a/eth2/types/src/beacon_state/committee_cache.rs +++ b/eth2/types/src/beacon_state/committee_cache.rs @@ -2,6 +2,7 @@ use super::BeaconState; use crate::*; use core::num::NonZeroUsize; use serde_derive::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; use std::ops::Range; use swap_or_not_shuffle::shuffle_list; @@ -9,7 +10,7 @@ mod tests; /// Computes and stores the shuffling for an epoch. Provides various getters to allow callers to /// read the committees for the given epoch. -#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize, Encode, Decode)] pub struct CommitteeCache { initialized_epoch: Option, shuffling: Vec, From 50dd963fddb9f5bade261d2c9d943c24afad043c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 14:16:45 +1000 Subject: [PATCH 46/76] Fix bug in epoch_processing --- .../src/per_epoch_processing/validator_statuses.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index a2ce292eb3..2cdd8700c9 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -233,7 +233,7 @@ impl ValidatorStatuses { let attestation_slot = state.get_attestation_slot(&a.data)?; let inclusion_slot = attestation_slot + a.inclusion_delay; let relative_epoch = - RelativeEpoch::from_slot(state.slot, inclusion_slot, spec.slots_per_epoch)?; + RelativeEpoch::from_slot(state.slot, attestation_slot, spec.slots_per_epoch)?; status.inclusion_info = Some(InclusionInfo { slot: inclusion_slot, distance: a.inclusion_delay, From 47128ea834620ea48eef04d21feb39d714d34ada Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 15:00:52 +1000 Subject: [PATCH 47/76] Fix bug in proposer rewards --- .../src/per_epoch_processing/validator_statuses.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index 2cdd8700c9..ed1b968d7d 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -233,12 +233,12 @@ impl ValidatorStatuses { let attestation_slot = state.get_attestation_slot(&a.data)?; let inclusion_slot = attestation_slot + a.inclusion_delay; let relative_epoch = - RelativeEpoch::from_slot(state.slot, attestation_slot, spec.slots_per_epoch)?; + RelativeEpoch::from_slot(state.slot, inclusion_slot, spec.slots_per_epoch)?; status.inclusion_info = Some(InclusionInfo { slot: inclusion_slot, distance: a.inclusion_delay, proposer_index: state.get_beacon_proposer_index( - attestation_slot, + inclusion_slot, relative_epoch, spec, )?, From f95711c15aa9449d79cba5ac820cbdf7f2a714e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 15:04:05 +1000 Subject: [PATCH 48/76] Add present_epoch metric --- beacon_node/http_server/src/metrics/local_metrics.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs index cd7de90f6f..fa69ee0c42 100644 --- a/beacon_node/http_server/src/metrics/local_metrics.rs +++ b/beacon_node/http_server/src/metrics/local_metrics.rs @@ -3,7 +3,7 @@ use prometheus::{IntGauge, Opts, Registry}; use slot_clock::SlotClock; use std::fs::File; use std::path::PathBuf; -use types::Slot; +use types::{EthSpec, Slot}; // If set to `true` will iterate and sum the balances of all validators in the state for each // scrape. @@ -11,6 +11,7 @@ const SHOULD_SUM_VALIDATOR_BALANCES: bool = true; pub struct LocalMetrics { present_slot: IntGauge, + present_epoch: IntGauge, best_slot: IntGauge, validator_count: IntGauge, justified_epoch: IntGauge, @@ -27,6 +28,10 @@ impl LocalMetrics { let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); IntGauge::with_opts(opts)? }, + present_epoch: { + let opts = Opts::new("present_epoch", "epoch_at_time_of_scrape"); + IntGauge::with_opts(opts)? + }, best_slot: { let opts = Opts::new("best_slot", "slot_of_block_at_chain_head"); IntGauge::with_opts(opts)? @@ -57,6 +62,7 @@ impl LocalMetrics { /// Registry this instance with the `registry`. pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { registry.register(Box::new(self.present_slot.clone()))?; + registry.register(Box::new(self.present_epoch.clone()))?; registry.register(Box::new(self.best_slot.clone()))?; registry.register(Box::new(self.validator_count.clone()))?; registry.register(Box::new(self.finalized_epoch.clone()))?; @@ -77,6 +83,8 @@ impl LocalMetrics { .unwrap_or_else(|_| None) .unwrap_or_else(|| Slot::new(0)); self.present_slot.set(present_slot.as_u64() as i64); + self.present_epoch + .set(present_slot.epoch(T::EthSpec::slots_per_epoch()).as_u64() as i64); self.best_slot.set(state.slot.as_u64() as i64); self.validator_count From f4b470999930762f1fe86152283f5aaf7fdd052d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 16:33:35 +1000 Subject: [PATCH 49/76] Publish attestations from RPC to P2P --- beacon_node/rpc/src/attestation.rs | 24 +++++++++++++++++++++++- beacon_node/rpc/src/lib.rs | 3 ++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index 27fcad7cda..0f585b7e75 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,6 +1,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; +use eth2_libp2p::PubsubMessage; use futures::Future; use grpcio::{RpcContext, RpcStatus, RpcStatusCode, UnarySink}; +use network::NetworkMessage; use protos::services::{ AttestationData as AttestationDataProto, ProduceAttestationDataRequest, ProduceAttestationDataResponse, PublishAttestationRequest, PublishAttestationResponse, @@ -14,6 +16,7 @@ use types::Attestation; #[derive(Clone)] pub struct AttestationServiceInstance { pub chain: Arc>, + pub network_chan: crossbeam_channel::Sender, pub log: slog::Logger, } @@ -124,7 +127,7 @@ impl AttestationService for AttestationServiceInstance { } }; - match self.chain.process_attestation(attestation) { + match self.chain.process_attestation(attestation.clone()) { Ok(_) => { // Attestation was successfully processed. info!( @@ -133,6 +136,25 @@ impl AttestationService for AttestationServiceInstance { "type" => "valid_attestation", ); + // TODO: Obtain topics from the network service properly. + let topic = types::TopicBuilder::new("beacon_chain".to_string()).build(); + let message = PubsubMessage::Attestation(attestation); + + // Publish the attestation to the p2p network via gossipsub. + self.network_chan + .send(NetworkMessage::Publish { + topics: vec![topic], + message: Box::new(message), + }) + .unwrap_or_else(|e| { + error!( + self.log, + "PublishAttestation"; + "type" => "failed to publish to gossipsub", + "error" => format!("{:?}", e) + ); + }); + resp.set_success(true); } Err(e) => { diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index 95c2e29163..f2f1b2abf3 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -46,7 +46,7 @@ pub fn start_server( let beacon_block_service = { let instance = BeaconBlockServiceInstance { chain: beacon_chain.clone(), - network_chan, + network_chan: network_chan.clone(), log: log.clone(), }; create_beacon_block_service(instance) @@ -61,6 +61,7 @@ pub fn start_server( let attestation_service = { let instance = AttestationServiceInstance { chain: beacon_chain.clone(), + network_chan, log: log.clone(), }; create_attestation_service(instance) From 8ab1d28c9f9acddd9868d3962ca869aa451dc003 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 4 Jun 2019 17:10:25 +1000 Subject: [PATCH 50/76] Fix attestation processing metrics --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 944079aa01..e8fa19ac6b 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -565,7 +565,7 @@ impl BeaconChain { .insert_attestation(attestation, &*self.state.read(), &T::EthSpec::spec()); if result.is_ok() { - self.metrics.attestation_production_successes.inc(); + self.metrics.attestation_processing_successes.inc(); } timer.observe_duration(); From df366f99dce6eb160e1ac2b1c8192830410bacc2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 5 Jun 2019 13:33:32 +1000 Subject: [PATCH 51/76] Add block iter to beacon chain --- beacon_node/beacon_chain/src/beacon_chain.rs | 13 +++ beacon_node/beacon_chain/src/iter.rs | 106 +++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + 3 files changed, 120 insertions(+) create mode 100644 beacon_node/beacon_chain/src/iter.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e8fa19ac6b..13aad81124 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2,6 +2,7 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::metrics::Metrics; use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; +use crate::iter::BlockRootsIterator; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; use operation_pool::DepositInsertStatus; @@ -226,6 +227,17 @@ impl BeaconChain { Ok(headers?) } + /// Iterate in reverse through all block roots starting from the current state, through to + /// genesis. + /// + /// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. + /// + /// Contains duplicate roots when skip slots are encountered. + pub fn iter_block_roots(&self) -> BlockRootsIterator { + BlockRootsIterator::from_state(self.store.clone(), self.state.read().clone()) + } + + /* /// Returns `count `beacon block roots, starting from `start_slot` with an /// interval of `skip` slots between each root. /// @@ -305,6 +317,7 @@ impl BeaconChain { Err(BeaconStateError::SlotOutOfBounds.into()) } } + */ /// Returns the block at the given root, if any. /// diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs new file mode 100644 index 0000000000..da81a6582f --- /dev/null +++ b/beacon_node/beacon_chain/src/iter.rs @@ -0,0 +1,106 @@ +use std::sync::Arc; +use store::Store; +use types::{BeaconState, BeaconStateError, EthSpec, Hash256, Slot}; + +/// Iterates backwards through block roots. +/// +/// Uses the `latest_block_roots` field of `BeaconState` to as the source of block roots and will +/// perform a lookup on the `Store` for a prior `BeaconState` if `latest_block_roots` has been +/// exhausted. +/// +/// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. +pub struct BlockRootsIterator { + store: Arc, + beacon_state: BeaconState, + slot: Slot, +} + +impl BlockRootsIterator { + /// Create a new iterator over all block roots in the given `beacon_state` and prior states. + pub fn from_state(store: Arc, beacon_state: BeaconState) -> Self { + Self { + slot: beacon_state.slot, + beacon_state, + store, + } + } +} + +impl Iterator for BlockRootsIterator { + type Item = Hash256; + + fn next(&mut self) -> Option { + if self.slot == 0 { + return None; + } + + let slot = self.slot - 1; + + match self.beacon_state.get_block_root(slot) { + Ok(root) => Some(*root), + Err(BeaconStateError::SlotOutOfBounds) => { + // Read a `BeaconState` from the store that has access to prior historical root. + self.beacon_state = { + // Read the earliest historic state in the current slot. + let earliest_historic_slot = + self.beacon_state.slot - Slot::from(T::slots_per_historical_root()); + + // Load the earlier state from disk. + let new_state_root = self + .beacon_state + .get_state_root(earliest_historic_slot) + .ok()?; + + let state_option = self.store.get(&new_state_root).ok()?; + state_option? + }; + + self.beacon_state.get_block_root(slot).ok().cloned() + } + _ => return None, + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use store::MemoryStore; + use types::{test_utils::TestingBeaconStateBuilder, FoundationEthSpec, Keypair}; + + fn get_state() -> BeaconState { + let builder = + TestingBeaconStateBuilder::from_single_keypair(0, &Keypair::random(), &T::spec()); + let (state, _keypairs) = builder.build(); + state + } + + #[test] + fn root_iter() { + let store = Arc::new(MemoryStore::open()); + + let mut state_a: BeaconState = get_state(); + let mut state_b: BeaconState = get_state(); + + let mut hashes = (0..).into_iter().map(|i| Hash256::from(i)); + + for root in &mut state_a.latest_block_roots[..] { + *root = hashes.next().unwrap() + } + for root in &mut state_b.latest_block_roots[..] { + *root = hashes.next().unwrap() + } + + let state_a_root = hashes.next().unwrap(); + state_b.latest_state_roots[0] = state_a_root; + store.put(&state_a_root, &state_a).unwrap(); + + let iter = BlockRootsIterator::from_state(store.clone(), state_b.clone()); + let mut collected: Vec = iter.collect(); + collected.reverse(); + + for (i, item) in collected.iter().enumerate() { + assert_eq!(*item, Hash256::from(i as u64)); + } + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c80dc4715b..bde541fce5 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,6 +1,7 @@ mod beacon_chain; mod checkpoint; mod errors; +pub mod iter; mod metrics; mod persisted_beacon_chain; From ed9f6558777ae4248df9da073c76a98a33387643 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 6 Jun 2019 00:27:17 -0400 Subject: [PATCH 52/76] Update `BeaconChain` iters --- beacon_node/beacon_chain/src/beacon_chain.rs | 17 ++++++++--- beacon_node/beacon_chain/src/iter.rs | 31 +++++++++++++++++--- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 13aad81124..d208139695 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1,8 +1,8 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; +use crate::iter::{BlockIterator, BlockRootsIterator}; use crate::metrics::Metrics; use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; -use crate::iter::BlockRootsIterator; use fork_choice::{ForkChoice, ForkChoiceError}; use log::{debug, trace}; use operation_pool::DepositInsertStatus; @@ -226,15 +226,24 @@ impl BeaconChain { Ok(headers?) } + /// Iterate in reverse (highest to lowest slot) through all blocks from the block at `slot` + /// through to the genesis block. + /// + /// Returns `None` for headers prior to genesis or when there is an error reading from `Store`. + /// + /// Contains duplicate headers when skip slots are encountered. + pub fn rev_iter_blocks(&self, slot: Slot) -> BlockIterator { + BlockIterator::new(self.store.clone(), self.state.read().clone(), slot) + } - /// Iterate in reverse through all block roots starting from the current state, through to + /// Iterates in reverse (highest to lowest slot) through all block roots from `slot` through to /// genesis. /// /// Returns `None` for roots prior to genesis or when there is an error reading from `Store`. /// /// Contains duplicate roots when skip slots are encountered. - pub fn iter_block_roots(&self) -> BlockRootsIterator { - BlockRootsIterator::from_state(self.store.clone(), self.state.read().clone()) + pub fn rev_iter_block_roots(&self, slot: Slot) -> BlockRootsIterator { + BlockRootsIterator::new(self.store.clone(), self.state.read().clone(), slot) } /* diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index da81a6582f..da939be2ad 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -1,6 +1,29 @@ use std::sync::Arc; use store::Store; -use types::{BeaconState, BeaconStateError, EthSpec, Hash256, Slot}; +use types::{BeaconBlock, BeaconState, BeaconStateError, EthSpec, Hash256, Slot}; + +/// Extends `BlockRootsIterator`, returning `BeaconBlock` instances, instead of their roots. +pub struct BlockIterator { + roots: BlockRootsIterator, +} + +impl BlockIterator { + /// Create a new iterator over all blocks in the given `beacon_state` and prior states. + pub fn new(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { + Self { + roots: BlockRootsIterator::new(store, beacon_state, start_slot), + } + } +} + +impl Iterator for BlockIterator { + type Item = BeaconBlock; + + fn next(&mut self) -> Option { + let root = self.roots.next()?; + self.roots.store.get(&root).ok()? + } +} /// Iterates backwards through block roots. /// @@ -17,9 +40,9 @@ pub struct BlockRootsIterator { impl BlockRootsIterator { /// Create a new iterator over all block roots in the given `beacon_state` and prior states. - pub fn from_state(store: Arc, beacon_state: BeaconState) -> Self { + pub fn new(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { Self { - slot: beacon_state.slot, + slot: start_slot, beacon_state, store, } @@ -95,7 +118,7 @@ mod test { state_b.latest_state_roots[0] = state_a_root; store.put(&state_a_root, &state_a).unwrap(); - let iter = BlockRootsIterator::from_state(store.clone(), state_b.clone()); + let iter = BlockRootsIterator::new(store.clone(), state_b.clone(), state_b.slot); let mut collected: Vec = iter.collect(); collected.reverse(); From f52d66a13662e0c67e751d444937b2a40a530135 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 6 Jun 2019 00:31:18 -0400 Subject: [PATCH 53/76] Fix bug in rev block iter --- beacon_node/beacon_chain/src/iter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index da939be2ad..a05e4186c6 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -53,7 +53,7 @@ impl Iterator for BlockRootsIterator { type Item = Hash256; fn next(&mut self) -> Option { - if self.slot == 0 { + if (self.slot == 0) || (self.slot > self.beacon_state.slot) { return None; } From af96dd08c89aa404cc715b21195a4610df4a219d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 6 Jun 2019 00:32:09 -0400 Subject: [PATCH 54/76] Simplify `simple_sync` code --- beacon_node/network/src/sync/simple_sync.rs | 260 ++++++-------------- 1 file changed, 77 insertions(+), 183 deletions(-) diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index af6cbdfc03..043f0beda0 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,8 +1,6 @@ use super::import_queue::ImportQueue; use crate::message_handler::NetworkContext; -use beacon_chain::{ - BeaconChain, BeaconChainError, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, -}; +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; @@ -33,51 +31,6 @@ pub struct PeerSyncInfo { best_slot: Slot, } -impl PeerSyncInfo { - /// Returns `true` if the has a different network ID to `other`. - fn has_different_network_id_to(&self, other: Self) -> bool { - self.network_id != other.network_id - } - - /// Returns `true` if the peer has a higher finalized epoch than `other`. - fn has_higher_finalized_epoch_than(&self, other: Self) -> bool { - self.latest_finalized_epoch > other.latest_finalized_epoch - } - - /// Returns `true` if the peer has a higher best slot than `other`. - fn has_higher_best_slot_than(&self, other: Self) -> bool { - self.best_slot > other.best_slot - } -} - -/// The status of a peers view on the chain, relative to some other view of the chain (presumably -/// our view). -#[derive(PartialEq, Clone, Copy, Debug)] -pub enum PeerStatus { - /// The peer is on a completely different chain. - DifferentNetworkId, - /// The peer lists a finalized epoch for which we have a different root. - FinalizedEpochNotInChain, - /// The peer has a higher finalized epoch. - HigherFinalizedEpoch, - /// The peer has a higher best slot. - HigherBestSlot, - /// The peer has the same or lesser view of the chain. We have nothing to request of them. - NotInteresting, -} - -impl PeerStatus { - pub fn should_handshake(self) -> bool { - match self { - PeerStatus::DifferentNetworkId => false, - PeerStatus::FinalizedEpochNotInChain => false, - PeerStatus::HigherFinalizedEpoch => true, - PeerStatus::HigherBestSlot => true, - PeerStatus::NotInteresting => true, - } - } -} - impl From for PeerSyncInfo { fn from(hello: HelloMessage) -> PeerSyncInfo { PeerSyncInfo { @@ -153,7 +106,7 @@ impl SimpleSync { /// /// Sends a `Hello` message to the peer. pub fn on_connect(&self, peer_id: PeerId, network: &mut NetworkContext) { - info!(self.log, "PeerConnect"; "peer" => format!("{:?}", peer_id)); + info!(self.log, "PeerConnected"; "peer" => format!("{:?}", peer_id)); network.send_rpc_request(peer_id, RPCRequest::Hello(hello_message(&self.chain))); } @@ -193,51 +146,6 @@ impl SimpleSync { self.process_hello(peer_id, hello, network); } - /// Returns a `PeerStatus` for some peer. - fn peer_status(&self, peer: PeerSyncInfo) -> PeerStatus { - let local = PeerSyncInfo::from(&self.chain); - - if peer.has_different_network_id_to(local) { - return PeerStatus::DifferentNetworkId; - } - - if local.has_higher_finalized_epoch_than(peer) { - let peer_finalized_slot = peer - .latest_finalized_epoch - .start_slot(T::EthSpec::spec().slots_per_epoch); - - let local_roots = self.chain.get_block_roots(peer_finalized_slot, 1, 0); - - if let Ok(local_roots) = local_roots { - if let Some(local_root) = local_roots.get(0) { - if *local_root != peer.latest_finalized_root { - return PeerStatus::FinalizedEpochNotInChain; - } - } else { - error!( - self.log, - "Cannot get root for peer finalized slot."; - "error" => "empty roots" - ); - } - } else { - error!( - self.log, - "Cannot get root for peer finalized slot."; - "error" => format!("{:?}", local_roots) - ); - } - } - - if peer.has_higher_finalized_epoch_than(local) { - PeerStatus::HigherFinalizedEpoch - } else if peer.has_higher_best_slot_than(local) { - PeerStatus::HigherBestSlot - } else { - PeerStatus::NotInteresting - } - } - /// Process a `Hello` message, requesting new blocks if appropriate. /// /// Disconnects the peer if required. @@ -251,52 +159,68 @@ impl SimpleSync { let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); - let remote_status = self.peer_status(remote); - if remote_status.should_handshake() { - info!(self.log, "HandshakeSuccess"; "peer" => format!("{:?}", peer_id)); - self.known_peers.insert(peer_id.clone(), remote); - } else { + let network_id_mismatch = local.network_id != remote.network_id; + let on_different_finalized_chain = (local.latest_finalized_epoch + >= remote.latest_finalized_epoch) + && (!self + .chain + .rev_iter_block_roots(local.best_slot) + .any(|root| root == remote.latest_finalized_root)); + + if network_id_mismatch || on_different_finalized_chain { info!( self.log, "HandshakeFailure"; "peer" => format!("{:?}", peer_id), "reason" => "network_id" ); network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); + return; + } else { + info!(self.log, "HandshakeSuccess"; "peer" => format!("{:?}", peer_id)); + self.known_peers.insert(peer_id.clone(), remote); } - // If required, send additional requests. - match remote_status { - PeerStatus::HigherFinalizedEpoch => { - let start_slot = remote - .latest_finalized_epoch - .start_slot(spec.slots_per_epoch); - let required_slots = start_slot - local.best_slot; + // If we have equal or better finalized epochs and best slots, we require nothing else from + // this peer. + if (remote.latest_finalized_epoch <= local.latest_finalized_epoch) + && (remote.best_slot <= local.best_slot) + { + return; + } - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); - } - PeerStatus::HigherBestSlot => { - let required_slots = remote.best_slot - local.best_slot; + // If the remote has a higher finalized epoch, request all block roots from our finalized + // epoch through to its best slot. + if remote.latest_finalized_epoch > local.latest_finalized_epoch { + let start_slot = local + .latest_finalized_epoch + .start_slot(spec.slots_per_epoch); + let required_slots = start_slot - remote.best_slot; - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot: local.best_slot + 1, - count: required_slots.into(), - }, - network, - ); - } - PeerStatus::FinalizedEpochNotInChain => {} - PeerStatus::DifferentNetworkId => {} - PeerStatus::NotInteresting => {} + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); + // If the remote has a greater best slot, request the roots between our best slot and their + // best slot. + } else if remote.best_root > local.best_root { + let start_slot = local + .latest_finalized_epoch + .start_slot(spec.slots_per_epoch); + let required_slots = start_slot - remote.best_slot; + + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); } } @@ -315,27 +239,12 @@ impl SimpleSync { "count" => req.count, ); - let roots = match self + let roots = self .chain - .get_block_roots(req.start_slot, req.count as usize, 0) - { - Ok(roots) => roots, - Err(e) => { - // TODO: return RPC error. - warn!( - self.log, - "RPCRequest"; "peer" => format!("{:?}", peer_id), - "req" => "BeaconBlockRoots", - "error" => format!("{:?}", e) - ); - return; - } - }; - - let roots = roots - .iter() + .rev_iter_block_roots(req.start_slot) + .take(req.count as usize) .enumerate() - .map(|(i, &block_root)| BlockRootSlot { + .map(|(i, block_root)| BlockRootSlot { slot: req.start_slot + Slot::from(i), block_root, }) @@ -426,24 +335,12 @@ impl SimpleSync { "count" => req.max_headers, ); - let headers = match get_block_headers( + let headers = get_block_headers( &self.chain, req.start_slot, req.max_headers as usize, req.skip_slots as usize, - ) { - Ok(headers) => headers, - Err(e) => { - // TODO: return RPC error. - warn!( - self.log, - "RPCRequest"; "peer" => format!("{:?}", peer_id), - "req" => "BeaconBlockHeaders", - "error" => format!("{:?}", e) - ); - return; - } - }; + ); network.send_rpc_response( peer_id, @@ -554,14 +451,15 @@ impl SimpleSync { ) -> bool { info!( self.log, - "NewGossipBlock"; + "GossipBlockReceived"; "peer" => format!("{:?}", peer_id), + "block_slot" => format!("{:?}", block.slot), ); // Ignore any block from a finalized slot. if self.slot_is_finalized(block.slot) { warn!( - self.log, "NewGossipBlock"; + self.log, "IgnoredGossipBlock"; "msg" => "new block slot is finalized.", "block_slot" => block.slot, ); @@ -572,22 +470,15 @@ impl SimpleSync { // Ignore any block that the chain already knows about. if self.chain_has_seen_block(&block_root) { - println!("this happened"); // TODO: Age confirm that we shouldn't forward a block if we already know of it. return false; } - debug!( - self.log, - "NewGossipBlock"; - "peer" => format!("{:?}", peer_id), - "msg" => "processing block", - ); match self.chain.process_block(block.clone()) { Ok(BlockProcessingOutcome::InvalidBlock(InvalidBlock::ParentUnknown)) => { // The block was valid and we processed it successfully. debug!( - self.log, "NewGossipBlock"; + self.log, "InvalidGossipBlock"; "msg" => "parent block unknown", "parent_root" => format!("{}", block.previous_block_root), "peer" => format!("{:?}", peer_id), @@ -614,7 +505,7 @@ impl SimpleSync { if block_slot - present_slot > FUTURE_SLOT_TOLERANCE { // The block is too far in the future, drop it. warn!( - self.log, "NewGossipBlock"; + self.log, "InvalidGossipBlock"; "msg" => "future block rejected", "present_slot" => present_slot, "block_slot" => block_slot, @@ -626,7 +517,7 @@ impl SimpleSync { } else { // The block is in the future, but not too far. warn!( - self.log, "NewGossipBlock"; + self.log, "FutureGossipBlock"; "msg" => "queuing future block", "present_slot" => present_slot, "block_slot" => block_slot, @@ -643,8 +534,8 @@ impl SimpleSync { if outcome.is_invalid() { // The peer has sent a block which is fundamentally invalid. warn!( - self.log, "NewGossipBlock"; - "msg" => "invalid block from peer", + self.log, "InvalidGossipBlock"; + "msg" => "peer sent objectively invalid block", "outcome" => format!("{:?}", outcome), "peer" => format!("{:?}", peer_id), ); @@ -655,8 +546,7 @@ impl SimpleSync { } else if outcome.sucessfully_processed() { // The block was valid and we processed it successfully. info!( - self.log, "NewGossipBlock"; - "msg" => "block import successful", + self.log, "ValidGossipBlock"; "peer" => format!("{:?}", peer_id), ); // Forward the block to peers @@ -665,7 +555,7 @@ impl SimpleSync { // The block wasn't necessarily invalid but we didn't process it successfully. // This condition shouldn't be reached. error!( - self.log, "NewGossipBlock"; + self.log, "InvalidGossipBlock"; "msg" => "unexpected condition in processing block.", "outcome" => format!("{:?}", outcome), ); @@ -679,7 +569,7 @@ impl SimpleSync { // Blocks should not be able to trigger errors, instead they should be flagged as // invalid. error!( - self.log, "NewGossipBlock"; + self.log, "InvalidGossipBlock"; "msg" => "internal error in processing block.", "error" => format!("{:?}", e), ); @@ -870,7 +760,11 @@ fn get_block_headers( start_slot: Slot, count: usize, skip: usize, -) -> Result, BeaconChainError> { - let roots = beacon_chain.get_block_roots(start_slot, count, skip)?; - beacon_chain.get_block_headers(&roots) +) -> Vec { + beacon_chain + .rev_iter_blocks(start_slot) + .step_by(skip + 1) + .take(count) + .map(|block| block.block_header()) + .collect() } From 1707aa5bf3ba3cd2777b5028e117d1396b69df1c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Thu, 6 Jun 2019 09:47:21 -0400 Subject: [PATCH 55/76] Fix bug in RPC wire encoding I broke this in a previous commit. --- beacon_node/eth2-libp2p/src/rpc/protocol.rs | 171 ++++++++------------ 1 file changed, 69 insertions(+), 102 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index 82257cc327..d75e53e6ce 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -1,6 +1,7 @@ use super::methods::*; use libp2p::core::{upgrade, InboundUpgrade, OutboundUpgrade, UpgradeInfo}; use ssz::{impl_decode_via_from, impl_encode_via_from, ssz_encode, Decode, Encode}; +use ssz_derive::{Decode, Encode}; use std::hash::{Hash, Hasher}; use std::io; use std::iter; @@ -31,7 +32,7 @@ impl Default for RPCProtocol { } /// A monotonic counter for ordering `RPCRequest`s. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Copy, Default)] pub struct RequestId(u64); impl RequestId { @@ -115,65 +116,67 @@ where } } +/// A helper structed used to obtain SSZ serialization for RPC messages. +#[derive(Encode, Decode, Default)] +struct SszContainer { + /// Note: the `is_request` field is not included in the spec. + /// + /// We are unable to determine a request from a response unless we add some flag to the + /// packet. Here we have added a bool (encoded as 1 byte) which is set to `1` if the + /// message is a request. + is_request: bool, + id: u64, + other: u16, + bytes: Vec, +} + // NOTE! // // This code has not been tested, it is a placeholder until we can update to the new libp2p // spec. fn decode(packet: Vec) -> Result { - let mut builder = ssz::SszDecoderBuilder::new(&packet); + let msg = SszContainer::from_ssz_bytes(&packet)?; - builder.register_type::()?; - builder.register_type::()?; - builder.register_type::()?; - builder.register_type::>()?; - - let mut decoder = builder.build()?; - - let request: bool = decoder.decode_next()?; - let id: RequestId = decoder.decode_next()?; - let method_id: u16 = decoder.decode_next()?; - let bytes: Vec = decoder.decode_next()?; - - if request { - let body = match RPCMethod::from(method_id) { - RPCMethod::Hello => RPCRequest::Hello(HelloMessage::from_ssz_bytes(&bytes)?), - RPCMethod::Goodbye => RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(&bytes)?), + if msg.is_request { + let body = match RPCMethod::from(msg.other) { + RPCMethod::Hello => RPCRequest::Hello(HelloMessage::from_ssz_bytes(&msg.bytes)?), + RPCMethod::Goodbye => RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(&msg.bytes)?), RPCMethod::BeaconBlockRoots => { - RPCRequest::BeaconBlockRoots(BeaconBlockRootsRequest::from_ssz_bytes(&bytes)?) - } - RPCMethod::BeaconBlockHeaders => { - RPCRequest::BeaconBlockHeaders(BeaconBlockHeadersRequest::from_ssz_bytes(&bytes)?) + RPCRequest::BeaconBlockRoots(BeaconBlockRootsRequest::from_ssz_bytes(&msg.bytes)?) } + RPCMethod::BeaconBlockHeaders => RPCRequest::BeaconBlockHeaders( + BeaconBlockHeadersRequest::from_ssz_bytes(&msg.bytes)?, + ), RPCMethod::BeaconBlockBodies => { - RPCRequest::BeaconBlockBodies(BeaconBlockBodiesRequest::from_ssz_bytes(&bytes)?) + RPCRequest::BeaconBlockBodies(BeaconBlockBodiesRequest::from_ssz_bytes(&msg.bytes)?) } RPCMethod::BeaconChainState => { - RPCRequest::BeaconChainState(BeaconChainStateRequest::from_ssz_bytes(&bytes)?) + RPCRequest::BeaconChainState(BeaconChainStateRequest::from_ssz_bytes(&msg.bytes)?) } RPCMethod::Unknown => return Err(DecodeError::UnknownRPCMethod), }; Ok(RPCEvent::Request { - id, - method_id, + id: RequestId::from(msg.id), + method_id: msg.other, body, }) } // we have received a response else { - let result = match RPCMethod::from(method_id) { - RPCMethod::Hello => RPCResponse::Hello(HelloMessage::from_ssz_bytes(&bytes)?), + let result = match RPCMethod::from(msg.other) { + RPCMethod::Hello => RPCResponse::Hello(HelloMessage::from_ssz_bytes(&msg.bytes)?), RPCMethod::BeaconBlockRoots => { - RPCResponse::BeaconBlockRoots(BeaconBlockRootsResponse::from_ssz_bytes(&bytes)?) - } - RPCMethod::BeaconBlockHeaders => { - RPCResponse::BeaconBlockHeaders(BeaconBlockHeadersResponse::from_ssz_bytes(&bytes)?) - } - RPCMethod::BeaconBlockBodies => { - RPCResponse::BeaconBlockBodies(BeaconBlockBodiesResponse::from_ssz_bytes(&packet)?) + RPCResponse::BeaconBlockRoots(BeaconBlockRootsResponse::from_ssz_bytes(&msg.bytes)?) } + RPCMethod::BeaconBlockHeaders => RPCResponse::BeaconBlockHeaders( + BeaconBlockHeadersResponse::from_ssz_bytes(&msg.bytes)?, + ), + RPCMethod::BeaconBlockBodies => RPCResponse::BeaconBlockBodies( + BeaconBlockBodiesResponse::from_ssz_bytes(&msg.bytes)?, + ), RPCMethod::BeaconChainState => { - RPCResponse::BeaconChainState(BeaconChainStateResponse::from_ssz_bytes(&packet)?) + RPCResponse::BeaconChainState(BeaconChainStateResponse::from_ssz_bytes(&msg.bytes)?) } // We should never receive a goodbye response; it is invalid. RPCMethod::Goodbye => return Err(DecodeError::UnknownRPCMethod), @@ -181,8 +184,8 @@ fn decode(packet: Vec) -> Result { }; Ok(RPCEvent::Response { - id, - method_id, + id: RequestId::from(msg.id), + method_id: msg.other, result, }) } @@ -208,80 +211,44 @@ impl Encode for RPCEvent { false } - // NOTE! - // - // This code has not been tested, it is a placeholder until we can update to the new libp2p - // spec. fn ssz_append(&self, buf: &mut Vec) { - let offset = ::ssz_fixed_len() - + ::ssz_fixed_len() - + as Encode>::ssz_fixed_len(); - - let mut encoder = ssz::SszEncoder::container(buf, offset); - - match self { + let container = match self { RPCEvent::Request { id, method_id, body, - } => { - encoder.append(&true); - encoder.append(id); - encoder.append(method_id); - - // Encode the `body` as a `Vec`. - match body { - RPCRequest::Hello(body) => { - encoder.append(&body.as_ssz_bytes()); - } - RPCRequest::Goodbye(body) => { - encoder.append(&body.as_ssz_bytes()); - } - RPCRequest::BeaconBlockRoots(body) => { - encoder.append(&body.as_ssz_bytes()); - } - RPCRequest::BeaconBlockHeaders(body) => { - encoder.append(&body.as_ssz_bytes()); - } - RPCRequest::BeaconBlockBodies(body) => { - encoder.append(&body.as_ssz_bytes()); - } - RPCRequest::BeaconChainState(body) => { - encoder.append(&body.as_ssz_bytes()); - } - } - } + } => SszContainer { + is_request: true, + id: (*id).into(), + other: (*method_id).into(), + bytes: match body { + RPCRequest::Hello(body) => body.as_ssz_bytes(), + RPCRequest::Goodbye(body) => body.as_ssz_bytes(), + RPCRequest::BeaconBlockRoots(body) => body.as_ssz_bytes(), + RPCRequest::BeaconBlockHeaders(body) => body.as_ssz_bytes(), + RPCRequest::BeaconBlockBodies(body) => body.as_ssz_bytes(), + RPCRequest::BeaconChainState(body) => body.as_ssz_bytes(), + }, + }, RPCEvent::Response { id, method_id, result, - } => { - encoder.append(&true); - encoder.append(id); - encoder.append(method_id); + } => SszContainer { + is_request: false, + id: (*id).into(), + other: (*method_id).into(), + bytes: match result { + RPCResponse::Hello(response) => response.as_ssz_bytes(), + RPCResponse::BeaconBlockRoots(response) => response.as_ssz_bytes(), + RPCResponse::BeaconBlockHeaders(response) => response.as_ssz_bytes(), + RPCResponse::BeaconBlockBodies(response) => response.as_ssz_bytes(), + RPCResponse::BeaconChainState(response) => response.as_ssz_bytes(), + }, + }, + }; - match result { - RPCResponse::Hello(response) => { - encoder.append(&response.as_ssz_bytes()); - } - RPCResponse::BeaconBlockRoots(response) => { - encoder.append(&response.as_ssz_bytes()); - } - RPCResponse::BeaconBlockHeaders(response) => { - encoder.append(&response.as_ssz_bytes()); - } - RPCResponse::BeaconBlockBodies(response) => { - encoder.append(&response.as_ssz_bytes()); - } - RPCResponse::BeaconChainState(response) => { - encoder.append(&response.as_ssz_bytes()); - } - } - } - } - - // Finalize the encoder, writing to `buf`. - encoder.finalize(); + container.ssz_append(buf) } } From 591c8ae219754b12ea27bb6bf7f15b5909690490 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 02:46:07 -0400 Subject: [PATCH 56/76] Reject re-processing the genesis block --- beacon_node/beacon_chain/src/beacon_chain.rs | 39 ++++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d208139695..a9374aa6a8 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -30,6 +30,8 @@ pub enum ValidBlock { #[derive(Debug, PartialEq)] pub enum InvalidBlock { + /// Don't re-process the genesis block. + GenesisBlock, /// The block slot is greater than the present slot. FutureSlot { present_slot: Slot, @@ -38,7 +40,7 @@ pub enum InvalidBlock { /// The block state_root does not match the generated state. StateRootMismatch, /// The blocks parent_root is unknown. - ParentUnknown, + ParentUnknown { parent: Hash256 }, /// There was an error whilst advancing the parent state to the present slot. This condition /// should not occur, it likely represents an internal error. SlotProcessingError(SlotProcessingError), @@ -61,9 +63,10 @@ impl BlockProcessingOutcome { match self { BlockProcessingOutcome::ValidBlock(_) => false, BlockProcessingOutcome::InvalidBlock(r) => match r { + InvalidBlock::GenesisBlock { .. } => true, InvalidBlock::FutureSlot { .. } => true, InvalidBlock::StateRootMismatch => true, - InvalidBlock::ParentUnknown => false, + InvalidBlock::ParentUnknown { .. } => false, InvalidBlock::SlotProcessingError(_) => false, InvalidBlock::PerBlockProcessingError(e) => match e { BlockProcessingError::Invalid(_) => true, @@ -131,6 +134,10 @@ impl BeaconChain { let genesis_block_root = genesis_block.block_header().canonical_root(); store.put(&genesis_block_root, &genesis_block)?; + // Also store the genesis block under the `ZERO_HASH` key. + let genesis_block_root = genesis_block.block_header().canonical_root(); + store.put(&spec.zero_hash, &genesis_block)?; + let canonical_head = RwLock::new(CheckPoint::new( genesis_block.clone(), genesis_block_root, @@ -205,7 +212,9 @@ impl BeaconChain { .iter() .map(|root| match self.get_block(root)? { Some(block) => Ok(block.body), - None => Err(Error::DBInconsistent("Missing block".into())), + None => Err(Error::DBInconsistent( + format!("Missing block: {}", root).into(), + )), }) .collect(); @@ -648,8 +657,18 @@ impl BeaconChain { self.metrics.block_processing_requests.inc(); let timer = self.metrics.block_processing_times.start_timer(); + if block.slot == 0 { + return Ok(BlockProcessingOutcome::InvalidBlock( + InvalidBlock::GenesisBlock, + )); + } + let block_root = block.block_header().canonical_root(); + if block_root == self.genesis_block_root { + return Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)); + } + let present_slot = self.present_slot(); if block.slot > present_slot { @@ -668,7 +687,9 @@ impl BeaconChain { Some(previous_block_root) => previous_block_root, None => { return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::ParentUnknown, + InvalidBlock::ParentUnknown { + parent: parent_block_root, + }, )); } }; @@ -754,9 +775,13 @@ impl BeaconChain { trace!("Finding attestations for new block..."); - let previous_block_root = *state - .get_block_root(state.slot - 1) - .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)?; + let previous_block_root = if state.slot > 0 { + *state + .get_block_root(state.slot - 1) + .map_err(|_| BlockProductionError::UnableToGetBlockRootFromState)? + } else { + state.latest_block_header.canonical_root() + }; let (proposer_slashings, attester_slashings) = self .op_pool From 4cecf05198180f00bb8ad7f3ae2a228393203daa Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 02:48:26 -0400 Subject: [PATCH 57/76] Fix beacon chain block iters --- beacon_node/beacon_chain/src/iter.rs | 37 +++++++++++----------- beacon_node/eth2-libp2p/src/rpc/methods.rs | 4 +-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index a05e4186c6..3147fb2078 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -57,28 +57,21 @@ impl Iterator for BlockRootsIterator { return None; } - let slot = self.slot - 1; + self.slot = self.slot - 1; - match self.beacon_state.get_block_root(slot) { + match self.beacon_state.get_block_root(self.slot) { Ok(root) => Some(*root), Err(BeaconStateError::SlotOutOfBounds) => { // Read a `BeaconState` from the store that has access to prior historical root. self.beacon_state = { - // Read the earliest historic state in the current slot. - let earliest_historic_slot = - self.beacon_state.slot - Slot::from(T::slots_per_historical_root()); + // Load the earlier state from disk. Skip forward one slot, because a state + // doesn't return it's own state root. + let new_state_root = self.beacon_state.get_state_root(self.slot + 1).ok()?; - // Load the earlier state from disk. - let new_state_root = self - .beacon_state - .get_state_root(earliest_historic_slot) - .ok()?; + self.store.get(&new_state_root).ok()? + }?; - let state_option = self.store.get(&new_state_root).ok()?; - state_option? - }; - - self.beacon_state.get_block_root(slot).ok().cloned() + self.beacon_state.get_block_root(self.slot).ok().cloned() } _ => return None, } @@ -101,10 +94,14 @@ mod test { #[test] fn root_iter() { let store = Arc::new(MemoryStore::open()); + let slots_per_historical_root = FoundationEthSpec::slots_per_historical_root(); let mut state_a: BeaconState = get_state(); let mut state_b: BeaconState = get_state(); + state_a.slot = Slot::from(slots_per_historical_root); + state_b.slot = Slot::from(slots_per_historical_root * 2); + let mut hashes = (0..).into_iter().map(|i| Hash256::from(i)); for root in &mut state_a.latest_block_roots[..] { @@ -118,12 +115,16 @@ mod test { state_b.latest_state_roots[0] = state_a_root; store.put(&state_a_root, &state_a).unwrap(); - let iter = BlockRootsIterator::new(store.clone(), state_b.clone(), state_b.slot); + let iter = BlockRootsIterator::new(store.clone(), state_b.clone(), state_b.slot - 1); let mut collected: Vec = iter.collect(); collected.reverse(); - for (i, item) in collected.iter().enumerate() { - assert_eq!(*item, Hash256::from(i as u64)); + let expected_len = 2 * FoundationEthSpec::slots_per_historical_root() - 1; + + assert_eq!(collected.len(), expected_len); + + for i in 0..expected_len { + assert_eq!(collected[i], Hash256::from(i as u64)); } } } diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index ef73157650..b752b74cbb 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -172,8 +172,8 @@ pub struct BeaconBlockRootsResponse { impl BeaconBlockRootsResponse { /// Returns `true` if each `self.roots.slot[i]` is higher than the preceeding `i`. pub fn slots_are_ascending(&self) -> bool { - for i in 1..self.roots.len() { - if self.roots[i - 1].slot >= self.roots[i].slot { + for window in self.roots.windows(2) { + if window[0].slot >= window[1].slot { return false; } } From 719dd72de65d2f4f8f9af16279206a7921c190cb Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 02:55:16 -0400 Subject: [PATCH 58/76] Fix recently introduced sync bugs --- beacon_node/network/Cargo.toml | 1 + beacon_node/network/src/sync/import_queue.rs | 4 +- beacon_node/network/src/sync/simple_sync.rs | 299 ++++++++++-------- .../src/per_slot_processing.rs | 3 +- eth2/types/src/beacon_state.rs | 2 +- 5 files changed, 179 insertions(+), 130 deletions(-) diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index 9cac126595..ebf71aa4e0 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -9,6 +9,7 @@ sloggers = "0.3.2" [dependencies] beacon_chain = { path = "../beacon_chain" } +store = { path = "../store" } eth2-libp2p = { path = "../eth2-libp2p" } version = { path = "../version" } types = { path = "../../eth2/types" } diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 5b03f58dff..16a277f0b5 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -166,7 +166,7 @@ impl ImportQueue { let mut required_bodies: Vec = vec![]; for header in headers { - let block_root = Hash256::from_slice(&header.tree_hash_root()[..]); + let block_root = Hash256::from_slice(&header.canonical_root()[..]); if self.chain_has_not_seen_block(&block_root) { self.insert_header(block_root, header, sender.clone()); @@ -250,7 +250,7 @@ impl ImportQueue { /// /// If the partial already existed, the `inserted` time is set to `now`. fn insert_full_block(&mut self, block: BeaconBlock, sender: PeerId) { - let block_root = Hash256::from_slice(&block.tree_hash_root()[..]); + let block_root = Hash256::from_slice(&block.canonical_root()[..]); let partial = PartialBeaconBlock { slot: block.slot, diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 043f0beda0..c3fb03ca1d 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -8,8 +8,11 @@ use slog::{debug, error, info, o, warn}; use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; +use store::Store; use tree_hash::TreeHash; -use types::{Attestation, BeaconBlock, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot}; +use types::{ + Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot, +}; /// The number of slots that we can import blocks ahead of us, before going into full Sync mode. const SLOT_IMPORT_TOLERANCE: u64 = 100; @@ -160,67 +163,88 @@ impl SimpleSync { let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); - let network_id_mismatch = local.network_id != remote.network_id; - let on_different_finalized_chain = (local.latest_finalized_epoch - >= remote.latest_finalized_epoch) - && (!self - .chain - .rev_iter_block_roots(local.best_slot) - .any(|root| root == remote.latest_finalized_root)); - - if network_id_mismatch || on_different_finalized_chain { + // Disconnect nodes who are on a different network. + if local.network_id != remote.network_id { info!( self.log, "HandshakeFailure"; "peer" => format!("{:?}", peer_id), "reason" => "network_id" ); network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); - return; + // Disconnect nodes if our finalized epoch is greater than thieirs, and their finalized + // epoch is not in our chain. Viz., they are on another chain. + // + // If the local or remote have a `latest_finalized_root == ZERO_HASH`, skips checks about + // the finalized_root. The logic is akward and I think we're better without it. + } else if (local.latest_finalized_epoch >= remote.latest_finalized_epoch) + && (!self + .chain + .rev_iter_block_roots(local.best_slot) + .any(|root| root == remote.latest_finalized_root)) + && (local.latest_finalized_root != spec.zero_hash) + && (remote.latest_finalized_root != spec.zero_hash) + { + info!( + self.log, "HandshakeFailure"; + "peer" => format!("{:?}", peer_id), + "reason" => "wrong_finalized_chain" + ); + network.disconnect(peer_id.clone(), GoodbyeReason::IrreleventNetwork); + // Process handshakes from peers that seem to be on our chain. } else { info!(self.log, "HandshakeSuccess"; "peer" => format!("{:?}", peer_id)); self.known_peers.insert(peer_id.clone(), remote); - } - // If we have equal or better finalized epochs and best slots, we require nothing else from - // this peer. - if (remote.latest_finalized_epoch <= local.latest_finalized_epoch) - && (remote.best_slot <= local.best_slot) - { - return; - } + // If we have equal or better finalized epochs and best slots, we require nothing else from + // this peer. + // + // We make an exception when our best slot is 0. Best slot does not indicate wether or + // not there is a block at slot zero. + if (remote.latest_finalized_epoch <= local.latest_finalized_epoch) + && (remote.best_slot <= local.best_slot) + && (local.best_slot > 0) + { + debug!(self.log, "Peer is naive"; "peer" => format!("{:?}", peer_id)); + return; + } - // If the remote has a higher finalized epoch, request all block roots from our finalized - // epoch through to its best slot. - if remote.latest_finalized_epoch > local.latest_finalized_epoch { - let start_slot = local - .latest_finalized_epoch - .start_slot(spec.slots_per_epoch); - let required_slots = start_slot - remote.best_slot; + // If the remote has a higher finalized epoch, request all block roots from our finalized + // epoch through to its best slot. + if remote.latest_finalized_epoch > local.latest_finalized_epoch { + debug!(self.log, "Peer has high finalized epoch"; "peer" => format!("{:?}", peer_id)); + let start_slot = local + .latest_finalized_epoch + .start_slot(spec.slots_per_epoch); + let required_slots = remote.best_slot - start_slot; - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); - // If the remote has a greater best slot, request the roots between our best slot and their - // best slot. - } else if remote.best_root > local.best_root { - let start_slot = local - .latest_finalized_epoch - .start_slot(spec.slots_per_epoch); - let required_slots = start_slot - remote.best_slot; + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); + // If the remote has a greater best slot, request the roots between our best slot and their + // best slot. + } else if remote.best_slot > local.best_slot { + debug!(self.log, "Peer has higher best slot"; "peer" => format!("{:?}", peer_id)); + let start_slot = local + .latest_finalized_epoch + .start_slot(spec.slots_per_epoch); + let required_slots = remote.best_slot - start_slot; - self.request_block_roots( - peer_id, - BeaconBlockRootsRequest { - start_slot, - count: required_slots.into(), - }, - network, - ); + self.request_block_roots( + peer_id, + BeaconBlockRootsRequest { + start_slot, + count: required_slots.into(), + }, + network, + ); + } else { + debug!(self.log, "Nothing to request from peer"; "peer" => format!("{:?}", peer_id)); + } } } @@ -237,19 +261,40 @@ impl SimpleSync { "BlockRootsRequest"; "peer" => format!("{:?}", peer_id), "count" => req.count, + "start_slot" => req.start_slot, ); - let roots = self + let mut roots: Vec = self .chain - .rev_iter_block_roots(req.start_slot) + .rev_iter_block_roots(req.start_slot + req.count) + .skip(1) .take(req.count as usize) + .collect(); + + if roots.len() as u64 != req.count { + debug!( + self.log, + "BlockRootsRequest"; + "peer" => format!("{:?}", peer_id), + "msg" => "Failed to return all requested hashes", + "requested" => req.count, + "returned" => roots.len(), + ); + } + + roots.reverse(); + + let mut roots: Vec = roots + .iter() .enumerate() .map(|(i, block_root)| BlockRootSlot { slot: req.start_slot + Slot::from(i), - block_root, + block_root: *block_root, }) .collect(); + roots.dedup_by_key(|brs| brs.block_root); + network.send_rpc_response( peer_id, request_id, @@ -335,12 +380,28 @@ impl SimpleSync { "count" => req.max_headers, ); - let headers = get_block_headers( - &self.chain, - req.start_slot, - req.max_headers as usize, - req.skip_slots as usize, - ); + let count = req.max_headers; + + // Collect the block roots. + // + // Instead of using `chain.rev_iter_blocks` we collect the roots first. This avoids + // unnecessary block deserialization when `req.skip_slots > 0`. + let mut roots: Vec = self + .chain + .rev_iter_block_roots(req.start_slot + (count - 1)) + .take(count as usize) + .collect(); + + roots.reverse(); + + let headers: Vec = roots + .into_iter() + .step_by(req.skip_slots as usize + 1) + .filter_map(|root| { + let block = self.chain.store.get::(&root).ok()?; + Some(block?.block_header()) + }) + .collect(); network.send_rpc_response( peer_id, @@ -388,27 +449,33 @@ impl SimpleSync { req: BeaconBlockBodiesRequest, network: &mut NetworkContext, ) { + let block_bodies: Vec = req + .block_roots + .iter() + .filter_map(|root| { + if let Ok(Some(block)) = self.chain.store.get::(root) { + Some(block.body) + } else { + debug!( + self.log, + "Peer requested unknown block"; + "peer" => format!("{:?}", peer_id), + "request_root" => format!("{:}", root), + ); + + None + } + }) + .collect(); + debug!( self.log, "BlockBodiesRequest"; "peer" => format!("{:?}", peer_id), - "count" => req.block_roots.len(), + "requested" => req.block_roots.len(), + "returned" => block_bodies.len(), ); - let block_bodies = match self.chain.get_block_bodies(&req.block_roots) { - Ok(bodies) => bodies, - Err(e) => { - // TODO: return RPC error. - warn!( - self.log, - "RPCRequest"; "peer" => format!("{:?}", peer_id), - "req" => "BeaconBlockBodies", - "error" => format!("{:?}", e) - ); - return; - } - }; - network.send_rpc_response( peer_id, request_id, @@ -449,18 +516,12 @@ impl SimpleSync { block: BeaconBlock, network: &mut NetworkContext, ) -> bool { - info!( - self.log, - "GossipBlockReceived"; - "peer" => format!("{:?}", peer_id), - "block_slot" => format!("{:?}", block.slot), - ); - // Ignore any block from a finalized slot. if self.slot_is_finalized(block.slot) { - warn!( - self.log, "IgnoredGossipBlock"; - "msg" => "new block slot is finalized.", + debug!( + self.log, "IgnoredFinalizedBlock"; + "source" => "gossip", + "msg" => "chain is finalized at block slot", "block_slot" => block.slot, ); return false; @@ -475,11 +536,11 @@ impl SimpleSync { } match self.chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::InvalidBlock(InvalidBlock::ParentUnknown)) => { + Ok(BlockProcessingOutcome::InvalidBlock(InvalidBlock::ParentUnknown { .. })) => { // The block was valid and we processed it successfully. debug!( - self.log, "InvalidGossipBlock"; - "msg" => "parent block unknown", + self.log, "ParentBlockUnknown"; + "source" => "gossip", "parent_root" => format!("{}", block.previous_block_root), "peer" => format!("{:?}", peer_id), ); @@ -505,8 +566,9 @@ impl SimpleSync { if block_slot - present_slot > FUTURE_SLOT_TOLERANCE { // The block is too far in the future, drop it. warn!( - self.log, "InvalidGossipBlock"; - "msg" => "future block rejected", + self.log, "FutureBlock"; + "source" => "gossip", + "msg" => "block for future slot rejected, check your time", "present_slot" => present_slot, "block_slot" => block_slot, "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, @@ -517,8 +579,9 @@ impl SimpleSync { } else { // The block is in the future, but not too far. warn!( - self.log, "FutureGossipBlock"; - "msg" => "queuing future block", + self.log, "QueuedFutureBlock"; + "source" => "gossip", + "msg" => "queuing future block, check your time", "present_slot" => present_slot, "block_slot" => block_slot, "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, @@ -534,7 +597,8 @@ impl SimpleSync { if outcome.is_invalid() { // The peer has sent a block which is fundamentally invalid. warn!( - self.log, "InvalidGossipBlock"; + self.log, "InvalidBlock"; + "source" => "gossip", "msg" => "peer sent objectively invalid block", "outcome" => format!("{:?}", outcome), "peer" => format!("{:?}", peer_id), @@ -546,7 +610,8 @@ impl SimpleSync { } else if outcome.sucessfully_processed() { // The block was valid and we processed it successfully. info!( - self.log, "ValidGossipBlock"; + self.log, "ImportedBlock"; + "source" => "gossip", "peer" => format!("{:?}", peer_id), ); // Forward the block to peers @@ -555,7 +620,8 @@ impl SimpleSync { // The block wasn't necessarily invalid but we didn't process it successfully. // This condition shouldn't be reached. error!( - self.log, "InvalidGossipBlock"; + self.log, "BlockProcessingFailure"; + "source" => "gossip", "msg" => "unexpected condition in processing block.", "outcome" => format!("{:?}", outcome), ); @@ -569,8 +635,9 @@ impl SimpleSync { // Blocks should not be able to trigger errors, instead they should be flagged as // invalid. error!( - self.log, "InvalidGossipBlock"; + self.log, "BlockProcessingError"; "msg" => "internal error in processing block.", + "source" => "gossip", "error" => format!("{:?}", e), ); // Do not forward the block to peers. @@ -584,19 +651,15 @@ impl SimpleSync { /// Not currently implemented. pub fn on_attestation_gossip( &mut self, - peer_id: PeerId, + _peer_id: PeerId, msg: Attestation, _network: &mut NetworkContext, ) { - info!( - self.log, - "NewAttestationGossip"; - "peer" => format!("{:?}", peer_id), - ); - match self.chain.process_attestation(msg) { - Ok(()) => info!(self.log, "ImportedAttestation"), - Err(e) => warn!(self.log, "InvalidAttestation"; "error" => format!("{:?}", e)), + Ok(()) => info!(self.log, "ImportedAttestation"; "source" => "gossip"), + Err(e) => { + warn!(self.log, "InvalidAttestation"; "source" => "gossip", "error" => format!("{:?}", e)) + } } } @@ -611,6 +674,9 @@ impl SimpleSync { // Loop through all of the complete blocks in the queue. for (block_root, block, sender) in self.import_queue.complete_blocks() { + let slot = block.slot; + let parent_root = block.previous_block_root; + match self.chain.process_block(block) { Ok(outcome) => { if outcome.is_invalid() { @@ -618,15 +684,12 @@ impl SimpleSync { warn!( self.log, "InvalidBlock"; - "sender_peer_id" => format!("{:?}", sender), + "sender_peer_id" => format!("{:?}", sender.clone()), + "block_root" => format!("{}", block_root), "reason" => format!("{:?}", outcome), ); network.disconnect(sender, GoodbyeReason::Fault); - break; - } - - // If this results to true, the item will be removed from the queue. - if outcome.sucessfully_processed() { + } else if outcome.sucessfully_processed() { successful += 1; self.import_queue.remove(block_root); } else { @@ -635,6 +698,8 @@ impl SimpleSync { "ProcessImportQueue"; "msg" => "Block not imported", "outcome" => format!("{:?}", outcome), + "block_slot" => format!("{:?}", slot), + "parent_root" => format!("{}", parent_root), "peer" => format!("{:?}", sender), ); } @@ -752,19 +817,3 @@ fn hello_message(beacon_chain: &BeaconChain) -> HelloMes best_slot: state.slot, } } - -/// Return a list of `BeaconBlockHeader` from the given `BeaconChain`, starting at `start_slot` and -/// returning `count` headers with a gap of `skip` slots between each. -fn get_block_headers( - beacon_chain: &BeaconChain, - start_slot: Slot, - count: usize, - skip: usize, -) -> Vec { - beacon_chain - .rev_iter_blocks(start_slot) - .step_by(skip + 1) - .take(count) - .map(|block| block.block_header()) - .collect() -} diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs index 97645ab8a9..0fc0742623 100644 --- a/eth2/state_processing/src/per_slot_processing.rs +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -1,5 +1,4 @@ use crate::*; -use tree_hash::SignedRoot; use types::*; #[derive(Debug, PartialEq)] @@ -44,7 +43,7 @@ fn cache_state(state: &mut BeaconState, spec: &ChainSpec) -> Resu // Store the previous slot's post state transition root. state.set_state_root(previous_slot, previous_slot_state_root)?; - let latest_block_root = Hash256::from_slice(&state.latest_block_header.signed_root()[..]); + let latest_block_root = state.latest_block_header.canonical_root(); state.set_block_root(previous_slot, latest_block_root)?; // Set the state slot back to what it should be. diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 471b82b20b..b937383280 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -111,7 +111,7 @@ where pub current_crosslinks: FixedLenVec, pub previous_crosslinks: FixedLenVec, pub latest_block_roots: FixedLenVec, - latest_state_roots: FixedLenVec, + pub latest_state_roots: FixedLenVec, latest_active_index_roots: FixedLenVec, latest_slashed_balances: FixedLenVec, pub latest_block_header: BeaconBlockHeader, From 39ec96ad82f28e1a70b074b32d5fdc7bbcf11b85 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 02:55:43 -0400 Subject: [PATCH 59/76] Run rustfmt --- beacon_node/store/src/impls/beacon_state.rs | 2 +- eth2/utils/ssz/src/decode/impls.rs | 6 ++++-- eth2/utils/ssz/tests/tests.rs | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs index 40d2cacb80..591663fe05 100644 --- a/beacon_node/store/src/impls/beacon_state.rs +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -2,7 +2,7 @@ use crate::*; use ssz::{Decode, DecodeError, Encode}; use ssz_derive::{Decode, Encode}; use std::convert::TryInto; -use types::beacon_state::{CACHED_EPOCHS, CommitteeCache}; +use types::beacon_state::{CommitteeCache, CACHED_EPOCHS}; /// A container for storing `BeaconState` components. #[derive(Encode, Decode)] diff --git a/eth2/utils/ssz/src/decode/impls.rs b/eth2/utils/ssz/src/decode/impls.rs index 7dd7fac3c1..0965ee3e54 100644 --- a/eth2/utils/ssz/src/decode/impls.rs +++ b/eth2/utils/ssz/src/decode/impls.rs @@ -1,6 +1,6 @@ use super::*; -use ethereum_types::{H256, U128, U256}; use core::num::NonZeroUsize; +use ethereum_types::{H256, U128, U256}; macro_rules! impl_decodable_for_uint { ($type: ident, $bit_size: expr) => { @@ -76,7 +76,9 @@ impl Decode for NonZeroUsize { let x = usize::from_ssz_bytes(bytes)?; if x == 0 { - Err(DecodeError::BytesInvalid("NonZeroUsize cannot be zero.".to_string())) + Err(DecodeError::BytesInvalid( + "NonZeroUsize cannot be zero.".to_string(), + )) } else { // `unwrap` is safe here as `NonZeroUsize::new()` succeeds if `x > 0` and this path // never executes when `x == 0`. diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index 06c6898171..9447cf5372 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -309,7 +309,7 @@ mod round_trip { 42, 00, 14, 00, 00, 00, 18, 00, 00, 00, 24, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00, // 23 24 25 26 27 // | 2nd list - 00, 00, 00, 00, 00, 00 + 00, 00, 00, 00, 00, 00, ]; assert_eq!(s.as_ssz_bytes(), bytes); From e73a31c37fbabde4ccc6de94f33fe466cfebd29a Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 19:44:27 -0400 Subject: [PATCH 60/76] Refactor ClientConfig, add serde to it --- beacon_node/Cargo.toml | 1 + beacon_node/client/Cargo.toml | 2 + beacon_node/client/src/beacon_chain_types.rs | 20 +-- beacon_node/client/src/client_config.rs | 179 +++++-------------- beacon_node/client/src/error.rs | 1 - beacon_node/client/src/lib.rs | 20 ++- beacon_node/eth2-libp2p/Cargo.toml | 3 + beacon_node/eth2-libp2p/src/config.rs | 47 +++-- beacon_node/eth2-libp2p/src/service.rs | 4 +- beacon_node/http_server/src/lib.rs | 22 ++- beacon_node/rpc/Cargo.toml | 2 + beacon_node/rpc/src/config.rs | 24 ++- beacon_node/src/main.rs | 26 +-- beacon_node/src/run.rs | 100 +++++++---- beacon_node/store/src/disk_db.rs | 1 - 15 files changed, 223 insertions(+), 229 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index d78a5b5960..f1f47bef3e 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" types = { path = "../eth2/types" } store = { path = "./store" } client = { path = "client" } +fork_choice = { path = "../eth2/fork_choice" } version = { path = "version" } clap = "2.32.0" slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index afff86bccc..6da832c337 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -15,6 +15,8 @@ prometheus = "^0.6" types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } slot_clock = { path = "../../eth2/utils/slot_clock" } +serde = "1.0" +serde_derive = "1.0" error-chain = "0.12.0" slog = "^2.2.3" ssz = { path = "../../eth2/utils/ssz" } diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index f84a7fdf65..37b1d261d9 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -18,42 +18,32 @@ const TESTNET_VALIDATOR_COUNT: usize = 16; /// Provides a new, initialized `BeaconChain` pub trait InitialiseBeaconChain { - fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain; + fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) + } } /// A testnet-suitable BeaconChainType, using `MemoryStore`. #[derive(Clone)] pub struct TestnetMemoryBeaconChainTypes; - impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { type Store = MemoryStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = OptimizedLMDGhost; type EthSpec = LighthouseTestnetEthSpec; } - -impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes { - fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) - } -} +impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes {} /// A testnet-suitable BeaconChainType, using `DiskStore`. #[derive(Clone)] pub struct TestnetDiskBeaconChainTypes; - impl BeaconChainTypes for TestnetDiskBeaconChainTypes { type Store = DiskStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = OptimizedLMDGhost; type EthSpec = LighthouseTestnetEthSpec; } - -impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes { - fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) - } -} +impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes {} /// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. fn maybe_load_from_store_for_testnet( diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index a34b83253d..65964a794c 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -1,164 +1,69 @@ use clap::ArgMatches; -use fork_choice::ForkChoiceAlgorithm; use http_server::HttpServerConfig; use network::NetworkConfig; -use slog::error; +use serde_derive::{Deserialize, Serialize}; use std::fs; -use std::net::SocketAddr; -use std::net::{IpAddr, Ipv4Addr}; use std::path::PathBuf; -use types::multiaddr::Protocol; -use types::multiaddr::ToMultiaddr; -use types::Multiaddr; -use types::{ChainSpec, EthSpec, LighthouseTestnetEthSpec}; -#[derive(Debug, Clone)] -pub enum DBType { - Memory, - Disk, -} - -/// Stores the client configuration for this Lighthouse instance. -#[derive(Debug, Clone)] +/// The core configuration of a Lighthouse beacon node. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClientConfig { - pub data_dir: PathBuf, - pub spec: ChainSpec, - pub net_conf: network::NetworkConfig, - pub fork_choice: ForkChoiceAlgorithm, - pub db_type: DBType, - pub db_name: PathBuf, - pub rpc_conf: rpc::RPCConfig, - pub http_conf: HttpServerConfig, //pub ipc_conf: + data_dir: String, + pub spec: String, + pub db_type: String, + db_name: String, + pub network: network::NetworkConfig, + pub rpc: rpc::RPCConfig, + pub http: HttpServerConfig, //pub ipc_conf: } impl Default for ClientConfig { - /// Build a new lighthouse configuration from defaults. fn default() -> Self { - let data_dir = { - let home = dirs::home_dir().expect("Unable to determine home dir."); - home.join(".lighthouse/") - }; - fs::create_dir_all(&data_dir) - .unwrap_or_else(|_| panic!("Unable to create {:?}", &data_dir)); - - let default_spec = LighthouseTestnetEthSpec::spec(); - let default_net_conf = NetworkConfig::new(default_spec.boot_nodes.clone()); - Self { - data_dir: data_dir.clone(), - // default to foundation for chain specs - spec: default_spec, - net_conf: default_net_conf, - // default to bitwise LMD Ghost - fork_choice: ForkChoiceAlgorithm::BitwiseLMDGhost, - // default to memory db for now - db_type: DBType::Memory, - // default db name for disk-based dbs - db_name: data_dir.join("chain_db"), - rpc_conf: rpc::RPCConfig::default(), - http_conf: HttpServerConfig::default(), + data_dir: ".lighthouse".to_string(), + spec: "testnet".to_string(), + db_type: "disk".to_string(), + db_name: "chain_db".to_string(), + // Note: there are no default bootnodes specified. + // Once bootnodes are established, add them here. + network: NetworkConfig::new(vec![]), + rpc: rpc::RPCConfig::default(), + http: HttpServerConfig::default(), } } } impl ClientConfig { - /// Parses the CLI arguments into a `Config` struct. - pub fn parse_args(args: ArgMatches, log: &slog::Logger) -> Result { - let mut config = ClientConfig::default(); + /// Returns the path to which the client may initialize an on-disk database. + pub fn db_path(&self) -> Option { + self.data_dir() + .and_then(|path| Some(path.join(&self.db_name))) + } - /* Network related arguments */ + /// Returns the core path for the client. + pub fn data_dir(&self) -> Option { + let path = dirs::home_dir()?.join(&self.data_dir); + fs::create_dir_all(&path).ok()?; + Some(path) + } - // Custom p2p listen port - if let Some(port_str) = args.value_of("port") { - if let Ok(port) = port_str.parse::() { - config.net_conf.listen_port = port; - // update the listening multiaddrs - for address in &mut config.net_conf.listen_addresses { - address.pop(); - address.append(Protocol::Tcp(port)); - } - } else { - error!(log, "Invalid port"; "port" => port_str); - return Err("Invalid port"); - } - } - // Custom listening address ipv4/ipv6 - // TODO: Handle list of addresses - if let Some(listen_address_str) = args.value_of("listen-address") { - if let Ok(listen_address) = listen_address_str.parse::() { - let multiaddr = SocketAddr::new(listen_address, config.net_conf.listen_port) - .to_multiaddr() - .expect("Invalid listen address format"); - config.net_conf.listen_addresses = vec![multiaddr]; - } else { - error!(log, "Invalid IP Address"; "Address" => listen_address_str); - return Err("Invalid IP Address"); - } - } - - // Custom bootnodes - if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - let boot_addresses_split = boot_addresses_str.split(','); - for boot_address in boot_addresses_split { - if let Ok(boot_address) = boot_address.parse::() { - config.net_conf.boot_nodes.append(&mut vec![boot_address]); - } else { - error!(log, "Invalid Bootnode multiaddress"; "Multiaddr" => boot_addresses_str); - return Err("Invalid IP Address"); - } - } - } - - /* Filesystem related arguments */ - - // Custom datadir + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. + /// + /// Returns an error if arguments are obviously invalid. May succeed even if some values are + /// invalid. + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { if let Some(dir) = args.value_of("datadir") { - config.data_dir = PathBuf::from(dir.to_string()); + self.data_dir = dir.to_string(); }; - /* RPC related arguments */ - - if args.is_present("rpc") { - config.rpc_conf.enabled = true; + if let Some(dir) = args.value_of("db") { + self.db_type = dir.to_string(); } - if let Some(rpc_address) = args.value_of("rpc-address") { - if let Ok(listen_address) = rpc_address.parse::() { - config.rpc_conf.listen_address = listen_address; - } else { - error!(log, "Invalid RPC listen address"; "Address" => rpc_address); - return Err("Invalid RPC listen address"); - } - } + self.network.apply_cli_args(args)?; + self.rpc.apply_cli_args(args)?; + self.http.apply_cli_args(args)?; - if let Some(rpc_port) = args.value_of("rpc-port") { - if let Ok(port) = rpc_port.parse::() { - config.rpc_conf.port = port; - } else { - error!(log, "Invalid RPC port"; "port" => rpc_port); - return Err("Invalid RPC port"); - } - } - - /* HTTP related arguments */ - - if args.is_present("http") { - config.http_conf.enabled = true; - } - - if let Some(listen_address) = args.value_of("http-address") { - config.http_conf.listen_address = listen_address.to_string(); - } - if let Some(listen_port) = args.value_of("http-port") { - config.http_conf.listen_port = listen_port.to_string(); - } - - match args.value_of("db") { - Some("disk") => config.db_type = DBType::Disk, - Some("memory") => config.db_type = DBType::Memory, - _ => unreachable!(), // clap prevents this. - }; - - Ok(config) + Ok(()) } } diff --git a/beacon_node/client/src/error.rs b/beacon_node/client/src/error.rs index b0272400ce..680ad8277d 100644 --- a/beacon_node/client/src/error.rs +++ b/beacon_node/client/src/error.rs @@ -6,5 +6,4 @@ error_chain! { links { Network(network::error::Error, network::error::ErrorKind); } - } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index d29d00ad49..36d1c7d1f8 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -6,7 +6,6 @@ pub mod error; pub mod notifier; use beacon_chain::BeaconChain; -use beacon_chain_types::InitialiseBeaconChain; use exit_future::Signal; use futures::{future::Future, Stream}; use network::Service as NetworkService; @@ -18,10 +17,12 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; +use types::EthSpec; pub use beacon_chain::BeaconChainTypes; +pub use beacon_chain_types::InitialiseBeaconChain; pub use beacon_chain_types::{TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes}; -pub use client_config::{ClientConfig, DBType}; +pub use client_config::ClientConfig; /// Main beacon node client service. This provides the connection and initialisation of the clients /// sub-services in multiple threads. @@ -57,6 +58,7 @@ where ) -> error::Result { let metrics_registry = Registry::new(); let store = Arc::new(store); + let spec = T::EthSpec::spec(); // Load a `BeaconChain` from the store, or create a new one if it does not exist. let beacon_chain = Arc::new(T::initialise_beacon_chain(store, log.clone())); @@ -97,7 +99,7 @@ where // Start the network service, libp2p and syncing threads // TODO: Add beacon_chain reference to network parameters - let network_config = &config.net_conf; + let network_config = &config.network; let network_logger = log.new(o!("Service" => "Network")); let (network, network_send) = NetworkService::new( beacon_chain.clone(), @@ -107,9 +109,9 @@ where )?; // spawn the RPC server - let rpc_exit_signal = if config.rpc_conf.enabled { + let rpc_exit_signal = if config.rpc.enabled { Some(rpc::start_server( - &config.rpc_conf, + &config.rpc, executor, network_send.clone(), beacon_chain.clone(), @@ -122,13 +124,13 @@ where // Start the `http_server` service. // // Note: presently we are ignoring the config and _always_ starting a HTTP server. - let http_exit_signal = if config.http_conf.enabled { + let http_exit_signal = if config.http.enabled { Some(http_server::start_service( - &config.http_conf, + &config.http, executor, network_send, beacon_chain.clone(), - config.db_name.clone(), + config.db_path().expect("unable to read datadir"), metrics_registry, &log, )) @@ -141,7 +143,7 @@ where // set up the validator work interval - start at next slot and proceed every slot let interval = { // Set the interval to start at the next slot, and every slot after - let slot_duration = Duration::from_secs(config.spec.seconds_per_slot); + let slot_duration = Duration::from_secs(spec.seconds_per_slot); //TODO: Handle checked add correctly Interval::new(Instant::now() + duration_to_next_slot, slot_duration) }; diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index d9c43b23c0..cc6393e388 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -6,9 +6,12 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } +clap = "2.32.0" # SigP repository until PR is merged libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "b3c32d9a821ae6cc89079499cc6e8a6bab0bffc3" } types = { path = "../../eth2/types" } +serde = "1.0" +serde_derive = "1.0" ssz = { path = "../../eth2/utils/ssz" } ssz_derive = { path = "../../eth2/utils/ssz_derive" } slog = "2.4.1" diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 2651006585..2a27018838 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -1,20 +1,22 @@ -use crate::Multiaddr; +use clap::ArgMatches; use libp2p::gossipsub::{GossipsubConfig, GossipsubConfigBuilder}; +use serde_derive::{Deserialize, Serialize}; +use types::multiaddr::{Error as MultiaddrError, Multiaddr}; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(default)] /// Network configuration for lighthouse. pub struct Config { - //TODO: stubbing networking initial params, change in the future /// IP address to listen on. - pub listen_addresses: Vec, - /// Listen port UDP/TCP. - pub listen_port: u16, + listen_addresses: Vec, /// Gossipsub configuration parameters. + #[serde(skip)] pub gs_config: GossipsubConfig, /// Configuration parameters for node identification protocol. + #[serde(skip)] pub identify_config: IdentifyConfig, /// List of nodes to initially connect to. - pub boot_nodes: Vec, + boot_nodes: Vec, /// Client version pub client_version: String, /// List of topics to subscribe to as strings @@ -25,15 +27,12 @@ impl Default for Config { /// Generate a default network configuration. fn default() -> Self { Config { - listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000" - .parse() - .expect("is a correct multi-address")], - listen_port: 9000, + listen_addresses: vec!["/ip4/127.0.0.1/tcp/9000".to_string()], gs_config: GossipsubConfigBuilder::new() .max_gossip_size(4_000_000) .build(), identify_config: IdentifyConfig::default(), - boot_nodes: Vec::new(), + boot_nodes: vec![], client_version: version::version(), topics: vec![String::from("beacon_chain")], } @@ -41,12 +40,34 @@ impl Default for Config { } impl Config { - pub fn new(boot_nodes: Vec) -> Self { + pub fn new(boot_nodes: Vec) -> Self { let mut conf = Config::default(); conf.boot_nodes = boot_nodes; conf } + + pub fn listen_addresses(&self) -> Result, MultiaddrError> { + self.listen_addresses.iter().map(|s| s.parse()).collect() + } + + pub fn boot_nodes(&self) -> Result, MultiaddrError> { + self.boot_nodes.iter().map(|s| s.parse()).collect() + } + + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if let Some(listen_address_str) = args.value_of("listen-address") { + let listen_addresses = listen_address_str.split(",").map(Into::into).collect(); + self.listen_addresses = listen_addresses; + } + + if let Some(boot_addresses_str) = args.value_of("boot-nodes") { + let boot_addresses = boot_addresses_str.split(",").map(Into::into).collect(); + self.boot_nodes = boot_addresses; + } + + Ok(()) + } } /// The configuration parameters for the Identify protocol diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 07a36e408c..b18a7dc512 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -57,7 +57,7 @@ impl Service { }; // listen on all addresses - for address in &config.listen_addresses { + for address in config.listen_addresses().expect("invalid listen multiaddr") { match Swarm::listen_on(&mut swarm, address.clone()) { Ok(mut listen_addr) => { listen_addr.append(Protocol::P2p(local_peer_id.clone().into())); @@ -68,7 +68,7 @@ impl Service { } // connect to boot nodes - these are currently stored as multiaddrs // Once we have discovery, can set to peerId - for bootnode in config.boot_nodes { + for bootnode in config.boot_nodes().expect("invalid boot node multiaddr") { match Swarm::dial_addr(&mut swarm, bootnode.clone()) { Ok(()) => debug!(log, "Dialing bootnode: {}", bootnode), Err(err) => debug!( diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs index fb94348265..ab1176d612 100644 --- a/beacon_node/http_server/src/lib.rs +++ b/beacon_node/http_server/src/lib.rs @@ -3,17 +3,19 @@ mod key; mod metrics; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use clap::ArgMatches; use futures::Future; use iron::prelude::*; use network::NetworkMessage; use prometheus::Registry; use router::Router; +use serde_derive::{Deserialize, Serialize}; use slog::{info, o, warn}; use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::TaskExecutor; -#[derive(PartialEq, Clone, Debug)] +#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct HttpServerConfig { pub enabled: bool, pub listen_address: String, @@ -30,6 +32,24 @@ impl Default for HttpServerConfig { } } +impl HttpServerConfig { + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if args.is_present("http") { + self.enabled = true; + } + + if let Some(listen_address) = args.value_of("http-address") { + self.listen_address = listen_address.to_string(); + } + + if let Some(listen_port) = args.value_of("http-port") { + self.listen_port = listen_port.to_string(); + } + + Ok(()) + } +} + /// Build the `iron` HTTP server, defining the core routes. pub fn create_iron_http_server( beacon_chain: Arc>, diff --git a/beacon_node/rpc/Cargo.toml b/beacon_node/rpc/Cargo.toml index a361c94abc..d707cc36d4 100644 --- a/beacon_node/rpc/Cargo.toml +++ b/beacon_node/rpc/Cargo.toml @@ -20,6 +20,8 @@ clap = "2.32.0" store = { path = "../store" } dirs = "1.0.3" futures = "0.1.23" +serde = "1.0" +serde_derive = "1.0" slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" diff --git a/beacon_node/rpc/src/config.rs b/beacon_node/rpc/src/config.rs index e21c2f7a89..0f031ddc60 100644 --- a/beacon_node/rpc/src/config.rs +++ b/beacon_node/rpc/src/config.rs @@ -1,7 +1,9 @@ +use clap::ArgMatches; +use serde_derive::{Deserialize, Serialize}; use std::net::Ipv4Addr; /// RPC Configuration -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { /// Enable the RPC server. pub enabled: bool, @@ -20,3 +22,23 @@ impl Default for Config { } } } + +impl Config { + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if args.is_present("rpc") { + self.enabled = true; + } + + if let Some(rpc_address) = args.value_of("rpc-address") { + self.listen_address = rpc_address + .parse::() + .map_err(|_| "rpc-address is not IPv4 address")?; + } + + if let Some(rpc_port) = args.value_of("rpc-port") { + self.port = rpc_port.parse::().map_err(|_| "rpc-port is not u16")?; + } + + Ok(()) + } +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 65f1899a01..68dbf6a741 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -4,7 +4,7 @@ mod run; use clap::{App, Arg}; use client::ClientConfig; -use slog::{error, o, Drain}; +use slog::{crit, o, Drain}; fn main() { let decorator = slog_term::TermDecorator::new().build(); @@ -29,21 +29,14 @@ fn main() { Arg::with_name("listen-address") .long("listen-address") .value_name("Listen Address") - .help("The Network address to listen for p2p connections.") - .takes_value(true), - ) - .arg( - Arg::with_name("port") - .long("port") - .value_name("PORT") - .help("Network listen port for p2p connections.") + .help("One or more comma-delimited multi-addresses to listen for p2p connections.") .takes_value(true), ) .arg( Arg::with_name("boot-nodes") .long("boot-nodes") .value_name("BOOTNODES") - .help("A list of comma separated multi addresses representing bootnodes to connect to.") + .help("One or more comma-delimited multi-addresses to bootstrap the p2p network.") .takes_value(true), ) // rpc related arguments @@ -101,11 +94,18 @@ fn main() { ) .get_matches(); - // invalid arguments, panic - let config = ClientConfig::parse_args(matches, &logger).unwrap(); + let mut config = ClientConfig::default(); + + match config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(logger, "Failed to parse CLI arguments"; "error" => s); + return; + } + }; match run::run_beacon_node(config, &logger) { Ok(_) => {} - Err(e) => error!(logger, "Beacon node failed because {:?}", e), + Err(e) => crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), } } diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index d8ff202bf2..f3d656ad31 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,11 +1,13 @@ use client::{ - error, notifier, BeaconChainTypes, Client, ClientConfig, DBType, TestnetDiskBeaconChainTypes, - TestnetMemoryBeaconChainTypes, + error, notifier, BeaconChainTypes, Client, ClientConfig, InitialiseBeaconChain, + TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes, }; use futures::sync::oneshot; use futures::Future; -use slog::info; +use slog::{error, info}; use std::cell::RefCell; +use std::path::Path; +use std::path::PathBuf; use store::{DiskStore, MemoryStore}; use tokio::runtime::Builder; use tokio::runtime::Runtime; @@ -19,51 +21,58 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul .build() .map_err(|e| format!("{:?}", e))?; - // Log configuration - info!(log, "Listening on {:?}", &config.net_conf.listen_addresses; - "data_dir" => &config.data_dir.to_str(), - "port" => &config.net_conf.listen_port); - let executor = runtime.executor(); - match config.db_type { - DBType::Disk => { - info!( - log, - "BeaconNode starting"; - "type" => "TestnetDiskBeaconChainTypes" - ); + let db_path: PathBuf = config + .db_path() + .ok_or_else::(|| "Unable to access database path".into())?; + let db_type = &config.db_type; + let spec = &config.spec; - let store = DiskStore::open(&config.db_name).expect("Unable to open DB."); + let other_config = config.clone(); - let client: Client = - Client::new(config, store, log.clone(), &executor)?; - - run(client, executor, runtime, log) + let result = match (db_type.as_str(), spec.as_str()) { + ("disk", "testnet") => { + run::(&db_path, config, executor, runtime, log) } - DBType::Memory => { - info!( - log, - "BeaconNode starting"; - "type" => "TestnetMemoryBeaconChainTypes" - ); - - let store = MemoryStore::open(); - - let client: Client = - Client::new(config, store, log.clone(), &executor)?; - - run(client, executor, runtime, log) + ("memory", "testnet") => { + run::(&db_path, config, executor, runtime, log) } + (db_type, spec) => { + error!(log, "Unknown runtime configuration"; "spec" => spec, "db_type" => db_type); + Err("Unknown specification and/or db_type.".into()) + } + }; + + if result.is_ok() { + info!( + log, + "Started beacon node"; + "p2p_listen_addresses" => format!("{:?}", &other_config.network.listen_addresses()), + "data_dir" => format!("{:?}", other_config.data_dir()), + "spec" => &other_config.spec, + "db_type" => &other_config.db_type, + ); } + + result } -pub fn run( - client: Client, +pub fn run( + db_path: &Path, + config: ClientConfig, executor: TaskExecutor, mut runtime: Runtime, log: &slog::Logger, -) -> error::Result<()> { +) -> error::Result<()> +where + T: BeaconChainTypes + InitialiseBeaconChain + Send + Sync + 'static + Clone, + T::Store: OpenDatabase, +{ + let store = T::Store::open_database(&db_path)?; + + let client: Client = Client::new(config, store, log.clone(), &executor)?; + // run service until ctrl-c let (ctrlc_send, ctrlc_oneshot) = oneshot::channel(); let ctrlc_send_c = RefCell::new(Some(ctrlc_send)); @@ -91,3 +100,22 @@ pub fn run( runtime.shutdown_on_idle().wait().unwrap(); Ok(()) } + +/// A convenience trait, providing a method to open a database. +/// +/// Panics if unable to open the database. +pub trait OpenDatabase: Sized { + fn open_database(path: &Path) -> error::Result; +} + +impl OpenDatabase for MemoryStore { + fn open_database(_path: &Path) -> error::Result { + Ok(MemoryStore::open()) + } +} + +impl OpenDatabase for DiskStore { + fn open_database(path: &Path) -> error::Result { + DiskStore::open(path).map_err(|e| format!("Unable to open database: {:?}", e).into()) + } +} diff --git a/beacon_node/store/src/disk_db.rs b/beacon_node/store/src/disk_db.rs index eb2b885c6b..669547ab97 100644 --- a/beacon_node/store/src/disk_db.rs +++ b/beacon_node/store/src/disk_db.rs @@ -1,6 +1,5 @@ extern crate rocksdb; -// use super::stores::COLUMNS; use super::{ClientDB, DBError, DBValue}; use rocksdb::Error as RocksError; use rocksdb::{Options, DB}; From f69d9093a3c93e484489d4953a8fef823f010f12 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 7 Jun 2019 21:00:34 -0400 Subject: [PATCH 61/76] Add concept of beacon node configuration TOML --- beacon_node/Cargo.toml | 2 + beacon_node/client/src/client_config.rs | 2 +- beacon_node/src/main.rs | 57 ++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index f1f47bef3e..3107068d9c 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -5,7 +5,9 @@ authors = ["Paul Hauner ", "Age Manning c, + Err(e) => { + crit!(logger, "Failed to load/generate a ChainConfig"; "error" => format!("{:?}", e)); + return; + } + }; match config.apply_cli_args(&matches) { Ok(()) => (), @@ -109,3 +123,42 @@ fn main() { Err(e) => crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), } } + +/// Loads a `ClientConfig` from file. If unable to load from file, generates a default +/// configuration and saves that as a sample file. +fn load_config(data_dir: Option<&str>) -> Result { + let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); + + let path = dirs::home_dir() + .ok_or_else(|| "Unable to locate home directory")? + .join(&data_dir); + fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; + + if let Ok(mut file) = File::open(path.join(CONFIG_FILENAME)) { + let mut contents = String::new(); + file.read_to_string(&mut contents).map_err(|e| { + format!( + "Unable to read existing {}. Error: {:?}", + CONFIG_FILENAME, e + ) + })?; + + toml::from_str(&contents).map_err(|_| format!("Unable to parse {}", CONFIG_FILENAME)) + } else { + let mut config = ClientConfig::default(); + config.data_dir = data_dir.to_string(); + + if let Ok(mut file) = File::create(path.join(SAMPLE_CONFIG_FILENAME)) { + let toml_encoded = toml::to_string(&config).map_err(|e| { + format!( + "Failed to write configuration to {}. Error: {:?}", + SAMPLE_CONFIG_FILENAME, e + ) + })?; + file.write_all(toml_encoded.as_bytes()) + .expect(&format!("Unable to write to {}", SAMPLE_CONFIG_FILENAME)); + } + + Ok(config) + } +} From e74d49fc8aa3b2943b63b6d3b83add023274bc2e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 07:57:25 -0400 Subject: [PATCH 62/76] Remove dupe info between ChainSpec and EthSpec --- beacon_node/beacon_chain/src/beacon_chain.rs | 91 ++++++++----------- beacon_node/beacon_chain/src/iter.rs | 7 +- beacon_node/client/src/beacon_chain_types.rs | 18 ++-- beacon_node/client/src/client_config.rs | 7 +- beacon_node/client/src/lib.rs | 11 ++- beacon_node/eth2-libp2p/src/behaviour.rs | 2 +- beacon_node/http_server/src/api.rs | 3 +- beacon_node/network/src/sync/simple_sync.rs | 10 +- beacon_node/rpc/src/beacon_node.rs | 3 +- beacon_node/rpc/src/validator.rs | 10 +- beacon_node/src/run.rs | 6 +- beacon_node/store/src/block_at_slot.rs | 8 +- eth2/fork_choice/benches/benches.rs | 2 +- eth2/fork_choice/examples/example.rs | 2 +- eth2/fork_choice/src/bitwise_lmd_ghost.rs | 2 +- eth2/fork_choice/src/optimized_lmd_ghost.rs | 2 +- eth2/fork_choice/src/slow_lmd_ghost.rs | 2 +- eth2/fork_choice/src/test_utils.rs | 6 +- eth2/fork_choice/tests/tests.rs | 4 +- eth2/operation_pool/src/lib.rs | 12 +-- .../benches/bench_epoch_processing.rs | 6 +- eth2/state_processing/benches/benches.rs | 6 +- .../src/per_block_processing.rs | 4 +- .../block_processing_builder.rs | 8 +- .../src/per_block_processing/tests.rs | 15 +-- .../validate_attestation.rs | 2 +- .../verify_proposer_slashing.rs | 12 +-- .../per_block_processing/verify_transfer.rs | 4 +- .../src/per_epoch_processing.rs | 2 +- .../src/per_epoch_processing/tests.rs | 4 +- .../validator_statuses.rs | 6 +- .../src/per_slot_processing.rs | 2 +- eth2/types/src/beacon_state.rs | 2 +- .../src/beacon_state/beacon_state_types.rs | 45 +++++---- .../types/src/beacon_state/committee_cache.rs | 17 +++- .../src/beacon_state/committee_cache/tests.rs | 28 +++--- eth2/types/src/beacon_state/tests.rs | 22 ++--- eth2/types/src/chain_spec.rs | 18 ++-- .../testing_attestation_data_builder.rs | 8 +- .../builders/testing_beacon_block_builder.rs | 34 ++++--- .../builders/testing_beacon_state_builder.rs | 8 +- .../testing_proposer_slashing_builder.rs | 7 +- .../builders/testing_transfer_builder.rs | 4 +- .../src/cases/epoch_processing_crosslinks.rs | 4 +- .../epoch_processing_registry_updates.rs | 2 +- .../src/cases/operations_attester_slashing.rs | 5 +- .../ef_tests/src/cases/operations_deposit.rs | 2 +- tests/ef_tests/src/cases/operations_exit.rs | 4 +- .../src/cases/operations_proposer_slashing.rs | 5 +- .../ef_tests/src/cases/operations_transfer.rs | 4 +- tests/ef_tests/src/cases/shuffling.rs | 2 +- tests/ef_tests/src/eth_specs.rs | 8 +- .../src/attestation_producer/mod.rs | 4 +- validator_client/src/block_producer/mod.rs | 10 +- validator_client/src/config.rs | 10 +- validator_client/src/duties/mod.rs | 2 +- validator_client/src/service.rs | 17 +++- 57 files changed, 299 insertions(+), 252 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a9374aa6a8..4d77d7aa5c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -96,6 +96,7 @@ pub trait BeaconChainTypes { /// Represents the "Beacon Chain" component of Ethereum 2.0. Allows import of blocks and block /// operations and chooses a canonical head. pub struct BeaconChain { + pub spec: ChainSpec, /// Persistent storage for blocks, states, etc. Typically an on-disk store, such as LevelDB. pub store: Arc, /// Reports the current slot, typically based upon the system clock. @@ -148,6 +149,7 @@ impl BeaconChain { genesis_state.build_all_caches(&spec)?; Ok(Self { + spec, store, slot_clock, op_pool: OperationPool::new(), @@ -160,7 +162,10 @@ impl BeaconChain { } /// Attempt to load an existing instance from the given `store`. - pub fn from_store(store: Arc) -> Result>, Error> { + pub fn from_store( + store: Arc, + spec: ChainSpec, + ) -> Result>, Error> { let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes()); let p: PersistedBeaconChain = match store.get(&key) { Err(e) => return Err(e.into()), @@ -168,8 +173,6 @@ impl BeaconChain { Ok(Some(p)) => p, }; - let spec = T::EthSpec::spec(); - let slot_clock = T::SlotClock::new( spec.genesis_slot, p.state.genesis_time, @@ -179,6 +182,7 @@ impl BeaconChain { let fork_choice = T::ForkChoice::new(store.clone()); Ok(Some(BeaconChain { + spec, store, slot_clock, op_pool: OperationPool::default(), @@ -363,10 +367,10 @@ impl BeaconChain { // If required, transition the new state to the present slot. for _ in state.slot.as_u64()..present_slot.as_u64() { - per_slot_processing(&mut state, &T::EthSpec::spec())?; + per_slot_processing(&mut state, &self.spec)?; } - state.build_all_caches(&T::EthSpec::spec())?; + state.build_all_caches(&self.spec)?; state }; @@ -400,7 +404,7 @@ impl BeaconChain { /// Ensures the current canonical `BeaconState` has been transitioned to match the `slot_clock`. pub fn catchup_state(&self) -> Result<(), Error> { - let spec = &T::EthSpec::spec(); + let spec = &self.spec; let present_slot = match self.slot_clock.present_slot() { Ok(Some(slot)) => slot, @@ -426,7 +430,7 @@ impl BeaconChain { /// /// Ideally this shouldn't be required, however we leave it here for testing. pub fn ensure_state_caches_are_built(&self) -> Result<(), Error> { - self.state.write().build_all_caches(&T::EthSpec::spec())?; + self.state.write().build_all_caches(&self.spec)?; Ok(()) } @@ -469,7 +473,7 @@ impl BeaconChain { /// genesis. pub fn slots_since_genesis(&self) -> Option { let now = self.read_slot_clock()?; - let genesis_slot = T::EthSpec::spec().genesis_slot; + let genesis_slot = self.spec.genesis_slot; if now < genesis_slot { None @@ -494,12 +498,12 @@ impl BeaconChain { pub fn block_proposer(&self, slot: Slot) -> Result { self.state .write() - .build_committee_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; + .build_committee_cache(RelativeEpoch::Current, &self.spec)?; let index = self.state.read().get_beacon_proposer_index( slot, RelativeEpoch::Current, - &T::EthSpec::spec(), + &self.spec, )?; Ok(index) @@ -530,7 +534,7 @@ impl BeaconChain { /// Produce an `AttestationData` that is valid for the present `slot` and given `shard`. pub fn produce_attestation_data(&self, shard: u64) -> Result { - let slots_per_epoch = T::EthSpec::spec().slots_per_epoch; + let slots_per_epoch = T::EthSpec::slots_per_epoch(); self.metrics.attestation_production_requests.inc(); let timer = self.metrics.attestation_production_times.start_timer(); @@ -591,9 +595,9 @@ impl BeaconChain { self.metrics.attestation_processing_requests.inc(); let timer = self.metrics.attestation_processing_times.start_timer(); - let result = - self.op_pool - .insert_attestation(attestation, &*self.state.read(), &T::EthSpec::spec()); + let result = self + .op_pool + .insert_attestation(attestation, &*self.state.read(), &self.spec); if result.is_ok() { self.metrics.attestation_processing_successes.inc(); @@ -610,19 +614,19 @@ impl BeaconChain { deposit: Deposit, ) -> Result { self.op_pool - .insert_deposit(deposit, &*self.state.read(), &T::EthSpec::spec()) + .insert_deposit(deposit, &*self.state.read(), &self.spec) } /// Accept some exit and queue it for inclusion in an appropriate block. pub fn process_voluntary_exit(&self, exit: VoluntaryExit) -> Result<(), ExitValidationError> { self.op_pool - .insert_voluntary_exit(exit, &*self.state.read(), &T::EthSpec::spec()) + .insert_voluntary_exit(exit, &*self.state.read(), &self.spec) } /// Accept some transfer and queue it for inclusion in an appropriate block. pub fn process_transfer(&self, transfer: Transfer) -> Result<(), TransferValidationError> { self.op_pool - .insert_transfer(transfer, &*self.state.read(), &T::EthSpec::spec()) + .insert_transfer(transfer, &*self.state.read(), &self.spec) } /// Accept some proposer slashing and queue it for inclusion in an appropriate block. @@ -630,11 +634,8 @@ impl BeaconChain { &self, proposer_slashing: ProposerSlashing, ) -> Result<(), ProposerSlashingValidationError> { - self.op_pool.insert_proposer_slashing( - proposer_slashing, - &*self.state.read(), - &T::EthSpec::spec(), - ) + self.op_pool + .insert_proposer_slashing(proposer_slashing, &*self.state.read(), &self.spec) } /// Accept some attester slashing and queue it for inclusion in an appropriate block. @@ -642,11 +643,8 @@ impl BeaconChain { &self, attester_slashing: AttesterSlashing, ) -> Result<(), AttesterSlashingValidationError> { - self.op_pool.insert_attester_slashing( - attester_slashing, - &*self.state.read(), - &T::EthSpec::spec(), - ) + self.op_pool + .insert_attester_slashing(attester_slashing, &*self.state.read(), &self.spec) } /// Accept some block and attempt to add it to block DAG. @@ -708,18 +706,18 @@ impl BeaconChain { // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, &T::EthSpec::spec()) { + if let Err(e) = per_slot_processing(&mut state, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::SlotProcessingError(e), )); } } - state.build_committee_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; + state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; // Apply the received block to its parent state (which has been transitioned into this // slot). - if let Err(e) = per_block_processing(&mut state, &block, &T::EthSpec::spec()) { + if let Err(e) = per_block_processing(&mut state, &block, &self.spec) { return Ok(BlockProcessingOutcome::InvalidBlock( InvalidBlock::PerBlockProcessingError(e), )); @@ -740,7 +738,7 @@ impl BeaconChain { // Register the new block with the fork choice service. self.fork_choice .write() - .add_block(&block, &block_root, &T::EthSpec::spec())?; + .add_block(&block, &block_root, &self.spec)?; // Execute the fork choice algorithm, enthroning a new head if discovered. // @@ -771,7 +769,7 @@ impl BeaconChain { let mut state = self.state.read().clone(); - state.build_committee_cache(RelativeEpoch::Current, &T::EthSpec::spec())?; + state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; trace!("Finding attestations for new block..."); @@ -783,9 +781,8 @@ impl BeaconChain { state.latest_block_header.canonical_root() }; - let (proposer_slashings, attester_slashings) = self - .op_pool - .get_slashings(&*self.state.read(), &T::EthSpec::spec()); + let (proposer_slashings, attester_slashings) = + self.op_pool.get_slashings(&*self.state.read(), &self.spec); let mut block = BeaconBlock { slot: state.slot, @@ -806,16 +803,12 @@ impl BeaconChain { attester_slashings, attestations: self .op_pool - .get_attestations(&*self.state.read(), &T::EthSpec::spec()), - deposits: self - .op_pool - .get_deposits(&*self.state.read(), &T::EthSpec::spec()), + .get_attestations(&*self.state.read(), &self.spec), + deposits: self.op_pool.get_deposits(&*self.state.read(), &self.spec), voluntary_exits: self .op_pool - .get_voluntary_exits(&*self.state.read(), &T::EthSpec::spec()), - transfers: self - .op_pool - .get_transfers(&*self.state.read(), &T::EthSpec::spec()), + .get_voluntary_exits(&*self.state.read(), &self.spec), + transfers: self.op_pool.get_transfers(&*self.state.read(), &self.spec), }, }; @@ -824,11 +817,7 @@ impl BeaconChain { block.body.attestations.len() ); - per_block_processing_without_verifying_block_signature( - &mut state, - &block, - &T::EthSpec::spec(), - )?; + per_block_processing_without_verifying_block_signature(&mut state, &block, &self.spec)?; let state_root = state.canonical_root(); @@ -849,7 +838,7 @@ impl BeaconChain { let justified_root = { let root = self.head().beacon_state.current_justified_root; - if root == T::EthSpec::spec().zero_hash { + if root == self.spec.zero_hash { self.genesis_block_root } else { root @@ -860,7 +849,7 @@ impl BeaconChain { let beacon_block_root = self .fork_choice .write() - .find_head(&justified_root, &T::EthSpec::spec())?; + .find_head(&justified_root, &self.spec)?; // End fork choice metrics timer. timer.observe_duration(); @@ -920,7 +909,7 @@ impl BeaconChain { loop { let beacon_block_root = last_slot.beacon_block.previous_block_root; - if beacon_block_root == T::EthSpec::spec().zero_hash { + if beacon_block_root == self.spec.zero_hash { break; // Genesis has been reached. } diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index 3147fb2078..48caaf6185 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -85,8 +85,11 @@ mod test { use types::{test_utils::TestingBeaconStateBuilder, FoundationEthSpec, Keypair}; fn get_state() -> BeaconState { - let builder = - TestingBeaconStateBuilder::from_single_keypair(0, &Keypair::random(), &T::spec()); + let builder = TestingBeaconStateBuilder::from_single_keypair( + 0, + &Keypair::random(), + &T::default_spec(), + ); let (state, _keypairs) = builder.build(); state } diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 37b1d261d9..9747b1dd81 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -10,7 +10,8 @@ use slot_clock::SlotClock; use std::sync::Arc; use tree_hash::TreeHash; use types::{ - test_utils::TestingBeaconStateBuilder, BeaconBlock, EthSpec, Hash256, LighthouseTestnetEthSpec, + test_utils::TestingBeaconStateBuilder, BeaconBlock, ChainSpec, EthSpec, Hash256, + LighthouseTestnetEthSpec, }; /// The number initial validators when starting the `LighthouseTestnet`. @@ -18,8 +19,12 @@ const TESTNET_VALIDATOR_COUNT: usize = 16; /// Provides a new, initialized `BeaconChain` pub trait InitialiseBeaconChain { - fn initialise_beacon_chain(store: Arc, log: Logger) -> BeaconChain { - maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, log) + fn initialise_beacon_chain( + store: Arc, + spec: ChainSpec, + log: Logger, + ) -> BeaconChain { + maybe_load_from_store_for_testnet::<_, T::Store, T::EthSpec>(store, spec, log) } } @@ -48,13 +53,14 @@ impl InitialiseBeaconChain for TestnetDiskBeaconChainTyp /// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. fn maybe_load_from_store_for_testnet( store: Arc, + spec: ChainSpec, log: Logger, ) -> BeaconChain where T: BeaconChainTypes, T::ForkChoice: ForkChoice, { - if let Ok(Some(beacon_chain)) = BeaconChain::from_store(store.clone()) { + if let Ok(Some(beacon_chain)) = BeaconChain::from_store(store.clone(), spec.clone()) { info!( log, "Loaded BeaconChain from store"; @@ -65,8 +71,6 @@ where beacon_chain } else { info!(log, "Initializing new BeaconChain from genesis"); - let spec = T::EthSpec::spec(); - let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists( TESTNET_VALIDATOR_COUNT, &spec, @@ -92,7 +96,7 @@ where slot_clock, genesis_state, genesis_block, - spec.clone(), + spec, fork_choice, ) .expect("Terminate if beacon chain generation fails") diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index baa9205ce8..a729531ad4 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -4,24 +4,26 @@ use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; +use types::ChainSpec; /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClientConfig { pub data_dir: String, - pub spec: String, + pub spec_constants: String, pub db_type: String, db_name: String, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, pub http: HttpServerConfig, //pub ipc_conf: + pub spec: ChainSpec, } impl Default for ClientConfig { fn default() -> Self { Self { data_dir: ".lighthouse".to_string(), - spec: "testnet".to_string(), + spec_constants: "testnet".to_string(), db_type: "disk".to_string(), db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. @@ -29,6 +31,7 @@ impl Default for ClientConfig { network: NetworkConfig::new(vec![]), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), + spec: ChainSpec::lighthouse_testnet(8), } } } diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 36d1c7d1f8..ae28858c5c 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -17,7 +17,6 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; -use types::EthSpec; pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::InitialiseBeaconChain; @@ -58,10 +57,14 @@ where ) -> error::Result { let metrics_registry = Registry::new(); let store = Arc::new(store); - let spec = T::EthSpec::spec(); + let seconds_per_slot = config.spec.seconds_per_slot; // Load a `BeaconChain` from the store, or create a new one if it does not exist. - let beacon_chain = Arc::new(T::initialise_beacon_chain(store, log.clone())); + let beacon_chain = Arc::new(T::initialise_beacon_chain( + store, + config.spec.clone(), + log.clone(), + )); // Registry all beacon chain metrics with the global registry. beacon_chain .metrics @@ -143,7 +146,7 @@ where // set up the validator work interval - start at next slot and proceed every slot let interval = { // Set the interval to start at the next slot, and every slot after - let slot_duration = Duration::from_secs(spec.seconds_per_slot); + let slot_duration = Duration::from_secs(seconds_per_slot); //TODO: Handle checked add correctly Interval::new(Instant::now() + duration_to_next_slot, slot_duration) }; diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 8f3a000e12..476cddfbb1 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -261,7 +261,7 @@ mod test { #[test] fn ssz_encoding() { - let original = PubsubMessage::Block(BeaconBlock::empty(&FoundationEthSpec::spec())); + let original = PubsubMessage::Block(BeaconBlock::empty(&FoundationEthSpec::default_spec())); let encoded = ssz_encode(&original); diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs index 2594f9c288..a910808998 100644 --- a/beacon_node/http_server/src/api.rs +++ b/beacon_node/http_server/src/api.rs @@ -10,7 +10,6 @@ use persistent::Read; use router::Router; use serde_json::json; use std::sync::Arc; -use types::EthSpec; /// Yields a handler for the HTTP API. pub fn build_handler( @@ -65,7 +64,7 @@ fn handle_fork(req: &mut Request) -> IronResult SimpleSync { hello: HelloMessage, network: &mut NetworkContext, ) { - let spec = T::EthSpec::spec(); + let spec = &self.chain.spec; let remote = PeerSyncInfo::from(hello); let local = PeerSyncInfo::from(&self.chain); @@ -214,7 +214,7 @@ impl SimpleSync { debug!(self.log, "Peer has high finalized epoch"; "peer" => format!("{:?}", peer_id)); let start_slot = local .latest_finalized_epoch - .start_slot(spec.slots_per_epoch); + .start_slot(T::EthSpec::slots_per_epoch()); let required_slots = remote.best_slot - start_slot; self.request_block_roots( @@ -231,7 +231,7 @@ impl SimpleSync { debug!(self.log, "Peer has higher best slot"; "peer" => format!("{:?}", peer_id)); let start_slot = local .latest_finalized_epoch - .start_slot(spec.slots_per_epoch); + .start_slot(T::EthSpec::slots_per_epoch()); let required_slots = remote.best_slot - start_slot; self.request_block_roots( @@ -795,7 +795,7 @@ impl SimpleSync { fn slot_is_finalized(&self, slot: Slot) -> bool { slot <= hello_message(&self.chain) .latest_finalized_epoch - .start_slot(T::EthSpec::spec().slots_per_epoch) + .start_slot(T::EthSpec::slots_per_epoch()) } /// Generates our current state in the form of a HELLO RPC message. @@ -806,7 +806,7 @@ impl SimpleSync { /// Build a `HelloMessage` representing the state of the given `beacon_chain`. fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { - let spec = T::EthSpec::spec(); + let spec = &beacon_chain.spec; let state = &beacon_chain.head().beacon_state; HelloMessage { diff --git a/beacon_node/rpc/src/beacon_node.rs b/beacon_node/rpc/src/beacon_node.rs index 8b49b193e8..631601ac95 100644 --- a/beacon_node/rpc/src/beacon_node.rs +++ b/beacon_node/rpc/src/beacon_node.rs @@ -5,7 +5,6 @@ use protos::services::{Empty, Fork, NodeInfoResponse}; use protos::services_grpc::BeaconNodeService; use slog::{trace, warn}; use std::sync::Arc; -use types::EthSpec; #[derive(Clone)] pub struct BeaconNodeServiceInstance { @@ -33,7 +32,7 @@ impl BeaconNodeService for BeaconNodeServiceInstance { fork.set_current_version(state_fork.current_version.to_vec()); fork.set_epoch(state_fork.epoch.into()); - let spec = T::EthSpec::spec(); + let spec = &self.chain.spec; node_info.set_fork(fork); node_info.set_genesis_time(genesis_time); diff --git a/beacon_node/rpc/src/validator.rs b/beacon_node/rpc/src/validator.rs index 2cd374d254..b13303e25c 100644 --- a/beacon_node/rpc/src/validator.rs +++ b/beacon_node/rpc/src/validator.rs @@ -14,7 +14,6 @@ pub struct ValidatorServiceInstance { pub chain: Arc>, pub log: slog::Logger, } -//TODO: Refactor Errors impl ValidatorService for ValidatorServiceInstance { /// For a list of validator public keys, this function returns the slot at which each @@ -29,14 +28,15 @@ impl ValidatorService for ValidatorServiceInstance { let validators = req.get_validators(); trace!(self.log, "RPC request"; "endpoint" => "GetValidatorDuties", "epoch" => req.get_epoch()); - let spec = T::EthSpec::spec(); + let spec = &self.chain.spec; let state = &self.chain.current_state(); let epoch = Epoch::from(req.get_epoch()); let mut resp = GetDutiesResponse::new(); let resp_validators = resp.mut_active_validators(); let relative_epoch = - match RelativeEpoch::from_epoch(state.slot.epoch(spec.slots_per_epoch), epoch) { + match RelativeEpoch::from_epoch(state.slot.epoch(T::EthSpec::slots_per_epoch()), epoch) + { Ok(v) => v, Err(e) => { // incorrect epoch @@ -52,7 +52,7 @@ impl ValidatorService for ValidatorServiceInstance { }; let validator_proposers: Result, _> = epoch - .slot_iter(spec.slots_per_epoch) + .slot_iter(T::EthSpec::slots_per_epoch()) .map(|slot| state.get_beacon_proposer_index(slot, relative_epoch, &spec)) .collect(); let validator_proposers = match validator_proposers { @@ -148,7 +148,7 @@ impl ValidatorService for ValidatorServiceInstance { // check if the validator needs to propose a block if let Some(slot) = validator_proposers.iter().position(|&v| val_index == v) { duty.set_block_production_slot( - epoch.start_slot(spec.slots_per_epoch).as_u64() + slot as u64, + epoch.start_slot(T::EthSpec::slots_per_epoch()).as_u64() + slot as u64, ); } else { // no blocks to propose this epoch diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index f3d656ad31..0638e9ec93 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -27,11 +27,11 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul .db_path() .ok_or_else::(|| "Unable to access database path".into())?; let db_type = &config.db_type; - let spec = &config.spec; + let spec_constants = &config.spec_constants; let other_config = config.clone(); - let result = match (db_type.as_str(), spec.as_str()) { + let result = match (db_type.as_str(), spec_constants.as_str()) { ("disk", "testnet") => { run::(&db_path, config, executor, runtime, log) } @@ -50,7 +50,7 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul "Started beacon node"; "p2p_listen_addresses" => format!("{:?}", &other_config.network.listen_addresses()), "data_dir" => format!("{:?}", other_config.data_dir()), - "spec" => &other_config.spec, + "spec_constants" => &other_config.spec_constants, "db_type" => &other_config.db_type, ); } diff --git a/beacon_node/store/src/block_at_slot.rs b/beacon_node/store/src/block_at_slot.rs index 260a351147..b7346e90db 100644 --- a/beacon_node/store/src/block_at_slot.rs +++ b/beacon_node/store/src/block_at_slot.rs @@ -61,7 +61,7 @@ mod tests { #[test] fn read_slot() { - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let test_slot = |slot: Slot| { let mut block = BeaconBlock::empty(&spec); @@ -85,7 +85,7 @@ mod tests { #[test] fn read_previous_block_root() { - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let test_root = |root: Hash256| { let mut block = BeaconBlock::empty(&spec); @@ -130,7 +130,7 @@ mod tests { fn chain_without_skips() { let n: usize = 10; let store = MemoryStore::open(); - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let slots: Vec = (0..n).collect(); let blocks_and_roots = build_chain(&store, &slots, &spec); @@ -154,7 +154,7 @@ mod tests { #[test] fn chain_with_skips() { let store = MemoryStore::open(); - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let slots = vec![0, 1, 2, 5]; diff --git a/eth2/fork_choice/benches/benches.rs b/eth2/fork_choice/benches/benches.rs index 065cde6552..b0495a80c9 100644 --- a/eth2/fork_choice/benches/benches.rs +++ b/eth2/fork_choice/benches/benches.rs @@ -19,7 +19,7 @@ fn setup( let store = MemoryStore::open(); let builder: TestingForkChoiceBuilder = TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - let spec = TestedEthSpec::spec(); + let spec = TestedEthSpec::default_spec(); (builder, spec) } diff --git a/eth2/fork_choice/examples/example.rs b/eth2/fork_choice/examples/example.rs index 927cf23b76..7e42ad144b 100644 --- a/eth2/fork_choice/examples/example.rs +++ b/eth2/fork_choice/examples/example.rs @@ -17,7 +17,7 @@ fn main() { .map(|_| builder.build()) .collect(); - let spec = &FoundationEthSpec::spec(); + let spec = &FoundationEthSpec::default_spec(); println!("Running {} times...", repetitions); for fc in fork_choosers { diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs index f78a8e780a..3ed57bf4dd 100644 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ b/eth2/fork_choice/src/bitwise_lmd_ghost.rs @@ -68,7 +68,7 @@ impl BitwiseLMDGhost { .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(spec.slots_per_epoch)); + current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); for index in active_validator_indices { let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs index b389d0981a..7a48c461e4 100644 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ b/eth2/fork_choice/src/optimized_lmd_ghost.rs @@ -68,7 +68,7 @@ impl OptimizedLMDGhost { .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(spec.slots_per_epoch)); + current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); for index in active_validator_indices { let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs index 7b5114887e..9b7a204002 100644 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ b/eth2/fork_choice/src/slow_lmd_ghost.rs @@ -40,7 +40,7 @@ impl SlowLMDGhost { .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(spec.slots_per_epoch)); + current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); for index in active_validator_indices { let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) diff --git a/eth2/fork_choice/src/test_utils.rs b/eth2/fork_choice/src/test_utils.rs index 76264fcb63..c3bf39930f 100644 --- a/eth2/fork_choice/src/test_utils.rs +++ b/eth2/fork_choice/src/test_utils.rs @@ -40,10 +40,10 @@ impl TestingForkChoiceBuilder { } fn get_state(validator_count: usize) -> BeaconState { - let spec = &T::spec(); + let spec = T::default_spec(); let builder: TestingBeaconStateBuilder = - TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec); + TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), &spec); let (state, _keypairs) = builder.build(); state } @@ -58,7 +58,7 @@ fn get_chain_of_blocks( validator_count: usize, store: Arc, ) -> Vec<(Hash256, BeaconBlock)> { - let spec = T::spec(); + let spec = T::default_spec(); let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![]; let mut unique_hashes = (0..).into_iter().map(|i| Hash256::from(i)); let mut random_block = BeaconBlock::random_for_test(&mut XorShiftRng::from_seed([42; 16])); diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 9270571ca3..59e4e0ee09 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -61,7 +61,7 @@ fn test_yaml_vectors>( let test_cases = load_test_cases_from_yaml(yaml_file_path); // default vars - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let zero_hash = Hash256::zero(); let eth1_data = Eth1Data { deposit_count: 0, @@ -204,7 +204,7 @@ where let store = Arc::new(MemoryStore::open()); let fork_choice = ForkChoice::new(store.clone()); - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let mut state_builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec); diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index 0affba3f44..0cd46e0183 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -676,7 +676,7 @@ mod tests { } fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState) { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let mut state = BeaconState::random_for_test(rng); @@ -721,21 +721,21 @@ mod tests { fn attestation_test_state( num_committees: usize, ) -> (BeaconState, Vec, ChainSpec) { - let spec = E::spec(); + let spec = E::default_spec(); let num_validators = - num_committees * spec.slots_per_epoch as usize * spec.target_committee_size; + num_committees * T::slots_per_epoch() as usize * spec.target_committee_size; let mut state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists( num_validators, &spec, ); - let slot_offset = 1000 * spec.slots_per_epoch + spec.slots_per_epoch / 2; + let slot_offset = 1000 * T::slots_per_epoch() + T::slots_per_epoch() / 2; let slot = spec.genesis_slot + slot_offset; state_builder.teleport_to_slot(slot, &spec); state_builder.build_caches(&spec).unwrap(); let (state, keypairs) = state_builder.build(); - (state, keypairs, FoundationEthSpec::spec()) + (state, keypairs, FoundationEthSpec::default_spec()) } #[test] @@ -852,7 +852,7 @@ mod tests { // But once we advance to more than an epoch after the attestation, it should prune it // out of existence. - state.slot += 2 * spec.slots_per_epoch; + state.slot += 2 * T::slots_per_epoch(); op_pool.prune_attestations(state); assert_eq!(op_pool.num_attestations(), 0); } diff --git a/eth2/state_processing/benches/bench_epoch_processing.rs b/eth2/state_processing/benches/bench_epoch_processing.rs index 9bff3a2e3e..23e662e09d 100644 --- a/eth2/state_processing/benches/bench_epoch_processing.rs +++ b/eth2/state_processing/benches/bench_epoch_processing.rs @@ -23,7 +23,7 @@ pub fn bench_epoch_processing_n_validators(c: &mut Criterion, validator_count: u TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); // Set the state to be just before an epoch transition. - let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + let target_slot = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); builder.teleport_to_slot(target_slot, &spec); // Builds all caches; benches will not contain shuffling/committee building times. @@ -38,10 +38,10 @@ pub fn bench_epoch_processing_n_validators(c: &mut Criterion, validator_count: u // Assert that the state has an attestations for each committee that is able to include an // attestation in the state. let committees_per_epoch = spec.get_epoch_committee_count(validator_count); - let committees_per_slot = committees_per_epoch / spec.slots_per_epoch; + let committees_per_slot = committees_per_epoch / T::slots_per_epoch(); let previous_epoch_attestations = committees_per_epoch; let current_epoch_attestations = - committees_per_slot * (spec.slots_per_epoch - spec.min_attestation_inclusion_delay); + committees_per_slot * (T::slots_per_epoch() - spec.min_attestation_inclusion_delay); assert_eq!( state.latest_attestations.len() as u64, previous_epoch_attestations + current_epoch_attestations, diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 0cf797147f..1a153abee7 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -34,7 +34,7 @@ pub fn block_processing_worst_case(c: &mut Criterion) { bench_builder.maximize_block_operations(&spec); // Set the state and block to be in the last slot of the 4th epoch. - let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); bench_builder.set_slot(last_slot_of_epoch, &spec); // Build all the state caches so the build times aren't included in the benches. @@ -67,13 +67,13 @@ pub fn block_processing_reasonable_case(c: &mut Criterion) { // Set the number of included operations to what we might expect normally. bench_builder.num_proposer_slashings = 0; bench_builder.num_attester_slashings = 0; - bench_builder.num_attestations = (spec.shard_count / spec.slots_per_epoch) as usize; + bench_builder.num_attestations = (spec.shard_count / T::slots_per_epoch()) as usize; bench_builder.num_deposits = 2; bench_builder.num_exits = 2; bench_builder.num_transfers = 2; // Set the state and block to be in the last slot of the 4th epoch. - let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); bench_builder.set_slot(last_slot_of_epoch, &spec); // Build all the state caches so the build times aren't included in the benches. diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 56238f9c2f..437cd459cf 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -140,7 +140,7 @@ pub fn verify_block_signature( [state.get_beacon_proposer_index(block.slot, RelativeEpoch::Current, spec)?]; let domain = spec.get_domain( - block.slot.epoch(spec.slots_per_epoch), + block.slot.epoch(T::slots_per_epoch()), Domain::BeaconProposer, &state.fork, ); @@ -172,7 +172,7 @@ pub fn process_randao( block.body.randao_reveal.verify( &state.current_epoch().tree_hash_root()[..], spec.get_domain( - block.slot.epoch(spec.slots_per_epoch), + block.slot.epoch(T::slots_per_epoch()), Domain::Randao, &state.fork ), diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 35e736d5fe..5a9d264ce1 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -55,11 +55,13 @@ impl BlockProcessingBuilder { let keypair = &keypairs[proposer_index]; match randao_sk { - Some(sk) => builder.set_randao_reveal(&sk, &state.fork, spec), - None => builder.set_randao_reveal(&keypair.sk, &state.fork, spec), + Some(sk) => builder.set_randao_reveal::(&sk, &state.fork, spec), + None => builder.set_randao_reveal::(&keypair.sk, &state.fork, spec), } - let block = self.block_builder.build(&keypair.sk, &state.fork, spec); + let block = self + .block_builder + .build::(&keypair.sk, &state.fork, spec); (block, state) } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 28ed9c4f0c..fe30560c67 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -9,7 +9,7 @@ pub const VALIDATOR_COUNT: usize = 10; #[test] fn valid_block_ok() { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let builder = get_builder(&spec); let (block, mut state) = builder.build(None, None, &spec); @@ -20,7 +20,7 @@ fn valid_block_ok() { #[test] fn invalid_block_header_state_slot() { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let builder = get_builder(&spec); let (mut block, mut state) = builder.build(None, None, &spec); @@ -39,7 +39,7 @@ fn invalid_block_header_state_slot() { #[test] fn invalid_parent_block_root() { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let builder = get_builder(&spec); let invalid_parent_root = Hash256::from([0xAA; 32]); let (block, mut state) = builder.build(None, Some(invalid_parent_root), &spec); @@ -59,14 +59,14 @@ fn invalid_parent_block_root() { #[test] fn invalid_block_signature() { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let builder = get_builder(&spec); let (mut block, mut state) = builder.build(None, None, &spec); // sign the block with a keypair that is not the expected proposer let keypair = Keypair::random(); let message = block.signed_root(); - let epoch = block.slot.epoch(spec.slots_per_epoch); + let epoch = block.slot.epoch(FoundationEthSpec::slots_per_epoch()); let domain = spec.get_domain(epoch, Domain::BeaconProposer, &state.fork); block.signature = Signature::new(&message, domain, &keypair.sk); @@ -82,7 +82,7 @@ fn invalid_block_signature() { #[test] fn invalid_randao_reveal_signature() { - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); let builder = get_builder(&spec); // sign randao reveal with random keypair @@ -104,7 +104,8 @@ fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); // Set the state and block to be in the last slot of the 4th epoch. - let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + let last_slot_of_epoch = + (spec.genesis_epoch + 4).end_slot(FoundationEthSpec::slots_per_epoch()); builder.set_slot(last_slot_of_epoch, &spec); builder.build_caches(&spec); diff --git a/eth2/state_processing/src/per_block_processing/validate_attestation.rs b/eth2/state_processing/src/per_block_processing/validate_attestation.rs index 1058c0d218..379b921f4d 100644 --- a/eth2/state_processing/src/per_block_processing/validate_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/validate_attestation.rs @@ -68,7 +68,7 @@ fn validate_attestation_parametric( } ); verify!( - state.slot <= attestation_slot + spec.slots_per_epoch, + state.slot <= attestation_slot + T::slots_per_epoch(), Invalid::IncludedTooLate { state: state.slot, attestation: attestation_slot diff --git a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs index 98a9a248cc..744427ad98 100644 --- a/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs +++ b/eth2/state_processing/src/per_block_processing/verify_proposer_slashing.rs @@ -21,8 +21,8 @@ pub fn verify_proposer_slashing( })?; verify!( - proposer_slashing.header_1.slot.epoch(spec.slots_per_epoch) - == proposer_slashing.header_2.slot.epoch(spec.slots_per_epoch), + proposer_slashing.header_1.slot.epoch(T::slots_per_epoch()) + == proposer_slashing.header_2.slot.epoch(T::slots_per_epoch()), Invalid::ProposalEpochMismatch( proposer_slashing.header_1.slot, proposer_slashing.header_2.slot @@ -40,7 +40,7 @@ pub fn verify_proposer_slashing( ); verify!( - verify_header_signature( + verify_header_signature::( &proposer_slashing.header_1, &proposer.pubkey, &state.fork, @@ -49,7 +49,7 @@ pub fn verify_proposer_slashing( Invalid::BadProposal1Signature ); verify!( - verify_header_signature( + verify_header_signature::( &proposer_slashing.header_2, &proposer.pubkey, &state.fork, @@ -66,7 +66,7 @@ pub fn verify_proposer_slashing( /// Returns `true` if the signature is valid. /// /// Spec v0.6.1 -fn verify_header_signature( +fn verify_header_signature( header: &BeaconBlockHeader, pubkey: &PublicKey, fork: &Fork, @@ -74,7 +74,7 @@ fn verify_header_signature( ) -> bool { let message = header.signed_root(); let domain = spec.get_domain( - header.slot.epoch(spec.slots_per_epoch), + header.slot.epoch(T::slots_per_epoch()), Domain::BeaconProposer, fork, ); diff --git a/eth2/state_processing/src/per_block_processing/verify_transfer.rs b/eth2/state_processing/src/per_block_processing/verify_transfer.rs index 15c142d90d..de4cef44f0 100644 --- a/eth2/state_processing/src/per_block_processing/verify_transfer.rs +++ b/eth2/state_processing/src/per_block_processing/verify_transfer.rs @@ -101,7 +101,7 @@ fn verify_transfer_parametric( .get(transfer.sender as usize) .ok_or_else(|| Error::Invalid(Invalid::FromValidatorUnknown(transfer.sender)))?; - let epoch = state.slot.epoch(spec.slots_per_epoch); + let epoch = state.slot.epoch(T::slots_per_epoch()); // Ensure one of the following is met: // @@ -136,7 +136,7 @@ fn verify_transfer_parametric( // Verify the transfer signature. let message = transfer.signed_root(); let domain = spec.get_domain( - transfer.slot.epoch(spec.slots_per_epoch), + transfer.slot.epoch(T::slots_per_epoch()), Domain::Transfer, &state.fork, ); diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index d261b8b47c..290c33804c 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -237,7 +237,7 @@ pub fn process_final_updates( state.slot -= 1; } - if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / spec.slots_per_epoch) == 0 { + if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / T::slots_per_epoch()) == 0 { let historical_batch = state.historical_batch(); state .historical_roots diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index 9841a5551a..0e5a81cced 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -8,12 +8,12 @@ use types::*; fn runs_without_error() { Builder::from_env(Env::default().default_filter_or("error")).init(); - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let mut builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); - let target_slot = (spec.genesis_epoch + 4).end_slot(spec.slots_per_epoch); + let target_slot = (spec.genesis_epoch + 4).end_slot(FewValidatorsEthSpec::slots_per_epoch()); builder.teleport_to_slot(target_slot, &spec); let (mut state, _keypairs) = builder.build(); diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index ed1b968d7d..76a485ea7b 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -233,7 +233,7 @@ impl ValidatorStatuses { let attestation_slot = state.get_attestation_slot(&a.data)?; let inclusion_slot = attestation_slot + a.inclusion_delay; let relative_epoch = - RelativeEpoch::from_slot(state.slot, inclusion_slot, spec.slots_per_epoch)?; + RelativeEpoch::from_slot(state.slot, inclusion_slot, T::slots_per_epoch())?; status.inclusion_info = Some(InclusionInfo { slot: inclusion_slot, distance: a.inclusion_delay, @@ -297,7 +297,7 @@ impl ValidatorStatuses { spec: &ChainSpec, ) -> Result<(), BeaconStateError> { // Loop through each slot in the previous epoch. - for slot in state.previous_epoch().slot_iter(spec.slots_per_epoch) { + for slot in state.previous_epoch().slot_iter(T::slots_per_epoch()) { let crosslink_committees_at_slot = state.get_crosslink_committees_at_slot(slot)?; // Loop through each committee in the slot. @@ -338,7 +338,7 @@ fn target_matches_epoch_start_block( epoch: Epoch, spec: &ChainSpec, ) -> Result { - let slot = epoch.start_slot(spec.slots_per_epoch); + let slot = epoch.start_slot(T::slots_per_epoch()); let state_boundary_root = *state.get_block_root(slot)?; Ok(a.data.target_root == state_boundary_root) diff --git a/eth2/state_processing/src/per_slot_processing.rs b/eth2/state_processing/src/per_slot_processing.rs index 0fc0742623..8adfe988bb 100644 --- a/eth2/state_processing/src/per_slot_processing.rs +++ b/eth2/state_processing/src/per_slot_processing.rs @@ -16,7 +16,7 @@ pub fn per_slot_processing( ) -> Result<(), Error> { cache_state(state, spec)?; - if (state.slot > spec.genesis_slot) && ((state.slot + 1) % spec.slots_per_epoch == 0) { + if (state.slot > spec.genesis_slot) && ((state.slot + 1) % T::slots_per_epoch() == 0) { per_epoch_processing(state, spec)?; } diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index b937383280..27a2e7a0e5 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -458,7 +458,7 @@ impl BeaconState { epoch: Epoch, spec: &ChainSpec, ) -> Result<&Hash256, BeaconStateError> { - self.get_block_root(epoch.start_slot(spec.slots_per_epoch)) + self.get_block_root(epoch.start_slot(T::slots_per_epoch())) } /// Sets the block root for some given slot. diff --git a/eth2/types/src/beacon_state/beacon_state_types.rs b/eth2/types/src/beacon_state/beacon_state_types.rs index ec6eb68bc4..fce0c6dff1 100644 --- a/eth2/types/src/beacon_state/beacon_state_types.rs +++ b/eth2/types/src/beacon_state/beacon_state_types.rs @@ -1,5 +1,5 @@ use crate::*; -use fixed_len_vec::typenum::{Unsigned, U1024, U8, U8192}; +use fixed_len_vec::typenum::{Unsigned, U0, U1024, U64, U8, U8192}; use serde_derive::{Deserialize, Serialize}; use std::fmt::Debug; @@ -9,14 +9,24 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq { type LatestRandaoMixesLength: Unsigned + Clone + Sync + Send + Debug + PartialEq; type LatestActiveIndexRootsLength: Unsigned + Clone + Sync + Send + Debug + PartialEq; type LatestSlashedExitLength: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /// Note: `SlotsPerEpoch` is not necessarily required to be a compile-time constant. We include + /// it here just for the convenience of not passing `slots_per_epoch` around all the time. + type SlotsPerEpoch: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type GenesisEpoch: Unsigned + Clone + Sync + Send + Debug + PartialEq; - fn spec() -> ChainSpec; + fn default_spec() -> ChainSpec; + + fn genesis_epoch() -> Epoch { + Epoch::new(Self::GenesisEpoch::to_u64()) + } /// Return the number of committees in one epoch. /// /// Spec v0.6.1 - fn get_epoch_committee_count(active_validator_count: usize) -> usize { - let target_committee_size = Self::spec().target_committee_size; + fn get_epoch_committee_count( + active_validator_count: usize, + target_committee_size: usize, + ) -> usize { let shard_count = Self::shard_count(); let slots_per_epoch = Self::slots_per_epoch() as usize; @@ -35,21 +45,14 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq { /// basic sense. This count is not required to provide any security guarantees regarding /// decentralization, entropy, etc. fn minimum_validator_count() -> usize { - Self::slots_per_epoch() as usize + Self::SlotsPerEpoch::to_usize() } /// Returns the `SLOTS_PER_EPOCH` constant for this specification. /// /// Spec v0.6.1 fn slots_per_epoch() -> u64 { - Self::spec().slots_per_epoch - } - - /// Returns the `SLOTS_PER_EPOCH` constant for this specification. - /// - /// Spec v0.6.1 - fn genesis_epoch() -> Epoch { - Self::spec().genesis_epoch + Self::SlotsPerEpoch::to_u64() } /// Returns the `SHARD_COUNT` constant for this specification. @@ -100,8 +103,10 @@ impl EthSpec for FoundationEthSpec { type LatestRandaoMixesLength = U8192; type LatestActiveIndexRootsLength = U8192; type LatestSlashedExitLength = U8192; + type SlotsPerEpoch = U64; + type GenesisEpoch = U0; - fn spec() -> ChainSpec { + fn default_spec() -> ChainSpec { ChainSpec::foundation() } } @@ -118,9 +123,11 @@ impl EthSpec for FewValidatorsEthSpec { type LatestRandaoMixesLength = U8192; type LatestActiveIndexRootsLength = U8192; type LatestSlashedExitLength = U8192; + type SlotsPerEpoch = U8; + type GenesisEpoch = U0; - fn spec() -> ChainSpec { - ChainSpec::few_validators() + fn default_spec() -> ChainSpec { + ChainSpec::few_validators(Self::slots_per_epoch()) } } @@ -136,9 +143,11 @@ impl EthSpec for LighthouseTestnetEthSpec { type LatestRandaoMixesLength = U8192; type LatestActiveIndexRootsLength = U8192; type LatestSlashedExitLength = U8192; + type SlotsPerEpoch = U8; + type GenesisEpoch = U0; - fn spec() -> ChainSpec { - ChainSpec::lighthouse_testnet() + fn default_spec() -> ChainSpec { + ChainSpec::lighthouse_testnet(Self::slots_per_epoch()) } } diff --git a/eth2/types/src/beacon_state/committee_cache.rs b/eth2/types/src/beacon_state/committee_cache.rs index d1e0e70307..78af727544 100644 --- a/eth2/types/src/beacon_state/committee_cache.rs +++ b/eth2/types/src/beacon_state/committee_cache.rs @@ -45,13 +45,18 @@ impl CommitteeCache { return Err(Error::InsufficientValidators); } - let committee_count = T::get_epoch_committee_count(active_validator_indices.len()) as usize; + let committee_count = T::get_epoch_committee_count( + active_validator_indices.len(), + spec.target_committee_size, + ) as usize; let shuffling_start_shard = match relative_epoch { RelativeEpoch::Current => state.latest_start_shard, RelativeEpoch::Previous => { - let committees_in_previous_epoch = - T::get_epoch_committee_count(active_validator_indices.len()) as u64; + let committees_in_previous_epoch = T::get_epoch_committee_count( + active_validator_indices.len(), + spec.target_committee_size, + ) as u64; (state.latest_start_shard + T::shard_count() as u64 - committees_in_previous_epoch) % T::shard_count() as u64 @@ -59,8 +64,10 @@ impl CommitteeCache { RelativeEpoch::Next => { let current_active_validators = get_active_validator_count(&state.validator_registry, state.current_epoch()); - let committees_in_current_epoch = - T::get_epoch_committee_count(current_active_validators) as u64; + let committees_in_current_epoch = T::get_epoch_committee_count( + current_active_validators, + spec.target_committee_size, + ) as u64; (state.latest_start_shard + committees_in_current_epoch) % T::shard_count() as u64 } diff --git a/eth2/types/src/beacon_state/committee_cache/tests.rs b/eth2/types/src/beacon_state/committee_cache/tests.rs index ecaa3d457d..e9b417e3d8 100644 --- a/eth2/types/src/beacon_state/committee_cache/tests.rs +++ b/eth2/types/src/beacon_state/committee_cache/tests.rs @@ -20,7 +20,7 @@ fn default_values() { } fn new_state(validator_count: usize, slot: Slot) -> BeaconState { - let spec = &T::spec(); + let spec = &T::default_spec(); let mut builder = TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec); @@ -35,7 +35,7 @@ fn new_state(validator_count: usize, slot: Slot) -> BeaconState { #[test] fn fails_without_validators() { let state = new_state::(0, Slot::new(0)); - let spec = &FewValidatorsEthSpec::spec(); + let spec = &FewValidatorsEthSpec::default_spec(); assert_eq!( CommitteeCache::initialized(&state, state.current_epoch(), &spec), @@ -46,7 +46,7 @@ fn fails_without_validators() { #[test] fn initializes_with_the_right_epoch() { let state = new_state::(16, Slot::new(0)); - let spec = &FewValidatorsEthSpec::spec(); + let spec = &FewValidatorsEthSpec::default_spec(); let cache = CommitteeCache::default(); assert_eq!(cache.initialized_epoch, None); @@ -68,7 +68,7 @@ fn shuffles_for_the_right_epoch() { let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch()); let mut state = new_state::(num_validators, slot); - let spec = &FewValidatorsEthSpec::spec(); + let spec = &FewValidatorsEthSpec::default_spec(); let distinct_hashes: Vec = (0..FewValidatorsEthSpec::latest_randao_mixes_length()) .into_iter() @@ -123,7 +123,7 @@ fn can_start_on_any_shard() { let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch()); let mut state = new_state::(num_validators, slot); - let spec = &FewValidatorsEthSpec::spec(); + let spec = &FewValidatorsEthSpec::default_spec(); for i in 0..FewValidatorsEthSpec::shard_count() as u64 { state.latest_start_shard = i; @@ -150,15 +150,17 @@ impl EthSpec for ExcessShardsEthSpec { type LatestRandaoMixesLength = U8192; type LatestActiveIndexRootsLength = U8192; type LatestSlashedExitLength = U8192; + type SlotsPerEpoch = U8; + type GenesisEpoch = U0; - fn spec() -> ChainSpec { - ChainSpec::few_validators() + fn default_spec() -> ChainSpec { + ChainSpec::few_validators(Self::slots_per_epoch()) } } #[test] fn starts_on_the_correct_shard() { - let spec = &ExcessShardsEthSpec::spec(); + let spec = &ExcessShardsEthSpec::default_spec(); let num_validators = ExcessShardsEthSpec::shard_count(); @@ -200,14 +202,16 @@ fn starts_on_the_correct_shard() { let previous_shards = ExcessShardsEthSpec::get_epoch_committee_count( get_active_validator_count(&state.validator_registry, previous_epoch), + spec.target_committee_size, ); let current_shards = ExcessShardsEthSpec::get_epoch_committee_count( get_active_validator_count(&state.validator_registry, current_epoch), + spec.target_committee_size, + ); + let next_shards = ExcessShardsEthSpec::get_epoch_committee_count( + get_active_validator_count(&state.validator_registry, next_epoch), + spec.target_committee_size, ); - let next_shards = ExcessShardsEthSpec::get_epoch_committee_count(get_active_validator_count( - &state.validator_registry, - next_epoch, - )); assert_eq!( previous_shards as usize, diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 588d24aa8b..581b9662ee 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -7,7 +7,7 @@ ssz_tests!(FoundationBeaconState); cached_tree_hash_tests!(FoundationBeaconState); fn test_beacon_proposer_index() { - let spec = T::spec(); + let spec = T::default_spec(); let relative_epoch = RelativeEpoch::Current; // Build a state for testing. @@ -61,7 +61,7 @@ fn beacon_proposer_index() { /// (current_epoch - LATEST_ACTIVE_INDEX_ROOTS_LENGTH + ACTIVATION_EXIT_DELAY, current_epoch + /// ACTIVATION_EXIT_DELAY] fn active_index_range(current_epoch: Epoch) -> RangeInclusive { - let delay = T::spec().activation_exit_delay; + let delay = T::default_spec().activation_exit_delay; let start: i32 = current_epoch.as_u64() as i32 - T::latest_active_index_roots() as i32 + delay as i32; @@ -79,7 +79,7 @@ fn active_index_range(current_epoch: Epoch) -> RangeInclusive /// Test getting an active index root at the start and end of the valid range, and one either side /// of that range. fn test_active_index(state_slot: Slot) { - let spec = T::spec(); + let spec = T::default_spec(); let builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(16, &spec); let (mut state, _keypairs) = builder.build(); @@ -133,8 +133,8 @@ fn test_cache_initialization<'a, T: EthSpec>( spec: &ChainSpec, ) { let slot = relative_epoch - .into_epoch(state.slot.epoch(spec.slots_per_epoch)) - .start_slot(spec.slots_per_epoch); + .into_epoch(state.slot.epoch(T::slots_per_epoch())) + .start_slot(T::slots_per_epoch()); // Assuming the cache isn't already built, assert that a call to a cache-using function fails. assert_eq!( @@ -166,13 +166,13 @@ fn test_cache_initialization<'a, T: EthSpec>( #[test] fn cache_initialization() { - let spec = FewValidatorsEthSpec::spec(); + let spec = FewValidatorsEthSpec::default_spec(); let builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(16, &spec); let (mut state, _keypairs) = builder.build(); - state.slot = (spec.genesis_epoch + 1).start_slot(spec.slots_per_epoch); + state.slot = (spec.genesis_epoch + 1).start_slot(FewValidatorsEthSpec::slots_per_epoch()); test_cache_initialization(&mut state, RelativeEpoch::Previous, &spec); test_cache_initialization(&mut state, RelativeEpoch::Current, &spec); @@ -234,7 +234,7 @@ mod committees { (start_shard..start_shard + T::shard_count() as u64).into_iter(); // Loop through all slots in the epoch being tested. - for slot in epoch.slot_iter(spec.slots_per_epoch) { + for slot in epoch.slot_iter(T::slots_per_epoch()) { let crosslink_committees = state.get_crosslink_committees_at_slot(slot).unwrap(); // Assert that the number of committees in this slot is consistent with the reported number @@ -290,7 +290,7 @@ mod committees { state_epoch: Epoch, cache_epoch: RelativeEpoch, ) { - let spec = &T::spec(); + let spec = &T::default_spec(); let mut builder = TestingBeaconStateBuilder::from_single_keypair( validator_count, @@ -298,7 +298,7 @@ mod committees { spec, ); - let slot = state_epoch.start_slot(spec.slots_per_epoch); + let slot = state_epoch.start_slot(T::slots_per_epoch()); builder.teleport_to_slot(slot, spec); let (mut state, _keypairs): (BeaconState, _) = builder.build(); @@ -325,7 +325,7 @@ mod committees { } fn committee_consistency_test_suite(cached_epoch: RelativeEpoch) { - let spec = T::spec(); + let spec = T::default_spec(); let validator_count = (T::shard_count() * spec.target_committee_size) + 1; diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 20aa6fcdbc..d9a0785a14 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,6 +1,6 @@ use crate::*; use int_to_bytes::int_to_bytes4; -use serde_derive::Deserialize; +use serde_derive::{Deserialize, Serialize}; use test_utils::u8_from_hex_str; /// Each of the BLS signature domains. @@ -18,7 +18,7 @@ pub enum Domain { /// Holds all the "constants" for a BeaconChain. /// /// Spec v0.6.1 -#[derive(PartialEq, Debug, Clone, Deserialize)] +#[derive(PartialEq, Debug, Clone, Serialize, Deserialize)] #[serde(default)] pub struct ChainSpec { /* @@ -59,7 +59,7 @@ pub struct ChainSpec { */ pub seconds_per_slot: u64, pub min_attestation_inclusion_delay: u64, - pub slots_per_epoch: u64, + //pub slots_per_epoch: u64, pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, pub slots_per_eth1_voting_period: u64, @@ -137,7 +137,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// /// Spec v0.6.1 - pub(crate) fn foundation() -> Self { + pub fn foundation() -> Self { Self { /* * Misc @@ -176,7 +176,7 @@ impl ChainSpec { */ seconds_per_slot: 6, min_attestation_inclusion_delay: 4, - slots_per_epoch: 64, + // slots_per_epoch: 64, min_seed_lookahead: Epoch::new(1), activation_exit_delay: 4, slots_per_eth1_voting_period: 1_024, @@ -226,7 +226,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Lighthouse testnet specification. /// /// Spec v0.4.0 - pub(crate) fn lighthouse_testnet() -> Self { + pub fn lighthouse_testnet(slots_per_epoch: u64) -> Self { /* * Lighthouse testnet bootnodes */ @@ -237,21 +237,19 @@ impl ChainSpec { Self { boot_nodes, chain_id: 2, // lighthouse testnet chain id - ..ChainSpec::few_validators() + ..ChainSpec::few_validators(slots_per_epoch) } } /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - pub(crate) fn few_validators() -> Self { + pub fn few_validators(slots_per_epoch: u64) -> Self { let genesis_slot = Slot::new(0); - let slots_per_epoch = 8; let genesis_epoch = genesis_slot.epoch(slots_per_epoch); Self { target_committee_size: 1, genesis_slot, genesis_epoch, - slots_per_epoch, ..ChainSpec::foundation() } } diff --git a/eth2/types/src/test_utils/builders/testing_attestation_data_builder.rs b/eth2/types/src/test_utils/builders/testing_attestation_data_builder.rs index 2150f5433a..0b4aa29876 100644 --- a/eth2/types/src/test_utils/builders/testing_attestation_data_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_attestation_data_builder.rs @@ -21,7 +21,7 @@ impl TestingAttestationDataBuilder { let previous_epoch = state.previous_epoch(); let is_previous_epoch = - state.slot.epoch(spec.slots_per_epoch) != slot.epoch(spec.slots_per_epoch); + state.slot.epoch(T::slots_per_epoch()) != slot.epoch(T::slots_per_epoch()); let source_epoch = if is_previous_epoch { state.previous_justified_epoch @@ -37,11 +37,11 @@ impl TestingAttestationDataBuilder { let target_root = if is_previous_epoch { *state - .get_block_root(previous_epoch.start_slot(spec.slots_per_epoch)) + .get_block_root(previous_epoch.start_slot(T::slots_per_epoch())) .unwrap() } else { *state - .get_block_root(current_epoch.start_slot(spec.slots_per_epoch)) + .get_block_root(current_epoch.start_slot(T::slots_per_epoch())) .unwrap() }; @@ -57,7 +57,7 @@ impl TestingAttestationDataBuilder { }; let source_root = *state - .get_block_root(source_epoch.start_slot(spec.slots_per_epoch)) + .get_block_root(source_epoch.start_slot(T::slots_per_epoch())) .unwrap(); let data = AttestationData { diff --git a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs index 941ad8fdd9..999e88aaef 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs @@ -36,9 +36,9 @@ impl TestingBeaconBlockBuilder { /// Signs the block. /// /// Modifying the block after signing may invalidate the signature. - pub fn sign(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { + pub fn sign(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { let message = self.block.signed_root(); - let epoch = self.block.slot.epoch(spec.slots_per_epoch); + let epoch = self.block.slot.epoch(T::slots_per_epoch()); let domain = spec.get_domain(epoch, Domain::BeaconProposer, fork); self.block.signature = Signature::new(&message, domain, sk); } @@ -46,8 +46,8 @@ impl TestingBeaconBlockBuilder { /// Sets the randao to be a signature across the blocks epoch. /// /// Modifying the block's slot after signing may invalidate the signature. - pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { - let epoch = self.block.slot.epoch(spec.slots_per_epoch); + pub fn set_randao_reveal(&mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) { + let epoch = self.block.slot.epoch(T::slots_per_epoch()); let message = epoch.tree_hash_root(); let domain = spec.get_domain(epoch, Domain::Randao, fork); self.block.body.randao_reveal = Signature::new(&message, domain, sk); @@ -59,14 +59,15 @@ impl TestingBeaconBlockBuilder { } /// Inserts a signed, valid `ProposerSlashing` for the validator. - pub fn insert_proposer_slashing( + pub fn insert_proposer_slashing( &mut self, validator_index: u64, secret_key: &SecretKey, fork: &Fork, spec: &ChainSpec, ) { - let proposer_slashing = build_proposer_slashing(validator_index, secret_key, fork, spec); + let proposer_slashing = + build_proposer_slashing::(validator_index, secret_key, fork, spec); self.block.body.proposer_slashings.push(proposer_slashing); } @@ -115,7 +116,7 @@ impl TestingBeaconBlockBuilder { // - The slot is too old to be included in a block at this slot. // - The `MAX_ATTESTATIONS`. loop { - if state.slot >= slot + spec.slots_per_epoch { + if state.slot >= slot + T::slots_per_epoch() { break; } @@ -194,7 +195,7 @@ impl TestingBeaconBlockBuilder { builder.set_index(index); builder.sign( &keypair, - state.slot.epoch(spec.slots_per_epoch), + state.slot.epoch(T::slots_per_epoch()), &state.fork, spec, ); @@ -211,7 +212,7 @@ impl TestingBeaconBlockBuilder { spec: &ChainSpec, ) { let mut builder = TestingVoluntaryExitBuilder::new( - state.slot.epoch(spec.slots_per_epoch), + state.slot.epoch(T::slots_per_epoch()), validator_index, ); @@ -234,14 +235,19 @@ impl TestingBeaconBlockBuilder { spec: &ChainSpec, ) { let mut builder = TestingTransferBuilder::new(from, to, amount, state.slot); - builder.sign(keypair, &state.fork, spec); + builder.sign::(keypair, &state.fork, spec); self.block.body.transfers.push(builder.build()) } /// Signs and returns the block, consuming the builder. - pub fn build(mut self, sk: &SecretKey, fork: &Fork, spec: &ChainSpec) -> BeaconBlock { - self.sign(sk, fork, spec); + pub fn build( + mut self, + sk: &SecretKey, + fork: &Fork, + spec: &ChainSpec, + ) -> BeaconBlock { + self.sign::(sk, fork, spec); self.block } @@ -254,7 +260,7 @@ impl TestingBeaconBlockBuilder { /// Builds an `ProposerSlashing` for some `validator_index`. /// /// Signs the message using a `BeaconChainHarness`. -fn build_proposer_slashing( +fn build_proposer_slashing( validator_index: u64, secret_key: &SecretKey, fork: &Fork, @@ -265,7 +271,7 @@ fn build_proposer_slashing( Signature::new(message, domain, secret_key) }; - TestingProposerSlashingBuilder::double_vote(validator_index, signer, spec) + TestingProposerSlashingBuilder::double_vote::(validator_index, signer, spec) } /// Builds an `AttesterSlashing` for some `validator_indices`. diff --git a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs index 20ed8a893a..dc2dd0a7c2 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -173,7 +173,7 @@ impl TestingBeaconStateBuilder { /// Sets the `BeaconState` to be in a slot, calling `teleport_to_epoch` to update the epoch. pub fn teleport_to_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.teleport_to_epoch(slot.epoch(spec.slots_per_epoch), spec); + self.teleport_to_epoch(slot.epoch(T::slots_per_epoch()), spec); self.state.slot = slot; } @@ -184,7 +184,7 @@ impl TestingBeaconStateBuilder { fn teleport_to_epoch(&mut self, epoch: Epoch, spec: &ChainSpec) { let state = &mut self.state; - let slot = epoch.start_slot(spec.slots_per_epoch); + let slot = epoch.start_slot(T::slots_per_epoch()); state.slot = slot; @@ -214,8 +214,8 @@ impl TestingBeaconStateBuilder { let current_epoch = state.current_epoch(); let previous_epoch = state.previous_epoch(); - let first_slot = previous_epoch.start_slot(spec.slots_per_epoch).as_u64(); - let last_slot = current_epoch.end_slot(spec.slots_per_epoch).as_u64() + let first_slot = previous_epoch.start_slot(T::slots_per_epoch()).as_u64(); + let last_slot = current_epoch.end_slot(T::slots_per_epoch()).as_u64() - spec.min_attestation_inclusion_delay; let last_slot = std::cmp::min(state.slot.as_u64(), last_slot); diff --git a/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs b/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs index 458082de2d..99c005472a 100644 --- a/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs @@ -17,8 +17,9 @@ impl TestingProposerSlashingBuilder { /// - `domain: Domain` /// /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). - pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing + pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing where + T: EthSpec, F: Fn(u64, &[u8], Epoch, Domain) -> Signature, { let slot = Slot::new(0); @@ -40,13 +41,13 @@ impl TestingProposerSlashingBuilder { header_1.signature = { let message = header_1.signed_root(); - let epoch = slot.epoch(spec.slots_per_epoch); + let epoch = slot.epoch(T::slots_per_epoch()); signer(proposer_index, &message[..], epoch, Domain::BeaconProposer) }; header_2.signature = { let message = header_2.signed_root(); - let epoch = slot.epoch(spec.slots_per_epoch); + let epoch = slot.epoch(T::slots_per_epoch()); signer(proposer_index, &message[..], epoch, Domain::BeaconProposer) }; diff --git a/eth2/types/src/test_utils/builders/testing_transfer_builder.rs b/eth2/types/src/test_utils/builders/testing_transfer_builder.rs index 2680f7b664..d3c3da19e6 100644 --- a/eth2/types/src/test_utils/builders/testing_transfer_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_transfer_builder.rs @@ -29,10 +29,10 @@ impl TestingTransferBuilder { /// Signs the transfer. /// /// The keypair must match that of the `from` validator index. - pub fn sign(&mut self, keypair: Keypair, fork: &Fork, spec: &ChainSpec) { + pub fn sign(&mut self, keypair: Keypair, fork: &Fork, spec: &ChainSpec) { self.transfer.pubkey = keypair.pk; let message = self.transfer.signed_root(); - let epoch = self.transfer.slot.epoch(spec.slots_per_epoch); + let epoch = self.transfer.slot.epoch(T::slots_per_epoch()); let domain = spec.get_domain(epoch, Domain::Transfer, fork); self.transfer.signature = Signature::new(&message, domain, &keypair.sk); diff --git a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs index fa530f9add..afdb9cc317 100644 --- a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs +++ b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs @@ -29,9 +29,9 @@ impl Case for EpochProcessingCrosslinks { let mut expected = self.post.clone(); // Processing requires the epoch cache. - state.build_all_caches(&E::spec()).unwrap(); + state.build_all_caches(&E::default_spec()).unwrap(); - let mut result = process_crosslinks(&mut state, &E::spec()).map(|_| state); + let mut result = process_crosslinks(&mut state, &E::default_spec()).map(|_| state); compare_beacon_state_results_without_caches(&mut result, &mut expected) } diff --git a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs index d91a7a4c3e..092a8a7092 100644 --- a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs +++ b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs @@ -27,7 +27,7 @@ impl Case for EpochProcessingRegistryUpdates { fn result(&self, _case_index: usize) -> Result<(), Error> { let mut state = self.pre.clone(); let mut expected = self.post.clone(); - let spec = &E::spec(); + let spec = &E::default_spec(); // Processing requires the epoch cache. state.build_all_caches(spec).unwrap(); diff --git a/tests/ef_tests/src/cases/operations_attester_slashing.rs b/tests/ef_tests/src/cases/operations_attester_slashing.rs index d8f1f06dc6..1b95f43590 100644 --- a/tests/ef_tests/src/cases/operations_attester_slashing.rs +++ b/tests/ef_tests/src/cases/operations_attester_slashing.rs @@ -31,9 +31,10 @@ impl Case for OperationsAttesterSlashing { let mut expected = self.post.clone(); // Processing requires the epoch cache. - state.build_all_caches(&E::spec()).unwrap(); + state.build_all_caches(&E::default_spec()).unwrap(); - let result = process_attester_slashings(&mut state, &[attester_slashing], &E::spec()); + let result = + process_attester_slashings(&mut state, &[attester_slashing], &E::default_spec()); let mut result = result.and_then(|_| Ok(state)); diff --git a/tests/ef_tests/src/cases/operations_deposit.rs b/tests/ef_tests/src/cases/operations_deposit.rs index 23b791ba53..8100590fdd 100644 --- a/tests/ef_tests/src/cases/operations_deposit.rs +++ b/tests/ef_tests/src/cases/operations_deposit.rs @@ -34,7 +34,7 @@ impl Case for OperationsDeposit { let deposit = self.deposit.clone(); let mut expected = self.post.clone(); - let result = process_deposits(&mut state, &[deposit], &E::spec()); + let result = process_deposits(&mut state, &[deposit], &E::default_spec()); let mut result = result.and_then(|_| Ok(state)); diff --git a/tests/ef_tests/src/cases/operations_exit.rs b/tests/ef_tests/src/cases/operations_exit.rs index 3d0f6b0102..cfe20d82b8 100644 --- a/tests/ef_tests/src/cases/operations_exit.rs +++ b/tests/ef_tests/src/cases/operations_exit.rs @@ -31,9 +31,9 @@ impl Case for OperationsExit { let mut expected = self.post.clone(); // Exit processing requires the epoch cache. - state.build_all_caches(&E::spec()).unwrap(); + state.build_all_caches(&E::default_spec()).unwrap(); - let result = process_exits(&mut state, &[exit], &E::spec()); + let result = process_exits(&mut state, &[exit], &E::default_spec()); let mut result = result.and_then(|_| Ok(state)); diff --git a/tests/ef_tests/src/cases/operations_proposer_slashing.rs b/tests/ef_tests/src/cases/operations_proposer_slashing.rs index 416a6f7c37..43293dfbcf 100644 --- a/tests/ef_tests/src/cases/operations_proposer_slashing.rs +++ b/tests/ef_tests/src/cases/operations_proposer_slashing.rs @@ -31,9 +31,10 @@ impl Case for OperationsProposerSlashing { let mut expected = self.post.clone(); // Processing requires the epoch cache. - state.build_all_caches(&E::spec()).unwrap(); + state.build_all_caches(&E::default_spec()).unwrap(); - let result = process_proposer_slashings(&mut state, &[proposer_slashing], &E::spec()); + let result = + process_proposer_slashings(&mut state, &[proposer_slashing], &E::default_spec()); let mut result = result.and_then(|_| Ok(state)); diff --git a/tests/ef_tests/src/cases/operations_transfer.rs b/tests/ef_tests/src/cases/operations_transfer.rs index 3ec96cd5cb..66318e7504 100644 --- a/tests/ef_tests/src/cases/operations_transfer.rs +++ b/tests/ef_tests/src/cases/operations_transfer.rs @@ -31,9 +31,9 @@ impl Case for OperationsTransfer { let mut expected = self.post.clone(); // Transfer processing requires the epoch cache. - state.build_all_caches(&E::spec()).unwrap(); + state.build_all_caches(&E::default_spec()).unwrap(); - let mut spec = E::spec(); + let mut spec = E::default_spec(); spec.max_transfers = 1; let result = process_transfers(&mut state, &[transfer], &spec); diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index ef8a1b934c..2920840c80 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -24,7 +24,7 @@ impl Case for Shuffling { if self.count == 0 { compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?; } else { - let spec = T::spec(); + let spec = T::default_spec(); let seed = hex::decode(&self.seed[2..]) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; diff --git a/tests/ef_tests/src/eth_specs.rs b/tests/ef_tests/src/eth_specs.rs index cdf8b94e8c..f0ee1e9f03 100644 --- a/tests/ef_tests/src/eth_specs.rs +++ b/tests/ef_tests/src/eth_specs.rs @@ -1,6 +1,6 @@ use serde_derive::{Deserialize, Serialize}; use types::{ - typenum::{U64, U8}, + typenum::{U0, U64, U8}, ChainSpec, EthSpec, FewValidatorsEthSpec, FoundationEthSpec, }; @@ -18,10 +18,12 @@ impl EthSpec for MinimalEthSpec { type LatestRandaoMixesLength = U64; type LatestActiveIndexRootsLength = U64; type LatestSlashedExitLength = U64; + type SlotsPerEpoch = U8; + type GenesisEpoch = U0; - fn spec() -> ChainSpec { + fn default_spec() -> ChainSpec { // TODO: this spec is likely incorrect! - let mut spec = FewValidatorsEthSpec::spec(); + let mut spec = FewValidatorsEthSpec::default_spec(); spec.shuffle_round_count = 10; spec } diff --git a/validator_client/src/attestation_producer/mod.rs b/validator_client/src/attestation_producer/mod.rs index 0a65c1f1e3..d59f383ef4 100644 --- a/validator_client/src/attestation_producer/mod.rs +++ b/validator_client/src/attestation_producer/mod.rs @@ -39,6 +39,8 @@ pub struct AttestationProducer<'a, B: BeaconNodeAttestation, S: Signer> { pub beacon_node: Arc, /// The signer to sign the block. pub signer: &'a S, + /// Used for caclulating epoch. + pub slots_per_epoch: u64, } impl<'a, B: BeaconNodeAttestation, S: Signer> AttestationProducer<'a, B, S> { @@ -78,7 +80,7 @@ impl<'a, B: BeaconNodeAttestation, S: Signer> AttestationProducer<'a, B, S> { /// The slash-protection code is not yet implemented. There is zero protection against /// slashing. pub fn produce_attestation(&mut self) -> Result { - let epoch = self.duty.slot.epoch(self.spec.slots_per_epoch); + let epoch = self.duty.slot.epoch(self.slots_per_epoch); let attestation = self .beacon_node diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index fc01b81265..9c19e35324 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -48,6 +48,8 @@ pub struct BlockProducer<'a, B: BeaconNodeBlock, S: Signer> { pub beacon_node: Arc, /// The signer to sign the block. pub signer: &'a S, + /// Used for caclulating epoch. + pub slots_per_epoch: u64, } impl<'a, B: BeaconNodeBlock, S: Signer> BlockProducer<'a, B, S> { @@ -84,7 +86,7 @@ impl<'a, B: BeaconNodeBlock, S: Signer> BlockProducer<'a, B, S> { /// The slash-protection code is not yet implemented. There is zero protection against /// slashing. pub fn produce_block(&mut self) -> Result { - let epoch = self.slot.epoch(self.spec.slots_per_epoch); + let epoch = self.slot.epoch(self.slots_per_epoch); let message = epoch.tree_hash_root(); let randao_reveal = match self.signer.sign_message( @@ -186,9 +188,9 @@ mod tests { let beacon_node = Arc::new(SimulatedBeaconNode::default()); let signer = Arc::new(LocalSigner::new(Keypair::random())); - let mut epoch_map = EpochMap::new(spec.slots_per_epoch); + let mut epoch_map = EpochMap::new(T::slots_per_epoch()); let produce_slot = Slot::new(100); - let produce_epoch = produce_slot.epoch(spec.slots_per_epoch); + let produce_epoch = produce_slot.epoch(T::slots_per_epoch()); epoch_map.map.insert(produce_epoch, produce_slot); let epoch_map = Arc::new(epoch_map); @@ -233,7 +235,7 @@ mod tests { ); // In an epoch without known duties... - let slot = (produce_epoch.as_u64() + 1) * spec.slots_per_epoch; + let slot = (produce_epoch.as_u64() + 1) * T::slots_per_epoch(); slot_clock.set_slot(slot); assert_eq!( block_proposer.poll(), diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 1e9450d599..f5d795ede5 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -19,6 +19,7 @@ pub struct Config { pub server: String, /// The chain specification that we are connecting to pub spec: ChainSpec, + pub slots_per_epoch: u64, } const DEFAULT_PRIVATE_KEY_FILENAME: &str = "private.key"; @@ -33,12 +34,13 @@ impl Default for Config { let server = "localhost:5051".to_string(); - let spec = FoundationEthSpec::spec(); + let spec = FoundationEthSpec::default_spec(); Self { data_dir, server, spec, + slots_per_epoch: FoundationEthSpec::slots_per_epoch(), } } } @@ -67,9 +69,9 @@ impl Config { if let Some(spec_str) = args.value_of("spec") { info!(log, "Using custom spec: {:?}", spec_str); config.spec = match spec_str { - "foundation" => FoundationEthSpec::spec(), - "few_validators" => FewValidatorsEthSpec::spec(), - "lighthouse_testnet" => LighthouseTestnetEthSpec::spec(), + "foundation" => FoundationEthSpec::default_spec(), + "few_validators" => FewValidatorsEthSpec::default_spec(), + "lighthouse_testnet" => LighthouseTestnetEthSpec::default_spec(), // Should be impossible due to clap's `possible_values(..)` function. _ => unreachable!(), }; diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index b2ddfd0b09..3f75c937d2 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -164,7 +164,7 @@ mod tests { #[test] pub fn polling() { let spec = Arc::new(ChainSpec::foundation()); - let duties_map = Arc::new(EpochDutiesMap::new(spec.slots_per_epoch)); + let duties_map = Arc::new(EpochDutiesMap::new(T::slots_per_epoch())); let keypair = Keypair::random(); let slot_clock = Arc::new(TestingSlotClock::new(0)); let beacon_node = Arc::new(TestBeaconNode::default()); diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 033394a0d3..99455808c3 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -47,6 +47,7 @@ pub struct Service { slot_clock: SystemTimeSlotClock, /// The current slot we are processing. current_slot: Slot, + slots_per_epoch: u64, /// The chain specification for this clients instance. spec: Arc, /// The duties manager which maintains the state of when to perform actions. @@ -177,7 +178,7 @@ impl Service { // Builds a mapping of Epoch -> Map(PublicKey, EpochDuty) // where EpochDuty contains slot numbers and attestation data that each validator needs to // produce work on. - let duties_map = RwLock::new(EpochDutiesMap::new(config.spec.slots_per_epoch)); + let duties_map = RwLock::new(EpochDutiesMap::new(config.slots_per_epoch)); // builds a manager which maintains the list of current duties for all known validators // and can check when a validator needs to perform a task. @@ -194,6 +195,7 @@ impl Service { fork, slot_clock, current_slot, + slots_per_epoch: config.slots_per_epoch, spec, duties_manager, beacon_block_client, @@ -204,7 +206,10 @@ impl Service { /// Initialise the service then run the core thread. // TODO: Improve handling of generic BeaconNode types, to stub grpcClient - pub fn start(config: ValidatorConfig, log: slog::Logger) -> error_chain::Result<()> { + pub fn start( + config: ValidatorConfig, + log: slog::Logger, + ) -> error_chain::Result<()> { // connect to the node and retrieve its properties and initialize the gRPC clients let mut service = Service::::initialize_service(config, log)?; @@ -274,7 +279,7 @@ impl Service { Ok(slot) => slot.expect("Genesis is in the future"), }; - let current_epoch = current_slot.epoch(self.spec.slots_per_epoch); + let current_epoch = current_slot.epoch(self.slots_per_epoch); // this is a fatal error. If the slot clock repeats, there is something wrong with // the timer, terminate immediately. @@ -291,7 +296,7 @@ impl Service { fn check_for_duties(&mut self) { let cloned_manager = self.duties_manager.clone(); let cloned_log = self.log.clone(); - let current_epoch = self.current_slot.epoch(self.spec.slots_per_epoch); + let current_epoch = self.current_slot.epoch(self.slots_per_epoch); // spawn a new thread separate to the runtime // TODO: Handle thread termination/timeout // TODO: Add duties thread back in, with channel to process duties in duty change. @@ -316,6 +321,7 @@ impl Service { let spec = self.spec.clone(); let beacon_node = self.beacon_block_client.clone(); let log = self.log.clone(); + let slots_per_epoch = self.slots_per_epoch; std::thread::spawn(move || { info!(log, "Producing a block"; "Validator"=> format!("{}", signers[signer_index])); let signer = &signers[signer_index]; @@ -325,6 +331,7 @@ impl Service { spec, beacon_node, signer, + slots_per_epoch, }; block_producer.handle_produce_block(log); }); @@ -337,6 +344,7 @@ impl Service { let spec = self.spec.clone(); let beacon_node = self.attestation_client.clone(); let log = self.log.clone(); + let slots_per_epoch = self.slots_per_epoch; std::thread::spawn(move || { info!(log, "Producing an attestation"; "Validator"=> format!("{}", signers[signer_index])); let signer = &signers[signer_index]; @@ -346,6 +354,7 @@ impl Service { spec, beacon_node, signer, + slots_per_epoch, }; attestation_producer.handle_produce_attestation(log); }); From caddeba81b7aa8e88eec3c47c0a5fc8ddd243eb3 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 08:17:42 -0400 Subject: [PATCH 63/76] Remove `genesis_epoch` from ChainSpec --- beacon_node/client/src/client_config.rs | 2 +- eth2/operation_pool/src/lib.rs | 2 +- .../benches/bench_epoch_processing.rs | 2 +- eth2/state_processing/benches/benches.rs | 4 ++-- eth2/state_processing/src/get_genesis_state.rs | 4 ++-- .../src/per_block_processing/tests.rs | 2 +- eth2/state_processing/src/per_epoch_processing.rs | 2 +- .../src/per_epoch_processing/apply_rewards.rs | 2 +- .../src/per_epoch_processing/tests.rs | 3 ++- eth2/types/src/beacon_state.rs | 10 +++++----- eth2/types/src/beacon_state/beacon_state_types.rs | 4 ++-- eth2/types/src/beacon_state/committee_cache/tests.rs | 2 +- eth2/types/src/beacon_state/tests.rs | 7 ++++--- eth2/types/src/chain_spec.rs | 12 ++++-------- eth2/types/src/fork.rs | 10 ++++------ .../builders/testing_beacon_state_builder.rs | 4 ++-- 16 files changed, 34 insertions(+), 38 deletions(-) diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index a729531ad4..6c42f1ea6e 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -31,7 +31,7 @@ impl Default for ClientConfig { network: NetworkConfig::new(vec![]), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), - spec: ChainSpec::lighthouse_testnet(8), + spec: ChainSpec::lighthouse_testnet(), } } } diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index 0cd46e0183..ae8a6ae11e 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -680,7 +680,7 @@ mod tests { let mut state = BeaconState::random_for_test(rng); - state.fork = Fork::genesis(&spec); + state.fork = Fork::genesis(FoundationEthSpec::genesis_epoch()); (spec, state) } diff --git a/eth2/state_processing/benches/bench_epoch_processing.rs b/eth2/state_processing/benches/bench_epoch_processing.rs index 23e662e09d..665aabbff9 100644 --- a/eth2/state_processing/benches/bench_epoch_processing.rs +++ b/eth2/state_processing/benches/bench_epoch_processing.rs @@ -23,7 +23,7 @@ pub fn bench_epoch_processing_n_validators(c: &mut Criterion, validator_count: u TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); // Set the state to be just before an epoch transition. - let target_slot = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); + let target_slot = (T::genesis_epoch() + 4).end_slot(T::slots_per_epoch()); builder.teleport_to_slot(target_slot, &spec); // Builds all caches; benches will not contain shuffling/committee building times. diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 1a153abee7..656ad9e4ef 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -34,7 +34,7 @@ pub fn block_processing_worst_case(c: &mut Criterion) { bench_builder.maximize_block_operations(&spec); // Set the state and block to be in the last slot of the 4th epoch. - let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); + let last_slot_of_epoch = (T::genesis_epoch() + 4).end_slot(T::slots_per_epoch()); bench_builder.set_slot(last_slot_of_epoch, &spec); // Build all the state caches so the build times aren't included in the benches. @@ -73,7 +73,7 @@ pub fn block_processing_reasonable_case(c: &mut Criterion) { bench_builder.num_transfers = 2; // Set the state and block to be in the last slot of the 4th epoch. - let last_slot_of_epoch = (spec.genesis_epoch + 4).end_slot(T::slots_per_epoch()); + let last_slot_of_epoch = (T::genesis_epoch() + 4).end_slot(T::slots_per_epoch()); bench_builder.set_slot(last_slot_of_epoch, &spec); // Build all the state caches so the build times aren't included in the benches. diff --git a/eth2/state_processing/src/get_genesis_state.rs b/eth2/state_processing/src/get_genesis_state.rs index 0fe78c1ed3..18a1e7c357 100644 --- a/eth2/state_processing/src/get_genesis_state.rs +++ b/eth2/state_processing/src/get_genesis_state.rs @@ -25,8 +25,8 @@ pub fn get_genesis_beacon_state( // Process genesis activations. for validator in &mut state.validator_registry { if validator.effective_balance >= spec.max_effective_balance { - validator.activation_eligibility_epoch = spec.genesis_epoch; - validator.activation_epoch = spec.genesis_epoch; + validator.activation_eligibility_epoch = T::genesis_epoch(); + validator.activation_epoch = T::genesis_epoch(); } } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index fe30560c67..c2a9840aa9 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -105,7 +105,7 @@ fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) // Set the state and block to be in the last slot of the 4th epoch. let last_slot_of_epoch = - (spec.genesis_epoch + 4).end_slot(FoundationEthSpec::slots_per_epoch()); + (FoundationEthSpec::genesis_epoch() + 4).end_slot(FoundationEthSpec::slots_per_epoch()); builder.set_slot(last_slot_of_epoch, &spec); builder.build_caches(&spec); diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 290c33804c..2d76577e1c 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -86,7 +86,7 @@ pub fn process_justification_and_finalization( total_balances: &TotalBalances, spec: &ChainSpec, ) -> Result<(), Error> { - if state.current_epoch() == spec.genesis_epoch { + if state.current_epoch() == T::genesis_epoch() { return Ok(()); } diff --git a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs index 7f98f3ae5b..7ddba6f38a 100644 --- a/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs +++ b/eth2/state_processing/src/per_epoch_processing/apply_rewards.rs @@ -39,7 +39,7 @@ pub fn process_rewards_and_penalties( winning_root_for_shards: &WinningRootHashSet, spec: &ChainSpec, ) -> Result<(), Error> { - if state.current_epoch() == spec.genesis_epoch { + if state.current_epoch() == T::genesis_epoch() { return Ok(()); } diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index 0e5a81cced..dffe2b593f 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -13,7 +13,8 @@ fn runs_without_error() { let mut builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); - let target_slot = (spec.genesis_epoch + 4).end_slot(FewValidatorsEthSpec::slots_per_epoch()); + let target_slot = (FewValidatorsEthSpec::genesis_epoch() + 4) + .end_slot(FewValidatorsEthSpec::slots_per_epoch()); builder.teleport_to_slot(target_slot, &spec); let (mut state, _keypairs) = builder.build(); diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 27a2e7a0e5..1c1fca6fb3 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -162,7 +162,7 @@ impl BeaconState { spec: &ChainSpec, ) -> BeaconState { let initial_crosslink = Crosslink { - epoch: spec.genesis_epoch, + epoch: T::genesis_epoch(), previous_crosslink_root: spec.zero_hash, crosslink_data_root: spec.zero_hash, }; @@ -171,7 +171,7 @@ impl BeaconState { // Misc slot: spec.genesis_slot, genesis_time, - fork: Fork::genesis(spec), + fork: Fork::genesis(T::genesis_epoch()), // Validator registry validator_registry: vec![], // Set later in the function. @@ -187,12 +187,12 @@ impl BeaconState { // Finality previous_epoch_attestations: vec![], current_epoch_attestations: vec![], - previous_justified_epoch: spec.genesis_epoch, - current_justified_epoch: spec.genesis_epoch, + previous_justified_epoch: T::genesis_epoch(), + current_justified_epoch: T::genesis_epoch(), previous_justified_root: spec.zero_hash, current_justified_root: spec.zero_hash, justification_bitfield: 0, - finalized_epoch: spec.genesis_epoch, + finalized_epoch: T::genesis_epoch(), finalized_root: spec.zero_hash, // Recent state diff --git a/eth2/types/src/beacon_state/beacon_state_types.rs b/eth2/types/src/beacon_state/beacon_state_types.rs index fce0c6dff1..6d0dd47688 100644 --- a/eth2/types/src/beacon_state/beacon_state_types.rs +++ b/eth2/types/src/beacon_state/beacon_state_types.rs @@ -127,7 +127,7 @@ impl EthSpec for FewValidatorsEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::few_validators(Self::slots_per_epoch()) + ChainSpec::few_validators() } } @@ -147,7 +147,7 @@ impl EthSpec for LighthouseTestnetEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::lighthouse_testnet(Self::slots_per_epoch()) + ChainSpec::lighthouse_testnet() } } diff --git a/eth2/types/src/beacon_state/committee_cache/tests.rs b/eth2/types/src/beacon_state/committee_cache/tests.rs index e9b417e3d8..8f1fbe1879 100644 --- a/eth2/types/src/beacon_state/committee_cache/tests.rs +++ b/eth2/types/src/beacon_state/committee_cache/tests.rs @@ -154,7 +154,7 @@ impl EthSpec for ExcessShardsEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::few_validators(Self::slots_per_epoch()) + ChainSpec::few_validators() } } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 581b9662ee..a668f99fe0 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -172,7 +172,8 @@ fn cache_initialization() { TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(16, &spec); let (mut state, _keypairs) = builder.build(); - state.slot = (spec.genesis_epoch + 1).start_slot(FewValidatorsEthSpec::slots_per_epoch()); + state.slot = (FewValidatorsEthSpec::genesis_epoch() + 1) + .start_slot(FewValidatorsEthSpec::slots_per_epoch()); test_cache_initialization(&mut state, RelativeEpoch::Previous, &spec); test_cache_initialization(&mut state, RelativeEpoch::Current, &spec); @@ -333,13 +334,13 @@ mod committees { committee_consistency_test::( validator_count as usize, - spec.genesis_epoch + 4, + T::genesis_epoch() + 4, cached_epoch, ); committee_consistency_test::( validator_count as usize, - spec.genesis_epoch + T::slots_per_historical_root() as u64 * T::slots_per_epoch() * 4, + T::genesis_epoch() + T::slots_per_historical_root() as u64 * T::slots_per_epoch() * 4, cached_epoch, ); } diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index d9a0785a14..a1267e37dd 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -48,7 +48,6 @@ pub struct ChainSpec { * Initial Values */ pub genesis_slot: Slot, - pub genesis_epoch: Epoch, pub far_future_epoch: Epoch, pub zero_hash: Hash256, #[serde(deserialize_with = "u8_from_hex_str")] @@ -166,7 +165,6 @@ impl ChainSpec { * Initial Values */ genesis_slot: Slot::new(0), - genesis_epoch: Epoch::new(0), far_future_epoch: Epoch::new(u64::max_value()), zero_hash: Hash256::zero(), bls_withdrawal_prefix_byte: 0, @@ -226,7 +224,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Lighthouse testnet specification. /// /// Spec v0.4.0 - pub fn lighthouse_testnet(slots_per_epoch: u64) -> Self { + pub fn lighthouse_testnet() -> Self { /* * Lighthouse testnet bootnodes */ @@ -237,19 +235,17 @@ impl ChainSpec { Self { boot_nodes, chain_id: 2, // lighthouse testnet chain id - ..ChainSpec::few_validators(slots_per_epoch) + ..ChainSpec::few_validators() } } /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - pub fn few_validators(slots_per_epoch: u64) -> Self { + pub fn few_validators() -> Self { let genesis_slot = Slot::new(0); - let genesis_epoch = genesis_slot.epoch(slots_per_epoch); Self { target_committee_size: 1, genesis_slot, - genesis_epoch, ..ChainSpec::foundation() } } @@ -272,7 +268,7 @@ mod tests { } fn test_domain(domain_type: Domain, raw_domain: u32, spec: &ChainSpec) { - let fork = Fork::genesis(&spec); + let fork = Fork::genesis(Epoch::new(0)); let epoch = Epoch::new(0); let domain = spec.get_domain(epoch, domain_type, &fork); diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index eb4e183f25..ec5f3af4cd 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -36,11 +36,11 @@ impl Fork { /// Initialize the `Fork` from the genesis parameters in the `spec`. /// /// Spec v0.6.1 - pub fn genesis(spec: &ChainSpec) -> Self { + pub fn genesis(genesis_epoch: Epoch) -> Self { Self { previous_version: [0; 4], current_version: [0; 4], - epoch: spec.genesis_epoch, + epoch: genesis_epoch, } } @@ -65,11 +65,9 @@ mod tests { fn test_genesis(epoch: Epoch) { let mut spec = ChainSpec::foundation(); - spec.genesis_epoch = epoch; + let fork = Fork::genesis(epoch); - let fork = Fork::genesis(&spec); - - assert_eq!(fork.epoch, spec.genesis_epoch, "epoch incorrect"); + assert_eq!(fork.epoch, epoch, "epoch incorrect"); assert_eq!( fork.previous_version, fork.current_version, "previous and current are not identical" diff --git a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs index dc2dd0a7c2..f9a4ea00c6 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -113,8 +113,8 @@ impl TestingBeaconStateBuilder { pubkey: keypair.pk.clone(), withdrawal_credentials, // All validators start active. - activation_eligibility_epoch: spec.genesis_epoch, - activation_epoch: spec.genesis_epoch, + activation_eligibility_epoch: T::genesis_epoch(), + activation_epoch: T::genesis_epoch(), exit_epoch: spec.far_future_epoch, withdrawable_epoch: spec.far_future_epoch, slashed: false, From 749f2fcb5fed6150bd7b3ef4066e0e9cda4f79da Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 08:49:04 -0400 Subject: [PATCH 64/76] Unify EthSpecs in `Mainnet` and `Minimal` --- beacon_node/beacon_chain/src/iter.rs | 10 ++--- beacon_node/client/src/beacon_chain_types.rs | 8 ++-- beacon_node/client/src/client_config.rs | 2 +- beacon_node/eth2-libp2p/src/behaviour.rs | 2 +- beacon_node/store/src/block_at_slot.rs | 8 ++-- eth2/fork_choice/benches/benches.rs | 4 +- eth2/fork_choice/examples/example.rs | 8 ++-- eth2/fork_choice/src/test_utils.rs | 9 ++--- eth2/fork_choice/tests/tests.rs | 14 +++---- eth2/operation_pool/src/lib.rs | 18 ++++----- .../benches/bench_epoch_processing.rs | 2 +- eth2/state_processing/benches/benches.rs | 4 +- .../src/per_block_processing/tests.rs | 16 ++++---- .../src/per_epoch_processing/tests.rs | 8 ++-- .../src/beacon_state/beacon_state_types.rs | 36 ++++------------- .../src/beacon_state/committee_cache/tests.rs | 30 +++++++------- eth2/types/src/beacon_state/tests.rs | 26 ++++++------ eth2/types/src/chain_spec.rs | 40 ++++++++----------- eth2/types/src/fork.rs | 2 +- eth2/types/src/historical_batch.rs | 2 +- tests/ef_tests/src/doc.rs | 2 +- tests/ef_tests/src/eth_specs.rs | 32 --------------- tests/ef_tests/src/lib.rs | 1 - validator_client/src/block_producer/mod.rs | 2 +- validator_client/src/config.rs | 13 +++--- validator_client/src/duties/mod.rs | 2 +- validator_client/src/main.rs | 4 +- 27 files changed, 119 insertions(+), 186 deletions(-) delete mode 100644 tests/ef_tests/src/eth_specs.rs diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index 48caaf6185..b54dd1d2aa 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -82,7 +82,7 @@ impl Iterator for BlockRootsIterator { mod test { use super::*; use store::MemoryStore; - use types::{test_utils::TestingBeaconStateBuilder, FoundationEthSpec, Keypair}; + use types::{test_utils::TestingBeaconStateBuilder, Keypair, MainnetEthSpec}; fn get_state() -> BeaconState { let builder = TestingBeaconStateBuilder::from_single_keypair( @@ -97,10 +97,10 @@ mod test { #[test] fn root_iter() { let store = Arc::new(MemoryStore::open()); - let slots_per_historical_root = FoundationEthSpec::slots_per_historical_root(); + let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root(); - let mut state_a: BeaconState = get_state(); - let mut state_b: BeaconState = get_state(); + let mut state_a: BeaconState = get_state(); + let mut state_b: BeaconState = get_state(); state_a.slot = Slot::from(slots_per_historical_root); state_b.slot = Slot::from(slots_per_historical_root * 2); @@ -122,7 +122,7 @@ mod test { let mut collected: Vec = iter.collect(); collected.reverse(); - let expected_len = 2 * FoundationEthSpec::slots_per_historical_root() - 1; + let expected_len = 2 * MainnetEthSpec::slots_per_historical_root() - 1; assert_eq!(collected.len(), expected_len); diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 9747b1dd81..133ebcc948 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -11,10 +11,10 @@ use std::sync::Arc; use tree_hash::TreeHash; use types::{ test_utils::TestingBeaconStateBuilder, BeaconBlock, ChainSpec, EthSpec, Hash256, - LighthouseTestnetEthSpec, + MinimalEthSpec, }; -/// The number initial validators when starting the `LighthouseTestnet`. +/// The number initial validators when starting the `Minimal`. const TESTNET_VALIDATOR_COUNT: usize = 16; /// Provides a new, initialized `BeaconChain` @@ -35,7 +35,7 @@ impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { type Store = MemoryStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = OptimizedLMDGhost; - type EthSpec = LighthouseTestnetEthSpec; + type EthSpec = MinimalEthSpec; } impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes {} @@ -46,7 +46,7 @@ impl BeaconChainTypes for TestnetDiskBeaconChainTypes { type Store = DiskStore; type SlotClock = SystemTimeSlotClock; type ForkChoice = OptimizedLMDGhost; - type EthSpec = LighthouseTestnetEthSpec; + type EthSpec = MinimalEthSpec; } impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes {} diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 6c42f1ea6e..1f875f02d4 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -31,7 +31,7 @@ impl Default for ClientConfig { network: NetworkConfig::new(vec![]), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), - spec: ChainSpec::lighthouse_testnet(), + spec: ChainSpec::minimal(), } } } diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 476cddfbb1..10b140c3b5 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -261,7 +261,7 @@ mod test { #[test] fn ssz_encoding() { - let original = PubsubMessage::Block(BeaconBlock::empty(&FoundationEthSpec::default_spec())); + let original = PubsubMessage::Block(BeaconBlock::empty(&MainnetEthSpec::default_spec())); let encoded = ssz_encode(&original); diff --git a/beacon_node/store/src/block_at_slot.rs b/beacon_node/store/src/block_at_slot.rs index b7346e90db..5a0dd68615 100644 --- a/beacon_node/store/src/block_at_slot.rs +++ b/beacon_node/store/src/block_at_slot.rs @@ -61,7 +61,7 @@ mod tests { #[test] fn read_slot() { - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); let test_slot = |slot: Slot| { let mut block = BeaconBlock::empty(&spec); @@ -85,7 +85,7 @@ mod tests { #[test] fn read_previous_block_root() { - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); let test_root = |root: Hash256| { let mut block = BeaconBlock::empty(&spec); @@ -130,7 +130,7 @@ mod tests { fn chain_without_skips() { let n: usize = 10; let store = MemoryStore::open(); - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); let slots: Vec = (0..n).collect(); let blocks_and_roots = build_chain(&store, &slots, &spec); @@ -154,7 +154,7 @@ mod tests { #[test] fn chain_with_skips() { let store = MemoryStore::open(); - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); let slots = vec![0, 1, 2, 5]; diff --git a/eth2/fork_choice/benches/benches.rs b/eth2/fork_choice/benches/benches.rs index b0495a80c9..f311e1ccbb 100644 --- a/eth2/fork_choice/benches/benches.rs +++ b/eth2/fork_choice/benches/benches.rs @@ -3,10 +3,10 @@ use criterion::{criterion_group, criterion_main, Benchmark}; use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; use std::sync::Arc; use store::MemoryStore; -use types::{ChainSpec, EthSpec, FoundationEthSpec}; +use types::{ChainSpec, EthSpec, MainnetEthSpec}; pub type TestedForkChoice = OptimizedLMDGhost; -pub type TestedEthSpec = FoundationEthSpec; +pub type TestedEthSpec = MainnetEthSpec; /// Helper function to setup a builder and spec. fn setup( diff --git a/eth2/fork_choice/examples/example.rs b/eth2/fork_choice/examples/example.rs index 7e42ad144b..a912c3753c 100644 --- a/eth2/fork_choice/examples/example.rs +++ b/eth2/fork_choice/examples/example.rs @@ -1,7 +1,7 @@ use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; use std::sync::Arc; use store::{MemoryStore, Store}; -use types::{BeaconBlock, ChainSpec, EthSpec, FoundationEthSpec, Hash256}; +use types::{BeaconBlock, ChainSpec, EthSpec, Hash256, MainnetEthSpec}; fn main() { let validator_count = 16; @@ -9,15 +9,15 @@ fn main() { let repetitions = 50; let store = MemoryStore::open(); - let builder: TestingForkChoiceBuilder = + let builder: TestingForkChoiceBuilder = TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - let fork_choosers: Vec> = (0..repetitions) + let fork_choosers: Vec> = (0..repetitions) .into_iter() .map(|_| builder.build()) .collect(); - let spec = &FoundationEthSpec::default_spec(); + let spec = &MainnetEthSpec::default_spec(); println!("Running {} times...", repetitions); for fc in fork_choosers { diff --git a/eth2/fork_choice/src/test_utils.rs b/eth2/fork_choice/src/test_utils.rs index c3bf39930f..840d7a76b5 100644 --- a/eth2/fork_choice/src/test_utils.rs +++ b/eth2/fork_choice/src/test_utils.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use store::Store; use types::{ test_utils::{SeedableRng, TestRandom, TestingBeaconStateBuilder, XorShiftRng}, - BeaconBlock, BeaconState, EthSpec, FoundationEthSpec, Hash256, Keypair, + BeaconBlock, BeaconState, EthSpec, Hash256, Keypair, MainnetEthSpec, }; /// Creates a chain of blocks and produces `ForkChoice` instances with pre-filled stores. @@ -16,11 +16,8 @@ pub struct TestingForkChoiceBuilder { impl TestingForkChoiceBuilder { pub fn new(validator_count: usize, chain_length: usize, store: Arc) -> Self { - let chain = get_chain_of_blocks::( - chain_length, - validator_count, - store.clone(), - ); + let chain = + get_chain_of_blocks::(chain_length, validator_count, store.clone()); Self { store, diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs index 59e4e0ee09..39e70a7ddb 100644 --- a/eth2/fork_choice/tests/tests.rs +++ b/eth2/fork_choice/tests/tests.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::{fs::File, io::prelude::*, path::PathBuf}; use types::test_utils::TestingBeaconStateBuilder; use types::{ - BeaconBlock, BeaconBlockBody, Eth1Data, EthSpec, FoundationEthSpec, Hash256, Keypair, Slot, + BeaconBlock, BeaconBlockBody, Eth1Data, EthSpec, Hash256, Keypair, MainnetEthSpec, Slot, }; use yaml_rust::yaml; @@ -22,7 +22,7 @@ fn test_optimized_lmd_ghost() { // set up logging // Builder::from_env(Env::default().default_filter_or("trace")).init(); - test_yaml_vectors::>( + test_yaml_vectors::>( "tests/lmd_ghost_test_vectors.yaml", 100, ); @@ -33,7 +33,7 @@ fn test_bitwise_lmd_ghost() { // set up logging //Builder::from_env(Env::default().default_filter_or("trace")).init(); - test_yaml_vectors::>( + test_yaml_vectors::>( "tests/bitwise_lmd_ghost_test_vectors.yaml", 100, ); @@ -41,7 +41,7 @@ fn test_bitwise_lmd_ghost() { #[test] fn test_slow_lmd_ghost() { - test_yaml_vectors::>( + test_yaml_vectors::>( "tests/lmd_ghost_test_vectors.yaml", 100, ); @@ -61,7 +61,7 @@ fn test_yaml_vectors>( let test_cases = load_test_cases_from_yaml(yaml_file_path); // default vars - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let zero_hash = Hash256::zero(); let eth1_data = Eth1Data { deposit_count: 0, @@ -204,9 +204,9 @@ where let store = Arc::new(MemoryStore::open()); let fork_choice = ForkChoice::new(store.clone()); - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); - let mut state_builder: TestingBeaconStateBuilder = + let mut state_builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec); state_builder.build_caches(&spec).unwrap(); let (state, _keypairs) = state_builder.build(); diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index ae8a6ae11e..0afd72918c 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -675,12 +675,12 @@ mod tests { .collect() } - fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState) { - let spec = FoundationEthSpec::default_spec(); + fn test_state(rng: &mut XorShiftRng) -> (ChainSpec, BeaconState) { + let spec = MainnetEthSpec::default_spec(); let mut state = BeaconState::random_for_test(rng); - state.fork = Fork::genesis(FoundationEthSpec::genesis_epoch()); + state.fork = Fork::genesis(MainnetEthSpec::genesis_epoch()); (spec, state) } @@ -735,13 +735,13 @@ mod tests { state_builder.build_caches(&spec).unwrap(); let (state, keypairs) = state_builder.build(); - (state, keypairs, FoundationEthSpec::default_spec()) + (state, keypairs, MainnetEthSpec::default_spec()) } #[test] fn test_attestation_score() { let (ref mut state, ref keypairs, ref spec) = - attestation_test_state::(1); + attestation_test_state::(1); let slot = state.slot - 1; let committees = state @@ -793,7 +793,7 @@ mod tests { #[test] fn attestation_aggregation_insert_get_prune() { let (ref mut state, ref keypairs, ref spec) = - attestation_test_state::(1); + attestation_test_state::(1); let op_pool = OperationPool::new(); @@ -861,7 +861,7 @@ mod tests { #[test] fn attestation_duplicate() { let (ref mut state, ref keypairs, ref spec) = - attestation_test_state::(1); + attestation_test_state::(1); let op_pool = OperationPool::new(); @@ -898,7 +898,7 @@ mod tests { #[test] fn attestation_pairwise_overlapping() { let (ref mut state, ref keypairs, ref spec) = - attestation_test_state::(1); + attestation_test_state::(1); let op_pool = OperationPool::new(); @@ -946,7 +946,7 @@ mod tests { let big_step_size = 4; let (ref mut state, ref keypairs, ref spec) = - attestation_test_state::(big_step_size); + attestation_test_state::(big_step_size); let op_pool = OperationPool::new(); diff --git a/eth2/state_processing/benches/bench_epoch_processing.rs b/eth2/state_processing/benches/bench_epoch_processing.rs index 665aabbff9..e89305ce44 100644 --- a/eth2/state_processing/benches/bench_epoch_processing.rs +++ b/eth2/state_processing/benches/bench_epoch_processing.rs @@ -17,7 +17,7 @@ pub const SMALL_BENCHING_SAMPLE_SIZE: usize = 10; /// Run the benchmarking suite on a foundation spec with 16,384 validators. pub fn bench_epoch_processing_n_validators(c: &mut Criterion, validator_count: usize) { - let spec = ChainSpec::foundation(); + let spec = ChainSpec::mainnet(); let mut builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); diff --git a/eth2/state_processing/benches/benches.rs b/eth2/state_processing/benches/benches.rs index 656ad9e4ef..3d884c3d80 100644 --- a/eth2/state_processing/benches/benches.rs +++ b/eth2/state_processing/benches/benches.rs @@ -25,7 +25,7 @@ pub fn block_processing_worst_case(c: &mut Criterion) { ); // Use the specifications from the Eth2.0 spec. - let spec = ChainSpec::foundation(); + let spec = ChainSpec::mainnet(); // Create a builder for configuring the block and state for benching. let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); @@ -59,7 +59,7 @@ pub fn block_processing_reasonable_case(c: &mut Criterion) { ); // Use the specifications from the Eth2.0 spec. - let spec = ChainSpec::foundation(); + let spec = ChainSpec::mainnet(); // Create a builder for configuring the block and state for benching. let mut bench_builder = BlockBenchingBuilder::new(VALIDATOR_COUNT, &spec); diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index c2a9840aa9..525637fea5 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -9,7 +9,7 @@ pub const VALIDATOR_COUNT: usize = 10; #[test] fn valid_block_ok() { - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let builder = get_builder(&spec); let (block, mut state) = builder.build(None, None, &spec); @@ -20,7 +20,7 @@ fn valid_block_ok() { #[test] fn invalid_block_header_state_slot() { - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let builder = get_builder(&spec); let (mut block, mut state) = builder.build(None, None, &spec); @@ -39,7 +39,7 @@ fn invalid_block_header_state_slot() { #[test] fn invalid_parent_block_root() { - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let builder = get_builder(&spec); let invalid_parent_root = Hash256::from([0xAA; 32]); let (block, mut state) = builder.build(None, Some(invalid_parent_root), &spec); @@ -59,14 +59,14 @@ fn invalid_parent_block_root() { #[test] fn invalid_block_signature() { - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let builder = get_builder(&spec); let (mut block, mut state) = builder.build(None, None, &spec); // sign the block with a keypair that is not the expected proposer let keypair = Keypair::random(); let message = block.signed_root(); - let epoch = block.slot.epoch(FoundationEthSpec::slots_per_epoch()); + let epoch = block.slot.epoch(MainnetEthSpec::slots_per_epoch()); let domain = spec.get_domain(epoch, Domain::BeaconProposer, &state.fork); block.signature = Signature::new(&message, domain, &keypair.sk); @@ -82,7 +82,7 @@ fn invalid_block_signature() { #[test] fn invalid_randao_reveal_signature() { - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); let builder = get_builder(&spec); // sign randao reveal with random keypair @@ -100,12 +100,12 @@ fn invalid_randao_reveal_signature() { ); } -fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { +fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { let mut builder = BlockProcessingBuilder::new(VALIDATOR_COUNT, &spec); // Set the state and block to be in the last slot of the 4th epoch. let last_slot_of_epoch = - (FoundationEthSpec::genesis_epoch() + 4).end_slot(FoundationEthSpec::slots_per_epoch()); + (MainnetEthSpec::genesis_epoch() + 4).end_slot(MainnetEthSpec::slots_per_epoch()); builder.set_slot(last_slot_of_epoch, &spec); builder.build_caches(&spec); diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index dffe2b593f..306acb7ee2 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -8,13 +8,13 @@ use types::*; fn runs_without_error() { Builder::from_env(Env::default().default_filter_or("error")).init(); - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); - let mut builder: TestingBeaconStateBuilder = + let mut builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_deterministic_keypairs(8, &spec); - let target_slot = (FewValidatorsEthSpec::genesis_epoch() + 4) - .end_slot(FewValidatorsEthSpec::slots_per_epoch()); + let target_slot = + (MinimalEthSpec::genesis_epoch() + 4).end_slot(MinimalEthSpec::slots_per_epoch()); builder.teleport_to_slot(target_slot, &spec); let (mut state, _keypairs) = builder.build(); diff --git a/eth2/types/src/beacon_state/beacon_state_types.rs b/eth2/types/src/beacon_state/beacon_state_types.rs index 6d0dd47688..d69350f9de 100644 --- a/eth2/types/src/beacon_state/beacon_state_types.rs +++ b/eth2/types/src/beacon_state/beacon_state_types.rs @@ -95,9 +95,9 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq { /// /// Spec v0.6.1 #[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct FoundationEthSpec; +pub struct MainnetEthSpec; -impl EthSpec for FoundationEthSpec { +impl EthSpec for MainnetEthSpec { type ShardCount = U1024; type SlotsPerHistoricalRoot = U8192; type LatestRandaoMixesLength = U8192; @@ -107,17 +107,17 @@ impl EthSpec for FoundationEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::foundation() + ChainSpec::mainnet() } } -pub type FoundationBeaconState = BeaconState; +pub type FoundationBeaconState = BeaconState; /// Ethereum Foundation specifications, modified to be suitable for < 1000 validators. #[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct FewValidatorsEthSpec; +pub struct MinimalEthSpec; -impl EthSpec for FewValidatorsEthSpec { +impl EthSpec for MinimalEthSpec { type ShardCount = U8; type SlotsPerHistoricalRoot = U8192; type LatestRandaoMixesLength = U8192; @@ -127,28 +127,8 @@ impl EthSpec for FewValidatorsEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::few_validators() + ChainSpec::minimal() } } -pub type FewValidatorsBeaconState = BeaconState; - -/// Specifications suitable for a small-scale (< 1000 validators) lighthouse testnet. -#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct LighthouseTestnetEthSpec; - -impl EthSpec for LighthouseTestnetEthSpec { - type ShardCount = U8; - type SlotsPerHistoricalRoot = U8192; - type LatestRandaoMixesLength = U8192; - type LatestActiveIndexRootsLength = U8192; - type LatestSlashedExitLength = U8192; - type SlotsPerEpoch = U8; - type GenesisEpoch = U0; - - fn default_spec() -> ChainSpec { - ChainSpec::lighthouse_testnet() - } -} - -pub type LighthouseTestnetBeaconState = BeaconState; +pub type MinimalBeaconState = BeaconState; diff --git a/eth2/types/src/beacon_state/committee_cache/tests.rs b/eth2/types/src/beacon_state/committee_cache/tests.rs index 8f1fbe1879..d30d0724d6 100644 --- a/eth2/types/src/beacon_state/committee_cache/tests.rs +++ b/eth2/types/src/beacon_state/committee_cache/tests.rs @@ -34,8 +34,8 @@ fn new_state(validator_count: usize, slot: Slot) -> BeaconState { #[test] fn fails_without_validators() { - let state = new_state::(0, Slot::new(0)); - let spec = &FewValidatorsEthSpec::default_spec(); + let state = new_state::(0, Slot::new(0)); + let spec = &MinimalEthSpec::default_spec(); assert_eq!( CommitteeCache::initialized(&state, state.current_epoch(), &spec), @@ -45,8 +45,8 @@ fn fails_without_validators() { #[test] fn initializes_with_the_right_epoch() { - let state = new_state::(16, Slot::new(0)); - let spec = &FewValidatorsEthSpec::default_spec(); + let state = new_state::(16, Slot::new(0)); + let spec = &MinimalEthSpec::default_spec(); let cache = CommitteeCache::default(); assert_eq!(cache.initialized_epoch, None); @@ -63,14 +63,14 @@ fn initializes_with_the_right_epoch() { #[test] fn shuffles_for_the_right_epoch() { - let num_validators = FewValidatorsEthSpec::minimum_validator_count() * 2; + let num_validators = MinimalEthSpec::minimum_validator_count() * 2; let epoch = Epoch::new(100_000_000); - let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch()); + let slot = epoch.start_slot(MinimalEthSpec::slots_per_epoch()); - let mut state = new_state::(num_validators, slot); - let spec = &FewValidatorsEthSpec::default_spec(); + let mut state = new_state::(num_validators, slot); + let spec = &MinimalEthSpec::default_spec(); - let distinct_hashes: Vec = (0..FewValidatorsEthSpec::latest_randao_mixes_length()) + let distinct_hashes: Vec = (0..MinimalEthSpec::latest_randao_mixes_length()) .into_iter() .map(|i| Hash256::from(i as u64)) .collect(); @@ -118,14 +118,14 @@ fn shuffles_for_the_right_epoch() { #[test] fn can_start_on_any_shard() { - let num_validators = FewValidatorsEthSpec::minimum_validator_count() * 2; + let num_validators = MinimalEthSpec::minimum_validator_count() * 2; let epoch = Epoch::new(100_000_000); - let slot = epoch.start_slot(FewValidatorsEthSpec::slots_per_epoch()); + let slot = epoch.start_slot(MinimalEthSpec::slots_per_epoch()); - let mut state = new_state::(num_validators, slot); - let spec = &FewValidatorsEthSpec::default_spec(); + let mut state = new_state::(num_validators, slot); + let spec = &MinimalEthSpec::default_spec(); - for i in 0..FewValidatorsEthSpec::shard_count() as u64 { + for i in 0..MinimalEthSpec::shard_count() as u64 { state.latest_start_shard = i; let cache = CommitteeCache::initialized(&state, state.current_epoch(), spec).unwrap(); @@ -154,7 +154,7 @@ impl EthSpec for ExcessShardsEthSpec { type GenesisEpoch = U0; fn default_spec() -> ChainSpec { - ChainSpec::few_validators() + ChainSpec::minimal() } } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index a668f99fe0..b3c641f614 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -53,7 +53,7 @@ fn test_beacon_proposer_index() { #[test] fn beacon_proposer_index() { - test_beacon_proposer_index::(); + test_beacon_proposer_index::(); } /// Should produce (note the set notation brackets): @@ -115,11 +115,11 @@ fn test_active_index(state_slot: Slot) { #[test] fn get_active_index_root_index() { - test_active_index::(Slot::new(0)); + test_active_index::(Slot::new(0)); - let epoch = Epoch::from(FoundationEthSpec::latest_active_index_roots() * 4); - let slot = epoch.start_slot(FoundationEthSpec::slots_per_epoch()); - test_active_index::(slot); + let epoch = Epoch::from(MainnetEthSpec::latest_active_index_roots() * 4); + let slot = epoch.start_slot(MainnetEthSpec::slots_per_epoch()); + test_active_index::(slot); } /// Test that @@ -166,14 +166,14 @@ fn test_cache_initialization<'a, T: EthSpec>( #[test] fn cache_initialization() { - let spec = FewValidatorsEthSpec::default_spec(); + let spec = MinimalEthSpec::default_spec(); - let builder: TestingBeaconStateBuilder = + let builder: TestingBeaconStateBuilder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(16, &spec); let (mut state, _keypairs) = builder.build(); - state.slot = (FewValidatorsEthSpec::genesis_epoch() + 1) - .start_slot(FewValidatorsEthSpec::slots_per_epoch()); + state.slot = + (MinimalEthSpec::genesis_epoch() + 1).start_slot(MinimalEthSpec::slots_per_epoch()); test_cache_initialization(&mut state, RelativeEpoch::Previous, &spec); test_cache_initialization(&mut state, RelativeEpoch::Current, &spec); @@ -203,7 +203,7 @@ fn tree_hash_cache() { #[cfg(test)] mod committees { use super::*; - use crate::beacon_state::FewValidatorsEthSpec; + use crate::beacon_state::MinimalEthSpec; use swap_or_not_shuffle::shuffle_list; fn execute_committee_consistency_test( @@ -347,16 +347,16 @@ mod committees { #[test] fn current_epoch_committee_consistency() { - committee_consistency_test_suite::(RelativeEpoch::Current); + committee_consistency_test_suite::(RelativeEpoch::Current); } #[test] fn previous_epoch_committee_consistency() { - committee_consistency_test_suite::(RelativeEpoch::Previous); + committee_consistency_test_suite::(RelativeEpoch::Previous); } #[test] fn next_epoch_committee_consistency() { - committee_consistency_test_suite::(RelativeEpoch::Next); + committee_consistency_test_suite::(RelativeEpoch::Next); } } diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index a1267e37dd..0a2602ea83 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -136,7 +136,7 @@ impl ChainSpec { /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification. /// /// Spec v0.6.1 - pub fn foundation() -> Self { + pub fn mainnet() -> Self { Self { /* * Misc @@ -217,43 +217,35 @@ impl ChainSpec { * Boot nodes */ boot_nodes: vec![], - chain_id: 1, // foundation chain id + chain_id: 1, // mainnet chain id } } - /// Returns a `ChainSpec` compatible with the Lighthouse testnet specification. - /// - /// Spec v0.4.0 - pub fn lighthouse_testnet() -> Self { - /* - * Lighthouse testnet bootnodes - */ + /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. + pub fn minimal() -> Self { + let genesis_slot = Slot::new(0); + + // Note: these bootnodes are placeholders. + // + // Should be updated once static bootnodes exist. let boot_nodes = vec!["/ip4/127.0.0.1/tcp/9000" .parse() .expect("correct multiaddr")]; Self { boot_nodes, - chain_id: 2, // lighthouse testnet chain id - ..ChainSpec::few_validators() - } - } - - /// Returns a `ChainSpec` compatible with the specification suitable for 8 validators. - pub fn few_validators() -> Self { - let genesis_slot = Slot::new(0); - - Self { target_committee_size: 1, + chain_id: 2, // lighthouse testnet chain id genesis_slot, - ..ChainSpec::foundation() + shuffle_round_count: 10, + ..ChainSpec::mainnet() } } } impl Default for ChainSpec { fn default() -> Self { - Self::foundation() + Self::mainnet() } } @@ -263,8 +255,8 @@ mod tests { use int_to_bytes::int_to_bytes8; #[test] - fn test_foundation_spec_can_be_constructed() { - let _ = ChainSpec::foundation(); + fn test_mainnet_spec_can_be_constructed() { + let _ = ChainSpec::mainnet(); } fn test_domain(domain_type: Domain, raw_domain: u32, spec: &ChainSpec) { @@ -281,7 +273,7 @@ mod tests { #[test] fn test_get_domain() { - let spec = ChainSpec::foundation(); + let spec = ChainSpec::mainnet(); test_domain(Domain::BeaconProposer, spec.domain_beacon_proposer, &spec); test_domain(Domain::Randao, spec.domain_randao, &spec); diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index ec5f3af4cd..60ab208ad2 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -63,7 +63,7 @@ mod tests { cached_tree_hash_tests!(Fork); fn test_genesis(epoch: Epoch) { - let mut spec = ChainSpec::foundation(); + let mut spec = ChainSpec::mainnet(); let fork = Fork::genesis(epoch); diff --git a/eth2/types/src/historical_batch.rs b/eth2/types/src/historical_batch.rs index 3480508dc7..0d8916216d 100644 --- a/eth2/types/src/historical_batch.rs +++ b/eth2/types/src/historical_batch.rs @@ -31,7 +31,7 @@ pub struct HistoricalBatch { mod tests { use super::*; - pub type FoundationHistoricalBatch = HistoricalBatch; + pub type FoundationHistoricalBatch = HistoricalBatch; ssz_tests!(FoundationHistoricalBatch); cached_tree_hash_tests!(FoundationHistoricalBatch); diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs index f69d1f998e..301ba91780 100644 --- a/tests/ef_tests/src/doc.rs +++ b/tests/ef_tests/src/doc.rs @@ -1,11 +1,11 @@ use crate::case_result::CaseResult; use crate::cases::*; use crate::doc_header::DocHeader; -use crate::eth_specs::{MainnetEthSpec, MinimalEthSpec}; use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode}; use crate::EfTest; use serde_derive::Deserialize; use std::{fs::File, io::prelude::*, path::PathBuf}; +use types::{MainnetEthSpec, MinimalEthSpec}; #[derive(Debug, Deserialize)] pub struct Doc { diff --git a/tests/ef_tests/src/eth_specs.rs b/tests/ef_tests/src/eth_specs.rs deleted file mode 100644 index f0ee1e9f03..0000000000 --- a/tests/ef_tests/src/eth_specs.rs +++ /dev/null @@ -1,32 +0,0 @@ -use serde_derive::{Deserialize, Serialize}; -use types::{ - typenum::{U0, U64, U8}, - ChainSpec, EthSpec, FewValidatorsEthSpec, FoundationEthSpec, -}; - -/// "Minimal" testing specification, as defined here: -/// -/// https://github.com/ethereum/eth2.0-specs/blob/v0.6.1/configs/constant_presets/minimal.yaml -/// -/// Spec v0.6.1 -#[derive(Clone, PartialEq, Debug, Default, Serialize, Deserialize)] -pub struct MinimalEthSpec; - -impl EthSpec for MinimalEthSpec { - type ShardCount = U8; - type SlotsPerHistoricalRoot = U64; - type LatestRandaoMixesLength = U64; - type LatestActiveIndexRootsLength = U64; - type LatestSlashedExitLength = U64; - type SlotsPerEpoch = U8; - type GenesisEpoch = U0; - - fn default_spec() -> ChainSpec { - // TODO: this spec is likely incorrect! - let mut spec = FewValidatorsEthSpec::default_spec(); - spec.shuffle_round_count = 10; - spec - } -} - -pub type MainnetEthSpec = FoundationEthSpec; diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index 942a6dbb78..ac238bb1c2 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -11,7 +11,6 @@ mod cases; mod doc; mod doc_header; mod error; -mod eth_specs; mod yaml_decode; /// Defined where an object can return the results of some test(s) adhering to the Ethereum diff --git a/validator_client/src/block_producer/mod.rs b/validator_client/src/block_producer/mod.rs index 9c19e35324..212db1f8eb 100644 --- a/validator_client/src/block_producer/mod.rs +++ b/validator_client/src/block_producer/mod.rs @@ -183,7 +183,7 @@ mod tests { pub fn polling() { let mut rng = XorShiftRng::from_seed([42; 16]); - let spec = Arc::new(ChainSpec::foundation()); + let spec = Arc::new(ChainSpec::mainnet()); let slot_clock = Arc::new(TestingSlotClock::new(0)); let beacon_node = Arc::new(SimulatedBeaconNode::default()); let signer = Arc::new(LocalSigner::new(Keypair::random())); diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index f5d795ede5..b6e2c5bb55 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -6,9 +6,7 @@ use std::fs; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; -use types::{ - ChainSpec, EthSpec, FewValidatorsEthSpec, FoundationEthSpec, LighthouseTestnetEthSpec, -}; +use types::{ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec}; /// Stores the core configuration for this validator instance. #[derive(Clone)] @@ -34,13 +32,13 @@ impl Default for Config { let server = "localhost:5051".to_string(); - let spec = FoundationEthSpec::default_spec(); + let spec = MainnetEthSpec::default_spec(); Self { data_dir, server, spec, - slots_per_epoch: FoundationEthSpec::slots_per_epoch(), + slots_per_epoch: MainnetEthSpec::slots_per_epoch(), } } } @@ -69,9 +67,8 @@ impl Config { if let Some(spec_str) = args.value_of("spec") { info!(log, "Using custom spec: {:?}", spec_str); config.spec = match spec_str { - "foundation" => FoundationEthSpec::default_spec(), - "few_validators" => FewValidatorsEthSpec::default_spec(), - "lighthouse_testnet" => LighthouseTestnetEthSpec::default_spec(), + "mainnet" => MainnetEthSpec::default_spec(), + "minimal" => MinimalEthSpec::default_spec(), // Should be impossible due to clap's `possible_values(..)` function. _ => unreachable!(), }; diff --git a/validator_client/src/duties/mod.rs b/validator_client/src/duties/mod.rs index 3f75c937d2..f0269a41f6 100644 --- a/validator_client/src/duties/mod.rs +++ b/validator_client/src/duties/mod.rs @@ -163,7 +163,7 @@ mod tests { #[test] pub fn polling() { - let spec = Arc::new(ChainSpec::foundation()); + let spec = Arc::new(ChainSpec::mainnet()); let duties_map = Arc::new(EpochDutiesMap::new(T::slots_per_epoch())); let keypair = Keypair::random(); let slot_clock = Arc::new(TestingSlotClock::new(0)); diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 038399936d..d755db4d24 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -46,8 +46,8 @@ fn main() { .short("s") .help("Configuration of Beacon Chain") .takes_value(true) - .possible_values(&["foundation", "few_validators", "lighthouse_testnet"]) - .default_value("lighthouse_testnet"), + .possible_values(&["mainnet", "minimal"]) + .default_value("minimal"), ) .get_matches(); From fd6766c26880faf34e8448a0833a58870c0d9762 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 09:46:04 -0400 Subject: [PATCH 65/76] Tidy beacon node runtime code --- beacon_node/Cargo.toml | 1 - beacon_node/client/src/beacon_chain_types.rs | 36 ++++++----------- beacon_node/client/src/lib.rs | 2 +- beacon_node/rpc/src/lib.rs | 3 +- beacon_node/src/run.rs | 41 +++++++++++++++----- beacon_node/store/src/leveldb_store.rs | 8 +++- beacon_node/store/src/memory_store.rs | 8 +++- 7 files changed, 59 insertions(+), 40 deletions(-) diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 3107068d9c..47abb6c6de 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -10,7 +10,6 @@ types = { path = "../eth2/types" } toml = "^0.5" store = { path = "./store" } client = { path = "client" } -fork_choice = { path = "../eth2/fork_choice" } version = { path = "version" } clap = "2.32.0" slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 133ebcc948..c55c04b443 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -1,18 +1,14 @@ use beacon_chain::{ - fork_choice::OptimizedLMDGhost, - slot_clock::SystemTimeSlotClock, - store::{DiskStore, MemoryStore, Store}, - BeaconChain, BeaconChainTypes, + fork_choice::OptimizedLMDGhost, slot_clock::SystemTimeSlotClock, store::Store, BeaconChain, + BeaconChainTypes, }; use fork_choice::ForkChoice; use slog::{info, Logger}; use slot_clock::SlotClock; +use std::marker::PhantomData; use std::sync::Arc; use tree_hash::TreeHash; -use types::{ - test_utils::TestingBeaconStateBuilder, BeaconBlock, ChainSpec, EthSpec, Hash256, - MinimalEthSpec, -}; +use types::{test_utils::TestingBeaconStateBuilder, BeaconBlock, ChainSpec, EthSpec, Hash256}; /// The number initial validators when starting the `Minimal`. const TESTNET_VALIDATOR_COUNT: usize = 16; @@ -28,27 +24,19 @@ pub trait InitialiseBeaconChain { } } -/// A testnet-suitable BeaconChainType, using `MemoryStore`. #[derive(Clone)] -pub struct TestnetMemoryBeaconChainTypes; -impl BeaconChainTypes for TestnetMemoryBeaconChainTypes { - type Store = MemoryStore; - type SlotClock = SystemTimeSlotClock; - type ForkChoice = OptimizedLMDGhost; - type EthSpec = MinimalEthSpec; +pub struct ClientType { + _phantom_t: PhantomData, + _phantom_u: PhantomData, } -impl InitialiseBeaconChain for TestnetMemoryBeaconChainTypes {} -/// A testnet-suitable BeaconChainType, using `DiskStore`. -#[derive(Clone)] -pub struct TestnetDiskBeaconChainTypes; -impl BeaconChainTypes for TestnetDiskBeaconChainTypes { - type Store = DiskStore; +impl BeaconChainTypes for ClientType { + type Store = S; type SlotClock = SystemTimeSlotClock; - type ForkChoice = OptimizedLMDGhost; - type EthSpec = MinimalEthSpec; + type ForkChoice = OptimizedLMDGhost; + type EthSpec = E; } -impl InitialiseBeaconChain for TestnetDiskBeaconChainTypes {} +impl InitialiseBeaconChain for ClientType {} /// Loads a `BeaconChain` from `store`, if it exists. Otherwise, create a new chain from genesis. fn maybe_load_from_store_for_testnet( diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index ae28858c5c..c4e1c25588 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -19,8 +19,8 @@ use tokio::runtime::TaskExecutor; use tokio::timer::Interval; pub use beacon_chain::BeaconChainTypes; +pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; -pub use beacon_chain_types::{TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes}; pub use client_config::ClientConfig; /// Main beacon node client service. This provides the connection and initialisation of the clients diff --git a/beacon_node/rpc/src/lib.rs b/beacon_node/rpc/src/lib.rs index f2f1b2abf3..11de6eb6a6 100644 --- a/beacon_node/rpc/src/lib.rs +++ b/beacon_node/rpc/src/lib.rs @@ -27,7 +27,8 @@ pub fn start_server( network_chan: crossbeam_channel::Sender, beacon_chain: Arc>, log: &slog::Logger, -) -> exit_future::Signal { +) -> exit_future::Signal +{ let log = log.new(o!("Service"=>"RPC")); let env = Arc::new(Environment::new(1)); diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 0638e9ec93..4e2ea08761 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,10 +1,9 @@ use client::{ - error, notifier, BeaconChainTypes, Client, ClientConfig, InitialiseBeaconChain, - TestnetDiskBeaconChainTypes, TestnetMemoryBeaconChainTypes, + error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, InitialiseBeaconChain, }; use futures::sync::oneshot; use futures::Future; -use slog::{error, info}; +use slog::{warn, error, info}; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; @@ -13,6 +12,7 @@ use tokio::runtime::Builder; use tokio::runtime::Runtime; use tokio::runtime::TaskExecutor; use tokio_timer::clock::Clock; +use types::{MainnetEthSpec, MinimalEthSpec}; pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Result<()> { let runtime = Builder::new() @@ -27,16 +27,22 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul .db_path() .ok_or_else::(|| "Unable to access database path".into())?; let db_type = &config.db_type; - let spec_constants = &config.spec_constants; + let spec_constants = config.spec_constants.clone(); let other_config = config.clone(); let result = match (db_type.as_str(), spec_constants.as_str()) { - ("disk", "testnet") => { - run::(&db_path, config, executor, runtime, log) + ("disk", "minimal") => { + run::>(&db_path, config, executor, runtime, log) } - ("memory", "testnet") => { - run::(&db_path, config, executor, runtime, log) + ("memory", "minimal") => { + run::>(&db_path, config, executor, runtime, log) + } + ("disk", "mainnet") => { + run::>(&db_path, config, executor, runtime, log) + } + ("memory", "mainnet") => { + run::>(&db_path, config, executor, runtime, log) } (db_type, spec) => { error!(log, "Unknown runtime configuration"; "spec" => spec, "db_type" => db_type); @@ -53,6 +59,23 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul "spec_constants" => &other_config.spec_constants, "db_type" => &other_config.db_type, ); + + // `SHUFFLE_ROUND_COUNT == 10` in minimal, this is not considered safe. + if spec_constants.as_str() == "minimal" { + warn!( + log, + "The minimal specification does not use cryptographically secure committee selection." + ) + } + + // Mainnet is not really complete, it still generates determinitic, unsafe initial + // validators. + if spec_constants.as_str() == "mainnet" { + warn!( + log, + "The mainnet specification uses unsafe validator keypairs." + ) + } } result @@ -66,7 +89,7 @@ pub fn run( log: &slog::Logger, ) -> error::Result<()> where - T: BeaconChainTypes + InitialiseBeaconChain + Send + Sync + 'static + Clone, + T: BeaconChainTypes + InitialiseBeaconChain + Clone + Send + Sync + 'static, T::Store: OpenDatabase, { let store = T::Store::open_database(&db_path)?; diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index 09aec46fa7..699861e3ae 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -5,10 +5,14 @@ use leveldb::database::Database; use leveldb::error::Error as LevelDBError; use leveldb::options::{Options, ReadOptions, WriteOptions}; use std::path::Path; +use std::sync::Arc; /// A wrapped leveldb database. +#[derive(Clone)] pub struct LevelDB { - db: Database, + // Note: this `Arc` is only included because of an artificial constraint by gRPC. Hopefully we + // can remove this one day. + db: Arc>, } impl LevelDB { @@ -18,7 +22,7 @@ impl LevelDB { options.create_if_missing = true; - let db = Database::open(path, options)?; + let db = Arc::new(Database::open(path, options)?); Ok(Self { db }) } diff --git a/beacon_node/store/src/memory_store.rs b/beacon_node/store/src/memory_store.rs index 086a16c269..048c054f52 100644 --- a/beacon_node/store/src/memory_store.rs +++ b/beacon_node/store/src/memory_store.rs @@ -1,19 +1,23 @@ use super::{Error, Store}; use parking_lot::RwLock; use std::collections::HashMap; +use std::sync::Arc; type DBHashMap = HashMap, Vec>; /// A thread-safe `HashMap` wrapper. +#[derive(Clone)] pub struct MemoryStore { - db: RwLock, + // Note: this `Arc` is only included because of an artificial constraint by gRPC. Hopefully we + // can remove this one day. + db: Arc>, } impl MemoryStore { /// Create a new, empty database. pub fn open() -> Self { Self { - db: RwLock::new(HashMap::new()), + db: Arc::new(RwLock::new(HashMap::new())), } } From d8fc5f31d8b842788b77662d305c66b5ad900038 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 13:17:03 -0400 Subject: [PATCH 66/76] Add Eth2Config to runtime --- beacon_node/Cargo.toml | 1 + beacon_node/client/src/client_config.rs | 7 +- beacon_node/client/src/eth2_config.rs | 48 +++++++++ beacon_node/client/src/lib.rs | 25 +++-- beacon_node/src/main.rs | 66 +++++++++--- beacon_node/src/run.rs | 100 ++++++++++-------- .../block_processing_builder.rs | 2 +- eth2/types/src/chain_spec.rs | 10 +- eth2/types/src/slot_epoch.rs | 1 + eth2/types/src/slot_epoch_macros.rs | 2 +- .../builders/testing_beacon_state_builder.rs | 21 +--- eth2/types/src/test_utils/mod.rs | 2 +- eth2/types/src/test_utils/serde_utils.rs | 12 ++- 13 files changed, 196 insertions(+), 101 deletions(-) create mode 100644 beacon_node/client/src/eth2_config.rs diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 47abb6c6de..77c9844aa1 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -12,6 +12,7 @@ store = { path = "./store" } client = { path = "client" } version = { path = "version" } clap = "2.32.0" +serde = "1.0" slog = { version = "^2.2.3" , features = ["max_level_trace", "release_max_level_debug"] } slog-term = "^2.4.0" slog-async = "^2.3.0" diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index 1f875f02d4..b9caff9f8a 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -4,26 +4,22 @@ use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; -use types::ChainSpec; /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClientConfig { pub data_dir: String, - pub spec_constants: String, pub db_type: String, db_name: String, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, - pub http: HttpServerConfig, //pub ipc_conf: - pub spec: ChainSpec, + pub http: HttpServerConfig, } impl Default for ClientConfig { fn default() -> Self { Self { data_dir: ".lighthouse".to_string(), - spec_constants: "testnet".to_string(), db_type: "disk".to_string(), db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. @@ -31,7 +27,6 @@ impl Default for ClientConfig { network: NetworkConfig::new(vec![]), rpc: rpc::RPCConfig::default(), http: HttpServerConfig::default(), - spec: ChainSpec::minimal(), } } } diff --git a/beacon_node/client/src/eth2_config.rs b/beacon_node/client/src/eth2_config.rs new file mode 100644 index 0000000000..ca5db98ba3 --- /dev/null +++ b/beacon_node/client/src/eth2_config.rs @@ -0,0 +1,48 @@ +use clap::ArgMatches; +use serde_derive::{Deserialize, Serialize}; +use std::time::SystemTime; +use types::ChainSpec; + +/// The core configuration of a Lighthouse beacon node. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct Eth2Config { + pub spec_constants: String, + pub spec: ChainSpec, +} + +impl Default for Eth2Config { + fn default() -> Self { + Self { + spec_constants: "minimal".to_string(), + spec: ChainSpec::minimal(), + } + } +} + +impl Eth2Config { + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. + /// + /// Returns an error if arguments are obviously invalid. May succeed even if some values are + /// invalid. + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if args.is_present("recent_genesis") { + self.spec.genesis_time = recent_genesis_time() + } + + Ok(()) + } +} + +/// Returns the system time, mod 30 minutes. +/// +/// Used for easily creating testnets. +fn recent_genesis_time() -> u64 { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(); + let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); + // genesis is now the last 30 minute block. + now - secs_after_last_period +} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index c4e1c25588..67aff3342e 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -3,6 +3,7 @@ extern crate slog; mod beacon_chain_types; mod client_config; pub mod error; +mod eth2_config; pub mod notifier; use beacon_chain::BeaconChain; @@ -22,12 +23,13 @@ pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; pub use client_config::ClientConfig; +pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients /// sub-services in multiple threads. pub struct Client { /// Configuration for the lighthouse client. - _config: ClientConfig, + _client_config: ClientConfig, /// The beacon chain for the running client. beacon_chain: Arc>, /// Reference to the network service. @@ -50,19 +52,20 @@ where { /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( - config: ClientConfig, + client_config: ClientConfig, + eth2_config: Eth2Config, store: T::Store, log: slog::Logger, executor: &TaskExecutor, ) -> error::Result { let metrics_registry = Registry::new(); let store = Arc::new(store); - let seconds_per_slot = config.spec.seconds_per_slot; + let seconds_per_slot = eth2_config.spec.seconds_per_slot; // Load a `BeaconChain` from the store, or create a new one if it does not exist. let beacon_chain = Arc::new(T::initialise_beacon_chain( store, - config.spec.clone(), + eth2_config.spec.clone(), log.clone(), )); // Registry all beacon chain metrics with the global registry. @@ -102,7 +105,7 @@ where // Start the network service, libp2p and syncing threads // TODO: Add beacon_chain reference to network parameters - let network_config = &config.network; + let network_config = &client_config.network; let network_logger = log.new(o!("Service" => "Network")); let (network, network_send) = NetworkService::new( beacon_chain.clone(), @@ -112,9 +115,9 @@ where )?; // spawn the RPC server - let rpc_exit_signal = if config.rpc.enabled { + let rpc_exit_signal = if client_config.rpc.enabled { Some(rpc::start_server( - &config.rpc, + &client_config.rpc, executor, network_send.clone(), beacon_chain.clone(), @@ -127,13 +130,13 @@ where // Start the `http_server` service. // // Note: presently we are ignoring the config and _always_ starting a HTTP server. - let http_exit_signal = if config.http.enabled { + let http_exit_signal = if client_config.http.enabled { Some(http_server::start_service( - &config.http, + &client_config.http, executor, network_send, beacon_chain.clone(), - config.db_path().expect("unable to read datadir"), + client_config.db_path().expect("unable to read datadir"), metrics_registry, &log, )) @@ -168,7 +171,7 @@ where } Ok(Client { - _config: config, + _client_config: client_config, beacon_chain, http_exit_signal, rpc_exit_signal, diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 8d2316d99e..fe29ecd41a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -3,16 +3,17 @@ extern crate slog; mod run; use clap::{App, Arg}; -use client::ClientConfig; +use client::{ClientConfig, Eth2Config}; use slog::{crit, o, Drain}; use std::fs; use std::fs::File; use std::io::prelude::*; -pub const SAMPLE_CONFIG_FILENAME: &str = "beacon_node_config.sample.toml"; -pub const CONFIG_FILENAME: &str = "beacon_node_config.toml"; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; +pub const CLIENT_CONFIG_FILENAME: &str = "client_config.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2_config.toml"; + fn main() { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::CompactFormat::new(decorator).build().fuse(); @@ -100,9 +101,17 @@ fn main() { .possible_values(&["disk", "memory"]) .default_value("memory"), ) + .arg( + Arg::with_name("recent_genesis") + .long("recent_genesis") + .help("When present, genesis will be within 30 minutes prior. Only for testing"), + ) .get_matches(); - let mut config = match load_config(matches.value_of("data_dir")) { + let mut client_config = match load_config::( + matches.value_of("data_dir"), + CLIENT_CONFIG_FILENAME, + ) { Ok(c) => c, Err(e) => { crit!(logger, "Failed to load/generate a ChainConfig"; "error" => format!("{:?}", e)); @@ -110,15 +119,38 @@ fn main() { } }; - match config.apply_cli_args(&matches) { + if let Some(data_dir) = matches.value_of("data_dir") { + client_config.data_dir = data_dir.to_string(); + } + + match client_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { - crit!(logger, "Failed to parse CLI arguments"; "error" => s); + crit!(logger, "Failed to parse ClientConfig CLI arguments"; "error" => s); return; } }; - match run::run_beacon_node(config, &logger) { + let mut eth2_config = match load_config::( + matches.value_of("data_dir"), + ETH2_CONFIG_FILENAME, + ) { + Ok(c) => c, + Err(e) => { + crit!(logger, "Failed to load/generate an Eth2Config"; "error" => format!("{:?}", e)); + return; + } + }; + + match eth2_config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(logger, "Failed to parse Eth2Config CLI arguments"; "error" => s); + return; + } + }; + + match run::run_beacon_node(client_config, eth2_config, &logger) { Ok(_) => {} Err(e) => crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), } @@ -126,7 +158,10 @@ fn main() { /// Loads a `ClientConfig` from file. If unable to load from file, generates a default /// configuration and saves that as a sample file. -fn load_config(data_dir: Option<&str>) -> Result { +fn load_config(data_dir: Option<&str>, config_filename: &str) -> Result +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); let path = dirs::home_dir() @@ -134,29 +169,28 @@ fn load_config(data_dir: Option<&str>) -> Result { .join(&data_dir); fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; - if let Ok(mut file) = File::open(path.join(CONFIG_FILENAME)) { + if let Ok(mut file) = File::open(path.join(config_filename)) { let mut contents = String::new(); file.read_to_string(&mut contents).map_err(|e| { format!( "Unable to read existing {}. Error: {:?}", - CONFIG_FILENAME, e + config_filename, e ) })?; - toml::from_str(&contents).map_err(|_| format!("Unable to parse {}", CONFIG_FILENAME)) + toml::from_str(&contents).map_err(|e| format!("Unable to parse {}: {:?}", config_filename, e)) } else { - let mut config = ClientConfig::default(); - config.data_dir = data_dir.to_string(); + let config = T::default(); - if let Ok(mut file) = File::create(path.join(SAMPLE_CONFIG_FILENAME)) { + if let Ok(mut file) = File::create(path.join(config_filename)) { let toml_encoded = toml::to_string(&config).map_err(|e| { format!( "Failed to write configuration to {}. Error: {:?}", - SAMPLE_CONFIG_FILENAME, e + config_filename, e ) })?; file.write_all(toml_encoded.as_bytes()) - .expect(&format!("Unable to write to {}", SAMPLE_CONFIG_FILENAME)); + .expect(&format!("Unable to write to {}", config_filename)); } Ok(config) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index 4e2ea08761..834f9a4281 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -1,9 +1,10 @@ use client::{ - error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, InitialiseBeaconChain, + error, notifier, BeaconChainTypes, Client, ClientConfig, ClientType, Eth2Config, + InitialiseBeaconChain, }; use futures::sync::oneshot; use futures::Future; -use slog::{warn, error, info}; +use slog::{error, info, warn}; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; @@ -14,7 +15,11 @@ use tokio::runtime::TaskExecutor; use tokio_timer::clock::Clock; use types::{MainnetEthSpec, MinimalEthSpec}; -pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Result<()> { +pub fn run_beacon_node( + client_config: ClientConfig, + eth2_config: Eth2Config, + log: &slog::Logger, +) -> error::Result<()> { let runtime = Builder::new() .name_prefix("main-") .clock(Clock::system()) @@ -23,29 +28,54 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul let executor = runtime.executor(); - let db_path: PathBuf = config + let db_path: PathBuf = client_config .db_path() .ok_or_else::(|| "Unable to access database path".into())?; - let db_type = &config.db_type; - let spec_constants = config.spec_constants.clone(); + let db_type = &client_config.db_type; + let spec_constants = eth2_config.spec_constants.clone(); - let other_config = config.clone(); + let other_client_config = client_config.clone(); + + warn!( + log, + "This software is EXPERIMENTAL and provides no guarantees or warranties." + ); let result = match (db_type.as_str(), spec_constants.as_str()) { - ("disk", "minimal") => { - run::>(&db_path, config, executor, runtime, log) - } - ("memory", "minimal") => { - run::>(&db_path, config, executor, runtime, log) - } - ("disk", "mainnet") => { - run::>(&db_path, config, executor, runtime, log) - } - ("memory", "mainnet") => { - run::>(&db_path, config, executor, runtime, log) - } + ("disk", "minimal") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("memory", "minimal") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("disk", "mainnet") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), + ("memory", "mainnet") => run::>( + &db_path, + client_config, + eth2_config, + executor, + runtime, + log, + ), (db_type, spec) => { - error!(log, "Unknown runtime configuration"; "spec" => spec, "db_type" => db_type); + error!(log, "Unknown runtime configuration"; "spec_constants" => spec, "db_type" => db_type); Err("Unknown specification and/or db_type.".into()) } }; @@ -54,28 +84,11 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul info!( log, "Started beacon node"; - "p2p_listen_addresses" => format!("{:?}", &other_config.network.listen_addresses()), - "data_dir" => format!("{:?}", other_config.data_dir()), - "spec_constants" => &other_config.spec_constants, - "db_type" => &other_config.db_type, + "p2p_listen_addresses" => format!("{:?}", &other_client_config.network.listen_addresses()), + "data_dir" => format!("{:?}", other_client_config.data_dir()), + "spec_constants" => &spec_constants, + "db_type" => &other_client_config.db_type, ); - - // `SHUFFLE_ROUND_COUNT == 10` in minimal, this is not considered safe. - if spec_constants.as_str() == "minimal" { - warn!( - log, - "The minimal specification does not use cryptographically secure committee selection." - ) - } - - // Mainnet is not really complete, it still generates determinitic, unsafe initial - // validators. - if spec_constants.as_str() == "mainnet" { - warn!( - log, - "The mainnet specification uses unsafe validator keypairs." - ) - } } result @@ -83,7 +96,8 @@ pub fn run_beacon_node(config: ClientConfig, log: &slog::Logger) -> error::Resul pub fn run( db_path: &Path, - config: ClientConfig, + client_config: ClientConfig, + eth2_config: Eth2Config, executor: TaskExecutor, mut runtime: Runtime, log: &slog::Logger, @@ -94,7 +108,7 @@ where { let store = T::Store::open_database(&db_path)?; - let client: Client = Client::new(config, store, log.clone(), &executor)?; + let client: Client = Client::new(client_config, eth2_config, store, log.clone(), &executor)?; // run service until ctrl-c let (ctrlc_send, ctrlc_oneshot) = oneshot::channel(); diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 5a9d264ce1..3675ef99e5 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -23,7 +23,7 @@ impl BlockProcessingBuilder { } pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.state_builder.teleport_to_slot(slot, &spec); + self.state_builder.teleport_to_slot(slot); } pub fn build_caches(&mut self, spec: &ChainSpec) { diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index 0a2602ea83..b16947328c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -1,7 +1,7 @@ use crate::*; use int_to_bytes::int_to_bytes4; use serde_derive::{Deserialize, Serialize}; -use test_utils::u8_from_hex_str; +use test_utils::{u8_from_hex_str, u8_to_hex_str}; /// Each of the BLS signature domains. /// @@ -48,17 +48,19 @@ pub struct ChainSpec { * Initial Values */ pub genesis_slot: Slot, + // Skipped because serde TOML can't handle u64::max_value, the typical value for this field. + #[serde(skip_serializing)] pub far_future_epoch: Epoch, pub zero_hash: Hash256, - #[serde(deserialize_with = "u8_from_hex_str")] + #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")] pub bls_withdrawal_prefix_byte: u8, /* * Time parameters */ + pub genesis_time: u64, pub seconds_per_slot: u64, pub min_attestation_inclusion_delay: u64, - //pub slots_per_epoch: u64, pub min_seed_lookahead: Epoch, pub activation_exit_delay: u64, pub slots_per_eth1_voting_period: u64, @@ -172,9 +174,9 @@ impl ChainSpec { /* * Time parameters */ + genesis_time: u32::max_value() as u64, seconds_per_slot: 6, min_attestation_inclusion_delay: 4, - // slots_per_epoch: 64, min_seed_lookahead: Epoch::new(1), activation_exit_delay: 4, slots_per_eth1_voting_period: 1_024, diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index 82cee0d75e..fbeb479d78 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -23,6 +23,7 @@ use std::iter::Iterator; use std::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Rem, Sub, SubAssign}; #[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] +#[serde(transparent)] pub struct Slot(u64); #[derive(Eq, Debug, Clone, Copy, Default, Serialize, Deserialize)] diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 1e24f8e997..5e02e40c17 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -184,7 +184,7 @@ macro_rules! impl_display { key: slog::Key, serializer: &mut slog::Serializer, ) -> slog::Result { - self.0.serialize(record, key, serializer) + slog::Value::serialize(&self.0, record, key, serializer) } } }; diff --git a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs index f9a4ea00c6..4f09275087 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_state_builder.rs @@ -6,7 +6,6 @@ use dirs; use log::debug; use rayon::prelude::*; use std::path::{Path, PathBuf}; -use std::time::SystemTime; pub const KEYPAIRS_FILE: &str = "keypairs.raw_keypairs"; @@ -123,20 +122,8 @@ impl TestingBeaconStateBuilder { }) .collect(); - // TODO: Testing only. Burn with fire later. - // set genesis to the last 30 minute block. - // this is used for testing only. Allows multiple nodes to connect within a 30min window - // and agree on a genesis - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(); - let secs_after_last_period = now.checked_rem(30 * 60).unwrap_or(0); - // genesis is now the last 30 minute block. - let genesis_time = now - secs_after_last_period; - let mut state = BeaconState::genesis( - genesis_time, + spec.genesis_time, Eth1Data { deposit_root: Hash256::zero(), deposit_count: 0, @@ -172,8 +159,8 @@ impl TestingBeaconStateBuilder { } /// Sets the `BeaconState` to be in a slot, calling `teleport_to_epoch` to update the epoch. - pub fn teleport_to_slot(&mut self, slot: Slot, spec: &ChainSpec) { - self.teleport_to_epoch(slot.epoch(T::slots_per_epoch()), spec); + pub fn teleport_to_slot(&mut self, slot: Slot) { + self.teleport_to_epoch(slot.epoch(T::slots_per_epoch())); self.state.slot = slot; } @@ -181,7 +168,7 @@ impl TestingBeaconStateBuilder { /// /// Sets all justification/finalization parameters to be be as "perfect" as possible (i.e., /// highest justified and finalized slots, full justification bitfield, etc). - fn teleport_to_epoch(&mut self, epoch: Epoch, spec: &ChainSpec) { + fn teleport_to_epoch(&mut self, epoch: Epoch) { let state = &mut self.state; let slot = epoch.start_slot(T::slots_per_epoch()); diff --git a/eth2/types/src/test_utils/mod.rs b/eth2/types/src/test_utils/mod.rs index ee8327be86..b5ec7a0270 100644 --- a/eth2/types/src/test_utils/mod.rs +++ b/eth2/types/src/test_utils/mod.rs @@ -14,5 +14,5 @@ pub use rand::{ RngCore, {prng::XorShiftRng, SeedableRng}, }; -pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str}; +pub use serde_utils::{fork_from_hex_str, graffiti_from_hex_str, u8_from_hex_str, u8_to_hex_str}; pub use test_random::TestRandom; diff --git a/eth2/types/src/test_utils/serde_utils.rs b/eth2/types/src/test_utils/serde_utils.rs index 5c0238c0bb..be8f002adb 100644 --- a/eth2/types/src/test_utils/serde_utils.rs +++ b/eth2/types/src/test_utils/serde_utils.rs @@ -1,5 +1,5 @@ use serde::de::Error; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, Serializer}; pub const FORK_BYTES_LEN: usize = 4; pub const GRAFFITI_BYTES_LEN: usize = 32; @@ -13,6 +13,16 @@ where u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom) } +pub fn u8_to_hex_str(byte: &u8, serializer: S) -> Result +where + S: Serializer, +{ + let mut hex: String = "0x".to_string(); + hex.push_str(&hex::encode(&[*byte])); + + serializer.serialize_str(&hex) +} + pub fn fork_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; FORK_BYTES_LEN], D::Error> where D: Deserializer<'de>, From eb23b003b48fa4d527b72b3d334ba52b62e650c6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 17:53:22 -0400 Subject: [PATCH 67/76] Ensure beacon node generates the right eth2 spec --- beacon_node/client/src/eth2_config.rs | 16 +++++ beacon_node/src/main.rs | 92 ++++++++++++++++++++------- 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/beacon_node/client/src/eth2_config.rs b/beacon_node/client/src/eth2_config.rs index ca5db98ba3..b16e8729e0 100644 --- a/beacon_node/client/src/eth2_config.rs +++ b/beacon_node/client/src/eth2_config.rs @@ -20,6 +20,22 @@ impl Default for Eth2Config { } } +impl Eth2Config { + pub fn mainnet() -> Self { + Self { + spec_constants: "mainnet".to_string(), + spec: ChainSpec::mainnet(), + } + } + + pub fn minimal() -> Self { + Self { + spec_constants: "minimal".to_string(), + spec: ChainSpec::minimal(), + } + } +} + impl Eth2Config { /// Apply the following arguments to `self`, replacing values if they are specified in `args`. /// diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index fe29ecd41a..ef47458c37 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -102,19 +102,37 @@ fn main() { .default_value("memory"), ) .arg( - Arg::with_name("recent_genesis") - .long("recent_genesis") + Arg::with_name("spec-constants") + .long("spec-constants") + .value_name("TITLE") + .help("The title of the spec constants for chain config..") + .takes_value(true) + .possible_values(&["mainnet", "minimal"]) + .default_value("minimal"), + ) + .arg( + Arg::with_name("recent-genesis") + .long("recent-genesis") .help("When present, genesis will be within 30 minutes prior. Only for testing"), ) .get_matches(); - let mut client_config = match load_config::( + // Attempt to lead the `ClientConfig` from disk. If it fails, write + let mut client_config = match read_from_file::( matches.value_of("data_dir"), CLIENT_CONFIG_FILENAME, ) { - Ok(c) => c, + Ok(Some(c)) => c, + Ok(None) => { + let default = ClientConfig::default(); + if let Err(e) = write_to_file(matches.value_of("data_dir"), CLIENT_CONFIG_FILENAME, &default) { + crit!(logger, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); + return; + } + default + }, Err(e) => { - crit!(logger, "Failed to load/generate a ChainConfig"; "error" => format!("{:?}", e)); + crit!(logger, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); return; } }; @@ -131,11 +149,23 @@ fn main() { } }; - let mut eth2_config = match load_config::( + let mut eth2_config = match read_from_file::( matches.value_of("data_dir"), ETH2_CONFIG_FILENAME, ) { - Ok(c) => c, + Ok(Some(c)) => c, + Ok(None) => { + let default = match matches.value_of("spec-constants") { + Some("mainnet") => Eth2Config::mainnet(), + Some("minimal") => Eth2Config::minimal(), + _ => unreachable!(), // Guarded by slog. + }; + if let Err(e) = write_to_file(matches.value_of("data_dir"), ETH2_CONFIG_FILENAME, &default) { + crit!(logger, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); + return; + } + default + } Err(e) => { crit!(logger, "Failed to load/generate an Eth2Config"; "error" => format!("{:?}", e)); return; @@ -156,9 +186,35 @@ fn main() { } } +/// Write a configuration to file. +fn write_to_file(data_dir: Option<&str>, config_filename: &str, config: &T) -> Result<(), String> +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ + let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); + + let path = dirs::home_dir() + .ok_or_else(|| "Unable to locate home directory")? + .join(&data_dir); + fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; + + if let Ok(mut file) = File::create(path.join(config_filename)) { + let toml_encoded = toml::to_string(&config).map_err(|e| { + format!( + "Failed to write configuration to {}. Error: {:?}", + config_filename, e + ) + })?; + file.write_all(toml_encoded.as_bytes()) + .expect(&format!("Unable to write to {}", config_filename)); + } + + Ok(()) +} + /// Loads a `ClientConfig` from file. If unable to load from file, generates a default /// configuration and saves that as a sample file. -fn load_config(data_dir: Option<&str>, config_filename: &str) -> Result +fn read_from_file(data_dir: Option<&str>, config_filename: &str) -> Result, String> where T: Default + serde::de::DeserializeOwned + serde::Serialize, { @@ -178,21 +234,11 @@ where ) })?; - toml::from_str(&contents).map_err(|e| format!("Unable to parse {}: {:?}", config_filename, e)) + let config = toml::from_str(&contents) + .map_err(|e| format!("Unable to parse {}: {:?}", config_filename, e))?; + + Ok(Some(config)) } else { - let config = T::default(); - - if let Ok(mut file) = File::create(path.join(config_filename)) { - let toml_encoded = toml::to_string(&config).map_err(|e| { - format!( - "Failed to write configuration to {}. Error: {:?}", - config_filename, e - ) - })?; - file.write_all(toml_encoded.as_bytes()) - .expect(&format!("Unable to write to {}", config_filename)); - } - - Ok(config) + Ok(None) } } From 3487b16ce58ef6802a7157bb43483beadc9d6600 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 8 Jun 2019 20:21:50 -0400 Subject: [PATCH 68/76] Add `eth2_config` crate, integrate into val client --- Cargo.toml | 1 + beacon_node/Cargo.toml | 1 + beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/client_config.rs | 6 +- beacon_node/client/src/lib.rs | 1 - beacon_node/src/main.rs | 113 +++++-------- eth2/utils/eth2_config/Cargo.toml | 13 ++ .../utils/eth2_config/src/lib.rs | 44 ++++- validator_client/Cargo.toml | 4 + validator_client/eth2_config.toml | 47 ++++++ validator_client/src/config.rs | 43 +++-- validator_client/src/main.rs | 156 ++++++++++++++++-- validator_client/src/service.rs | 65 +++++--- 13 files changed, 366 insertions(+), 129 deletions(-) create mode 100644 eth2/utils/eth2_config/Cargo.toml rename beacon_node/client/src/eth2_config.rs => eth2/utils/eth2_config/src/lib.rs (54%) create mode 100644 validator_client/eth2_config.toml diff --git a/Cargo.toml b/Cargo.toml index e9acb2be47..43c13ac8c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "eth2/utils/cached_tree_hash", "eth2/utils/compare_fields", "eth2/utils/compare_fields_derive", + "eth2/utils/eth2_config", "eth2/utils/fixed_len_vec", "eth2/utils/hashing", "eth2/utils/honey-badger-split", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 77c9844aa1..3edbf636d7 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [dependencies] dirs = "1.0.3" +eth2_config = { path = "../eth2/utils/eth2_config" } types = { path = "../eth2/types" } toml = "^0.5" store = { path = "./store" } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 6da832c337..2b6f44e949 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -14,6 +14,7 @@ fork_choice = { path = "../../eth2/fork_choice" } prometheus = "^0.6" types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } +eth2_config = { path = "../../eth2/utils/eth2_config" } slot_clock = { path = "../../eth2/utils/slot_clock" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/client/src/client_config.rs b/beacon_node/client/src/client_config.rs index b9caff9f8a..166725b61c 100644 --- a/beacon_node/client/src/client_config.rs +++ b/beacon_node/client/src/client_config.rs @@ -8,7 +8,7 @@ use std::path::PathBuf; /// The core configuration of a Lighthouse beacon node. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ClientConfig { - pub data_dir: String, + pub data_dir: PathBuf, pub db_type: String, db_name: String, pub network: network::NetworkConfig, @@ -19,7 +19,7 @@ pub struct ClientConfig { impl Default for ClientConfig { fn default() -> Self { Self { - data_dir: ".lighthouse".to_string(), + data_dir: PathBuf::from(".lighthouse"), db_type: "disk".to_string(), db_name: "chain_db".to_string(), // Note: there are no default bootnodes specified. @@ -51,7 +51,7 @@ impl ClientConfig { /// invalid. pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { if let Some(dir) = args.value_of("datadir") { - self.data_dir = dir.to_string(); + self.data_dir = PathBuf::from(dir); }; if let Some(dir) = args.value_of("db") { diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 67aff3342e..92ed6e0227 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -3,7 +3,6 @@ extern crate slog; mod beacon_chain_types; mod client_config; pub mod error; -mod eth2_config; pub mod notifier; use beacon_chain::BeaconChain; diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index ef47458c37..20217049d6 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -2,12 +2,12 @@ extern crate slog; mod run; -use clap::{App, Arg}; +use clap::{App, Arg, ArgMatches}; use client::{ClientConfig, Eth2Config}; +use eth2_config::{read_from_file, write_to_file}; use slog::{crit, o, Drain}; use std::fs; -use std::fs::File; -use std::io::prelude::*; +use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; @@ -105,7 +105,8 @@ fn main() { Arg::with_name("spec-constants") .long("spec-constants") .value_name("TITLE") - .help("The title of the spec constants for chain config..") + .short("s") + .help("The title of the spec constants for chain config.") .takes_value(true) .possible_values(&["mainnet", "minimal"]) .default_value("minimal"), @@ -113,34 +114,44 @@ fn main() { .arg( Arg::with_name("recent-genesis") .long("recent-genesis") + .short("r") .help("When present, genesis will be within 30 minutes prior. Only for testing"), ) .get_matches(); - // Attempt to lead the `ClientConfig` from disk. If it fails, write - let mut client_config = match read_from_file::( - matches.value_of("data_dir"), - CLIENT_CONFIG_FILENAME, - ) { + let data_dir = match get_data_dir(&matches) { + Ok(dir) => dir, + Err(e) => { + crit!(logger, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + return; + } + }; + + let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); + + // Attempt to lead the `ClientConfig` from disk. + // + // If file doesn't exist, create a new, default one. + let mut client_config = match read_from_file::(client_config_path.clone()) { Ok(Some(c)) => c, Ok(None) => { let default = ClientConfig::default(); - if let Err(e) = write_to_file(matches.value_of("data_dir"), CLIENT_CONFIG_FILENAME, &default) { + if let Err(e) = write_to_file(client_config_path, &default) { crit!(logger, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); return; } default - }, + } Err(e) => { crit!(logger, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); return; } }; - if let Some(data_dir) = matches.value_of("data_dir") { - client_config.data_dir = data_dir.to_string(); - } + // Ensure the `data_dir` in the config matches that supplied to the CLI. + client_config.data_dir = data_dir.clone(); + // Update the client config with any CLI args. match client_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { @@ -149,10 +160,12 @@ fn main() { } }; - let mut eth2_config = match read_from_file::( - matches.value_of("data_dir"), - ETH2_CONFIG_FILENAME, - ) { + let eth2_config_path = data_dir.join(ETH2_CONFIG_FILENAME); + + // Attempt to load the `Eth2Config` from file. + // + // If the file doesn't exist, create a default one depending on the CLI flags. + let mut eth2_config = match read_from_file::(eth2_config_path.clone()) { Ok(Some(c)) => c, Ok(None) => { let default = match matches.value_of("spec-constants") { @@ -160,7 +173,7 @@ fn main() { Some("minimal") => Eth2Config::minimal(), _ => unreachable!(), // Guarded by slog. }; - if let Err(e) = write_to_file(matches.value_of("data_dir"), ETH2_CONFIG_FILENAME, &default) { + if let Err(e) = write_to_file(eth2_config_path, &default) { crit!(logger, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); return; } @@ -172,6 +185,7 @@ fn main() { } }; + // Update the eth2 config with any CLI flags. match eth2_config.apply_cli_args(&matches) { Ok(()) => (), Err(s) => { @@ -186,59 +200,14 @@ fn main() { } } -/// Write a configuration to file. -fn write_to_file(data_dir: Option<&str>, config_filename: &str, config: &T) -> Result<(), String> -where - T: Default + serde::de::DeserializeOwned + serde::Serialize, -{ - let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); - - let path = dirs::home_dir() - .ok_or_else(|| "Unable to locate home directory")? - .join(&data_dir); - fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; - - if let Ok(mut file) = File::create(path.join(config_filename)) { - let toml_encoded = toml::to_string(&config).map_err(|e| { - format!( - "Failed to write configuration to {}. Error: {:?}", - config_filename, e - ) - })?; - file.write_all(toml_encoded.as_bytes()) - .expect(&format!("Unable to write to {}", config_filename)); - } - - Ok(()) -} - -/// Loads a `ClientConfig` from file. If unable to load from file, generates a default -/// configuration and saves that as a sample file. -fn read_from_file(data_dir: Option<&str>, config_filename: &str) -> Result, String> -where - T: Default + serde::de::DeserializeOwned + serde::Serialize, -{ - let data_dir = data_dir.unwrap_or_else(|| DEFAULT_DATA_DIR); - - let path = dirs::home_dir() - .ok_or_else(|| "Unable to locate home directory")? - .join(&data_dir); - fs::create_dir_all(&path).map_err(|_| "Unable to open data_dir")?; - - if let Ok(mut file) = File::open(path.join(config_filename)) { - let mut contents = String::new(); - file.read_to_string(&mut contents).map_err(|e| { - format!( - "Unable to read existing {}. Error: {:?}", - config_filename, e - ) - })?; - - let config = toml::from_str(&contents) - .map_err(|e| format!("Unable to parse {}: {:?}", config_filename, e))?; - - Ok(Some(config)) +fn get_data_dir(args: &ArgMatches) -> Result { + if let Some(data_dir) = args.value_of("data_dir") { + Ok(PathBuf::from(data_dir)) } else { - Ok(None) + let path = dirs::home_dir() + .ok_or_else(|| "Unable to locate home directory")? + .join(&DEFAULT_DATA_DIR); + fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; + Ok(path) } } diff --git a/eth2/utils/eth2_config/Cargo.toml b/eth2/utils/eth2_config/Cargo.toml new file mode 100644 index 0000000000..5af385e2d6 --- /dev/null +++ b/eth2/utils/eth2_config/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "eth2_config" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +clap = "2.32.0" +dirs = "1.0.3" +serde = "1.0" +serde_derive = "1.0" +toml = "^0.5" +types = { path = "../../types" } diff --git a/beacon_node/client/src/eth2_config.rs b/eth2/utils/eth2_config/src/lib.rs similarity index 54% rename from beacon_node/client/src/eth2_config.rs rename to eth2/utils/eth2_config/src/lib.rs index b16e8729e0..df4229629d 100644 --- a/beacon_node/client/src/eth2_config.rs +++ b/eth2/utils/eth2_config/src/lib.rs @@ -1,5 +1,8 @@ use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; +use std::fs::File; +use std::io::prelude::*; +use std::path::PathBuf; use std::time::SystemTime; use types::ChainSpec; @@ -42,7 +45,7 @@ impl Eth2Config { /// Returns an error if arguments are obviously invalid. May succeed even if some values are /// invalid. pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { - if args.is_present("recent_genesis") { + if args.is_present("recent-genesis") { self.spec.genesis_time = recent_genesis_time() } @@ -62,3 +65,42 @@ fn recent_genesis_time() -> u64 { // genesis is now the last 30 minute block. now - secs_after_last_period } + +/// Write a configuration to file. +pub fn write_to_file(path: PathBuf, config: &T) -> Result<(), String> +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ + if let Ok(mut file) = File::create(path.clone()) { + let toml_encoded = toml::to_string(&config).map_err(|e| { + format!( + "Failed to write configuration to {:?}. Error: {:?}", + path, e + ) + })?; + file.write_all(toml_encoded.as_bytes()) + .expect(&format!("Unable to write to {:?}", path)); + } + + Ok(()) +} + +/// Loads a `ClientConfig` from file. If unable to load from file, generates a default +/// configuration and saves that as a sample file. +pub fn read_from_file(path: PathBuf) -> Result, String> +where + T: Default + serde::de::DeserializeOwned + serde::Serialize, +{ + if let Ok(mut file) = File::open(path.clone()) { + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|e| format!("Unable to read {:?}. Error: {:?}", path, e))?; + + let config = toml::from_str(&contents) + .map_err(|e| format!("Unable to parse {:?}: {:?}", path, e))?; + + Ok(Some(config)) + } else { + Ok(None) + } +} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 559460c8bc..06ee02f538 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -15,6 +15,7 @@ path = "src/lib.rs" [dependencies] bls = { path = "../eth2/utils/bls" } ssz = { path = "../eth2/utils/ssz" } +eth2_config = { path = "../eth2/utils/eth2_config" } tree_hash = { path = "../eth2/utils/tree_hash" } clap = "2.32.0" dirs = "1.0.3" @@ -23,11 +24,14 @@ protobuf = "2.0.2" protos = { path = "../protos" } slot_clock = { path = "../eth2/utils/slot_clock" } types = { path = "../eth2/types" } +serde = "1.0" +serde_derive = "1.0" slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" tokio = "0.1.18" tokio-timer = "0.2.10" +toml = "^0.5" error-chain = "0.12.0" bincode = "^1.1.2" futures = "0.1.25" diff --git a/validator_client/eth2_config.toml b/validator_client/eth2_config.toml new file mode 100644 index 0000000000..49d4e1bd38 --- /dev/null +++ b/validator_client/eth2_config.toml @@ -0,0 +1,47 @@ +spec_constants = "minimal" + +[spec] +target_committee_size = 1 +max_indices_per_attestation = 4096 +min_per_epoch_churn_limit = 4 +churn_limit_quotient = 65536 +base_rewards_per_epoch = 5 +shuffle_round_count = 10 +deposit_contract_tree_depth = 32 +min_deposit_amount = 1000000000 +max_effective_balance = 32000000000 +ejection_balance = 16000000000 +effective_balance_increment = 1000000000 +genesis_slot = 0 +zero_hash = "0x0000000000000000000000000000000000000000000000000000000000000000" +bls_withdrawal_prefix_byte = "0x00" +genesis_time = 4294967295 +seconds_per_slot = 6 +min_attestation_inclusion_delay = 4 +min_seed_lookahead = 1 +activation_exit_delay = 4 +slots_per_eth1_voting_period = 1024 +slots_per_historical_root = 8192 +min_validator_withdrawability_delay = 256 +persistent_committee_period = 2048 +max_crosslink_epochs = 64 +min_epochs_to_inactivity_penalty = 4 +base_reward_quotient = 32 +whistleblowing_reward_quotient = 512 +proposer_reward_quotient = 8 +inactivity_penalty_quotient = 33554432 +min_slashing_penalty_quotient = 32 +max_proposer_slashings = 16 +max_attester_slashings = 1 +max_attestations = 128 +max_deposits = 16 +max_voluntary_exits = 16 +max_transfers = 0 +domain_beacon_proposer = 0 +domain_randao = 1 +domain_attestation = 2 +domain_deposit = 3 +domain_voluntary_exit = 4 +domain_transfer = 5 +boot_nodes = ["/ip4/127.0.0.1/tcp/9000"] +chain_id = 2 diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index b6e2c5bb55..46ceaaf80a 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -1,22 +1,22 @@ use bincode; use bls::Keypair; use clap::ArgMatches; +use serde_derive::{Deserialize, Serialize}; use slog::{debug, error, info}; use std::fs; use std::fs::File; use std::io::{Error, ErrorKind}; use std::path::PathBuf; -use types::{ChainSpec, EthSpec, MainnetEthSpec, MinimalEthSpec}; +use types::{EthSpec, MainnetEthSpec}; /// Stores the core configuration for this validator instance. -#[derive(Clone)] +#[derive(Clone, Serialize, Deserialize)] pub struct Config { /// The data directory, which stores all validator databases pub data_dir: PathBuf, /// The server at which the Beacon Node can be contacted pub server: String, - /// The chain specification that we are connecting to - pub spec: ChainSpec, + /// The number of slots per epoch. pub slots_per_epoch: u64, } @@ -25,25 +25,33 @@ const DEFAULT_PRIVATE_KEY_FILENAME: &str = "private.key"; impl Default for Config { /// Build a new configuration from defaults. fn default() -> Self { - let data_dir = { - let home = dirs::home_dir().expect("Unable to determine home directory."); - home.join(".lighthouse-validator") - }; - - let server = "localhost:5051".to_string(); - - let spec = MainnetEthSpec::default_spec(); - Self { - data_dir, - server, - spec, + data_dir: PathBuf::from(".lighthouse-validator"), + server: "localhost:5051".to_string(), slots_per_epoch: MainnetEthSpec::slots_per_epoch(), } } } impl Config { + /// Apply the following arguments to `self`, replacing values if they are specified in `args`. + /// + /// Returns an error if arguments are obviously invalid. May succeed even if some values are + /// invalid. + pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { + if let Some(datadir) = args.value_of("datadir") { + self.data_dir = PathBuf::from(datadir); + }; + + if let Some(srv) = args.value_of("server") { + self.server = srv.to_string(); + }; + + Ok(()) + // + } + + /* /// Build a new configuration from defaults, which are overrided by arguments provided. pub fn parse_args(args: &ArgMatches, log: &slog::Logger) -> Result { let mut config = Config::default(); @@ -80,12 +88,13 @@ impl Config { Ok(config) } + */ /// Try to load keys from validator_dir, returning None if none are found or an error. #[allow(dead_code)] pub fn fetch_keys(&self, log: &slog::Logger) -> Option> { let key_pairs: Vec = fs::read_dir(&self.data_dir) - .unwrap() + .ok()? .filter_map(|validator_dir| { let validator_dir = validator_dir.ok()?; diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index d755db4d24..44dede100a 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -7,11 +7,19 @@ mod service; mod signer; use crate::config::Config as ValidatorClientConfig; +use std::fs; use crate::service::Service as ValidatorService; -use clap::{App, Arg}; +use clap::{App, Arg, ArgMatches}; +use eth2_config::{read_from_file, write_to_file, Eth2Config}; use protos::services_grpc::ValidatorServiceClient; -use slog::{error, info, o, Drain}; -use types::Keypair; +use slog::{crit, error, info, o, Drain}; +use std::path::PathBuf; +use types::{Keypair, MainnetEthSpec, MinimalEthSpec}; + +pub const DEFAULT_SPEC: &str = "minimal"; +pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator"; +pub const CLIENT_CONFIG_FILENAME: &str = "client_config.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2_config.toml"; fn main() { // Logging @@ -30,7 +38,16 @@ fn main() { .long("datadir") .value_name("DIR") .help("Data directory for keys and databases.") - .takes_value(true), + .takes_value(true) + ) + .arg( + Arg::with_name("eth-config") + .long("eth-config") + .short("e") + .value_name("DIR") + .help(&format!("Directory containing {}.", ETH2_CONFIG_FILENAME)) + .takes_value(true) + .default_value(ETH2_CONFIG_FILENAME), ) .arg( Arg::with_name("server") @@ -40,24 +57,139 @@ fn main() { .takes_value(true), ) .arg( - Arg::with_name("spec") - .long("spec") - .value_name("spec") + Arg::with_name("spec-constants") + .long("spec-constants") + .value_name("TITLE") .short("s") - .help("Configuration of Beacon Chain") + .help("The title of the spec constants for chain config.") .takes_value(true) .possible_values(&["mainnet", "minimal"]) .default_value("minimal"), ) .get_matches(); - let config = ValidatorClientConfig::parse_args(&matches, &log) - .expect("Unable to build a configuration for the validator client."); + let data_dir = match get_data_dir(&matches) { + Ok(dir) => dir, + Err(e) => { + crit!(log, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + return + } + }; + + let client_config_path = data_dir.join(CLIENT_CONFIG_FILENAME); + + // Attempt to lead the `ClientConfig` from disk. + // + // If file doesn't exist, create a new, default one. + let mut client_config = match read_from_file::( + client_config_path.clone(), + ) { + Ok(Some(c)) => c, + Ok(None) => { + let default = ValidatorClientConfig::default(); + if let Err(e) = write_to_file(client_config_path.clone(), &default) { + crit!(log, "Failed to write default ClientConfig to file"; "error" => format!("{:?}", e)); + return; + } + default + } + Err(e) => { + crit!(log, "Failed to load a ChainConfig file"; "error" => format!("{:?}", e)); + return; + } + }; + + // Ensure the `data_dir` in the config matches that supplied to the CLI. + client_config.data_dir = data_dir.clone(); + + // Update the client config with any CLI args. + match client_config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); + return; + } + }; + + let eth2_config_path: PathBuf = matches + .value_of("eth-config") + .and_then(|s| Some(PathBuf::from(s))) + .unwrap_or_else(|| data_dir.join(ETH2_CONFIG_FILENAME)); + + // Attempt to load the `Eth2Config` from file. + // + // If the file doesn't exist, create a default one depending on the CLI flags. + let mut eth2_config = match read_from_file::( + eth2_config_path.clone() + ) { + Ok(Some(c)) => c, + Ok(None) => { + let default = match matches.value_of("spec-constants") { + Some("mainnet") => Eth2Config::mainnet(), + Some("minimal") => Eth2Config::minimal(), + _ => unreachable!(), // Guarded by slog. + }; + if let Err(e) = write_to_file(eth2_config_path, &default) { + crit!(log, "Failed to write default Eth2Config to file"; "error" => format!("{:?}", e)); + return; + } + default + } + Err(e) => { + crit!(log, "Failed to instantiate an Eth2Config"; "error" => format!("{:?}", e)); + return; + } + }; + + // Update the eth2 config with any CLI flags. + match eth2_config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(log, "Failed to parse Eth2Config CLI arguments"; "error" => s); + return; + } + }; + + info!( + log, + "Starting validator client"; + "datadir" => client_config.data_dir.to_str(), + "spec_constants" => ð2_config.spec_constants, + ); + + let result = match eth2_config.spec_constants.as_str() { + "mainnet" => ValidatorService::::start::( + client_config, + eth2_config, + log.clone(), + ), + "minimal" => ValidatorService::::start::( + client_config, + eth2_config, + log.clone(), + ), + other => { + crit!(log, "Unknown spec constants"; "title" => other); + return; + } + }; // start the validator service. // this specifies the GRPC and signer type to use as the duty manager beacon node. - match ValidatorService::::start(config, log.clone()) { + match result { Ok(_) => info!(log, "Validator client shutdown successfully."), - Err(e) => error!(log, "Validator exited due to: {}", e.to_string()), + Err(e) => crit!(log, "Validator client exited with error"; "error" => e.to_string()), + } +} + +fn get_data_dir(args: &ArgMatches) -> Result { + if let Some(data_dir) = args.value_of("data_dir") { + Ok(PathBuf::from(data_dir)) + } else { + let path = dirs::home_dir() + .ok_or_else(|| "Unable to locate home directory")? + .join(&DEFAULT_DATA_DIR); + fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; + Ok(path) } } diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 99455808c3..8dbb82b37f 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -16,6 +16,7 @@ use crate::error as error_chain; use crate::error::ErrorKind; use crate::signer::Signer; use bls::Keypair; +use eth2_config::Eth2Config; use grpcio::{ChannelBuilder, EnvBuilder}; use protos::services::Empty; use protos::services_grpc::{ @@ -31,7 +32,7 @@ use tokio::prelude::*; use tokio::runtime::Builder; use tokio::timer::Interval; use tokio_timer::clock::Clock; -use types::{ChainSpec, Epoch, Fork, Slot}; +use types::{ChainSpec, Epoch, EthSpec, Fork, Slot}; /// A fixed amount of time after a slot to perform operations. This gives the node time to complete /// per-slot processes. @@ -66,8 +67,9 @@ impl Service { /// /// This tries to connect to a beacon node. Once connected, it initialised the gRPC clients /// and returns an instance of the service. - fn initialize_service( - config: ValidatorConfig, + fn initialize_service( + client_config: ValidatorConfig, + eth2_config: Eth2Config, log: slog::Logger, ) -> error_chain::Result> { // initialise the beacon node client to check for a connection @@ -75,7 +77,7 @@ impl Service { let env = Arc::new(EnvBuilder::new().build()); // Beacon node gRPC beacon node endpoints. let beacon_node_client = { - let ch = ChannelBuilder::new(env.clone()).connect(&config.server); + let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server); BeaconNodeServiceClient::new(ch) }; @@ -103,12 +105,12 @@ impl Service { return Err("Genesis time in the future".into()); } // verify the node's chain id - if config.spec.chain_id != info.chain_id as u8 { + if eth2_config.spec.chain_id != info.chain_id as u8 { error!( log, "Beacon Node's genesis time is in the future. No work to do.\n Exiting" ); - return Err(format!("Beacon node has the wrong chain id. Expected chain id: {}, node's chain id: {}", config.spec.chain_id, info.chain_id).into()); + return Err(format!("Beacon node has the wrong chain id. Expected chain id: {}, node's chain id: {}", eth2_config.spec.chain_id, info.chain_id).into()); } break info; } @@ -136,7 +138,7 @@ impl Service { // Beacon node gRPC beacon block endpoints. let beacon_block_client = { - let ch = ChannelBuilder::new(env.clone()).connect(&config.server); + let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server); let beacon_block_service_client = Arc::new(BeaconBlockServiceClient::new(ch)); // a wrapper around the service client to implement the beacon block node trait Arc::new(BeaconBlockGrpcClient::new(beacon_block_service_client)) @@ -144,33 +146,42 @@ impl Service { // Beacon node gRPC validator endpoints. let validator_client = { - let ch = ChannelBuilder::new(env.clone()).connect(&config.server); + let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server); Arc::new(ValidatorServiceClient::new(ch)) }; //Beacon node gRPC attester endpoints. let attestation_client = { - let ch = ChannelBuilder::new(env.clone()).connect(&config.server); + let ch = ChannelBuilder::new(env.clone()).connect(&client_config.server); Arc::new(AttestationServiceClient::new(ch)) }; // build the validator slot clock - let slot_clock = - SystemTimeSlotClock::new(genesis_slot, genesis_time, config.spec.seconds_per_slot); + let slot_clock = SystemTimeSlotClock::new( + genesis_slot, + genesis_time, + eth2_config.spec.seconds_per_slot, + ); let current_slot = slot_clock .present_slot() .map_err(ErrorKind::SlotClockError)? - .expect("Genesis must be in the future"); + .ok_or_else::(|| { + "Genesis is not in the past. Exiting.".into() + })?; /* Generate the duties manager */ // Load generated keypairs - let keypairs = match config.fetch_keys(&log) { + let keypairs = match client_config.fetch_keys(&log) { Some(kps) => Arc::new(kps), - None => panic!("No key pairs found, cannot start validator client without at least one. Try running `./account_manager generate` first.") + None => { + return Err("Unable to locate validator key pairs, nothing to do.".into()); + } }; + let slots_per_epoch = T::slots_per_epoch(); + // TODO: keypairs are randomly generated; they should be loaded from a file or generated. // https://github.com/sigp/lighthouse/issues/160 //let keypairs = Arc::new(generate_deterministic_keypairs(8)); @@ -178,7 +189,7 @@ impl Service { // Builds a mapping of Epoch -> Map(PublicKey, EpochDuty) // where EpochDuty contains slot numbers and attestation data that each validator needs to // produce work on. - let duties_map = RwLock::new(EpochDutiesMap::new(config.slots_per_epoch)); + let duties_map = RwLock::new(EpochDutiesMap::new(slots_per_epoch)); // builds a manager which maintains the list of current duties for all known validators // and can check when a validator needs to perform a task. @@ -189,13 +200,13 @@ impl Service { beacon_node: validator_client, }); - let spec = Arc::new(config.spec); + let spec = Arc::new(eth2_config.spec); Ok(Service { fork, slot_clock, current_slot, - slots_per_epoch: config.slots_per_epoch, + slots_per_epoch, spec, duties_manager, beacon_block_client, @@ -206,13 +217,17 @@ impl Service { /// Initialise the service then run the core thread. // TODO: Improve handling of generic BeaconNode types, to stub grpcClient - pub fn start( - config: ValidatorConfig, + pub fn start( + client_config: ValidatorConfig, + eth2_config: Eth2Config, log: slog::Logger, ) -> error_chain::Result<()> { // connect to the node and retrieve its properties and initialize the gRPC clients - let mut service = - Service::::initialize_service(config, log)?; + let mut service = Service::::initialize_service::( + client_config, + eth2_config, + log, + )?; // we have connected to a node and established its parameters. Spin up the core service @@ -227,7 +242,9 @@ impl Service { .slot_clock .duration_to_next_slot() .map_err(|e| format!("System clock error: {:?}", e))? - .expect("Cannot start before genesis"); + .ok_or_else::(|| { + "Genesis is not in the past. Exiting.".into() + })?; // set up the validator work interval - start at next slot and proceed every slot let interval = { @@ -276,7 +293,9 @@ impl Service { error!(self.log, "SystemTimeError {:?}", e); return Err("Could not read system time".into()); } - Ok(slot) => slot.expect("Genesis is in the future"), + Ok(slot) => slot.ok_or_else::(|| { + "Genesis is not in the past. Exiting.".into() + })?, }; let current_epoch = current_slot.epoch(self.slots_per_epoch); From ab12787610792c33d49fb7602cc0b6b5924725f4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 9 Jun 2019 04:34:56 -0400 Subject: [PATCH 69/76] Update account manager config parsing --- account_manager/Cargo.toml | 1 + account_manager/src/main.rs | 45 +++++++++++++++---- beacon_node/Cargo.toml | 1 - beacon_node/src/main.rs | 14 +----- .../block_processing_builder.rs | 2 +- .../validator_statuses.rs | 5 +-- eth2/types/src/beacon_state.rs | 6 +-- eth2/types/src/fork.rs | 3 +- eth2/utils/eth2_config/src/lib.rs | 13 ++++++ validator_client/Cargo.toml | 1 - validator_client/src/config.rs | 40 ----------------- validator_client/src/main.rs | 31 ++++--------- 12 files changed, 66 insertions(+), 96 deletions(-) diff --git a/account_manager/Cargo.toml b/account_manager/Cargo.toml index 7b561869a7..48504d89ad 100644 --- a/account_manager/Cargo.toml +++ b/account_manager/Cargo.toml @@ -12,3 +12,4 @@ slog-term = "^2.4.0" slog-async = "^2.3.0" validator_client = { path = "../validator_client" } types = { path = "../eth2/types" } +eth2_config = { path = "../eth2/utils/eth2_config" } diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index aa691d31af..d820321013 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -1,9 +1,13 @@ use bls::Keypair; use clap::{App, Arg, SubCommand}; -use slog::{debug, info, o, Drain}; +use slog::{crit, debug, info, o, Drain}; use std::path::PathBuf; use types::test_utils::generate_deterministic_keypair; use validator_client::Config as ValidatorClientConfig; +use eth2_config::{get_data_dir}; + +pub const DEFAULT_DATA_DIR: &str = ".lighthouse-account-manager"; +pub const CLIENT_CONFIG_FILENAME: &str = "account-manager-config.toml"; fn main() { // Logging @@ -20,6 +24,7 @@ fn main() { .arg( Arg::with_name("datadir") .long("datadir") + .short("d") .value_name("DIR") .help("Data directory for keys and databases.") .takes_value(true), @@ -52,19 +57,43 @@ fn main() { .help("If supplied along with `index`, generates keys `i..i + n`.") .takes_value(true) .default_value("1"), - ), + ) ) .get_matches(); - let config = ValidatorClientConfig::parse_args(&matches, &log) - .expect("Unable to build a configuration for the account manager."); + let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { + Ok(dir) => dir, + Err(e) => { + crit!(log, "Failed to initialize data dir"; "error" => format!("{:?}", e)); + return + } + }; + + let mut client_config = ValidatorClientConfig::default(); + + if let Err(e) = client_config.apply_cli_args(&matches) { + crit!(log, "Failed to apply CLI args"; "error" => format!("{:?}", e)); + return + }; + + // Ensure the `data_dir` in the config matches that supplied to the CLI. + client_config.data_dir = data_dir.clone(); + + // Update the client config with any CLI args. + match client_config.apply_cli_args(&matches) { + Ok(()) => (), + Err(s) => { + crit!(log, "Failed to parse ClientConfig CLI arguments"; "error" => s); + return; + } + }; // Log configuration info!(log, ""; - "data_dir" => &config.data_dir.to_str()); + "data_dir" => &client_config.data_dir.to_str()); match matches.subcommand() { - ("generate", Some(_)) => generate_random(&config, &log), + ("generate", Some(_)) => generate_random(&client_config, &log), ("generate_deterministic", Some(m)) => { if let Some(string) = m.value_of("validator index") { let i: usize = string.parse().expect("Invalid validator index"); @@ -72,9 +101,9 @@ fn main() { let n: usize = string.parse().expect("Invalid end validator count"); let indices: Vec = (i..i + n).collect(); - generate_deterministic_multiple(&indices, &config, &log) + generate_deterministic_multiple(&indices, &client_config, &log) } else { - generate_deterministic(i, &config, &log) + generate_deterministic(i, &client_config, &log) } } } diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 3edbf636d7..309f162e55 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -5,7 +5,6 @@ authors = ["Paul Hauner ", "Age Manning crit!(logger, "Beacon node failed to start"; "reason" => format!("{:}", e)), } } - -fn get_data_dir(args: &ArgMatches) -> Result { - if let Some(data_dir) = args.value_of("data_dir") { - Ok(PathBuf::from(data_dir)) - } else { - let path = dirs::home_dir() - .ok_or_else(|| "Unable to locate home directory")? - .join(&DEFAULT_DATA_DIR); - fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; - Ok(path) - } -} diff --git a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs index 3675ef99e5..05a5a2de24 100644 --- a/eth2/state_processing/src/per_block_processing/block_processing_builder.rs +++ b/eth2/state_processing/src/per_block_processing/block_processing_builder.rs @@ -22,7 +22,7 @@ impl BlockProcessingBuilder { } } - pub fn set_slot(&mut self, slot: Slot, spec: &ChainSpec) { + pub fn set_slot(&mut self, slot: Slot) { self.state_builder.teleport_to_slot(slot); } diff --git a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs index 76a485ea7b..45ef3419bc 100644 --- a/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs +++ b/eth2/state_processing/src/per_epoch_processing/validator_statuses.rs @@ -223,7 +223,7 @@ impl ValidatorStatuses { if is_from_epoch(a, state.current_epoch()) { status.is_current_epoch_attester = true; - if target_matches_epoch_start_block(a, state, state.current_epoch(), spec)? { + if target_matches_epoch_start_block(a, state, state.current_epoch())? { status.is_current_epoch_target_attester = true; } } else if is_from_epoch(a, state.previous_epoch()) { @@ -244,7 +244,7 @@ impl ValidatorStatuses { )?, }); - if target_matches_epoch_start_block(a, state, state.previous_epoch(), spec)? { + if target_matches_epoch_start_block(a, state, state.previous_epoch())? { status.is_previous_epoch_target_attester = true; } @@ -336,7 +336,6 @@ fn target_matches_epoch_start_block( a: &PendingAttestation, state: &BeaconState, epoch: Epoch, - spec: &ChainSpec, ) -> Result { let slot = epoch.start_slot(T::slots_per_epoch()); let state_boundary_root = *state.get_block_root(slot)?; diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 1c1fca6fb3..850cfcaa66 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -453,11 +453,7 @@ impl BeaconState { /// /// Spec v0.6.0 // FIXME(sproul): name swap with get_block_root - pub fn get_block_root_at_epoch( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result<&Hash256, BeaconStateError> { + pub fn get_block_root_at_epoch(&self, epoch: Epoch) -> Result<&Hash256, BeaconStateError> { self.get_block_root(epoch.start_slot(T::slots_per_epoch())) } diff --git a/eth2/types/src/fork.rs b/eth2/types/src/fork.rs index 60ab208ad2..41a233a434 100644 --- a/eth2/types/src/fork.rs +++ b/eth2/types/src/fork.rs @@ -1,6 +1,6 @@ use crate::{ test_utils::{fork_from_hex_str, TestRandom}, - ChainSpec, Epoch, + Epoch, }; use serde_derive::{Deserialize, Serialize}; @@ -58,6 +58,7 @@ impl Fork { #[cfg(test)] mod tests { use super::*; + use crate::ChainSpec; ssz_tests!(Fork); cached_tree_hash_tests!(Fork); diff --git a/eth2/utils/eth2_config/src/lib.rs b/eth2/utils/eth2_config/src/lib.rs index df4229629d..c4d4736bdb 100644 --- a/eth2/utils/eth2_config/src/lib.rs +++ b/eth2/utils/eth2_config/src/lib.rs @@ -1,5 +1,6 @@ use clap::ArgMatches; use serde_derive::{Deserialize, Serialize}; +use std::fs; use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; @@ -104,3 +105,15 @@ where Ok(None) } } + +pub fn get_data_dir(args: &ArgMatches, default_data_dir: PathBuf) -> Result { + if let Some(data_dir) = args.value_of("data_dir") { + Ok(PathBuf::from(data_dir)) + } else { + let path = dirs::home_dir() + .ok_or_else(|| "Unable to locate home directory")? + .join(&default_data_dir); + fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; + Ok(path) + } +} diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 06ee02f538..1784bdcb1e 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -18,7 +18,6 @@ ssz = { path = "../eth2/utils/ssz" } eth2_config = { path = "../eth2/utils/eth2_config" } tree_hash = { path = "../eth2/utils/tree_hash" } clap = "2.32.0" -dirs = "1.0.3" grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } protobuf = "2.0.2" protos = { path = "../protos" } diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 46ceaaf80a..d7664c1613 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -48,48 +48,8 @@ impl Config { }; Ok(()) - // } - /* - /// Build a new configuration from defaults, which are overrided by arguments provided. - pub fn parse_args(args: &ArgMatches, log: &slog::Logger) -> Result { - let mut config = Config::default(); - - // Use the specified datadir, or default in the home directory - if let Some(datadir) = args.value_of("datadir") { - config.data_dir = PathBuf::from(datadir); - info!(log, "Using custom data dir: {:?}", &config.data_dir); - }; - - fs::create_dir_all(&config.data_dir) - .unwrap_or_else(|_| panic!("Unable to create {:?}", &config.data_dir)); - - if let Some(srv) = args.value_of("server") { - //TODO: Validate the server value, to ensure it makes sense. - config.server = srv.to_string(); - info!(log, "Using custom server: {:?}", &config.server); - }; - - // TODO: Permit loading a custom spec from file. - if let Some(spec_str) = args.value_of("spec") { - info!(log, "Using custom spec: {:?}", spec_str); - config.spec = match spec_str { - "mainnet" => MainnetEthSpec::default_spec(), - "minimal" => MinimalEthSpec::default_spec(), - // Should be impossible due to clap's `possible_values(..)` function. - _ => unreachable!(), - }; - }; - // Log configuration - info!(log, ""; - "data_dir" => &config.data_dir.to_str(), - "server" => &config.server); - - Ok(config) - } - */ - /// Try to load keys from validator_dir, returning None if none are found or an error. #[allow(dead_code)] pub fn fetch_keys(&self, log: &slog::Logger) -> Option> { diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 44dede100a..8a07894d5b 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -7,10 +7,9 @@ mod service; mod signer; use crate::config::Config as ValidatorClientConfig; -use std::fs; use crate::service::Service as ValidatorService; -use clap::{App, Arg, ArgMatches}; -use eth2_config::{read_from_file, write_to_file, Eth2Config}; +use clap::{App, Arg}; +use eth2_config::{get_data_dir, read_from_file, write_to_file, Eth2Config}; use protos::services_grpc::ValidatorServiceClient; use slog::{crit, error, info, o, Drain}; use std::path::PathBuf; @@ -18,8 +17,8 @@ use types::{Keypair, MainnetEthSpec, MinimalEthSpec}; pub const DEFAULT_SPEC: &str = "minimal"; pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator"; -pub const CLIENT_CONFIG_FILENAME: &str = "client_config.toml"; -pub const ETH2_CONFIG_FILENAME: &str = "eth2_config.toml"; +pub const CLIENT_CONFIG_FILENAME: &str = "client-config.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2-config.toml"; fn main() { // Logging @@ -38,7 +37,7 @@ fn main() { .long("datadir") .value_name("DIR") .help("Data directory for keys and databases.") - .takes_value(true) + .takes_value(true), ) .arg( Arg::with_name("eth-config") @@ -68,11 +67,11 @@ fn main() { ) .get_matches(); - let data_dir = match get_data_dir(&matches) { + let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { Ok(dir) => dir, Err(e) => { crit!(log, "Failed to initialize data dir"; "error" => format!("{:?}", e)); - return + return; } }; @@ -119,9 +118,7 @@ fn main() { // Attempt to load the `Eth2Config` from file. // // If the file doesn't exist, create a default one depending on the CLI flags. - let mut eth2_config = match read_from_file::( - eth2_config_path.clone() - ) { + let mut eth2_config = match read_from_file::(eth2_config_path.clone()) { Ok(Some(c)) => c, Ok(None) => { let default = match matches.value_of("spec-constants") { @@ -181,15 +178,3 @@ fn main() { Err(e) => crit!(log, "Validator client exited with error"; "error" => e.to_string()), } } - -fn get_data_dir(args: &ArgMatches) -> Result { - if let Some(data_dir) = args.value_of("data_dir") { - Ok(PathBuf::from(data_dir)) - } else { - let path = dirs::home_dir() - .ok_or_else(|| "Unable to locate home directory")? - .join(&DEFAULT_DATA_DIR); - fs::create_dir_all(&path).map_err(|_| "Unable to create data_dir")?; - Ok(path) - } -} From a662c3a940c3f704b47232440f7ed9c8a7e7ac66 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 9 Jun 2019 06:26:34 -0400 Subject: [PATCH 70/76] Fix various compile errors and warnings --- beacon_node/src/main.rs | 5 ++--- eth2/operation_pool/src/lib.rs | 8 ++++---- .../src/per_block_processing/tests.rs | 2 +- eth2/state_processing/src/per_epoch_processing.rs | 15 +++++++-------- .../src/per_epoch_processing/tests.rs | 2 +- .../src/beacon_state/committee_cache/tests.rs | 2 +- eth2/types/src/beacon_state/tests.rs | 2 +- 7 files changed, 17 insertions(+), 19 deletions(-) diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 7827f7d26f..ef70dd300e 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -2,11 +2,10 @@ extern crate slog; mod run; -use clap::{App, Arg, ArgMatches}; +use clap::{App, Arg}; use client::{ClientConfig, Eth2Config}; use eth2_config::{get_data_dir, read_from_file, write_to_file}; use slog::{crit, o, Drain}; -use std::fs; use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; @@ -119,7 +118,7 @@ fn main() { ) .get_matches(); - let data_dir = match get_data_dir(&matches) { + let data_dir = match get_data_dir(&matches, PathBuf::from(DEFAULT_DATA_DIR)) { Ok(dir) => dir, Err(e) => { crit!(logger, "Failed to initialize data dir"; "error" => format!("{:?}", e)); diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index 0afd72918c..ec7d5aa905 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -724,14 +724,14 @@ mod tests { let spec = E::default_spec(); let num_validators = - num_committees * T::slots_per_epoch() as usize * spec.target_committee_size; + num_committees * E::slots_per_epoch() as usize * spec.target_committee_size; let mut state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists( num_validators, &spec, ); - let slot_offset = 1000 * T::slots_per_epoch() + T::slots_per_epoch() / 2; + let slot_offset = 1000 * E::slots_per_epoch() + E::slots_per_epoch() / 2; let slot = spec.genesis_slot + slot_offset; - state_builder.teleport_to_slot(slot, &spec); + state_builder.teleport_to_slot(slot); state_builder.build_caches(&spec).unwrap(); let (state, keypairs) = state_builder.build(); @@ -852,7 +852,7 @@ mod tests { // But once we advance to more than an epoch after the attestation, it should prune it // out of existence. - state.slot += 2 * T::slots_per_epoch(); + state.slot += 2 * MainnetEthSpec::slots_per_epoch(); op_pool.prune_attestations(state); assert_eq!(op_pool.num_attestations(), 0); } diff --git a/eth2/state_processing/src/per_block_processing/tests.rs b/eth2/state_processing/src/per_block_processing/tests.rs index 525637fea5..6c9593c496 100644 --- a/eth2/state_processing/src/per_block_processing/tests.rs +++ b/eth2/state_processing/src/per_block_processing/tests.rs @@ -106,7 +106,7 @@ fn get_builder(spec: &ChainSpec) -> (BlockProcessingBuilder) { // Set the state and block to be in the last slot of the 4th epoch. let last_slot_of_epoch = (MainnetEthSpec::genesis_epoch() + 4).end_slot(MainnetEthSpec::slots_per_epoch()); - builder.set_slot(last_slot_of_epoch, &spec); + builder.set_slot(last_slot_of_epoch); builder.build_caches(&spec); (builder) diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 2d76577e1c..f7a4a1c50c 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -42,7 +42,7 @@ pub fn per_epoch_processing( validator_statuses.process_attestations(&state, spec)?; // Justification and finalization. - process_justification_and_finalization(state, &validator_statuses.total_balances, spec)?; + process_justification_and_finalization(state, &validator_statuses.total_balances)?; // Crosslinks. let winning_root_for_shards = process_crosslinks(state, spec)?; @@ -84,7 +84,6 @@ pub fn per_epoch_processing( pub fn process_justification_and_finalization( state: &mut BeaconState, total_balances: &TotalBalances, - spec: &ChainSpec, ) -> Result<(), Error> { if state.current_epoch() == T::genesis_epoch() { return Ok(()); @@ -104,14 +103,14 @@ pub fn process_justification_and_finalization( if total_balances.previous_epoch_target_attesters * 3 >= total_balances.previous_epoch * 2 { state.current_justified_epoch = previous_epoch; state.current_justified_root = - *state.get_block_root_at_epoch(state.current_justified_epoch, spec)?; + *state.get_block_root_at_epoch(state.current_justified_epoch)?; state.justification_bitfield |= 2; } // If the current epoch gets justified, fill the last bit. if total_balances.current_epoch_target_attesters * 3 >= total_balances.current_epoch * 2 { state.current_justified_epoch = current_epoch; state.current_justified_root = - *state.get_block_root_at_epoch(state.current_justified_epoch, spec)?; + *state.get_block_root_at_epoch(state.current_justified_epoch)?; state.justification_bitfield |= 1; } @@ -120,22 +119,22 @@ pub fn process_justification_and_finalization( // The 2nd/3rd/4th most recent epochs are all justified, the 2nd using the 4th as source. if (bitfield >> 1) % 8 == 0b111 && old_previous_justified_epoch == current_epoch - 3 { state.finalized_epoch = old_previous_justified_epoch; - state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch, spec)?; + state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch)?; } // The 2nd/3rd most recent epochs are both justified, the 2nd using the 3rd as source. if (bitfield >> 1) % 4 == 0b11 && state.previous_justified_epoch == current_epoch - 2 { state.finalized_epoch = old_previous_justified_epoch; - state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch, spec)?; + state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch)?; } // The 1st/2nd/3rd most recent epochs are all justified, the 1st using the 2nd as source. if bitfield % 8 == 0b111 && state.current_justified_epoch == current_epoch - 2 { state.finalized_epoch = old_current_justified_epoch; - state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch, spec)?; + state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch)?; } // The 1st/2nd most recent epochs are both justified, the 1st using the 2nd as source. if bitfield % 4 == 0b11 && state.current_justified_epoch == current_epoch - 1 { state.finalized_epoch = old_current_justified_epoch; - state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch, spec)?; + state.finalized_root = *state.get_block_root_at_epoch(state.finalized_epoch)?; } Ok(()) diff --git a/eth2/state_processing/src/per_epoch_processing/tests.rs b/eth2/state_processing/src/per_epoch_processing/tests.rs index 306acb7ee2..9fdc82c6f3 100644 --- a/eth2/state_processing/src/per_epoch_processing/tests.rs +++ b/eth2/state_processing/src/per_epoch_processing/tests.rs @@ -15,7 +15,7 @@ fn runs_without_error() { let target_slot = (MinimalEthSpec::genesis_epoch() + 4).end_slot(MinimalEthSpec::slots_per_epoch()); - builder.teleport_to_slot(target_slot, &spec); + builder.teleport_to_slot(target_slot); let (mut state, _keypairs) = builder.build(); diff --git a/eth2/types/src/beacon_state/committee_cache/tests.rs b/eth2/types/src/beacon_state/committee_cache/tests.rs index d30d0724d6..ca58f8926a 100644 --- a/eth2/types/src/beacon_state/committee_cache/tests.rs +++ b/eth2/types/src/beacon_state/committee_cache/tests.rs @@ -25,7 +25,7 @@ fn new_state(validator_count: usize, slot: Slot) -> BeaconState { let mut builder = TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), spec); - builder.teleport_to_slot(slot, spec); + builder.teleport_to_slot(slot); let (state, _keypairs) = builder.build(); diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index b3c641f614..316a901512 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -300,7 +300,7 @@ mod committees { ); let slot = state_epoch.start_slot(T::slots_per_epoch()); - builder.teleport_to_slot(slot, spec); + builder.teleport_to_slot(slot); let (mut state, _keypairs): (BeaconState, _) = builder.build(); From 4a871dbcc77cdbd18ffb1ecb63bcba6268107e20 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 9 Jun 2019 06:35:36 -0400 Subject: [PATCH 71/76] Change default config filenames --- beacon_node/src/main.rs | 4 ++-- validator_client/src/main.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index ef70dd300e..dbbfbbea4c 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -10,8 +10,8 @@ use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; -pub const CLIENT_CONFIG_FILENAME: &str = "client_config.toml"; -pub const ETH2_CONFIG_FILENAME: &str = "eth2_config.toml"; +pub const CLIENT_CONFIG_FILENAME: &str = "becaon-node.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn main() { let decorator = slog_term::TermDecorator::new().build(); diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 8a07894d5b..6a310cddbe 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -17,8 +17,8 @@ use types::{Keypair, MainnetEthSpec, MinimalEthSpec}; pub const DEFAULT_SPEC: &str = "minimal"; pub const DEFAULT_DATA_DIR: &str = ".lighthouse-validator"; -pub const CLIENT_CONFIG_FILENAME: &str = "client-config.toml"; -pub const ETH2_CONFIG_FILENAME: &str = "eth2-config.toml"; +pub const CLIENT_CONFIG_FILENAME: &str = "validator-client.toml"; +pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn main() { // Logging From a9284bec188911766ac9fcf4a1db3ef9223fec15 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 9 Jun 2019 06:41:51 -0400 Subject: [PATCH 72/76] Fix type in default confi filename --- beacon_node/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index dbbfbbea4c..d6274befc8 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -10,7 +10,7 @@ use std::path::PathBuf; pub const DEFAULT_DATA_DIR: &str = ".lighthouse"; -pub const CLIENT_CONFIG_FILENAME: &str = "becaon-node.toml"; +pub const CLIENT_CONFIG_FILENAME: &str = "beacon-node.toml"; pub const ETH2_CONFIG_FILENAME: &str = "eth2-spec.toml"; fn main() { From e550c0218fc774a338f794d5628f6ccb31dfdb85 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jun 2019 11:01:25 -0400 Subject: [PATCH 73/76] Add various fixes to clippy lints Thou shalt appease clippy --- beacon_node/beacon_chain/src/beacon_chain.rs | 4 +--- beacon_node/beacon_chain/src/iter.rs | 4 ++-- beacon_node/eth2-libp2p/src/config.rs | 4 ++-- beacon_node/eth2-libp2p/src/rpc/protocol.rs | 6 +++--- beacon_node/network/src/message_handler.rs | 4 ++-- eth2/fork_choice/src/test_utils.rs | 2 +- .../verify_indexed_attestation.rs | 4 ++-- eth2/state_processing/src/per_epoch_processing.rs | 2 +- eth2/types/src/beacon_state.rs | 2 +- eth2/types/src/beacon_state/committee_cache.rs | 1 - eth2/types/src/chain_spec.rs | 2 +- eth2/types/src/slot_epoch.rs | 2 +- .../builders/testing_beacon_block_builder.rs | 2 +- .../builders/testing_proposer_slashing_builder.rs | 2 +- eth2/types/src/test_utils/serde_utils.rs | 1 + eth2/utils/eth2_config/src/lib.rs | 2 +- tests/ef_tests/src/case_result.rs | 11 ++++------- tests/ef_tests/src/cases.rs | 2 +- tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs | 4 ++-- tests/ef_tests/src/cases/bls_aggregate_sigs.rs | 4 ++-- tests/ef_tests/src/cases/bls_g2_compressed.rs | 8 ++++---- tests/ef_tests/src/cases/bls_g2_uncompressed.rs | 8 ++++---- tests/ef_tests/src/cases/bls_priv_to_pub.rs | 4 ++-- tests/ef_tests/src/cases/bls_sign_msg.rs | 8 ++++---- .../ef_tests/src/cases/epoch_processing_crosslinks.rs | 4 ++-- .../src/cases/epoch_processing_registry_updates.rs | 4 ++-- .../src/cases/operations_attester_slashing.rs | 4 ++-- tests/ef_tests/src/cases/operations_deposit.rs | 4 ++-- tests/ef_tests/src/cases/operations_exit.rs | 4 ++-- .../src/cases/operations_proposer_slashing.rs | 4 ++-- tests/ef_tests/src/cases/operations_transfer.rs | 4 ++-- tests/ef_tests/src/cases/shuffling.rs | 5 ++--- tests/ef_tests/src/cases/ssz_generic.rs | 10 +++------- tests/ef_tests/src/cases/ssz_static.rs | 2 +- tests/ef_tests/src/doc.rs | 6 +++--- tests/ef_tests/src/yaml_decode.rs | 8 ++++---- tests/ef_tests/src/yaml_decode/utils.rs | 2 +- 37 files changed, 72 insertions(+), 82 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 4d77d7aa5c..945c376175 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -216,9 +216,7 @@ impl BeaconChain { .iter() .map(|root| match self.get_block(root)? { Some(block) => Ok(block.body), - None => Err(Error::DBInconsistent( - format!("Missing block: {}", root).into(), - )), + None => Err(Error::DBInconsistent(format!("Missing block: {}", root))), }) .collect(); diff --git a/beacon_node/beacon_chain/src/iter.rs b/beacon_node/beacon_chain/src/iter.rs index b54dd1d2aa..1b5e382b02 100644 --- a/beacon_node/beacon_chain/src/iter.rs +++ b/beacon_node/beacon_chain/src/iter.rs @@ -57,7 +57,7 @@ impl Iterator for BlockRootsIterator { return None; } - self.slot = self.slot - 1; + self.slot -= 1; match self.beacon_state.get_block_root(self.slot) { Ok(root) => Some(*root), @@ -73,7 +73,7 @@ impl Iterator for BlockRootsIterator { self.beacon_state.get_block_root(self.slot).ok().cloned() } - _ => return None, + _ => None, } } } diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 2a27018838..ee2add75eb 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -57,12 +57,12 @@ impl Config { pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { if let Some(listen_address_str) = args.value_of("listen-address") { - let listen_addresses = listen_address_str.split(",").map(Into::into).collect(); + let listen_addresses = listen_address_str.split(',').map(Into::into).collect(); self.listen_addresses = listen_addresses; } if let Some(boot_addresses_str) = args.value_of("boot-nodes") { - let boot_addresses = boot_addresses_str.split(",").map(Into::into).collect(); + let boot_addresses = boot_addresses_str.split(',').map(Into::into).collect(); self.boot_nodes = boot_addresses; } diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index d75e53e6ce..de52f964e5 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -42,7 +42,7 @@ impl RequestId { } /// Return the previous id. - pub fn previous(&self) -> Self { + pub fn previous(self) -> Self { Self(self.0 - 1) } } @@ -220,7 +220,7 @@ impl Encode for RPCEvent { } => SszContainer { is_request: true, id: (*id).into(), - other: (*method_id).into(), + other: *method_id, bytes: match body { RPCRequest::Hello(body) => body.as_ssz_bytes(), RPCRequest::Goodbye(body) => body.as_ssz_bytes(), @@ -237,7 +237,7 @@ impl Encode for RPCEvent { } => SszContainer { is_request: false, id: (*id).into(), - other: (*method_id).into(), + other: *method_id, bytes: match result { RPCResponse::Hello(response) => response.as_ssz_bytes(), RPCResponse::BeaconBlockRoots(response) => response.as_ssz_bytes(), diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index adafae145d..40a396c3b0 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -155,7 +155,7 @@ impl MessageHandler { if self .network_context .outstanding_outgoing_request_ids - .remove(&(peer_id.clone(), id.clone())) + .remove(&(peer_id.clone(), id)) .is_none() { warn!( @@ -250,7 +250,7 @@ impl NetworkContext { let id = self.generate_request_id(&peer_id); self.outstanding_outgoing_request_ids - .insert((peer_id.clone(), id.clone()), Instant::now()); + .insert((peer_id.clone(), id), Instant::now()); self.send_rpc_event( peer_id, diff --git a/eth2/fork_choice/src/test_utils.rs b/eth2/fork_choice/src/test_utils.rs index 840d7a76b5..8ef20108aa 100644 --- a/eth2/fork_choice/src/test_utils.rs +++ b/eth2/fork_choice/src/test_utils.rs @@ -57,7 +57,7 @@ fn get_chain_of_blocks( ) -> Vec<(Hash256, BeaconBlock)> { let spec = T::default_spec(); let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![]; - let mut unique_hashes = (0..).into_iter().map(|i| Hash256::from(i)); + let mut unique_hashes = (0..).map(Hash256::from); let mut random_block = BeaconBlock::random_for_test(&mut XorShiftRng::from_seed([42; 16])); random_block.previous_block_root = Hash256::zero(); let beacon_state = get_state::(validator_count); diff --git a/eth2/state_processing/src/per_block_processing/verify_indexed_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_indexed_attestation.rs index 6581e516de..f06f1e9000 100644 --- a/eth2/state_processing/src/per_block_processing/verify_indexed_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_indexed_attestation.rs @@ -49,7 +49,7 @@ fn verify_indexed_attestation_parametric( ); // Check that nobody signed with custody bit 1 (to be removed in phase 1) - if custody_bit_1_indices.len() > 0 { + if !custody_bit_1_indices.is_empty() { invalid!(Invalid::CustodyBitfieldHasSetBits); } @@ -96,7 +96,7 @@ where state .validator_registry .get(validator_idx as usize) - .ok_or(Error::Invalid(Invalid::UnknownValidator(validator_idx))) + .ok_or_else(|| Error::Invalid(Invalid::UnknownValidator(validator_idx))) .map(|validator| { aggregate_pubkey.add(&validator.pubkey); aggregate_pubkey diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index f7a4a1c50c..5110207a3c 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -156,7 +156,7 @@ pub fn process_crosslinks( state.previous_crosslinks = state.current_crosslinks.clone(); - for relative_epoch in vec![RelativeEpoch::Previous, RelativeEpoch::Current] { + for &relative_epoch in &[RelativeEpoch::Previous, RelativeEpoch::Current] { let epoch = relative_epoch.into_epoch(state.current_epoch()); for offset in 0..state.get_epoch_committee_count(relative_epoch)? { let shard = diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index 850cfcaa66..1fbab8580b 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -422,7 +422,7 @@ impl BeaconState { }; let effective_balance = self.validator_registry[candidate_index].effective_balance; if (effective_balance * MAX_RANDOM_BYTE) - >= (spec.max_effective_balance * random_byte as u64) + >= (spec.max_effective_balance * u64::from(random_byte)) { break candidate_index; } diff --git a/eth2/types/src/beacon_state/committee_cache.rs b/eth2/types/src/beacon_state/committee_cache.rs index 78af727544..aa921da82f 100644 --- a/eth2/types/src/beacon_state/committee_cache.rs +++ b/eth2/types/src/beacon_state/committee_cache.rs @@ -162,7 +162,6 @@ impl CommitteeCache { let i = self.shuffled_position(validator_index)?; (0..self.committee_count) - .into_iter() .map(|nth_committee| (nth_committee, self.compute_committee_range(nth_committee))) .find(|(_, range)| { if let Some(range) = range { diff --git a/eth2/types/src/chain_spec.rs b/eth2/types/src/chain_spec.rs index b16947328c..89ea97070c 100644 --- a/eth2/types/src/chain_spec.rs +++ b/eth2/types/src/chain_spec.rs @@ -174,7 +174,7 @@ impl ChainSpec { /* * Time parameters */ - genesis_time: u32::max_value() as u64, + genesis_time: u64::from(u32::max_value()), seconds_per_slot: 6, min_attestation_inclusion_delay: 4, min_seed_lookahead: Epoch::new(1), diff --git a/eth2/types/src/slot_epoch.rs b/eth2/types/src/slot_epoch.rs index fbeb479d78..9a7808da4c 100644 --- a/eth2/types/src/slot_epoch.rs +++ b/eth2/types/src/slot_epoch.rs @@ -77,7 +77,7 @@ impl Epoch { /// Position of some slot inside an epoch, if any. /// /// E.g., the first `slot` in `epoch` is at position `0`. - pub fn position(&self, slot: Slot, slots_per_epoch: u64) -> Option { + pub fn position(self, slot: Slot, slots_per_epoch: u64) -> Option { let start = self.start_slot(slots_per_epoch); let end = self.end_slot(slots_per_epoch); diff --git a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs index 999e88aaef..36bbe2d37c 100644 --- a/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_beacon_block_builder.rs @@ -271,7 +271,7 @@ fn build_proposer_slashing( Signature::new(message, domain, secret_key) }; - TestingProposerSlashingBuilder::double_vote::(validator_index, signer, spec) + TestingProposerSlashingBuilder::double_vote::(validator_index, signer) } /// Builds an `AttesterSlashing` for some `validator_indices`. diff --git a/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs b/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs index 99c005472a..67668d130d 100644 --- a/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs +++ b/eth2/types/src/test_utils/builders/testing_proposer_slashing_builder.rs @@ -17,7 +17,7 @@ impl TestingProposerSlashingBuilder { /// - `domain: Domain` /// /// Where domain is a domain "constant" (e.g., `spec.domain_attestation`). - pub fn double_vote(proposer_index: u64, signer: F, spec: &ChainSpec) -> ProposerSlashing + pub fn double_vote(proposer_index: u64, signer: F) -> ProposerSlashing where T: EthSpec, F: Fn(u64, &[u8], Epoch, Domain) -> Signature, diff --git a/eth2/types/src/test_utils/serde_utils.rs b/eth2/types/src/test_utils/serde_utils.rs index be8f002adb..079551b583 100644 --- a/eth2/types/src/test_utils/serde_utils.rs +++ b/eth2/types/src/test_utils/serde_utils.rs @@ -13,6 +13,7 @@ where u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom) } +#[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref. pub fn u8_to_hex_str(byte: &u8, serializer: S) -> Result where S: Serializer, diff --git a/eth2/utils/eth2_config/src/lib.rs b/eth2/utils/eth2_config/src/lib.rs index c4d4736bdb..9d50a95c18 100644 --- a/eth2/utils/eth2_config/src/lib.rs +++ b/eth2/utils/eth2_config/src/lib.rs @@ -80,7 +80,7 @@ where ) })?; file.write_all(toml_encoded.as_bytes()) - .expect(&format!("Unable to write to {:?}", path)); + .unwrap_or_else(|_| panic!("Unable to write to {:?}", path)); } Ok(()) diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index cd40ac8cea..88fd353a14 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -28,13 +28,10 @@ pub fn compare_beacon_state_results_without_caches( result: &mut Result, E>, expected: &mut Option>, ) -> Result<(), Error> { - match (result.as_mut(), expected.as_mut()) { - (Ok(ref mut result), Some(ref mut expected)) => { - result.drop_all_caches(); - expected.drop_all_caches(); - } - _ => (), - }; + if let (Ok(ref mut result), Some(ref mut expected)) = (result.as_mut(), expected.as_mut()) { + result.drop_all_caches(); + expected.drop_all_caches(); + } compare_result_detailed(&result, &expected) } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index df1a9428ba..5b83ea7930 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -70,7 +70,7 @@ where impl YamlDecode for Cases { /// Decodes a YAML list of test cases - fn yaml_decode(yaml: &String) -> Result { + fn yaml_decode(yaml: &str) -> Result { let mut p = 0; let mut elems: Vec<&str> = yaml .match_indices("\n- ") diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index 6cd37ec36e..6e38743f2f 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -10,8 +10,8 @@ pub struct BlsAggregatePubkeys { } impl YamlDecode for BlsAggregatePubkeys { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index 5b69a6134e..eeecab82cd 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -10,8 +10,8 @@ pub struct BlsAggregateSigs { } impl YamlDecode for BlsAggregateSigs { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index e6895ca1af..185cb58f35 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -16,8 +16,8 @@ pub struct BlsG2Compressed { } impl YamlDecode for BlsG2Compressed { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } @@ -46,13 +46,13 @@ impl Case for BlsG2Compressed { } // Converts a vector to u64 (from big endian) -fn bytes_to_u64(array: &Vec) -> u64 { +fn bytes_to_u64(array: &[u8]) -> u64 { let mut result: u64 = 0; for (i, value) in array.iter().rev().enumerate() { if i == 8 { break; } - result += u64::pow(2, i as u32 * 8) * (*value as u64); + result += u64::pow(2, i as u32 * 8) * u64::from(*value); } result } diff --git a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs index 93b1e1c51e..962b6aac39 100644 --- a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs @@ -16,8 +16,8 @@ pub struct BlsG2Uncompressed { } impl YamlDecode for BlsG2Uncompressed { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } @@ -56,13 +56,13 @@ impl Case for BlsG2Uncompressed { } // Converts a vector to u64 (from big endian) -fn bytes_to_u64(array: &Vec) -> u64 { +fn bytes_to_u64(array: &[u8]) -> u64 { let mut result: u64 = 0; for (i, value) in array.iter().rev().enumerate() { if i == 8 { break; } - result += u64::pow(2, i as u32 * 8) * (*value as u64); + result += u64::pow(2, i as u32 * 8) * u64::from(*value); } result } diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index c558d01428..d72a43bbbc 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -10,8 +10,8 @@ pub struct BlsPrivToPub { } impl YamlDecode for BlsPrivToPub { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index a361f2523c..e62c3550fa 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -17,8 +17,8 @@ pub struct BlsSign { } impl YamlDecode for BlsSign { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } @@ -46,13 +46,13 @@ impl Case for BlsSign { } // Converts a vector to u64 (from big endian) -fn bytes_to_u64(array: &Vec) -> u64 { +fn bytes_to_u64(array: &[u8]) -> u64 { let mut result: u64 = 0; for (i, value) in array.iter().rev().enumerate() { if i == 8 { break; } - result += u64::pow(2, i as u32 * 8) * (*value as u64); + result += u64::pow(2, i as u32 * 8) * u64::from(*value); } result } diff --git a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs index afdb9cc317..bf1564b97c 100644 --- a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs +++ b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs @@ -14,8 +14,8 @@ pub struct EpochProcessingCrosslinks { } impl YamlDecode for EpochProcessingCrosslinks { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs index 092a8a7092..02311656e3 100644 --- a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs +++ b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs @@ -14,8 +14,8 @@ pub struct EpochProcessingRegistryUpdates { } impl YamlDecode for EpochProcessingRegistryUpdates { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/operations_attester_slashing.rs b/tests/ef_tests/src/cases/operations_attester_slashing.rs index 1b95f43590..8f2338054b 100644 --- a/tests/ef_tests/src/cases/operations_attester_slashing.rs +++ b/tests/ef_tests/src/cases/operations_attester_slashing.rs @@ -15,8 +15,8 @@ pub struct OperationsAttesterSlashing { } impl YamlDecode for OperationsAttesterSlashing { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/operations_deposit.rs b/tests/ef_tests/src/cases/operations_deposit.rs index 8100590fdd..8357067059 100644 --- a/tests/ef_tests/src/cases/operations_deposit.rs +++ b/tests/ef_tests/src/cases/operations_deposit.rs @@ -16,8 +16,8 @@ pub struct OperationsDeposit { } impl YamlDecode for OperationsDeposit { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/operations_exit.rs b/tests/ef_tests/src/cases/operations_exit.rs index cfe20d82b8..f38f327ea5 100644 --- a/tests/ef_tests/src/cases/operations_exit.rs +++ b/tests/ef_tests/src/cases/operations_exit.rs @@ -15,8 +15,8 @@ pub struct OperationsExit { } impl YamlDecode for OperationsExit { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/operations_proposer_slashing.rs b/tests/ef_tests/src/cases/operations_proposer_slashing.rs index 43293dfbcf..cf498079f9 100644 --- a/tests/ef_tests/src/cases/operations_proposer_slashing.rs +++ b/tests/ef_tests/src/cases/operations_proposer_slashing.rs @@ -15,8 +15,8 @@ pub struct OperationsProposerSlashing { } impl YamlDecode for OperationsProposerSlashing { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/operations_transfer.rs b/tests/ef_tests/src/cases/operations_transfer.rs index 66318e7504..966023b2a8 100644 --- a/tests/ef_tests/src/cases/operations_transfer.rs +++ b/tests/ef_tests/src/cases/operations_transfer.rs @@ -15,8 +15,8 @@ pub struct OperationsTransfer { } impl YamlDecode for OperationsTransfer { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index 2920840c80..d7ff40e596 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -14,8 +14,8 @@ pub struct Shuffling { } impl YamlDecode for Shuffling { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } @@ -30,7 +30,6 @@ impl Case for Shuffling { // Test get_permuted_index let shuffling = (0..self.count) - .into_iter() .map(|i| { get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap() }) diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index 09aba39f1c..ca49d21060 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -15,8 +15,8 @@ pub struct SszGeneric { } impl YamlDecode for SszGeneric { - fn yaml_decode(yaml: &String) -> Result { - Ok(serde_yaml::from_str(&yaml.as_str()).unwrap()) + fn yaml_decode(yaml: &str) -> Result { + Ok(serde_yaml::from_str(yaml).unwrap()) } } @@ -45,11 +45,7 @@ impl Case for SszGeneric { } /// Execute a `ssz_generic` test case. -fn ssz_generic_test( - should_be_ok: bool, - ssz: &String, - value: &Option, -) -> Result<(), Error> +fn ssz_generic_test(should_be_ok: bool, ssz: &str, value: &Option) -> Result<(), Error> where T: Decode + YamlDecode + Debug + PartialEq, { diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 374b90bd27..3365a51e13 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -55,7 +55,7 @@ where } impl YamlDecode for SszStatic { - fn yaml_decode(yaml: &String) -> Result { + fn yaml_decode(yaml: &str) -> Result { serde_yaml::from_str(yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } } diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs index 301ba91780..3f4284501d 100644 --- a/tests/ef_tests/src/doc.rs +++ b/tests/ef_tests/src/doc.rs @@ -136,14 +136,14 @@ pub fn print_failures(doc: &Doc, results: &[CaseResult]) { println!("Test Failure"); println!("Title: {}", header.title); println!("File: {:?}", doc.path); - println!(""); + println!(); println!( "{} tests, {} failures, {} passes.", results.len(), failures.len(), results.len() - failures.len() ); - println!(""); + println!(); for failure in failures { let error = failure.result.clone().unwrap_err(); @@ -157,5 +157,5 @@ pub fn print_failures(doc: &Doc, results: &[CaseResult]) { ); println!("{}", error.message()); } - println!(""); + println!(); } diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs index 974df8311a..c89dd92a9e 100644 --- a/tests/ef_tests/src/yaml_decode.rs +++ b/tests/ef_tests/src/yaml_decode.rs @@ -8,14 +8,14 @@ pub use utils::*; pub trait YamlDecode: Sized { /// Decode an object from the test specification YAML. - fn yaml_decode(string: &String) -> Result; + fn yaml_decode(string: &str) -> Result; } /// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. macro_rules! impl_via_parse { ($ty: ty) => { impl YamlDecode for $ty { - fn yaml_decode(string: &String) -> Result { + fn yaml_decode(string: &str) -> Result { string .parse::() .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) @@ -34,7 +34,7 @@ impl_via_parse!(u64); macro_rules! impl_via_from_dec_str { ($ty: ty) => { impl YamlDecode for $ty { - fn yaml_decode(string: &String) -> Result { + fn yaml_decode(string: &str) -> Result { Self::from_dec_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } } @@ -48,7 +48,7 @@ impl_via_from_dec_str!(U256); macro_rules! impl_via_serde_yaml { ($ty: ty) => { impl YamlDecode for $ty { - fn yaml_decode(string: &String) -> Result { + fn yaml_decode(string: &str) -> Result { serde_yaml::from_str(string) .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) } diff --git a/tests/ef_tests/src/yaml_decode/utils.rs b/tests/ef_tests/src/yaml_decode/utils.rs index 059d3b5d21..7b6caac728 100644 --- a/tests/ef_tests/src/yaml_decode/utils.rs +++ b/tests/ef_tests/src/yaml_decode/utils.rs @@ -3,7 +3,7 @@ pub fn yaml_split_header_and_cases(mut yaml: String) -> (String, String) { // + 1 to skip the \n we used for matching. let mut test_cases = yaml.split_off(test_cases_start + 1); - let end_of_first_line = test_cases.find("\n").unwrap(); + let end_of_first_line = test_cases.find('\n').unwrap(); let test_cases = test_cases.split_off(end_of_first_line + 1); (yaml, test_cases) From 059699736b2f3d720a6c100625a889db00d3e02b Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jun 2019 21:37:41 -0400 Subject: [PATCH 74/76] Return errors instead of panic in libp2p --- beacon_node/eth2-libp2p/src/service.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index b18a7dc512..18f7ca98ce 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -57,7 +57,10 @@ impl Service { }; // listen on all addresses - for address in config.listen_addresses().expect("invalid listen multiaddr") { + for address in config + .listen_addresses() + .map_err(|e| format!("Invalid listen multiaddr: {}", e))? + { match Swarm::listen_on(&mut swarm, address.clone()) { Ok(mut listen_addr) => { listen_addr.append(Protocol::P2p(local_peer_id.clone().into())); @@ -68,7 +71,10 @@ impl Service { } // connect to boot nodes - these are currently stored as multiaddrs // Once we have discovery, can set to peerId - for bootnode in config.boot_nodes().expect("invalid boot node multiaddr") { + for bootnode in config + .boot_nodes() + .map_err(|e| format!("Invalid boot node multiaddr: {:?}", e))? + { match Swarm::dial_addr(&mut swarm, bootnode.clone()) { Ok(()) => debug!(log, "Dialing bootnode: {}", bootnode), Err(err) => debug!( From b9e832216ba7a5feb820e645352c1fdc09c5a89e Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 10 Jun 2019 21:37:59 -0400 Subject: [PATCH 75/76] Change name of VC CLI param --- validator_client/src/main.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/validator_client/src/main.rs b/validator_client/src/main.rs index 6a310cddbe..f749154385 100644 --- a/validator_client/src/main.rs +++ b/validator_client/src/main.rs @@ -40,13 +40,12 @@ fn main() { .takes_value(true), ) .arg( - Arg::with_name("eth-config") - .long("eth-config") + Arg::with_name("eth2-spec") + .long("eth2-spec") .short("e") - .value_name("DIR") - .help(&format!("Directory containing {}.", ETH2_CONFIG_FILENAME)) - .takes_value(true) - .default_value(ETH2_CONFIG_FILENAME), + .value_name("TOML_FILE") + .help("Path to Ethereum 2.0 specifications file.") + .takes_value(true), ) .arg( Arg::with_name("server") @@ -111,7 +110,7 @@ fn main() { }; let eth2_config_path: PathBuf = matches - .value_of("eth-config") + .value_of("eth2-spec") .and_then(|s| Some(PathBuf::from(s))) .unwrap_or_else(|| data_dir.join(ETH2_CONFIG_FILENAME)); From 4c0724fba6abe7a353c8033bb3681cc58d29373d Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 12 Jun 2019 23:54:16 -0400 Subject: [PATCH 76/76] Refactor block processing results, some sync logic --- beacon_node/beacon_chain/src/beacon_chain.rs | 205 +++-------- beacon_node/beacon_chain/src/lib.rs | 4 +- beacon_node/network/src/sync/import_queue.rs | 2 +- beacon_node/network/src/sync/simple_sync.rs | 337 +++++++++---------- beacon_node/rpc/src/beacon_block.rs | 26 +- 5 files changed, 215 insertions(+), 359 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 945c376175..dc2cc16df9 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -15,7 +15,7 @@ use state_processing::per_block_processing::errors::{ }; use state_processing::{ per_block_processing, per_block_processing_without_verifying_block_signature, - per_slot_processing, BlockProcessingError, SlotProcessingError, + per_slot_processing, BlockProcessingError, }; use std::sync::Arc; use store::{Error as DBError, Store}; @@ -23,15 +23,11 @@ use tree_hash::TreeHash; use types::*; #[derive(Debug, PartialEq)] -pub enum ValidBlock { - /// The block was successfully processed. +pub enum BlockProcessingOutcome { + /// Block was valid and imported into the block graph. Processed, -} - -#[derive(Debug, PartialEq)] -pub enum InvalidBlock { - /// Don't re-process the genesis block. - GenesisBlock, + /// The blocks parent_root is unknown. + ParentUnknown { parent: Hash256 }, /// The block slot is greater than the present slot. FutureSlot { present_slot: Slot, @@ -39,53 +35,16 @@ pub enum InvalidBlock { }, /// The block state_root does not match the generated state. StateRootMismatch, - /// The blocks parent_root is unknown. - ParentUnknown { parent: Hash256 }, - /// There was an error whilst advancing the parent state to the present slot. This condition - /// should not occur, it likely represents an internal error. - SlotProcessingError(SlotProcessingError), + /// The block was a genesis block, these blocks cannot be re-imported. + GenesisBlock, + /// The slot is finalized, no need to import. + FinalizedSlot, + /// Block is already known, no need to re-import. + BlockIsAlreadyKnown, /// The block could not be applied to the state, it is invalid. PerBlockProcessingError(BlockProcessingError), } -#[derive(Debug, PartialEq)] -pub enum BlockProcessingOutcome { - /// The block was successfully validated. - ValidBlock(ValidBlock), - /// The block was not successfully validated. - InvalidBlock(InvalidBlock), -} - -impl BlockProcessingOutcome { - /// Returns `true` if the block was objectively invalid and we should disregard the peer who - /// sent it. - pub fn is_invalid(&self) -> bool { - match self { - BlockProcessingOutcome::ValidBlock(_) => false, - BlockProcessingOutcome::InvalidBlock(r) => match r { - InvalidBlock::GenesisBlock { .. } => true, - InvalidBlock::FutureSlot { .. } => true, - InvalidBlock::StateRootMismatch => true, - InvalidBlock::ParentUnknown { .. } => false, - InvalidBlock::SlotProcessingError(_) => false, - InvalidBlock::PerBlockProcessingError(e) => match e { - BlockProcessingError::Invalid(_) => true, - BlockProcessingError::BeaconStateError(_) => false, - }, - }, - } - } - - /// Returns `true` if the block was successfully processed and can be removed from any import - /// queues or temporary storage. - pub fn sucessfully_processed(&self) -> bool { - match self { - BlockProcessingOutcome::ValidBlock(_) => true, - _ => false, - } - } -} - pub trait BeaconChainTypes { type Store: store::Store; type SlotClock: slot_clock::SlotClock; @@ -257,88 +216,6 @@ impl BeaconChain { BlockRootsIterator::new(self.store.clone(), self.state.read().clone(), slot) } - /* - /// Returns `count `beacon block roots, starting from `start_slot` with an - /// interval of `skip` slots between each root. - /// - /// ## Errors: - /// - /// - `SlotOutOfBounds`: Unable to return the full specified range. - /// - `SlotOutOfBounds`: Unable to load a state from the DB. - /// - `SlotOutOfBounds`: Start slot is higher than the first slot. - /// - Other: BeaconState` is inconsistent. - pub fn get_block_roots( - &self, - earliest_slot: Slot, - count: usize, - skip: usize, - ) -> Result, Error> { - let step_by = Slot::from(skip + 1); - - let mut roots: Vec = vec![]; - - // The state for reading block roots. Will be updated with an older state if slots go too - // far back in history. - let mut state = self.state.read().clone(); - - // The final slot in this series, will be reduced by `skip` each loop iteration. - let mut slot = earliest_slot + Slot::from(count * (skip + 1)) - 1; - - // If the highest slot requested is that of the current state insert the root of the - // head block, unless the head block's slot is not matching. - if slot == state.slot && self.head().beacon_block.slot == slot { - roots.push(self.head().beacon_block_root); - - slot -= step_by; - } else if slot >= state.slot { - return Err(BeaconStateError::SlotOutOfBounds.into()); - } - - loop { - // If the slot is within the range of the current state's block roots, append the root - // to the output vec. - // - // If we get `SlotOutOfBounds` error, load the oldest available historic - // state from the DB. - match state.get_block_root(slot) { - Ok(root) => { - if slot < earliest_slot { - break; - } else { - roots.push(*root); - slot -= step_by; - } - } - Err(BeaconStateError::SlotOutOfBounds) => { - // Read the earliest historic state in the current slot. - let earliest_historic_slot = - state.slot - Slot::from(T::EthSpec::slots_per_historical_root()); - // Load the earlier state from disk. - let new_state_root = state.get_state_root(earliest_historic_slot)?; - - // Break if the DB is unable to load the state. - state = match self.store.get(&new_state_root) { - Ok(Some(state)) => state, - _ => break, - } - } - Err(e) => return Err(e.into()), - }; - } - - // Return the results if they pass a sanity check. - if (slot <= earliest_slot) && (roots.len() == count) { - // Reverse the ordering of the roots. We extracted them in reverse order to make it - // simpler to lookup historic states. - // - // This is a potential optimisation target. - Ok(roots.iter().rev().cloned().collect()) - } else { - Err(BeaconStateError::SlotOutOfBounds.into()) - } - } - */ - /// Returns the block at the given root, if any. /// /// ## Errors @@ -649,31 +526,39 @@ impl BeaconChain { /// /// Will accept blocks from prior slots, however it will reject any block from a future slot. pub fn process_block(&self, block: BeaconBlock) -> Result { - debug!("Processing block with slot {}...", block.slot); self.metrics.block_processing_requests.inc(); let timer = self.metrics.block_processing_times.start_timer(); + let finalized_slot = self + .state + .read() + .finalized_epoch + .start_slot(T::EthSpec::slots_per_epoch()); + if block.slot <= finalized_slot { + return Ok(BlockProcessingOutcome::FinalizedSlot); + } + if block.slot == 0 { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::GenesisBlock, - )); + return Ok(BlockProcessingOutcome::GenesisBlock); } let block_root = block.block_header().canonical_root(); if block_root == self.genesis_block_root { - return Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)); + return Ok(BlockProcessingOutcome::GenesisBlock); } let present_slot = self.present_slot(); if block.slot > present_slot { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::FutureSlot { - present_slot, - block_slot: block.slot, - }, - )); + return Ok(BlockProcessingOutcome::FutureSlot { + present_slot, + block_slot: block.slot, + }); + } + + if self.store.exists::(&block_root)? { + return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown); } // Load the blocks parent block from the database, returning invalid if that block is not @@ -682,11 +567,9 @@ impl BeaconChain { let parent_block: BeaconBlock = match self.store.get(&parent_block_root)? { Some(previous_block_root) => previous_block_root, None => { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::ParentUnknown { - parent: parent_block_root, - }, - )); + return Ok(BlockProcessingOutcome::ParentUnknown { + parent: parent_block_root, + }); } }; @@ -704,29 +587,25 @@ impl BeaconChain { // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { - if let Err(e) = per_slot_processing(&mut state, &self.spec) { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::SlotProcessingError(e), - )); - } + per_slot_processing(&mut state, &self.spec)?; } state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; // Apply the received block to its parent state (which has been transitioned into this // slot). - if let Err(e) = per_block_processing(&mut state, &block, &self.spec) { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::PerBlockProcessingError(e), - )); + match per_block_processing(&mut state, &block, &self.spec) { + Err(BlockProcessingError::BeaconStateError(e)) => { + return Err(Error::BeaconStateError(e)) + } + Err(e) => return Ok(BlockProcessingOutcome::PerBlockProcessingError(e)), + _ => {} } let state_root = state.canonical_root(); if block.state_root != state_root { - return Ok(BlockProcessingOutcome::InvalidBlock( - InvalidBlock::StateRootMismatch, - )); + return Ok(BlockProcessingOutcome::StateRootMismatch); } // Store the block and state. @@ -750,7 +629,7 @@ impl BeaconChain { .observe(block.body.attestations.len() as f64); timer.observe_duration(); - Ok(BlockProcessingOutcome::ValidBlock(ValidBlock::Processed)) + Ok(BlockProcessingOutcome::Processed) } /// Produce a new block at the present slot. diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index bde541fce5..21edb78598 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -5,9 +5,7 @@ pub mod iter; mod metrics; mod persisted_beacon_chain; -pub use self::beacon_chain::{ - BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock, ValidBlock, -}; +pub use self::beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; pub use fork_choice; diff --git a/beacon_node/network/src/sync/import_queue.rs b/beacon_node/network/src/sync/import_queue.rs index 16a277f0b5..90c354cfd5 100644 --- a/beacon_node/network/src/sync/import_queue.rs +++ b/beacon_node/network/src/sync/import_queue.rs @@ -212,7 +212,7 @@ impl ImportQueue { // Case 2: there was no partial with a matching block root. // // A new partial is added. This case permits adding a header without already known the - // root -- this is not possible in the wire protocol however we support it anyway. + // root. self.partials.push(PartialBeaconBlock { slot: header.slot, block_root, diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 7f6421e327..403a8c54b8 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,6 +1,6 @@ use super::import_queue::ImportQueue; use crate::message_handler::NetworkContext; -use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome, InvalidBlock}; +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; @@ -9,7 +9,6 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; use store::Store; -use tree_hash::TreeHash; use types::{ Attestation, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, Epoch, EthSpec, Hash256, Slot, }; @@ -24,6 +23,9 @@ const QUEUE_STALE_SECS: u64 = 600; /// Otherwise we queue it. const FUTURE_SLOT_TOLERANCE: u64 = 1; +const SHOULD_FORWARD_GOSSIP_BLOCK: bool = true; +const SHOULD_NOT_FORWARD_GOSSIP_BLOCK: bool = false; + /// Keeps track of syncing information for known connected peers. #[derive(Clone, Copy, Debug)] pub struct PeerSyncInfo { @@ -393,6 +395,7 @@ impl SimpleSync { .collect(); roots.reverse(); + roots.dedup(); let headers: Vec = roots .into_iter() @@ -509,6 +512,8 @@ impl SimpleSync { /// Process a gossip message declaring a new block. /// + /// Attempts to apply to block to the beacon chain. May queue the block for later processing. + /// /// Returns a `bool` which, if `true`, indicates we should forward the block to our peers. pub fn on_block_gossip( &mut self, @@ -516,133 +521,35 @@ impl SimpleSync { block: BeaconBlock, network: &mut NetworkContext, ) -> bool { - // Ignore any block from a finalized slot. - if self.slot_is_finalized(block.slot) { - debug!( - self.log, "IgnoredFinalizedBlock"; - "source" => "gossip", - "msg" => "chain is finalized at block slot", - "block_slot" => block.slot, - ); - return false; - } + if let Some(outcome) = + self.process_block(peer_id.clone(), block.clone(), network, &"gossip") + { + match outcome { + BlockProcessingOutcome::Processed => SHOULD_FORWARD_GOSSIP_BLOCK, + BlockProcessingOutcome::ParentUnknown { .. } => { + self.import_queue + .enqueue_full_blocks(vec![block], peer_id.clone()); - let block_root = Hash256::from_slice(&block.tree_hash_root()); - - // Ignore any block that the chain already knows about. - if self.chain_has_seen_block(&block_root) { - // TODO: Age confirm that we shouldn't forward a block if we already know of it. - return false; - } - - match self.chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::InvalidBlock(InvalidBlock::ParentUnknown { .. })) => { - // The block was valid and we processed it successfully. - debug!( - self.log, "ParentBlockUnknown"; - "source" => "gossip", - "parent_root" => format!("{}", block.previous_block_root), - "peer" => format!("{:?}", peer_id), - ); - // Queue the block for later processing. - self.import_queue - .enqueue_full_blocks(vec![block], peer_id.clone()); - // Send a hello to learn of the clients best slot so we can then sync the require - // parent(s). - network.send_rpc_request( - peer_id.clone(), - RPCRequest::Hello(hello_message(&self.chain)), - ); - // Forward the block onto our peers. - // - // Note: this may need to be changed if we decide to only forward blocks if we have - // all required info. - true - } - Ok(BlockProcessingOutcome::InvalidBlock(InvalidBlock::FutureSlot { - present_slot, - block_slot, - })) => { - if block_slot - present_slot > FUTURE_SLOT_TOLERANCE { - // The block is too far in the future, drop it. - warn!( - self.log, "FutureBlock"; - "source" => "gossip", - "msg" => "block for future slot rejected, check your time", - "present_slot" => present_slot, - "block_slot" => block_slot, - "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, - "peer" => format!("{:?}", peer_id), - ); - // Do not forward the block around to peers. - false - } else { - // The block is in the future, but not too far. - warn!( - self.log, "QueuedFutureBlock"; - "source" => "gossip", - "msg" => "queuing future block, check your time", - "present_slot" => present_slot, - "block_slot" => block_slot, - "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, - "peer" => format!("{:?}", peer_id), - ); - // Queue the block for later processing. - self.import_queue.enqueue_full_blocks(vec![block], peer_id); - // Forward the block around to peers. - true + SHOULD_FORWARD_GOSSIP_BLOCK } - } - Ok(outcome) => { - if outcome.is_invalid() { - // The peer has sent a block which is fundamentally invalid. - warn!( - self.log, "InvalidBlock"; - "source" => "gossip", - "msg" => "peer sent objectively invalid block", - "outcome" => format!("{:?}", outcome), - "peer" => format!("{:?}", peer_id), - ); - // Disconnect the peer - network.disconnect(peer_id, GoodbyeReason::Fault); - // Do not forward the block to peers. - false - } else if outcome.sucessfully_processed() { - // The block was valid and we processed it successfully. - info!( - self.log, "ImportedBlock"; - "source" => "gossip", - "peer" => format!("{:?}", peer_id), - ); - // Forward the block to peers - true - } else { - // The block wasn't necessarily invalid but we didn't process it successfully. - // This condition shouldn't be reached. - error!( - self.log, "BlockProcessingFailure"; - "source" => "gossip", - "msg" => "unexpected condition in processing block.", - "outcome" => format!("{:?}", outcome), - ); - // Do not forward the block on. - false + BlockProcessingOutcome::FutureSlot { + present_slot, + block_slot, + } if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot => { + self.import_queue + .enqueue_full_blocks(vec![block], peer_id.clone()); + + SHOULD_FORWARD_GOSSIP_BLOCK } - } - Err(e) => { - // We encountered an error whilst processing the block. + // Note: known blocks are forwarded on the gossip network. // - // Blocks should not be able to trigger errors, instead they should be flagged as - // invalid. - error!( - self.log, "BlockProcessingError"; - "msg" => "internal error in processing block.", - "source" => "gossip", - "error" => format!("{:?}", e), - ); - // Do not forward the block to peers. - false + // We rely upon the lower layers (libp2p) to stop loops occuring from re-gossiped + // blocks. + BlockProcessingOutcome::BlockIsAlreadyKnown => SHOULD_FORWARD_GOSSIP_BLOCK, + _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, } + } else { + SHOULD_NOT_FORWARD_GOSSIP_BLOCK } } @@ -669,57 +576,32 @@ impl SimpleSync { /// the queue. pub fn process_import_queue(&mut self, network: &mut NetworkContext) { let mut successful = 0; - let mut invalid = 0; - let mut errored = 0; // Loop through all of the complete blocks in the queue. for (block_root, block, sender) in self.import_queue.complete_blocks() { - let slot = block.slot; - let parent_root = block.previous_block_root; + let processing_result = self.process_block(sender, block.clone(), network, &"gossip"); - match self.chain.process_block(block) { - Ok(outcome) => { - if outcome.is_invalid() { - invalid += 1; - warn!( - self.log, - "InvalidBlock"; - "sender_peer_id" => format!("{:?}", sender.clone()), - "block_root" => format!("{}", block_root), - "reason" => format!("{:?}", outcome), - ); - network.disconnect(sender, GoodbyeReason::Fault); - } else if outcome.sucessfully_processed() { - successful += 1; - self.import_queue.remove(block_root); - } else { - debug!( - self.log, - "ProcessImportQueue"; - "msg" => "Block not imported", - "outcome" => format!("{:?}", outcome), - "block_slot" => format!("{:?}", slot), - "parent_root" => format!("{}", parent_root), - "peer" => format!("{:?}", sender), - ); - } - } - Err(e) => { - errored += 1; - error!(self.log, "BlockProcessingError"; "error" => format!("{:?}", e)); - } + let should_dequeue = match processing_result { + Some(BlockProcessingOutcome::ParentUnknown { .. }) => false, + Some(BlockProcessingOutcome::FutureSlot { + present_slot, + block_slot, + }) if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot => false, + _ => true, + }; + + if processing_result == Some(BlockProcessingOutcome::Processed) { + successful += 1; + } + + if should_dequeue { + self.import_queue.remove(block_root); } } if successful > 0 { info!(self.log, "Imported {} blocks", successful) } - if invalid > 0 { - warn!(self.log, "Rejected {} invalid blocks", invalid) - } - if errored > 0 { - warn!(self.log, "Failed to process {} blocks", errored) - } } /// Request some `BeaconBlockRoots` from the remote peer. @@ -791,17 +673,128 @@ impl SimpleSync { }) } - /// Returns `true` if the given slot is finalized in our chain. - fn slot_is_finalized(&self, slot: Slot) -> bool { - slot <= hello_message(&self.chain) - .latest_finalized_epoch - .start_slot(T::EthSpec::slots_per_epoch()) - } - /// Generates our current state in the form of a HELLO RPC message. pub fn generate_hello(&self) -> HelloMessage { hello_message(&self.chain) } + + /// Processes the `block` that was received from `peer_id`. + /// + /// If the block was submitted to the beacon chain without internal error, `Some(outcome)` is + /// returned, otherwise `None` is returned. Note: `Some(_)` does not necessarily indicate that + /// the block was successfully processed or valid. + /// + /// This function performs the following duties: + /// + /// - Attempting to import the block into the beacon chain. + /// - Logging + /// - Requesting unavailable blocks (e.g., if parent is unknown). + /// - Disconnecting faulty nodes. + /// + /// This function does not remove processed blocks from the import queue. + fn process_block( + &mut self, + peer_id: PeerId, + block: BeaconBlock, + network: &mut NetworkContext, + source: &str, + ) -> Option { + let processing_result = self.chain.process_block(block.clone()); + + if let Ok(outcome) = processing_result { + match outcome { + BlockProcessingOutcome::Processed => { + info!( + self.log, "Imported block from network"; + "source" => source, + "slot" => block.slot, + "peer" => format!("{:?}", peer_id), + ); + } + BlockProcessingOutcome::ParentUnknown { parent } => { + // The block was valid and we processed it successfully. + debug!( + self.log, "ParentBlockUnknown"; + "source" => source, + "parent_root" => format!("{}", parent), + "peer" => format!("{:?}", peer_id), + ); + + // Send a hello to learn of the clients best slot so we can then sync the require + // parent(s). + network.send_rpc_request( + peer_id.clone(), + RPCRequest::Hello(hello_message(&self.chain)), + ); + + // Explicitly request the parent block from the peer. + // + // It is likely that this is duplicate work, given we already send a hello + // request. However, I believe there are some edge-cases where the hello + // message doesn't suffice, so we perform this request as well. + self.request_block_headers( + peer_id, + BeaconBlockHeadersRequest { + start_root: parent, + start_slot: block.slot - 1, + max_headers: 1, + skip_slots: 0, + }, + network, + ) + } + BlockProcessingOutcome::FutureSlot { + present_slot, + block_slot, + } => { + if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot { + // The block is too far in the future, drop it. + warn!( + self.log, "FutureBlock"; + "source" => source, + "msg" => "block for future slot rejected, check your time", + "present_slot" => present_slot, + "block_slot" => block_slot, + "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, + "peer" => format!("{:?}", peer_id), + ); + network.disconnect(peer_id, GoodbyeReason::Fault); + } else { + // The block is in the future, but not too far. + debug!( + self.log, "QueuedFutureBlock"; + "source" => source, + "msg" => "queuing future block, check your time", + "present_slot" => present_slot, + "block_slot" => block_slot, + "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, + "peer" => format!("{:?}", peer_id), + ); + } + } + _ => { + debug!( + self.log, "InvalidBlock"; + "source" => source, + "msg" => "peer sent invalid block", + "outcome" => format!("{:?}", outcome), + "peer" => format!("{:?}", peer_id), + ); + } + } + + Some(outcome) + } else { + error!( + self.log, "BlockProcessingFailure"; + "source" => source, + "msg" => "unexpected condition in processing block.", + "outcome" => format!("{:?}", processing_result) + ); + + None + } + } } /// Build a `HelloMessage` representing the state of the given `beacon_chain`. diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index c28c4f111b..d36cb1f313 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -1,4 +1,4 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use crossbeam_channel; use eth2_libp2p::PubsubMessage; use futures::Future; @@ -95,14 +95,12 @@ impl BeaconBlockService for BeaconBlockServiceInstance { Ok(block) => { match self.chain.process_block(block.clone()) { Ok(outcome) => { - if outcome.sucessfully_processed() { + if outcome == BlockProcessingOutcome::Processed { // Block was successfully processed. info!( self.log, - "PublishBeaconBlock"; - "type" => "valid_block", + "Valid block from RPC"; "block_slot" => block.slot, - "outcome" => format!("{:?}", outcome) ); // TODO: Obtain topics from the network service properly. @@ -126,12 +124,11 @@ impl BeaconBlockService for BeaconBlockServiceInstance { }); resp.set_success(true); - } else if outcome.is_invalid() { - // Block was invalid. + } else { + // Block was not successfully processed. warn!( self.log, - "PublishBeaconBlock"; - "type" => "invalid_block", + "Invalid block from RPC"; "outcome" => format!("{:?}", outcome) ); @@ -139,17 +136,6 @@ impl BeaconBlockService for BeaconBlockServiceInstance { resp.set_msg( format!("InvalidBlock: {:?}", outcome).as_bytes().to_vec(), ); - } else { - // Some failure during processing. - warn!( - self.log, - "PublishBeaconBlock"; - "type" => "unable_to_import", - "outcome" => format!("{:?}", outcome) - ); - - resp.set_success(false); - resp.set_msg(format!("other: {:?}", outcome).as_bytes().to_vec()); } } Err(e) => {