From cd26a19a702c98432b9cf2db93523b3fa4db2d51 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 14 Aug 2019 10:55:24 +1000 Subject: [PATCH 1/3] Attestation processing (#497) * Renamed fork_choice::process_attestation_from_block * Processing attestation in fork choice * Retrieving state from store and checking signature * Looser check on beacon state validity. * Cleaned up get_attestation_state * Expanded fork choice api to provide latest validator message. * Checking if the an attestation contains a latest message * Correct process_attestation error handling. * Copy paste error in comment fixed. * Tidy ancestor iterators * Getting attestation slot via helper method * Refactored attestation creation in test utils * Revert "Refactored attestation creation in test utils" This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306. * Integration tests for free attestation processing * Implicit conflicts resolved. * formatting * Do first pass on Grants code * Add another attestation processing test * Tidy attestation processing * Remove old code fragment * Add non-compiling half finished changes * Simplify, fix bugs, add tests for chain iters * Remove attestation processing from op pool * Fix bug with fork choice, tidy * Fix overly restrictive check in fork choice. * Ensure committee cache is build during attn proc * Ignore unknown blocks at fork choice * Various minor fixes * Make fork choice write lock in to read lock * Remove unused method * Tidy comments * Fix attestation prod. target roots change * Fix compile error in store iters * Reject any attestation prior to finalization * Fix minor PR comments * Remove duplicated attestation finalization check * Remove awkward `let` statement --- beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 265 +++++++++++++++--- beacon_node/beacon_chain/src/errors.rs | 11 + beacon_node/beacon_chain/src/fork_choice.rs | 44 ++- beacon_node/beacon_chain/src/lib.rs | 4 +- beacon_node/beacon_chain/src/test_utils.rs | 71 +++-- beacon_node/beacon_chain/tests/tests.rs | 214 +++++++++++++- beacon_node/network/src/sync/simple_sync.rs | 20 +- beacon_node/rpc/src/attestation.rs | 34 ++- beacon_node/store/src/iter.rs | 36 +-- eth2/lmd_ghost/src/lib.rs | 3 + eth2/lmd_ghost/src/reduced_tree.rs | 21 +- eth2/operation_pool/src/lib.rs | 20 +- .../src/per_block_processing.rs | 12 +- .../verify_attestation.rs | 117 ++++---- eth2/types/src/beacon_block.rs | 5 + 16 files changed, 695 insertions(+), 183 deletions(-) diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index d3de16b0f0..778224a3d4 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -24,3 +24,4 @@ lmd_ghost = { path = "../../eth2/lmd_ghost" } [dev-dependencies] rand = "0.5.5" +lazy_static = "1.3.0" diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 28ba5fe48b..96d3065306 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -11,9 +11,12 @@ use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::{RwLock, RwLockReadGuard}; use slog::{error, info, warn, Logger}; use slot_clock::SlotClock; -use state_processing::per_block_processing::errors::{ - AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, - ExitValidationError, ProposerSlashingValidationError, TransferValidationError, +use state_processing::per_block_processing::{ + errors::{ + AttestationValidationError, AttesterSlashingValidationError, DepositValidationError, + ExitValidationError, ProposerSlashingValidationError, TransferValidationError, + }, + verify_attestation_for_state, VerifySignatures, }; use state_processing::{ per_block_processing, per_block_processing_without_verifying_block_signature, @@ -54,6 +57,26 @@ pub enum BlockProcessingOutcome { PerBlockProcessingError(BlockProcessingError), } +#[derive(Debug, PartialEq)] +pub enum AttestationProcessingOutcome { + Processed, + UnknownHeadBlock { + beacon_block_root: Hash256, + }, + /// The attestation is attesting to a state that is later than itself. (Viz., attesting to the + /// future). + AttestsToFutureState { + state: Slot, + attestation: Slot, + }, + /// The slot is finalized, no need to import. + FinalizedSlot { + attestation: Epoch, + finalized: Epoch, + }, + Invalid(AttestationValidationError), +} + pub trait BeaconChainTypes { type Store: store::Store; type SlotClock: slot_clock::SlotClock; @@ -237,15 +260,12 @@ impl BeaconChain { /// - Iterator returns `(Hash256, Slot)`. /// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot /// returned may be earlier than the wall-clock slot. - pub fn rev_iter_block_roots( - &self, - slot: Slot, - ) -> ReverseBlockRootIterator { + pub fn rev_iter_block_roots(&self) -> ReverseBlockRootIterator { let state = &self.head().beacon_state; let block_root = self.head().beacon_block_root; let block_slot = state.slot; - let iter = BlockRootsIterator::owned(self.store.clone(), state.clone(), slot); + let iter = BlockRootsIterator::owned(self.store.clone(), state.clone()); ReverseBlockRootIterator::new((block_root, block_slot), iter) } @@ -259,15 +279,12 @@ impl BeaconChain { /// - Iterator returns `(Hash256, Slot)`. /// - As this iterator starts at the `head` of the chain (viz., the best block), the first slot /// returned may be earlier than the wall-clock slot. - pub fn rev_iter_state_roots( - &self, - slot: Slot, - ) -> ReverseStateRootIterator { + pub fn rev_iter_state_roots(&self) -> ReverseStateRootIterator { let state = &self.head().beacon_state; let state_root = self.head().beacon_state_root; let state_slot = state.slot; - let iter = StateRootsIterator::owned(self.store.clone(), state.clone(), slot); + let iter = StateRootsIterator::owned(self.store.clone(), state.clone()); ReverseStateRootIterator::new((state_root, state_slot), iter) } @@ -484,6 +501,7 @@ impl BeaconChain { } else { *state.get_block_root(current_epoch_start_slot)? }; + let target = Checkpoint { epoch: state.current_epoch(), root: target_root, @@ -513,38 +531,212 @@ impl BeaconChain { }) } - /// Accept a new attestation from the network. + /// Accept a new, potentially invalid attestation from the network. /// - /// If valid, the attestation is added to the `op_pool` and aggregated with another attestation - /// if possible. + /// If valid, the attestation is added to `self.op_pool` and `self.fork_choice`. + /// + /// Returns an `Ok(AttestationProcessingOutcome)` if the chain was able to make a determination + /// about the `attestation` (whether it was invalid or not). Returns an `Err` if there was an + /// error during this process and no determination was able to be made. + /// + /// ## Notes + /// + /// - Whilst the `attestation` is added to fork choice, the head is not updated. That must be + /// done separately. pub fn process_attestation( &self, attestation: Attestation, - ) -> Result<(), AttestationValidationError> { + ) -> Result { + // From the store, load the attestation's "head block". + // + // An honest validator would have set this block to be the head of the chain (i.e., the + // result of running fork choice). + if let Some(attestation_head_block) = self + .store + .get::>(&attestation.data.beacon_block_root)? + { + // Attempt to process the attestation using the `self.head()` state. + // + // This is purely an effort to avoid loading a `BeaconState` unnecessarily from the DB. + // Take a read lock on the head beacon state. + let state = &self.head().beacon_state; + + // If it turns out that the attestation was made using the head state, then there + // is no need to load a state from the database to process the attestation. + // + // Note: use the epoch of the target because it indicates which epoch the + // attestation was created in. You cannot use the epoch of the head block, because + // the block doesn't necessarily need to be in the same epoch as the attestation + // (e.g., if there are skip slots between the epoch the block was created in and + // the epoch for the attestation). + // + // This check also ensures that the slot for `data.beacon_block_root` is not higher + // than `state.root` by ensuring that the block is in the history of `state`. + if state.current_epoch() == attestation.data.target.epoch + && (attestation.data.beacon_block_root == self.head().beacon_block_root + || state + .get_block_root(attestation_head_block.slot) + .map(|root| *root == attestation.data.beacon_block_root) + .unwrap_or_else(|_| false)) + { + // The head state is able to be used to validate this attestation. No need to load + // anything from the database. + return self.process_attestation_for_state_and_block( + attestation.clone(), + state, + &attestation_head_block, + ); + } + + // Ensure the read-lock from `self.head()` is dropped. + // + // This is likely unnecessary, however it remains as a reminder to ensure this lock + // isn't hogged. + std::mem::drop(state); + + // Use the `data.beacon_block_root` to load the state from the latest non-skipped + // slot preceding the attestation's creation. + // + // This state is guaranteed to be in the same chain as the attestation, but it's + // not guaranteed to be from the same slot or epoch as the attestation. + let mut state: BeaconState = self + .store + .get(&attestation_head_block.state_root)? + .ok_or_else(|| Error::MissingBeaconState(attestation_head_block.state_root))?; + + // Ensure the state loaded from the database matches the state of the attestation + // head block. + // + // The state needs to be advanced from the current slot through to the epoch in + // which the attestation was created in. It would be an error to try and use + // `state.get_attestation_data_slot(..)` because the state matching the + // `data.beacon_block_root` isn't necessarily in a nearby epoch to the attestation + // (e.g., if there were lots of skip slots since the head of the chain and the + // epoch creation epoch). + for _ in state.slot.as_u64() + ..attestation + .data + .target + .epoch + .start_slot(T::EthSpec::slots_per_epoch()) + .as_u64() + { + per_slot_processing(&mut state, &self.spec)?; + } + + state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + + let attestation_slot = state.get_attestation_data_slot(&attestation.data)?; + + // Reject any attestation where the `state` loaded from `data.beacon_block_root` + // has a higher slot than the attestation. + // + // Permitting this would allow for attesters to vote on _future_ slots. + if attestation_slot > state.slot { + Ok(AttestationProcessingOutcome::AttestsToFutureState { + state: state.slot, + attestation: attestation_slot, + }) + } else { + self.process_attestation_for_state_and_block( + attestation, + &state, + &attestation_head_block, + ) + } + } else { + // Drop any attestation where we have not processed `attestation.data.beacon_block_root`. + // + // This is likely overly restrictive, we could store the attestation for later + // processing. + warn!( + self.log, + "Dropped attestation for unknown block"; + "block" => format!("{}", attestation.data.beacon_block_root) + ); + Ok(AttestationProcessingOutcome::UnknownHeadBlock { + beacon_block_root: attestation.data.beacon_block_root, + }) + } + } + + /// Verifies the `attestation` against the `state` to which it is attesting. + /// + /// Updates fork choice with any new latest messages, but _does not_ find or update the head. + /// + /// ## Notes + /// + /// The given `state` must fulfil one of the following conditions: + /// + /// - `state` corresponds to the `block.state_root` identified by + /// `attestation.data.beacon_block_root`. (Viz., `attestation` was created using `state`). + /// - `state.slot` is in the same epoch as `data.target.epoch` and + /// `attestation.data.beacon_block_root` is in the history of `state`. + /// + /// Additionally, `attestation.data.beacon_block_root` **must** be available to read in + /// `self.store` _and_ be the root of the given `block`. + /// + /// If the given conditions are not fulfilled, the function may error or provide a false + /// negative (indicating that a given `attestation` is invalid when it is was validly formed). + fn process_attestation_for_state_and_block( + &self, + attestation: Attestation, + state: &BeaconState, + block: &BeaconBlock, + ) -> Result { self.metrics.attestation_processing_requests.inc(); let timer = self.metrics.attestation_processing_times.start_timer(); - let result = self - .op_pool - .insert_attestation(attestation, &*self.state.read(), &self.spec); + // Find the highest between: + // + // - The highest valid finalized epoch we've ever seen (i.e., the head). + // - The finalized epoch that this attestation was created against. + let finalized_epoch = std::cmp::max( + self.head().beacon_state.finalized_checkpoint.epoch, + state.finalized_checkpoint.epoch, + ); + + let result = if block.slot <= finalized_epoch.start_slot(T::EthSpec::slots_per_epoch()) { + // Ignore any attestation where the slot of `data.beacon_block_root` is equal to or + // prior to the finalized epoch. + // + // For any valid attestation if the `beacon_block_root` is prior to finalization, then + // all other parameters (source, target, etc) must all be prior to finalization and + // therefore no longer interesting. + Ok(AttestationProcessingOutcome::FinalizedSlot { + attestation: block.slot.epoch(T::EthSpec::slots_per_epoch()), + finalized: finalized_epoch, + }) + } else if let Err(e) = + verify_attestation_for_state(state, &attestation, &self.spec, VerifySignatures::True) + { + warn!( + self.log, + "Invalid attestation"; + "state_epoch" => state.current_epoch(), + "error" => format!("{:?}", e), + ); + + Ok(AttestationProcessingOutcome::Invalid(e)) + } else { + // Provide the attestation to fork choice, updating the validator latest messages but + // _without_ finding and updating the head. + self.fork_choice + .process_attestation(&state, &attestation, block)?; + + // Provide the valid attestation to op pool, which may choose to retain the + // attestation for inclusion in a future block. + self.op_pool + .insert_attestation(attestation, state, &self.spec)?; + + // Update the metrics. + self.metrics.attestation_processing_successes.inc(); + + Ok(AttestationProcessingOutcome::Processed) + }; timer.observe_duration(); - if result.is_ok() { - self.metrics.attestation_processing_successes.inc(); - } - - // TODO: process attestation. Please consider: - // - // - Because a block was not added to the op pool does not mean it's invalid (it might - // just be old). - // - The attestation should be rejected if we don't know the block (ideally it should be - // queued, but this may be overkill). - // - The attestation _must_ be validated against it's state before being added to fork - // choice. - // - You can avoid verifying some attestations by first checking if they're a latest - // message. This would involve expanding the `LmdGhost` API. - result } @@ -612,7 +804,7 @@ impl BeaconChain { return Ok(BlockProcessingOutcome::GenesisBlock); } - let block_root = block.block_header().canonical_root(); + let block_root = block.canonical_root(); if block_root == self.genesis_block_root { return Ok(BlockProcessingOutcome::GenesisBlock); @@ -658,6 +850,7 @@ impl BeaconChain { per_slot_processing(&mut state, &self.spec)?; } + state.build_committee_cache(RelativeEpoch::Previous, &self.spec)?; state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; // Apply the received block to its parent state (which has been transitioned into this diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 0d619d7f2d..7a51fc4258 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -1,5 +1,8 @@ use crate::fork_choice::Error as ForkChoiceError; use crate::metrics::Error as MetricsError; +use state_processing::per_block_processing::errors::{ + AttestationValidationError, IndexedAttestationValidationError, +}; use state_processing::BlockProcessingError; use state_processing::SlotProcessingError; use types::*; @@ -23,6 +26,7 @@ pub enum BeaconChainError { previous_epoch: Epoch, new_epoch: Epoch, }, + UnableToFindTargetRoot(Slot), BeaconStateError(BeaconStateError), DBInconsistent(String), DBError(store::Error), @@ -31,6 +35,11 @@ pub enum BeaconChainError { MissingBeaconState(Hash256), SlotProcessingError(SlotProcessingError), MetricsError(String), + NoStateForAttestation { + beacon_block_root: Hash256, + }, + AttestationValidationError(AttestationValidationError), + IndexedAttestationValidationError(IndexedAttestationValidationError), } easy_from_to!(SlotProcessingError, BeaconChainError); @@ -53,3 +62,5 @@ pub enum BlockProductionError { easy_from_to!(BlockProcessingError, BlockProductionError); easy_from_to!(BeaconStateError, BlockProductionError); easy_from_to!(SlotProcessingError, BlockProductionError); +easy_from_to!(AttestationValidationError, BeaconChainError); +easy_from_to!(IndexedAttestationValidationError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/fork_choice.rs b/beacon_node/beacon_chain/src/fork_choice.rs index 74778be32e..edd426f296 100644 --- a/beacon_node/beacon_chain/src/fork_choice.rs +++ b/beacon_node/beacon_chain/src/fork_choice.rs @@ -3,7 +3,9 @@ use lmd_ghost::LmdGhost; use state_processing::common::get_attesting_indices; use std::sync::Arc; use store::{Error as StoreError, Store}; -use types::{Attestation, BeaconBlock, BeaconState, BeaconStateError, Epoch, EthSpec, Hash256}; +use types::{ + Attestation, BeaconBlock, BeaconState, BeaconStateError, Epoch, EthSpec, Hash256, Slot, +}; type Result = std::result::Result; @@ -17,8 +19,8 @@ pub enum Error { } pub struct ForkChoice { - backend: T::LmdGhost, store: Arc, + backend: T::LmdGhost, /// Used for resolving the `0x00..00` alias back to genesis. /// /// Does not necessarily need to be the _actual_ genesis, it suffices to be the finalized root @@ -117,18 +119,34 @@ impl ForkChoice { // // https://github.com/ethereum/eth2.0-specs/blob/v0.7.0/specs/core/0_fork-choice.md for attestation in &block.body.attestations { - self.process_attestation_from_block(state, attestation)?; + // If the `data.beacon_block_root` block is not known to us, simply ignore the latest + // vote. + if let Some(block) = self + .store + .get::>(&attestation.data.beacon_block_root)? + { + self.process_attestation(state, attestation, &block)?; + } } + // This does not apply a vote to the block, it just makes fork choice aware of the block so + // it can still be identified as the head even if it doesn't have any votes. + // + // A case where a block without any votes can be the head is where it is the only child of + // a block that has the majority of votes applied to it. self.backend.process_block(block, block_root)?; Ok(()) } - fn process_attestation_from_block( + /// Process an attestation which references `block` in `attestation.data.beacon_block_root`. + /// + /// Assumes the attestation is valid. + pub fn process_attestation( &self, state: &BeaconState, attestation: &Attestation, + block: &BeaconBlock, ) -> Result<()> { let block_hash = attestation.data.beacon_block_root; @@ -147,26 +165,26 @@ impl ForkChoice { // to genesis just by being present in the chain. // // Additionally, don't add any block hash to fork choice unless we have imported the block. - if block_hash != Hash256::zero() - && self - .store - .exists::>(&block_hash) - .unwrap_or(false) - { + if block_hash != Hash256::zero() { let validator_indices = get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?; - let block_slot = state.get_attestation_data_slot(&attestation.data)?; - for validator_index in validator_indices { self.backend - .process_attestation(validator_index, block_hash, block_slot)?; + .process_attestation(validator_index, block_hash, block.slot)?; } } Ok(()) } + /// Returns the latest message for a given validator, if any. + /// + /// Returns `(block_root, block_slot)`. + pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> { + self.backend.latest_message(validator_index) + } + /// Inform the fork choice that the given block (and corresponding root) have been finalized so /// it may prune it's storage. /// diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c2efcad130..3188760a42 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -7,7 +7,9 @@ mod metrics; mod persisted_beacon_chain; pub mod test_utils; -pub use self::beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; +pub use self::beacon_chain::{ + AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BlockProcessingOutcome, +}; pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; pub use lmd_ghost; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index cdcd8bb21e..298c637dbd 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -84,14 +84,30 @@ where { /// Instantiate a new harness with `validator_count` initial validators. pub fn new(validator_count: usize) -> Self { + let state_builder = TestingBeaconStateBuilder::from_default_keypairs_file_if_exists( + validator_count, + &E::default_spec(), + ); + let (genesis_state, keypairs) = state_builder.build(); + + Self::from_state_and_keypairs(genesis_state, keypairs) + } + + /// Instantiate a new harness with an initial validator for each key supplied. + pub fn from_keypairs(keypairs: Vec) -> Self { + let state_builder = TestingBeaconStateBuilder::from_keypairs(keypairs, &E::default_spec()); + let (genesis_state, keypairs) = state_builder.build(); + + Self::from_state_and_keypairs(genesis_state, keypairs) + } + + /// Instantiate a new harness with the given genesis state and a keypair for each of the + /// initial validators in the given state. + pub fn from_state_and_keypairs(genesis_state: BeaconState, keypairs: Vec) -> Self { let spec = E::default_spec(); let store = Arc::new(MemoryStore::open()); - let state_builder = - TestingBeaconStateBuilder::from_default_keypairs_file_if_exists(validator_count, &spec); - let (genesis_state, keypairs) = state_builder.build(); - let mut genesis_block = BeaconBlock::empty(&spec); genesis_block.state_root = Hash256::from_slice(&genesis_state.tree_hash_root()); @@ -178,12 +194,7 @@ where if let BlockProcessingOutcome::Processed { block_root } = outcome { head_block_root = Some(block_root); - self.add_attestations_to_op_pool( - &attestation_strategy, - &new_state, - block_root, - slot, - ); + self.add_free_attestations(&attestation_strategy, &new_state, block_root, slot); } else { panic!("block should be successfully processed: {:?}", outcome); } @@ -198,7 +209,7 @@ where fn get_state_at_slot(&self, state_slot: Slot) -> BeaconState { let state_root = self .chain - .rev_iter_state_roots(self.chain.head().beacon_state.slot - 1) + .rev_iter_state_roots() .find(|(_hash, slot)| *slot == state_slot) .map(|(hash, _slot)| hash) .expect("could not find state root"); @@ -263,16 +274,38 @@ where (block, state) } - /// Adds attestations to the `BeaconChain` operations pool to be included in future blocks. + /// Adds attestations to the `BeaconChain` operations pool and fork choice. /// /// The `attestation_strategy` dictates which validators should attest. - fn add_attestations_to_op_pool( + fn add_free_attestations( &self, attestation_strategy: &AttestationStrategy, state: &BeaconState, head_block_root: Hash256, head_block_slot: Slot, ) { + self.get_free_attestations( + attestation_strategy, + state, + head_block_root, + head_block_slot, + ) + .into_iter() + .for_each(|attestation| { + self.chain + .process_attestation(attestation) + .expect("should process attestation"); + }); + } + + /// Generates a `Vec` for some attestation strategy and head_block. + pub fn get_free_attestations( + &self, + attestation_strategy: &AttestationStrategy, + state: &BeaconState, + head_block_root: Hash256, + head_block_slot: Slot, + ) -> Vec> { let spec = &self.spec; let fork = &state.fork; @@ -281,6 +314,8 @@ where AttestationStrategy::SomeValidators(vec) => vec.clone(), }; + let mut vec = vec![]; + state .get_crosslink_committees_at_slot(state.slot) .expect("should get committees") @@ -326,19 +361,17 @@ where agg_sig }; - let attestation = Attestation { + vec.push(Attestation { aggregation_bits, data, custody_bits, signature, - }; - - self.chain - .process_attestation(attestation) - .expect("should process attestation"); + }) } } }); + + vec } /// Creates two forks: diff --git a/beacon_node/beacon_chain/tests/tests.rs b/beacon_node/beacon_chain/tests/tests.rs index babdbe5e1f..22b667f159 100644 --- a/beacon_node/beacon_chain/tests/tests.rs +++ b/beacon_node/beacon_chain/tests/tests.rs @@ -1,29 +1,104 @@ #![cfg(not(debug_assertions))] +#[macro_use] +extern crate lazy_static; + use beacon_chain::test_utils::{ AttestationStrategy, BeaconChainHarness, BlockStrategy, CommonTypes, PersistedBeaconChain, BEACON_CHAIN_DB_KEY, }; +use beacon_chain::AttestationProcessingOutcome; use lmd_ghost::ThreadSafeReducedTree; use rand::Rng; use store::{MemoryStore, Store}; use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; -use types::{Deposit, EthSpec, Hash256, MinimalEthSpec, Slot}; +use types::{Deposit, EthSpec, Hash256, Keypair, MinimalEthSpec, RelativeEpoch, Slot}; // Should ideally be divisible by 3. pub const VALIDATOR_COUNT: usize = 24; +lazy_static! { + /// A cached set of keys. + static ref KEYPAIRS: Vec = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT); +} + type TestForkChoice = ThreadSafeReducedTree; fn get_harness(validator_count: usize) -> BeaconChainHarness { - let harness = BeaconChainHarness::new(validator_count); + let harness = BeaconChainHarness::from_keypairs(KEYPAIRS[0..validator_count].to_vec()); - // Move past the zero slot. harness.advance_slot(); harness } +#[test] +fn iterators() { + let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1; + + let harness = get_harness(VALIDATOR_COUNT); + + harness.extend_chain( + num_blocks_produced as usize, + BlockStrategy::OnCanonicalHead, + // No need to produce attestations for this test. + AttestationStrategy::SomeValidators(vec![]), + ); + + let block_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_block_roots().collect(); + let state_roots: Vec<(Hash256, Slot)> = harness.chain.rev_iter_state_roots().collect(); + + assert_eq!( + block_roots.len(), + state_roots.len(), + "should be an equal amount of block and state roots" + ); + + assert!( + block_roots.iter().any(|(_root, slot)| *slot == 0), + "should contain genesis block root" + ); + assert!( + state_roots.iter().any(|(_root, slot)| *slot == 0), + "should contain genesis state root" + ); + + assert_eq!( + block_roots.len(), + num_blocks_produced as usize + 1, + "should contain all produced blocks, plus the genesis block" + ); + + block_roots.windows(2).for_each(|x| { + assert_eq!( + x[1].1, + x[0].1 - 1, + "block root slots should be decreasing by one" + ) + }); + state_roots.windows(2).for_each(|x| { + assert_eq!( + x[1].1, + x[0].1 - 1, + "state root slots should be decreasing by one" + ) + }); + + let head = &harness.chain.head(); + + assert_eq!( + *block_roots.first().expect("should have some block roots"), + (head.beacon_block_root, head.beacon_block.slot), + "first block root and slot should be for the head block" + ); + + assert_eq!( + *state_roots.first().expect("should have some state roots"), + (head.beacon_state_root, head.beacon_state.slot), + "first state root and slot should be for the head state" + ); +} + #[test] fn chooses_fork() { let harness = get_harness(VALIDATOR_COUNT); @@ -251,3 +326,136 @@ fn roundtrip_operation_pool() { assert_eq!(harness.chain.op_pool, restored_op_pool); } + +#[test] +fn free_attestations_added_to_fork_choice_some_none() { + let num_blocks_produced = MinimalEthSpec::slots_per_epoch() / 2; + + let harness = get_harness(VALIDATOR_COUNT); + + harness.extend_chain( + num_blocks_produced as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ); + + let state = &harness.chain.head().beacon_state; + let fork_choice = &harness.chain.fork_choice; + + let validator_slots: Vec<(usize, Slot)> = (0..VALIDATOR_COUNT) + .into_iter() + .map(|validator_index| { + let slot = state + .get_attestation_duties(validator_index, RelativeEpoch::Current) + .expect("should get attester duties") + .unwrap() + .slot; + + (validator_index, slot) + }) + .collect(); + + for (validator, slot) in validator_slots.clone() { + let latest_message = fork_choice.latest_message(validator); + + if slot <= num_blocks_produced && slot != 0 { + assert_eq!( + latest_message.unwrap().1, + slot, + "Latest message slot for {} should be equal to slot {}.", + validator, + slot + ) + } else { + assert!( + latest_message.is_none(), + "Latest message slot should be None." + ) + } + } +} + +#[test] +fn attestations_with_increasing_slots() { + let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 5; + + let harness = get_harness(VALIDATOR_COUNT); + + let mut attestations = vec![]; + + for _ in 0..num_blocks_produced { + harness.extend_chain( + 2, + BlockStrategy::OnCanonicalHead, + // Don't produce & include any attestations (we'll collect them later). + AttestationStrategy::SomeValidators(vec![]), + ); + + attestations.append(&mut harness.get_free_attestations( + &AttestationStrategy::AllValidators, + &harness.chain.head().beacon_state, + harness.chain.head().beacon_block_root, + harness.chain.head().beacon_block.slot, + )); + + harness.advance_slot(); + } + + for attestation in attestations { + assert_eq!( + harness.chain.process_attestation(attestation), + Ok(AttestationProcessingOutcome::Processed) + ) + } +} + +#[test] +fn free_attestations_added_to_fork_choice_all_updated() { + let num_blocks_produced = MinimalEthSpec::slots_per_epoch() * 2 - 1; + + let harness = get_harness(VALIDATOR_COUNT); + + harness.extend_chain( + num_blocks_produced as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ); + + let state = &harness.chain.head().beacon_state; + let fork_choice = &harness.chain.fork_choice; + + let validators: Vec = (0..VALIDATOR_COUNT).collect(); + let slots: Vec = validators + .iter() + .map(|&v| { + state + .get_attestation_duties(v, RelativeEpoch::Current) + .expect("should get attester duties") + .unwrap() + .slot + }) + .collect(); + let validator_slots: Vec<(&usize, Slot)> = validators.iter().zip(slots).collect(); + + for (validator, slot) in validator_slots { + let latest_message = fork_choice.latest_message(*validator); + + assert_eq!( + latest_message.unwrap().1, + slot, + "Latest message slot should be equal to attester duty." + ); + + if slot != num_blocks_produced { + let block_root = state + .get_block_root(slot) + .expect("Should get block root at slot"); + + assert_eq!( + latest_message.unwrap().0, + *block_root, + "Latest message block root should be equal to block at slot." + ); + } + } +} diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index c3271888a8..3cf43b7a06 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -266,8 +266,7 @@ impl SimpleSync { fn root_at_slot(&self, target_slot: Slot) -> Option { self.chain - .rev_iter_block_roots(target_slot) - .take(1) + .rev_iter_block_roots() .find(|(_root, slot)| *slot == target_slot) .map(|(root, _slot)| root) } @@ -280,8 +279,6 @@ impl SimpleSync { req: BeaconBlockRootsRequest, network: &mut NetworkContext, ) { - let state = &self.chain.head().beacon_state; - debug!( self.log, "BlockRootsRequest"; @@ -292,8 +289,9 @@ impl SimpleSync { let mut roots: Vec = self .chain - .rev_iter_block_roots(std::cmp::min(req.start_slot + req.count, state.slot)) + .rev_iter_block_roots() .take_while(|(_root, slot)| req.start_slot <= *slot) + .filter(|(_root, slot)| *slot < req.start_slot + req.count) .map(|(block_root, slot)| BlockRootSlot { slot, block_root }) .collect(); @@ -391,8 +389,6 @@ impl SimpleSync { req: BeaconBlockHeadersRequest, network: &mut NetworkContext, ) { - let state = &self.chain.head().beacon_state; - debug!( self.log, "BlockHeadersRequest"; @@ -405,8 +401,9 @@ impl SimpleSync { // Collect the block roots. let mut roots: Vec = self .chain - .rev_iter_block_roots(std::cmp::min(req.start_slot + count, state.slot)) + .rev_iter_block_roots() .take_while(|(_root, slot)| req.start_slot <= *slot) + .filter(|(_root, slot)| *slot < req.start_slot + count) .map(|(root, _slot)| root) .collect(); @@ -631,7 +628,12 @@ impl SimpleSync { _network: &mut NetworkContext, ) { match self.chain.process_attestation(msg) { - Ok(()) => info!(self.log, "ImportedAttestation"; "source" => "gossip"), + Ok(outcome) => info!( + self.log, + "Processed attestation"; + "source" => "gossip", + "outcome" => format!("{:?}", outcome) + ), Err(e) => { warn!(self.log, "InvalidAttestation"; "source" => "gossip", "error" => format!("{:?}", e)) } diff --git a/beacon_node/rpc/src/attestation.rs b/beacon_node/rpc/src/attestation.rs index 1f507f0f3f..f442e247dd 100644 --- a/beacon_node/rpc/src/attestation.rs +++ b/beacon_node/rpc/src/attestation.rs @@ -1,4 +1,4 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes}; +use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes}; use eth2_libp2p::PubsubMessage; use eth2_libp2p::Topic; use eth2_libp2p::BEACON_ATTESTATION_TOPIC; @@ -163,7 +163,7 @@ impl AttestationService for AttestationServiceInstance { resp.set_success(true); } - Err(e) => { + Err(BeaconChainError::AttestationValidationError(e)) => { // Attestation was invalid warn!( self.log, @@ -174,6 +174,36 @@ impl AttestationService for AttestationServiceInstance { resp.set_success(false); resp.set_msg(format!("InvalidAttestation: {:?}", e).as_bytes().to_vec()); } + Err(BeaconChainError::IndexedAttestationValidationError(e)) => { + // Indexed attestation was invalid + warn!( + self.log, + "PublishAttestation"; + "type" => "invalid_attestation", + "error" => format!("{:?}", e), + ); + resp.set_success(false); + resp.set_msg( + format!("InvalidIndexedAttestation: {:?}", e) + .as_bytes() + .to_vec(), + ); + } + Err(e) => { + // Some other error + warn!( + self.log, + "PublishAttestation"; + "type" => "beacon_chain_error", + "error" => format!("{:?}", e), + ); + resp.set_success(false); + resp.set_msg( + format!("There was a beacon chain error: {:?}", e) + .as_bytes() + .to_vec(), + ); + } }; let error_log = self.log.clone(); diff --git a/beacon_node/store/src/iter.rs b/beacon_node/store/src/iter.rs index 4e47fceb22..c97241903f 100644 --- a/beacon_node/store/src/iter.rs +++ b/beacon_node/store/src/iter.rs @@ -15,21 +15,21 @@ pub trait AncestorIter { } impl<'a, U: Store, E: EthSpec> AncestorIter> for BeaconBlock { - /// Iterates across all the prior block roots of `self`, starting at the most recent and ending + /// Iterates across all available prior block roots of `self`, starting at the most recent and ending /// at genesis. fn try_iter_ancestor_roots(&self, store: Arc) -> Option> { let state = store.get::>(&self.state_root).ok()??; - Some(BlockRootsIterator::owned(store, state, self.slot)) + Some(BlockRootsIterator::owned(store, state)) } } impl<'a, U: Store, E: EthSpec> AncestorIter> for BeaconState { - /// Iterates across all the prior state roots of `self`, starting at the most recent and ending + /// Iterates across all available prior state roots of `self`, starting at the most recent and ending /// at genesis. fn try_iter_ancestor_roots(&self, store: Arc) -> Option> { // The `self.clone()` here is wasteful. - Some(StateRootsIterator::owned(store, self.clone(), self.slot)) + Some(StateRootsIterator::owned(store, self.clone())) } } @@ -41,19 +41,19 @@ pub struct StateRootsIterator<'a, T: EthSpec, U> { } impl<'a, T: EthSpec, U: Store> StateRootsIterator<'a, T, U> { - pub fn new(store: Arc, beacon_state: &'a BeaconState, start_slot: Slot) -> Self { + pub fn new(store: Arc, beacon_state: &'a BeaconState) -> Self { Self { store, + slot: beacon_state.slot, beacon_state: Cow::Borrowed(beacon_state), - slot: start_slot + 1, } } - pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { + pub fn owned(store: Arc, beacon_state: BeaconState) -> Self { Self { store, + slot: beacon_state.slot, beacon_state: Cow::Owned(beacon_state), - slot: start_slot + 1, } } } @@ -97,16 +97,16 @@ pub struct BlockIterator<'a, T: EthSpec, U> { impl<'a, T: EthSpec, U: Store> BlockIterator<'a, T, U> { /// Create a new iterator over all blocks in the given `beacon_state` and prior states. - pub fn new(store: Arc, beacon_state: &'a BeaconState, start_slot: Slot) -> Self { + pub fn new(store: Arc, beacon_state: &'a BeaconState) -> Self { Self { - roots: BlockRootsIterator::new(store, beacon_state, start_slot), + roots: BlockRootsIterator::new(store, beacon_state), } } /// Create a new iterator over all blocks in the given `beacon_state` and prior states. - pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { + pub fn owned(store: Arc, beacon_state: BeaconState) -> Self { Self { - roots: BlockRootsIterator::owned(store, beacon_state, start_slot), + roots: BlockRootsIterator::owned(store, beacon_state), } } } @@ -137,20 +137,20 @@ pub struct BlockRootsIterator<'a, T: EthSpec, U> { impl<'a, T: EthSpec, U: Store> BlockRootsIterator<'a, T, U> { /// Create a new iterator over all block roots in the given `beacon_state` and prior states. - pub fn new(store: Arc, beacon_state: &'a BeaconState, start_slot: Slot) -> Self { + pub fn new(store: Arc, beacon_state: &'a BeaconState) -> Self { Self { store, + slot: beacon_state.slot, beacon_state: Cow::Borrowed(beacon_state), - slot: start_slot + 1, } } /// Create a new iterator over all block roots in the given `beacon_state` and prior states. - pub fn owned(store: Arc, beacon_state: BeaconState, start_slot: Slot) -> Self { + pub fn owned(store: Arc, beacon_state: BeaconState) -> Self { Self { store, + slot: beacon_state.slot, beacon_state: Cow::Owned(beacon_state), - slot: start_slot + 1, } } } @@ -227,7 +227,7 @@ mod test { state_b.state_roots[0] = state_a_root; store.put(&state_a_root, &state_a).unwrap(); - let iter = BlockRootsIterator::new(store.clone(), &state_b, state_b.slot - 1); + let iter = BlockRootsIterator::new(store.clone(), &state_b); assert!( iter.clone().find(|(_root, slot)| *slot == 0).is_some(), @@ -276,7 +276,7 @@ mod test { store.put(&state_a_root, &state_a).unwrap(); store.put(&state_b_root, &state_b).unwrap(); - let iter = StateRootsIterator::new(store.clone(), &state_b, state_b.slot - 1); + let iter = StateRootsIterator::new(store.clone(), &state_b); assert!( iter.clone().find(|(_root, slot)| *slot == 0).is_some(), diff --git a/eth2/lmd_ghost/src/lib.rs b/eth2/lmd_ghost/src/lib.rs index de9bdd8605..95cd0679c1 100644 --- a/eth2/lmd_ghost/src/lib.rs +++ b/eth2/lmd_ghost/src/lib.rs @@ -43,4 +43,7 @@ pub trait LmdGhost: Send + Sync { finalized_block: &BeaconBlock, finalized_block_root: Hash256, ) -> Result<()>; + + /// Returns the latest message for a given validator index. + fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)>; } diff --git a/eth2/lmd_ghost/src/reduced_tree.rs b/eth2/lmd_ghost/src/reduced_tree.rs index a3cf4e1051..deda02e1fd 100644 --- a/eth2/lmd_ghost/src/reduced_tree.rs +++ b/eth2/lmd_ghost/src/reduced_tree.rs @@ -109,6 +109,10 @@ where .update_root(new_block.slot, new_root) .map_err(|e| format!("update_finalized_root failed: {:?}", e)) } + + fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> { + self.core.read().latest_message(validator_index) + } } struct ReducedTree { @@ -254,6 +258,13 @@ where Ok(head_node.block_hash) } + pub fn latest_message(&self, validator_index: usize) -> Option<(Hash256, Slot)> { + match self.latest_votes.get_ref(validator_index) { + Some(Some(v)) => Some((v.hash.clone(), v.slot.clone())), + _ => None, + } + } + fn find_head_from<'a>(&'a self, start_node: &'a Node) -> Result<&'a Node> { if start_node.does_not_have_children() { Ok(start_node) @@ -600,11 +611,7 @@ where let block = self.get_block(child)?; let state = self.get_state(block.state_root)?; - Ok(BlockRootsIterator::owned( - self.store.clone(), - state, - block.slot - 1, - )) + Ok(BlockRootsIterator::owned(self.store.clone(), state)) } /// Verify the integrity of `self`. Returns `Ok(())` if the tree has integrity, otherwise returns `Err(description)`. @@ -769,6 +776,10 @@ where &self.0[i] } + pub fn get_ref(&self, i: usize) -> Option<&T> { + self.0.get(i) + } + pub fn insert(&mut self, i: usize, element: T) { self.ensure(i); self.0[i] = element; diff --git a/eth2/operation_pool/src/lib.rs b/eth2/operation_pool/src/lib.rs index 92d5fb1683..ba9ca81c07 100644 --- a/eth2/operation_pool/src/lib.rs +++ b/eth2/operation_pool/src/lib.rs @@ -15,9 +15,10 @@ use state_processing::per_block_processing::errors::{ ExitValidationError, ProposerSlashingValidationError, TransferValidationError, }; use state_processing::per_block_processing::{ - get_slashable_indices_modular, verify_attestation, verify_attestation_time_independent_only, + get_slashable_indices_modular, verify_attestation_for_block_inclusion, verify_attester_slashing, verify_exit, verify_exit_time_independent_only, verify_proposer_slashing, verify_transfer, verify_transfer_time_independent_only, + VerifySignatures, }; use std::collections::{btree_map::Entry, hash_map, BTreeMap, HashMap, HashSet}; use std::marker::PhantomData; @@ -64,15 +65,16 @@ impl OperationPool { } /// Insert an attestation into the pool, aggregating it with existing attestations if possible. + /// + /// ## Note + /// + /// This function assumes the given `attestation` is valid. pub fn insert_attestation( &self, attestation: Attestation, state: &BeaconState, spec: &ChainSpec, ) -> Result<(), AttestationValidationError> { - // Check that attestation signatures are valid. - verify_attestation_time_independent_only(state, &attestation, spec)?; - let id = AttestationId::from_data(&attestation.data, state, spec); // Take a write lock on the attestations map. @@ -128,7 +130,15 @@ impl OperationPool { }) .flat_map(|(_, attestations)| attestations) // That are valid... - .filter(|attestation| verify_attestation(state, attestation, spec).is_ok()) + .filter(|attestation| { + verify_attestation_for_block_inclusion( + state, + attestation, + spec, + VerifySignatures::True, + ) + .is_ok() + }) .map(|att| AttMaxCover::new(att, earliest_attestation_validators(att, state))); maximum_cover(valid_attestations, T::MaxAttestations::to_usize()) diff --git a/eth2/state_processing/src/per_block_processing.rs b/eth2/state_processing/src/per_block_processing.rs index 3c89215550..a64158ac98 100644 --- a/eth2/state_processing/src/per_block_processing.rs +++ b/eth2/state_processing/src/per_block_processing.rs @@ -15,8 +15,7 @@ pub use is_valid_indexed_attestation::{ is_valid_indexed_attestation, is_valid_indexed_attestation_without_signature, }; pub use verify_attestation::{ - verify_attestation, verify_attestation_time_independent_only, - verify_attestation_without_signature, + verify_attestation_for_block_inclusion, verify_attestation_for_state, }; pub use verify_deposit::{ get_existing_validator_index, verify_deposit_merkle_proof, verify_deposit_signature, @@ -37,6 +36,12 @@ mod verify_exit; mod verify_proposer_slashing; mod verify_transfer; +#[derive(PartialEq)] +pub enum VerifySignatures { + True, + False, +} + /// Updates the state for a new block, whilst validating that the block is valid. /// /// Returns `Ok(())` if the block is valid and the state was successfully updated. Otherwise @@ -312,7 +317,8 @@ pub fn process_attestations( .par_iter() .enumerate() .try_for_each(|(i, attestation)| { - verify_attestation(state, attestation, spec).map_err(|e| e.into_with_index(i)) + verify_attestation_for_block_inclusion(state, attestation, spec, VerifySignatures::True) + .map_err(|e| e.into_with_index(i)) })?; // Update the state in series. diff --git a/eth2/state_processing/src/per_block_processing/verify_attestation.rs b/eth2/state_processing/src/per_block_processing/verify_attestation.rs index af25300457..74dbefa232 100644 --- a/eth2/state_processing/src/per_block_processing/verify_attestation.rs +++ b/eth2/state_processing/src/per_block_processing/verify_attestation.rs @@ -1,4 +1,5 @@ use super::errors::{AttestationInvalid as Invalid, AttestationValidationError as Error}; +use super::VerifySignatures; use crate::common::get_indexed_attestation; use crate::per_block_processing::{ is_valid_indexed_attestation, is_valid_indexed_attestation_without_signature, @@ -6,67 +7,25 @@ use crate::per_block_processing::{ use tree_hash::TreeHash; use types::*; -/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the -/// given state. +/// Returns `Ok(())` if the given `attestation` is valid to be included in a block that is applied +/// to `state`. Otherwise, returns a descriptive `Err`. /// -/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. +/// Optionally verifies the aggregate signature, depending on `verify_signatures`. /// /// Spec v0.8.0 -pub fn verify_attestation( +pub fn verify_attestation_for_block_inclusion( state: &BeaconState, attestation: &Attestation, spec: &ChainSpec, -) -> Result<(), Error> { - verify_attestation_parametric(state, attestation, spec, true, false) -} - -/// Like `verify_attestation` but doesn't run checks which may become true in future states. -pub fn verify_attestation_time_independent_only( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), Error> { - verify_attestation_parametric(state, attestation, spec, true, true) -} - -/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the -/// given state, without validating the aggregate signature. -/// -/// Returns `Ok(())` if the `Attestation` is valid, otherwise indicates the reason for invalidity. -/// -/// Spec v0.8.0 -pub fn verify_attestation_without_signature( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, -) -> Result<(), Error> { - verify_attestation_parametric(state, attestation, spec, false, false) -} - -/// Indicates if an `Attestation` is valid to be included in a block in the current epoch of the -/// given state, optionally validating the aggregate signature. -/// -/// -/// Spec v0.8.0 -fn verify_attestation_parametric( - state: &BeaconState, - attestation: &Attestation, - spec: &ChainSpec, - verify_signature: bool, - time_independent_only: bool, + verify_signatures: VerifySignatures, ) -> Result<(), Error> { let data = &attestation.data; - verify!( - data.crosslink.shard < T::ShardCount::to_u64(), - Invalid::BadShard - ); // Check attestation slot. let attestation_slot = state.get_attestation_data_slot(&data)?; verify!( - time_independent_only - || attestation_slot + spec.min_attestation_inclusion_delay <= state.slot, + attestation_slot + spec.min_attestation_inclusion_delay <= state.slot, Invalid::IncludedTooEarly { state: state.slot, delay: spec.min_attestation_inclusion_delay, @@ -81,27 +40,47 @@ fn verify_attestation_parametric( } ); - // Verify the Casper FFG vote and crosslink data. - if !time_independent_only { - let parent_crosslink = verify_casper_ffg_vote(attestation, state)?; + verify_attestation_for_state(state, attestation, spec, verify_signatures) +} - verify!( - data.crosslink.parent_root == Hash256::from_slice(&parent_crosslink.tree_hash_root()), - Invalid::BadParentCrosslinkHash - ); - verify!( - data.crosslink.start_epoch == parent_crosslink.end_epoch, - Invalid::BadParentCrosslinkStartEpoch - ); - verify!( - data.crosslink.end_epoch - == std::cmp::min( - data.target.epoch, - parent_crosslink.end_epoch + spec.max_epochs_per_crosslink - ), - Invalid::BadParentCrosslinkEndEpoch - ); - } +/// Returns `Ok(())` if `attestation` is a valid attestation to the chain that precedes the given +/// `state`. +/// +/// Returns a descriptive `Err` if the attestation is malformed or does not accurately reflect the +/// prior blocks in `state`. +/// +/// Spec v0.8.0 +pub fn verify_attestation_for_state( + state: &BeaconState, + attestation: &Attestation, + spec: &ChainSpec, + verify_signature: VerifySignatures, +) -> Result<(), Error> { + let data = &attestation.data; + verify!( + data.crosslink.shard < T::ShardCount::to_u64(), + Invalid::BadShard + ); + + // Verify the Casper FFG vote and crosslink data. + let parent_crosslink = verify_casper_ffg_vote(attestation, state)?; + + verify!( + data.crosslink.parent_root == Hash256::from_slice(&parent_crosslink.tree_hash_root()), + Invalid::BadParentCrosslinkHash + ); + verify!( + data.crosslink.start_epoch == parent_crosslink.end_epoch, + Invalid::BadParentCrosslinkStartEpoch + ); + verify!( + data.crosslink.end_epoch + == std::cmp::min( + data.target.epoch, + parent_crosslink.end_epoch + spec.max_epochs_per_crosslink + ), + Invalid::BadParentCrosslinkEndEpoch + ); // Crosslink data root is zero (to be removed in phase 1). verify!( @@ -111,7 +90,7 @@ fn verify_attestation_parametric( // Check signature and bitfields let indexed_attestation = get_indexed_attestation(state, attestation)?; - if verify_signature { + if verify_signature == VerifySignatures::True { is_valid_indexed_attestation(state, &indexed_attestation, spec)?; } else { is_valid_indexed_attestation_without_signature(state, &indexed_attestation, spec)?; diff --git a/eth2/types/src/beacon_block.rs b/eth2/types/src/beacon_block.rs index 3f6ec92198..500bde6e43 100644 --- a/eth2/types/src/beacon_block.rs +++ b/eth2/types/src/beacon_block.rs @@ -61,6 +61,11 @@ impl BeaconBlock { } } + /// Returns the epoch corresponding to `self.slot`. + pub fn epoch(&self) -> Epoch { + self.slot.epoch(T::slots_per_epoch()) + } + /// Returns the `signed_root` of the block. /// /// Spec v0.8.1 From c4ced3e0d25f40e14c2f47a43285da4faf350a01 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 19 Aug 2019 21:02:34 +1000 Subject: [PATCH 2/3] Fix block processing blowup, upgrade metrics (#500) * Renamed fork_choice::process_attestation_from_block * Processing attestation in fork choice * Retrieving state from store and checking signature * Looser check on beacon state validity. * Cleaned up get_attestation_state * Expanded fork choice api to provide latest validator message. * Checking if the an attestation contains a latest message * Correct process_attestation error handling. * Copy paste error in comment fixed. * Tidy ancestor iterators * Getting attestation slot via helper method * Refactored attestation creation in test utils * Revert "Refactored attestation creation in test utils" This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306. * Integration tests for free attestation processing * Implicit conflicts resolved. * formatting * Do first pass on Grants code * Add another attestation processing test * Tidy attestation processing * Remove old code fragment * Add non-compiling half finished changes * Simplify, fix bugs, add tests for chain iters * Remove attestation processing from op pool * Fix bug with fork choice, tidy * Fix overly restrictive check in fork choice. * Ensure committee cache is build during attn proc * Ignore unknown blocks at fork choice * Various minor fixes * Make fork choice write lock in to read lock * Remove unused method * Tidy comments * Fix attestation prod. target roots change * Fix compile error in store iters * Reject any attestation prior to finalization * Begin metrics refactor * Move beacon_chain to new metrics structure. * Make metrics not panic if already defined * Use global prometheus gather at rest api * Unify common metric fns into a crate * Add heavy metering to block processing * Remove hypen from prometheus metric name * Add more beacon chain metrics * Add beacon chain persistence metric * Prune op pool on finalization * Add extra prom beacon chain metrics * Prefix BeaconChain metrics with "beacon_" * Add more store metrics * Add basic metrics to libp2p * Add metrics to HTTP server * Remove old `http_server` crate * Update metrics names to be more like standard * Fix broken beacon chain metrics, add slot clock metrics * Add lighthouse_metrics gather fn * Remove http args * Fix wrong state given to op pool prune * Make prom metric names more consistent * Add more metrics, tidy existing metrics * Fix store block read metrics * Tidy attestation metrics * Fix minor PR comments * Allow travis failures on beta (see desc) There's a non-backward compatible change in `cargo fmt`. Stable and beta do not agree. * Tidy `lighthouse_metrics` docs * Fix typo --- .travis.yml | 1 + Cargo.toml | 2 +- beacon_node/beacon_chain/Cargo.toml | 3 +- beacon_node/beacon_chain/src/beacon_chain.rs | 150 +++++-- beacon_node/beacon_chain/src/errors.rs | 8 - beacon_node/beacon_chain/src/fork_choice.rs | 20 +- beacon_node/beacon_chain/src/lib.rs | 5 + beacon_node/beacon_chain/src/metrics.rs | 411 ++++++++++++------ beacon_node/client/Cargo.toml | 1 - beacon_node/client/src/config.rs | 4 - beacon_node/client/src/lib.rs | 28 +- beacon_node/eth2-libp2p/Cargo.toml | 2 + beacon_node/eth2-libp2p/src/discovery.rs | 8 + beacon_node/eth2-libp2p/src/lib.rs | 4 + beacon_node/eth2-libp2p/src/metrics.rs | 20 + beacon_node/http_server/Cargo.toml | 23 - beacon_node/http_server/src/api.rs | 71 --- beacon_node/http_server/src/key.rs | 33 -- beacon_node/http_server/src/lib.rs | 145 ------ beacon_node/http_server/src/metrics.rs | 72 --- .../http_server/src/metrics/local_metrics.rs | 154 ------- beacon_node/rest_api/Cargo.toml | 4 + beacon_node/rest_api/src/config.rs | 2 +- beacon_node/rest_api/src/lib.rs | 37 +- beacon_node/rest_api/src/metrics.rs | 69 +++ beacon_node/src/main.rs | 22 - beacon_node/store/Cargo.toml | 2 + beacon_node/store/src/impls.rs | 20 +- beacon_node/store/src/impls/beacon_state.rs | 21 +- beacon_node/store/src/leveldb_store.rs | 22 +- beacon_node/store/src/lib.rs | 4 + beacon_node/store/src/metrics.rs | 106 +++++ docs/config_examples/beacon-node.toml | 10 +- eth2/utils/lighthouse_metrics/Cargo.toml | 11 + eth2/utils/lighthouse_metrics/src/lib.rs | 129 ++++++ eth2/utils/slot_clock/Cargo.toml | 2 + eth2/utils/slot_clock/src/lib.rs | 10 +- eth2/utils/slot_clock/src/metrics.rs | 32 ++ .../slot_clock/src/system_time_slot_clock.rs | 4 + .../slot_clock/src/testing_slot_clock.rs | 4 + 40 files changed, 912 insertions(+), 764 deletions(-) create mode 100644 beacon_node/eth2-libp2p/src/metrics.rs delete mode 100644 beacon_node/http_server/Cargo.toml delete mode 100644 beacon_node/http_server/src/api.rs delete mode 100644 beacon_node/http_server/src/key.rs delete mode 100644 beacon_node/http_server/src/lib.rs delete mode 100644 beacon_node/http_server/src/metrics.rs delete mode 100644 beacon_node/http_server/src/metrics/local_metrics.rs create mode 100644 beacon_node/rest_api/src/metrics.rs create mode 100644 beacon_node/store/src/metrics.rs create mode 100644 eth2/utils/lighthouse_metrics/Cargo.toml create mode 100644 eth2/utils/lighthouse_metrics/src/lib.rs create mode 100644 eth2/utils/slot_clock/src/metrics.rs diff --git a/.travis.yml b/.travis.yml index def7435a1f..b9754eb1eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ rust: - nightly matrix: allow_failures: + - rust: beta - rust: nightly fast_finish: true install: diff --git a/Cargo.toml b/Cargo.toml index f5ee02a173..f087539e6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ members = [ "eth2/utils/eth2_interop_keypairs", "eth2/utils/logging", "eth2/utils/eth2_hashing", + "eth2/utils/lighthouse_metrics", "eth2/utils/merkle_proof", "eth2/utils/int_to_bytes", "eth2/utils/serde_hex", @@ -25,7 +26,6 @@ members = [ "beacon_node", "beacon_node/store", "beacon_node/client", - "beacon_node/http_server", "beacon_node/rest_api", "beacon_node/network", "beacon_node/eth2-libp2p", diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 778224a3d4..1d3fc03b81 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -7,7 +7,8 @@ edition = "2018" [dependencies] store = { path = "../store" } parking_lot = "0.7" -prometheus = "^0.6" +lazy_static = "1.3.0" +lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } log = "0.4" operation_pool = { path = "../../eth2/operation_pool" } serde = "1.0" diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 96d3065306..e8e2f49dd2 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2,7 +2,7 @@ use crate::checkpoint::CheckPoint; use crate::errors::{BeaconChainError as Error, BlockProductionError}; use crate::fork_choice::{Error as ForkChoiceError, ForkChoice}; use crate::iter::{ReverseBlockRootIterator, ReverseStateRootIterator}; -use crate::metrics::Metrics; +use crate::metrics; use crate::persisted_beacon_chain::{PersistedBeaconChain, BEACON_CHAIN_DB_KEY}; use lmd_ghost::LmdGhost; use log::trace; @@ -106,8 +106,6 @@ pub struct BeaconChain { /// A state-machine that is updated with information from the network and chooses a canonical /// head block. pub fork_choice: ForkChoice, - /// Stores metrics about this `BeaconChain`. - pub metrics: Metrics, /// Logging to CLI, etc. log: Logger, } @@ -157,7 +155,6 @@ impl BeaconChain { canonical_head, genesis_block_root, fork_choice: ForkChoice::new(store.clone(), &genesis_block, genesis_block_root), - metrics: Metrics::new()?, store, log, }) @@ -195,7 +192,6 @@ impl BeaconChain { canonical_head: RwLock::new(p.canonical_head), state: RwLock::new(p.state), genesis_block_root: p.genesis_block_root, - metrics: Metrics::new()?, store, log, })) @@ -203,6 +199,8 @@ impl BeaconChain { /// Attempt to save this instance to `self.store`. pub fn persist(&self) -> Result<(), Error> { + let timer = metrics::start_timer(&metrics::PERSIST_CHAIN); + let p: PersistedBeaconChain = PersistedBeaconChain { canonical_head: self.canonical_head.read().clone(), op_pool: PersistedOperationPool::from_operation_pool(&self.op_pool), @@ -213,6 +211,8 @@ impl BeaconChain { let key = Hash256::from_slice(&BEACON_CHAIN_DB_KEY.as_bytes()); self.store.put(&key, &p)?; + metrics::stop_timer(timer); + Ok(()) } @@ -472,8 +472,8 @@ impl BeaconChain { state: &BeaconState, ) -> Result { // Collect some metrics. - self.metrics.attestation_production_requests.inc(); - let timer = self.metrics.attestation_production_times.start_timer(); + metrics::inc_counter(&metrics::ATTESTATION_PRODUCTION_REQUESTS); + let timer = metrics::start_timer(&metrics::ATTESTATION_PRODUCTION_TIMES); let slots_per_epoch = T::EthSpec::slots_per_epoch(); let current_epoch_start_slot = state.current_epoch().start_slot(slots_per_epoch); @@ -520,8 +520,8 @@ impl BeaconChain { }; // Collect some metrics. - self.metrics.attestation_production_successes.inc(); - timer.observe_duration(); + metrics::inc_counter(&metrics::ATTESTATION_PRODUCTION_SUCCESSES); + metrics::stop_timer(timer); Ok(AttestationData { beacon_block_root: head_block_root, @@ -547,11 +547,14 @@ impl BeaconChain { &self, attestation: Attestation, ) -> Result { + metrics::inc_counter(&metrics::ATTESTATION_PROCESSING_REQUESTS); + let timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_TIMES); + // From the store, load the attestation's "head block". // // An honest validator would have set this block to be the head of the chain (i.e., the // result of running fork choice). - if let Some(attestation_head_block) = self + let result = if let Some(attestation_head_block) = self .store .get::>(&attestation.data.beacon_block_root)? { @@ -657,7 +660,15 @@ impl BeaconChain { Ok(AttestationProcessingOutcome::UnknownHeadBlock { beacon_block_root: attestation.data.beacon_block_root, }) + }; + + metrics::stop_timer(timer); + + if let Ok(AttestationProcessingOutcome::Processed) = &result { + metrics::inc_counter(&metrics::ATTESTATION_PROCESSING_SUCCESSES); } + + result } /// Verifies the `attestation` against the `state` to which it is attesting. @@ -684,9 +695,6 @@ impl BeaconChain { state: &BeaconState, block: &BeaconBlock, ) -> Result { - self.metrics.attestation_processing_requests.inc(); - let timer = self.metrics.attestation_processing_times.start_timer(); - // Find the highest between: // // - The highest valid finalized epoch we've ever seen (i.e., the head). @@ -696,7 +704,17 @@ impl BeaconChain { state.finalized_checkpoint.epoch, ); - let result = if block.slot <= finalized_epoch.start_slot(T::EthSpec::slots_per_epoch()) { + // A helper function to allow attestation processing to be metered. + let verify_attestation_for_state = |state, attestation, spec, verify_signatures| { + let timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_CORE); + + let result = verify_attestation_for_state(state, attestation, spec, verify_signatures); + + metrics::stop_timer(timer); + result + }; + + if block.slot <= finalized_epoch.start_slot(T::EthSpec::slots_per_epoch()) { // Ignore any attestation where the slot of `data.beacon_block_root` is equal to or // prior to the finalized epoch. // @@ -730,14 +748,10 @@ impl BeaconChain { .insert_attestation(attestation, state, &self.spec)?; // Update the metrics. - self.metrics.attestation_processing_successes.inc(); + metrics::inc_counter(&metrics::ATTESTATION_PROCESSING_SUCCESSES); Ok(AttestationProcessingOutcome::Processed) - }; - - timer.observe_duration(); - - result + } } /// Accept some deposit and queue it for inclusion in an appropriate block. @@ -786,8 +800,8 @@ impl BeaconChain { &self, block: BeaconBlock, ) -> Result { - self.metrics.block_processing_requests.inc(); - let timer = self.metrics.block_processing_times.start_timer(); + metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS); + let full_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_TIMES); let finalized_slot = self .state @@ -804,8 +818,12 @@ impl BeaconChain { return Ok(BlockProcessingOutcome::GenesisBlock); } + let block_root_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_BLOCK_ROOT); + let block_root = block.canonical_root(); + metrics::stop_timer(block_root_timer); + if block_root == self.genesis_block_root { return Ok(BlockProcessingOutcome::GenesisBlock); } @@ -825,6 +843,10 @@ impl BeaconChain { return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown); } + // Records the time taken to load the block and state from the database during block + // processing. + let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ); + // Load the blocks parent block from the database, returning invalid if that block is not // found. let parent_block: BeaconBlock = match self.store.get(&block.parent_root)? { @@ -844,15 +866,27 @@ impl BeaconChain { .get(&parent_state_root)? .ok_or_else(|| Error::DBInconsistent(format!("Missing state {}", parent_state_root)))?; + metrics::stop_timer(db_read_timer); + + let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE); + // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; for _ in state.slot.as_u64()..block.slot.as_u64() { per_slot_processing(&mut state, &self.spec)?; } + metrics::stop_timer(catchup_timer); + + let committee_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_COMMITTEE); + state.build_committee_cache(RelativeEpoch::Previous, &self.spec)?; state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + metrics::stop_timer(committee_timer); + + let core_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CORE); + // Apply the received block to its parent state (which has been transitioned into this // slot). match per_block_processing(&mut state, &block, &self.spec) { @@ -863,16 +897,29 @@ impl BeaconChain { _ => {} } + metrics::stop_timer(core_timer); + + let state_root_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_STATE_ROOT); + let state_root = state.canonical_root(); if block.state_root != state_root { return Ok(BlockProcessingOutcome::StateRootMismatch); } + metrics::stop_timer(state_root_timer); + + let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE); + // Store the block and state. self.store.put(&block_root, &block)?; self.store.put(&state_root, &state)?; + metrics::stop_timer(db_write_timer); + + let fork_choice_register_timer = + metrics::start_timer(&metrics::BLOCK_PROCESSING_FORK_CHOICE_REGISTER); + // Register the new block with the fork choice service. if let Err(e) = self.fork_choice.process_block(&state, &block, block_root) { error!( @@ -884,6 +931,11 @@ impl BeaconChain { ) } + metrics::stop_timer(fork_choice_register_timer); + + let find_head_timer = + metrics::start_timer(&metrics::BLOCK_PROCESSING_FORK_CHOICE_FIND_HEAD); + // Execute the fork choice algorithm, enthroning a new head if discovered. // // Note: in the future we may choose to run fork-choice less often, potentially based upon @@ -896,11 +948,14 @@ impl BeaconChain { ) }; - self.metrics.block_processing_successes.inc(); - self.metrics - .operations_per_block_attestation - .observe(block.body.attestations.len() as f64); - timer.observe_duration(); + metrics::stop_timer(find_head_timer); + + metrics::inc_counter(&metrics::BLOCK_PROCESSING_SUCCESSES); + metrics::observe( + &metrics::OPERATIONS_PER_BLOCK_ATTESTATION, + block.body.attestations.len() as f64, + ); + metrics::stop_timer(full_timer); Ok(BlockProcessingOutcome::Processed { block_root }) } @@ -935,8 +990,8 @@ impl BeaconChain { produce_at_slot: Slot, randao_reveal: Signature, ) -> Result<(BeaconBlock, BeaconState), BlockProductionError> { - self.metrics.block_production_requests.inc(); - let timer = self.metrics.block_production_times.start_timer(); + metrics::inc_counter(&metrics::BLOCK_PRODUCTION_REQUESTS); + let timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_TIMES); // If required, transition the new state to the present slot. while state.slot < produce_at_slot { @@ -988,28 +1043,25 @@ impl BeaconChain { block.state_root = state_root; - self.metrics.block_production_successes.inc(); - timer.observe_duration(); + metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); + metrics::stop_timer(timer); Ok((block, state)) } /// Execute the fork choice algorithm and enthrone the result as the canonical head. pub fn fork_choice(&self) -> Result<(), Error> { - self.metrics.fork_choice_requests.inc(); + metrics::inc_counter(&metrics::FORK_CHOICE_REQUESTS); // Start fork choice metrics timer. - let timer = self.metrics.fork_choice_times.start_timer(); + let timer = metrics::start_timer(&metrics::FORK_CHOICE_TIMES); // Determine the root of the block that is the head of the chain. let beacon_block_root = self.fork_choice.find_head(&self)?; - // End fork choice metrics timer. - timer.observe_duration(); - // If a new head was chosen. - if beacon_block_root != self.head().beacon_block_root { - self.metrics.fork_choice_changed_head.inc(); + let result = if beacon_block_root != self.head().beacon_block_root { + metrics::inc_counter(&metrics::FORK_CHOICE_CHANGED_HEAD); let beacon_block: BeaconBlock = self .store @@ -1027,7 +1079,7 @@ impl BeaconChain { // If we switched to a new chain (instead of building atop the present chain). if self.head().beacon_block_root != beacon_block.parent_root { - self.metrics.fork_choice_reorg_count.inc(); + metrics::inc_counter(&metrics::FORK_CHOICE_REORG_COUNT); warn!( self.log, "Beacon chain re-org"; @@ -1071,11 +1123,22 @@ impl BeaconChain { } } else { Ok(()) + }; + + // End fork choice metrics timer. + metrics::stop_timer(timer); + + if let Err(_) = result { + metrics::inc_counter(&metrics::FORK_CHOICE_ERRORS); } + + result } /// Update the canonical head to `new_head`. fn update_canonical_head(&self, new_head: CheckPoint) -> Result<(), Error> { + let timer = metrics::start_timer(&metrics::UPDATE_HEAD_TIMES); + // Update the checkpoint that stores the head of the chain at the time it received the // block. *self.canonical_head.write() = new_head; @@ -1102,6 +1165,8 @@ impl BeaconChain { // Save `self` to `self.store`. self.persist()?; + metrics::stop_timer(timer); + Ok(()) } @@ -1129,6 +1194,13 @@ impl BeaconChain { self.fork_choice .process_finalization(&finalized_block, finalized_block_root)?; + let finalized_state = self + .store + .get::>(&finalized_block.state_root)? + .ok_or_else(|| Error::MissingBeaconState(finalized_block.state_root))?; + + self.op_pool.prune_all(&finalized_state, &self.spec); + Ok(()) } } diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 7a51fc4258..22df90397e 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -1,5 +1,4 @@ use crate::fork_choice::Error as ForkChoiceError; -use crate::metrics::Error as MetricsError; use state_processing::per_block_processing::errors::{ AttestationValidationError, IndexedAttestationValidationError, }; @@ -34,7 +33,6 @@ pub enum BeaconChainError { MissingBeaconBlock(Hash256), MissingBeaconState(Hash256), SlotProcessingError(SlotProcessingError), - MetricsError(String), NoStateForAttestation { beacon_block_root: Hash256, }, @@ -44,12 +42,6 @@ pub enum BeaconChainError { easy_from_to!(SlotProcessingError, BeaconChainError); -impl From for BeaconChainError { - fn from(e: MetricsError) -> BeaconChainError { - BeaconChainError::MetricsError(format!("{:?}", e)) - } -} - #[derive(Debug, PartialEq)] pub enum BlockProductionError { UnableToGetBlockRootFromState, diff --git a/beacon_node/beacon_chain/src/fork_choice.rs b/beacon_node/beacon_chain/src/fork_choice.rs index edd426f296..77fdaacdc5 100644 --- a/beacon_node/beacon_chain/src/fork_choice.rs +++ b/beacon_node/beacon_chain/src/fork_choice.rs @@ -1,4 +1,4 @@ -use crate::{BeaconChain, BeaconChainTypes}; +use crate::{metrics, BeaconChain, BeaconChainTypes}; use lmd_ghost::LmdGhost; use state_processing::common::get_attesting_indices; use std::sync::Arc; @@ -46,6 +46,8 @@ impl ForkChoice { } pub fn find_head(&self, chain: &BeaconChain) -> Result { + let timer = metrics::start_timer(&metrics::FORK_CHOICE_FIND_HEAD_TIMES); + let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch()); // From the specification: @@ -97,9 +99,14 @@ impl ForkChoice { .map(|v| v.effective_balance) }; - self.backend + let result = self + .backend .find_head(start_block_slot, start_block_root, weight) - .map_err(Into::into) + .map_err(Into::into); + + metrics::stop_timer(timer); + + result } /// Process all attestations in the given `block`. @@ -112,6 +119,7 @@ impl ForkChoice { block: &BeaconBlock, block_root: Hash256, ) -> Result<()> { + let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_BLOCK_TIMES); // Note: we never count the block as a latest message, only attestations. // // I (Paul H) do not have an explicit reference to this, but I derive it from this @@ -136,6 +144,8 @@ impl ForkChoice { // a block that has the majority of votes applied to it. self.backend.process_block(block, block_root)?; + metrics::stop_timer(timer); + Ok(()) } @@ -148,6 +158,8 @@ impl ForkChoice { attestation: &Attestation, block: &BeaconBlock, ) -> Result<()> { + let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_ATTESTATION_TIMES); + let block_hash = attestation.data.beacon_block_root; // Ignore any attestations to the zero hash. @@ -175,6 +187,8 @@ impl ForkChoice { } } + metrics::stop_timer(timer); + Ok(()) } diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 3188760a42..cc7725dd83 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,3 +1,7 @@ +#![recursion_limit = "128"] // For lazy-static +#[macro_use] +extern crate lazy_static; + mod beacon_chain; mod checkpoint; mod errors; @@ -13,6 +17,7 @@ pub use self::beacon_chain::{ pub use self::checkpoint::CheckPoint; pub use self::errors::{BeaconChainError, BlockProductionError}; pub use lmd_ghost; +pub use metrics::scrape_for_metrics; pub use parking_lot; pub use slot_clock; pub use state_processing::per_block_processing::errors::{ diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index fa1718ebfb..6efa4b3f2b 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1,143 +1,276 @@ -pub use prometheus::Error; -use prometheus::{Histogram, HistogramOpts, IntCounter, Opts, Registry}; +use crate::{BeaconChain, BeaconChainTypes}; +pub use lighthouse_metrics::*; +use types::{BeaconState, Epoch, Hash256, Slot}; -pub struct Metrics { - pub block_processing_requests: IntCounter, - pub block_processing_successes: IntCounter, - pub block_processing_times: Histogram, - pub block_production_requests: IntCounter, - pub block_production_successes: IntCounter, - pub block_production_times: Histogram, - pub attestation_production_requests: IntCounter, - pub attestation_production_successes: IntCounter, - pub attestation_production_times: Histogram, - pub attestation_processing_requests: IntCounter, - pub attestation_processing_successes: IntCounter, - pub attestation_processing_times: Histogram, - pub fork_choice_requests: IntCounter, - pub fork_choice_changed_head: IntCounter, - pub fork_choice_reorg_count: IntCounter, - pub fork_choice_times: Histogram, - pub operations_per_block_attestation: Histogram, +lazy_static! { + /* + * Block Processing + */ + pub static ref BLOCK_PROCESSING_REQUESTS: Result = try_create_int_counter( + "beacon_block_processing_requests_total", + "Count of blocks submitted for processing" + ); + pub static ref BLOCK_PROCESSING_SUCCESSES: Result = try_create_int_counter( + "beacon_block_processing_successes_total", + "Count of blocks processed without error" + ); + pub static ref BLOCK_PROCESSING_TIMES: Result = + try_create_histogram("beacon_block_processing_seconds", "Full runtime of block processing"); + pub static ref BLOCK_PROCESSING_BLOCK_ROOT: Result = try_create_histogram( + "beacon_block_processing_block_root_seconds", + "Time spent calculating the block root when processing a block." + ); + pub static ref BLOCK_PROCESSING_DB_READ: Result = try_create_histogram( + "beacon_block_processing_db_read_seconds", + "Time spent loading block and state from DB for block processing" + ); + pub static ref BLOCK_PROCESSING_CATCHUP_STATE: Result = try_create_histogram( + "beacon_block_processing_catch_up_state_seconds", + "Time spent skipping slots on a state before processing a block." + ); + pub static ref BLOCK_PROCESSING_COMMITTEE: Result = try_create_histogram( + "beacon_block_processing_committee_building_seconds", + "Time spent building/obtaining committees for block processing." + ); + pub static ref BLOCK_PROCESSING_CORE: Result = try_create_histogram( + "beacon_block_processing_core_seconds", + "Time spent doing the core per_block_processing state processing." + ); + pub static ref BLOCK_PROCESSING_STATE_ROOT: Result = try_create_histogram( + "beacon_block_processing_state_root_seconds", + "Time spent calculating the state root when processing a block." + ); + pub static ref BLOCK_PROCESSING_DB_WRITE: Result = try_create_histogram( + "beacon_block_processing_db_write_seconds", + "Time spent writing a newly processed block and state to DB" + ); + pub static ref BLOCK_PROCESSING_FORK_CHOICE_REGISTER: Result = try_create_histogram( + "beacon_block_processing_fork_choice_register_seconds", + "Time spent registering the new block with fork choice (but not finding head)" + ); + pub static ref BLOCK_PROCESSING_FORK_CHOICE_FIND_HEAD: Result = try_create_histogram( + "beacon_block_processing_fork_choice_find_head_seconds", + "Time spent finding the new head after processing a new block" + ); + + /* + * Block Production + */ + pub static ref BLOCK_PRODUCTION_REQUESTS: Result = try_create_int_counter( + "beacon_block_production_requests_total", + "Count of all block production requests" + ); + pub static ref BLOCK_PRODUCTION_SUCCESSES: Result = try_create_int_counter( + "beacon_block_production_successes_total", + "Count of blocks successfully produced." + ); + pub static ref BLOCK_PRODUCTION_TIMES: Result = + try_create_histogram("beacon_block_production_seconds", "Full runtime of block production"); + + /* + * Block Statistics + */ + pub static ref OPERATIONS_PER_BLOCK_ATTESTATION: Result = try_create_histogram( + "beacon_operations_per_block_attestation_total", + "Number of attestations in a block" + ); + + /* + * Attestation Processing + */ + pub static ref ATTESTATION_PROCESSING_REQUESTS: Result = try_create_int_counter( + "beacon_attestation_processing_requests_total", + "Count of all attestations submitted for processing" + ); + pub static ref ATTESTATION_PROCESSING_SUCCESSES: Result = try_create_int_counter( + "beacon_attestation_processing_successes_total", + "total_attestation_processing_successes" + ); + pub static ref ATTESTATION_PROCESSING_TIMES: Result = try_create_histogram( + "beacon_attestation_processing_seconds", + "Full runtime of attestation processing" + ); + pub static ref ATTESTATION_PROCESSING_CORE: Result = try_create_histogram( + "beacon_attestation_processing_core_seconds", + "Time spent on the core spec processing of attestation processing" + ); + + /* + * Attestation Production + */ + pub static ref ATTESTATION_PRODUCTION_REQUESTS: Result = try_create_int_counter( + "beacon_attestation_production_requests_total", + "Count of all attestation production requests" + ); + pub static ref ATTESTATION_PRODUCTION_SUCCESSES: Result = try_create_int_counter( + "beacon_attestation_production_successes_total", + "Count of attestations processed without error" + ); + pub static ref ATTESTATION_PRODUCTION_TIMES: Result = try_create_histogram( + "beacon_attestation_production_seconds", + "Full runtime of attestation production" + ); + + /* + * Fork Choice + */ + pub static ref FORK_CHOICE_REQUESTS: Result = try_create_int_counter( + "beacon_fork_choice_requests_total", + "Count of occasions where fork choice has tried to find a head" + ); + pub static ref FORK_CHOICE_ERRORS: Result = try_create_int_counter( + "beacon_fork_choice_errors_total", + "Count of occasions where fork choice has returned an error when trying to find a head" + ); + pub static ref FORK_CHOICE_CHANGED_HEAD: Result = try_create_int_counter( + "beacon_fork_choice_changed_head_total", + "Count of occasions fork choice has found a new head" + ); + pub static ref FORK_CHOICE_REORG_COUNT: Result = try_create_int_counter( + "beacon_fork_choice_reorg_total", + "Count of occasions fork choice has switched to a different chain" + ); + pub static ref FORK_CHOICE_TIMES: Result = + try_create_histogram("beacon_fork_choice_seconds", "Full runtime of fork choice"); + pub static ref FORK_CHOICE_FIND_HEAD_TIMES: Result = + try_create_histogram("beacon_fork_choice_find_head_seconds", "Full runtime of fork choice find_head function"); + pub static ref FORK_CHOICE_PROCESS_BLOCK_TIMES: Result = try_create_histogram( + "beacon_fork_choice_process_block_seconds", + "Time taken to add a block and all attestations to fork choice" + ); + pub static ref FORK_CHOICE_PROCESS_ATTESTATION_TIMES: Result = try_create_histogram( + "beacon_fork_choice_process_attestation_seconds", + "Time taken to add an attestation to fork choice" + ); + + /* + * Persisting BeaconChain to disk + */ + pub static ref PERSIST_CHAIN: Result = + try_create_histogram("beacon_persist_chain", "Time taken to update the canonical head"); + + /* + * Chain Head + */ + pub static ref UPDATE_HEAD_TIMES: Result = + try_create_histogram("beacon_update_head_seconds", "Time taken to update the canonical head"); + pub static ref HEAD_STATE_SLOT: Result = + try_create_int_gauge("beacon_head_state_slot", "Slot of the block at the head of the chain"); + pub static ref HEAD_STATE_ROOT: Result = + try_create_int_gauge("beacon_head_state_root", "Root of the block at the head of the chain"); + pub static ref HEAD_STATE_LATEST_BLOCK_SLOT: Result = + try_create_int_gauge("beacon_head_state_latest_block_slot", "Latest block slot at the head of the chain"); + pub static ref HEAD_STATE_CURRENT_JUSTIFIED_ROOT: Result = + try_create_int_gauge("beacon_head_state_current_justified_root", "Current justified root at the head of the chain"); + pub static ref HEAD_STATE_CURRENT_JUSTIFIED_EPOCH: Result = + try_create_int_gauge("beacon_head_state_current_justified_epoch", "Current justified epoch at the head of the chain"); + pub static ref HEAD_STATE_PREVIOUS_JUSTIFIED_ROOT: Result = + try_create_int_gauge("beacon_head_state_previous_justified_root", "Previous justified root at the head of the chain"); + pub static ref HEAD_STATE_PREVIOUS_JUSTIFIED_EPOCH: Result = + try_create_int_gauge("beacon_head_state_previous_justified_epoch", "Previous justified epoch at the head of the chain"); + pub static ref HEAD_STATE_FINALIZED_ROOT: Result = + try_create_int_gauge("beacon_head_state_finalized_root", "Finalized root at the head of the chain"); + pub static ref HEAD_STATE_FINALIZED_EPOCH: Result = + try_create_int_gauge("beacon_head_state_finalized_epoch", "Finalized epoch at the head of the chain"); + pub static ref HEAD_STATE_SHARDS: Result = + try_create_int_gauge("beacon_head_state_shard_total", "Count of shards in the beacon chain"); + pub static ref HEAD_STATE_TOTAL_VALIDATORS: Result = + try_create_int_gauge("beacon_head_state_total_validators_total", "Count of validators at the head of the chain"); + pub static ref HEAD_STATE_ACTIVE_VALIDATORS: Result = + try_create_int_gauge("beacon_head_state_active_validators_total", "Count of active validators at the head of the chain"); + pub static ref HEAD_STATE_VALIDATOR_BALANCES: Result = + try_create_int_gauge("beacon_head_state_validator_balances_total", "Sum of all validator balances at the head of the chain"); + pub static ref HEAD_STATE_SLASHED_VALIDATORS: Result = + try_create_int_gauge("beacon_head_state_slashed_validators_total", "Count of all slashed validators at the head of the chain"); + pub static ref HEAD_STATE_WITHDRAWN_VALIDATORS: Result = + try_create_int_gauge("beacon_head_state_withdrawn_validators_total", "Sum of all validator balances at the head of the chain"); + pub static ref HEAD_STATE_ETH1_DEPOSIT_INDEX: Result = + try_create_int_gauge("beacon_head_state_eth1_deposit_index", "Eth1 deposit index at the head of the chain"); } -impl Metrics { - pub fn new() -> Result { - Ok(Self { - block_processing_requests: { - let opts = Opts::new("block_processing_requests", "total_blocks_processed"); - IntCounter::with_opts(opts)? - }, - block_processing_successes: { - let opts = Opts::new("block_processing_successes", "total_valid_blocks_processed"); - IntCounter::with_opts(opts)? - }, - block_processing_times: { - let opts = HistogramOpts::new("block_processing_times", "block_processing_time"); - Histogram::with_opts(opts)? - }, - block_production_requests: { - let opts = Opts::new("block_production_requests", "attempts_to_produce_new_block"); - IntCounter::with_opts(opts)? - }, - block_production_successes: { - let opts = Opts::new("block_production_successes", "blocks_successfully_produced"); - IntCounter::with_opts(opts)? - }, - block_production_times: { - let opts = HistogramOpts::new("block_production_times", "block_production_time"); - Histogram::with_opts(opts)? - }, - attestation_production_requests: { - let opts = Opts::new( - "attestation_production_requests", - "total_attestation_production_requests", - ); - IntCounter::with_opts(opts)? - }, - attestation_production_successes: { - let opts = Opts::new( - "attestation_production_successes", - "total_attestation_production_successes", - ); - IntCounter::with_opts(opts)? - }, - attestation_production_times: { - let opts = HistogramOpts::new( - "attestation_production_times", - "attestation_production_time", - ); - Histogram::with_opts(opts)? - }, - attestation_processing_requests: { - let opts = Opts::new( - "attestation_processing_requests", - "total_attestation_processing_requests", - ); - IntCounter::with_opts(opts)? - }, - attestation_processing_successes: { - let opts = Opts::new( - "attestation_processing_successes", - "total_attestation_processing_successes", - ); - IntCounter::with_opts(opts)? - }, - attestation_processing_times: { - let opts = HistogramOpts::new( - "attestation_processing_times", - "attestation_processing_time", - ); - Histogram::with_opts(opts)? - }, - fork_choice_requests: { - let opts = Opts::new("fork_choice_requests", "total_times_fork_choice_called"); - IntCounter::with_opts(opts)? - }, - fork_choice_changed_head: { - let opts = Opts::new( - "fork_choice_changed_head", - "total_times_fork_choice_chose_a_new_head", - ); - IntCounter::with_opts(opts)? - }, - fork_choice_reorg_count: { - let opts = Opts::new("fork_choice_reorg_count", "number_of_reorgs"); - IntCounter::with_opts(opts)? - }, - fork_choice_times: { - let opts = HistogramOpts::new("fork_choice_time", "total_time_to_run_fork_choice"); - Histogram::with_opts(opts)? - }, - operations_per_block_attestation: { - let opts = HistogramOpts::new( - "operations_per_block_attestation", - "count_of_attestations_per_block", - ); - Histogram::with_opts(opts)? - }, - }) - } - - pub fn register(&self, registry: &Registry) -> Result<(), Error> { - registry.register(Box::new(self.block_processing_requests.clone()))?; - registry.register(Box::new(self.block_processing_successes.clone()))?; - registry.register(Box::new(self.block_processing_times.clone()))?; - registry.register(Box::new(self.block_production_requests.clone()))?; - registry.register(Box::new(self.block_production_successes.clone()))?; - registry.register(Box::new(self.block_production_times.clone()))?; - registry.register(Box::new(self.attestation_production_requests.clone()))?; - registry.register(Box::new(self.attestation_production_successes.clone()))?; - registry.register(Box::new(self.attestation_production_times.clone()))?; - registry.register(Box::new(self.attestation_processing_requests.clone()))?; - registry.register(Box::new(self.attestation_processing_successes.clone()))?; - registry.register(Box::new(self.attestation_processing_times.clone()))?; - registry.register(Box::new(self.fork_choice_requests.clone()))?; - registry.register(Box::new(self.fork_choice_changed_head.clone()))?; - registry.register(Box::new(self.fork_choice_reorg_count.clone()))?; - registry.register(Box::new(self.fork_choice_times.clone()))?; - registry.register(Box::new(self.operations_per_block_attestation.clone()))?; - - Ok(()) - } +/// Scrape the `beacon_chain` for metrics that are not constantly updated (e.g., the present slot, +/// head state info, etc) and update the Prometheus `DEFAULT_REGISTRY`. +pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { + scrape_head_state::( + &beacon_chain.head().beacon_state, + beacon_chain.head().beacon_state_root, + ); +} + +/// Scrape the given `state` assuming it's the head state, updating the `DEFAULT_REGISTRY`. +fn scrape_head_state(state: &BeaconState, state_root: Hash256) { + set_gauge_by_slot(&HEAD_STATE_SLOT, state.slot); + set_gauge_by_hash(&HEAD_STATE_ROOT, state_root); + set_gauge_by_slot( + &HEAD_STATE_LATEST_BLOCK_SLOT, + state.latest_block_header.slot, + ); + set_gauge_by_hash( + &HEAD_STATE_CURRENT_JUSTIFIED_ROOT, + state.current_justified_checkpoint.root, + ); + set_gauge_by_epoch( + &HEAD_STATE_CURRENT_JUSTIFIED_EPOCH, + state.current_justified_checkpoint.epoch, + ); + set_gauge_by_hash( + &HEAD_STATE_PREVIOUS_JUSTIFIED_ROOT, + state.previous_justified_checkpoint.root, + ); + set_gauge_by_epoch( + &HEAD_STATE_PREVIOUS_JUSTIFIED_EPOCH, + state.previous_justified_checkpoint.epoch, + ); + set_gauge_by_hash(&HEAD_STATE_FINALIZED_ROOT, state.finalized_checkpoint.root); + set_gauge_by_epoch( + &HEAD_STATE_FINALIZED_EPOCH, + state.finalized_checkpoint.epoch, + ); + set_gauge_by_usize(&HEAD_STATE_SHARDS, state.previous_crosslinks.len()); + set_gauge_by_usize(&HEAD_STATE_TOTAL_VALIDATORS, state.validators.len()); + set_gauge_by_u64( + &HEAD_STATE_VALIDATOR_BALANCES, + state.balances.iter().fold(0_u64, |acc, i| acc + i), + ); + set_gauge_by_usize( + &HEAD_STATE_ACTIVE_VALIDATORS, + state + .validators + .iter() + .filter(|v| v.is_active_at(state.current_epoch())) + .count(), + ); + set_gauge_by_usize( + &HEAD_STATE_SLASHED_VALIDATORS, + state.validators.iter().filter(|v| v.slashed).count(), + ); + set_gauge_by_usize( + &HEAD_STATE_WITHDRAWN_VALIDATORS, + state + .validators + .iter() + .filter(|v| v.is_withdrawable_at(state.current_epoch())) + .count(), + ); + set_gauge_by_u64(&HEAD_STATE_ETH1_DEPOSIT_INDEX, state.eth1_deposit_index); +} + +fn set_gauge_by_slot(gauge: &Result, value: Slot) { + set_gauge(gauge, value.as_u64() as i64); +} + +fn set_gauge_by_epoch(gauge: &Result, value: Epoch) { + set_gauge(gauge, value.as_u64() as i64); +} + +fn set_gauge_by_hash(gauge: &Result, value: Hash256) { + set_gauge(gauge, value.to_low_u64_le() as i64); +} + +fn set_gauge_by_usize(gauge: &Result, value: usize) { + set_gauge(gauge, value as i64); +} + +fn set_gauge_by_u64(gauge: &Result, value: u64) { + set_gauge(gauge, value as i64); } diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 8c72fa4171..b13f175a9b 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -7,7 +7,6 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } -http_server = { path = "../http_server" } rpc = { path = "../rpc" } rest_api = { path = "../rest_api" } prometheus = "^0.6" diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index ee62b62815..fcc2cc7dac 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,6 +1,5 @@ use crate::Eth2Config; use clap::ArgMatches; -use http_server::HttpServerConfig; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; use slog::{info, o, Drain}; @@ -25,7 +24,6 @@ pub struct Config { pub genesis_state: GenesisState, pub network: network::NetworkConfig, pub rpc: rpc::RPCConfig, - pub http: HttpServerConfig, pub rest_api: rest_api::ApiConfig, } @@ -59,7 +57,6 @@ impl Default for Config { db_name: "chain_db".to_string(), network: NetworkConfig::new(), rpc: rpc::RPCConfig::default(), - http: HttpServerConfig::default(), rest_api: rest_api::ApiConfig::default(), spec_constants: TESTNET_SPEC_CONSTANTS.into(), genesis_state: GenesisState::RecentGenesis { @@ -143,7 +140,6 @@ impl Config { self.network.apply_cli_args(args)?; self.rpc.apply_cli_args(args)?; - self.http.apply_cli_args(args)?; self.rest_api.apply_cli_args(args)?; if let Some(log_file) = args.value_of("logfile") { diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 65ba071fa1..5c37ac3e9c 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -10,7 +10,6 @@ use beacon_chain::BeaconChain; use exit_future::Signal; use futures::{future::Future, Stream}; use network::Service as NetworkService; -use prometheus::Registry; use slog::{error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; @@ -36,8 +35,6 @@ pub struct Client { pub network: Arc>, /// Signal to terminate the RPC server. pub rpc_exit_signal: Option, - /// Signal to terminate the HTTP server. - pub http_exit_signal: Option, /// Signal to terminate the slot timer. pub slot_timer_exit_signal: Option, /// Signal to terminate the API @@ -60,7 +57,6 @@ where log: slog::Logger, executor: &TaskExecutor, ) -> error::Result { - let metrics_registry = Registry::new(); let store = Arc::new(store); let seconds_per_slot = eth2_config.spec.seconds_per_slot; @@ -71,11 +67,6 @@ where eth2_config.spec.clone(), log.clone(), )?); - // Registry all beacon chain metrics with the global registry. - beacon_chain - .metrics - .register(&metrics_registry) - .expect("Failed to registry metrics"); if beacon_chain.read_slot_clock().is_none() { panic!("Cannot start client before genesis!") @@ -124,29 +115,13 @@ where None }; - // Start the `http_server` service. - // - // Note: presently we are ignoring the config and _always_ starting a HTTP server. - let http_exit_signal = if client_config.http.enabled { - Some(http_server::start_service( - &client_config.http, - executor, - network_send, - beacon_chain.clone(), - client_config.db_path().expect("unable to read datadir"), - metrics_registry, - &log, - )) - } else { - None - }; - // Start the `rest_api` service let api_exit_signal = if client_config.rest_api.enabled { match rest_api::start_server( &client_config.rest_api, executor, beacon_chain.clone(), + client_config.db_path().expect("unable to read datadir"), &log, ) { Ok(s) => Some(s), @@ -188,7 +163,6 @@ where Ok(Client { _client_config: client_config, beacon_chain, - http_exit_signal, rpc_exit_signal, slot_timer_exit_signal: Some(slot_timer_exit_signal), api_exit_signal, diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index 794b097128..006b895a1c 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -26,3 +26,5 @@ smallvec = "0.6.10" fnv = "1.0.6" unsigned-varint = "0.2.2" bytes = "0.4.12" +lazy_static = "1.3.0" +lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 4c1794945d..ca98db3246 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -1,3 +1,4 @@ +use crate::metrics; use crate::{error, NetworkConfig}; /// This manages the discovery and management of peers. /// @@ -159,10 +160,16 @@ where fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) { self.connected_peers.insert(peer_id); + + metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64); } fn inject_disconnected(&mut self, peer_id: &PeerId, _endpoint: ConnectedPoint) { self.connected_peers.remove(peer_id); + + metrics::inc_counter(&metrics::PEER_DISCONNECT_EVENT_COUNT); + metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64); } fn inject_replaced( @@ -217,6 +224,7 @@ where } Discv5Event::SocketUpdated(socket) => { info!(self.log, "Address updated"; "IP" => format!("{}",socket.ip())); + metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT); let mut address = Multiaddr::from(socket.ip()); address.push(Protocol::Tcp(self.tcp_port)); let enr = self.discovery.local_enr(); diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 54a4f2a998..33d5ba9ed9 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -2,10 +2,14 @@ /// all required libp2p functionality. /// /// This crate builds and manages the libp2p services required by the beacon node. +#[macro_use] +extern crate lazy_static; + pub mod behaviour; mod config; mod discovery; pub mod error; +mod metrics; pub mod rpc; mod service; diff --git a/beacon_node/eth2-libp2p/src/metrics.rs b/beacon_node/eth2-libp2p/src/metrics.rs new file mode 100644 index 0000000000..b678ef6b41 --- /dev/null +++ b/beacon_node/eth2-libp2p/src/metrics.rs @@ -0,0 +1,20 @@ +pub use lighthouse_metrics::*; + +lazy_static! { + pub static ref ADDRESS_UPDATE_COUNT: Result = try_create_int_counter( + "libp2p_address_update_total", + "Count of libp2p socked updated events (when our view of our IP address has changed)" + ); + pub static ref PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_peer_connected_peers_total", + "Count of libp2p peers currently connected" + ); + pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( + "libp2p_peer_connect_event_total", + "Count of libp2p peer connect events (not the current number of connected peers)" + ); + pub static ref PEER_DISCONNECT_EVENT_COUNT: Result = try_create_int_counter( + "libp2p_peer_disconnect_event_total", + "Count of libp2p peer disconnect events" + ); +} diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml deleted file mode 100644 index e87ff29972..0000000000 --- a/beacon_node/http_server/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "http_server" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -beacon_chain = { path = "../beacon_chain" } -iron = "^0.6" -router = "^0.6" -network = { path = "../network" } -types = { path = "../../eth2/types" } -slot_clock = { path = "../../eth2/utils/slot_clock" } -persistent = "^0.4" -prometheus = { version = "^0.6", features = ["process"] } -clap = "2.32.0" -futures = "0.1.23" -serde = "1.0" -serde_derive = "1.0" -serde_json = "1.0" -slog = { version = "^2.2.3" , features = ["max_level_trace"] } -tokio = "0.1.17" -exit-future = "0.1.4" diff --git a/beacon_node/http_server/src/api.rs b/beacon_node/http_server/src/api.rs deleted file mode 100644 index 8cb023b02c..0000000000 --- a/beacon_node/http_server/src/api.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::{key::BeaconChainKey, map_persistent_err_to_500}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use iron::prelude::*; -use iron::{ - headers::{CacheControl, CacheDirective, ContentType}, - status::Status, - AfterMiddleware, Handler, IronResult, Request, Response, -}; -use persistent::Read; -use router::Router; -use serde_json::json; -use std::sync::Arc; - -/// Yields a handler for the HTTP API. -pub fn build_handler( - beacon_chain: Arc>, -) -> impl Handler { - let mut router = Router::new(); - - router.get("/node/fork", handle_fork::, "fork"); - - let mut chain = Chain::new(router); - - // Insert `BeaconChain` so it may be accessed in a request. - chain.link(Read::>::both(beacon_chain.clone())); - // Set the content-type headers. - chain.link_after(SetJsonContentType); - // Set the cache headers. - chain.link_after(SetCacheDirectives); - - chain -} - -/// Sets the `cache-control` headers on _all_ responses, unless they are already set. -struct SetCacheDirectives; -impl AfterMiddleware for SetCacheDirectives { - fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult { - // This is run for every requests, AFTER all handlers have been executed - if resp.headers.get::() == None { - resp.headers.set(CacheControl(vec![ - CacheDirective::NoCache, - CacheDirective::NoStore, - ])); - } - Ok(resp) - } -} - -/// Sets the `content-type` headers on _all_ responses, unless they are already set. -struct SetJsonContentType; -impl AfterMiddleware for SetJsonContentType { - fn after(&self, _req: &mut Request, mut resp: Response) -> IronResult { - if resp.headers.get::() == None { - resp.headers.set(ContentType::json()); - } - Ok(resp) - } -} - -fn handle_fork(req: &mut Request) -> IronResult { - let beacon_chain = req - .get::>>() - .map_err(map_persistent_err_to_500)?; - - let response = json!({ - "fork": beacon_chain.head().beacon_state.fork, - "network_id": beacon_chain.spec.network_id - }); - - Ok(Response::with((Status::Ok, response.to_string()))) -} diff --git a/beacon_node/http_server/src/key.rs b/beacon_node/http_server/src/key.rs deleted file mode 100644 index a69da6747f..0000000000 --- a/beacon_node/http_server/src/key.rs +++ /dev/null @@ -1,33 +0,0 @@ -use crate::metrics::LocalMetrics; -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use iron::typemap::Key; -use prometheus::Registry; -use std::marker::PhantomData; -use std::path::PathBuf; -use std::sync::Arc; - -pub struct BeaconChainKey { - _phantom: PhantomData, -} - -impl Key for BeaconChainKey { - type Value = Arc>; -} - -pub struct MetricsRegistryKey; - -impl Key for MetricsRegistryKey { - type Value = Registry; -} - -pub struct LocalMetricsKey; - -impl Key for LocalMetricsKey { - type Value = LocalMetrics; -} - -pub struct DBPathKey; - -impl Key for DBPathKey { - type Value = PathBuf; -} diff --git a/beacon_node/http_server/src/lib.rs b/beacon_node/http_server/src/lib.rs deleted file mode 100644 index f1d006a5bc..0000000000 --- a/beacon_node/http_server/src/lib.rs +++ /dev/null @@ -1,145 +0,0 @@ -mod api; -mod key; -mod metrics; - -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use clap::ArgMatches; -use futures::Future; -use iron::prelude::*; -use network::NetworkMessage; -use prometheus::Registry; -use router::Router; -use serde_derive::{Deserialize, Serialize}; -use slog::{info, o, warn}; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::runtime::TaskExecutor; -use tokio::sync::mpsc; - -#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] -pub struct HttpServerConfig { - pub enabled: bool, - pub listen_address: String, - pub listen_port: String, -} - -impl Default for HttpServerConfig { - fn default() -> Self { - Self { - enabled: false, - listen_address: "127.0.0.1".to_string(), - listen_port: "5052".to_string(), - } - } -} - -impl HttpServerConfig { - pub fn apply_cli_args(&mut self, args: &ArgMatches) -> Result<(), &'static str> { - if args.is_present("http") { - self.enabled = true; - } - - if let Some(listen_address) = args.value_of("http-address") { - self.listen_address = listen_address.to_string(); - } - - if let Some(listen_port) = args.value_of("http-port") { - self.listen_port = listen_port.to_string(); - } - - Ok(()) - } -} - -/// Build the `iron` HTTP server, defining the core routes. -pub fn create_iron_http_server( - beacon_chain: Arc>, - db_path: PathBuf, - metrics_registry: Registry, -) -> Iron { - let mut router = Router::new(); - - // A `GET` request to `/metrics` is handled by the `metrics` module. - router.get( - "/metrics", - metrics::build_handler(beacon_chain.clone(), db_path, metrics_registry), - "metrics", - ); - - // Any request to all other endpoints is handled by the `api` module. - router.any("/*", api::build_handler(beacon_chain.clone()), "api"); - - Iron::new(router) -} - -/// Start the HTTP service on the tokio `TaskExecutor`. -pub fn start_service( - config: &HttpServerConfig, - executor: &TaskExecutor, - _network_chan: mpsc::UnboundedSender, - beacon_chain: Arc>, - db_path: PathBuf, - metrics_registry: Registry, - log: &slog::Logger, -) -> exit_future::Signal { - let log = log.new(o!("Service"=>"HTTP")); - - // Create: - // - `shutdown_trigger` a one-shot to shut down this service. - // - `wait_for_shutdown` a future that will wait until someone calls shutdown. - let (shutdown_trigger, wait_for_shutdown) = exit_future::signal(); - - // Create an `iron` http, without starting it yet. - let iron = create_iron_http_server(beacon_chain, db_path, metrics_registry); - - // Create a HTTP server future. - // - // 1. Start the HTTP server - // 2. Build an exit future that will shutdown the server when requested. - // 3. Return the exit future, so the caller may shutdown the service when desired. - let http_service = { - let listen_address = format!("{}:{}", config.listen_address, config.listen_port); - // Start the HTTP server - let server_start_result = iron.http(listen_address.clone()); - - if server_start_result.is_ok() { - info!(log, "HTTP server running on {}", listen_address); - } else { - warn!(log, "HTTP server failed to start on {}", listen_address); - } - - // Build a future that will shutdown the HTTP server when the `shutdown_trigger` is - // triggered. - wait_for_shutdown.and_then(move |_| { - info!(log, "HTTP server shutting down"); - - if let Ok(mut server) = server_start_result { - // According to the documentation, `server.close()` "doesn't work" and the server - // keeps listening. - // - // It is being called anyway, because it seems like the right thing to do. If you - // know this has negative side-effects, please create an issue to discuss. - // - // See: https://docs.rs/iron/0.6.0/iron/struct.Listening.html#impl - match server.close() { - _ => (), - }; - } - info!(log, "HTTP server shutdown complete."); - Ok(()) - }) - }; - - // Attach the HTTP server to the executor. - executor.spawn(http_service); - - shutdown_trigger -} - -/// Helper function for mapping a failure to read state to a 500 server error. -fn map_persistent_err_to_500(e: persistent::PersistentError) -> iron::error::IronError { - iron::error::IronError { - error: Box::new(e), - response: iron::Response::with(iron::status::Status::InternalServerError), - } -} diff --git a/beacon_node/http_server/src/metrics.rs b/beacon_node/http_server/src/metrics.rs deleted file mode 100644 index 1b1ed1f3d4..0000000000 --- a/beacon_node/http_server/src/metrics.rs +++ /dev/null @@ -1,72 +0,0 @@ -use crate::{ - key::{BeaconChainKey, DBPathKey, LocalMetricsKey, MetricsRegistryKey}, - map_persistent_err_to_500, -}; -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use iron::prelude::*; -use iron::{status::Status, Handler, IronResult, Request, Response}; -use persistent::Read; -use prometheus::{Encoder, Registry, TextEncoder}; -use std::path::PathBuf; -use std::sync::Arc; - -pub use local_metrics::LocalMetrics; - -mod local_metrics; - -/// Yields a handler for the metrics endpoint. -pub fn build_handler( - beacon_chain: Arc>, - db_path: PathBuf, - metrics_registry: Registry, -) -> impl Handler { - let mut chain = Chain::new(handle_metrics::); - - let local_metrics = LocalMetrics::new().unwrap(); - local_metrics.register(&metrics_registry).unwrap(); - - chain.link(Read::>::both(beacon_chain)); - chain.link(Read::::both(metrics_registry)); - chain.link(Read::::both(local_metrics)); - chain.link(Read::::both(db_path)); - - chain -} - -/// Handle a request for Prometheus metrics. -/// -/// Returns a text string containing all metrics. -fn handle_metrics(req: &mut Request) -> IronResult { - let beacon_chain = req - .get::>>() - .map_err(map_persistent_err_to_500)?; - - let r = req - .get::>() - .map_err(map_persistent_err_to_500)?; - - let local_metrics = req - .get::>() - .map_err(map_persistent_err_to_500)?; - - let db_path = req - .get::>() - .map_err(map_persistent_err_to_500)?; - - // Update metrics that are calculated on each scrape. - local_metrics.update(&beacon_chain, &db_path); - - let mut buffer = vec![]; - let encoder = TextEncoder::new(); - - // Gather `DEFAULT_REGISTRY` metrics. - encoder.encode(&prometheus::gather(), &mut buffer).unwrap(); - - // Gather metrics from our registry. - let metric_families = r.gather(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - let prom_string = String::from_utf8(buffer).unwrap(); - - Ok(Response::with((Status::Ok, prom_string))) -} diff --git a/beacon_node/http_server/src/metrics/local_metrics.rs b/beacon_node/http_server/src/metrics/local_metrics.rs deleted file mode 100644 index b342cca81c..0000000000 --- a/beacon_node/http_server/src/metrics/local_metrics.rs +++ /dev/null @@ -1,154 +0,0 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes}; -use prometheus::{IntGauge, Opts, Registry}; -use slot_clock::SlotClock; -use std::fs; -use std::path::PathBuf; -use types::{EthSpec, Slot}; - -// If set to `true` will iterate and sum the balances of all validators in the state for each -// scrape. -const SHOULD_SUM_VALIDATOR_BALANCES: bool = true; - -pub struct LocalMetrics { - present_slot: IntGauge, - present_epoch: IntGauge, - best_slot: IntGauge, - best_beacon_block_root: IntGauge, - justified_beacon_block_root: IntGauge, - finalized_beacon_block_root: IntGauge, - validator_count: IntGauge, - justified_epoch: IntGauge, - finalized_epoch: IntGauge, - validator_balances_sum: IntGauge, - database_size: IntGauge, -} - -impl LocalMetrics { - /// Create a new instance. - pub fn new() -> Result { - Ok(Self { - present_slot: { - let opts = Opts::new("present_slot", "slot_at_time_of_scrape"); - IntGauge::with_opts(opts)? - }, - present_epoch: { - let opts = Opts::new("present_epoch", "epoch_at_time_of_scrape"); - IntGauge::with_opts(opts)? - }, - best_slot: { - let opts = Opts::new("best_slot", "slot_of_block_at_chain_head"); - IntGauge::with_opts(opts)? - }, - best_beacon_block_root: { - let opts = Opts::new("best_beacon_block_root", "root_of_block_at_chain_head"); - IntGauge::with_opts(opts)? - }, - justified_beacon_block_root: { - let opts = Opts::new( - "justified_beacon_block_root", - "root_of_block_at_justified_head", - ); - IntGauge::with_opts(opts)? - }, - finalized_beacon_block_root: { - let opts = Opts::new( - "finalized_beacon_block_root", - "root_of_block_at_finalized_head", - ); - IntGauge::with_opts(opts)? - }, - validator_count: { - let opts = Opts::new("validator_count", "number_of_validators"); - IntGauge::with_opts(opts)? - }, - justified_epoch: { - let opts = Opts::new("justified_epoch", "state_justified_epoch"); - IntGauge::with_opts(opts)? - }, - finalized_epoch: { - let opts = Opts::new("finalized_epoch", "state_finalized_epoch"); - IntGauge::with_opts(opts)? - }, - validator_balances_sum: { - let opts = Opts::new("validator_balances_sum", "sum_of_all_validator_balances"); - IntGauge::with_opts(opts)? - }, - database_size: { - let opts = Opts::new("database_size", "size_of_on_disk_db_in_mb"); - IntGauge::with_opts(opts)? - }, - }) - } - - /// Registry this instance with the `registry`. - pub fn register(&self, registry: &Registry) -> Result<(), prometheus::Error> { - registry.register(Box::new(self.present_slot.clone()))?; - registry.register(Box::new(self.present_epoch.clone()))?; - registry.register(Box::new(self.best_slot.clone()))?; - registry.register(Box::new(self.best_beacon_block_root.clone()))?; - registry.register(Box::new(self.justified_beacon_block_root.clone()))?; - registry.register(Box::new(self.finalized_beacon_block_root.clone()))?; - registry.register(Box::new(self.validator_count.clone()))?; - registry.register(Box::new(self.finalized_epoch.clone()))?; - registry.register(Box::new(self.justified_epoch.clone()))?; - registry.register(Box::new(self.validator_balances_sum.clone()))?; - registry.register(Box::new(self.database_size.clone()))?; - - Ok(()) - } - - /// Update the metrics in `self` to the latest values. - pub fn update(&self, beacon_chain: &BeaconChain, db_path: &PathBuf) { - let state = &beacon_chain.head().beacon_state; - - let present_slot = beacon_chain - .slot_clock - .present_slot() - .unwrap_or_else(|_| None) - .unwrap_or_else(|| Slot::new(0)); - self.present_slot.set(present_slot.as_u64() as i64); - self.present_epoch - .set(present_slot.epoch(T::EthSpec::slots_per_epoch()).as_u64() as i64); - - self.best_slot.set(state.slot.as_u64() as i64); - self.best_beacon_block_root - .set(beacon_chain.head().beacon_block_root.to_low_u64_le() as i64); - self.justified_beacon_block_root.set( - beacon_chain - .head() - .beacon_state - .current_justified_checkpoint - .root - .to_low_u64_le() as i64, - ); - self.finalized_beacon_block_root.set( - beacon_chain - .head() - .beacon_state - .finalized_checkpoint - .root - .to_low_u64_le() as i64, - ); - self.validator_count.set(state.validators.len() as i64); - self.justified_epoch - .set(state.current_justified_checkpoint.epoch.as_u64() as i64); - self.finalized_epoch - .set(state.finalized_checkpoint.epoch.as_u64() as i64); - if SHOULD_SUM_VALIDATOR_BALANCES { - self.validator_balances_sum - .set(state.balances.iter().sum::() as i64); - } - let db_size = if let Ok(iter) = fs::read_dir(db_path) { - iter.filter_map(Result::ok) - .map(size_of_dir_entry) - .fold(0_u64, |sum, val| sum + val) - } else { - 0 - }; - self.database_size.set(db_size as i64); - } -} - -fn size_of_dir_entry(dir: fs::DirEntry) -> u64 { - dir.metadata().map(|m| m.len()).unwrap_or(0) -} diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index fb6cb84134..c7026014c4 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -18,8 +18,12 @@ state_processing = { path = "../../eth2/state_processing" } types = { path = "../../eth2/types" } clap = "2.32.0" http = "^0.1.17" +prometheus = { version = "^0.6", features = ["process"] } hyper = "0.12.32" futures = "0.1" exit-future = "0.1.3" tokio = "0.1.17" url = "2.0" +lazy_static = "1.3.0" +lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } +slot_clock = { path = "../../eth2/utils/slot_clock" } diff --git a/beacon_node/rest_api/src/config.rs b/beacon_node/rest_api/src/config.rs index c4a9c738a0..90ac0821b1 100644 --- a/beacon_node/rest_api/src/config.rs +++ b/beacon_node/rest_api/src/config.rs @@ -18,7 +18,7 @@ impl Default for Config { Config { enabled: true, // rest_api enabled by default listen_address: Ipv4Addr::new(127, 0, 0, 1), - port: 1248, + port: 5052, } } } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index a94a8cdf4a..57019deea0 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -1,8 +1,10 @@ -extern crate futures; -extern crate hyper; +#[macro_use] +extern crate lazy_static; + mod beacon; mod config; mod helpers; +mod metrics; mod node; mod url_query; @@ -12,6 +14,8 @@ use hyper::rt::Future; use hyper::service::service_fn_ok; use hyper::{Body, Method, Response, Server, StatusCode}; use slog::{info, o, warn}; +use std::ops::Deref; +use std::path::PathBuf; use std::sync::Arc; use tokio::runtime::TaskExecutor; use url_query::UrlQuery; @@ -67,6 +71,7 @@ pub fn start_server( config: &ApiConfig, executor: &TaskExecutor, beacon_chain: Arc>, + db_path: PathBuf, log: &slog::Logger, ) -> Result { let log = log.new(o!("Service" => "Api")); @@ -80,6 +85,8 @@ pub fn start_server( Ok(()) }); + let db_path = DBPath(db_path); + // Get the address to bind to let bind_addr = (config.listen_address, config.port).into(); @@ -90,12 +97,17 @@ pub fn start_server( let service = move || { let log = server_log.clone(); let beacon_chain = server_bc.clone(); + let db_path = db_path.clone(); // Create a simple handler for the router, inject our stateful objects into the request. service_fn_ok(move |mut req| { + metrics::inc_counter(&metrics::REQUEST_COUNT); + let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); + req.extensions_mut().insert::(log.clone()); req.extensions_mut() .insert::>>(beacon_chain.clone()); + req.extensions_mut().insert::(db_path.clone()); let path = req.uri().path().to_string(); @@ -103,14 +115,16 @@ pub fn start_server( let result = match (req.method(), path.as_ref()) { (&Method::GET, "/beacon/state") => beacon::get_state::(req), (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), + (&Method::GET, "/metrics") => metrics::get_prometheus::(req), (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), _ => Err(ApiError::MethodNotAllowed(path.clone())), }; - match result { + let response = match result { // Return the `hyper::Response`. Ok(response) => { + metrics::inc_counter(&metrics::SUCCESS_COUNT); slog::debug!(log, "Request successful: {:?}", path); response } @@ -119,7 +133,11 @@ pub fn start_server( slog::debug!(log, "Request failure: {:?}", path); e.into() } - } + }; + + metrics::stop_timer(timer); + + response }) }; @@ -152,3 +170,14 @@ fn success_response(body: Body) -> Response { .body(body) .expect("We should always be able to make response from the success body.") } + +#[derive(Clone)] +pub struct DBPath(PathBuf); + +impl Deref for DBPath { + type Target = PathBuf; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs new file mode 100644 index 0000000000..064359337a --- /dev/null +++ b/beacon_node/rest_api/src/metrics.rs @@ -0,0 +1,69 @@ +use crate::{success_response, ApiError, ApiResult, DBPath}; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use hyper::{Body, Request}; +use prometheus::{Encoder, TextEncoder}; +use std::sync::Arc; + +pub use lighthouse_metrics::*; + +lazy_static! { + pub static ref REQUEST_RESPONSE_TIME: Result = try_create_histogram( + "http_server_request_duration_seconds", + "Time taken to build a response to a HTTP request" + ); + pub static ref REQUEST_COUNT: Result = try_create_int_counter( + "http_server_request_total", + "Total count of HTTP requests received" + ); + pub static ref SUCCESS_COUNT: Result = try_create_int_counter( + "http_server_success_total", + "Total count of HTTP 200 responses sent" + ); +} + +/// Returns the full set of Prometheus metrics for the Beacon Node application. +/// +/// # Note +/// +/// This is a HTTP handler method. +pub fn get_prometheus(req: Request) -> ApiResult { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + let db_path = req + .extensions() + .get::() + .ok_or_else(|| ApiError::ServerError("DBPath extension missing".to_string()))?; + + // There are two categories of metrics: + // + // - Dynamically updated: things like histograms and event counters that are updated on the + // fly. + // - Statically updated: things which are only updated at the time of the scrape (used where we + // can avoid cluttering up code with metrics calls). + // + // The `lighthouse_metrics` crate has a `DEFAULT_REGISTRY` global singleton (via `lazy_static`) + // which keeps the state of all the metrics. Dynamically updated things will already be + // up-to-date in the registry (because they update themselves) however statically updated + // things need to be "scraped". + // + // We proceed by, first updating all the static metrics using `scrape_for_metrics(..)`. Then, + // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into + // a string that can be returned via HTTP. + + slot_clock::scrape_for_metrics::(&beacon_chain.slot_clock); + store::scrape_for_metrics(&db_path); + beacon_chain::scrape_for_metrics(&beacon_chain); + + encoder + .encode(&lighthouse_metrics::gather(), &mut buffer) + .unwrap(); + + String::from_utf8(buffer) + .map(|string| success_response(Body::from(string))) + .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e))) +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 2e3ad06918..9a52f2638c 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -128,28 +128,6 @@ fn main() { .help("Listen port for RPC endpoint.") .takes_value(true), ) - /* - * HTTP server parameters. - */ - .arg( - Arg::with_name("http") - .long("http") - .help("Enable the HTTP server.") - .takes_value(false), - ) - .arg( - Arg::with_name("http-address") - .long("http-address") - .value_name("Address") - .help("Listen address for the HTTP server.") - .takes_value(true), - ) - .arg( - Arg::with_name("http-port") - .long("http-port") - .help("Listen port for the HTTP server.") - .takes_value(true), - ) /* Client related arguments */ .arg( Arg::with_name("api") diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index 9607e8b8e5..cd9711253e 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -15,3 +15,5 @@ eth2_ssz = "0.1" eth2_ssz_derive = "0.1" tree_hash = "0.1" types = { path = "../../eth2/types" } +lazy_static = "1.3.0" +lighthouse_metrics = { path = "../../eth2/utils/lighthouse_metrics" } diff --git a/beacon_node/store/src/impls.rs b/beacon_node/store/src/impls.rs index e88b70f396..ed724480c3 100644 --- a/beacon_node/store/src/impls.rs +++ b/beacon_node/store/src/impls.rs @@ -9,10 +9,26 @@ impl StoreItem for BeaconBlock { } fn as_store_bytes(&self) -> Vec { - self.as_ssz_bytes() + let timer = metrics::start_timer(&metrics::BEACON_BLOCK_WRITE_TIMES); + let bytes = self.as_ssz_bytes(); + + metrics::stop_timer(timer); + metrics::inc_counter(&metrics::BEACON_BLOCK_WRITE_COUNT); + metrics::inc_counter_by(&metrics::BEACON_BLOCK_WRITE_BYTES, bytes.len() as i64); + + bytes } fn from_store_bytes(bytes: &mut [u8]) -> Result { - Self::from_ssz_bytes(bytes).map_err(Into::into) + let timer = metrics::start_timer(&metrics::BEACON_BLOCK_READ_TIMES); + + let len = bytes.len(); + let result = Self::from_ssz_bytes(bytes).map_err(Into::into); + + metrics::stop_timer(timer); + metrics::inc_counter(&metrics::BEACON_BLOCK_READ_COUNT); + metrics::inc_counter_by(&metrics::BEACON_BLOCK_READ_BYTES, len as i64); + + result } } diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs index 591663fe05..69e83cd636 100644 --- a/beacon_node/store/src/impls/beacon_state.rs +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -53,12 +53,29 @@ impl StoreItem for BeaconState { } fn as_store_bytes(&self) -> Vec { + let timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES); + let container = StorageContainer::new(self); - container.as_ssz_bytes() + let bytes = container.as_ssz_bytes(); + + metrics::stop_timer(timer); + metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT); + metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64); + + bytes } fn from_store_bytes(bytes: &mut [u8]) -> Result { + let timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES); + + let len = bytes.len(); let container = StorageContainer::from_ssz_bytes(bytes)?; - container.try_into() + let result = container.try_into(); + + metrics::stop_timer(timer); + metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT); + metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, len as i64); + + result } } diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index 699861e3ae..a085d845a8 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -1,4 +1,5 @@ use super::*; +use crate::metrics; use db_key::Key; use leveldb::database::kv::KV; use leveldb::database::Database; @@ -62,15 +63,27 @@ impl Store for LevelDB { fn get_bytes(&self, col: &str, key: &[u8]) -> Result>, Error> { let column_key = Self::get_key_for_col(col, key); - self.db + metrics::inc_counter(&metrics::DISK_DB_READ_COUNT); + + let result = self + .db .get(self.read_options(), column_key) - .map_err(Into::into) + .map_err(Into::into); + + if let Ok(Some(bytes)) = &result { + metrics::inc_counter_by(&metrics::DISK_DB_READ_BYTES, bytes.len() as i64) + } + + result } /// Store some `value` in `column`, indexed with `key`. fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> { let column_key = Self::get_key_for_col(col, key); + metrics::inc_counter(&metrics::DISK_DB_WRITE_COUNT); + metrics::inc_counter_by(&metrics::DISK_DB_WRITE_BYTES, val.len() as i64); + self.db .put(self.write_options(), column_key, val) .map_err(Into::into) @@ -80,6 +93,8 @@ impl Store for LevelDB { fn key_exists(&self, col: &str, key: &[u8]) -> Result { let column_key = Self::get_key_for_col(col, key); + metrics::inc_counter(&metrics::DISK_DB_EXISTS_COUNT); + self.db .get(self.read_options(), column_key) .map_err(Into::into) @@ -89,6 +104,9 @@ impl Store for LevelDB { /// Removes `key` from `column`. fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> { let column_key = Self::get_key_for_col(col, key); + + metrics::inc_counter(&metrics::DISK_DB_DELETE_COUNT); + self.db .delete(self.write_options(), column_key) .map_err(Into::into) diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 5b8d583200..9c0e3cbaec 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -7,18 +7,22 @@ //! //! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See //! tests for implementation examples. +#[macro_use] +extern crate lazy_static; mod block_at_slot; mod errors; mod impls; mod leveldb_store; mod memory_store; +mod metrics; pub mod iter; pub use self::leveldb_store::LevelDB as DiskStore; pub use self::memory_store::MemoryStore; pub use errors::Error; +pub use metrics::scrape_for_metrics; pub use types::*; /// An object capable of storing and retrieving objects implementing `StoreItem`. diff --git a/beacon_node/store/src/metrics.rs b/beacon_node/store/src/metrics.rs new file mode 100644 index 0000000000..90237824d2 --- /dev/null +++ b/beacon_node/store/src/metrics.rs @@ -0,0 +1,106 @@ +pub use lighthouse_metrics::{set_gauge, try_create_int_gauge, *}; + +use std::fs; +use std::path::PathBuf; + +lazy_static! { + /* + * General + */ + pub static ref DISK_DB_SIZE: Result = + try_create_int_gauge("store_disk_db_size", "Size of the on-disk database (bytes)"); + pub static ref DISK_DB_WRITE_BYTES: Result = try_create_int_counter( + "store_disk_db_write_bytes_total", + "Number of bytes attempted to be written to the on-disk DB" + ); + pub static ref DISK_DB_READ_BYTES: Result = try_create_int_counter( + "store_disk_db_read_bytes_total", + "Number of bytes read from the on-disk DB" + ); + pub static ref DISK_DB_READ_COUNT: Result = try_create_int_counter( + "store_disk_db_read_count_total", + "Total number of reads to the on-disk DB" + ); + pub static ref DISK_DB_WRITE_COUNT: Result = try_create_int_counter( + "store_disk_db_write_count_total", + "Total number of writes to the on-disk DB" + ); + pub static ref DISK_DB_EXISTS_COUNT: Result = try_create_int_counter( + "store_disk_db_exists_count_total", + "Total number of checks if a key is in the on-disk DB" + ); + pub static ref DISK_DB_DELETE_COUNT: Result = try_create_int_counter( + "store_disk_db_delete_count_total", + "Total number of deletions from the on-disk DB" + ); + /* + * Beacon State + */ + pub static ref BEACON_STATE_READ_TIMES: Result = try_create_histogram( + "store_beacon_state_read_overhead_seconds", + "Overhead on reading a beacon state from the DB (e.g., decoding)" + ); + pub static ref BEACON_STATE_READ_COUNT: Result = try_create_int_counter( + "store_beacon_state_read_total", + "Total number of beacon state reads from the DB" + ); + pub static ref BEACON_STATE_READ_BYTES: Result = try_create_int_counter( + "store_beacon_state_read_bytes_total", + "Total number of beacon state bytes read from the DB" + ); + pub static ref BEACON_STATE_WRITE_TIMES: Result = try_create_histogram( + "store_beacon_state_write_overhead_seconds", + "Overhead on writing a beacon state to the DB (e.g., encoding)" + ); + pub static ref BEACON_STATE_WRITE_COUNT: Result = try_create_int_counter( + "store_beacon_state_write_total", + "Total number of beacon state writes the DB" + ); + pub static ref BEACON_STATE_WRITE_BYTES: Result = try_create_int_counter( + "store_beacon_state_write_bytes_total", + "Total number of beacon state bytes written to the DB" + ); + /* + * Beacon Block + */ + pub static ref BEACON_BLOCK_READ_TIMES: Result = try_create_histogram( + "store_beacon_block_read_overhead_seconds", + "Overhead on reading a beacon block from the DB (e.g., decoding)" + ); + pub static ref BEACON_BLOCK_READ_COUNT: Result = try_create_int_counter( + "store_beacon_block_read_total", + "Total number of beacon block reads from the DB" + ); + pub static ref BEACON_BLOCK_READ_BYTES: Result = try_create_int_counter( + "store_beacon_block_read_bytes_total", + "Total number of beacon block bytes read from the DB" + ); + pub static ref BEACON_BLOCK_WRITE_TIMES: Result = try_create_histogram( + "store_beacon_block_write_overhead_seconds", + "Overhead on writing a beacon block to the DB (e.g., encoding)" + ); + pub static ref BEACON_BLOCK_WRITE_COUNT: Result = try_create_int_counter( + "store_beacon_block_write_total", + "Total number of beacon block writes the DB" + ); + pub static ref BEACON_BLOCK_WRITE_BYTES: Result = try_create_int_counter( + "store_beacon_block_write_bytes_total", + "Total number of beacon block bytes written to the DB" + ); +} + +/// Updates the global metrics registry with store-related information. +pub fn scrape_for_metrics(db_path: &PathBuf) { + let db_size = if let Ok(iter) = fs::read_dir(db_path) { + iter.filter_map(std::result::Result::ok) + .map(size_of_dir_entry) + .fold(0_u64, |sum, val| sum + val) + } else { + 0 + }; + set_gauge(&DISK_DB_SIZE, db_size as i64); +} + +fn size_of_dir_entry(dir: fs::DirEntry) -> u64 { + dir.metadata().map(|m| m.len()).unwrap_or(0) +} diff --git a/docs/config_examples/beacon-node.toml b/docs/config_examples/beacon-node.toml index 3c9f8b613c..f0863934e7 100644 --- a/docs/config_examples/beacon-node.toml +++ b/docs/config_examples/beacon-node.toml @@ -78,14 +78,6 @@ enabled = false listen_address = "127.0.0.1" port = 5051 -# -# Legacy HTTP server configuration. To be removed. -# -[http] -enabled = false -listen_address = "127.0.0.1" -listen_port = "5052" - # # RESTful HTTP API server configuration. # @@ -95,4 +87,4 @@ enabled = true # The listen port for the HTTP server. listen_address = "127.0.0.1" # The listen port for the HTTP server. -port = 1248 +port = 5052 diff --git a/eth2/utils/lighthouse_metrics/Cargo.toml b/eth2/utils/lighthouse_metrics/Cargo.toml new file mode 100644 index 0000000000..0a24a96fb5 --- /dev/null +++ b/eth2/utils/lighthouse_metrics/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "lighthouse_metrics" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +lazy_static = "1.3.0" +prometheus = "^0.6" diff --git a/eth2/utils/lighthouse_metrics/src/lib.rs b/eth2/utils/lighthouse_metrics/src/lib.rs new file mode 100644 index 0000000000..9022a5b684 --- /dev/null +++ b/eth2/utils/lighthouse_metrics/src/lib.rs @@ -0,0 +1,129 @@ +//! A wrapper around the `prometheus` crate that provides a global, `lazy_static` metrics registry +//! and functions to add and use the following components (more info at +//! [Prometheus docs](https://prometheus.io/docs/concepts/metric_types/)): +//! +//! - `Histogram`: used with `start_timer(..)` and `stop_timer(..)` to record durations (e.g., +//! block processing time). +//! - `IncCounter`: used to represent an ideally ever-growing, never-shrinking integer (e.g., +//! number of block processing requests). +//! - `IntGauge`: used to represent an varying integer (e.g., number of attestations per block). +//! +//! ## Important +//! +//! Metrics will fail if two items have the same `name`. All metrics must have a unique `name`. +//! Because we use a global registry there is no namespace per crate, it's one big global space. +//! +//! See the [Prometheus naming best practices](https://prometheus.io/docs/practices/naming/) when +//! choosing metric names. +//! +//! ## Example +//! +//! ```rust +//! #[macro_use] +//! extern crate lazy_static; +//! use lighthouse_metrics::*; +//! +//! // These metrics are "magically" linked to the global registry defined in `lighthouse_metrics`. +//! lazy_static! { +//! pub static ref RUN_COUNT: Result = try_create_int_counter( +//! "runs_total", +//! "Total number of runs" +//! ); +//! pub static ref CURRENT_VALUE: Result = try_create_int_gauge( +//! "current_value", +//! "The current value" +//! ); +//! pub static ref RUN_TIME: Result = +//! try_create_histogram("run_seconds", "Time taken (measured to high precision)"); +//! } +//! +//! +//! fn main() { +//! for i in 0..100 { +//! inc_counter(&RUN_COUNT); +//! let timer = start_timer(&RUN_TIME); +//! +//! for j in 0..10 { +//! set_gauge(&CURRENT_VALUE, j); +//! println!("Howdy partner"); +//! } +//! +//! stop_timer(timer); +//! } +//! } +//! ``` + +use prometheus::{HistogramOpts, HistogramTimer, Opts}; + +pub use prometheus::{Histogram, IntCounter, IntGauge, Result}; + +/// Collect all the metrics for reporting. +pub fn gather() -> Vec { + prometheus::gather() +} + +/// Attempts to crate an `IntCounter`, returning `Err` if the registry does not accept the counter +/// (potentially due to naming conflict). +pub fn try_create_int_counter(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let counter = IntCounter::with_opts(opts)?; + prometheus::register(Box::new(counter.clone()))?; + Ok(counter) +} + +/// Attempts to crate an `IntGauge`, returning `Err` if the registry does not accept the counter +/// (potentially due to naming conflict). +pub fn try_create_int_gauge(name: &str, help: &str) -> Result { + let opts = Opts::new(name, help); + let gauge = IntGauge::with_opts(opts)?; + prometheus::register(Box::new(gauge.clone()))?; + Ok(gauge) +} + +/// Attempts to crate a `Histogram`, returning `Err` if the registry does not accept the counter +/// (potentially due to naming conflict). +pub fn try_create_histogram(name: &str, help: &str) -> Result { + let opts = HistogramOpts::new(name, help); + let histogram = Histogram::with_opts(opts)?; + prometheus::register(Box::new(histogram.clone()))?; + Ok(histogram) +} + +/// Starts a timer for the given `Histogram`, stopping when it gets dropped or given to `stop_timer(..)`. +pub fn start_timer(histogram: &Result) -> Option { + if let Ok(histogram) = histogram { + Some(histogram.start_timer()) + } else { + None + } +} + +/// Stops a timer created with `start_timer(..)`. +pub fn stop_timer(timer: Option) { + timer.map(|t| t.observe_duration()); +} + +pub fn inc_counter(counter: &Result) { + if let Ok(counter) = counter { + counter.inc(); + } +} + +pub fn inc_counter_by(counter: &Result, value: i64) { + if let Ok(counter) = counter { + counter.inc_by(value); + } +} + +pub fn set_gauge(gauge: &Result, value: i64) { + if let Ok(gauge) = gauge { + gauge.set(value); + } +} + +/// Sets the value of a `Histogram` manually. +pub fn observe(histogram: &Result, value: f64) { + if let Ok(histogram) = histogram { + histogram.observe(value); + } +} diff --git a/eth2/utils/slot_clock/Cargo.toml b/eth2/utils/slot_clock/Cargo.toml index 31a4357251..c4b9df5edd 100644 --- a/eth2/utils/slot_clock/Cargo.toml +++ b/eth2/utils/slot_clock/Cargo.toml @@ -6,3 +6,5 @@ edition = "2018" [dependencies] types = { path = "../../types" } +lazy_static = "1.3.0" +lighthouse_metrics = { path = "../lighthouse_metrics" } diff --git a/eth2/utils/slot_clock/src/lib.rs b/eth2/utils/slot_clock/src/lib.rs index 7b86684fa4..871743c9e6 100644 --- a/eth2/utils/slot_clock/src/lib.rs +++ b/eth2/utils/slot_clock/src/lib.rs @@ -1,9 +1,15 @@ +#[macro_use] +extern crate lazy_static; + +mod metrics; mod system_time_slot_clock; mod testing_slot_clock; +use std::time::Duration; + pub use crate::system_time_slot_clock::{Error as SystemTimeSlotClockError, SystemTimeSlotClock}; pub use crate::testing_slot_clock::{Error as TestingSlotClockError, TestingSlotClock}; -use std::time::Duration; +pub use metrics::scrape_for_metrics; pub use types::Slot; pub trait SlotClock: Send + Sync + Sized { @@ -17,4 +23,6 @@ pub trait SlotClock: Send + Sync + Sized { fn present_slot(&self) -> Result, Self::Error>; fn duration_to_next_slot(&self) -> Result, Self::Error>; + + fn slot_duration_millis(&self) -> u64; } diff --git a/eth2/utils/slot_clock/src/metrics.rs b/eth2/utils/slot_clock/src/metrics.rs new file mode 100644 index 0000000000..e0d3923e00 --- /dev/null +++ b/eth2/utils/slot_clock/src/metrics.rs @@ -0,0 +1,32 @@ +use crate::SlotClock; +pub use lighthouse_metrics::*; +use types::{EthSpec, Slot}; + +lazy_static! { + pub static ref PRESENT_SLOT: Result = + try_create_int_gauge("slotclock_present_slot", "The present wall-clock slot"); + pub static ref PRESENT_EPOCH: Result = + try_create_int_gauge("slotclock_present_epoch", "The present wall-clock epoch"); + pub static ref SLOTS_PER_EPOCH: Result = + try_create_int_gauge("slotclock_slots_per_epoch", "Slots per epoch (constant)"); + pub static ref MILLISECONDS_PER_SLOT: Result = try_create_int_gauge( + "slotclock_slot_time_milliseconds", + "The duration in milliseconds between each slot" + ); +} + +/// Update the global metrics `DEFAULT_REGISTRY` with info from the slot clock. +pub fn scrape_for_metrics(clock: &U) { + let present_slot = match clock.present_slot() { + Ok(Some(slot)) => slot, + _ => Slot::new(0), + }; + + set_gauge(&PRESENT_SLOT, present_slot.as_u64() as i64); + set_gauge( + &PRESENT_EPOCH, + present_slot.epoch(T::slots_per_epoch()).as_u64() as i64, + ); + set_gauge(&SLOTS_PER_EPOCH, T::slots_per_epoch() as i64); + set_gauge(&MILLISECONDS_PER_SLOT, clock.slot_duration_millis() as i64); +} diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 7c184b02bf..c493a8be83 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -52,6 +52,10 @@ impl SlotClock for SystemTimeSlotClock { fn duration_to_next_slot(&self) -> Result, Error> { duration_to_next_slot(self.genesis_seconds, self.slot_duration_seconds) } + + fn slot_duration_millis(&self) -> u64 { + self.slot_duration_seconds * 1000 + } } impl From for Error { diff --git a/eth2/utils/slot_clock/src/testing_slot_clock.rs b/eth2/utils/slot_clock/src/testing_slot_clock.rs index ab00d2baa7..f741d3b87a 100644 --- a/eth2/utils/slot_clock/src/testing_slot_clock.rs +++ b/eth2/utils/slot_clock/src/testing_slot_clock.rs @@ -40,6 +40,10 @@ impl SlotClock for TestingSlotClock { fn duration_to_next_slot(&self) -> Result, Error> { Ok(Some(Duration::from_secs(1))) } + + fn slot_duration_millis(&self) -> u64 { + 0 + } } #[cfg(test)] From 0c3fdcd57c2ffb08d36a2af4f038fe0ac92cc7d8 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 23 Aug 2019 15:53:53 +1000 Subject: [PATCH 3/3] Bootstrap (#501) * Renamed fork_choice::process_attestation_from_block * Processing attestation in fork choice * Retrieving state from store and checking signature * Looser check on beacon state validity. * Cleaned up get_attestation_state * Expanded fork choice api to provide latest validator message. * Checking if the an attestation contains a latest message * Correct process_attestation error handling. * Copy paste error in comment fixed. * Tidy ancestor iterators * Getting attestation slot via helper method * Refactored attestation creation in test utils * Revert "Refactored attestation creation in test utils" This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306. * Integration tests for free attestation processing * Implicit conflicts resolved. * formatting * Do first pass on Grants code * Add another attestation processing test * Tidy attestation processing * Remove old code fragment * Add non-compiling half finished changes * Simplify, fix bugs, add tests for chain iters * Remove attestation processing from op pool * Fix bug with fork choice, tidy * Fix overly restrictive check in fork choice. * Ensure committee cache is build during attn proc * Ignore unknown blocks at fork choice * Various minor fixes * Make fork choice write lock in to read lock * Remove unused method * Tidy comments * Fix attestation prod. target roots change * Fix compile error in store iters * Reject any attestation prior to finalization * Begin metrics refactor * Move beacon_chain to new metrics structure. * Make metrics not panic if already defined * Use global prometheus gather at rest api * Unify common metric fns into a crate * Add heavy metering to block processing * Remove hypen from prometheus metric name * Add more beacon chain metrics * Add beacon chain persistence metric * Prune op pool on finalization * Add extra prom beacon chain metrics * Prefix BeaconChain metrics with "beacon_" * Add more store metrics * Add basic metrics to libp2p * Add metrics to HTTP server * Remove old `http_server` crate * Update metrics names to be more like standard * Fix broken beacon chain metrics, add slot clock metrics * Add lighthouse_metrics gather fn * Remove http args * Fix wrong state given to op pool prune * Make prom metric names more consistent * Add more metrics, tidy existing metrics * Fix store block read metrics * Tidy attestation metrics * Fix minor PR comments * Fix minor PR comments * Remove duplicated attestation finalization check * Remove awkward `let` statement * Add first attempts at HTTP bootstrap * Add beacon_block methods to rest api * Fix serde for block.body.grafitti * Allow travis failures on beta (see desc) There's a non-backward compatible change in `cargo fmt`. Stable and beta do not agree. * Add network routes to API * Fix rustc warnings * Add best_slot method * Add --bootstrap arg to beacon node * Get bootstrapper working for ENR address * Store intermediate states during block processing * Allow bootstrapper to scrape libp2p address * Update bootstrapper libp2p address finding * Add comments * Tidy API to be more consistent with recent decisions * Address some review comments * Make BeaconChainTypes Send + Sync + 'static * Add `/network/listen_port` API endpoint * Abandon starting the node if libp2p doesn't start * Update bootstrapper for API changes * Remove unnecessary trait bounds --- beacon_node/beacon_chain/src/beacon_chain.rs | 27 ++- beacon_node/beacon_chain/src/test_utils.rs | 4 +- beacon_node/client/Cargo.toml | 3 + beacon_node/client/src/beacon_chain_types.rs | 17 +- beacon_node/client/src/bootstrapper.rs | 210 +++++++++++++++++++ beacon_node/client/src/config.rs | 40 +++- beacon_node/client/src/lib.rs | 7 +- beacon_node/client/src/notifier.rs | 6 +- beacon_node/eth2-libp2p/src/behaviour.rs | 4 + beacon_node/eth2-libp2p/src/discovery.rs | 9 + beacon_node/eth2-libp2p/src/lib.rs | 3 +- beacon_node/eth2-libp2p/src/service.rs | 19 +- beacon_node/network/src/service.rs | 50 ++++- beacon_node/rest_api/Cargo.toml | 2 + beacon_node/rest_api/src/beacon.rs | 150 ++++++++++++- beacon_node/rest_api/src/helpers.rs | 28 ++- beacon_node/rest_api/src/lib.rs | 30 ++- beacon_node/rest_api/src/network.rs | 108 ++++++++++ beacon_node/rest_api/src/spec.rs | 27 +++ beacon_node/src/main.rs | 15 ++ beacon_node/src/run.rs | 9 +- eth2/types/src/beacon_block_body.rs | 7 +- eth2/types/src/utils/serde_utils.rs | 16 +- 23 files changed, 742 insertions(+), 49 deletions(-) create mode 100644 beacon_node/client/src/bootstrapper.rs create mode 100644 beacon_node/rest_api/src/network.rs create mode 100644 beacon_node/rest_api/src/spec.rs diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e8e2f49dd2..5feefd8417 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -77,7 +77,7 @@ pub enum AttestationProcessingOutcome { Invalid(AttestationValidationError), } -pub trait BeaconChainTypes { +pub trait BeaconChainTypes: Send + Sync + 'static { type Store: store::Store; type SlotClock: slot_clock::SlotClock; type LmdGhost: LmdGhost; @@ -870,9 +870,16 @@ impl BeaconChain { let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE); + // Keep a list of any states that were "skipped" (block-less) in between the parent state + // slot and the block slot. These will need to be stored in the database. + let mut intermediate_states = vec![]; + // Transition the parent state to the block slot. let mut state: BeaconState = parent_state; - for _ in state.slot.as_u64()..block.slot.as_u64() { + for i in state.slot.as_u64()..block.slot.as_u64() { + if i > 0 { + intermediate_states.push(state.clone()); + } per_slot_processing(&mut state, &self.spec)?; } @@ -911,6 +918,22 @@ impl BeaconChain { let db_write_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_WRITE); + // Store all the states between the parent block state and this blocks slot before storing + // the final state. + for (i, intermediate_state) in intermediate_states.iter().enumerate() { + // To avoid doing an unnecessary tree hash, use the following (slot + 1) state's + // state_roots field to find the root. + let following_state = match intermediate_states.get(i + 1) { + Some(following_state) => following_state, + None => &state, + }; + let intermediate_state_root = + following_state.get_state_root(intermediate_state.slot)?; + + self.store + .put(&intermediate_state_root, intermediate_state)?; + } + // Store the block and state. self.store.put(&block_root, &block)?; self.store.put(&state_root, &state)?; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 298c637dbd..09f4749ea3 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -54,7 +54,7 @@ where impl BeaconChainTypes for CommonTypes where - L: LmdGhost, + L: LmdGhost + 'static, E: EthSpec, { type Store = MemoryStore; @@ -69,7 +69,7 @@ where /// Used for testing. pub struct BeaconChainHarness where - L: LmdGhost, + L: LmdGhost + 'static, E: EthSpec, { pub chain: BeaconChain>, diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index b13f175a9b..9b5a9cf42c 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -7,6 +7,7 @@ edition = "2018" [dependencies] beacon_chain = { path = "../beacon_chain" } network = { path = "../network" } +eth2-libp2p = { path = "../eth2-libp2p" } rpc = { path = "../rpc" } rest_api = { path = "../rest_api" } prometheus = "^0.6" @@ -26,3 +27,5 @@ clap = "2.32.0" dirs = "1.0.3" exit-future = "0.1.3" futures = "0.1.25" +reqwest = "0.9" +url = "1.2" diff --git a/beacon_node/client/src/beacon_chain_types.rs b/beacon_node/client/src/beacon_chain_types.rs index 0b86c95838..5168c067a9 100644 --- a/beacon_node/client/src/beacon_chain_types.rs +++ b/beacon_node/client/src/beacon_chain_types.rs @@ -1,3 +1,4 @@ +use crate::bootstrapper::Bootstrapper; use crate::error::Result; use crate::{config::GenesisState, ClientConfig}; use beacon_chain::{ @@ -35,7 +36,11 @@ pub struct ClientType { _phantom_u: PhantomData, } -impl BeaconChainTypes for ClientType { +impl BeaconChainTypes for ClientType +where + S: Store + 'static, + E: EthSpec, +{ type Store = S; type SlotClock = SystemTimeSlotClock; type LmdGhost = ThreadSafeReducedTree; @@ -74,6 +79,16 @@ where serde_yaml::from_reader(file) .map_err(|e| format!("Unable to parse YAML genesis state file: {:?}", e))? } + GenesisState::HttpBootstrap { server } => { + let bootstrapper = Bootstrapper::from_server_string(server.to_string()) + .map_err(|e| format!("Failed to initialize bootstrap client: {}", e))?; + + let (state, _block) = bootstrapper + .genesis() + .map_err(|e| format!("Failed to bootstrap genesis state: {}", e))?; + + state + } }; let mut genesis_block = BeaconBlock::empty(&spec); diff --git a/beacon_node/client/src/bootstrapper.rs b/beacon_node/client/src/bootstrapper.rs new file mode 100644 index 0000000000..c94d9a51d8 --- /dev/null +++ b/beacon_node/client/src/bootstrapper.rs @@ -0,0 +1,210 @@ +use eth2_libp2p::{ + multiaddr::{Multiaddr, Protocol}, + Enr, +}; +use reqwest::{Error as HttpError, Url}; +use serde::Deserialize; +use std::borrow::Cow; +use std::net::Ipv4Addr; +use types::{BeaconBlock, BeaconState, Checkpoint, EthSpec, Hash256, Slot}; +use url::Host; + +#[derive(Debug)] +enum Error { + InvalidUrl, + HttpError(HttpError), +} + +impl From for Error { + fn from(e: HttpError) -> Error { + Error::HttpError(e) + } +} + +/// Used to load "bootstrap" information from the HTTP API of another Lighthouse beacon node. +/// +/// Bootstrapping information includes things like genesis and finalized states and blocks, and +/// libp2p connection details. +pub struct Bootstrapper { + url: Url, +} + +impl Bootstrapper { + /// Parses the given `server` as a URL, instantiating `Self`. + pub fn from_server_string(server: String) -> Result { + Ok(Self { + url: Url::parse(&server).map_err(|e| format!("Invalid bootstrap server url: {}", e))?, + }) + } + + /// Build a multiaddr using the HTTP server URL that is not guaranteed to be correct. + /// + /// The address is created by querying the HTTP server for its listening libp2p addresses. + /// Then, we find the first TCP port in those addresses and combine the port with the URL of + /// the server. + /// + /// For example, the server `http://192.168.0.1` might end up with a `best_effort_multiaddr` of + /// `/ipv4/192.168.0.1/tcp/9000` if the server advertises a listening address of + /// `/ipv4/172.0.0.1/tcp/9000`. + pub fn best_effort_multiaddr(&self) -> Option { + let tcp_port = self.listen_port().ok()?; + + let mut multiaddr = Multiaddr::with_capacity(2); + + match self.url.host()? { + Host::Ipv4(addr) => multiaddr.push(Protocol::Ip4(addr)), + Host::Domain(s) => multiaddr.push(Protocol::Dns4(Cow::Borrowed(s))), + _ => return None, + }; + + multiaddr.push(Protocol::Tcp(tcp_port)); + + Some(multiaddr) + } + + /// Returns the IPv4 address of the server URL, unless it contains a FQDN. + pub fn server_ipv4_addr(&self) -> Option { + match self.url.host()? { + Host::Ipv4(addr) => Some(addr), + _ => None, + } + } + + /// Returns the servers ENR address. + pub fn enr(&self) -> Result { + get_enr(self.url.clone()).map_err(|e| format!("Unable to get ENR: {:?}", e)) + } + + /// Returns the servers listening libp2p addresses. + pub fn listen_port(&self) -> Result { + get_listen_port(self.url.clone()).map_err(|e| format!("Unable to get listen port: {:?}", e)) + } + + /// Returns the genesis block and state. + pub fn genesis(&self) -> Result<(BeaconState, BeaconBlock), String> { + let genesis_slot = Slot::new(0); + + let block = get_block(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), genesis_slot) + .map_err(|e| format!("Unable to get genesis state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } + + /// Returns the most recent finalized state and block. + pub fn finalized(&self) -> Result<(BeaconState, BeaconBlock), String> { + let slots_per_epoch = get_slots_per_epoch(self.url.clone()) + .map_err(|e| format!("Unable to get slots per epoch: {:?}", e))?; + let finalized_slot = get_finalized_slot(self.url.clone(), slots_per_epoch.as_u64()) + .map_err(|e| format!("Unable to get finalized slot: {:?}", e))?; + + let block = get_block(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized block: {:?}", e))? + .beacon_block; + let state = get_state(self.url.clone(), finalized_slot) + .map_err(|e| format!("Unable to get finalized state: {:?}", e))? + .beacon_state; + + Ok((state, block)) + } +} + +fn get_slots_per_epoch(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("spec").push("slots_per_epoch"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_finalized_slot(mut url: Url, slots_per_epoch: u64) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("latest_finalized_checkpoint"); + }) + .map_err(|_| Error::InvalidUrl)?; + + let checkpoint: Checkpoint = reqwest::get(url)?.error_for_status()?.json()?; + + Ok(checkpoint.epoch.start_slot(slots_per_epoch)) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct StateResponse { + pub root: Hash256, + pub beacon_state: BeaconState, +} + +fn get_state(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("state"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +#[derive(Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlockResponse { + pub root: Hash256, + pub beacon_block: BeaconBlock, +} + +fn get_block(mut url: Url, slot: Slot) -> Result, Error> { + url.path_segments_mut() + .map(|mut url| { + url.push("beacon").push("block"); + }) + .map_err(|_| Error::InvalidUrl)?; + + url.query_pairs_mut() + .append_pair("slot", &format!("{}", slot.as_u64())); + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_enr(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("enr"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} + +fn get_listen_port(mut url: Url) -> Result { + url.path_segments_mut() + .map(|mut url| { + url.push("network").push("listen_port"); + }) + .map_err(|_| Error::InvalidUrl)?; + + reqwest::get(url)? + .error_for_status()? + .json() + .map_err(Into::into) +} diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index fcc2cc7dac..ea8186dbc9 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,8 +1,8 @@ -use crate::Eth2Config; +use crate::{Bootstrapper, Eth2Config}; use clap::ArgMatches; use network::NetworkConfig; use serde_derive::{Deserialize, Serialize}; -use slog::{info, o, Drain}; +use slog::{info, o, warn, Drain}; use std::fs::{self, OpenOptions}; use std::path::PathBuf; use std::sync::Mutex; @@ -46,6 +46,8 @@ pub enum GenesisState { }, /// Load a YAML-encoded genesis state from a file. Yaml { file: PathBuf }, + /// Use a HTTP server (running our REST-API) to load genesis and finalized states and blocks. + HttpBootstrap { server: String }, } impl Default for Config { @@ -147,6 +149,40 @@ impl Config { self.update_logger(log)?; }; + // If the `--bootstrap` flag is provided, overwrite the default configuration. + if let Some(server) = args.value_of("bootstrap") { + do_bootstrapping(self, server.to_string(), &log)?; + } + Ok(()) } } + +/// Perform the HTTP bootstrapping procedure, reading an ENR and multiaddr from the HTTP server and +/// adding them to the `config`. +fn do_bootstrapping(config: &mut Config, server: String, log: &slog::Logger) -> Result<(), String> { + // Set the genesis state source. + config.genesis_state = GenesisState::HttpBootstrap { + server: server.to_string(), + }; + + let bootstrapper = Bootstrapper::from_server_string(server.to_string())?; + + config.network.boot_nodes.push(bootstrapper.enr()?); + + if let Some(server_multiaddr) = bootstrapper.best_effort_multiaddr() { + info!( + log, + "Estimated bootstrapper libp2p address"; + "multiaddr" => format!("{:?}", server_multiaddr) + ); + config.network.libp2p_nodes.push(server_multiaddr); + } else { + warn!( + log, + "Unable to estimate a bootstrapper libp2p address, this node may not find any peers." + ); + } + + Ok(()) +} diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 5c37ac3e9c..6405e05e71 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -1,6 +1,7 @@ extern crate slog; mod beacon_chain_types; +mod bootstrapper; mod config; pub mod error; @@ -21,7 +22,8 @@ use tokio::timer::Interval; pub use beacon_chain::BeaconChainTypes; pub use beacon_chain_types::ClientType; pub use beacon_chain_types::InitialiseBeaconChain; -pub use config::Config as ClientConfig; +pub use bootstrapper::Bootstrapper; +pub use config::{Config as ClientConfig, GenesisState}; pub use eth2_config::Eth2Config; /// Main beacon node client service. This provides the connection and initialisation of the clients @@ -47,7 +49,7 @@ pub struct Client { impl Client where - T: BeaconChainTypes + InitialiseBeaconChain + Clone + 'static, + T: BeaconChainTypes + InitialiseBeaconChain + Clone, { /// Generate an instance of the client. Spawn and link all internal sub-processes. pub fn new( @@ -121,6 +123,7 @@ where &client_config.rest_api, executor, beacon_chain.clone(), + network.clone(), client_config.db_path().expect("unable to read datadir"), &log, ) { diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 1c7cf38670..78e50ac79d 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -17,11 +17,7 @@ pub const WARN_PEER_COUNT: usize = 1; /// durations. /// /// Presently unused, but remains for future use. -pub fn run( - client: &Client, - executor: TaskExecutor, - exit: Exit, -) { +pub fn run(client: &Client, executor: TaskExecutor, exit: Exit) { // notification heartbeat let interval = Interval::new( Instant::now(), diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index b87f8a0613..9158fe4858 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -78,6 +78,10 @@ impl Behaviour { log: behaviour_log, }) } + + pub fn discovery(&self) -> &Discovery { + &self.discovery + } } // Implement the NetworkBehaviourEventProcess trait so that we can derive NetworkBehaviour for Behaviour diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index ca98db3246..87d5dd5581 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -103,6 +103,10 @@ impl Discovery { }) } + pub fn local_enr(&self) -> &Enr { + self.discovery.local_enr() + } + /// Manually search for peers. This restarts the discovery round, sparking multiple rapid /// queries. pub fn discover_peers(&mut self) { @@ -120,6 +124,11 @@ impl Discovery { self.connected_peers.len() } + /// The current number of connected libp2p peers. + pub fn connected_peer_set(&self) -> &HashSet { + &self.connected_peers + } + /// Search for new peers using the underlying discovery mechanism. fn find_peers(&mut self) { // pick a random NodeId diff --git a/beacon_node/eth2-libp2p/src/lib.rs b/beacon_node/eth2-libp2p/src/lib.rs index 33d5ba9ed9..4c84469cea 100644 --- a/beacon_node/eth2-libp2p/src/lib.rs +++ b/beacon_node/eth2-libp2p/src/lib.rs @@ -17,12 +17,13 @@ pub use behaviour::PubsubMessage; pub use config::{ Config as NetworkConfig, BEACON_ATTESTATION_TOPIC, BEACON_BLOCK_TOPIC, SHARD_TOPIC_PREFIX, }; +pub use libp2p::enr::Enr; pub use libp2p::gossipsub::{Topic, TopicHash}; pub use libp2p::multiaddr; pub use libp2p::Multiaddr; pub use libp2p::{ gossipsub::{GossipsubConfig, GossipsubConfigBuilder}, - PeerId, + PeerId, Swarm, }; pub use rpc::RPCEvent; pub use service::Libp2pEvent; diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 316aa05798..e208dbecac 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -16,7 +16,7 @@ use libp2p::core::{ upgrade::{InboundUpgradeExt, OutboundUpgradeExt}, }; use libp2p::{core, secio, PeerId, Swarm, Transport}; -use slog::{debug, info, trace, warn}; +use slog::{crit, debug, info, trace, warn}; use std::fs::File; use std::io::prelude::*; use std::io::{Error, ErrorKind}; @@ -33,7 +33,7 @@ pub struct Service { //TODO: Make this private pub swarm: Swarm, /// This node's PeerId. - _local_peer_id: PeerId, + pub local_peer_id: PeerId, /// The libp2p logger handle. pub log: slog::Logger, } @@ -69,10 +69,15 @@ impl Service { log_address.push(Protocol::P2p(local_peer_id.clone().into())); info!(log, "Listening on: {}", log_address); } - Err(err) => warn!( - log, - "Cannot listen on: {} because: {:?}", listen_multiaddr, err - ), + Err(err) => { + crit!( + log, + "Unable to listen on libp2p address"; + "error" => format!("{:?}", err), + "listen_multiaddr" => format!("{}", listen_multiaddr), + ); + return Err("Libp2p was unable to listen on the given listen address.".into()); + } }; // attempt to connect to user-input libp2p nodes @@ -113,7 +118,7 @@ impl Service { info!(log, "Subscribed to topics: {:?}", subscribed_topics); Ok(Service { - _local_peer_id: local_peer_id, + local_peer_id, swarm, log, }) diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index e5ca2a9175..152f4dc77d 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -5,7 +5,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes}; use core::marker::PhantomData; use eth2_libp2p::Service as LibP2PService; use eth2_libp2p::Topic; -use eth2_libp2p::{Libp2pEvent, PeerId}; +use eth2_libp2p::{Enr, Libp2pEvent, Multiaddr, PeerId, Swarm}; use eth2_libp2p::{PubsubMessage, RPCEvent}; use futures::prelude::*; use futures::Stream; @@ -18,6 +18,7 @@ use tokio::sync::{mpsc, oneshot}; /// Service that handles communication between internal services and the eth2_libp2p network service. pub struct Service { libp2p_service: Arc>, + libp2p_port: u16, _libp2p_exit: oneshot::Sender<()>, _network_send: mpsc::UnboundedSender, _phantom: PhantomData, //message_handler: MessageHandler, @@ -56,6 +57,7 @@ impl Service { )?; let network_service = Service { libp2p_service, + libp2p_port: config.libp2p_port, _libp2p_exit: libp2p_exit, _network_send: network_send.clone(), _phantom: PhantomData, @@ -64,6 +66,52 @@ impl Service { Ok((Arc::new(network_service), network_send)) } + /// Returns the local ENR from the underlying Discv5 behaviour that external peers may connect + /// to. + pub fn local_enr(&self) -> Enr { + self.libp2p_service + .lock() + .swarm + .discovery() + .local_enr() + .clone() + } + + /// Returns the local libp2p PeerID. + pub fn local_peer_id(&self) -> PeerId { + self.libp2p_service.lock().local_peer_id.clone() + } + + /// Returns the list of `Multiaddr` that the underlying libp2p instance is listening on. + pub fn listen_multiaddrs(&self) -> Vec { + Swarm::listeners(&self.libp2p_service.lock().swarm) + .cloned() + .collect() + } + + /// Returns the libp2p port that this node has been configured to listen using. + pub fn listen_port(&self) -> u16 { + self.libp2p_port + } + + /// Returns the number of libp2p connected peers. + pub fn connected_peers(&self) -> usize { + self.libp2p_service.lock().swarm.connected_peers() + } + + /// Returns the set of `PeerId` that are connected via libp2p. + pub fn connected_peer_set(&self) -> Vec { + self.libp2p_service + .lock() + .swarm + .discovery() + .connected_peer_set() + .iter() + .cloned() + .collect() + } + + /// Provides a reference to the underlying libp2p service. pub fn libp2p_service(&self) -> Arc> { self.libp2p_service.clone() } diff --git a/beacon_node/rest_api/Cargo.toml b/beacon_node/rest_api/Cargo.toml index c7026014c4..cac196d9cb 100644 --- a/beacon_node/rest_api/Cargo.toml +++ b/beacon_node/rest_api/Cargo.toml @@ -7,6 +7,8 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] beacon_chain = { path = "../beacon_chain" } +network = { path = "../network" } +eth2-libp2p = { path = "../eth2-libp2p" } store = { path = "../store" } version = { path = "../version" } serde = { version = "1.0", features = ["derive"] } diff --git a/beacon_node/rest_api/src/beacon.rs b/beacon_node/rest_api/src/beacon.rs index cef23abe81..1c66a2819f 100644 --- a/beacon_node/rest_api/src/beacon.rs +++ b/beacon_node/rest_api/src/beacon.rs @@ -2,9 +2,114 @@ use super::{success_response, ApiResult}; use crate::{helpers::*, ApiError, UrlQuery}; use beacon_chain::{BeaconChain, BeaconChainTypes}; use hyper::{Body, Request}; +use serde::Serialize; use std::sync::Arc; use store::Store; -use types::BeaconState; +use types::{BeaconBlock, BeaconState, EthSpec, Hash256, Slot}; + +#[derive(Serialize)] +pub struct HeadResponse { + pub slot: Slot, + pub block_root: Hash256, + pub state_root: Hash256, +} + +/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. +pub fn get_head(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let head = HeadResponse { + slot: beacon_chain.head().beacon_state.slot, + block_root: beacon_chain.head().beacon_block_root, + state_root: beacon_chain.head().beacon_state_root, + }; + + let json: String = serde_json::to_string(&head) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize HeadResponse: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +#[derive(Serialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlockResponse { + pub root: Hash256, + pub beacon_block: BeaconBlock, +} + +/// HTTP handler to return a `BeaconBlock` at a given `root` or `slot`. +pub fn get_block(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let query_params = ["root", "slot"]; + let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; + + let block_root = match (key.as_ref(), value) { + ("slot", value) => { + let target = parse_slot(&value)?; + + block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + })? + } + ("root", value) => parse_root(&value)?, + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), + }; + + let block = beacon_chain + .store + .get::>(&block_root)? + .ok_or_else(|| { + ApiError::NotFound(format!( + "Unable to find BeaconBlock for root {}", + block_root + )) + })?; + + let response = BlockResponse { + root: block_root, + beacon_block: block, + }; + + let json: String = serde_json::to_string(&response).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize BlockResponse: {:?}", e)) + })?; + + Ok(success_response(Body::from(json))) +} + +/// HTTP handler to return a `BeaconBlock` root at a given `slot`. +pub fn get_block_root(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let slot_string = UrlQuery::from_request(&req)?.only_one("slot")?; + let target = parse_slot(&slot_string)?; + + let root = block_root_at_slot(&beacon_chain, target).ok_or_else(|| { + ApiError::NotFound(format!("Unable to find BeaconBlock for slot {}", target)) + })?; + + let json: String = serde_json::to_string(&root) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize root: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +#[derive(Serialize)] +#[serde(bound = "T: EthSpec")] +pub struct StateResponse { + pub root: Hash256, + pub beacon_state: BeaconState, +} /// HTTP handler to return a `BeaconState` at a given `root` or `slot`. /// @@ -19,26 +124,34 @@ pub fn get_state(req: Request) -> ApiResult let query_params = ["root", "slot"]; let (key, value) = UrlQuery::from_request(&req)?.first_of(&query_params)?; - let state: BeaconState = match (key.as_ref(), value) { + let (root, state): (Hash256, BeaconState) = match (key.as_ref(), value) { ("slot", value) => state_at_slot(&beacon_chain, parse_slot(&value)?)?, ("root", value) => { let root = &parse_root(&value)?; - beacon_chain + let state = beacon_chain .store .get(root)? - .ok_or_else(|| ApiError::NotFound(format!("No state for root: {}", root)))? + .ok_or_else(|| ApiError::NotFound(format!("No state for root: {}", root)))?; + + (*root, state) } - _ => unreachable!("Guarded by UrlQuery::from_request()"), + _ => return Err(ApiError::ServerError("Unexpected query parameter".into())), }; - let json: String = serde_json::to_string(&state) - .map_err(|e| ApiError::ServerError(format!("Unable to serialize BeaconState: {:?}", e)))?; + let response = StateResponse { + root, + beacon_state: state, + }; + + let json: String = serde_json::to_string(&response).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize StateResponse: {:?}", e)) + })?; Ok(success_response(Body::from(json))) } -/// HTTP handler to return a `BeaconState` root at a given or `slot`. +/// HTTP handler to return a `BeaconState` root at a given `slot`. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. @@ -58,3 +171,24 @@ pub fn get_state_root(req: Request) -> ApiR Ok(success_response(Body::from(json))) } + +/// HTTP handler to return the highest finalized slot. +pub fn get_latest_finalized_checkpoint( + req: Request, +) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let checkpoint = beacon_chain + .head() + .beacon_state + .finalized_checkpoint + .clone(); + + let json: String = serde_json::to_string(&checkpoint) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize checkpoint: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} diff --git a/beacon_node/rest_api/src/helpers.rs b/beacon_node/rest_api/src/helpers.rs index 2a429076c7..5365086df7 100644 --- a/beacon_node/rest_api/src/helpers.rs +++ b/beacon_node/rest_api/src/helpers.rs @@ -31,22 +31,40 @@ pub fn parse_root(string: &str) -> Result { } } -/// Returns a `BeaconState` in the canonical chain of `beacon_chain` at the given `slot`, if -/// possible. +/// Returns the root of the `BeaconBlock` in the canonical chain of `beacon_chain` at the given +/// `slot`, if possible. +/// +/// May return a root for a previous slot, in the case of skip slots. +pub fn block_root_at_slot( + beacon_chain: &BeaconChain, + target: Slot, +) -> Option { + beacon_chain + .rev_iter_block_roots() + .take_while(|(_root, slot)| *slot >= target) + .find(|(_root, slot)| *slot == target) + .map(|(root, _slot)| root) +} + +/// Returns a `BeaconState` and it's root in the canonical chain of `beacon_chain` at the given +/// `slot`, if possible. /// /// Will not return a state if the request slot is in the future. Will return states higher than /// the current head by skipping slots. pub fn state_at_slot( beacon_chain: &BeaconChain, slot: Slot, -) -> Result, ApiError> { +) -> Result<(Hash256, BeaconState), ApiError> { let head_state = &beacon_chain.head().beacon_state; if head_state.slot == slot { // The request slot is the same as the best block (head) slot. // I'm not sure if this `.clone()` will be optimized out. If not, it seems unnecessary. - Ok(beacon_chain.head().beacon_state.clone()) + Ok(( + beacon_chain.head().beacon_state_root, + beacon_chain.head().beacon_state.clone(), + )) } else { let root = state_root_at_slot(beacon_chain, slot)?; @@ -55,7 +73,7 @@ pub fn state_at_slot( .get(&root)? .ok_or_else(|| ApiError::NotFound(format!("Unable to find state at root {}", root)))?; - Ok(state) + Ok((root, state)) } } diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 57019deea0..964dd79982 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -1,15 +1,18 @@ #[macro_use] extern crate lazy_static; +extern crate network as client_network; mod beacon; mod config; mod helpers; mod metrics; +mod network; mod node; +mod spec; mod url_query; use beacon_chain::{BeaconChain, BeaconChainTypes}; -pub use config::Config as ApiConfig; +use client_network::Service as NetworkService; use hyper::rt::Future; use hyper::service::service_fn_ok; use hyper::{Body, Method, Response, Server, StatusCode}; @@ -20,6 +23,9 @@ use std::sync::Arc; use tokio::runtime::TaskExecutor; use url_query::UrlQuery; +pub use beacon::{BlockResponse, HeadResponse, StateResponse}; +pub use config::Config as ApiConfig; + #[derive(PartialEq, Debug)] pub enum ApiError { MethodNotAllowed(String), @@ -67,10 +73,11 @@ impl From for ApiError { } } -pub fn start_server( +pub fn start_server( config: &ApiConfig, executor: &TaskExecutor, beacon_chain: Arc>, + network_service: Arc>, db_path: PathBuf, log: &slog::Logger, ) -> Result { @@ -98,6 +105,7 @@ pub fn start_server( let log = server_log.clone(); let beacon_chain = server_bc.clone(); let db_path = db_path.clone(); + let network_service = network_service.clone(); // Create a simple handler for the router, inject our stateful objects into the request. service_fn_ok(move |mut req| { @@ -108,16 +116,34 @@ pub fn start_server( req.extensions_mut() .insert::>>(beacon_chain.clone()); req.extensions_mut().insert::(db_path.clone()); + req.extensions_mut() + .insert::>>(network_service.clone()); let path = req.uri().path().to_string(); // Route the request to the correct handler. let result = match (req.method(), path.as_ref()) { + (&Method::GET, "/beacon/head") => beacon::get_head::(req), + (&Method::GET, "/beacon/block") => beacon::get_block::(req), + (&Method::GET, "/beacon/block_root") => beacon::get_block_root::(req), + (&Method::GET, "/beacon/latest_finalized_checkpoint") => { + beacon::get_latest_finalized_checkpoint::(req) + } (&Method::GET, "/beacon/state") => beacon::get_state::(req), (&Method::GET, "/beacon/state_root") => beacon::get_state_root::(req), (&Method::GET, "/metrics") => metrics::get_prometheus::(req), + (&Method::GET, "/network/enr") => network::get_enr::(req), + (&Method::GET, "/network/peer_count") => network::get_peer_count::(req), + (&Method::GET, "/network/peer_id") => network::get_peer_id::(req), + (&Method::GET, "/network/peers") => network::get_peer_list::(req), + (&Method::GET, "/network/listen_port") => network::get_listen_port::(req), + (&Method::GET, "/network/listen_addresses") => { + network::get_listen_addresses::(req) + } (&Method::GET, "/node/version") => node::get_version(req), (&Method::GET, "/node/genesis_time") => node::get_genesis_time::(req), + (&Method::GET, "/spec") => spec::get_spec::(req), + (&Method::GET, "/spec/slots_per_epoch") => spec::get_slots_per_epoch::(req), _ => Err(ApiError::MethodNotAllowed(path.clone())), }; diff --git a/beacon_node/rest_api/src/network.rs b/beacon_node/rest_api/src/network.rs new file mode 100644 index 0000000000..a3e4c5ee72 --- /dev/null +++ b/beacon_node/rest_api/src/network.rs @@ -0,0 +1,108 @@ +use crate::{success_response, ApiError, ApiResult, NetworkService}; +use beacon_chain::BeaconChainTypes; +use eth2_libp2p::{Enr, Multiaddr, PeerId}; +use hyper::{Body, Request}; +use std::sync::Arc; + +/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// +/// Returns a list of `Multiaddr`, serialized according to their `serde` impl. +pub fn get_listen_addresses(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let multiaddresses: Vec = network.listen_multiaddrs(); + + Ok(success_response(Body::from( + serde_json::to_string(&multiaddresses) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the list of libp2p multiaddr the client is listening on. +/// +/// Returns a list of `Multiaddr`, serialized according to their `serde` impl. +pub fn get_listen_port(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + Ok(success_response(Body::from( + serde_json::to_string(&network.listen_port()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize port: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the Discv5 ENR from the client's libp2p service. +/// +/// ENR is encoded as base64 string. +pub fn get_enr(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let enr: Enr = network.local_enr(); + + Ok(success_response(Body::from( + serde_json::to_string(&enr.to_base64()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the `PeerId` from the client's libp2p service. +/// +/// PeerId is encoded as base58 string. +pub fn get_peer_id(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let peer_id: PeerId = network.local_peer_id(); + + Ok(success_response(Body::from( + serde_json::to_string(&peer_id.to_base58()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the number of peers connected in the client's libp2p service. +pub fn get_peer_count(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let connected_peers: usize = network.connected_peers(); + + Ok(success_response(Body::from( + serde_json::to_string(&connected_peers) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize Enr: {:?}", e)))?, + ))) +} + +/// HTTP handle to return the list of peers connected to the client's libp2p service. +/// +/// Peers are presented as a list of `PeerId::to_string()`. +pub fn get_peer_list(req: Request) -> ApiResult { + let network = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("NetworkService extension missing".to_string()))?; + + let connected_peers: Vec = network + .connected_peer_set() + .iter() + .map(PeerId::to_string) + .collect(); + + Ok(success_response(Body::from( + serde_json::to_string(&connected_peers).map_err(|e| { + ApiError::ServerError(format!("Unable to serialize Vec: {:?}", e)) + })?, + ))) +} diff --git a/beacon_node/rest_api/src/spec.rs b/beacon_node/rest_api/src/spec.rs new file mode 100644 index 0000000000..d0c8e4368d --- /dev/null +++ b/beacon_node/rest_api/src/spec.rs @@ -0,0 +1,27 @@ +use super::{success_response, ApiResult}; +use crate::ApiError; +use beacon_chain::{BeaconChain, BeaconChainTypes}; +use hyper::{Body, Request}; +use std::sync::Arc; +use types::EthSpec; + +/// HTTP handler to return the full spec object. +pub fn get_spec(req: Request) -> ApiResult { + let beacon_chain = req + .extensions() + .get::>>() + .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?; + + let json: String = serde_json::to_string(&beacon_chain.spec) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize spec: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} + +/// HTTP handler to return the full spec object. +pub fn get_slots_per_epoch(_req: Request) -> ApiResult { + let json: String = serde_json::to_string(&T::EthSpec::slots_per_epoch()) + .map_err(|e| ApiError::ServerError(format!("Unable to serialize epoch: {:?}", e)))?; + + Ok(success_response(Body::from(json))) +} diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index 9a52f2638c..04366baa7a 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -200,6 +200,16 @@ fn main() { .help("Sets the verbosity level") .takes_value(true), ) + /* + * Bootstrap. + */ + .arg( + Arg::with_name("bootstrap") + .long("bootstrap") + .value_name("HTTP_SERVER") + .help("Load the genesis state and libp2p address from the HTTP API of another Lighthouse node.") + .takes_value(true) + ) .get_matches(); // build the initial logger @@ -227,6 +237,11 @@ fn main() { let mut log = slog::Logger::root(drain.fuse(), o!()); + warn!( + log, + "Ethereum 2.0 is pre-release. This software is experimental." + ); + let data_dir = match matches .value_of("datadir") .and_then(|v| Some(PathBuf::from(v))) diff --git a/beacon_node/src/run.rs b/beacon_node/src/run.rs index c16d23e5f1..f88cb7460b 100644 --- a/beacon_node/src/run.rs +++ b/beacon_node/src/run.rs @@ -4,7 +4,7 @@ use client::{ }; use futures::sync::oneshot; use futures::Future; -use slog::{error, info, warn}; +use slog::{error, info}; use std::cell::RefCell; use std::path::Path; use std::path::PathBuf; @@ -42,11 +42,6 @@ pub fn run_beacon_node( let other_client_config = client_config.clone(); - warn!( - log, - "Ethereum 2.0 is pre-release. This software is experimental." - ); - info!( log, "BeaconNode init"; @@ -123,7 +118,7 @@ fn run( log: &slog::Logger, ) -> error::Result<()> where - T: BeaconChainTypes + InitialiseBeaconChain + Clone + Send + Sync + 'static, + T: BeaconChainTypes + InitialiseBeaconChain + Clone, T::Store: OpenDatabase, { let store = T::Store::open_database(&db_path)?; diff --git a/eth2/types/src/beacon_block_body.rs b/eth2/types/src/beacon_block_body.rs index 64dc229ed8..c1f66b816c 100644 --- a/eth2/types/src/beacon_block_body.rs +++ b/eth2/types/src/beacon_block_body.rs @@ -1,5 +1,5 @@ use crate::test_utils::TestRandom; -use crate::utils::graffiti_from_hex_str; +use crate::utils::{graffiti_from_hex_str, graffiti_to_hex_str}; use crate::*; use serde_derive::{Deserialize, Serialize}; @@ -16,7 +16,10 @@ use tree_hash_derive::TreeHash; pub struct BeaconBlockBody { pub randao_reveal: Signature, pub eth1_data: Eth1Data, - #[serde(deserialize_with = "graffiti_from_hex_str")] + #[serde( + serialize_with = "graffiti_to_hex_str", + deserialize_with = "graffiti_from_hex_str" + )] pub graffiti: [u8; 32], pub proposer_slashings: VariableList, pub attester_slashings: VariableList, T::MaxAttesterSlashings>, diff --git a/eth2/types/src/utils/serde_utils.rs b/eth2/types/src/utils/serde_utils.rs index 4b46fc0dc6..a9b27d75b5 100644 --- a/eth2/types/src/utils/serde_utils.rs +++ b/eth2/types/src/utils/serde_utils.rs @@ -46,8 +46,20 @@ where Ok(array) } -// #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref. -pub fn fork_to_hex_str(bytes: &[u8; 4], serializer: S) -> Result +pub fn fork_to_hex_str(bytes: &[u8; FORK_BYTES_LEN], serializer: S) -> Result +where + S: Serializer, +{ + let mut hex_string: String = "0x".to_string(); + hex_string.push_str(&hex::encode(&bytes)); + + serializer.serialize_str(&hex_string) +} + +pub fn graffiti_to_hex_str( + bytes: &[u8; GRAFFITI_BYTES_LEN], + serializer: S, +) -> Result where S: Serializer, {