gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Testnet compatible network upgrade (#587)

Browse Source 
* Create libp2p instance

* Change logger to stdlog

* test_connection initial commit

* Add gossipsub test

* Delete tests in network crate

* Add test module

* Clean tests

* Remove dependency on discovery

* Working publish between 2 nodes
TODO: Publish should be called just once

* Working 2 peer gossipsub test with additional events

* Cleanup test

* Add rpc test

* Star topology discovery WIP

* build_nodes builds and connects n nodes. Increase nodes in gossipsub test

* Add unsubscribe method and expose reference to gossipsub object for gossipsub tests

* Add gossipsub message forwarding test

* Fix gossipsub forward test

* Test improvements
* Remove discovery tests
* Simplify gossipsub forward test topology
* Add helper functions for topology building

* Clean up tests

* Update naming to new network spec

* Correct ssz encoding of protocol names

* Further additions to network upgrade

* Initial network spec update WIP

* Temp commit

* Builds one side of the streamed RPC responses

* Temporary commit

* Propagates streaming changes up into message handler

* Intermediate network update

* Partial update in upgrading to the new network spec

* Update dependencies, remove redundant deps

* Correct sync manager for block stream handling

* Re-write of RPC handler, improves efficiency and corrects bugs

* Stream termination update

* Completed refactor of rpc handler

* Remove crates

* Correct compile issues associated with test merge

* Build basic tests and testing structure for eth2-libp2p

* Enhance RPC tests and add logging

* Complete RPC testing framework and STATUS test

* Decoding bug fixes, log improvements, stream test

* Clean up RPC handler logging

* Decoder bug fix, empty block stream test

* Add BlocksByRoot RPC test

* Add Goodbye RPC test

* Syncing and stream handling bug fixes and performance improvements

* Applies discv5 bug fixes

* Adds DHT IP filtering for lighthouse - currently disabled

* Adds randomized network propagation as a CLI arg

* Add disconnect functionality

* Adds attestation handling and parent lookup

* Adds RPC error handling for the sync manager

* Allow parent's blocks to be already processed

* Update github workflow

* Adds reviewer suggestions
This commit is contained in:
Age Manning
2019-11-27 12:47:46 +11:00
committed by Paul Hauner
parent bf2eeae3f2
commit 97aa8b75b8
25 changed files with 2239 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
beacon_node/network/src/message_handler.rs
View File

@@ -4,7 +4,7 @@ use crate::sync::MessageProcessor;
use beacon_chain::{BeaconChain, BeaconChainTypes};
use eth2_libp2p::{
behaviour::PubsubMessage,
rpc::{RPCError, RPCErrorResponse, RPCRequest, RPCResponse, RequestId},
rpc::{RPCError, RPCErrorResponse, RPCRequest, RPCResponse, RequestId, ResponseTermination},
PeerId, RPCEvent,
};
use futures::future::Future;
@@ -115,9 +115,9 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
/// A new RPC request has been received from the network.
fn handle_rpc_request(&mut self, peer_id: PeerId, request_id: RequestId, request: RPCRequest) {
match request {
RPCRequest::Hello(hello_message) => {
RPCRequest::Status(status_message) => {
self.message_processor
.on_hello_request(peer_id, request_id, hello_message)
.on_status_request(peer_id, request_id, status_message)
}
RPCRequest::Goodbye(goodbye_reason) => {
debug!(
@@ -127,12 +127,12 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
);
self.message_processor.on_disconnect(peer_id);
}
RPCRequest::BeaconBlocks(request) => self
RPCRequest::BlocksByRange(request) => self
.message_processor
.on_beacon_blocks_request(peer_id, request_id, request),
RPCRequest::RecentBeaconBlocks(request) => self
.on_blocks_by_range_request(peer_id, request_id, request),
RPCRequest::BlocksByRoot(request) => self
.message_processor
.on_recent_beacon_blocks_request(peer_id, request_id, request),
.on_blocks_by_root_request(peer_id, request_id, request),
}
}
@@ -147,27 +147,30 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
// an error could have occurred.
match error_response {
RPCErrorResponse::InvalidRequest(error) => {
warn!(self.log, "Peer indicated invalid request";"peer_id" => format!("{:?}", peer_id), "error" => error.as_string())
warn!(self.log, "Peer indicated invalid request";"peer_id" => format!("{:?}", peer_id), "error" => error.as_string());
self.handle_rpc_error(peer_id, request_id, RPCError::RPCErrorResponse);
}
RPCErrorResponse::ServerError(error) => {
warn!(self.log, "Peer internal server error";"peer_id" => format!("{:?}", peer_id), "error" => error.as_string())
warn!(self.log, "Peer internal server error";"peer_id" => format!("{:?}", peer_id), "error" => error.as_string());
self.handle_rpc_error(peer_id, request_id, RPCError::RPCErrorResponse);
}
RPCErrorResponse::Unknown(error) => {
warn!(self.log, "Unknown peer error";"peer" => format!("{:?}", peer_id), "error" => error.as_string())
warn!(self.log, "Unknown peer error";"peer" => format!("{:?}", peer_id), "error" => error.as_string());
self.handle_rpc_error(peer_id, request_id, RPCError::RPCErrorResponse);
}
RPCErrorResponse::Success(response) => {
match response {
RPCResponse::Hello(hello_message) => {
RPCResponse::Status(status_message) => {
self.message_processor
.on_hello_response(peer_id, hello_message);
.on_status_response(peer_id, status_message);
}
RPCResponse::BeaconBlocks(response) => {
match self.decode_beacon_blocks(&response) {
Ok(beacon_blocks) => {
self.message_processor.on_beacon_blocks_response(
RPCResponse::BlocksByRange(response) => {
match self.decode_beacon_block(response) {
Ok(beacon_block) => {
self.message_processor.on_blocks_by_range_response(
peer_id,
request_id,
beacon_blocks,
Some(beacon_block),
);
}
Err(e) => {
@@ -176,13 +179,13 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
}
}
}
RPCResponse::RecentBeaconBlocks(response) => {
match self.decode_beacon_blocks(&response) {
Ok(beacon_blocks) => {
self.message_processor.on_recent_beacon_blocks_response(
RPCResponse::BlocksByRoot(response) => {
match self.decode_beacon_block(response) {
Ok(beacon_block) => {
self.message_processor.on_blocks_by_root_response(
peer_id,
request_id,
beacon_blocks,
Some(beacon_block),
);
}
Err(e) => {
@@ -191,6 +194,22 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
}
}
}
RPCResponse::Goodbye => {
// A goodbye was successfully sent, ignore it
}
}
}
RPCErrorResponse::StreamTermination(response_type) => {
// have received a stream termination, notify the processing functions
match response_type {
ResponseTermination::BlocksByRange => {
self.message_processor
.on_blocks_by_range_response(peer_id, request_id, None);
}
ResponseTermination::BlocksByRoot => {
self.message_processor
.on_blocks_by_root_response(peer_id, request_id, None);
}
}
}
}
@@ -198,8 +217,8 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
/// Handle various RPC errors
fn handle_rpc_error(&mut self, peer_id: PeerId, request_id: RequestId, error: RPCError) {
//TODO: Handle error correctly
warn!(self.log, "RPC Error"; "Peer" => format!("{:?}", peer_id), "request_id" => format!("{}", request_id), "Error" => format!("{:?}", error));
self.message_processor.on_rpc_error(peer_id, request_id);
}
/// Handle RPC messages
@@ -338,16 +357,13 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
/* Req/Resp Domain Decoding */
/// Verifies and decodes an ssz-encoded list of `BeaconBlock`s. This list may contain empty
/// entries encoded with an SSZ NULL.
fn decode_beacon_blocks(
/// Verifies and decodes an ssz-encoded `BeaconBlock`. If `None` is passed, this represents a
/// stream termination.
fn decode_beacon_block(
&self,
beacon_blocks: &[u8],
) -> Result<Vec<BeaconBlock<T::EthSpec>>, DecodeError> {
if beacon_blocks.is_empty() {
return Ok(Vec::new());
}
beacon_block: Vec<u8>,
) -> Result<BeaconBlock<T::EthSpec>, DecodeError> {
//TODO: Implement faster block verification before decoding entirely
Vec::from_ssz_bytes(&beacon_blocks)
BeaconBlock::from_ssz_bytes(&beacon_block)
}
}

115
beacon_node/network/src/service.rs
View File

@@ -4,13 +4,15 @@ use crate::NetworkConfig;
use beacon_chain::{BeaconChain, BeaconChainTypes};
use core::marker::PhantomData;
use eth2_libp2p::Service as LibP2PService;
use eth2_libp2p::Topic;
use eth2_libp2p::{Enr, Libp2pEvent, Multiaddr, PeerId, Swarm};
use eth2_libp2p::{
rpc::{RPCErrorResponse, RPCRequest, RPCResponse},
ConnectedPoint, Enr, Libp2pEvent, Multiaddr, NetworkBehaviour, PeerId, Swarm, Topic,
};
use eth2_libp2p::{PubsubMessage, RPCEvent};
use futures::prelude::*;
use futures::Stream;
use parking_lot::Mutex;
use slog::{debug, info, trace};
use parking_lot::{Mutex, MutexGuard};
use slog::{debug, info, trace, warn};
use std::sync::Arc;
use tokio::runtime::TaskExecutor;
use tokio::sync::{mpsc, oneshot};
@@ -53,6 +55,7 @@ impl<T: BeaconChainTypes> Service<T> {
message_handler_send,
executor,
network_log,
config.propagation_percentage,
)?;
let network_service = Service {
libp2p_service,
@@ -122,6 +125,7 @@ fn spawn_service(
message_handler_send: mpsc::UnboundedSender<HandlerMessage>,
executor: &TaskExecutor,
log: slog::Logger,
propagation_percentage: Option<u8>,
) -> error::Result<tokio::sync::oneshot::Sender<()>> {
let (network_exit, exit_rx) = tokio::sync::oneshot::channel();
@@ -132,6 +136,7 @@ fn spawn_service(
network_recv,
message_handler_send,
log.clone(),
propagation_percentage,
)
// allow for manual termination
.select(exit_rx.then(|_| Ok(())))
@@ -150,33 +155,70 @@ fn network_service(
mut network_recv: mpsc::UnboundedReceiver<NetworkMessage>,
mut message_handler_send: mpsc::UnboundedSender<HandlerMessage>,
log: slog::Logger,
propagation_percentage: Option<u8>,
) -> impl futures::Future<Item = (), Error = eth2_libp2p::error::Error> {
futures::future::poll_fn(move || -> Result<_, eth2_libp2p::error::Error> {
// if the network channel is not ready, try the swarm
// keep a list of peers to disconnect, once all channels are processed, remove the peers.
let mut peers_to_ban = Vec::new();
// processes the network channel before processing the libp2p swarm
loop {
// poll the network channel
match network_recv.poll() {
Ok(Async::Ready(Some(message))) => match message {
NetworkMessage::RPC(peer_id, rpc_event) => {
trace!(log, "{}", rpc_event);
trace!(log, "Sending RPC"; "RPC" => format!("{}", rpc_event));
libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event);
}
NetworkMessage::Propagate {
propagation_source,
message_id,
} => {
trace!(log, "Propagating gossipsub message";
"propagation_peer" => format!("{:?}", propagation_source),
"message_id" => format!("{}", message_id),
);
libp2p_service
.lock()
.swarm
.propagate_message(&propagation_source, message_id);
// TODO: Remove this for mainnet
// randomly prevents propagation
let mut should_send = true;
if let Some(percentage) = propagation_percentage {
// not exact percentage but close enough
let rand = rand::random::<u8>() % 100;
if rand > percentage {
// don't propagate
should_send = false;
}
}
if !should_send {
info!(log, "Random filter did not propagate message");
} else {
trace!(log, "Propagating gossipsub message";
"propagation_peer" => format!("{:?}", propagation_source),
"message_id" => format!("{}", message_id),
);
libp2p_service
.lock()
.swarm
.propagate_message(&propagation_source, message_id);
}
}
NetworkMessage::Publish { topics, message } => {
debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics));
libp2p_service.lock().swarm.publish(&topics, message);
// TODO: Remove this for mainnet
// randomly prevents propagation
let mut should_send = true;
if let Some(percentage) = propagation_percentage {
// not exact percentage but close enough
let rand = rand::random::<u8>() % 100;
if rand > percentage {
// don't propagate
should_send = false;
}
}
if !should_send {
info!(log, "Random filter did not publish message");
} else {
debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics));
libp2p_service.lock().swarm.publish(&topics, message);
}
}
NetworkMessage::Disconnect { peer_id } => {
peers_to_ban.push(peer_id);
}
},
Ok(Async::NotReady) => break,
@@ -194,7 +236,19 @@ fn network_service(
match libp2p_service.lock().poll() {
Ok(Async::Ready(Some(event))) => match event {
Libp2pEvent::RPC(peer_id, rpc_event) => {
trace!(log, "{}", rpc_event);
trace!(log, "Received RPC"; "RPC" => format!("{}", rpc_event));
// if we received or sent a Goodbye message, drop and ban the peer
match rpc_event {
RPCEvent::Request(_, RPCRequest::Goodbye(_))
| RPCEvent::Response(
_,
RPCErrorResponse::Success(RPCResponse::Goodbye),
) => {
peers_to_ban.push(peer_id.clone());
}
_ => {}
};
message_handler_send
.try_send(HandlerMessage::RPC(peer_id, rpc_event))
.map_err(|_| "Failed to send RPC to handler")?;
@@ -221,6 +275,7 @@ fn network_service(
.try_send(HandlerMessage::PubsubMessage(id, source, message))
.map_err(|_| "Failed to send pubsub message to handler")?;
}
Libp2pEvent::PeerSubscribed(_, _) => {}
},
Ok(Async::Ready(None)) => unreachable!("Stream never ends"),
Ok(Async::NotReady) => break,
@@ -228,10 +283,32 @@ fn network_service(
}
}
while !peers_to_ban.is_empty() {
let service = libp2p_service.lock();
disconnect_peer(service, peers_to_ban.pop().expect("element exists"), &log);
}
Ok(Async::NotReady)
})
}
fn disconnect_peer(mut service: MutexGuard<LibP2PService>, peer_id: PeerId, log: &slog::Logger) {
warn!(log, "Disconnecting and banning peer"; "peer_id" => format!("{:?}", peer_id));
Swarm::ban_peer_id(&mut service.swarm, peer_id.clone());
// TODO: Correctly notify protocols of the disconnect
// TOOD: Also remove peer from the DHT: https://github.com/sigp/lighthouse/issues/629
let dummy_connected_point = ConnectedPoint::Dialer {
address: "/ip4/0.0.0.0"
.parse::<Multiaddr>()
.expect("valid multiaddr"),
};
service
.swarm
.inject_disconnected(&peer_id, dummy_connected_point);
// inform the behaviour that the peer has been banned
service.swarm.peer_banned(peer_id);
}
/// Types of messages that the network service can receive.
#[derive(Debug)]
pub enum NetworkMessage {
@@ -242,9 +319,11 @@ pub enum NetworkMessage {
topics: Vec<Topic>,
message: PubsubMessage,
},
/// Propagate a received gossipsub message
/// Propagate a received gossipsub message.
Propagate {
propagation_source: PeerId,
message_id: String,
},
/// Disconnect and bans a peer id.
Disconnect { peer_id: PeerId },
}

558
beacon_node/network/src/sync/manager.rs
View File

@@ -1,6 +1,6 @@
//! The `SyncManager` facilities the block syncing logic of lighthouse. The current networking
//! specification provides two methods from which to obtain blocks from peers. The `BeaconBlocks`
//! request and the `RecentBeaconBlocks` request. The former is used to obtain a large number of
//! specification provides two methods from which to obtain blocks from peers. The `BlocksByRange`
//! request and the `BlocksByRoot` request. The former is used to obtain a large number of
//! blocks and the latter allows for searching for blocks given a block-hash.
//!
//! These two RPC methods are designed for two type of syncing.
@@ -9,7 +9,6 @@
//!
//! Both of these syncing strategies are built into the `SyncManager`.
//!
//!
//! Currently the long-range (batch) syncing method functions by opportunistically downloading
//! batches blocks from all peers who know about a chain that we do not. When a new peer connects
//! which has a later head that is greater than `SLOT_IMPORT_TOLERANCE` from our current head slot,
@@ -19,7 +18,7 @@
//!
//! Batch Syncing
//!
//! This syncing process start by requesting `MAX_BLOCKS_PER_REQUEST` blocks from a peer with an
//! This syncing process start by requesting `BLOCKS_PER_REQUEST` blocks from a peer with an
//! unknown chain (with a greater slot height) starting from our current head slot. If the earliest
//! block returned is known to us, then the group of blocks returned form part of a known chain,
//! and we process this batch of blocks, before requesting more batches forward and processing
@@ -52,14 +51,23 @@
//! queued for lookup. A round-robin approach is used to request the parent from the known list of
//! fully sync'd peers. If `PARENT_FAIL_TOLERANCE` attempts at requesting the block fails, we
//! drop the propagated block and downvote the peer that sent it to us.
//!
//! Block Lookup
//!
//! To keep the logic maintained to the syncing thread (and manage the request_ids), when a block needs to be searched for (i.e
//! if an attestation references an unknown block) this manager can search for the block and
//! subsequently search for parents if needed.
use super::simple_sync::{hello_message, NetworkContext, PeerSyncInfo, FUTURE_SLOT_TOLERANCE};
use super::message_processor::{
status_message, NetworkContext, PeerSyncInfo, FUTURE_SLOT_TOLERANCE,
};
use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome};
use eth2_libp2p::rpc::methods::*;
use eth2_libp2p::rpc::{RPCRequest, RequestId};
use eth2_libp2p::PeerId;
use fnv::FnvHashMap;
use futures::prelude::*;
use slog::{debug, info, trace, warn, Logger};
use slog::{crit, debug, info, trace, warn, Logger};
use smallvec::SmallVec;
use std::collections::{HashMap, HashSet};
use std::ops::{Add, Sub};
@@ -68,14 +76,17 @@ use tokio::sync::{mpsc, oneshot};
use types::{BeaconBlock, EthSpec, Hash256, Slot};
/// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch
/// is requested. Currently the value is small for testing. This will be incremented for
/// production.
const MAX_BLOCKS_PER_REQUEST: u64 = 50;
/// is requested. There is a timeout for each batch request. If this value is too high, we will
/// downvote peers with poor bandwidth. This can be set arbitrarily high, in which case the
/// responder will fill the response up to the max request size, assuming they have the bandwidth
/// to do so.
//TODO: Make this dynamic based on peer's bandwidth
const BLOCKS_PER_REQUEST: u64 = 50;
/// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync
/// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a
/// fully sync'd peer.
const SLOT_IMPORT_TOLERANCE: usize = 10;
const SLOT_IMPORT_TOLERANCE: usize = 20;
/// How many attempts we try to find a parent of a block before we give up trying .
const PARENT_FAIL_TOLERANCE: usize = 3;
/// The maximum depth we will search for a parent block. In principle we should have sync'd any
@@ -91,24 +102,33 @@ const EMPTY_BATCH_TOLERANCE: usize = 100;
pub enum SyncMessage<T: EthSpec> {
/// A useful peer has been discovered.
AddPeer(PeerId, PeerSyncInfo),
/// A `BeaconBlocks` response has been received.
BeaconBlocksResponse {
/// A `BlocksByRange` response has been received.
BlocksByRangeResponse {
peer_id: PeerId,
request_id: RequestId,
beacon_blocks: Vec<BeaconBlock<T>>,
beacon_block: Option<BeaconBlock<T>>,
},
/// A `RecentBeaconBlocks` response has been received.
RecentBeaconBlocksResponse {
/// A `BlocksByRoot` response has been received.
BlocksByRootResponse {
peer_id: PeerId,
request_id: RequestId,
beacon_blocks: Vec<BeaconBlock<T>>,
beacon_block: Option<BeaconBlock<T>>,
},
/// A block with an unknown parent has been received.
UnknownBlock(PeerId, BeaconBlock<T>),
/// A peer has sent an object that references a block that is unknown. This triggers the
/// manager to attempt to find the block matching the unknown hash.
UnknownBlockHash(PeerId, Hash256),
/// A peer has disconnected.
Disconnect(PeerId),
/// An RPC Error has occurred on a request.
_RPCError(RequestId),
RPCError(PeerId, RequestId),
}
#[derive(PartialEq)]
@@ -116,46 +136,42 @@ pub enum SyncMessage<T: EthSpec> {
enum BlockRequestsState {
/// The object is queued to be downloaded from a peer but has not yet been requested.
Queued,
/// The batch or parent has been requested with the `RequestId` and we are awaiting a response.
Pending(RequestId),
/// The downloaded blocks are ready to be processed by the beacon chain. For a batch process
/// this means we have found a common chain.
ReadyToProcess,
/// The batch is complete, simply drop without downvoting the peer.
Complete,
/// A failure has occurred and we will drop and downvote the peer that caused the request.
Failed,
}
/// The state of batch requests.
enum SyncDirection {
/// The batch has just been initialised and we need to check to see if a backward sync is
/// required on first batch response.
Initial,
/// We are syncing forwards, the next batch should contain higher slot numbers than is
/// predecessor.
Forwards,
/// We are syncing backwards and looking for a common ancestor chain before we can start
/// processing the downloaded blocks.
Backwards,
}
/// `BlockRequests` keep track of the long-range (batch) sync process per peer.
struct BlockRequests<T: EthSpec> {
/// The peer's head slot and the target of this batch download.
target_head_slot: Slot,
/// The peer's head root, used to specify which chain of blocks we are downloading from the
/// blocks.
/// The peer's head root, used to specify which chain of blocks we are downloading from.
target_head_root: Hash256,
/// The blocks that we have currently downloaded from the peer that are yet to be processed.
downloaded_blocks: Vec<BeaconBlock<T>>,
/// The number of blocks successfully processed in this request.
blocks_processed: usize,
/// The number of empty batches we have consecutively received. If a peer returns more than
/// EMPTY_BATCHES_TOLERANCE, they are dropped.
consecutive_empty_batches: usize,
/// The current state of this batch request.
state: BlockRequestsState,
/// Specifies the current direction of this batch request.
sync_direction: SyncDirection,
/// The current `start_slot` of the batched block request.
current_start_slot: Slot,
}
@@ -164,12 +180,15 @@ struct BlockRequests<T: EthSpec> {
struct ParentRequests<T: EthSpec> {
/// The blocks that have currently been downloaded.
downloaded_blocks: Vec<BeaconBlock<T>>,
/// The number of failed attempts to retrieve a parent block. If too many attempts occur, this
/// lookup is failed and rejected.
failed_attempts: usize,
/// The peer who last submitted a block. If the chain ends or fails, this is the peer that is
/// downvoted.
last_submitted_peer: PeerId,
/// The current state of the parent lookup.
state: BlockRequestsState,
}
@@ -177,13 +196,16 @@ struct ParentRequests<T: EthSpec> {
impl<T: EthSpec> BlockRequests<T> {
/// Gets the next start slot for a batch and transitions the state to a Queued state.
fn update_start_slot(&mut self) {
match self.sync_direction {
SyncDirection::Initial | SyncDirection::Forwards => {
self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST);
}
SyncDirection::Backwards => {
self.current_start_slot -= Slot::from(MAX_BLOCKS_PER_REQUEST);
}
// the last request may not have returned all the required blocks (hit the rpc size
// limit). If so, start from the last returned slot
if !self.downloaded_blocks.is_empty()
&& self.downloaded_blocks[self.downloaded_blocks.len() - 1].slot
> self.current_start_slot
{
self.current_start_slot = self.downloaded_blocks[self.downloaded_blocks.len() - 1].slot
+ Slot::from(BLOCKS_PER_REQUEST);
} else {
self.current_start_slot += Slot::from(BLOCKS_PER_REQUEST);
}
self.state = BlockRequestsState::Queued;
}
@@ -195,9 +217,11 @@ enum ManagerState {
/// The manager is performing a long-range (batch) sync. In this mode, parent lookups are
/// disabled.
Syncing,
/// The manager is up to date with all known peers and is connected to at least one
/// fully-syncing peer. In this state, parent lookups are enabled.
Regular,
/// No useful peers are connected. Long-range sync's cannot proceed and we have no useful
/// peers to download parents for. More peers need to be connected before we can proceed.
Stalled,
@@ -210,23 +234,34 @@ enum ManagerState {
pub struct SyncManager<T: BeaconChainTypes> {
/// A weak reference to the underlying beacon chain.
chain: Weak<BeaconChain<T>>,
/// The current state of the import manager.
state: ManagerState,
/// A receiving channel sent by the message processor thread.
input_channel: mpsc::UnboundedReceiver<SyncMessage<T::EthSpec>>,
/// A network context to contact the network service.
network: NetworkContext,
/// A collection of `BlockRequest` per peer that is currently being downloaded. Used in the
/// long-range (batch) sync process.
import_queue: HashMap<PeerId, BlockRequests<T::EthSpec>>,
/// A collection of parent block lookups.
parent_queue: SmallVec<[ParentRequests<T::EthSpec>; 3]>,
/// A collection of block hashes being searched for
single_block_lookups: FnvHashMap<RequestId, Hash256>,
/// The collection of known, connected, fully-sync'd peers.
full_peers: HashSet<PeerId>,
/// The current request Id. This is used to keep track of responses to various outbound
/// The current request id. This is used to keep track of responses to various outbound
/// requests. This is an internal accounting mechanism, request id's are never sent to any
/// peers.
current_req_id: usize,
/// The logger for the import manager.
log: Logger,
}
@@ -256,6 +291,7 @@ pub fn spawn<T: BeaconChainTypes>(
network,
import_queue: HashMap::new(),
parent_queue: SmallVec::new(),
single_block_lookups: FnvHashMap::default(),
full_peers: HashSet::new(),
current_req_id: 0,
log: log.clone(),
@@ -281,14 +317,12 @@ impl<T: BeaconChainTypes> SyncManager<T> {
///
/// This function handles the logic associated with the connection of a new peer. If the peer
/// is sufficiently ahead of our current head, a long-range (batch) sync is started and
/// batches of blocks are queued to download from the peer. Batched blocks begin at our
/// current head. If the resulting downloaded blocks are part of our current chain, we
/// continue with a forward sync. If not, we download blocks (in batches) backwards until we
/// reach a common ancestor. Batches are then processed and downloaded sequentially forwards.
/// batches of blocks are queued to download from the peer. Batched blocks begin at our latest
/// finalized head.
///
/// If the peer is within the `SLOT_IMPORT_TOLERANCE`, then it's head is sufficiently close to
/// ours that we consider it fully sync'd with respect to our current chain.
pub fn add_peer(&mut self, peer_id: PeerId, remote: PeerSyncInfo) {
fn add_peer(&mut self, peer_id: PeerId, remote: PeerSyncInfo) {
// ensure the beacon chain still exists
let chain = match self.chain.upgrade() {
Some(chain) => chain,
@@ -313,7 +347,6 @@ impl<T: BeaconChainTypes> SyncManager<T> {
// remove the peer from the queue if it exists
self.import_queue.remove(&peer_id);
self.add_full_peer(peer_id);
//
return;
}
@@ -340,35 +373,27 @@ impl<T: BeaconChainTypes> SyncManager<T> {
} else {
// not already downloading blocks from this peer
let block_requests = BlockRequests {
target_head_slot: remote.head_slot, // this should be larger than the current head. It is checked in the SyncManager before add_peer is called
target_head_slot: remote.head_slot, // this should be larger than the current head. It is checked before add_peer is called
target_head_root: remote.head_root,
consecutive_empty_batches: 0,
downloaded_blocks: Vec::new(),
blocks_processed: 0,
state: BlockRequestsState::Queued,
sync_direction: SyncDirection::Initial,
current_start_slot: chain.best_slot(),
current_start_slot: local
.finalized_epoch
.start_slot(T::EthSpec::slots_per_epoch()),
};
self.import_queue.insert(peer_id, block_requests);
}
}
/// A `BeaconBlocks` request has received a response. This function process the response.
pub fn beacon_blocks_response(
/// A `BlocksByRange` request has received a response. This function process the response.
fn blocks_by_range_response(
&mut self,
peer_id: PeerId,
request_id: RequestId,
mut blocks: Vec<BeaconBlock<T::EthSpec>>,
block: Option<BeaconBlock<T::EthSpec>>,
) {
// ensure the underlying chain still exists
let chain = match self.chain.upgrade() {
Some(chain) => chain,
None => {
trace!(self.log, "Chain dropped. Sync terminating");
return;
}
};
// find the request associated with this response
let block_requests = match self
.import_queue
@@ -378,33 +403,42 @@ impl<T: BeaconChainTypes> SyncManager<T> {
Some(req) => req,
_ => {
// No pending request, invalid request_id or coding error
warn!(self.log, "BeaconBlocks response unknown"; "request_id" => request_id);
warn!(self.log, "BlocksByRange response unknown"; "request_id" => request_id);
return;
}
};
// If we are syncing up to a target head block, at least the target head block should be
// returned. If we are syncing back to our last finalized block the request should return
// at least the last block we received (last known block). In diagram form:
//
// unknown blocks requested blocks downloaded blocks
// |-------------------|------------------------|------------------------|
// ^finalized slot ^ requested start slot ^ last known block ^ remote head
// add the downloaded block
if let Some(downloaded_block) = block {
// add the block to the request
block_requests.downloaded_blocks.push(downloaded_block);
return;
}
// the batch has finished processing, or terminated early
// TODO: The following requirement may need to be relaxed as a node could fork and prune
// their old head, given to us during a STATUS.
// If we are syncing up to a target head block, at least the target head block should be
// returned.
let blocks = &block_requests.downloaded_blocks;
if blocks.is_empty() {
debug!(self.log, "BeaconBlocks response was empty"; "request_id" => request_id);
debug!(self.log, "BlocksByRange response was empty"; "request_id" => request_id);
block_requests.consecutive_empty_batches += 1;
if block_requests.consecutive_empty_batches >= EMPTY_BATCH_TOLERANCE {
warn!(self.log, "Peer returned too many empty block batches";
"peer" => format!("{:?}", peer_id));
block_requests.state = BlockRequestsState::Failed;
} else if block_requests.current_start_slot + MAX_BLOCKS_PER_REQUEST
} else if block_requests.current_start_slot + BLOCKS_PER_REQUEST
>= block_requests.target_head_slot
{
warn!(self.log, "Peer did not return blocks it claimed to possess";
"peer" => format!("{:?}", peer_id));
block_requests.state = BlockRequestsState::Failed;
// This could be due to a re-org causing the peer to prune their head. In this
// instance, we try to process what is currently downloaded, if there are blocks
// downloaded.
block_requests.state = BlockRequestsState::Complete;
} else {
// this batch was empty, request the next batch
block_requests.update_start_slot();
}
return;
@@ -416,12 +450,9 @@ impl<T: BeaconChainTypes> SyncManager<T> {
// Note that the order of blocks is verified in block processing
let last_sent_slot = blocks[blocks.len() - 1].slot;
if block_requests.current_start_slot > blocks[0].slot
|| block_requests
.current_start_slot
.add(MAX_BLOCKS_PER_REQUEST)
< last_sent_slot
|| block_requests.current_start_slot.add(BLOCKS_PER_REQUEST) < last_sent_slot
{
warn!(self.log, "BeaconBlocks response returned out of range blocks";
warn!(self.log, "BlocksByRange response returned out of range blocks";
"request_id" => request_id,
"response_initial_slot" => blocks[0].slot,
"requested_initial_slot" => block_requests.current_start_slot);
@@ -431,87 +462,35 @@ impl<T: BeaconChainTypes> SyncManager<T> {
return;
}
// Determine if more blocks need to be downloaded. There are a few cases:
// - We are in initial sync mode - We have requested blocks and need to determine if this
// is part of a known chain to determine the whether to start syncing backwards or continue
// syncing forwards.
// - We are syncing backwards and need to verify if we have found a common ancestor in
// order to start processing the downloaded blocks.
// - We are syncing forwards. We mark this as complete and check if any further blocks are
// required to download when processing the batch.
match block_requests.sync_direction {
SyncDirection::Initial => {
block_requests.downloaded_blocks.append(&mut blocks);
// this batch is the first batch downloaded. Check if we can process or if we need
// to backwards search.
//TODO: Decide which is faster. Reading block from db and comparing or calculating
//the hash tree root and comparing.
let earliest_slot = block_requests.downloaded_blocks[0].slot;
if Some(block_requests.downloaded_blocks[0].canonical_root())
== chain.root_at_slot(earliest_slot)
{
// we have a common head, start processing and begin a forwards sync
block_requests.sync_direction = SyncDirection::Forwards;
block_requests.state = BlockRequestsState::ReadyToProcess;
return;
}
// no common head, begin a backwards search
block_requests.sync_direction = SyncDirection::Backwards;
block_requests.current_start_slot =
std::cmp::min(chain.best_slot(), block_requests.downloaded_blocks[0].slot);
block_requests.update_start_slot();
}
SyncDirection::Forwards => {
// continue processing all blocks forwards, verify the end in the processing
block_requests.downloaded_blocks.append(&mut blocks);
block_requests.state = BlockRequestsState::ReadyToProcess;
}
SyncDirection::Backwards => {
block_requests.downloaded_blocks.splice(..0, blocks);
// verify the request hasn't failed by having no common ancestor chain
// get our local finalized_slot
let local_finalized_slot = {
let state = &chain.head().beacon_state;
state
.finalized_checkpoint
.epoch
.start_slot(T::EthSpec::slots_per_epoch())
};
if local_finalized_slot >= block_requests.current_start_slot {
warn!(self.log, "Peer returned an unknown chain."; "request_id" => request_id);
block_requests.state = BlockRequestsState::Failed;
return;
}
// check if we have reached a common chain ancestor
let earliest_slot = block_requests.downloaded_blocks[0].slot;
if Some(block_requests.downloaded_blocks[0].canonical_root())
== chain.root_at_slot(earliest_slot)
{
// we have a common head, start processing and begin a forwards sync
block_requests.sync_direction = SyncDirection::Forwards;
block_requests.state = BlockRequestsState::ReadyToProcess;
return;
}
// no common chain, haven't passed last_finalized_head, so continue backwards
// search
block_requests.update_start_slot();
}
}
// Process this batch
block_requests.state = BlockRequestsState::ReadyToProcess;
}
pub fn recent_blocks_response(
/// The response to a `BlocksByRoot` request.
/// The current implementation takes one block at a time. As blocks are streamed, any
/// subsequent blocks will simply be ignored.
/// There are two reasons we could have received a BlocksByRoot response
/// - We requested a single hash and have received a response for the single_block_lookup
/// - We are looking up parent blocks in parent lookup search
fn blocks_by_root_response(
&mut self,
peer_id: PeerId,
request_id: RequestId,
mut blocks: Vec<BeaconBlock<T::EthSpec>>,
block: Option<BeaconBlock<T::EthSpec>>,
) {
// check if this is a single block lookup - i.e we were searching for a specific hash
if block.is_some() {
if let Some(block_hash) = self.single_block_lookups.remove(&request_id) {
self.single_block_lookup_response(
peer_id,
block.expect("block exists"),
block_hash,
);
return;
}
}
// this should be a response to a parent request search
// find the request
let parent_request = match self
.parent_queue
@@ -520,40 +499,78 @@ impl<T: BeaconChainTypes> SyncManager<T> {
{
Some(req) => req,
None => {
// No pending request, invalid request_id or coding error
warn!(self.log, "RecentBeaconBlocks response unknown"; "request_id" => request_id);
if block.is_some() {
// No pending request, invalid request_id or coding error
warn!(self.log, "BlocksByRoot response unknown"; "request_id" => request_id);
}
// it could be a stream termination None, in which case we just ignore it
return;
}
};
match block {
Some(block) => {
// add the block to response
parent_request.downloaded_blocks.push(block);
// if an empty response is given, the peer didn't have the requested block, try again
if blocks.is_empty() {
parent_request.failed_attempts += 1;
parent_request.state = BlockRequestsState::Queued;
parent_request.last_submitted_peer = peer_id;
return;
// queue for processing
parent_request.state = BlockRequestsState::ReadyToProcess;
}
None => {
// if an empty response is given, the peer didn't have the requested block, try again
parent_request.failed_attempts += 1;
parent_request.state = BlockRequestsState::Queued;
parent_request.last_submitted_peer = peer_id;
return;
}
}
// currently only support a single block lookup. Reject any response that has more than 1
// block
if blocks.len() != 1 {
//TODO: Potentially downvote the peer
debug!(self.log, "Peer sent more than 1 parent. Ignoring";
"peer_id" => format!("{:?}", peer_id),
"no_parents" => blocks.len()
);
return;
}
// add the block to response
parent_request
.downloaded_blocks
.push(blocks.pop().expect("must exist"));
// queue for processing
parent_request.state = BlockRequestsState::ReadyToProcess;
}
/// Processes the response obtained from a single block lookup search. If the block is
/// processed or errors, the search ends. If the blocks parent is unknown, a block parent
/// lookup search is started.
fn single_block_lookup_response(
&mut self,
peer_id: PeerId,
block: BeaconBlock<T::EthSpec>,
expected_block_hash: Hash256,
) {
// verify the hash is correct and try and process the block
if expected_block_hash != block.canonical_root() {
// the peer that sent this, sent us the wrong block
downvote_peer(&mut self.network, &self.log, peer_id);
return;
}
// we have the correct block, try and process it
if let Some(chain) = self.chain.upgrade() {
match chain.process_block(block.clone()) {
Ok(outcome) => {
match outcome {
BlockProcessingOutcome::Processed { block_root } => {
info!(self.log, "Processed block"; "block" => format!("{}", block_root));
}
BlockProcessingOutcome::ParentUnknown { parent: _ } => {
// We don't know of the blocks parent, begin a parent lookup search
self.add_unknown_block(peer_id, block);
}
BlockProcessingOutcome::BlockIsAlreadyKnown => {
trace!(self.log, "Single block lookup already known");
}
_ => {
warn!(self.log, "Single block lookup failed"; "outcome" => format!("{:?}", outcome));
downvote_peer(&mut self.network, &self.log, peer_id);
}
}
}
Err(e) => {
warn!(self.log, "Unexpected block processing error"; "error" => format!("{:?}", e));
}
}
}
}
/// A block has been sent to us that has an unknown parent. This begins a parent lookup search
/// to find the parent or chain of parents that match our current chain.
fn add_unknown_block(&mut self, peer_id: PeerId, block: BeaconBlock<T::EthSpec>) {
// if we are not in regular sync mode, ignore this block
if self.state != ManagerState::Regular {
@@ -583,8 +600,53 @@ impl<T: BeaconChainTypes> SyncManager<T> {
self.parent_queue.push(req);
}
fn inject_error(&mut self, _id: RequestId) {
//TODO: Remove block state from pending
/// A request to search for a block hash has been received. This function begins a BlocksByRoot
/// request to find the requested block.
fn search_for_block(&mut self, peer_id: PeerId, block_hash: Hash256) {
let request_id = self.current_req_id;
self.single_block_lookups.insert(request_id, block_hash);
self.current_req_id += 1;
let request = BlocksByRootRequest {
block_roots: vec![block_hash],
};
blocks_by_root_request(&mut self.network, &self.log, peer_id, request_id, request);
}
fn inject_error(&mut self, peer_id: PeerId, request_id: RequestId) {
trace!(self.log, "Sync manager received a failed RPC");
// remove any single block lookups
self.single_block_lookups.remove(&request_id);
// find the request associated with this response
if let Some(block_requests) = self
.import_queue
.get_mut(&peer_id)
.filter(|r| r.state == BlockRequestsState::Pending(request_id))
{
// TODO: Potentially implement a tolerance. For now, we try to process what have been
// downloaded
if !block_requests.downloaded_blocks.is_empty() {
block_requests.current_start_slot = block_requests
.downloaded_blocks
.last()
.expect("is not empty")
.slot;
block_requests.state = BlockRequestsState::ReadyToProcess;
} else {
block_requests.state = BlockRequestsState::Failed;
}
};
// increment the failure of a parent lookup if the request matches a parent search
if let Some(parent_req) = self
.parent_queue
.iter_mut()
.find(|request| request.state == BlockRequestsState::Pending(request_id))
{
parent_req.failed_attempts += 1;
parent_req.state = BlockRequestsState::Queued;
parent_req.last_submitted_peer = peer_id;
}
}
fn peer_disconnect(&mut self, peer_id: &PeerId) {
@@ -626,23 +688,32 @@ impl<T: BeaconChainTypes> SyncManager<T> {
fn process_potential_block_requests(&mut self) {
// check if an outbound request is required
// Managing a fixed number of outbound requests is maintained at the RPC protocol libp2p
// layer and not needed here. Therefore we create many outbound requests and let the RPC
// handle the number of simultaneous requests. Request all queued objects.
// handle the number of simultaneous requests.
// Request all queued objects.
// remove any failed batches
let debug_log = &self.log;
let full_peer_ref = &mut self.full_peers;
self.import_queue.retain(|peer_id, block_request| {
if let BlockRequestsState::Failed = block_request.state {
debug!(debug_log, "Block import from peer failed";
"peer_id" => format!("{:?}", peer_id),
"downloaded_blocks" => block_request.blocks_processed
);
full_peer_ref.remove(peer_id);
false
} else {
true
match block_request.state {
BlockRequestsState::Failed => {
debug!(debug_log, "Block import from peer failed";
"peer_id" => format!("{:?}", peer_id),
"downloaded_blocks" => block_request.blocks_processed
);
full_peer_ref.remove(peer_id);
false
}
BlockRequestsState::Complete => {
debug!(debug_log, "Block import from peer completed";
"peer_id" => format!("{:?}", peer_id),
);
false
}
_ => true, // keep all other states
}
});
@@ -653,13 +724,13 @@ impl<T: BeaconChainTypes> SyncManager<T> {
block_requests.state = BlockRequestsState::Pending(request_id);
self.current_req_id += 1;
let request = BeaconBlocksRequest {
let request = BlocksByRangeRequest {
head_block_root: block_requests.target_head_root,
start_slot: block_requests.current_start_slot.as_u64(),
count: MAX_BLOCKS_PER_REQUEST,
count: BLOCKS_PER_REQUEST,
step: 0,
};
request_blocks(
blocks_by_range_request(
&mut self.network,
&self.log,
peer_id.clone(),
@@ -684,9 +755,12 @@ impl<T: BeaconChainTypes> SyncManager<T> {
if block_requests.state == BlockRequestsState::ReadyToProcess {
let downloaded_blocks =
std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new());
let last_element = downloaded_blocks.len() - 1;
let end_slot = downloaded_blocks
.last()
.expect("Batches to be processed should not be empty")
.slot;
let total_blocks = downloaded_blocks.len();
let start_slot = downloaded_blocks[0].slot;
let end_slot = downloaded_blocks[last_element].slot;
match process_blocks(chain_ref.clone(), downloaded_blocks, log_ref) {
Ok(()) => {
@@ -694,15 +768,15 @@ impl<T: BeaconChainTypes> SyncManager<T> {
"peer" => format!("{:?}", peer_id),
"start_slot" => start_slot,
"end_slot" => end_slot,
"no_blocks" => last_element + 1,
"no_blocks" => total_blocks,
);
block_requests.blocks_processed += last_element + 1;
block_requests.blocks_processed += total_blocks;
// check if the batch is complete, by verifying if we have reached the
// target head
if end_slot >= block_requests.target_head_slot {
// Completed, re-hello the peer to ensure we are up to the latest head
hello_peer(network_ref, log_ref, chain_ref.clone(), peer_id.clone());
// Completed, re-status the peer to ensure we are up to the latest head
status_peer(network_ref, log_ref, chain_ref.clone(), peer_id.clone());
// remove the request
false
} else {
@@ -718,7 +792,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
"peer" => format!("{:?}", peer_id),
"start_slot" => start_slot,
"end_slot" => end_slot,
"no_blocks" => last_element + 1,
"no_blocks" => total_blocks,
"error" => format!("{:?}", e),
);
downvote_peer(network_ref, log_ref, peer_id.clone());
@@ -771,14 +845,14 @@ impl<T: BeaconChainTypes> SyncManager<T> {
self.current_req_id += 1;
let last_element_index = parent_request.downloaded_blocks.len() - 1;
let parent_hash = parent_request.downloaded_blocks[last_element_index].parent_root;
let request = RecentBeaconBlocksRequest {
let request = BlocksByRootRequest {
block_roots: vec![parent_hash],
};
// select a random fully synced peer to attempt to download the parent block
let peer_id = self.full_peers.iter().next().expect("List is not empty");
recent_blocks_request(
blocks_by_root_request(
&mut self.network,
&self.log,
peer_id.clone(),
@@ -800,17 +874,29 @@ impl<T: BeaconChainTypes> SyncManager<T> {
.filter(|req| req.state == BlockRequestsState::ReadyToProcess)
{
// verify the last added block is the parent of the last requested block
let last_index = completed_request.downloaded_blocks.len() - 1;
let expected_hash = completed_request.downloaded_blocks[last_index].parent_root;
// Note: the length must be greater than 1 so this cannot panic.
let block_hash = completed_request.downloaded_blocks[last_index - 1].canonical_root();
if completed_request.downloaded_blocks.len() < 2 {
crit!(
self.log,
"There must be at least two blocks in a parent request lookup at all times"
);
panic!("There must be at least two blocks in parent request lookup at all time");
// fail loudly
}
let previous_index = completed_request.downloaded_blocks.len() - 2;
let expected_hash = completed_request.downloaded_blocks[previous_index].parent_root;
// Note: the length must be greater than 2 so this cannot panic.
let block_hash = completed_request
.downloaded_blocks
.last()
.expect("Complete batch cannot be empty")
.canonical_root();
if block_hash != expected_hash {
// remove the head block
let _ = completed_request.downloaded_blocks.pop();
completed_request.state = BlockRequestsState::Queued;
//TODO: Potentially downvote the peer
let peer = completed_request.last_submitted_peer.clone();
debug!(self.log, "Peer sent invalid parent. Ignoring";
debug!(self.log, "Peer sent invalid parent.";
"peer_id" => format!("{:?}",peer),
"received_block" => format!("{}", block_hash),
"expected_parent" => format!("{}", expected_hash),
@@ -836,7 +922,8 @@ impl<T: BeaconChainTypes> SyncManager<T> {
re_run_poll = true;
break;
}
Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {}
Ok(BlockProcessingOutcome::Processed { block_root: _ })
| Ok(BlockProcessingOutcome::BlockIsAlreadyKnown { .. }) => {}
Ok(outcome) => {
// it's a future slot or an invalid block, remove it and try again
completed_request.failed_attempts += 1;
@@ -891,7 +978,7 @@ impl<T: BeaconChainTypes> SyncManager<T> {
/* Network Context Helper Functions */
fn hello_peer<T: BeaconChainTypes>(
fn status_peer<T: BeaconChainTypes>(
network: &mut NetworkContext,
log: &slog::Logger,
chain: Weak<BeaconChain<T>>,
@@ -899,26 +986,26 @@ fn hello_peer<T: BeaconChainTypes>(
) {
trace!(
log,
"RPC Request";
"method" => "HELLO",
"Sending Status Request";
"method" => "STATUS",
"peer" => format!("{:?}", peer_id)
);
if let Some(chain) = chain.upgrade() {
network.send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&chain)));
network.send_rpc_request(None, peer_id, RPCRequest::Status(status_message(&chain)));
}
}
fn request_blocks(
fn blocks_by_range_request(
network: &mut NetworkContext,
log: &slog::Logger,
peer_id: PeerId,
request_id: RequestId,
request: BeaconBlocksRequest,
request: BlocksByRangeRequest,
) {
trace!(
log,
"RPC Request";
"method" => "BeaconBlocks",
"Sending BlocksByRange Request";
"method" => "BlocksByRange",
"id" => request_id,
"count" => request.count,
"peer" => format!("{:?}", peer_id)
@@ -926,28 +1013,28 @@ fn request_blocks(
network.send_rpc_request(
Some(request_id),
peer_id.clone(),
RPCRequest::BeaconBlocks(request),
RPCRequest::BlocksByRange(request),
);
}
fn recent_blocks_request(
fn blocks_by_root_request(
network: &mut NetworkContext,
log: &slog::Logger,
peer_id: PeerId,
request_id: RequestId,
request: RecentBeaconBlocksRequest,
request: BlocksByRootRequest,
) {
trace!(
log,
"RPC Request";
"method" => "RecentBeaconBlocks",
"Sending BlocksByRoot Request";
"method" => "BlocksByRoot",
"count" => request.block_roots.len(),
"peer" => format!("{:?}", peer_id)
);
network.send_rpc_request(
Some(request_id),
peer_id.clone(),
RPCRequest::RecentBeaconBlocks(request),
RPCRequest::BlocksByRoot(request),
);
}
@@ -1080,28 +1167,31 @@ impl<T: BeaconChainTypes> Future for SyncManager<T> {
SyncMessage::AddPeer(peer_id, info) => {
self.add_peer(peer_id, info);
}
SyncMessage::BeaconBlocksResponse {
SyncMessage::BlocksByRangeResponse {
peer_id,
request_id,
beacon_blocks,
beacon_block,
} => {
self.beacon_blocks_response(peer_id, request_id, beacon_blocks);
self.blocks_by_range_response(peer_id, request_id, beacon_block);
}
SyncMessage::RecentBeaconBlocksResponse {
SyncMessage::BlocksByRootResponse {
peer_id,
request_id,
beacon_blocks,
beacon_block,
} => {
self.recent_blocks_response(peer_id, request_id, beacon_blocks);
self.blocks_by_root_response(peer_id, request_id, beacon_block);
}
SyncMessage::UnknownBlock(peer_id, block) => {
self.add_unknown_block(peer_id, block);
}
SyncMessage::UnknownBlockHash(peer_id, block_hash) => {
self.search_for_block(peer_id, block_hash);
}
SyncMessage::Disconnect(peer_id) => {
self.peer_disconnect(&peer_id);
}
SyncMessage::_RPCError(request_id) => {
self.inject_error(request_id);
SyncMessage::RPCError(peer_id, request_id) => {
self.inject_error(peer_id, request_id);
}
},
Ok(Async::NotReady) => break,

305
beacon_node/network/src/sync/simple_sync.rs → beacon_node/network/src/sync/message_processor.rs
View File

@@ -14,7 +14,6 @@ use tokio::sync::{mpsc, oneshot};
use tree_hash::SignedRoot;
use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot};
//TODO: Put a maximum limit on the number of block that can be requested.
//TODO: Rate limit requests
/// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it.
@@ -34,21 +33,21 @@ pub struct PeerSyncInfo {
pub head_slot: Slot,
}
impl From<HelloMessage> for PeerSyncInfo {
fn from(hello: HelloMessage) -> PeerSyncInfo {
impl From<StatusMessage> for PeerSyncInfo {
fn from(status: StatusMessage) -> PeerSyncInfo {
PeerSyncInfo {
fork_version: hello.fork_version,
finalized_root: hello.finalized_root,
finalized_epoch: hello.finalized_epoch,
head_root: hello.head_root,
head_slot: hello.head_slot,
fork_version: status.fork_version,
finalized_root: status.finalized_root,
finalized_epoch: status.finalized_epoch,
head_root: status.head_root,
head_slot: status.head_slot,
}
}
}
impl<T: BeaconChainTypes> From<&Arc<BeaconChain<T>>> for PeerSyncInfo {
fn from(chain: &Arc<BeaconChain<T>>) -> PeerSyncInfo {
Self::from(hello_message(chain))
Self::from(status_message(chain))
}
}
@@ -111,49 +110,58 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
self.send_to_sync(SyncMessage::Disconnect(peer_id));
}
/// Handle the connection of a new peer.
///
/// Sends a `Hello` message to the peer.
pub fn on_connect(&mut self, peer_id: PeerId) {
self.network
.send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&self.chain)));
/// An error occurred during an RPC request. The state is maintained by the sync manager, so
/// this function notifies the sync manager of the error.
pub fn on_rpc_error(&mut self, peer_id: PeerId, request_id: RequestId) {
self.send_to_sync(SyncMessage::RPCError(peer_id, request_id));
}
/// Handle a `Hello` request.
/// Handle the connection of a new peer.
///
/// Processes the `HelloMessage` from the remote peer and sends back our `Hello`.
pub fn on_hello_request(
/// Sends a `Status` message to the peer.
pub fn on_connect(&mut self, peer_id: PeerId) {
self.network.send_rpc_request(
None,
peer_id,
RPCRequest::Status(status_message(&self.chain)),
);
}
/// Handle a `Status` request.
///
/// Processes the `Status` from the remote peer and sends back our `Status`.
pub fn on_status_request(
&mut self,
peer_id: PeerId,
request_id: RequestId,
hello: HelloMessage,
status: StatusMessage,
) {
// ignore hello responses if we are shutting down
trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id));
// ignore status responses if we are shutting down
trace!(self.log, "StatusRequest"; "peer" => format!("{:?}", peer_id));
// Say hello back.
// Say status back.
self.network.send_rpc_response(
peer_id.clone(),
request_id,
RPCResponse::Hello(hello_message(&self.chain)),
RPCResponse::Status(status_message(&self.chain)),
);
self.process_hello(peer_id, hello);
self.process_status(peer_id, status);
}
/// Process a `Hello` response from a peer.
pub fn on_hello_response(&mut self, peer_id: PeerId, hello: HelloMessage) {
trace!(self.log, "HelloResponse"; "peer" => format!("{:?}", peer_id));
/// Process a `Status` response from a peer.
pub fn on_status_response(&mut self, peer_id: PeerId, status: StatusMessage) {
trace!(self.log, "StatusResponse"; "peer" => format!("{:?}", peer_id));
// Process the hello message, without sending back another hello.
self.process_hello(peer_id, hello);
// Process the status message, without sending back another status.
self.process_status(peer_id, status);
}
/// Process a `Hello` message, requesting new blocks if appropriate.
/// Process a `Status` message, requesting new blocks if appropriate.
///
/// Disconnects the peer if required.
fn process_hello(&mut self, peer_id: PeerId, hello: HelloMessage) {
let remote = PeerSyncInfo::from(hello);
fn process_status(&mut self, peer_id: PeerId, status: StatusMessage) {
let remote = PeerSyncInfo::from(status);
let local = PeerSyncInfo::from(&self.chain);
let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch());
@@ -161,11 +169,27 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
if local.fork_version != remote.fork_version {
// The node is on a different network/fork, disconnect them.
debug!(
self.log, "HandshakeFailure";
self.log, "Handshake Failure";
"peer" => format!("{:?}", peer_id),
"reason" => "network_id"
);
self.network
.disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork);
} else if remote.head_slot
> self.chain.slot().unwrap_or_else(|_| Slot::from(0u64)) + FUTURE_SLOT_TOLERANCE
{
// Note: If the slot_clock cannot be read, this will not error. Other system
// components will deal with an invalid slot clock error.
// The remotes head is on a slot that is significantly ahead of ours. This could be
// because they are using a different genesis time, or that theirs or our system
// clock is incorrect.
debug!(
self.log, "Handshake Failure";
"peer" => format!("{:?}", peer_id),
"reason" => "different system clocks or genesis time"
);
self.network
.disconnect(peer_id.clone(), GoodbyeReason::IrrelevantNetwork);
} else if remote.finalized_epoch <= local.finalized_epoch
@@ -179,7 +203,7 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
//
// Therefore, the node is on a different chain and we should not communicate with them.
debug!(
self.log, "HandshakeFailure";
self.log, "Handshake Failure";
"peer" => format!("{:?}", peer_id),
"reason" => "different finalized chain"
);
@@ -236,57 +260,57 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
}
}
/// Handle a `RecentBeaconBlocks` request from the peer.
pub fn on_recent_beacon_blocks_request(
/// Handle a `BlocksByRoot` request from the peer.
pub fn on_blocks_by_root_request(
&mut self,
peer_id: PeerId,
request_id: RequestId,
request: RecentBeaconBlocksRequest,
request: BlocksByRootRequest,
) {
let blocks: Vec<BeaconBlock<_>> = request
.block_roots
.iter()
.filter_map(|root| {
if let Ok(Some(block)) = self.chain.store.get::<BeaconBlock<T::EthSpec>>(root) {
Some(block)
} else {
debug!(
self.log,
"Peer requested unknown block";
"peer" => format!("{:?}", peer_id),
"request_root" => format!("{:}", root),
);
None
}
})
.collect();
let mut send_block_count = 0;
for root in request.block_roots.iter() {
if let Ok(Some(block)) = self.chain.store.get::<BeaconBlock<T::EthSpec>>(root) {
self.network.send_rpc_response(
peer_id.clone(),
request_id,
RPCResponse::BlocksByRoot(block.as_ssz_bytes()),
);
send_block_count += 1;
} else {
debug!(
self.log,
"Peer requested unknown block";
"peer" => format!("{:?}", peer_id),
"request_root" => format!("{:}", root),
);
}
}
debug!(
self.log,
"RecentBeaconBlocksRequest";
"Received BlocksByRoot Request";
"peer" => format!("{:?}", peer_id),
"requested" => request.block_roots.len(),
"returned" => blocks.len(),
"returned" => send_block_count,
);
self.network.send_rpc_response(
// send stream termination
self.network.send_rpc_error_response(
peer_id,
request_id,
RPCResponse::BeaconBlocks(blocks.as_ssz_bytes()),
)
RPCErrorResponse::StreamTermination(ResponseTermination::BlocksByRoot),
);
}
/// Handle a `BeaconBlocks` request from the peer.
pub fn on_beacon_blocks_request(
/// Handle a `BlocksByRange` request from the peer.
pub fn on_blocks_by_range_request(
&mut self,
peer_id: PeerId,
request_id: RequestId,
req: BeaconBlocksRequest,
req: BlocksByRangeRequest,
) {
debug!(
self.log,
"BeaconBlocksRequest";
"Received BlocksByRange Request";
"peer" => format!("{:?}", peer_id),
"count" => req.count,
"start_slot" => req.start_slot,
@@ -297,6 +321,10 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
// In the current implementation we read from the db then filter out out-of-range blocks.
// Improving the db schema to prevent this would be ideal.
//TODO: This really needs to be read forward for infinite streams
// We should be reading the first block from the db, sending, then reading the next... we
// need a forwards iterator!!
let mut blocks: Vec<BeaconBlock<T::EthSpec>> = self
.chain
.rev_iter_block_roots()
@@ -313,7 +341,6 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
"Block in the chain is not in the store";
"request_root" => format!("{:}", root),
);
None
}
})
@@ -323,63 +350,82 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
blocks.reverse();
blocks.dedup_by_key(|brs| brs.slot);
debug!(
self.log,
"BeaconBlocksRequest response";
"peer" => format!("{:?}", peer_id),
"msg" => "Failed to return all requested hashes",
"start_slot" => req.start_slot,
"current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(),
"requested" => req.count,
"returned" => blocks.len(),
);
if blocks.len() < (req.count as usize) {
debug!(
self.log,
"Sending BlocksByRange Response";
"peer" => format!("{:?}", peer_id),
"msg" => "Failed to return all requested blocks",
"start_slot" => req.start_slot,
"current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(),
"requested" => req.count,
"returned" => blocks.len(),
);
} else {
trace!(
self.log,
"Sending BlocksByRange Response";
"peer" => format!("{:?}", peer_id),
"start_slot" => req.start_slot,
"current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(),
"requested" => req.count,
"returned" => blocks.len(),
);
}
self.network.send_rpc_response(
for block in blocks {
self.network.send_rpc_response(
peer_id.clone(),
request_id,
RPCResponse::BlocksByRange(block.as_ssz_bytes()),
);
}
// send the stream terminator
self.network.send_rpc_error_response(
peer_id,
request_id,
RPCResponse::BeaconBlocks(blocks.as_ssz_bytes()),
)
RPCErrorResponse::StreamTermination(ResponseTermination::BlocksByRange),
);
}
/// Handle a `BeaconBlocks` response from the peer.
pub fn on_beacon_blocks_response(
/// Handle a `BlocksByRange` response from the peer.
/// A `beacon_block` behaves as a stream which is terminated on a `None` response.
pub fn on_blocks_by_range_response(
&mut self,
peer_id: PeerId,
request_id: RequestId,
beacon_blocks: Vec<BeaconBlock<T::EthSpec>>,
beacon_block: Option<BeaconBlock<T::EthSpec>>,
) {
debug!(
trace!(
self.log,
"BeaconBlocksResponse";
"Received BlocksByRange Response";
"peer" => format!("{:?}", peer_id),
"count" => beacon_blocks.len(),
);
self.send_to_sync(SyncMessage::BeaconBlocksResponse {
self.send_to_sync(SyncMessage::BlocksByRangeResponse {
peer_id,
request_id,
beacon_blocks,
beacon_block,
});
}
/// Handle a `RecentBeaconBlocks` response from the peer.
pub fn on_recent_beacon_blocks_response(
/// Handle a `BlocksByRoot` response from the peer.
pub fn on_blocks_by_root_response(
&mut self,
peer_id: PeerId,
request_id: RequestId,
beacon_blocks: Vec<BeaconBlock<T::EthSpec>>,
beacon_block: Option<BeaconBlock<T::EthSpec>>,
) {
debug!(
trace!(
self.log,
"RecentBeaconBlocksResponse";
"Received BlocksByRoot Response";
"peer" => format!("{:?}", peer_id),
"count" => beacon_blocks.len(),
);
self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse {
self.send_to_sync(SyncMessage::BlocksByRootResponse {
peer_id,
request_id,
beacon_blocks,
beacon_block,
});
}
@@ -447,24 +493,36 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
/// Process a gossip message declaring a new attestation.
///
/// Not currently implemented.
pub fn on_attestation_gossip(&mut self, _peer_id: PeerId, msg: Attestation<T::EthSpec>) {
pub fn on_attestation_gossip(&mut self, peer_id: PeerId, msg: Attestation<T::EthSpec>) {
match self.chain.process_attestation(msg.clone()) {
Ok(outcome) => {
info!(
self.log,
"Processed attestation";
"source" => "gossip",
"outcome" => format!("{:?}", outcome)
);
if outcome != AttestationProcessingOutcome::Processed {
trace!(
Ok(outcome) => match outcome {
AttestationProcessingOutcome::Processed => {
info!(
self.log,
"Invalid gossip attestation ssz";
"ssz" => format!("0x{}", hex::encode(msg.as_ssz_bytes())),
"Processed attestation";
"source" => "gossip",
"outcome" => format!("{:?}", outcome)
);
}
}
AttestationProcessingOutcome::UnknownHeadBlock { beacon_block_root } => {
// TODO: Maintain this attestation and re-process once sync completes
debug!(
self.log,
"Attestation for unknown block";
"peer_id" => format!("{:?}", peer_id),
"block" => format!("{}", beacon_block_root)
);
// we don't know the block, get the sync manager to handle the block lookup
self.send_to_sync(SyncMessage::UnknownBlockHash(peer_id, beacon_block_root));
}
AttestationProcessingOutcome::AttestsToFutureState { .. }
| AttestationProcessingOutcome::FinalizedSlot { .. } => {} // ignore the attestation
AttestationProcessingOutcome::Invalid { .. }
| AttestationProcessingOutcome::EmptyAggregationBitfield { .. } => {
// the peer has sent a bad attestation. Remove them.
self.network.disconnect(peer_id, GoodbyeReason::Fault);
}
},
Err(e) => {
trace!(
self.log,
@@ -477,11 +535,11 @@ impl<T: BeaconChainTypes> MessageProcessor<T> {
}
}
/// Build a `HelloMessage` representing the state of the given `beacon_chain`.
pub(crate) fn hello_message<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) -> HelloMessage {
/// Build a `StatusMessage` representing the state of the given `beacon_chain`.
pub(crate) fn status_message<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>) -> StatusMessage {
let state = &beacon_chain.head().beacon_state;
HelloMessage {
StatusMessage {
fork_version: state.fork.current_version,
finalized_root: state.finalized_checkpoint.root,
finalized_epoch: state.finalized_checkpoint.epoch,
@@ -510,8 +568,15 @@ impl NetworkContext {
"reason" => format!("{:?}", reason),
"peer_id" => format!("{:?}", peer_id),
);
self.send_rpc_request(None, peer_id, RPCRequest::Goodbye(reason))
// TODO: disconnect peers.
self.send_rpc_request(None, peer_id.clone(), RPCRequest::Goodbye(reason));
self.network_send
.try_send(NetworkMessage::Disconnect { peer_id })
.unwrap_or_else(|_| {
warn!(
self.log,
"Could not send a Disconnect to the network service"
)
});
}
pub fn send_rpc_request(
@@ -525,7 +590,7 @@ impl NetworkContext {
self.send_rpc_event(peer_id, RPCEvent::Request(request_id, rpc_request));
}
//TODO: Handle Error responses
/// Convenience function to wrap successful RPC Responses.
pub fn send_rpc_response(
&mut self,
peer_id: PeerId,
@@ -538,6 +603,16 @@ impl NetworkContext {
);
}
/// Send an RPCErrorResponse. This handles errors and stream terminations.
pub fn send_rpc_error_response(
&mut self,
peer_id: PeerId,
request_id: RequestId,
rpc_error_response: RPCErrorResponse,
) {
self.send_rpc_event(peer_id, RPCEvent::Response(request_id, rpc_error_response));
}
fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) {
self.network_send
.try_send(NetworkMessage::RPC(peer_id, rpc_event))

10
beacon_node/network/src/sync/mod.rs
View File

@@ -1,10 +1,10 @@
//! Syncing for lighthouse.
//!
//! Stores the various syncing methods for the beacon chain.
mod manager;
/// Syncing for lighthouse.
///
/// Stores the various syncing methods for the beacon chain.
mod simple_sync;
mod message_processor;
pub use simple_sync::MessageProcessor;
pub use message_processor::MessageProcessor;
/// Currently implemented sync methods.
pub enum SyncMethod {