mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-23 14:54:45 +00:00
Remove sync await points (#3036)
## Issue Addressed Removes the await points in sync waiting for a processor response for rpc block processing. Built on top of #3029 This also handles a couple of bugs in the previous code and adds a relatively comprehensive test suite.
This commit is contained in:
Divma
637
beacon_node/network/src/sync/block_lookups/mod.rs
Normal file
637
beacon_node/network/src/sync/block_lookups/mod.rs
Normal file
@@ -0,0 +1,637 @@
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::time::Duration;
|
||||
|
||||
use beacon_chain::{BeaconChainTypes, BlockError};
|
||||
use fnv::FnvHashMap;
|
||||
use lighthouse_network::{PeerAction, PeerId};
|
||||
use lru_cache::LRUCache;
|
||||
use slog::{crit, debug, error, trace, warn, Logger};
|
||||
use smallvec::SmallVec;
|
||||
use store::{Hash256, SignedBeaconBlock};
|
||||
use strum::AsStaticRef;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
|
||||
use crate::metrics;
|
||||
|
||||
use self::{
|
||||
parent_lookup::{ParentLookup, VerifyError},
|
||||
single_block_lookup::SingleBlockRequest,
|
||||
};
|
||||
|
||||
use super::BatchProcessResult;
|
||||
use super::{
|
||||
manager::{BlockProcessType, Id},
|
||||
network_context::SyncNetworkContext,
|
||||
};
|
||||
|
||||
mod parent_lookup;
|
||||
mod single_block_lookup;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
const FAILED_CHAINS_CACHE_SIZE: usize = 500;
|
||||
const SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS: u8 = 3;
|
||||
|
||||
pub(crate) struct BlockLookups<T: BeaconChainTypes> {
|
||||
/// A collection of parent block lookups.
|
||||
parent_queue: SmallVec<[ParentLookup<T::EthSpec>; 3]>,
|
||||
|
||||
/// A cache of failed chain lookups to prevent duplicate searches.
|
||||
failed_chains: LRUCache<Hash256>,
|
||||
|
||||
/// A collection of block hashes being searched for and a flag indicating if a result has been
|
||||
/// received or not.
|
||||
///
|
||||
/// The flag allows us to determine if the peer returned data or sent us nothing.
|
||||
single_block_lookups: FnvHashMap<Id, SingleBlockRequest<SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS>>,
|
||||
|
||||
/// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
|
||||
beacon_processor_send: mpsc::Sender<WorkEvent<T>>,
|
||||
|
||||
/// The logger for the import manager.
|
||||
log: Logger,
|
||||
}
|
||||
|
||||
impl<T: BeaconChainTypes> BlockLookups<T> {
|
||||
pub fn new(beacon_processor_send: mpsc::Sender<WorkEvent<T>>, log: Logger) -> Self {
|
||||
Self {
|
||||
parent_queue: Default::default(),
|
||||
failed_chains: LRUCache::new(FAILED_CHAINS_CACHE_SIZE),
|
||||
single_block_lookups: Default::default(),
|
||||
beacon_processor_send,
|
||||
log,
|
||||
}
|
||||
}
|
||||
|
||||
/* Lookup requests */
|
||||
|
||||
pub fn search_block(
|
||||
&mut self,
|
||||
hash: Hash256,
|
||||
peer_id: PeerId,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
// Do not re-request a block that is already being requested
|
||||
if self
|
||||
.single_block_lookups
|
||||
.values_mut()
|
||||
.any(|single_block_request| single_block_request.add_peer(&hash, &peer_id))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
debug!(
|
||||
self.log,
|
||||
"Searching for block";
|
||||
"peer_id" => %peer_id,
|
||||
"block" => %hash
|
||||
);
|
||||
|
||||
let mut single_block_request = SingleBlockRequest::new(hash, peer_id);
|
||||
|
||||
let (peer_id, request) = single_block_request.request_block().unwrap();
|
||||
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, request) {
|
||||
self.single_block_lookups
|
||||
.insert(request_id, single_block_request);
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_SINGLE_BLOCK_LOOKUPS,
|
||||
self.single_block_lookups.len() as i64,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_parent(
|
||||
&mut self,
|
||||
block: Box<SignedBeaconBlock<T::EthSpec>>,
|
||||
peer_id: PeerId,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let block_root = block.canonical_root();
|
||||
let parent_root = block.parent_root();
|
||||
// If this block or it's parent is part of a known failed chain, ignore it.
|
||||
if self.failed_chains.contains(&parent_root) || self.failed_chains.contains(&block_root) {
|
||||
debug!(self.log, "Block is from a past failed chain. Dropping";
|
||||
"block_root" => ?block_root, "block_slot" => block.slot());
|
||||
return;
|
||||
}
|
||||
|
||||
// Make sure this block is not already downloaded, and that neither it or its parent is
|
||||
// being searched for.
|
||||
if self.parent_queue.iter_mut().any(|parent_req| {
|
||||
parent_req.contains_block(&block)
|
||||
|| parent_req.add_peer(&block_root, &peer_id)
|
||||
|| parent_req.add_peer(&parent_root, &peer_id)
|
||||
}) {
|
||||
// we are already searching for this block, ignore it
|
||||
return;
|
||||
}
|
||||
|
||||
let parent_lookup = ParentLookup::new(*block, peer_id);
|
||||
self.request_parent(parent_lookup, cx);
|
||||
}
|
||||
|
||||
/* Lookup responses */
|
||||
|
||||
pub fn single_block_lookup_response(
|
||||
&mut self,
|
||||
id: Id,
|
||||
peer_id: PeerId,
|
||||
block: Option<Box<SignedBeaconBlock<T::EthSpec>>>,
|
||||
seen_timestamp: Duration,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let mut request = match self.single_block_lookups.entry(id) {
|
||||
Entry::Occupied(req) => req,
|
||||
Entry::Vacant(_) => {
|
||||
if block.is_some() {
|
||||
crit!(
|
||||
self.log,
|
||||
"Block returned for single block lookup not present"
|
||||
);
|
||||
#[cfg(debug_assertions)]
|
||||
panic!("block returned for single block lookup not present");
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match request.get_mut().verify_block(block) {
|
||||
Ok(Some(block)) => {
|
||||
// This is the correct block, send it for processing
|
||||
if self
|
||||
.send_block_for_processing(
|
||||
block,
|
||||
seen_timestamp,
|
||||
BlockProcessType::SingleBlock { id },
|
||||
)
|
||||
.is_err()
|
||||
{
|
||||
// Remove to avoid inconsistencies
|
||||
self.single_block_lookups.remove(&id);
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// request finished correctly, it will be removed after the block is processed.
|
||||
}
|
||||
Err(error) => {
|
||||
let msg: &str = error.as_static();
|
||||
cx.report_peer(peer_id, PeerAction::LowToleranceError, msg);
|
||||
// Remove the request, if it can be retried it will be added with a new id.
|
||||
let mut req = request.remove();
|
||||
|
||||
debug!(self.log, "Single block lookup failed";
|
||||
"peer_id" => %peer_id, "error" => msg, "block_root" => %req.hash);
|
||||
// try the request again if possible
|
||||
if let Ok((peer_id, request)) = req.request_block() {
|
||||
if let Ok(id) = cx.single_block_lookup_request(peer_id, request) {
|
||||
self.single_block_lookups.insert(id, req);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_SINGLE_BLOCK_LOOKUPS,
|
||||
self.single_block_lookups.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn parent_lookup_response(
|
||||
&mut self,
|
||||
id: Id,
|
||||
peer_id: PeerId,
|
||||
block: Option<Box<SignedBeaconBlock<T::EthSpec>>>,
|
||||
seen_timestamp: Duration,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let mut parent_lookup = if let Some(pos) = self
|
||||
.parent_queue
|
||||
.iter()
|
||||
.position(|request| request.pending_response(id))
|
||||
{
|
||||
self.parent_queue.remove(pos)
|
||||
} else {
|
||||
if block.is_some() {
|
||||
debug!(self.log, "Response for a parent lookup request that was not found"; "peer_id" => %peer_id);
|
||||
}
|
||||
return;
|
||||
};
|
||||
|
||||
match parent_lookup.verify_block(block, &self.failed_chains) {
|
||||
Ok(Some(block)) => {
|
||||
// Block is correct, send to the beacon processor.
|
||||
let chain_hash = parent_lookup.chain_hash();
|
||||
if self
|
||||
.send_block_for_processing(
|
||||
block,
|
||||
seen_timestamp,
|
||||
BlockProcessType::ParentLookup { chain_hash },
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
self.parent_queue.push(parent_lookup)
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
// Request finished successfully, nothing else to do. It will be removed after the
|
||||
// processing result arrives.
|
||||
self.parent_queue.push(parent_lookup);
|
||||
}
|
||||
Err(e) => match e {
|
||||
VerifyError::RootMismatch
|
||||
| VerifyError::NoBlockReturned
|
||||
| VerifyError::ExtraBlocksReturned => {
|
||||
let e = e.as_static();
|
||||
warn!(self.log, "Peer sent invalid response to parent request.";
|
||||
"peer_id" => %peer_id, "reason" => e);
|
||||
|
||||
// We do not tolerate these kinds of errors. We will accept a few but these are signs
|
||||
// of a faulty peer.
|
||||
cx.report_peer(peer_id, PeerAction::LowToleranceError, e);
|
||||
|
||||
// We try again if possible.
|
||||
self.request_parent(parent_lookup, cx);
|
||||
}
|
||||
VerifyError::PreviousFailure { parent_root } => {
|
||||
self.failed_chains.insert(parent_lookup.chain_hash());
|
||||
debug!(
|
||||
self.log,
|
||||
"Parent chain ignored due to past failure";
|
||||
"block" => %parent_root,
|
||||
);
|
||||
// Add the root block to failed chains
|
||||
self.failed_chains.insert(parent_lookup.chain_hash());
|
||||
|
||||
cx.report_peer(
|
||||
peer_id,
|
||||
PeerAction::MidToleranceError,
|
||||
"bbroot_failed_chains",
|
||||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_PARENT_BLOCK_LOOKUPS,
|
||||
self.parent_queue.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
/* Error responses */
|
||||
|
||||
#[allow(clippy::needless_collect)] // false positive
|
||||
pub fn peer_disconnected(&mut self, peer_id: &PeerId, cx: &mut SyncNetworkContext<T::EthSpec>) {
|
||||
/* Check disconnection for single block lookups */
|
||||
// better written after https://github.com/rust-lang/rust/issues/59618
|
||||
let remove_retry_ids: Vec<Id> = self
|
||||
.single_block_lookups
|
||||
.iter_mut()
|
||||
.filter_map(|(id, req)| {
|
||||
if req.check_peer_disconnected(peer_id).is_err() {
|
||||
Some(*id)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
for mut req in remove_retry_ids
|
||||
.into_iter()
|
||||
.map(|id| self.single_block_lookups.remove(&id).unwrap())
|
||||
.collect::<Vec<_>>()
|
||||
{
|
||||
// retry the request
|
||||
match req.request_block() {
|
||||
Ok((peer_id, block_request)) => {
|
||||
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, block_request) {
|
||||
self.single_block_lookups.insert(request_id, req);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
trace!(self.log, "Single block request failed on peer disconnection";
|
||||
"block_root" => %req.hash, "peer_id" => %peer_id, "reason" => e.as_static());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Check disconnection for parent lookups */
|
||||
while let Some(pos) = self
|
||||
.parent_queue
|
||||
.iter_mut()
|
||||
.position(|req| req.check_peer_disconnected(peer_id).is_err())
|
||||
{
|
||||
let parent_lookup = self.parent_queue.remove(pos);
|
||||
trace!(self.log, "Parent lookup's peer disconnected"; &parent_lookup);
|
||||
self.request_parent(parent_lookup, cx);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parent_lookup_failed(
|
||||
&mut self,
|
||||
id: Id,
|
||||
peer_id: PeerId,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
if let Some(pos) = self
|
||||
.parent_queue
|
||||
.iter()
|
||||
.position(|request| request.pending_response(id))
|
||||
{
|
||||
let mut parent_lookup = self.parent_queue.remove(pos);
|
||||
parent_lookup.download_failed();
|
||||
trace!(self.log, "Parent lookup request failed"; &parent_lookup);
|
||||
self.request_parent(parent_lookup, cx);
|
||||
} else {
|
||||
return debug!(self.log, "RPC failure for a parent lookup request that was not found"; "peer_id" => %peer_id);
|
||||
};
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_PARENT_BLOCK_LOOKUPS,
|
||||
self.parent_queue.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn single_block_lookup_failed(&mut self, id: Id, cx: &mut SyncNetworkContext<T::EthSpec>) {
|
||||
if let Some(mut request) = self.single_block_lookups.remove(&id) {
|
||||
request.register_failure();
|
||||
trace!(self.log, "Single block lookup failed"; "block" => %request.hash);
|
||||
if let Ok((peer_id, block_request)) = request.request_block() {
|
||||
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, block_request) {
|
||||
self.single_block_lookups.insert(request_id, request);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_SINGLE_BLOCK_LOOKUPS,
|
||||
self.single_block_lookups.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
/* Processing responses */
|
||||
|
||||
pub fn single_block_processed(
|
||||
&mut self,
|
||||
id: Id,
|
||||
result: Result<(), BlockError<T::EthSpec>>,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let mut req = match self.single_block_lookups.remove(&id) {
|
||||
Some(req) => req,
|
||||
None => {
|
||||
#[cfg(debug_assertions)]
|
||||
panic!("block processed for single block lookup not present");
|
||||
#[cfg(not(debug_assertions))]
|
||||
return crit!(
|
||||
self.log,
|
||||
"Block processed for single block lookup not present"
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let root = req.hash;
|
||||
let peer_id = match req.processing_peer() {
|
||||
Ok(peer) => peer,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
if let Err(e) = &result {
|
||||
trace!(self.log, "Single block processing failed"; "block" => %root, "error" => %e);
|
||||
} else {
|
||||
trace!(self.log, "Single block processing succeeded"; "block" => %root);
|
||||
}
|
||||
|
||||
match result {
|
||||
Err(e) => match e {
|
||||
BlockError::BlockIsAlreadyKnown => {
|
||||
// No error here
|
||||
}
|
||||
BlockError::BeaconChainError(e) => {
|
||||
// Internal error
|
||||
error!(self.log, "Beacon chain error processing single block"; "block_root" => %root, "error" => ?e);
|
||||
}
|
||||
BlockError::ParentUnknown(block) => {
|
||||
self.search_parent(block, peer_id, cx);
|
||||
}
|
||||
other => {
|
||||
warn!(self.log, "Peer sent invalid block in single block lookup"; "root" => %root, "error" => ?other, "peer_id" => %peer_id);
|
||||
cx.report_peer(
|
||||
peer_id,
|
||||
PeerAction::MidToleranceError,
|
||||
"single_block_failure",
|
||||
);
|
||||
|
||||
// Try it again if possible.
|
||||
req.register_failure();
|
||||
if let Ok((peer_id, request)) = req.request_block() {
|
||||
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, request) {
|
||||
// insert with the new id
|
||||
self.single_block_lookups.insert(request_id, req);
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(()) => {
|
||||
// No error here
|
||||
}
|
||||
}
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_SINGLE_BLOCK_LOOKUPS,
|
||||
self.single_block_lookups.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn parent_block_processed(
|
||||
&mut self,
|
||||
chain_hash: Hash256,
|
||||
result: Result<(), BlockError<T::EthSpec>>,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let (mut parent_lookup, peer_id) = if let Some((pos, peer)) = self
|
||||
.parent_queue
|
||||
.iter()
|
||||
.enumerate()
|
||||
.find_map(|(pos, request)| {
|
||||
request
|
||||
.get_processing_peer(chain_hash)
|
||||
.map(|peer| (pos, peer))
|
||||
}) {
|
||||
(self.parent_queue.remove(pos), peer)
|
||||
} else {
|
||||
#[cfg(debug_assertions)]
|
||||
panic!(
|
||||
"Process response for a parent lookup request that was not found. Chain_hash: {}",
|
||||
chain_hash
|
||||
);
|
||||
#[cfg(not(debug_assertions))]
|
||||
return crit!(self.log, "Process response for a parent lookup request that was not found"; "chain_hash" => %chain_hash);
|
||||
};
|
||||
|
||||
if let Err(e) = &result {
|
||||
trace!(self.log, "Parent block processing failed"; &parent_lookup, "error" => %e);
|
||||
} else {
|
||||
trace!(self.log, "Parent block processing succeeded"; &parent_lookup);
|
||||
}
|
||||
|
||||
match result {
|
||||
Err(BlockError::ParentUnknown(block)) => {
|
||||
// need to keep looking for parents
|
||||
// add the block back to the queue and continue the search
|
||||
parent_lookup.add_block(*block);
|
||||
self.request_parent(parent_lookup, cx);
|
||||
}
|
||||
Ok(_) | Err(BlockError::BlockIsAlreadyKnown { .. }) => {
|
||||
let chain_hash = parent_lookup.chain_hash();
|
||||
let blocks = parent_lookup.chain_blocks();
|
||||
let process_id = ChainSegmentProcessId::ParentLookup(chain_hash);
|
||||
|
||||
match self
|
||||
.beacon_processor_send
|
||||
.try_send(WorkEvent::chain_segment(process_id, blocks))
|
||||
{
|
||||
Ok(_) => {
|
||||
self.parent_queue.push(parent_lookup);
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
self.log,
|
||||
"Failed to send chain segment to processor";
|
||||
"error" => ?e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(outcome) => {
|
||||
// all else we consider the chain a failure and downvote the peer that sent
|
||||
// us the last block
|
||||
warn!(
|
||||
self.log, "Invalid parent chain";
|
||||
"score_adjustment" => %PeerAction::MidToleranceError,
|
||||
"outcome" => ?outcome,
|
||||
"last_peer" => %peer_id,
|
||||
);
|
||||
|
||||
// Add this chain to cache of failed chains
|
||||
self.failed_chains.insert(chain_hash);
|
||||
|
||||
// This currently can be a host of errors. We permit this due to the partial
|
||||
// ambiguity.
|
||||
cx.report_peer(peer_id, PeerAction::MidToleranceError, "parent_request_err");
|
||||
}
|
||||
}
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_PARENT_BLOCK_LOOKUPS,
|
||||
self.parent_queue.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
pub fn parent_chain_processed(
|
||||
&mut self,
|
||||
chain_hash: Hash256,
|
||||
result: BatchProcessResult,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
let parent_lookup = if let Some(pos) = self
|
||||
.parent_queue
|
||||
.iter()
|
||||
.position(|request| request.chain_hash() == chain_hash)
|
||||
{
|
||||
self.parent_queue.remove(pos)
|
||||
} else {
|
||||
#[cfg(debug_assertions)]
|
||||
panic!(
|
||||
"Chain process response for a parent lookup request that was not found. Chain_hash: {}",
|
||||
chain_hash
|
||||
);
|
||||
#[cfg(not(debug_assertions))]
|
||||
return crit!(self.log, "Chain process response for a parent lookup request that was not found"; "chain_hash" => %chain_hash);
|
||||
};
|
||||
|
||||
debug!(self.log, "Parent chain processed"; "chain_hash" => %chain_hash, "result" => ?result);
|
||||
match result {
|
||||
BatchProcessResult::Success(_) => {
|
||||
// nothing to do.
|
||||
}
|
||||
BatchProcessResult::Failed {
|
||||
imported_blocks: _,
|
||||
peer_action,
|
||||
} => {
|
||||
self.failed_chains.insert(parent_lookup.chain_hash());
|
||||
if let Some(peer_action) = peer_action {
|
||||
for &peer_id in parent_lookup.used_peers() {
|
||||
cx.report_peer(peer_id, peer_action, "parent_chain_failure")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_PARENT_BLOCK_LOOKUPS,
|
||||
self.parent_queue.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
/* Helper functions */
|
||||
|
||||
fn send_block_for_processing(
|
||||
&mut self,
|
||||
block: Box<SignedBeaconBlock<T::EthSpec>>,
|
||||
duration: Duration,
|
||||
process_type: BlockProcessType,
|
||||
) -> Result<(), ()> {
|
||||
trace!(self.log, "Sending block for processing"; "block" => %block.canonical_root(), "process" => ?process_type);
|
||||
let event = WorkEvent::rpc_beacon_block(block, duration, process_type);
|
||||
if let Err(e) = self.beacon_processor_send.try_send(event) {
|
||||
error!(
|
||||
self.log,
|
||||
"Failed to send sync block to processor";
|
||||
"error" => ?e
|
||||
);
|
||||
return Err(());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn request_parent(
|
||||
&mut self,
|
||||
mut parent_lookup: ParentLookup<T::EthSpec>,
|
||||
cx: &mut SyncNetworkContext<T::EthSpec>,
|
||||
) {
|
||||
match parent_lookup.request_parent(cx) {
|
||||
Err(e) => {
|
||||
debug!(self.log, "Failed to request parent"; &parent_lookup, "error" => e.as_static());
|
||||
match e {
|
||||
parent_lookup::RequestError::SendFailed(_) => {
|
||||
// Probably shutting down, nothing to do here. Drop the request
|
||||
}
|
||||
parent_lookup::RequestError::ChainTooLong
|
||||
| parent_lookup::RequestError::TooManyAttempts => {
|
||||
self.failed_chains.insert(parent_lookup.chain_hash());
|
||||
// This indicates faulty peers.
|
||||
for &peer_id in parent_lookup.used_peers() {
|
||||
cx.report_peer(peer_id, PeerAction::LowToleranceError, e.as_static())
|
||||
}
|
||||
}
|
||||
parent_lookup::RequestError::NoPeers => {
|
||||
// This happens if the peer disconnects while the block is being
|
||||
// processed. Drop the request without extra penalty
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
debug!(self.log, "Requesting parent"; &parent_lookup);
|
||||
self.parent_queue.push(parent_lookup)
|
||||
}
|
||||
}
|
||||
|
||||
// We remove and add back again requests so we want this updated regardless of outcome.
|
||||
metrics::set_gauge(
|
||||
&metrics::SYNC_PARENT_BLOCK_LOOKUPS,
|
||||
self.parent_queue.len() as i64,
|
||||
);
|
||||
}
|
||||
}
|
||||
201
beacon_node/network/src/sync/block_lookups/parent_lookup.rs
Normal file
201
beacon_node/network/src/sync/block_lookups/parent_lookup.rs
Normal file
@@ -0,0 +1,201 @@
|
||||
use lighthouse_network::PeerId;
|
||||
use store::{EthSpec, Hash256, SignedBeaconBlock};
|
||||
use strum::AsStaticStr;
|
||||
|
||||
use crate::sync::{
|
||||
manager::{Id, SLOT_IMPORT_TOLERANCE},
|
||||
network_context::SyncNetworkContext,
|
||||
};
|
||||
|
||||
use super::single_block_lookup::{self, SingleBlockRequest};
|
||||
|
||||
/// How many attempts we try to find a parent of a block before we give up trying .
|
||||
pub(crate) const PARENT_FAIL_TOLERANCE: u8 = 5;
|
||||
/// The maximum depth we will search for a parent block. In principle we should have sync'd any
|
||||
/// canonical chain to its head once the peer connects. A chain should not appear where it's depth
|
||||
/// is further back than the most recent head slot.
|
||||
pub(crate) const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2;
|
||||
|
||||
/// Maintains a sequential list of parents to lookup and the lookup's current state.
|
||||
pub(crate) struct ParentLookup<T: EthSpec> {
|
||||
/// The root of the block triggering this parent request.
|
||||
chain_hash: Hash256,
|
||||
/// The blocks that have currently been downloaded.
|
||||
downloaded_blocks: Vec<SignedBeaconBlock<T>>,
|
||||
/// Request of the last parent.
|
||||
current_parent_request: SingleBlockRequest<PARENT_FAIL_TOLERANCE>,
|
||||
/// Id of the last parent request.
|
||||
current_parent_request_id: Option<Id>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
|
||||
pub enum VerifyError {
|
||||
RootMismatch,
|
||||
NoBlockReturned,
|
||||
ExtraBlocksReturned,
|
||||
PreviousFailure { parent_root: Hash256 },
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum RequestError {
|
||||
SendFailed(&'static str),
|
||||
ChainTooLong,
|
||||
TooManyAttempts,
|
||||
NoPeers,
|
||||
}
|
||||
|
||||
impl<T: EthSpec> ParentLookup<T> {
|
||||
pub fn contains_block(&self, block: &SignedBeaconBlock<T>) -> bool {
|
||||
self.downloaded_blocks
|
||||
.iter()
|
||||
.any(|d_block| d_block == block)
|
||||
}
|
||||
|
||||
pub fn new(block: SignedBeaconBlock<T>, peer_id: PeerId) -> Self {
|
||||
let current_parent_request = SingleBlockRequest::new(block.parent_root(), peer_id);
|
||||
|
||||
Self {
|
||||
chain_hash: block.canonical_root(),
|
||||
downloaded_blocks: vec![block],
|
||||
current_parent_request,
|
||||
current_parent_request_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Attempts to request the next unknown parent. If the request fails, it should be removed.
|
||||
pub fn request_parent(&mut self, cx: &mut SyncNetworkContext<T>) -> Result<(), RequestError> {
|
||||
// check to make sure this request hasn't failed
|
||||
if self.downloaded_blocks.len() >= PARENT_DEPTH_TOLERANCE {
|
||||
return Err(RequestError::ChainTooLong);
|
||||
}
|
||||
|
||||
let (peer_id, request) = self.current_parent_request.request_block()?;
|
||||
match cx.parent_lookup_request(peer_id, request) {
|
||||
Ok(request_id) => {
|
||||
self.current_parent_request_id = Some(request_id);
|
||||
Ok(())
|
||||
}
|
||||
Err(reason) => {
|
||||
self.current_parent_request_id = None;
|
||||
Err(RequestError::SendFailed(reason))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn check_peer_disconnected(&mut self, peer_id: &PeerId) -> Result<(), ()> {
|
||||
self.current_parent_request.check_peer_disconnected(peer_id)
|
||||
}
|
||||
|
||||
pub fn add_block(&mut self, block: SignedBeaconBlock<T>) {
|
||||
let next_parent = block.parent_root();
|
||||
self.downloaded_blocks.push(block);
|
||||
self.current_parent_request.hash = next_parent;
|
||||
self.current_parent_request.state = single_block_lookup::State::AwaitingDownload;
|
||||
self.current_parent_request_id = None;
|
||||
}
|
||||
|
||||
pub fn pending_response(&self, req_id: Id) -> bool {
|
||||
self.current_parent_request_id == Some(req_id)
|
||||
}
|
||||
|
||||
/// Get the parent lookup's chain hash.
|
||||
pub fn chain_hash(&self) -> Hash256 {
|
||||
self.chain_hash
|
||||
}
|
||||
|
||||
pub fn download_failed(&mut self) {
|
||||
self.current_parent_request.register_failure();
|
||||
self.current_parent_request_id = None;
|
||||
}
|
||||
|
||||
pub fn chain_blocks(&mut self) -> Vec<SignedBeaconBlock<T>> {
|
||||
std::mem::take(&mut self.downloaded_blocks)
|
||||
}
|
||||
|
||||
/// Verifies that the received block is what we requested. If so, parent lookup now waits for
|
||||
/// the processing result of the block.
|
||||
pub fn verify_block(
|
||||
&mut self,
|
||||
block: Option<Box<SignedBeaconBlock<T>>>,
|
||||
failed_chains: &lru_cache::LRUCache<Hash256>,
|
||||
) -> Result<Option<Box<SignedBeaconBlock<T>>>, VerifyError> {
|
||||
let block = self.current_parent_request.verify_block(block)?;
|
||||
|
||||
// check if the parent of this block isn't in the failed cache. If it is, this chain should
|
||||
// be dropped and the peer downscored.
|
||||
if let Some(parent_root) = block.as_ref().map(|block| block.parent_root()) {
|
||||
if failed_chains.contains(&parent_root) {
|
||||
self.current_parent_request.register_failure();
|
||||
self.current_parent_request_id = None;
|
||||
return Err(VerifyError::PreviousFailure { parent_root });
|
||||
}
|
||||
}
|
||||
|
||||
Ok(block)
|
||||
}
|
||||
|
||||
pub fn get_processing_peer(&self, chain_hash: Hash256) -> Option<PeerId> {
|
||||
if self.chain_hash == chain_hash {
|
||||
return self.current_parent_request.processing_peer().ok();
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn failed_attempts(&self) -> u8 {
|
||||
self.current_parent_request.failed_attempts
|
||||
}
|
||||
|
||||
pub fn add_peer(&mut self, block_root: &Hash256, peer_id: &PeerId) -> bool {
|
||||
self.current_parent_request.add_peer(block_root, peer_id)
|
||||
}
|
||||
|
||||
pub fn used_peers(&self) -> impl Iterator<Item = &PeerId> + '_ {
|
||||
self.current_parent_request.used_peers.iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<super::single_block_lookup::VerifyError> for VerifyError {
|
||||
fn from(e: super::single_block_lookup::VerifyError) -> Self {
|
||||
use super::single_block_lookup::VerifyError as E;
|
||||
match e {
|
||||
E::RootMismatch => VerifyError::RootMismatch,
|
||||
E::NoBlockReturned => VerifyError::NoBlockReturned,
|
||||
E::ExtraBlocksReturned => VerifyError::ExtraBlocksReturned,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<super::single_block_lookup::LookupRequestError> for RequestError {
|
||||
fn from(e: super::single_block_lookup::LookupRequestError) -> Self {
|
||||
use super::single_block_lookup::LookupRequestError as E;
|
||||
match e {
|
||||
E::TooManyAttempts => RequestError::TooManyAttempts,
|
||||
E::NoPeers => RequestError::NoPeers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: EthSpec> slog::KV for ParentLookup<T> {
|
||||
fn serialize(
|
||||
&self,
|
||||
record: &slog::Record,
|
||||
serializer: &mut dyn slog::Serializer,
|
||||
) -> slog::Result {
|
||||
serializer.emit_arguments("chain_hash", &format_args!("{}", self.chain_hash))?;
|
||||
slog::Value::serialize(&self.current_parent_request, record, "parent", serializer)?;
|
||||
serializer.emit_usize("downloaded_blocks", self.downloaded_blocks.len())?;
|
||||
slog::Result::Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestError {
|
||||
pub fn as_static(&self) -> &'static str {
|
||||
match self {
|
||||
RequestError::SendFailed(e) => e,
|
||||
RequestError::ChainTooLong => "chain_too_long",
|
||||
RequestError::TooManyAttempts => "too_many_attempts",
|
||||
RequestError::NoPeers => "no_peers",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
use std::collections::HashSet;
|
||||
|
||||
use lighthouse_network::{rpc::BlocksByRootRequest, PeerId};
|
||||
use rand::seq::IteratorRandom;
|
||||
use ssz_types::VariableList;
|
||||
use store::{EthSpec, Hash256, SignedBeaconBlock};
|
||||
use strum::AsStaticStr;
|
||||
|
||||
/// Object representing a single block lookup request.
|
||||
#[derive(PartialEq, Eq)]
|
||||
pub struct SingleBlockRequest<const MAX_ATTEMPTS: u8> {
|
||||
/// The hash of the requested block.
|
||||
pub hash: Hash256,
|
||||
/// State of this request.
|
||||
pub state: State,
|
||||
/// Peers that should have this block.
|
||||
pub available_peers: HashSet<PeerId>,
|
||||
/// Peers from which we have requested this block.
|
||||
pub used_peers: HashSet<PeerId>,
|
||||
/// How many times have we attempted this block.
|
||||
pub failed_attempts: u8,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum State {
|
||||
AwaitingDownload,
|
||||
Downloading { peer_id: PeerId },
|
||||
Processing { peer_id: PeerId },
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
|
||||
pub enum VerifyError {
|
||||
RootMismatch,
|
||||
NoBlockReturned,
|
||||
ExtraBlocksReturned,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
|
||||
pub enum LookupRequestError {
|
||||
TooManyAttempts,
|
||||
NoPeers,
|
||||
}
|
||||
|
||||
impl<const MAX_ATTEMPTS: u8> SingleBlockRequest<MAX_ATTEMPTS> {
|
||||
pub fn new(hash: Hash256, peer_id: PeerId) -> Self {
|
||||
Self {
|
||||
hash,
|
||||
state: State::AwaitingDownload,
|
||||
available_peers: HashSet::from([peer_id]),
|
||||
used_peers: HashSet::default(),
|
||||
failed_attempts: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register_failure(&mut self) {
|
||||
self.failed_attempts += 1;
|
||||
self.state = State::AwaitingDownload;
|
||||
}
|
||||
|
||||
pub fn add_peer(&mut self, hash: &Hash256, peer_id: &PeerId) -> bool {
|
||||
let is_useful = &self.hash == hash;
|
||||
if is_useful {
|
||||
self.available_peers.insert(*peer_id);
|
||||
}
|
||||
is_useful
|
||||
}
|
||||
|
||||
/// If a peer disconnects, this request could be failed. If so, an error is returned
|
||||
pub fn check_peer_disconnected(&mut self, dc_peer_id: &PeerId) -> Result<(), ()> {
|
||||
self.available_peers.remove(dc_peer_id);
|
||||
if let State::Downloading { peer_id } = &self.state {
|
||||
if peer_id == dc_peer_id {
|
||||
// Peer disconnected before providing a block
|
||||
self.register_failure();
|
||||
return Err(());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verifies if the received block matches the requested one.
|
||||
/// Returns the block for processing if the response is what we expected.
|
||||
pub fn verify_block<T: EthSpec>(
|
||||
&mut self,
|
||||
block: Option<Box<SignedBeaconBlock<T>>>,
|
||||
) -> Result<Option<Box<SignedBeaconBlock<T>>>, VerifyError> {
|
||||
match self.state {
|
||||
State::AwaitingDownload => {
|
||||
self.register_failure();
|
||||
Err(VerifyError::ExtraBlocksReturned)
|
||||
}
|
||||
State::Downloading { peer_id } => match block {
|
||||
Some(block) => {
|
||||
if block.canonical_root() != self.hash {
|
||||
// return an error and drop the block
|
||||
self.register_failure();
|
||||
Err(VerifyError::RootMismatch)
|
||||
} else {
|
||||
// Return the block for processing.
|
||||
self.state = State::Processing { peer_id };
|
||||
Ok(Some(block))
|
||||
}
|
||||
}
|
||||
None => {
|
||||
self.register_failure();
|
||||
Err(VerifyError::NoBlockReturned)
|
||||
}
|
||||
},
|
||||
State::Processing { peer_id: _ } => match block {
|
||||
Some(_) => {
|
||||
// We sent the block for processing and received an extra block.
|
||||
self.register_failure();
|
||||
Err(VerifyError::ExtraBlocksReturned)
|
||||
}
|
||||
None => {
|
||||
// This is simply the stream termination and we are already processing the
|
||||
// block
|
||||
Ok(None)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request_block(&mut self) -> Result<(PeerId, BlocksByRootRequest), LookupRequestError> {
|
||||
debug_assert!(matches!(self.state, State::AwaitingDownload));
|
||||
if self.failed_attempts <= MAX_ATTEMPTS {
|
||||
if let Some(&peer_id) = self.available_peers.iter().choose(&mut rand::thread_rng()) {
|
||||
let request = BlocksByRootRequest {
|
||||
block_roots: VariableList::from(vec![self.hash]),
|
||||
};
|
||||
self.state = State::Downloading { peer_id };
|
||||
self.used_peers.insert(peer_id);
|
||||
Ok((peer_id, request))
|
||||
} else {
|
||||
Err(LookupRequestError::NoPeers)
|
||||
}
|
||||
} else {
|
||||
Err(LookupRequestError::TooManyAttempts)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn processing_peer(&self) -> Result<PeerId, ()> {
|
||||
if let State::Processing { peer_id } = &self.state {
|
||||
Ok(*peer_id)
|
||||
} else {
|
||||
Err(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<const MAX_ATTEMPTS: u8> slog::Value for SingleBlockRequest<MAX_ATTEMPTS> {
|
||||
fn serialize(
|
||||
&self,
|
||||
record: &slog::Record,
|
||||
key: slog::Key,
|
||||
serializer: &mut dyn slog::Serializer,
|
||||
) -> slog::Result {
|
||||
serializer.emit_str("request", key)?;
|
||||
serializer.emit_arguments("hash", &format_args!("{}", self.hash))?;
|
||||
match &self.state {
|
||||
State::AwaitingDownload => {
|
||||
"awaiting_download".serialize(record, "state", serializer)?
|
||||
}
|
||||
State::Downloading { peer_id } => {
|
||||
serializer.emit_arguments("downloading_peer", &format_args!("{}", peer_id))?
|
||||
}
|
||||
State::Processing { peer_id } => {
|
||||
serializer.emit_arguments("processing_peer", &format_args!("{}", peer_id))?
|
||||
}
|
||||
}
|
||||
slog::Result::Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
|
||||
use types::MinimalEthSpec as E;
|
||||
|
||||
fn rand_block() -> SignedBeaconBlock<E> {
|
||||
let mut rng = XorShiftRng::from_seed([42; 16]);
|
||||
SignedBeaconBlock::from_block(
|
||||
types::BeaconBlock::Base(types::BeaconBlockBase {
|
||||
..<_>::random_for_test(&mut rng)
|
||||
}),
|
||||
types::Signature::random_for_test(&mut rng),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_happy_path() {
|
||||
let peer_id = PeerId::random();
|
||||
let block = rand_block();
|
||||
|
||||
let mut sl = SingleBlockRequest::<4>::new(block.canonical_root(), peer_id);
|
||||
sl.request_block().unwrap();
|
||||
sl.verify_block(Some(Box::new(block))).unwrap().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_attempts() {
|
||||
let peer_id = PeerId::random();
|
||||
let block = rand_block();
|
||||
|
||||
let mut sl = SingleBlockRequest::<4>::new(block.canonical_root(), peer_id);
|
||||
sl.register_failure();
|
||||
}
|
||||
}
|
||||
460
beacon_node/network/src/sync/block_lookups/tests.rs
Normal file
460
beacon_node/network/src/sync/block_lookups/tests.rs
Normal file
@@ -0,0 +1,460 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::service::RequestId;
|
||||
use crate::sync::manager::RequestId as SyncId;
|
||||
use crate::NetworkMessage;
|
||||
|
||||
use super::*;
|
||||
|
||||
use beacon_chain::builder::Witness;
|
||||
use beacon_chain::eth1_chain::CachingEth1Backend;
|
||||
use lighthouse_network::{NetworkGlobals, Request};
|
||||
use slog::{Drain, Level};
|
||||
use slot_clock::SystemTimeSlotClock;
|
||||
use store::MemoryStore;
|
||||
use types::test_utils::{SeedableRng, TestRandom, XorShiftRng};
|
||||
use types::MinimalEthSpec as E;
|
||||
|
||||
type T = Witness<SystemTimeSlotClock, CachingEth1Backend<E>, E, MemoryStore<E>, MemoryStore<E>>;
|
||||
|
||||
struct TestRig {
|
||||
beacon_processor_rx: mpsc::Receiver<WorkEvent<T>>,
|
||||
network_rx: mpsc::UnboundedReceiver<NetworkMessage<E>>,
|
||||
rng: XorShiftRng,
|
||||
}
|
||||
|
||||
const D: Duration = Duration::new(0, 0);
|
||||
|
||||
impl TestRig {
|
||||
fn test_setup(log_level: Option<Level>) -> (BlockLookups<T>, SyncNetworkContext<E>, Self) {
|
||||
let log = {
|
||||
let decorator = slog_term::TermDecorator::new().build();
|
||||
let drain = slog_term::FullFormat::new(decorator).build().fuse();
|
||||
let drain = slog_async::Async::new(drain).build().fuse();
|
||||
|
||||
if let Some(log_level) = log_level {
|
||||
slog::Logger::root(drain.filter_level(log_level).fuse(), slog::o!())
|
||||
} else {
|
||||
slog::Logger::root(drain.filter(|_| false).fuse(), slog::o!())
|
||||
}
|
||||
};
|
||||
|
||||
let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(100);
|
||||
let (network_tx, network_rx) = mpsc::unbounded_channel();
|
||||
let rng = XorShiftRng::from_seed([42; 16]);
|
||||
let rig = TestRig {
|
||||
beacon_processor_rx,
|
||||
network_rx,
|
||||
rng,
|
||||
};
|
||||
let bl = BlockLookups::new(
|
||||
beacon_processor_tx,
|
||||
log.new(slog::o!("component" => "block_lookups")),
|
||||
);
|
||||
let cx = {
|
||||
let globals = Arc::new(NetworkGlobals::new_test_globals(&log));
|
||||
SyncNetworkContext::new(
|
||||
network_tx,
|
||||
globals,
|
||||
log.new(slog::o!("component" => "network_context")),
|
||||
)
|
||||
};
|
||||
|
||||
(bl, cx, rig)
|
||||
}
|
||||
|
||||
fn rand_block(&mut self) -> SignedBeaconBlock<E> {
|
||||
SignedBeaconBlock::from_block(
|
||||
types::BeaconBlock::Base(types::BeaconBlockBase {
|
||||
..<_>::random_for_test(&mut self.rng)
|
||||
}),
|
||||
types::Signature::random_for_test(&mut self.rng),
|
||||
)
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn expect_block_request(&mut self) -> Id {
|
||||
match self.network_rx.try_recv() {
|
||||
Ok(NetworkMessage::SendRequest {
|
||||
peer_id: _,
|
||||
request: Request::BlocksByRoot(_request),
|
||||
request_id: RequestId::Sync(SyncId::SingleBlock { id }),
|
||||
}) => id,
|
||||
other => {
|
||||
panic!("Expected block request, found {:?}", other);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn expect_parent_request(&mut self) -> Id {
|
||||
match self.network_rx.try_recv() {
|
||||
Ok(NetworkMessage::SendRequest {
|
||||
peer_id: _,
|
||||
request: Request::BlocksByRoot(_request),
|
||||
request_id: RequestId::Sync(SyncId::ParentLookup { id }),
|
||||
}) => id,
|
||||
other => panic!("Expected parent request, found {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn expect_block_process(&mut self) {
|
||||
match self.beacon_processor_rx.try_recv() {
|
||||
Ok(work) => {
|
||||
assert_eq!(work.work_type(), crate::beacon_processor::RPC_BLOCK);
|
||||
}
|
||||
other => panic!("Expected block process, found {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn expect_parent_chain_process(&mut self) {
|
||||
match self.beacon_processor_rx.try_recv() {
|
||||
Ok(work) => {
|
||||
assert_eq!(work.work_type(), crate::beacon_processor::CHAIN_SEGMENT);
|
||||
}
|
||||
other => panic!("Expected chain segment process, found {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
fn expect_empty_network(&mut self) {
|
||||
assert_eq!(
|
||||
self.network_rx.try_recv().expect_err("must err"),
|
||||
mpsc::error::TryRecvError::Empty
|
||||
);
|
||||
}
|
||||
|
||||
#[track_caller]
|
||||
pub fn expect_penalty(&mut self) {
|
||||
match self.network_rx.try_recv() {
|
||||
Ok(NetworkMessage::ReportPeer { .. }) => {}
|
||||
other => panic!("Expected peer penalty, found {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn block_with_parent(&mut self, parent_root: Hash256) -> SignedBeaconBlock<E> {
|
||||
SignedBeaconBlock::from_block(
|
||||
types::BeaconBlock::Base(types::BeaconBlockBase {
|
||||
parent_root,
|
||||
..<_>::random_for_test(&mut self.rng)
|
||||
}),
|
||||
types::Signature::random_for_test(&mut self.rng),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_block_lookup_happy_path() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let block = rig.rand_block();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_block(block.canonical_root(), peer_id, &mut cx);
|
||||
let id = rig.expect_block_request();
|
||||
|
||||
// The peer provides the correct block, should not be penalized. Now the block should be sent
|
||||
// for processing.
|
||||
bl.single_block_lookup_response(id, peer_id, Some(Box::new(block)), D, &mut cx);
|
||||
rig.expect_empty_network();
|
||||
rig.expect_block_process();
|
||||
|
||||
// The request should still be active.
|
||||
assert_eq!(bl.single_block_lookups.len(), 1);
|
||||
|
||||
// Send the stream termination. Peer should have not been penalized, and the request removed
|
||||
// after processing.
|
||||
bl.single_block_lookup_response(id, peer_id, None, D, &mut cx);
|
||||
bl.single_block_processed(id, Ok(()), &mut cx);
|
||||
rig.expect_empty_network();
|
||||
assert_eq!(bl.single_block_lookups.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_block_lookup_empty_response() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let block_hash = Hash256::random();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_block(block_hash, peer_id, &mut cx);
|
||||
let id = rig.expect_block_request();
|
||||
|
||||
// The peer does not have the block. It should be penalized.
|
||||
bl.single_block_lookup_response(id, peer_id, None, D, &mut cx);
|
||||
rig.expect_penalty();
|
||||
|
||||
rig.expect_block_request(); // it should be retried
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_block_lookup_wrong_response() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let block_hash = Hash256::random();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_block(block_hash, peer_id, &mut cx);
|
||||
let id = rig.expect_block_request();
|
||||
|
||||
// Peer sends something else. It should be penalized.
|
||||
let bad_block = rig.rand_block();
|
||||
bl.single_block_lookup_response(id, peer_id, Some(Box::new(bad_block)), D, &mut cx);
|
||||
rig.expect_penalty();
|
||||
rig.expect_block_request(); // should be retried
|
||||
|
||||
// Send the stream termination. This should not produce an additional penalty.
|
||||
bl.single_block_lookup_response(id, peer_id, None, D, &mut cx);
|
||||
rig.expect_empty_network();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_block_lookup_failure() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let block_hash = Hash256::random();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_block(block_hash, peer_id, &mut cx);
|
||||
let id = rig.expect_block_request();
|
||||
|
||||
// The request fails. RPC failures are handled elsewhere so we should not penalize the peer.
|
||||
bl.single_block_lookup_failed(id, &mut cx);
|
||||
rig.expect_block_request();
|
||||
rig.expect_empty_network();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_single_block_lookup_becomes_parent_request() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let block = rig.rand_block();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_block(block.canonical_root(), peer_id, &mut cx);
|
||||
let id = rig.expect_block_request();
|
||||
|
||||
// The peer provides the correct block, should not be penalized. Now the block should be sent
|
||||
// for processing.
|
||||
bl.single_block_lookup_response(id, peer_id, Some(Box::new(block.clone())), D, &mut cx);
|
||||
rig.expect_empty_network();
|
||||
rig.expect_block_process();
|
||||
|
||||
// The request should still be active.
|
||||
assert_eq!(bl.single_block_lookups.len(), 1);
|
||||
|
||||
// Send the stream termination. Peer should have not been penalized, and the request moved to a
|
||||
// parent request after processing.
|
||||
bl.single_block_processed(id, Err(BlockError::ParentUnknown(Box::new(block))), &mut cx);
|
||||
assert_eq!(bl.single_block_lookups.len(), 0);
|
||||
rig.expect_parent_request();
|
||||
rig.expect_empty_network();
|
||||
assert_eq!(bl.parent_queue.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_happy_path() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let parent = rig.rand_block();
|
||||
let block = rig.block_with_parent(parent.canonical_root());
|
||||
let chain_hash = block.canonical_root();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_parent(Box::new(block), peer_id, &mut cx);
|
||||
let id = rig.expect_parent_request();
|
||||
|
||||
// Peer sends the right block, it should be sent for processing. Peer should not be penalized.
|
||||
bl.parent_lookup_response(id, peer_id, Some(Box::new(parent)), D, &mut cx);
|
||||
rig.expect_block_process();
|
||||
rig.expect_empty_network();
|
||||
|
||||
// Processing succeeds, now the rest of the chain should be sent for processing.
|
||||
bl.parent_block_processed(chain_hash, Err(BlockError::BlockIsAlreadyKnown), &mut cx);
|
||||
rig.expect_parent_chain_process();
|
||||
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
|
||||
assert_eq!(bl.parent_queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_wrong_response() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let parent = rig.rand_block();
|
||||
let block = rig.block_with_parent(parent.canonical_root());
|
||||
let chain_hash = block.canonical_root();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_parent(Box::new(block), peer_id, &mut cx);
|
||||
let id1 = rig.expect_parent_request();
|
||||
|
||||
// Peer sends the wrong block, peer should be penalized and the block re-requested.
|
||||
let bad_block = rig.rand_block();
|
||||
bl.parent_lookup_response(id1, peer_id, Some(Box::new(bad_block)), D, &mut cx);
|
||||
rig.expect_penalty();
|
||||
let id2 = rig.expect_parent_request();
|
||||
|
||||
// Send the stream termination for the first request. This should not produce extra penalties.
|
||||
bl.parent_lookup_response(id1, peer_id, None, D, &mut cx);
|
||||
rig.expect_empty_network();
|
||||
|
||||
// Send the right block this time.
|
||||
bl.parent_lookup_response(id2, peer_id, Some(Box::new(parent)), D, &mut cx);
|
||||
rig.expect_block_process();
|
||||
|
||||
// Processing succeeds, now the rest of the chain should be sent for processing.
|
||||
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
|
||||
rig.expect_parent_chain_process();
|
||||
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
|
||||
assert_eq!(bl.parent_queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_empty_response() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let parent = rig.rand_block();
|
||||
let block = rig.block_with_parent(parent.canonical_root());
|
||||
let chain_hash = block.canonical_root();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_parent(Box::new(block), peer_id, &mut cx);
|
||||
let id1 = rig.expect_parent_request();
|
||||
|
||||
// Peer sends an empty response, peer should be penalized and the block re-requested.
|
||||
bl.parent_lookup_response(id1, peer_id, None, D, &mut cx);
|
||||
rig.expect_penalty();
|
||||
let id2 = rig.expect_parent_request();
|
||||
|
||||
// Send the right block this time.
|
||||
bl.parent_lookup_response(id2, peer_id, Some(Box::new(parent)), D, &mut cx);
|
||||
rig.expect_block_process();
|
||||
|
||||
// Processing succeeds, now the rest of the chain should be sent for processing.
|
||||
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
|
||||
rig.expect_parent_chain_process();
|
||||
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
|
||||
assert_eq!(bl.parent_queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_rpc_failure() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let parent = rig.rand_block();
|
||||
let block = rig.block_with_parent(parent.canonical_root());
|
||||
let chain_hash = block.canonical_root();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_parent(Box::new(block), peer_id, &mut cx);
|
||||
let id1 = rig.expect_parent_request();
|
||||
|
||||
// The request fails. It should be tried again.
|
||||
bl.parent_lookup_failed(id1, peer_id, &mut cx);
|
||||
let id2 = rig.expect_parent_request();
|
||||
|
||||
// Send the right block this time.
|
||||
bl.parent_lookup_response(id2, peer_id, Some(Box::new(parent)), D, &mut cx);
|
||||
rig.expect_block_process();
|
||||
|
||||
// Processing succeeds, now the rest of the chain should be sent for processing.
|
||||
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
|
||||
rig.expect_parent_chain_process();
|
||||
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
|
||||
assert_eq!(bl.parent_queue.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_too_many_attempts() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
|
||||
let parent = rig.rand_block();
|
||||
let block = rig.block_with_parent(parent.canonical_root());
|
||||
let chain_hash = block.canonical_root();
|
||||
let peer_id = PeerId::random();
|
||||
|
||||
// Trigger the request
|
||||
bl.search_parent(Box::new(block), peer_id, &mut cx);
|
||||
for i in 1..=parent_lookup::PARENT_FAIL_TOLERANCE + 1 {
|
||||
let id = rig.expect_parent_request();
|
||||
match i % 2 {
|
||||
// make sure every error is accounted for
|
||||
0 => {
|
||||
// The request fails. It should be tried again.
|
||||
bl.parent_lookup_failed(id, peer_id, &mut cx);
|
||||
}
|
||||
_ => {
|
||||
// Send a bad block this time. It should be tried again.
|
||||
let bad_block = rig.rand_block();
|
||||
bl.parent_lookup_response(id, peer_id, Some(Box::new(bad_block)), D, &mut cx);
|
||||
rig.expect_penalty();
|
||||
}
|
||||
}
|
||||
if i < parent_lookup::PARENT_FAIL_TOLERANCE {
|
||||
assert_eq!(bl.parent_queue[0].failed_attempts(), dbg!(i));
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(bl.parent_queue.len(), 0);
|
||||
assert!(bl.failed_chains.contains(&chain_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_too_deep() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
let mut blocks =
|
||||
Vec::<SignedBeaconBlock<E>>::with_capacity(parent_lookup::PARENT_DEPTH_TOLERANCE);
|
||||
while blocks.len() < parent_lookup::PARENT_DEPTH_TOLERANCE {
|
||||
let parent = blocks
|
||||
.last()
|
||||
.map(|b| b.canonical_root())
|
||||
.unwrap_or_else(Hash256::random);
|
||||
let block = rig.block_with_parent(parent);
|
||||
blocks.push(block);
|
||||
}
|
||||
|
||||
let peer_id = PeerId::random();
|
||||
let trigger_block = blocks.pop().unwrap();
|
||||
let chain_hash = trigger_block.canonical_root();
|
||||
bl.search_parent(Box::new(trigger_block), peer_id, &mut cx);
|
||||
|
||||
for block in blocks.into_iter().rev() {
|
||||
let id = rig.expect_parent_request();
|
||||
// the block
|
||||
bl.parent_lookup_response(id, peer_id, Some(Box::new(block.clone())), D, &mut cx);
|
||||
// the stream termination
|
||||
bl.parent_lookup_response(id, peer_id, None, D, &mut cx);
|
||||
// the processing request
|
||||
rig.expect_block_process();
|
||||
// the processing result
|
||||
bl.parent_block_processed(
|
||||
chain_hash,
|
||||
Err(BlockError::ParentUnknown(Box::new(block))),
|
||||
&mut cx,
|
||||
)
|
||||
}
|
||||
|
||||
rig.expect_penalty();
|
||||
assert!(bl.failed_chains.contains(&chain_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parent_lookup_disconnection() {
|
||||
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
|
||||
let peer_id = PeerId::random();
|
||||
let trigger_block = rig.rand_block();
|
||||
bl.search_parent(Box::new(trigger_block), peer_id, &mut cx);
|
||||
bl.peer_disconnected(&peer_id, &mut cx);
|
||||
assert!(bl.parent_queue.is_empty());
|
||||
}
|
||||
Reference in New Issue
Block a user