Merge remote-tracking branch 'origin/unstable' into tree-states

This commit is contained in:
Michael Sproul
2022-05-24 10:01:05 +10:00
237 changed files with 8506 additions and 3598 deletions

View File

@@ -29,7 +29,7 @@ error-chain = "0.12.4"
tokio = { version = "1.14.0", features = ["full"] }
tokio-stream = "0.1.3"
smallvec = "1.6.1"
rand = "0.7.3"
rand = "0.8.5"
fnv = "1.0.7"
rlp = "0.5.0"
lazy_static = "1.4.0"
@@ -41,5 +41,6 @@ itertools = "0.10.0"
num_cpus = "1.13.0"
lru_cache = { path = "../../common/lru_cache" }
if-addrs = "0.6.4"
strum = "0.21.0"
strum = "0.24.0"
tokio-util = { version = "0.6.3", features = ["time"] }
derivative = "2.2.0"

View File

@@ -42,6 +42,7 @@ use crate::sync::manager::BlockProcessType;
use crate::{metrics, service::NetworkMessage, sync::SyncMessage};
use beacon_chain::parking_lot::Mutex;
use beacon_chain::{BeaconChain, BeaconChainTypes, GossipVerifiedBlock};
use derivative::Derivative;
use futures::stream::{Stream, StreamExt};
use futures::task::Poll;
use lighthouse_network::{
@@ -51,7 +52,6 @@ use lighthouse_network::{
use logging::TimeLatch;
use slog::{crit, debug, error, trace, warn, Logger};
use std::collections::VecDeque;
use std::fmt;
use std::pin::Pin;
use std::sync::{Arc, Weak};
use std::task::Context;
@@ -331,17 +331,13 @@ impl DuplicateCache {
}
/// An event to be processed by the manager task.
#[derive(Derivative)]
#[derivative(Debug(bound = "T: BeaconChainTypes"))]
pub struct WorkEvent<T: BeaconChainTypes> {
drop_during_sync: bool,
work: Work<T>,
}
impl<T: BeaconChainTypes> fmt::Debug for WorkEvent<T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
impl<T: BeaconChainTypes> WorkEvent<T> {
/// Create a new `Work` event for some unaggregated attestation.
pub fn unaggregated_attestation(
@@ -615,7 +611,8 @@ impl<T: BeaconChainTypes> std::convert::From<ReadyWork<T>> for WorkEvent<T> {
}
/// A consensus message (or multiple) from the network that requires processing.
#[derive(Debug)]
#[derive(Derivative)]
#[derivative(Debug(bound = "T: BeaconChainTypes"))]
pub enum Work<T: BeaconChainTypes> {
GossipAttestation {
message_id: MessageId,
@@ -1344,6 +1341,7 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
"worker" => worker_id,
);
let sub_executor = executor.clone();
executor.spawn_blocking(
move || {
let _worker_timer = worker_timer;
@@ -1520,7 +1518,15 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
peer_id,
request_id,
request,
} => worker.handle_blocks_by_range_request(peer_id, request_id, request),
} => {
return worker.handle_blocks_by_range_request(
sub_executor,
send_idle_on_drop,
peer_id,
request_id,
request,
)
}
/*
* Processing of blocks by roots requests from other peers.
*/
@@ -1528,7 +1534,15 @@ impl<T: BeaconChainTypes> BeaconProcessor<T> {
peer_id,
request_id,
request,
} => worker.handle_blocks_by_root_request(peer_id, request_id, request),
} => {
return worker.handle_blocks_by_root_request(
sub_executor,
send_idle_on_drop,
peer_id,
request_id,
request,
)
}
Work::UnknownBlockAttestation {
message_id,
peer_id,

View File

@@ -20,7 +20,7 @@ use std::cmp;
use std::iter::Iterator;
use std::sync::Arc;
use std::time::Duration;
use tokio::runtime::Runtime;
use tokio::runtime::Handle;
use tokio::sync::mpsc;
use types::{
Attestation, AttesterSlashing, EthSpec, MainnetEthSpec, ProposerSlashing, SignedBeaconBlock,
@@ -324,20 +324,19 @@ impl TestRig {
.unwrap();
}
fn runtime(&mut self) -> Arc<Runtime> {
fn handle(&mut self) -> Handle {
self.environment
.as_mut()
.unwrap()
.core_context()
.executor
.runtime()
.upgrade()
.handle()
.unwrap()
}
/// Assert that the `BeaconProcessor` doesn't produce any events in the given `duration`.
pub fn assert_no_events_for(&mut self, duration: Duration) {
self.runtime().block_on(async {
self.handle().block_on(async {
tokio::select! {
_ = tokio::time::sleep(duration) => (),
event = self.work_journal_rx.recv() => panic!(
@@ -360,7 +359,7 @@ impl TestRig {
.iter()
.all(|ev| ev != &WORKER_FREED && ev != &NOTHING_TO_DO));
let (events, worker_freed_remaining) = self.runtime().block_on(async {
let (events, worker_freed_remaining) = self.handle().block_on(async {
let mut events = Vec::with_capacity(expected.len());
let mut worker_freed_remaining = expected.len();
@@ -415,7 +414,7 @@ impl TestRig {
/// We won't attempt to listen for any more than `expected.len()` events. As such, it makes sense
/// to use the `NOTHING_TO_DO` event to ensure that execution has completed.
pub fn assert_event_journal_with_timeout(&mut self, expected: &[&str], timeout: Duration) {
let events = self.runtime().block_on(async {
let events = self.handle().block_on(async {
let mut events = Vec::with_capacity(expected.len());
let drain_future = async {

View File

@@ -1,4 +1,4 @@
use crate::beacon_processor::worker::FUTURE_SLOT_TOLERANCE;
use crate::beacon_processor::{worker::FUTURE_SLOT_TOLERANCE, SendOnDrop};
use crate::service::NetworkMessage;
use crate::status::ToStatusMessage;
use crate::sync::SyncMessage;
@@ -9,6 +9,7 @@ use lighthouse_network::rpc::*;
use lighthouse_network::{PeerId, PeerRequestId, ReportSource, Response, SyncInfo};
use slog::{debug, error, warn};
use slot_clock::SlotClock;
use task_executor::TaskExecutor;
use types::{Epoch, EthSpec, Hash256, Slot};
use super::Worker;
@@ -122,38 +123,71 @@ impl<T: BeaconChainTypes> Worker<T> {
/// Handle a `BlocksByRoot` request from the peer.
pub fn handle_blocks_by_root_request(
&self,
self,
executor: TaskExecutor,
send_on_drop: SendOnDrop,
peer_id: PeerId,
request_id: PeerRequestId,
request: BlocksByRootRequest,
) {
let mut send_block_count = 0;
for root in request.block_roots.iter() {
if let Ok(Some(block)) = self.chain.get_block_checking_early_attester_cache(root) {
self.send_response(
peer_id,
Response::BlocksByRoot(Some(Box::new(block))),
request_id,
);
send_block_count += 1;
} else {
debug!(self.log, "Peer requested unknown block";
// Fetching blocks is async because it may have to hit the execution layer for payloads.
executor.spawn(
async move {
let mut send_block_count = 0;
for root in request.block_roots.iter() {
match self
.chain
.get_block_checking_early_attester_cache(root)
.await
{
Ok(Some(block)) => {
self.send_response(
peer_id,
Response::BlocksByRoot(Some(Box::new(block))),
request_id,
);
send_block_count += 1;
}
Ok(None) => {
debug!(
self.log,
"Peer requested unknown block";
"peer" => %peer_id,
"request_root" => ?root
);
}
Err(e) => {
debug!(
self.log,
"Error fetching block for peer";
"peer" => %peer_id,
"request_root" => ?root,
"error" => ?e,
);
}
}
}
debug!(
self.log,
"Received BlocksByRoot Request";
"peer" => %peer_id,
"request_root" => ?root);
}
}
debug!(self.log, "Received BlocksByRoot Request";
"peer" => %peer_id,
"requested" => request.block_roots.len(),
"returned" => send_block_count);
"requested" => request.block_roots.len(),
"returned" => send_block_count
);
// send stream termination
self.send_response(peer_id, Response::BlocksByRoot(None), request_id);
// send stream termination
self.send_response(peer_id, Response::BlocksByRoot(None), request_id);
drop(send_on_drop);
},
"load_blocks_by_root_blocks",
)
}
/// Handle a `BlocksByRange` request from the peer.
pub fn handle_blocks_by_range_request(
&self,
self,
executor: TaskExecutor,
send_on_drop: SendOnDrop,
peer_id: PeerId,
request_id: PeerRequestId,
mut req: BlocksByRangeRequest,
@@ -228,54 +262,84 @@ impl<T: BeaconChainTypes> Worker<T> {
// remove all skip slots
let block_roots = block_roots.into_iter().flatten().collect::<Vec<_>>();
let mut blocks_sent = 0;
for root in block_roots {
if let Ok(Some(block)) = self.chain.store.get_block(&root) {
// Due to skip slots, blocks could be out of the range, we ensure they are in the
// range before sending
if block.slot() >= req.start_slot
&& block.slot() < req.start_slot + req.count * req.step
{
blocks_sent += 1;
self.send_network_message(NetworkMessage::SendResponse {
peer_id,
response: Response::BlocksByRange(Some(Box::new(block))),
id: request_id,
});
// Fetching blocks is async because it may have to hit the execution layer for payloads.
executor.spawn(
async move {
let mut blocks_sent = 0;
for root in block_roots {
match self.chain.get_block(&root).await {
Ok(Some(block)) => {
// Due to skip slots, blocks could be out of the range, we ensure they
// are in the range before sending
if block.slot() >= req.start_slot
&& block.slot() < req.start_slot + req.count * req.step
{
blocks_sent += 1;
self.send_network_message(NetworkMessage::SendResponse {
peer_id,
response: Response::BlocksByRange(Some(Box::new(block))),
id: request_id,
});
}
}
Ok(None) => {
error!(
self.log,
"Block in the chain is not in the store";
"request_root" => ?root
);
break;
}
Err(e) => {
error!(
self.log,
"Error fetching block for peer";
"block_root" => ?root,
"error" => ?e
);
break;
}
}
}
} else {
error!(self.log, "Block in the chain is not in the store";
"request_root" => ?root);
}
}
let current_slot = self
.chain
.slot()
.unwrap_or_else(|_| self.chain.slot_clock.genesis_slot());
let current_slot = self
.chain
.slot()
.unwrap_or_else(|_| self.chain.slot_clock.genesis_slot());
if blocks_sent < (req.count as usize) {
debug!(self.log, "BlocksByRange Response processed";
"peer" => %peer_id,
"msg" => "Failed to return all requested blocks",
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blocks_sent);
} else {
debug!(self.log, "BlocksByRange Response processed";
"peer" => %peer_id,
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blocks_sent);
}
if blocks_sent < (req.count as usize) {
debug!(
self.log,
"BlocksByRange Response processed";
"peer" => %peer_id,
"msg" => "Failed to return all requested blocks",
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blocks_sent
);
} else {
debug!(
self.log,
"BlocksByRange Response processed";
"peer" => %peer_id,
"start_slot" => req.start_slot,
"current_slot" => current_slot,
"requested" => req.count,
"returned" => blocks_sent
);
}
// send the stream terminator
self.send_network_message(NetworkMessage::SendResponse {
peer_id,
response: Response::BlocksByRange(None),
id: request_id,
});
// send the stream terminator
self.send_network_message(NetworkMessage::SendResponse {
peer_id,
response: Response::BlocksByRange(None),
id: request_id,
});
drop(send_on_drop);
},
"load_blocks_by_range_blocks",
);
}
}

View File

@@ -138,7 +138,7 @@ impl<T: BeaconChainTypes> Worker<T> {
let end_slot = downloaded_blocks.last().map(|b| b.slot().as_u64());
let sent_blocks = downloaded_blocks.len();
match self.process_backfill_blocks(&downloaded_blocks) {
match self.process_backfill_blocks(downloaded_blocks) {
(_, Ok(_)) => {
debug!(self.log, "Backfill batch processed";
"batch_epoch" => epoch,
@@ -223,9 +223,10 @@ impl<T: BeaconChainTypes> Worker<T> {
/// Helper function to process backfill block batches which only consumes the chain and blocks to process.
fn process_backfill_blocks(
&self,
blocks: &[SignedBeaconBlock<T::EthSpec>],
blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
) -> (usize, Result<(), ChainSegmentFailed>) {
match self.chain.import_historical_block_batch(blocks) {
let blinded_blocks = blocks.into_iter().map(Into::into).collect();
match self.chain.import_historical_block_batch(blinded_blocks) {
Ok(imported_blocks) => {
metrics::inc_counter(
&metrics::BEACON_PROCESSOR_BACKFILL_CHAIN_SEGMENT_SUCCESS_TOTAL,

View File

@@ -9,7 +9,6 @@ use lighthouse_network::{
Gossipsub, NetworkGlobals,
};
use std::sync::Arc;
use strum::AsStaticRef;
use strum::IntoEnumIterator;
use types::EthSpec;
@@ -357,12 +356,12 @@ pub fn update_gossip_metrics<T: EthSpec>(
for client_kind in ClientKind::iter() {
set_gauge_vec(
&BEACON_BLOCK_MESH_PEERS_PER_CLIENT,
&[&client_kind.to_string()],
&[client_kind.as_ref()],
0_i64,
);
set_gauge_vec(
&BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,
&[&client_kind.to_string()],
&[client_kind.as_ref()],
0_i64,
);
}
@@ -377,7 +376,7 @@ pub fn update_gossip_metrics<T: EthSpec>(
.peers
.read()
.peer_info(peer_id)
.map(|peer_info| peer_info.client().kind.as_static())
.map(|peer_info| peer_info.client().kind.into())
.unwrap_or_else(|| "Unknown");
if let Some(v) =
get_int_gauge(&BEACON_BLOCK_MESH_PEERS_PER_CLIENT, &[client])
@@ -392,7 +391,7 @@ pub fn update_gossip_metrics<T: EthSpec>(
.peers
.read()
.peer_info(peer_id)
.map(|peer_info| peer_info.client().kind.as_static())
.map(|peer_info| peer_info.client().kind.into())
.unwrap_or_else(|| "Unknown");
if let Some(v) = get_int_gauge(
&BEACON_AGGREGATE_AND_PROOF_MESH_PEERS_PER_CLIENT,

View File

@@ -4,11 +4,10 @@ use std::time::Duration;
use beacon_chain::{BeaconChainTypes, BlockError};
use fnv::FnvHashMap;
use lighthouse_network::{PeerAction, PeerId};
use lru_cache::LRUCache;
use lru_cache::LRUTimeCache;
use slog::{crit, debug, error, trace, warn, Logger};
use smallvec::SmallVec;
use store::{Hash256, SignedBeaconBlock};
use strum::AsStaticRef;
use tokio::sync::mpsc;
use crate::beacon_processor::{ChainSegmentProcessId, WorkEvent};
@@ -30,7 +29,7 @@ mod single_block_lookup;
#[cfg(test)]
mod tests;
const FAILED_CHAINS_CACHE_SIZE: usize = 500;
const FAILED_CHAINS_CACHE_EXPIRY_SECONDS: u64 = 60;
const SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS: u8 = 3;
pub(crate) struct BlockLookups<T: BeaconChainTypes> {
@@ -38,7 +37,7 @@ pub(crate) struct BlockLookups<T: BeaconChainTypes> {
parent_queue: SmallVec<[ParentLookup<T::EthSpec>; 3]>,
/// A cache of failed chain lookups to prevent duplicate searches.
failed_chains: LRUCache<Hash256>,
failed_chains: LRUTimeCache<Hash256>,
/// A collection of block hashes being searched for and a flag indicating if a result has been
/// received or not.
@@ -57,7 +56,9 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
pub fn new(beacon_processor_send: mpsc::Sender<WorkEvent<T>>, log: Logger) -> Self {
Self {
parent_queue: Default::default(),
failed_chains: LRUCache::new(FAILED_CHAINS_CACHE_SIZE),
failed_chains: LRUTimeCache::new(Duration::from_secs(
FAILED_CHAINS_CACHE_EXPIRY_SECONDS,
)),
single_block_lookups: Default::default(),
beacon_processor_send,
log,
@@ -176,7 +177,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
// request finished correctly, it will be removed after the block is processed.
}
Err(error) => {
let msg: &str = error.as_static();
let msg: &str = error.into();
cx.report_peer(peer_id, PeerAction::LowToleranceError, msg);
// Remove the request, if it can be retried it will be added with a new id.
let mut req = request.remove();
@@ -219,7 +220,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
return;
};
match parent_lookup.verify_block(block, &self.failed_chains) {
match parent_lookup.verify_block(block, &mut self.failed_chains) {
Ok(Some(block)) => {
// Block is correct, send to the beacon processor.
let chain_hash = parent_lookup.chain_hash();
@@ -243,7 +244,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
VerifyError::RootMismatch
| VerifyError::NoBlockReturned
| VerifyError::ExtraBlocksReturned => {
let e = e.as_static();
let e = e.into();
warn!(self.log, "Peer sent invalid response to parent request.";
"peer_id" => %peer_id, "reason" => e);
@@ -310,8 +311,13 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
}
}
Err(e) => {
trace!(self.log, "Single block request failed on peer disconnection";
"block_root" => %req.hash, "peer_id" => %peer_id, "reason" => e.as_static());
trace!(
self.log,
"Single block request failed on peer disconnection";
"block_root" => %req.hash,
"peer_id" => %peer_id,
"reason" => <&str>::from(e),
);
}
}
}
@@ -402,8 +408,8 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
trace!(self.log, "Single block processing succeeded"; "block" => %root);
}
match result {
Err(e) => match e {
if let Err(e) = result {
match e {
BlockError::BlockIsAlreadyKnown => {
// No error here
}
@@ -431,9 +437,6 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
}
}
}
},
Ok(()) => {
// No error here
}
}

View File

@@ -1,6 +1,6 @@
use lighthouse_network::PeerId;
use store::{EthSpec, Hash256, SignedBeaconBlock};
use strum::AsStaticStr;
use strum::IntoStaticStr;
use crate::sync::{
manager::{Id, SLOT_IMPORT_TOLERANCE},
@@ -28,7 +28,7 @@ pub(crate) struct ParentLookup<T: EthSpec> {
current_parent_request_id: Option<Id>,
}
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
pub enum VerifyError {
RootMismatch,
NoBlockReturned,
@@ -117,7 +117,7 @@ impl<T: EthSpec> ParentLookup<T> {
pub fn verify_block(
&mut self,
block: Option<Box<SignedBeaconBlock<T>>>,
failed_chains: &lru_cache::LRUCache<Hash256>,
failed_chains: &mut lru_cache::LRUTimeCache<Hash256>,
) -> Result<Option<Box<SignedBeaconBlock<T>>>, VerifyError> {
let block = self.current_parent_request.verify_block(block)?;

View File

@@ -4,7 +4,7 @@ use lighthouse_network::{rpc::BlocksByRootRequest, PeerId};
use rand::seq::IteratorRandom;
use ssz_types::VariableList;
use store::{EthSpec, Hash256, SignedBeaconBlock};
use strum::AsStaticStr;
use strum::IntoStaticStr;
/// Object representing a single block lookup request.
#[derive(PartialEq, Eq)]
@@ -28,14 +28,14 @@ pub enum State {
Processing { peer_id: PeerId },
}
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
pub enum VerifyError {
RootMismatch,
NoBlockReturned,
ExtraBlocksReturned,
}
#[derive(Debug, PartialEq, Eq, AsStaticStr)]
#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
pub enum LookupRequestError {
TooManyAttempts,
NoPeers,

View File

@@ -26,8 +26,9 @@ const BATCH_BUFFER_SIZE: u8 = 5;
/// A return type for functions that act on a `Chain` which informs the caller whether the chain
/// has been completed and should be removed or to be kept if further processing is
/// required.
#[must_use = "Should be checked, since a failed chain must be removed. A chain that requested
being removed and continued is now in an inconsistent state"]
///
/// Should be checked, since a failed chain must be removed. A chain that requested being removed
/// and continued is now in an inconsistent state.
pub type ProcessingResult = Result<KeepChain, RemoveChain>;
/// Reasons for removing a chain

View File

@@ -49,13 +49,18 @@ use crate::sync::manager::Id;
use crate::sync::network_context::SyncNetworkContext;
use crate::sync::BatchProcessResult;
use beacon_chain::{BeaconChain, BeaconChainTypes};
use lighthouse_network::rpc::GoodbyeReason;
use lighthouse_network::PeerId;
use lighthouse_network::SyncInfo;
use slog::{crit, debug, error, trace};
use lru_cache::LRUTimeCache;
use slog::{crit, debug, error, trace, warn};
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::mpsc;
use types::{Epoch, EthSpec, SignedBeaconBlock, Slot};
use types::{Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot};
/// For how long we store failed finalized chains to prevent retries.
const FAILED_CHAINS_EXPIRY_SECONDS: u64 = 30;
/// The primary object dealing with long range/batch syncing. This contains all the active and
/// non-active chains that need to be processed before the syncing is considered complete. This
@@ -69,6 +74,8 @@ pub struct RangeSync<T: BeaconChainTypes, C = BeaconChain<T>> {
/// A collection of chains that need to be downloaded. This stores any head or finalized chains
/// that need to be downloaded.
chains: ChainCollection<T, C>,
/// Chains that have failed and are stored to prevent being retried.
failed_chains: LRUTimeCache<Hash256>,
/// A multi-threaded, non-blocking processor for applying messages to the beacon chain.
beacon_processor_send: mpsc::Sender<BeaconWorkEvent<T>>,
/// The syncing logger.
@@ -88,6 +95,9 @@ where
RangeSync {
beacon_chain: beacon_chain.clone(),
chains: ChainCollection::new(beacon_chain, log.clone()),
failed_chains: LRUTimeCache::new(std::time::Duration::from_secs(
FAILED_CHAINS_EXPIRY_SECONDS,
)),
awaiting_head_peers: HashMap::new(),
beacon_processor_send,
log,
@@ -128,6 +138,14 @@ where
// determine which kind of sync to perform and set up the chains
match RangeSyncType::new(self.beacon_chain.as_ref(), &local_info, &remote_info) {
RangeSyncType::Finalized => {
// Make sure we have not recently tried this chain
if self.failed_chains.contains(&remote_info.finalized_root) {
debug!(self.log, "Disconnecting peer that belongs to previously failed chain";
"failed_root" => %remote_info.finalized_root, "peer_id" => %peer_id);
network.goodbye_peer(peer_id, GoodbyeReason::IrrelevantNetwork);
return;
}
// Finalized chain search
debug!(self.log, "Finalization sync peer joined"; "peer_id" => %peer_id);
self.awaiting_head_peers.remove(&peer_id);
@@ -338,6 +356,13 @@ where
debug!(self.log, "Chain removed"; "sync_type" => ?sync_type, &chain, "reason" => ?remove_reason, "op" => op);
}
if let RemoveChain::ChainFailed(_) = remove_reason {
if RangeSyncType::Finalized == sync_type {
warn!(self.log, "Chain failed! Syncing to its head won't be retried for at least the next {} seconds", FAILED_CHAINS_EXPIRY_SECONDS; &chain);
self.failed_chains.insert(chain.target_head_root);
}
}
network.status_peers(self.beacon_chain.as_ref(), chain.peers());
let local = match self.beacon_chain.status_message() {