gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 18:32:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ignored sync jobs 2 (#3317)

Browse Source 
## Issue Addressed

Duplicate of #3269. Making this since @divagant-martian opened the previous PR and she can't approve her own PR 😄 


Co-authored-by: Diva M <divma@protonmail.com>
This commit is contained in:
Pawan Dhananjay
2022-07-15 07:31:20 +00:00
parent 98a9626ef5
commit 28b0ff27ff
8 changed files with 396 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
beacon_node/network/src/sync/block_lookups/mod.rs
View File

@@ -19,6 +19,7 @@ use self::{
single_block_lookup::SingleBlockRequest,
};
use super::manager::BlockProcessResult;
use super::BatchProcessResult;
use super::{
manager::{BlockProcessType, Id},
@@ -247,7 +248,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
| VerifyError::ExtraBlocksReturned => {
let e = e.into();
warn!(self.log, "Peer sent invalid response to parent request.";
"peer_id" => %peer_id, "reason" => e);
"peer_id" => %peer_id, "reason" => %e);
// We do not tolerate these kinds of errors. We will accept a few but these are signs
// of a faulty peer.
@@ -381,7 +382,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
pub fn single_block_processed(
&mut self,
id: Id,
result: Result<(), BlockError<T::EthSpec>>,
result: BlockProcessResult<T::EthSpec>,
cx: &mut SyncNetworkContext<T::EthSpec>,
) {
let mut req = match self.single_block_lookups.remove(&id) {
@@ -403,52 +404,62 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
Err(_) => return,
};
if let Err(e) = &result {
trace!(self.log, "Single block processing failed"; "block" => %root, "error" => %e);
} else {
trace!(self.log, "Single block processing succeeded"; "block" => %root);
}
if let Err(e) = result {
match e {
BlockError::BlockIsAlreadyKnown => {
// No error here
}
BlockError::BeaconChainError(e) => {
// Internal error
error!(self.log, "Beacon chain error processing single block"; "block_root" => %root, "error" => ?e);
}
BlockError::ParentUnknown(block) => {
self.search_parent(block, peer_id, cx);
}
e @ BlockError::ExecutionPayloadError(ExecutionPayloadError::RequestFailed(_))
| e @ BlockError::ExecutionPayloadError(
ExecutionPayloadError::NoExecutionConnection,
) => {
// These errors indicate that the execution layer is offline
// and failed to validate the execution payload. Do not downscore peer.
debug!(
self.log,
"Single block lookup failed. Execution layer is offline";
"root" => %root,
"error" => ?e
);
}
other => {
warn!(self.log, "Peer sent invalid block in single block lookup"; "root" => %root, "error" => ?other, "peer_id" => %peer_id);
cx.report_peer(
peer_id,
PeerAction::MidToleranceError,
"single_block_failure",
);
// Try it again if possible.
req.register_failure();
if let Ok((peer_id, request)) = req.request_block() {
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, request) {
// insert with the new id
self.single_block_lookups.insert(request_id, req);
match result {
BlockProcessResult::Ok => {
trace!(self.log, "Single block processing succeeded"; "block" => %root);
}
BlockProcessResult::Ignored => {
// Beacon processor signalled to ignore the block processing result.
// This implies that the cpu is overloaded. Drop the request.
warn!(
self.log,
"Single block processing was ignored, cpu might be overloaded";
"action" => "dropping single block request"
);
}
BlockProcessResult::Err(e) => {
trace!(self.log, "Single block processing failed"; "block" => %root, "error" => %e);
match e {
BlockError::BlockIsAlreadyKnown => {
// No error here
}
BlockError::BeaconChainError(e) => {
// Internal error
error!(self.log, "Beacon chain error processing single block"; "block_root" => %root, "error" => ?e);
}
BlockError::ParentUnknown(block) => {
self.search_parent(block, peer_id, cx);
}
e @ BlockError::ExecutionPayloadError(ExecutionPayloadError::RequestFailed(
_,
))
| e @ BlockError::ExecutionPayloadError(
ExecutionPayloadError::NoExecutionConnection,
) => {
// These errors indicate that the execution layer is offline
// and failed to validate the execution payload. Do not downscore peer.
debug!(
self.log,
"Single block lookup failed. Execution layer is offline";
"root" => %root,
"error" => ?e
);
}
other => {
warn!(self.log, "Peer sent invalid block in single block lookup"; "root" => %root, "error" => ?other, "peer_id" => %peer_id);
cx.report_peer(
peer_id,
PeerAction::MidToleranceError,
"single_block_failure",
);
// Try it again if possible.
req.register_failure();
if let Ok((peer_id, request)) = req.request_block() {
if let Ok(request_id) = cx.single_block_lookup_request(peer_id, request)
{
// insert with the new id
self.single_block_lookups.insert(request_id, req);
}
}
}
}
@@ -464,7 +475,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
pub fn parent_block_processed(
&mut self,
chain_hash: Hash256,
result: Result<(), BlockError<T::EthSpec>>,
result: BlockProcessResult<T::EthSpec>,
cx: &mut SyncNetworkContext<T::EthSpec>,
) {
let (mut parent_lookup, peer_id) = if let Some((pos, peer)) = self
@@ -487,20 +498,32 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
return crit!(self.log, "Process response for a parent lookup request that was not found"; "chain_hash" => %chain_hash);
};
if let Err(e) = &result {
trace!(self.log, "Parent block processing failed"; &parent_lookup, "error" => %e);
} else {
trace!(self.log, "Parent block processing succeeded"; &parent_lookup);
match &result {
BlockProcessResult::Ok => {
trace!(self.log, "Parent block processing succeeded"; &parent_lookup)
}
BlockProcessResult::Err(e) => {
trace!(self.log, "Parent block processing failed"; &parent_lookup, "error" => %e)
}
BlockProcessResult::Ignored => {
trace!(
self.log,
"Parent block processing job was ignored";
"action" => "re-requesting block",
&parent_lookup
);
}
}
match result {
Err(BlockError::ParentUnknown(block)) => {
BlockProcessResult::Err(BlockError::ParentUnknown(block)) => {
// need to keep looking for parents
// add the block back to the queue and continue the search
parent_lookup.add_block(block);
self.request_parent(parent_lookup, cx);
}
Ok(_) | Err(BlockError::BlockIsAlreadyKnown { .. }) => {
BlockProcessResult::Ok
| BlockProcessResult::Err(BlockError::BlockIsAlreadyKnown { .. }) => {
let chain_hash = parent_lookup.chain_hash();
let blocks = parent_lookup.chain_blocks();
let process_id = ChainSegmentProcessId::ParentLookup(chain_hash);
@@ -521,8 +544,10 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
}
}
}
Err(e @ BlockError::ExecutionPayloadError(ExecutionPayloadError::RequestFailed(_)))
| Err(
BlockProcessResult::Err(
e @ BlockError::ExecutionPayloadError(ExecutionPayloadError::RequestFailed(_)),
)
| BlockProcessResult::Err(
e @ BlockError::ExecutionPayloadError(ExecutionPayloadError::NoExecutionConnection),
) => {
// These errors indicate that the execution layer is offline
@@ -534,7 +559,7 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
"error" => ?e
);
}
Err(outcome) => {
BlockProcessResult::Err(outcome) => {
// all else we consider the chain a failure and downvote the peer that sent
// us the last block
warn!(
@@ -551,6 +576,15 @@ impl<T: BeaconChainTypes> BlockLookups<T> {
// ambiguity.
cx.report_peer(peer_id, PeerAction::MidToleranceError, "parent_request_err");
}
BlockProcessResult::Ignored => {
// Beacon processor signalled to ignore the block processing result.
// This implies that the cpu is overloaded. Drop the request.
warn!(
self.log,
"Parent block processing was ignored, cpu might be overloaded";
"action" => "dropping parent request"
);
}
}
metrics::set_gauge(

71
beacon_node/network/src/sync/block_lookups/tests.rs
View File

@@ -168,7 +168,7 @@ fn test_single_block_lookup_happy_path() {
// Send the stream termination. Peer should have not been penalized, and the request removed
// after processing.
bl.single_block_lookup_response(id, peer_id, None, D, &mut cx);
bl.single_block_processed(id, Ok(()), &mut cx);
bl.single_block_processed(id, Ok(()).into(), &mut cx);
rig.expect_empty_network();
assert_eq!(bl.single_block_lookups.len(), 0);
}
@@ -252,7 +252,11 @@ fn test_single_block_lookup_becomes_parent_request() {
// Send the stream termination. Peer should have not been penalized, and the request moved to a
// parent request after processing.
bl.single_block_processed(id, Err(BlockError::ParentUnknown(Arc::new(block))), &mut cx);
bl.single_block_processed(
id,
BlockError::ParentUnknown(Arc::new(block)).into(),
&mut cx,
);
assert_eq!(bl.single_block_lookups.len(), 0);
rig.expect_parent_request();
rig.expect_empty_network();
@@ -278,7 +282,7 @@ fn test_parent_lookup_happy_path() {
rig.expect_empty_network();
// Processing succeeds, now the rest of the chain should be sent for processing.
bl.parent_block_processed(chain_hash, Err(BlockError::BlockIsAlreadyKnown), &mut cx);
bl.parent_block_processed(chain_hash, BlockError::BlockIsAlreadyKnown.into(), &mut cx);
rig.expect_parent_chain_process();
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
assert_eq!(bl.parent_queue.len(), 0);
@@ -312,7 +316,7 @@ fn test_parent_lookup_wrong_response() {
rig.expect_block_process();
// Processing succeeds, now the rest of the chain should be sent for processing.
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
bl.parent_block_processed(chain_hash, Ok(()).into(), &mut cx);
rig.expect_parent_chain_process();
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
assert_eq!(bl.parent_queue.len(), 0);
@@ -341,7 +345,7 @@ fn test_parent_lookup_empty_response() {
rig.expect_block_process();
// Processing succeeds, now the rest of the chain should be sent for processing.
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
bl.parent_block_processed(chain_hash, Ok(()).into(), &mut cx);
rig.expect_parent_chain_process();
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
assert_eq!(bl.parent_queue.len(), 0);
@@ -369,7 +373,7 @@ fn test_parent_lookup_rpc_failure() {
rig.expect_block_process();
// Processing succeeds, now the rest of the chain should be sent for processing.
bl.parent_block_processed(chain_hash, Ok(()), &mut cx);
bl.parent_block_processed(chain_hash, Ok(()).into(), &mut cx);
rig.expect_parent_chain_process();
bl.parent_chain_processed(chain_hash, BatchProcessResult::Success(true), &mut cx);
assert_eq!(bl.parent_queue.len(), 0);
@@ -440,7 +444,7 @@ fn test_parent_lookup_too_deep() {
// the processing result
bl.parent_block_processed(
chain_hash,
Err(BlockError::ParentUnknown(Arc::new(block))),
BlockError::ParentUnknown(Arc::new(block)).into(),
&mut cx,
)
}
@@ -458,3 +462,56 @@ fn test_parent_lookup_disconnection() {
bl.peer_disconnected(&peer_id, &mut cx);
assert!(bl.parent_queue.is_empty());
}
#[test]
fn test_single_block_lookup_ignored_response() {
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
let block = rig.rand_block();
let peer_id = PeerId::random();
// Trigger the request
bl.search_block(block.canonical_root(), peer_id, &mut cx);
let id = rig.expect_block_request();
// The peer provides the correct block, should not be penalized. Now the block should be sent
// for processing.
bl.single_block_lookup_response(id, peer_id, Some(Arc::new(block)), D, &mut cx);
rig.expect_empty_network();
rig.expect_block_process();
// The request should still be active.
assert_eq!(bl.single_block_lookups.len(), 1);
// Send the stream termination. Peer should have not been penalized, and the request removed
// after processing.
bl.single_block_lookup_response(id, peer_id, None, D, &mut cx);
// Send an Ignored response, the request should be dropped
bl.single_block_processed(id, BlockProcessResult::Ignored, &mut cx);
rig.expect_empty_network();
assert_eq!(bl.single_block_lookups.len(), 0);
}
#[test]
fn test_parent_lookup_ignored_response() {
let (mut bl, mut cx, mut rig) = TestRig::test_setup(None);
let parent = rig.rand_block();
let block = rig.block_with_parent(parent.canonical_root());
let chain_hash = block.canonical_root();
let peer_id = PeerId::random();
// Trigger the request
bl.search_parent(Arc::new(block), peer_id, &mut cx);
let id = rig.expect_parent_request();
// Peer sends the right block, it should be sent for processing. Peer should not be penalized.
bl.parent_lookup_response(id, peer_id, Some(Arc::new(parent)), D, &mut cx);
rig.expect_block_process();
rig.expect_empty_network();
// Return an Ignored result. The request should be dropped
bl.parent_block_processed(chain_hash, BlockProcessResult::Ignored, &mut cx);
rig.expect_empty_network();
assert_eq!(bl.parent_queue.len(), 0);
}