gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Sync lookup dedup range and blobs (#5561)

Browse Source 
* Handle sync range blocks as blocks and blobs

* Merge range sync and backfill sync handling

* Update tests

* Add no_blobs_into_responses test

* Address @realbigsean comments

* Merge remote-tracking branch 'origin/unstable' into sync-lookup-dedup-range-and-blobs
This commit is contained in:
Lion - dapplion
2024-04-13 00:39:11 +09:00
committed by GitHub
parent 5fdd3b39bb
commit 6fb0b2ed78
7 changed files with 254 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files

339
beacon_node/network/src/sync/network_context.rs
View File

@@ -21,14 +21,20 @@ use tokio::sync::mpsc;
use types::{BlobSidecar, EthSpec, SignedBeaconBlock};
pub struct BlocksAndBlobsByRangeResponse<E: EthSpec> {
pub batch_id: BatchId,
pub sender_id: RangeRequestId,
pub responses: Result<Vec<RpcBlock<E>>, String>,
pub request_type: ByRangeRequestType,
}
pub struct BlocksAndBlobsByRangeRequest<E: EthSpec> {
pub chain_id: ChainId,
pub batch_id: BatchId,
pub block_blob_info: BlocksAndBlobsRequestInfo<E>,
#[derive(Debug, Clone, Copy)]
pub enum RangeRequestId {
RangeSync {
chain_id: ChainId,
batch_id: BatchId,
},
BackfillSync {
batch_id: BatchId,
},
}
/// Wraps a Network channel to employ various RPC related network functionality for the Sync manager. This includes management of a global RPC request Id.
@@ -39,18 +45,9 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
/// A sequential ID for all RPC requests.
request_id: Id,
/// BlocksByRange requests made by the range syncing algorithm.
range_requests: FnvHashMap<Id, (ChainId, BatchId)>,
/// BlocksByRange requests made by backfill syncing.
backfill_requests: FnvHashMap<Id, BatchId>,
/// BlocksByRange requests paired with BlobsByRange requests made by the range.
range_blocks_and_blobs_requests: FnvHashMap<Id, BlocksAndBlobsByRangeRequest<T::EthSpec>>,
/// BlocksByRange requests paired with BlobsByRange requests made by the backfill sync.
backfill_blocks_and_blobs_requests:
FnvHashMap<Id, (BatchId, BlocksAndBlobsRequestInfo<T::EthSpec>)>,
/// BlocksByRange requests paired with BlobsByRange
range_blocks_and_blobs_requests:
FnvHashMap<Id, (RangeRequestId, BlocksAndBlobsRequestInfo<T::EthSpec>)>,
/// Whether the ee is online. If it's not, we don't allow access to the
/// `beacon_processor_send`.
@@ -94,10 +91,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
network_send,
execution_engine_state: EngineState::Online, // always assume `Online` at the start
request_id: 1,
range_requests: FnvHashMap::default(),
backfill_requests: FnvHashMap::default(),
range_blocks_and_blobs_requests: FnvHashMap::default(),
backfill_blocks_and_blobs_requests: FnvHashMap::default(),
network_beacon_processor,
chain,
log,
@@ -148,266 +142,85 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
peer_id: PeerId,
batch_type: ByRangeRequestType,
request: BlocksByRangeRequest,
chain_id: ChainId,
batch_id: BatchId,
) -> Result<Id, &'static str> {
match batch_type {
ByRangeRequestType::Blocks => {
trace!(
self.log,
"Sending BlocksByRange request";
"method" => "BlocksByRange",
"count" => request.count(),
"peer" => %peer_id,
);
let request = Request::BlocksByRange(request);
let id = self.next_id();
let request_id = RequestId::Sync(SyncRequestId::RangeBlocks { id });
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request,
request_id,
})?;
self.range_requests.insert(id, (chain_id, batch_id));
Ok(id)
}
ByRangeRequestType::BlocksAndBlobs => {
debug!(
self.log,
"Sending BlocksByRange and BlobsByRange requests";
"method" => "Mixed by range request",
"count" => request.count(),
"peer" => %peer_id,
);
let id = self.next_id();
trace!(
self.log,
"Sending BlocksByRange request";
"method" => "BlocksByRange",
"count" => request.count(),
"peer" => %peer_id,
);
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: Request::BlocksByRange(request.clone()),
request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }),
})?;
// create the shared request id. This is fine since the rpc handles substream ids.
let id = self.next_id();
let request_id = RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id });
if matches!(batch_type, ByRangeRequestType::BlocksAndBlobs) {
debug!(
self.log,
"Sending BlobsByRange requests";
"method" => "BlobsByRange",
"count" => request.count(),
"peer" => %peer_id,
);
// Create the blob request based on the blob request.
let blobs_request = Request::BlobsByRange(BlobsByRangeRequest {
// Create the blob request based on the blocks request.
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: Request::BlobsByRange(BlobsByRangeRequest {
start_slot: *request.start_slot(),
count: *request.count(),
});
let blocks_request = Request::BlocksByRange(request);
// Send both requests. Make sure both can be sent.
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: blocks_request,
request_id,
})?;
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: blobs_request,
request_id,
})?;
let block_blob_info = BlocksAndBlobsRequestInfo::default();
self.range_blocks_and_blobs_requests.insert(
id,
BlocksAndBlobsByRangeRequest {
chain_id,
batch_id,
block_blob_info,
},
);
Ok(id)
}
}),
request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }),
})?;
}
Ok(id)
}
/// A blocks by range request sent by the backfill sync algorithm
pub fn backfill_blocks_by_range_request(
/// A blocks by range request sent by the range sync algorithm
pub fn blocks_and_blobs_by_range_request(
&mut self,
peer_id: PeerId,
batch_type: ByRangeRequestType,
request: BlocksByRangeRequest,
batch_id: BatchId,
sender_id: RangeRequestId,
) -> Result<Id, &'static str> {
match batch_type {
ByRangeRequestType::Blocks => {
trace!(
self.log,
"Sending backfill BlocksByRange request";
"method" => "BlocksByRange",
"count" => request.count(),
"peer" => %peer_id,
);
let request = Request::BlocksByRange(request);
let id = self.next_id();
let request_id = RequestId::Sync(SyncRequestId::BackFillBlocks { id });
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request,
request_id,
})?;
self.backfill_requests.insert(id, batch_id);
Ok(id)
}
ByRangeRequestType::BlocksAndBlobs => {
debug!(
self.log,
"Sending backfill BlocksByRange and BlobsByRange requests";
"method" => "Mixed by range request",
"count" => request.count(),
"peer" => %peer_id,
);
// create the shared request id. This is fine since the rpc handles substream ids.
let id = self.next_id();
let request_id = RequestId::Sync(SyncRequestId::BackFillBlockAndBlobs { id });
// Create the blob request based on the blob request.
let blobs_request = Request::BlobsByRange(BlobsByRangeRequest {
start_slot: *request.start_slot(),
count: *request.count(),
});
let blocks_request = Request::BlocksByRange(request);
// Send both requests. Make sure both can be sent.
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: blocks_request,
request_id,
})?;
self.send_network_msg(NetworkMessage::SendRequest {
peer_id,
request: blobs_request,
request_id,
})?;
let block_blob_info = BlocksAndBlobsRequestInfo::default();
self.backfill_blocks_and_blobs_requests
.insert(id, (batch_id, block_blob_info));
Ok(id)
}
}
let id = self.blocks_by_range_request(peer_id, batch_type, request)?;
self.range_blocks_and_blobs_requests
.insert(id, (sender_id, BlocksAndBlobsRequestInfo::new(batch_type)));
Ok(id)
}
/// Response for a request that is only for blocks.
pub fn range_sync_block_only_response(
&mut self,
request_id: Id,
is_stream_terminator: bool,
) -> Option<(ChainId, BatchId)> {
if is_stream_terminator {
self.range_requests.remove(&request_id)
} else {
self.range_requests.get(&request_id).copied()
}
}
/// Received a blocks by range response for a request that couples blocks and blobs.
pub fn range_sync_block_and_blob_response(
&mut self,
request_id: Id,
block_or_blob: BlockOrBlob<T::EthSpec>,
) -> Option<(ChainId, BlocksAndBlobsByRangeResponse<T::EthSpec>)> {
match self.range_blocks_and_blobs_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let req = entry.get_mut();
let info = &mut req.block_blob_info;
match block_or_blob {
BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block),
BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
}
if info.is_finished() {
// If the request is finished, dequeue everything
let BlocksAndBlobsByRangeRequest {
chain_id,
batch_id,
block_blob_info,
} = entry.remove();
Some((
chain_id,
BlocksAndBlobsByRangeResponse {
batch_id,
responses: block_blob_info.into_responses(),
},
))
} else {
None
}
}
Entry::Vacant(_) => None,
}
}
pub fn range_sync_request_failed(
&mut self,
request_id: Id,
batch_type: ByRangeRequestType,
) -> Option<(ChainId, BatchId)> {
let req = match batch_type {
ByRangeRequestType::BlocksAndBlobs => self
.range_blocks_and_blobs_requests
.remove(&request_id)
.map(|req| (req.chain_id, req.batch_id)),
ByRangeRequestType::Blocks => self.range_requests.remove(&request_id),
};
if let Some(req) = req {
pub fn range_request_failed(&mut self, request_id: Id) -> Option<RangeRequestId> {
let sender_id = self
.range_blocks_and_blobs_requests
.remove(&request_id)
.map(|(sender_id, _info)| sender_id);
if let Some(sender_id) = sender_id {
debug!(
self.log,
"Range sync request failed";
"Sync range request failed";
"request_id" => request_id,
"batch_type" => ?batch_type,
"chain_id" => ?req.0,
"batch_id" => ?req.1
"sender_id" => ?sender_id
);
Some(req)
Some(sender_id)
} else {
debug!(self.log, "Range sync request failed"; "request_id" => request_id, "batch_type" => ?batch_type);
debug!(self.log, "Sync range request failed"; "request_id" => request_id);
None
}
}
pub fn backfill_request_failed(
&mut self,
request_id: Id,
batch_type: ByRangeRequestType,
) -> Option<BatchId> {
let batch_id = match batch_type {
ByRangeRequestType::BlocksAndBlobs => self
.backfill_blocks_and_blobs_requests
.remove(&request_id)
.map(|(batch_id, _info)| batch_id),
ByRangeRequestType::Blocks => self.backfill_requests.remove(&request_id),
};
if let Some(batch_id) = batch_id {
debug!(
self.log,
"Backfill sync request failed";
"request_id" => request_id,
"batch_type" => ?batch_type,
"batch_id" => ?batch_id
);
Some(batch_id)
} else {
debug!(self.log, "Backfill sync request failed"; "request_id" => request_id, "batch_type" => ?batch_type);
None
}
}
/// Response for a request that is only for blocks.
pub fn backfill_sync_only_blocks_response(
&mut self,
request_id: Id,
is_stream_terminator: bool,
) -> Option<BatchId> {
if is_stream_terminator {
self.backfill_requests.remove(&request_id)
} else {
self.backfill_requests.get(&request_id).copied()
}
}
/// Received a blocks by range or blobs by range response for a request that couples blocks '
/// and blobs.
pub fn backfill_sync_block_and_blob_response(
pub fn range_block_and_blob_response(
&mut self,
request_id: Id,
block_or_blob: BlockOrBlob<T::EthSpec>,
) -> Option<BlocksAndBlobsByRangeResponse<T::EthSpec>> {
match self.backfill_blocks_and_blobs_requests.entry(request_id) {
match self.range_blocks_and_blobs_requests.entry(request_id) {
Entry::Occupied(mut entry) => {
let (_, info) = entry.get_mut();
match block_or_blob {
@@ -416,12 +229,12 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
}
if info.is_finished() {
// If the request is finished, dequeue everything
let (batch_id, info) = entry.remove();
let responses = info.into_responses();
let (sender_id, info) = entry.remove();
let request_type = info.get_request_type();
Some(BlocksAndBlobsByRangeResponse {
batch_id,
responses,
sender_id,
request_type,
responses: info.into_responses(),
})
} else {
None
@@ -586,18 +399,10 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
pub fn insert_range_blocks_and_blobs_request(
&mut self,
id: Id,
request: BlocksAndBlobsByRangeRequest<T::EthSpec>,
sender_id: RangeRequestId,
info: BlocksAndBlobsRequestInfo<T::EthSpec>,
) {
self.range_blocks_and_blobs_requests.insert(id, request);
}
pub fn insert_backfill_blocks_and_blobs_requests(
&mut self,
id: Id,
batch_id: BatchId,
request: BlocksAndBlobsRequestInfo<T::EthSpec>,
) {
self.backfill_blocks_and_blobs_requests
.insert(id, (batch_id, request));
self.range_blocks_and_blobs_requests
.insert(id, (sender_id, info));
}
}