Single blob lookups (#4152)

* some blob reprocessing work

* remove ForceBlockLookup

* reorder enum match arms in sync manager

* a lot more reprocessing work

* impl logic for triggerng blob lookups along with block lookups

* deal with rpc blobs in groups per block in the da checker. don't cache missing blob ids in the da checker.

* make single block lookup generic

* more work

* add delayed processing logic and combine some requests

* start fixing some compile errors

* fix compilation in main block lookup mod

* much work

* get things compiling

* parent blob lookups

* fix compile

* revert red/stevie changes

* fix up sync manager delay message logic

* add peer usefulness enum

* should remove lookup refactor

* consolidate retry error handling

* improve peer scoring during certain failures in parent lookups

* improve retry code

* drop parent lookup if either req has a peer disconnect during download

* refactor single block processed method

* processing peer refactor

* smol bugfix

* fix some todos

* fix lints

* fix lints

* fix compile in lookup tests

* fix lints

* fix lints

* fix existing block lookup tests

* renamings

* fix after merge

* cargo fmt

* compilation fix in beacon chain tests

* fix

* refactor lookup tests to work with multiple forks and response types

* make tests into macros

* wrap availability check error

* fix compile after merge

* add random blobs

* start fixing up lookup verify error handling

* some bug fixes and the start of deneb only tests

* make tests work for all forks

* track information about peer source

* error refactoring

* improve peer scoring

* fix test compilation

* make sure blobs are sent for processing after stream termination, delete copied tests

* add some tests and fix a bug

* smol bugfixes and moar tests

* add tests and fix some things

* compile after merge

* lots of refactoring

* retry on invalid block/blob

* merge unknown parent messages before current slot lookup

* get tests compiling

* penalize blob peer on invalid blobs

* Check disk on in-memory cache miss

* Update beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

* Update beacon_node/network/src/sync/network_context.rs

Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com>

* fix bug in matching blocks and blobs in range sync

* pr feedback

* fix conflicts

* upgrade logs from warn to crit when we receive incorrect response in range

* synced_and_connected_within_tolerance -> should_search_for_block

* remove todo

* Fix Broken Overflow Tests

* fix merge conflicts

* checkpoint sync without alignment

* add import

* query for checkpoint state by slot rather than state root (teku doesn't serve by state root)

* get state first and query by most recent block root

* simplify delay logic

* rename unknown parent sync message variants

* rename parameter, block_slot -> slot

* add some docs to the lookup module

* use interval instead of sleep

* drop request if blocks and blobs requests both return `None` for `Id`

* clean up `find_single_lookup` logic

* add lookup source enum

* clean up `find_single_lookup` logic

* add docs to find_single_lookup_request

* move LookupSource our of param where unnecessary

* remove unnecessary todo

* query for block by `state.latest_block_header.slot`

* fix lint

* fix test

* fix test

* fix observed  blob sidecars test

* PR updates

* use optional params instead of a closure

* create lookup and trigger request in separate method calls

* remove `LookupSource`

* make sure duplicate lookups are not dropped

---------

Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>
Co-authored-by: Mark Mackey <mark@sigmaprime.io>
Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com>

beacon_node/network/src/sync/network_context.rs

@@ -7,11 +7,11 @@ use super::range_sync::{BatchId, ByRangeRequestType, ChainId};
 use crate::beacon_processor::WorkEvent;
 use crate::service::{NetworkMessage, RequestId};
 use crate::status::ToStatusMessage;
-use crate::sync::block_lookups::ForceBlockRequest;
+use crate::sync::block_lookups::{BlobRequestId, BlockRequestId};
 use beacon_chain::blob_verification::BlockWrapper;
 use beacon_chain::{BeaconChain, BeaconChainTypes, EngineState};
 use fnv::FnvHashMap;
-use lighthouse_network::rpc::methods::BlobsByRangeRequest;
+use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest};
 use lighthouse_network::rpc::{BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason};
 use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Request};
 use slog::{debug, trace, warn};
@@ -62,7 +62,7 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
     /// Channel to send work to the beacon processor.
     beacon_processor_send: mpsc::Sender<WorkEvent<T>>,
 
-    chain: Arc<BeaconChain<T>>,
+    pub chain: Arc<BeaconChain<T>>,
 
     /// Logger for the `SyncNetworkContext`.
     log: slog::Logger,
@@ -71,7 +71,7 @@ pub struct SyncNetworkContext<T: BeaconChainTypes> {
 /// Small enumeration to make dealing with block and blob requests easier.
 pub enum BlockOrBlob<T: EthSpec> {
     Block(Option<Arc<SignedBeaconBlock<T>>>),
-    Sidecar(Option<Arc<BlobSidecar<T>>>),
+    Blob(Option<Arc<BlobSidecar<T>>>),
 }
 
 impl<T: EthSpec> From<Option<Arc<SignedBeaconBlock<T>>>> for BlockOrBlob<T> {
@@ -82,7 +82,7 @@ impl<T: EthSpec> From<Option<Arc<SignedBeaconBlock<T>>>> for BlockOrBlob<T> {
 
 impl<T: EthSpec> From<Option<Arc<BlobSidecar<T>>>> for BlockOrBlob<T> {
     fn from(blob: Option<Arc<BlobSidecar<T>>>) -> Self {
-        BlockOrBlob::Sidecar(blob)
+        BlockOrBlob::Blob(blob)
     }
 }
 
@@ -187,7 +187,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
 
                 // create the shared request id. This is fine since the rpc handles substream ids.
                 let id = self.next_id();
-                let request_id = RequestId::Sync(SyncRequestId::RangeBlobs { id });
+                let request_id = RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id });
 
                 // Create the blob request based on the blob request.
                 let blobs_request = Request::BlobsByRange(BlobsByRangeRequest {
@@ -260,7 +260,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
 
                 // create the shared request id. This is fine since the rpc handles substream ids.
                 let id = self.next_id();
-                let request_id = RequestId::Sync(SyncRequestId::BackFillBlobs { id });
+                let request_id = RequestId::Sync(SyncRequestId::BackFillBlockAndBlobs { id });
 
                 // Create the blob request based on the blob request.
                 let blobs_request = Request::BlobsByRange(BlobsByRangeRequest {
@@ -289,7 +289,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
     }
 
     /// Response for a request that is only for blocks.
-    pub fn range_sync_block_response(
+    pub fn range_sync_block_only_response(
         &mut self,
         request_id: Id,
         is_stream_terminator: bool,
@@ -313,7 +313,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
                 let info = &mut req.block_blob_info;
                 match block_or_blob {
                     BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block),
-                    BlockOrBlob::Sidecar(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
+                    BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
                 }
                 if info.is_finished() {
                     // If the request is finished, dequeue everything
@@ -390,7 +390,7 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
                 let (_, info) = entry.get_mut();
                 match block_or_blob {
                     BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block),
-                    BlockOrBlob::Sidecar(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
+                    BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar),
                 }
                 if info.is_finished() {
                     // If the request is finished, dequeue everything
@@ -409,83 +409,101 @@ impl<T: BeaconChainTypes> SyncNetworkContext<T> {
         }
     }
 
-    /// Sends a blocks by root request for a single block lookup.
+    /// Sends a blocks by root request for a parent request.
     pub fn single_block_lookup_request(
         &mut self,
         peer_id: PeerId,
         request: BlocksByRootRequest,
     ) -> Result<Id, &'static str> {
-        let request = if self
-            .chain
-            .is_data_availability_check_required()
-            .map_err(|_| "Unable to read slot clock")?
-        {
-            trace!(
-                self.log,
-                "Sending BlobsByRoot Request";
-                "method" => "BlobsByRoot",
-                "count" => request.block_roots.len(),
-                "peer" => %peer_id
-            );
-            unimplemented!("There is no longer such thing as a single block lookup, since we nede to ask for blobs and blocks separetely");
-        } else {
-            trace!(
-                self.log,
-                "Sending BlocksByRoot Request";
-                "method" => "BlocksByRoot",
-                "count" => request.block_roots.len(),
-                "peer" => %peer_id
-            );
-            Request::BlocksByRoot(request)
-        };
         let id = self.next_id();
         let request_id = RequestId::Sync(SyncRequestId::SingleBlock { id });
+
+        trace!(
+            self.log,
+            "Sending BlocksByRoot Request";
+            "method" => "BlocksByRoot",
+            "count" => request.block_roots.len(),
+            "peer" => %peer_id
+        );
+
         self.send_network_msg(NetworkMessage::SendRequest {
             peer_id,
-            request,
+            request: Request::BlocksByRoot(request),
+            request_id,
+        })?;
+        Ok(id)
+    }
+
+    /// Sends a blobs by root request for a parent request.
+    pub fn single_blobs_lookup_request(
+        &mut self,
+        peer_id: PeerId,
+        request: BlobsByRootRequest,
+    ) -> Result<Id, &'static str> {
+        let id = self.next_id();
+        let request_id = RequestId::Sync(SyncRequestId::SingleBlock { id });
+
+        trace!(
+            self.log,
+            "Sending BlobsByRoot Request";
+            "method" => "BlobsByRoot",
+            "count" => request.blob_ids.len(),
+            "peer" => %peer_id
+        );
+
+        self.send_network_msg(NetworkMessage::SendRequest {
+            peer_id,
+            request: Request::BlobsByRoot(request),
             request_id,
         })?;
         Ok(id)
     }
 
     /// Sends a blocks by root request for a parent request.
-    pub fn parent_lookup_request(
+    pub fn parent_lookup_block_request(
         &mut self,
         peer_id: PeerId,
         request: BlocksByRootRequest,
-        force_block_request: ForceBlockRequest,
-    ) -> Result<Id, &'static str> {
-        let request = if self
-            .chain
-            .is_data_availability_check_required()
-            .map_err(|_| "Unable to read slot clock")?
-            && matches!(force_block_request, ForceBlockRequest::False)
-        {
-            trace!(
-                self.log,
-                "Sending BlobsByRoot Request";
-                "method" => "BlobsByRoot",
-                "count" => request.block_roots.len(),
-                "peer" => %peer_id
-            );
-            unimplemented!(
-                "Parent requests now need to interleave blocks and blobs or something like that."
-            )
-        } else {
-            trace!(
-                self.log,
-                "Sending BlocksByRoot Request";
-                "method" => "BlocksByRoot",
-                "count" => request.block_roots.len(),
-                "peer" => %peer_id
-            );
-            Request::BlocksByRoot(request)
-        };
+    ) -> Result<BlockRequestId, &'static str> {
         let id = self.next_id();
         let request_id = RequestId::Sync(SyncRequestId::ParentLookup { id });
+
+        trace!(
+            self.log,
+            "Sending parent BlocksByRoot Request";
+            "method" => "BlocksByRoot",
+            "count" => request.block_roots.len(),
+            "peer" => %peer_id
+        );
+
         self.send_network_msg(NetworkMessage::SendRequest {
             peer_id,
-            request,
+            request: Request::BlocksByRoot(request),
+            request_id,
+        })?;
+        Ok(id)
+    }
+
+    /// Sends a blocks by root request for a parent request.
+    pub fn parent_lookup_blobs_request(
+        &mut self,
+        peer_id: PeerId,
+        request: BlobsByRootRequest,
+    ) -> Result<BlobRequestId, &'static str> {
+        let id = self.next_id();
+        let request_id = RequestId::Sync(SyncRequestId::ParentLookup { id });
+
+        trace!(
+            self.log,
+            "Sending parent BlobsByRoot Request";
+            "method" => "BlobsByRoot",
+            "count" => request.blob_ids.len(),
+            "peer" => %peer_id
+        );
+
+        self.send_network_msg(NetworkMessage::SendRequest {
+            peer_id,
+            request: Request::BlobsByRoot(request),
             request_id,
         })?;
         Ok(id)