Handle sync lookup request streams in network context (#5583)

* by-root-stream-terminator

* Fix tests

* Resolve merge conflicts

* Log report reason

* Some lints and bugfixes (#23)

* fix lints

* bug fixes

* Fix tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into handle-sync-lookup-requests

* Pr 5583 review (#24)

* add bad state warn log

* add rust docs to new fields in `SyncNetworkContext`

* remove timestamp todo

* add back lookup verify error

* remove TODOs

beacon_node/network/src/sync/network_context/requests.rs

@@ -0,0 +1,149 @@
+use beacon_chain::get_block_root;
+use lighthouse_network::rpc::{methods::BlobsByRootRequest, BlocksByRootRequest};
+use std::sync::Arc;
+use strum::IntoStaticStr;
+use types::{
+    blob_sidecar::BlobIdentifier, BlobSidecar, ChainSpec, EthSpec, Hash256, SignedBeaconBlock,
+};
+
+#[derive(Debug, PartialEq, Eq, IntoStaticStr)]
+pub enum LookupVerifyError {
+    NoResponseReturned,
+    TooManyResponses,
+    UnrequestedBlockRoot(Hash256),
+    UnrequestedBlobIndex(u64),
+    InvalidInclusionProof,
+    DuplicateData,
+}
+
+pub struct ActiveBlocksByRootRequest {
+    request: BlocksByRootSingleRequest,
+    resolved: bool,
+}
+
+impl ActiveBlocksByRootRequest {
+    pub fn new(request: BlocksByRootSingleRequest) -> Self {
+        Self {
+            request,
+            resolved: false,
+        }
+    }
+
+    /// Append a response to the single chunk request. If the chunk is valid, the request is
+    /// resolved immediately.
+    /// The active request SHOULD be dropped after `add_response` returns an error
+    pub fn add_response<E: EthSpec>(
+        &mut self,
+        block: Arc<SignedBeaconBlock<E>>,
+    ) -> Result<Arc<SignedBeaconBlock<E>>, LookupVerifyError> {
+        if self.resolved {
+            return Err(LookupVerifyError::TooManyResponses);
+        }
+
+        let block_root = get_block_root(&block);
+        if self.request.0 != block_root {
+            return Err(LookupVerifyError::UnrequestedBlockRoot(block_root));
+        }
+
+        // Valid data, blocks by root expects a single response
+        self.resolved = true;
+        Ok(block)
+    }
+
+    pub fn terminate(self) -> Result<(), LookupVerifyError> {
+        if self.resolved {
+            Ok(())
+        } else {
+            Err(LookupVerifyError::NoResponseReturned)
+        }
+    }
+}
+
+#[derive(Debug, Copy, Clone)]
+pub struct BlocksByRootSingleRequest(pub Hash256);
+
+impl BlocksByRootSingleRequest {
+    pub fn into_request(self, spec: &ChainSpec) -> BlocksByRootRequest {
+        BlocksByRootRequest::new(vec![self.0], spec)
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct BlobsByRootSingleBlockRequest {
+    pub block_root: Hash256,
+    pub indices: Vec<u64>,
+}
+
+impl BlobsByRootSingleBlockRequest {
+    pub fn into_request(self, spec: &ChainSpec) -> BlobsByRootRequest {
+        BlobsByRootRequest::new(
+            self.indices
+                .into_iter()
+                .map(|index| BlobIdentifier {
+                    block_root: self.block_root,
+                    index,
+                })
+                .collect(),
+            spec,
+        )
+    }
+}
+
+pub struct ActiveBlobsByRootRequest<E: EthSpec> {
+    request: BlobsByRootSingleBlockRequest,
+    blobs: Vec<Arc<BlobSidecar<E>>>,
+    resolved: bool,
+}
+
+impl<E: EthSpec> ActiveBlobsByRootRequest<E> {
+    pub fn new(request: BlobsByRootSingleBlockRequest) -> Self {
+        Self {
+            request,
+            blobs: vec![],
+            resolved: false,
+        }
+    }
+
+    /// Appends a chunk to this multi-item request. If all expected chunks are received, this
+    /// method returns `Some`, resolving the request before the stream terminator.
+    /// The active request SHOULD be dropped after `add_response` returns an error
+    pub fn add_response(
+        &mut self,
+        blob: Arc<BlobSidecar<E>>,
+    ) -> Result<Option<Vec<Arc<BlobSidecar<E>>>>, LookupVerifyError> {
+        if self.resolved {
+            return Err(LookupVerifyError::TooManyResponses);
+        }
+
+        let block_root = blob.block_root();
+        if self.request.block_root != block_root {
+            return Err(LookupVerifyError::UnrequestedBlockRoot(block_root));
+        }
+        if !blob.verify_blob_sidecar_inclusion_proof().unwrap_or(false) {
+            return Err(LookupVerifyError::InvalidInclusionProof);
+        }
+        if !self.request.indices.contains(&blob.index) {
+            return Err(LookupVerifyError::UnrequestedBlobIndex(blob.index));
+        }
+        if self.blobs.iter().any(|b| b.index == blob.index) {
+            return Err(LookupVerifyError::DuplicateData);
+        }
+
+        self.blobs.push(blob);
+        if self.blobs.len() >= self.request.indices.len() {
+            // All expected chunks received, return result early
+            self.resolved = true;
+            Ok(Some(std::mem::take(&mut self.blobs)))
+        } else {
+            Ok(None)
+        }
+    }
+
+    pub fn terminate(self) -> Option<Vec<Arc<BlobSidecar<E>>>> {
+        if self.resolved {
+            None
+        } else {
+            Some(self.blobs)
+        }
+    }
+}