Single blob lookups (#4152)

* some blob reprocessing work

* remove ForceBlockLookup

* reorder enum match arms in sync manager

* a lot more reprocessing work

* impl logic for triggerng blob lookups along with block lookups

* deal with rpc blobs in groups per block in the da checker. don't cache missing blob ids in the da checker.

* make single block lookup generic

* more work

* add delayed processing logic and combine some requests

* start fixing some compile errors

* fix compilation in main block lookup mod

* much work

* get things compiling

* parent blob lookups

* fix compile

* revert red/stevie changes

* fix up sync manager delay message logic

* add peer usefulness enum

* should remove lookup refactor

* consolidate retry error handling

* improve peer scoring during certain failures in parent lookups

* improve retry code

* drop parent lookup if either req has a peer disconnect during download

* refactor single block processed method

* processing peer refactor

* smol bugfix

* fix some todos

* fix lints

* fix lints

* fix compile in lookup tests

* fix lints

* fix lints

* fix existing block lookup tests

* renamings

* fix after merge

* cargo fmt

* compilation fix in beacon chain tests

* fix

* refactor lookup tests to work with multiple forks and response types

* make tests into macros

* wrap availability check error

* fix compile after merge

* add random blobs

* start fixing up lookup verify error handling

* some bug fixes and the start of deneb only tests

* make tests work for all forks

* track information about peer source

* error refactoring

* improve peer scoring

* fix test compilation

* make sure blobs are sent for processing after stream termination, delete copied tests

* add some tests and fix a bug

* smol bugfixes and moar tests

* add tests and fix some things

* compile after merge

* lots of refactoring

* retry on invalid block/blob

* merge unknown parent messages before current slot lookup

* get tests compiling

* penalize blob peer on invalid blobs

* Check disk on in-memory cache miss

* Update beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs

* Update beacon_node/network/src/sync/network_context.rs

Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com>

* fix bug in matching blocks and blobs in range sync

* pr feedback

* fix conflicts

* upgrade logs from warn to crit when we receive incorrect response in range

* synced_and_connected_within_tolerance -> should_search_for_block

* remove todo

* Fix Broken Overflow Tests

* fix merge conflicts

* checkpoint sync without alignment

* add import

* query for checkpoint state by slot rather than state root (teku doesn't serve by state root)

* get state first and query by most recent block root

* simplify delay logic

* rename unknown parent sync message variants

* rename parameter, block_slot -> slot

* add some docs to the lookup module

* use interval instead of sleep

* drop request if blocks and blobs requests both return `None` for `Id`

* clean up `find_single_lookup` logic

* add lookup source enum

* clean up `find_single_lookup` logic

* add docs to find_single_lookup_request

* move LookupSource our of param where unnecessary

* remove unnecessary todo

* query for block by `state.latest_block_header.slot`

* fix lint

* fix test

* fix test

* fix observed  blob sidecars test

* PR updates

* use optional params instead of a closure

* create lookup and trigger request in separate method calls

* remove `LookupSource`

* make sure duplicate lookups are not dropped

---------

Co-authored-by: Pawan Dhananjay <pawandhananjay@gmail.com>
Co-authored-by: Mark Mackey <mark@sigmaprime.io>
Co-authored-by: Divma <26765164+divagant-martian@users.noreply.github.com>

consensus/types/src/blob_sidecar.rs

@@ -6,13 +6,15 @@ use kzg::{KzgCommitment, KzgProof};
 use serde_derive::{Deserialize, Serialize};
 use ssz::Encode;
 use ssz_derive::{Decode, Encode};
-use ssz_types::VariableList;
+use ssz_types::{FixedVector, VariableList};
 use std::sync::Arc;
 use test_random_derive::TestRandom;
 use tree_hash_derive::TreeHash;
 
 /// Container of the data that identifies an individual blob.
-#[derive(Serialize, Deserialize, Encode, Decode, TreeHash, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(
+    Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash,
+)]
 pub struct BlobIdentifier {
     pub block_root: Hash256,
     pub index: u64,
@@ -73,6 +75,8 @@ impl<T: EthSpec> Ord for BlobSidecar<T> {
 }
 
 pub type BlobSidecarList<T> = VariableList<Arc<BlobSidecar<T>>, <T as EthSpec>::MaxBlobsPerBlock>;
+pub type FixedBlobSidecarList<T> =
+    FixedVector<Option<Arc<BlobSidecar<T>>>, <T as EthSpec>::MaxBlobsPerBlock>;
 pub type Blobs<T> = VariableList<Blob<T>, <T as EthSpec>::MaxBlobsPerBlock>;
 
 impl<T: EthSpec> SignedRoot for BlobSidecar<T> {}

consensus/types/src/eth_spec.rs

@@ -105,7 +105,7 @@ pub trait EthSpec:
     /*
      * New in Deneb
      */
-    type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq;
+    type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin;
     type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq;
     type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq;
     /*
@@ -255,6 +255,11 @@ pub trait EthSpec:
     fn max_blobs_per_block() -> usize {
         Self::MaxBlobsPerBlock::to_usize()
     }
+
+    /// Returns the `BYTES_PER_BLOB` constant for this specification.
+    fn bytes_per_blob() -> usize {
+        Self::BytesPerBlob::to_usize()
+    }
 }
 
 /// Macro to inherit some type values from another EthSpec.

consensus/types/src/signed_beacon_block.rs

@@ -1,3 +1,4 @@
+use crate::blob_sidecar::BlobIdentifier;
 use crate::*;
 use bls::Signature;
 use derivative::Derivative;
@@ -248,6 +249,38 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
     pub fn canonical_root(&self) -> Hash256 {
         self.message().tree_hash_root()
     }
+
+    pub fn num_expected_blobs(&self) -> usize {
+        self.message()
+            .body()
+            .blob_kzg_commitments()
+            .map(|c| c.len())
+            .unwrap_or(0)
+    }
+
+    pub fn get_expected_blob_ids(&self, block_root: Option<Hash256>) -> Vec<BlobIdentifier> {
+        self.get_filtered_blob_ids(block_root, |_, _| true)
+    }
+
+    /// If the filter returns `true` the id for the corresponding index and root will be included.
+    pub fn get_filtered_blob_ids(
+        &self,
+        block_root: Option<Hash256>,
+        filter: impl Fn(usize, Hash256) -> bool,
+    ) -> Vec<BlobIdentifier> {
+        let block_root = block_root.unwrap_or_else(|| self.canonical_root());
+        let num_blobs_expected = self.num_expected_blobs();
+        let mut blob_ids = Vec::with_capacity(num_blobs_expected);
+        for i in 0..num_blobs_expected {
+            if filter(i, block_root) {
+                blob_ids.push(BlobIdentifier {
+                    block_root,
+                    index: i as u64,
+                });
+            }
+        }
+        blob_ids
+    }
 }
 
 // We can convert pre-Bellatrix blocks without payloads into blocks with payloads.

consensus/types/src/test_utils/test_random/bitfield.rs

@@ -4,8 +4,21 @@ use smallvec::smallvec;
 
 impl<N: Unsigned + Clone> TestRandom for BitList<N> {
     fn random_for_test(rng: &mut impl RngCore) -> Self {
-        let mut raw_bytes = smallvec![0; std::cmp::max(1, (N::to_usize() + 7) / 8)];
+        let initial_len = std::cmp::max(1, (N::to_usize() + 7) / 8);
+        let mut raw_bytes = smallvec![0; initial_len];
         rng.fill_bytes(&mut raw_bytes);
+
+        let non_zero_bytes = raw_bytes
+            .iter()
+            .enumerate()
+            .rev()
+            .find_map(|(i, byte)| (*byte > 0).then_some(i + 1))
+            .unwrap_or(0);
+
+        if non_zero_bytes < initial_len {
+            raw_bytes.truncate(non_zero_bytes);
+        }
+
         Self::from_bytes(raw_bytes).expect("we generate a valid BitList")
     }
 }