add processing and processed caching to the DA checker (#4732)

* add processing and processed caching to the DA checker

* move processing cache out of critical cache

* get it compiling

* fix lints

* add docs to `AvailabilityView`

* some self review

* fix lints

* fix beacon chain tests

* cargo fmt

* make availability view easier to implement, start on testing

* move child component cache and finish test

* cargo fix

* cargo fix

* cargo fix

* fmt and lint

* make blob commitments not optional, rename some caches, add missing blobs struct

* Update beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs

Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>

* marks review feedback and other general cleanup

* cargo fix

* improve availability view docs

* some renames

* some renames and docs

* fix should delay lookup logic

* get rid of some wrapper methods

* fix up single lookup changes

* add a couple docs

* add single blob merge method and improve process_... docs

* update some names

* lints

* fix merge

* remove blob indices from lookup creation log

* remove blob indices from lookup creation log

* delayed lookup logging improvement

* check fork choice before doing any blob processing

* remove unused dep

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/network/src/sync/block_lookups/delayed_lookup.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* remove duplicate deps

* use gen range in random blobs geneartor

* rename processing cache fields

* require block root in rpc block construction and check block root consistency

* send peers as vec in single message

* spawn delayed lookup service from network beacon processor

* fix tests

---------

Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>
Co-authored-by: Michael Sproul <micsproul@gmail.com>

beacon_node/beacon_chain/src/test_utils.rs

@@ -60,6 +60,7 @@ use task_executor::{test_utils::TestRuntime, ShutdownReason};
 use tree_hash::TreeHash;
 use types::sync_selection_proof::SyncSelectionProof;
 pub use types::test_utils::generate_deterministic_keypairs;
+use types::test_utils::TestRandom;
 use types::{typenum::U4294967296, *};
 
 // 4th September 2019
@@ -709,14 +710,14 @@ where
         let block = self.chain.head_beacon_block();
         let block_root = block.canonical_root();
         let blobs = self.chain.get_blobs(&block_root).unwrap();
-        RpcBlock::new(block, Some(blobs)).unwrap()
+        RpcBlock::new(Some(block_root), block, Some(blobs)).unwrap()
     }
 
     pub fn get_full_block(&self, block_root: &Hash256) -> RpcBlock<E> {
         let block = self.chain.get_blinded_block(block_root).unwrap().unwrap();
         let full_block = self.chain.store.make_full_block(block_root, block).unwrap();
         let blobs = self.chain.get_blobs(block_root).unwrap();
-        RpcBlock::new(Arc::new(full_block), Some(blobs)).unwrap()
+        RpcBlock::new(Some(*block_root), Arc::new(full_block), Some(blobs)).unwrap()
     }
 
     pub fn get_all_validators(&self) -> Vec<usize> {
@@ -1922,7 +1923,7 @@ where
             .chain
             .process_block(
                 block_root,
-                RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(),
+                RpcBlock::new(Some(block_root), Arc::new(block), blobs_without_signatures).unwrap(),
                 NotifyExecutionLayer::Yes,
                 || Ok(()),
             )
@@ -1947,11 +1948,12 @@ where
                     .collect::<Vec<_>>(),
             )
         });
+        let block_root = block.canonical_root();
         let block_hash: SignedBeaconBlockHash = self
             .chain
             .process_block(
-                block.canonical_root(),
-                RpcBlock::new(Arc::new(block), blobs_without_signatures).unwrap(),
+                block_root,
+                RpcBlock::new(Some(block_root), Arc::new(block), blobs_without_signatures).unwrap(),
                 NotifyExecutionLayer::Yes,
                 || Ok(()),
             )
@@ -2513,3 +2515,64 @@ pub fn build_log(level: slog::Level, enabled: bool) -> Logger {
         Logger::root(drain.filter(|_| false).fuse(), o!())
     }
 }
+
+pub enum NumBlobs {
+    Random,
+    None,
+}
+
+pub fn generate_rand_block_and_blobs<E: EthSpec>(
+    fork_name: ForkName,
+    num_blobs: NumBlobs,
+    kzg: &Kzg<E::Kzg>,
+    rng: &mut impl Rng,
+) -> (SignedBeaconBlock<E, FullPayload<E>>, Vec<BlobSidecar<E>>) {
+    let inner = map_fork_name!(fork_name, BeaconBlock, <_>::random_for_test(rng));
+    let mut block = SignedBeaconBlock::from_block(inner, types::Signature::random_for_test(rng));
+    let mut blob_sidecars = vec![];
+    if let Ok(message) = block.message_deneb_mut() {
+        // Get either zero blobs or a random number of blobs between 1 and Max Blobs.
+        let payload: &mut FullPayloadDeneb<E> = &mut message.body.execution_payload;
+        let num_blobs = match num_blobs {
+            NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()),
+            NumBlobs::None => 0,
+        };
+        let (bundle, transactions) =
+            execution_layer::test_utils::generate_random_blobs::<E, _>(num_blobs, kzg, rng)
+                .unwrap();
+
+        payload.execution_payload.transactions = <_>::default();
+        for tx in Vec::from(transactions) {
+            payload.execution_payload.transactions.push(tx).unwrap();
+        }
+        message.body.blob_kzg_commitments = bundle.commitments.clone();
+
+        let eth2::types::BlobsBundle {
+            commitments,
+            proofs,
+            blobs,
+        } = bundle;
+
+        let block_root = block.canonical_root();
+
+        for (index, ((blob, kzg_commitment), kzg_proof)) in blobs
+            .into_iter()
+            .zip(commitments.into_iter())
+            .zip(proofs.into_iter())
+            .enumerate()
+        {
+            blob_sidecars.push(BlobSidecar {
+                block_root,
+                index: index as u64,
+                slot: block.slot(),
+                block_parent_root: block.parent_root(),
+                proposer_index: block.message().proposer_index(),
+                blob: blob.clone(),
+                kzg_commitment,
+                kzg_proof,
+            });
+        }
+    }
+
+    (block, blob_sidecars)
+}