add processing and processed caching to the DA checker (#4732)

* add processing and processed caching to the DA checker

* move processing cache out of critical cache

* get it compiling

* fix lints

* add docs to `AvailabilityView`

* some self review

* fix lints

* fix beacon chain tests

* cargo fmt

* make availability view easier to implement, start on testing

* move child component cache and finish test

* cargo fix

* cargo fix

* cargo fix

* fmt and lint

* make blob commitments not optional, rename some caches, add missing blobs struct

* Update beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs

Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>

* marks review feedback and other general cleanup

* cargo fix

* improve availability view docs

* some renames

* some renames and docs

* fix should delay lookup logic

* get rid of some wrapper methods

* fix up single lookup changes

* add a couple docs

* add single blob merge method and improve process_... docs

* update some names

* lints

* fix merge

* remove blob indices from lookup creation log

* remove blob indices from lookup creation log

* delayed lookup logging improvement

* check fork choice before doing any blob processing

* remove unused dep

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Update beacon_node/network/src/sync/block_lookups/delayed_lookup.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* remove duplicate deps

* use gen range in random blobs geneartor

* rename processing cache fields

* require block root in rpc block construction and check block root consistency

* send peers as vec in single message

* spawn delayed lookup service from network beacon processor

* fix tests

---------

Co-authored-by: ethDreamer <37123614+ethDreamer@users.noreply.github.com>
Co-authored-by: Michael Sproul <micsproul@gmail.com>

beacon_node/beacon_chain/src/blob_verification.rs

@@ -9,11 +9,12 @@ use crate::beacon_chain::{
 use crate::block_verification::cheap_state_advance_to_obtain_committees;
 use crate::data_availability_checker::AvailabilityCheckError;
 use crate::kzg_utils::{validate_blob, validate_blobs};
-use crate::BeaconChainError;
+use crate::{metrics, BeaconChainError};
 use kzg::Kzg;
 use slog::{debug, warn};
 use ssz_derive::{Decode, Encode};
 use ssz_types::VariableList;
+use tree_hash::TreeHash;
 use types::blob_sidecar::BlobIdentifier;
 use types::{
     BeaconStateError, BlobSidecar, BlobSidecarList, CloneConfig, EthSpec, Hash256,
@@ -172,6 +173,9 @@ impl<T: BeaconChainTypes> GossipVerifiedBlob<T> {
     pub fn to_blob(self) -> Arc<BlobSidecar<T::EthSpec>> {
         self.blob.message
     }
+    pub fn as_blob(&self) -> &BlobSidecar<T::EthSpec> {
+        &self.blob.message
+    }
     pub fn signed_blob(&self) -> SignedBlobSidecar<T::EthSpec> {
         self.blob.clone()
     }
@@ -203,6 +207,8 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
         });
     }
 
+    let blob_root = get_blob_root(&signed_blob_sidecar);
+
     // Verify that the sidecar is not from a future slot.
     let latest_permissible_slot = chain
         .slot_clock
@@ -393,7 +399,7 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
             .ok_or_else(|| GossipBlobError::UnknownValidator(proposer_index as u64))?;
 
         signed_blob_sidecar.verify_signature(
-            None,
+            Some(blob_root),
             pubkey,
             &fork,
             chain.genesis_validators_root,
@@ -473,6 +479,15 @@ impl<T: EthSpec> KzgVerifiedBlob<T> {
     }
 }
 
+#[cfg(test)]
+impl<T: EthSpec> KzgVerifiedBlob<T> {
+    pub fn new(blob: BlobSidecar<T>) -> Self {
+        Self {
+            blob: Arc::new(blob),
+        }
+    }
+}
+
 /// Complete kzg verification for a `GossipVerifiedBlob`.
 ///
 /// Returns an error if the kzg verification check fails.
@@ -518,3 +533,16 @@ pub fn verify_kzg_for_blob_list<T: EthSpec>(
         Err(AvailabilityCheckError::KzgVerificationFailed)
     }
 }
+
+/// Returns the canonical root of the given `blob`.
+///
+/// Use this function to ensure that we report the blob hashing time Prometheus metric.
+pub fn get_blob_root<E: EthSpec>(blob: &SignedBlobSidecar<E>) -> Hash256 {
+    let blob_root_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_BLOB_ROOT);
+
+    let blob_root = blob.message.tree_hash_root();
+
+    metrics::stop_timer(blob_root_timer);
+
+    blob_root
+}