Add timing for block availability (#5510)

* Add timing for block availability

* Attestation metrics analysis

* Prettier printing

* Add some metrics and timings to track late blocks

* Update to latest unstable

* fmt

* Merge latest unstable

* Small tweaks

* Try pushing blob timing down into verification

* Simplify for clippy

beacon_node/beacon_chain/src/blob_verification.rs

@@ -14,6 +14,7 @@ use merkle_proof::MerkleTreeError;
 use slog::{debug, warn};
 use ssz_derive::{Decode, Encode};
 use ssz_types::VariableList;
+use std::time::Duration;
 use tree_hash::TreeHash;
 use types::blob_sidecar::BlobIdentifier;
 use types::{
@@ -214,7 +215,10 @@ impl<T: BeaconChainTypes> GossipVerifiedBlob<T> {
     pub fn __assumed_valid(blob: Arc<BlobSidecar<T::EthSpec>>) -> Self {
         Self {
             block_root: blob.block_root(),
-            blob: KzgVerifiedBlob { blob },
+            blob: KzgVerifiedBlob {
+                blob,
+                seen_timestamp: Duration::from_secs(0),
+            },
         }
     }
     pub fn id(&self) -> BlobIdentifier {
@@ -260,6 +264,8 @@ impl<T: BeaconChainTypes> GossipVerifiedBlob<T> {
 #[ssz(struct_behaviour = "transparent")]
 pub struct KzgVerifiedBlob<E: EthSpec> {
     blob: Arc<BlobSidecar<E>>,
+    #[ssz(skip_serializing, skip_deserializing)]
+    seen_timestamp: Duration,
 }
 
 impl<E: EthSpec> PartialOrd for KzgVerifiedBlob<E> {
@@ -275,8 +281,12 @@ impl<E: EthSpec> Ord for KzgVerifiedBlob<E> {
 }
 
 impl<E: EthSpec> KzgVerifiedBlob<E> {
-    pub fn new(blob: Arc<BlobSidecar<E>>, kzg: &Kzg) -> Result<Self, KzgError> {
-        verify_kzg_for_blob(blob, kzg)
+    pub fn new(
+        blob: Arc<BlobSidecar<E>>,
+        kzg: &Kzg,
+        seen_timestamp: Duration,
+    ) -> Result<Self, KzgError> {
+        verify_kzg_for_blob(blob, kzg, seen_timestamp)
     }
     pub fn to_blob(self) -> Arc<BlobSidecar<E>> {
         self.blob
@@ -294,12 +304,18 @@ impl<E: EthSpec> KzgVerifiedBlob<E> {
     pub fn blob_index(&self) -> u64 {
         self.blob.index
     }
+    pub fn seen_timestamp(&self) -> Duration {
+        self.seen_timestamp
+    }
     /// Construct a `KzgVerifiedBlob` that is assumed to be valid.
     ///
     /// This should ONLY be used for testing.
     #[cfg(test)]
     pub fn __assumed_valid(blob: Arc<BlobSidecar<E>>) -> Self {
-        Self { blob }
+        Self {
+            blob,
+            seen_timestamp: Duration::from_secs(0),
+        }
     }
 }
 
@@ -309,9 +325,13 @@ impl<E: EthSpec> KzgVerifiedBlob<E> {
 pub fn verify_kzg_for_blob<E: EthSpec>(
     blob: Arc<BlobSidecar<E>>,
     kzg: &Kzg,
+    seen_timestamp: Duration,
 ) -> Result<KzgVerifiedBlob<E>, KzgError> {
     validate_blob::<E>(kzg, &blob.blob, blob.kzg_commitment, blob.kzg_proof)?;
-    Ok(KzgVerifiedBlob { blob })
+    Ok(KzgVerifiedBlob {
+        blob,
+        seen_timestamp,
+    })
 }
 
 pub struct KzgVerifiedBlobList<E: EthSpec> {
@@ -322,13 +342,17 @@ impl<E: EthSpec> KzgVerifiedBlobList<E> {
     pub fn new<I: IntoIterator<Item = Arc<BlobSidecar<E>>>>(
         blob_list: I,
         kzg: &Kzg,
+        seen_timestamp: Duration,
     ) -> Result<Self, KzgError> {
         let blobs = blob_list.into_iter().collect::<Vec<_>>();
         verify_kzg_for_blob_list(blobs.iter(), kzg)?;
         Ok(Self {
             verified_blobs: blobs
                 .into_iter()
-                .map(|blob| KzgVerifiedBlob { blob })
+                .map(|blob| KzgVerifiedBlob {
+                    blob,
+                    seen_timestamp,
+                })
                 .collect(),
         })
     }
@@ -374,6 +398,8 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
     let blob_epoch = blob_slot.epoch(T::EthSpec::slots_per_epoch());
     let signed_block_header = &blob_sidecar.signed_block_header;
 
+    let seen_timestamp = chain.slot_clock.now_duration().unwrap_or_default();
+
     // This condition is not possible if we have received the blob from the network
     // since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network.
     // We include this check only for completeness.
@@ -641,8 +667,8 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes>(
         .kzg
         .as_ref()
         .ok_or(GossipBlobError::KzgNotInitialized)?;
-    let kzg_verified_blob =
-        KzgVerifiedBlob::new(blob_sidecar, kzg).map_err(GossipBlobError::KzgError)?;
+    let kzg_verified_blob = KzgVerifiedBlob::new(blob_sidecar, kzg, seen_timestamp)
+        .map_err(GossipBlobError::KzgError)?;
 
     Ok(GossipVerifiedBlob {
         block_root,