add blobs cache and fix some block production

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -6,6 +6,7 @@ use crate::attestation_verification::{
 use crate::attester_cache::{AttesterCache, AttesterCacheKey};
 use crate::beacon_proposer_cache::compute_proposer_duties_from_head;
 use crate::beacon_proposer_cache::BeaconProposerCache;
+use crate::blob_cache::BlobCache;
 use crate::block_times_cache::BlockTimesCache;
 use crate::block_verification::{
     check_block_is_finalized_descendant, check_block_relevancy, get_block_root,
@@ -389,6 +390,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
     pub slasher: Option<Arc<Slasher<T::EthSpec>>>,
     /// Provides monitoring of a set of explicitly defined validators.
     pub validator_monitor: RwLock<ValidatorMonitor<T::EthSpec>>,
+    pub blob_cache: BlobCache<T::EthSpec>,
 }
 
 type BeaconBlockAndState<T, Payload> = (BeaconBlock<T, Payload>, BeaconState<T>);
@@ -2360,8 +2362,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         };
 
         while let Some((_root, block)) = filtered_chain_segment.first() {
-            let block: &SignedBeaconBlock<T::EthSpec> = block.block();
-
             // Determine the epoch of the first block in the remaining segment.
             let start_epoch = block.slot().epoch(T::EthSpec::slots_per_epoch());
 
@@ -2449,7 +2449,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     let slot = block.slot();
                     let graffiti_string = block.message().body().graffiti().as_utf8_lossy();
 
-                    match GossipVerifiedBlock::new(block, &chain) {
+                    match GossipVerifiedBlock::new(block, blobs, &chain) {
                         Ok(verified) => {
                             debug!(
                                 chain.log,
@@ -2505,6 +2505,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // Increment the Prometheus counter for block processing requests.
         metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS);
 
+        let slot = unverified_block.block().slot();
+
         // A small closure to group the verification and import errors.
         let chain = self.clone();
         let import_block = async move {
@@ -2515,8 +2517,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 .await
         };
 
-        let slot = unverified_block.block().slot();
-
         // Verify and import the block.
         match import_block.await {
             // The block was successfully verified and imported. Yay.
@@ -2525,7 +2525,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     self.log,
                     "Beacon block imported";
                     "block_root" => ?block_root,
-                     "block_slot" => %block.slot(),
+                     "block_slot" => slot,
                 );
 
                 // Increment the Prometheus counter for block processing successes.
@@ -3693,6 +3693,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             prepare_payload_handle: _,
         } = partial_beacon_block;
 
+        let (payload, kzg_commitments_opt, blobs) = block_contents.deconstruct();
+
         let inner_block = match &state {
             BeaconState::Base(_) => BeaconBlock::Base(BeaconBlockBase {
                 slot,
@@ -3746,8 +3748,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     voluntary_exits: voluntary_exits.into(),
                     sync_aggregate: sync_aggregate
                         .ok_or(BlockProductionError::MissingSyncAggregate)?,
-                    execution_payload: block_contents
-                        .to_payload()
+                    execution_payload: payload
                         .try_into()
                         .map_err(|_| BlockProductionError::InvalidPayloadFork)?,
                 },
@@ -3768,16 +3769,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                     voluntary_exits: voluntary_exits.into(),
                     sync_aggregate: sync_aggregate
                         .ok_or(BlockProductionError::MissingSyncAggregate)?,
-                    execution_payload: block_contents
-                        .to_payload()
+                    execution_payload: payload
                         .try_into()
                         .map_err(|_| BlockProductionError::InvalidPayloadFork)?,
                 },
             }),
             BeaconState::Eip4844(_) => {
-                let kzg_commitments = block_contents
-                    .kzg_commitments()
-                    .ok_or(BlockProductionError::InvalidPayloadFork)?;
+                let kzg_commitments =
+                    kzg_commitments_opt.ok_or(BlockProductionError::InvalidPayloadFork)?;
                 BeaconBlock::Eip4844(BeaconBlockEip4844 {
                     slot,
                     proposer_index,
@@ -3794,11 +3793,10 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         voluntary_exits: voluntary_exits.into(),
                         sync_aggregate: sync_aggregate
                             .ok_or(BlockProductionError::MissingSyncAggregate)?,
-                        execution_payload: block_contents
-                            .to_payload()
+                        execution_payload: payload
                             .try_into()
                             .map_err(|_| BlockProductionError::InvalidPayloadFork)?,
-                        blob_kzg_commitments: VariableList::from(kzg_commitments.to_vec()),
+                        blob_kzg_commitments: VariableList::from(kzg_commitments),
                     },
                 })
             }
@@ -3828,8 +3826,13 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             ProduceBlockVerification::VerifyRandao => BlockSignatureStrategy::VerifyRandao,
             ProduceBlockVerification::NoVerification => BlockSignatureStrategy::NoVerification,
         };
+
         // Use a context without block root or proposer index so that both are checked.
-        let mut ctxt = ConsensusContext::new(block.slot());
+        let mut ctxt = ConsensusContext::new(block.slot())
+            //FIXME(sean) This is a hack beacuse `valdiate blobs sidecar requires the block root`
+            // which we won't have until after the state root is calculated.
+            .set_blobs_sidecar_validated(true);
+
         per_block_processing(
             &mut state,
             &block,
@@ -3847,6 +3850,20 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let (mut block, _) = block.deconstruct();
         *block.state_root_mut() = state_root;
 
+        //FIXME(sean)
+        // - generate kzg proof
+        // - validate blobs then cache them
+        if let Some(blobs) = blobs {
+            let beacon_block_root = block.canonical_root();
+            let blobs_sidecar = BlobsSidecar {
+                beacon_block_slot: slot,
+                beacon_block_root,
+                blobs: VariableList::from(blobs),
+                kzg_aggregate_proof: KzgProof::default(),
+            };
+            self.blob_cache.put(beacon_block_root, blobs_sidecar);
+        }
+
         metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES);
 
         trace!(

beacon_node/beacon_chain/src/blob_cache.rs

@@ -0,0 +1,32 @@
+use lru::LruCache;
+use parking_lot::Mutex;
+use tree_hash::TreeHash;
+use types::{BlobsSidecar, EthSpec, ExecutionPayload, Hash256};
+
+pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10;
+
+/// A cache blobs by beacon block root.
+pub struct BlobCache<T: EthSpec> {
+    blobs: Mutex<LruCache<BlobCacheId, BlobsSidecar<T>>>,
+}
+
+#[derive(Hash, PartialEq, Eq)]
+struct BlobCacheId(Hash256);
+
+impl<T: EthSpec> Default for BlobCache<T> {
+    fn default() -> Self {
+        BlobCache {
+            blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)),
+        }
+    }
+}
+
+impl<T: EthSpec> BlobCache<T> {
+    pub fn put(&self, beacon_block: Hash256, blobs: BlobsSidecar<T>) -> Option<BlobsSidecar<T>> {
+        self.blobs.lock().put(BlobCacheId(beacon_block), blobs)
+    }
+
+    pub fn pop(&self, root: &Hash256) -> Option<BlobsSidecar<T>> {
+        self.blobs.lock().pop(&BlobCacheId(*root))
+    }
+}

beacon_node/beacon_chain/src/blob_verification.rs

@@ -7,6 +7,7 @@ use crate::BeaconChainError;
 use bls::PublicKey;
 use types::{consts::eip4844::BLS_MODULUS, BeaconStateError, BlobsSidecar, Hash256, Slot};
 
+#[derive(Debug)]
 pub enum BlobError {
     /// The blob sidecar is from a slot that is later than the current slot (with respect to the
     /// gossip clock disparity).
@@ -82,7 +83,7 @@ impl From<BeaconStateError> for BlobError {
 
 pub fn validate_blob_for_gossip<T: BeaconChainTypes>(
     blob_sidecar: &BlobsSidecar<T::EthSpec>,
-    chain: &Arc<BeaconChain<T>>,
+    chain: &BeaconChain<T>,
 ) -> Result<(), BlobError> {
     let blob_slot = blob_sidecar.beacon_block_slot;
     // Do not gossip or process blobs from future or past slots.

beacon_node/beacon_chain/src/block_verification.rs

@@ -42,7 +42,7 @@
 //!            END
 //!
 //! ```
-use crate::blob_verification::validate_blob_for_gossip;
+use crate::blob_verification::{validate_blob_for_gossip, BlobError};
 use crate::eth1_finalization_cache::Eth1FinalizationData;
 use crate::execution_payload::{
     is_optimistic_candidate_block, validate_execution_payload_for_gossip, validate_merge_block,
@@ -51,6 +51,7 @@ use crate::execution_payload::{
 use crate::snapshot_cache::PreProcessingSnapshot;
 use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS;
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
+use crate::BlockError::BlobValidation;
 use crate::{
     beacon_chain::{
         BeaconForkChoice, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, MAXIMUM_GOSSIP_CLOCK_DISPARITY,
@@ -138,7 +139,10 @@ pub enum BlockError<T: EthSpec> {
     /// its parent.
     ParentUnknown(Arc<SignedBeaconBlock<T>>),
     /// The block skips too many slots and is a DoS risk.
-    TooManySkippedSlots { parent_slot: Slot, block_slot: Slot },
+    TooManySkippedSlots {
+        parent_slot: Slot,
+        block_slot: Slot,
+    },
     /// The block slot is greater than the present slot.
     ///
     /// ## Peer scoring
@@ -153,7 +157,10 @@ pub enum BlockError<T: EthSpec> {
     /// ## Peer scoring
     ///
     /// The peer has incompatible state transition logic and is faulty.
-    StateRootMismatch { block: Hash256, local: Hash256 },
+    StateRootMismatch {
+        block: Hash256,
+        local: Hash256,
+    },
     /// The block was a genesis block, these blocks cannot be re-imported.
     GenesisBlock,
     /// The slot is finalized, no need to import.
@@ -172,7 +179,9 @@ pub enum BlockError<T: EthSpec> {
     ///
     /// It's unclear if this block is valid, but it conflicts with finality and shouldn't be
     /// imported.
-    NotFinalizedDescendant { block_parent_root: Hash256 },
+    NotFinalizedDescendant {
+        block_parent_root: Hash256,
+    },
     /// Block is already known, no need to re-import.
     ///
     /// ## Peer scoring
@@ -185,7 +194,10 @@ pub enum BlockError<T: EthSpec> {
     ///
     /// The `proposer` has already proposed a block at this slot. The existing block may or may not
     /// be equal to the given block.
-    RepeatProposal { proposer: u64, slot: Slot },
+    RepeatProposal {
+        proposer: u64,
+        slot: Slot,
+    },
     /// The block slot exceeds the MAXIMUM_BLOCK_SLOT_NUMBER.
     ///
     /// ## Peer scoring
@@ -200,7 +212,10 @@ pub enum BlockError<T: EthSpec> {
     /// ## Peer scoring
     ///
     /// The block is invalid and the peer is faulty.
-    IncorrectBlockProposer { block: u64, local_shuffling: u64 },
+    IncorrectBlockProposer {
+        block: u64,
+        local_shuffling: u64,
+    },
     /// The proposal signature in invalid.
     ///
     /// ## Peer scoring
@@ -224,7 +239,10 @@ pub enum BlockError<T: EthSpec> {
     /// ## Peer scoring
     ///
     /// The block is invalid and the peer is faulty.
-    BlockIsNotLaterThanParent { block_slot: Slot, parent_slot: Slot },
+    BlockIsNotLaterThanParent {
+        block_slot: Slot,
+        parent_slot: Slot,
+    },
     /// At least one block in the chain segment did not have it's parent root set to the root of
     /// the prior block.
     ///
@@ -280,7 +298,10 @@ pub enum BlockError<T: EthSpec> {
     ///
     /// The peer sent us an invalid block, but I'm not really sure how to score this in an
     /// "optimistic" sync world.
-    ParentExecutionPayloadInvalid { parent_root: Hash256 },
+    ParentExecutionPayloadInvalid {
+        parent_root: Hash256,
+    },
+    BlobValidation(BlobError),
 }
 
 /// Returned when block validation failed due to some issue verifying
@@ -625,7 +646,7 @@ type PayloadVerificationHandle<E> =
 /// `BeaconChain` immediately after it is instantiated.
 pub struct ExecutionPendingBlock<T: BeaconChainTypes> {
     pub block: Arc<SignedBeaconBlock<T::EthSpec>>,
-    pub blobs: Option<Box<BlobsSidecar<T::EthSpec>>>,
+    pub blobs: Option<Arc<BlobsSidecar<T::EthSpec>>>,
     pub block_root: Hash256,
     pub state: BeaconState<T::EthSpec>,
     pub parent_block: SignedBeaconBlock<T::EthSpec, BlindedPayload<T::EthSpec>>,
@@ -884,7 +905,7 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
         validate_execution_payload_for_gossip(&parent_block, block.message(), chain)?;
 
         if let Some(blobs_sidecar) = blobs.as_ref() {
-            validate_blob_for_gossip(blobs_sidecar, chain)?;
+            validate_blob_for_gossip(blobs_sidecar, chain).map_err(BlobValidation)?;
             //FIXME(sean) validate blobs sidecar
         }
 
@@ -1058,7 +1079,7 @@ impl<T: BeaconChainTypes> IntoExecutionPendingBlock<T> for SignatureVerifiedBloc
 
         ExecutionPendingBlock::from_signature_verified_components(
             block,
-            self.consensus_context.blobs(),
+            self.consensus_context.blobs_sidecar(),
             block_root,
             parent,
             self.consensus_context,
@@ -1103,7 +1124,7 @@ impl<T: BeaconChainTypes> ExecutionPendingBlock<T> {
     /// Returns an error if the block is invalid, or if the block was unable to be verified.
     pub fn from_signature_verified_components(
         block: Arc<SignedBeaconBlock<T::EthSpec>>,
-        blobs: Option<Box<BlobsSidecar<T::EthSpec>>>,
+        blobs: Option<Arc<BlobsSidecar<T::EthSpec>>>,
         block_root: Hash256,
         parent: PreProcessingSnapshot<T::EthSpec>,
         mut consensus_context: ConsensusContext<T::EthSpec>,

beacon_node/beacon_chain/src/builder.rs

@@ -1,4 +1,5 @@
 use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY};
+use crate::blob_cache::BlobCache;
 use crate::eth1_chain::{CachingEth1Backend, SszEth1};
 use crate::eth1_finalization_cache::Eth1FinalizationCache;
 use crate::fork_choice_signal::ForkChoiceSignalTx;
@@ -810,6 +811,7 @@ where
             graffiti: self.graffiti,
             slasher: self.slasher.clone(),
             validator_monitor: RwLock::new(validator_monitor),
+            blob_cache: BlobCache::default(),
         };
 
         let head = beacon_chain.head_snapshot();

beacon_node/beacon_chain/src/lib.rs

@@ -5,6 +5,7 @@ mod beacon_chain;
 mod beacon_fork_choice_store;
 pub mod beacon_proposer_cache;
 mod beacon_snapshot;
+pub mod blob_cache;
 pub mod blob_verification;
 pub mod block_reward;
 mod block_times_cache;