Merge remote-tracking branch 'origin/deneb-free-blobs' into tree-states

beacon_node/beacon_chain/src/historical_blocks.rs

@@ -1,3 +1,4 @@
+use crate::data_availability_checker::AvailableBlock;
 use crate::{errors::BeaconChainError as Error, metrics, BeaconChain, BeaconChainTypes};
 use itertools::Itertools;
 use slog::debug;
@@ -7,10 +8,9 @@ use state_processing::{
 };
 use std::borrow::Cow;
 use std::iter;
-use std::sync::Arc;
 use std::time::Duration;
 use store::{get_key_for_col, AnchorInfo, DBColumn, KeyValueStore, KeyValueStoreOp};
-use types::{Hash256, SignedBlindedBeaconBlock, Slot};
+use types::{Hash256, Slot};
 
 /// Use a longer timeout on the pubkey cache.
 ///
@@ -59,27 +59,30 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
     /// Return the number of blocks successfully imported.
     pub fn import_historical_block_batch(
         &self,
-        blocks: Vec<Arc<SignedBlindedBeaconBlock<T::EthSpec>>>,
+        mut blocks: Vec<AvailableBlock<T::EthSpec>>,
     ) -> Result<usize, Error> {
         let anchor_info = self
             .store
             .get_anchor_info()
             .ok_or(HistoricalBlockError::NoAnchorInfo)?;
+        let blob_info = self.store.get_blob_info();
 
         // Take all blocks with slots less than the oldest block slot.
-        let num_relevant =
-            blocks.partition_point(|block| block.slot() < anchor_info.oldest_block_slot);
-        let blocks_to_import = &blocks
-            .get(..num_relevant)
-            .ok_or(HistoricalBlockError::IndexOutOfBounds)?;
+        let num_relevant = blocks.partition_point(|available_block| {
+            available_block.block().slot() < anchor_info.oldest_block_slot
+        });
 
-        if blocks_to_import.len() != blocks.len() {
+        let total_blocks = blocks.len();
+        blocks.truncate(num_relevant);
+        let blocks_to_import = blocks;
+
+        if blocks_to_import.len() != total_blocks {
             debug!(
                 self.log,
                 "Ignoring some historic blocks";
                 "oldest_block_slot" => anchor_info.oldest_block_slot,
-                "total_blocks" => blocks.len(),
-                "ignored" => blocks.len().saturating_sub(blocks_to_import.len()),
+                "total_blocks" => total_blocks,
+                "ignored" => total_blocks.saturating_sub(blocks_to_import.len()),
             );
         }
 
@@ -87,12 +90,23 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             return Ok(0);
         }
 
+        let n_blobs_lists_to_import = blocks_to_import
+            .iter()
+            .filter(|available_block| available_block.blobs().is_some())
+            .count();
+
         let mut expected_block_root = anchor_info.oldest_block_parent;
         let mut prev_block_slot = anchor_info.oldest_block_slot;
 
-        let mut cold_batch = Vec::with_capacity(blocks.len());
+        let mut new_oldest_blob_slot = blob_info.oldest_blob_slot;
+
+        let mut cold_batch = Vec::with_capacity(blocks_to_import.len());
+        let mut hot_batch = Vec::with_capacity(blocks_to_import.len() + n_blobs_lists_to_import);
+        let mut signed_blocks = Vec::with_capacity(blocks_to_import.len());
+
+        for available_block in blocks_to_import.into_iter().rev() {
+            let (block, maybe_blobs) = available_block.deconstruct();
 
-        for block in blocks_to_import.iter().rev() {
             // Check chain integrity.
             let block_root = block.canonical_root();
 
@@ -104,9 +118,19 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 .into());
             }
 
-            // Store block in the hot database without payload.
-            self.store
-                .blinded_block_as_cold_kv_store_ops(&block_root, block, &mut cold_batch)?;
+            let blinded_block = block.clone_as_blinded();
+            // Store block in the freezer database without payload.
+            self.store.blinded_block_as_cold_kv_store_ops(
+                &block_root,
+                &blinded_block,
+                &mut cold_batch,
+            )?;
+            // Store the blobs too
+            if let Some(blobs) = maybe_blobs {
+                new_oldest_blob_slot = Some(block.slot());
+                self.store
+                    .blobs_as_kv_store_ops(&block_root, blobs, &mut hot_batch);
+            }
 
             // Store block roots, including at all skip slots in the freezer DB.
             for slot in (block.slot().as_usize() + 1..prev_block_slot.as_usize()).rev() {
@@ -134,7 +158,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                 expected_block_root = Hash256::zero();
                 break;
             }
+            signed_blocks.push(block);
         }
+        chunk_writer.write(&mut cold_batch)?;
+        // these were pushed in reverse order so we reverse again
+        signed_blocks.reverse();
 
         // Verify signatures in one batch, holding the pubkey cache lock for the shortest duration
         // possible. For each block fetch the parent root from its successor. Slicing from index 1
@@ -145,25 +173,29 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .validator_pubkey_cache
             .try_read_for(PUBKEY_CACHE_LOCK_TIMEOUT)
             .ok_or(HistoricalBlockError::ValidatorPubkeyCacheTimeout)?;
-        let block_roots = blocks_to_import
+        let block_roots = signed_blocks
             .get(1..)
             .ok_or(HistoricalBlockError::IndexOutOfBounds)?
             .iter()
             .map(|block| block.parent_root())
             .chain(iter::once(anchor_info.oldest_block_parent));
-        let signature_set = blocks_to_import
+        let signature_set = signed_blocks
             .iter()
             .zip_eq(block_roots)
-            .map(|(block, block_root)| {
-                block_proposal_signature_set_from_parts(
-                    block,
-                    Some(block_root),
-                    block.message().proposer_index(),
-                    &self.spec.fork_at_epoch(block.message().epoch()),
-                    self.genesis_validators_root,
-                    |validator_index| pubkey_cache.get(validator_index).cloned().map(Cow::Owned),
-                    &self.spec,
-                )
+            .filter_map(|(block, block_root)| {
+                (block_root != self.genesis_block_root).then(|| {
+                    block_proposal_signature_set_from_parts(
+                        block,
+                        Some(block_root),
+                        block.message().proposer_index(),
+                        &self.spec.fork_at_epoch(block.message().epoch()),
+                        self.genesis_validators_root,
+                        |validator_index| {
+                            pubkey_cache.get(validator_index).cloned().map(Cow::Owned)
+                        },
+                        &self.spec,
+                    )
+                })
             })
             .collect::<Result<Vec<_>, _>>()
             .map_err(HistoricalBlockError::SignatureSet)
@@ -181,6 +213,22 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // Write the I/O batch to disk.
         self.store.cold_db.do_atomically(cold_batch)?;
 
+        let mut anchor_and_blob_batch = Vec::with_capacity(2);
+
+        // Update the blob info.
+        if new_oldest_blob_slot != blob_info.oldest_blob_slot {
+            if let Some(oldest_blob_slot) = new_oldest_blob_slot {
+                let new_blob_info = BlobInfo {
+                    oldest_blob_slot: Some(oldest_blob_slot),
+                    ..blob_info.clone()
+                };
+                anchor_and_blob_batch.push(
+                    self.store
+                        .compare_and_set_blob_info(blob_info, new_blob_info)?,
+                );
+            }
+        }
+
         // Update the anchor.
         let new_anchor = AnchorInfo {
             oldest_block_slot: prev_block_slot,
@@ -188,8 +236,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             ..anchor_info
         };
         let backfill_complete = new_anchor.block_backfill_complete(self.genesis_backfill_slot);
-        self.store
-            .compare_and_set_anchor_info_with_write(Some(anchor_info), Some(new_anchor))?;
+        anchor_and_blob_batch.push(
+            self.store
+                .compare_and_set_anchor_info(Some(anchor_info), Some(new_anchor))?,
+        );
+        self.store.hot_db.do_atomically(anchor_and_blob_batch)?;
 
         // If backfill has completed and the chain is configured to reconstruct historic states,
         // send a message to the background migrator instructing it to begin reconstruction.
@@ -201,6 +252,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             self.store_migrator.process_reconstruction();
         }
 
-        Ok(blocks_to_import.len())
+        Ok(num_relevant)
     }
 }