Make max_blobs_per_block a config parameter (#6329)

* First pass

* Add restrictions to RuntimeVariableList api

* Use empty_uninitialized and fix warnings

* Fix some todos

* Merge branch 'unstable' into max-blobs-preset

* Fix take impl on RuntimeFixedList

* cleanup

* Fix test compilations

* Fix some more tests

* Fix test from unstable

* Merge branch 'unstable' into max-blobs-preset

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset

* Remove footgun function

* Minor simplifications

* Move from preset to config

* Fix typo

* Revert "Remove footgun function"

This reverts commit de01f923c7.

* Try fixing tests

* Thread through ChainSpec

* Fix release tests

* Move RuntimeFixedVector into module and rename

* Add test

* Remove empty RuntimeVarList awefullness

* Fix tests

* Simplify BlobSidecarListFromRoot

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset

* Bump quota to account for new target (6)

* Remove clone

* Fix issue from review

* Try to remove ugliness

* Merge branch 'unstable' into max-blobs-preset

* Fix max value

* Fix doctest

* Fix formatting

* Fix max check

* Delete hardcoded max_blobs_per_block in RPC limits

* Merge remote-tracking branch 'origin/unstable' into max-blobs-preset

beacon_node/store/src/blob_sidecar_list_from_root.rs

@@ -0,0 +1,42 @@
+use std::sync::Arc;
+use types::{BlobSidecar, BlobSidecarList, EthSpec};
+
+#[derive(Debug, Clone)]
+pub enum BlobSidecarListFromRoot<E: EthSpec> {
+    /// Valid root that exists in the DB, but has no blobs associated with it.
+    NoBlobs,
+    /// Contains > 1 blob for the requested root.
+    Blobs(BlobSidecarList<E>),
+    /// No root exists in the db or cache for the requested root.
+    NoRoot,
+}
+
+impl<E: EthSpec> From<BlobSidecarList<E>> for BlobSidecarListFromRoot<E> {
+    fn from(value: BlobSidecarList<E>) -> Self {
+        Self::Blobs(value)
+    }
+}
+
+impl<E: EthSpec> BlobSidecarListFromRoot<E> {
+    pub fn blobs(self) -> Option<BlobSidecarList<E>> {
+        match self {
+            Self::NoBlobs | Self::NoRoot => None,
+            Self::Blobs(blobs) => Some(blobs),
+        }
+    }
+
+    #[allow(clippy::len_without_is_empty)]
+    pub fn len(&self) -> usize {
+        match self {
+            Self::NoBlobs | Self::NoRoot => 0,
+            Self::Blobs(blobs) => blobs.len(),
+        }
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &Arc<BlobSidecar<E>>> {
+        match self {
+            Self::NoBlobs | Self::NoRoot => [].iter(),
+            Self::Blobs(list) => list.iter(),
+        }
+    }
+}

beacon_node/store/src/hot_cold_store.rs

@@ -14,8 +14,8 @@ use crate::metadata::{
 };
 use crate::state_cache::{PutStateOutcome, StateCache};
 use crate::{
-    get_data_column_key, get_key_for_col, DBColumn, DatabaseBlock, Error, ItemStore,
-    KeyValueStoreOp, StoreItem, StoreOp,
+    get_data_column_key, get_key_for_col, BlobSidecarListFromRoot, DBColumn, DatabaseBlock, Error,
+    ItemStore, KeyValueStoreOp, StoreItem, StoreOp,
 };
 use crate::{metrics, parse_data_column_key};
 use itertools::{process_results, Itertools};
@@ -1280,9 +1280,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
                 StoreOp::PutBlobs(_, _) | StoreOp::PutDataColumns(_, _) => true,
                 StoreOp::DeleteBlobs(block_root) => {
                     match self.get_blobs(block_root) {
-                        Ok(Some(blob_sidecar_list)) => {
+                        Ok(BlobSidecarListFromRoot::Blobs(blob_sidecar_list)) => {
                             blobs_to_delete.push((*block_root, blob_sidecar_list));
                         }
+                        Ok(BlobSidecarListFromRoot::NoBlobs | BlobSidecarListFromRoot::NoRoot) => {}
                         Err(e) => {
                             error!(
                                 self.log, "Error getting blobs";
@@ -1290,7 +1291,6 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
                                 "error" => ?e
                             );
                         }
-                        _ => (),
                     }
                     true
                 }
@@ -2045,11 +2045,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     }
 
     /// Fetch blobs for a given block from the store.
-    pub fn get_blobs(&self, block_root: &Hash256) -> Result<Option<BlobSidecarList<E>>, Error> {
+    pub fn get_blobs(&self, block_root: &Hash256) -> Result<BlobSidecarListFromRoot<E>, Error> {
         // Check the cache.
         if let Some(blobs) = self.block_cache.lock().get_blobs(block_root) {
             metrics::inc_counter(&metrics::BEACON_BLOBS_CACHE_HIT_COUNT);
-            return Ok(Some(blobs.clone()));
+            return Ok(blobs.clone().into());
         }
 
         match self
@@ -2057,13 +2057,27 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
             .get_bytes(DBColumn::BeaconBlob.into(), block_root.as_slice())?
         {
             Some(ref blobs_bytes) => {
-                let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?;
-                self.block_cache
-                    .lock()
-                    .put_blobs(*block_root, blobs.clone());
-                Ok(Some(blobs))
+                // We insert a VariableList of BlobSidecars into the db, but retrieve
+                // a plain vec since we don't know the length limit of the list without
+                // knowing the slot.
+                // The encoding of a VariableList is the same as a regular vec.
+                let blobs: Vec<Arc<BlobSidecar<E>>> = Vec::<_>::from_ssz_bytes(blobs_bytes)?;
+                if let Some(max_blobs_per_block) = blobs
+                    .first()
+                    .map(|blob| self.spec.max_blobs_per_block(blob.epoch()))
+                {
+                    let blobs = BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize);
+                    self.block_cache
+                        .lock()
+                        .put_blobs(*block_root, blobs.clone());
+
+                    Ok(BlobSidecarListFromRoot::Blobs(blobs))
+                } else {
+                    // This always implies that there were no blobs for this block_root
+                    Ok(BlobSidecarListFromRoot::NoBlobs)
+                }
             }
-            None => Ok(None),
+            None => Ok(BlobSidecarListFromRoot::NoRoot),
         }
     }
 

beacon_node/store/src/impls/execution_payload.rs

@@ -1,7 +1,7 @@
 use crate::{DBColumn, Error, StoreItem};
 use ssz::{Decode, Encode};
 use types::{
-    BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella,
+    EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella,
     ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadFulu,
 };
 
@@ -27,7 +27,6 @@ impl_store_item!(ExecutionPayloadCapella);
 impl_store_item!(ExecutionPayloadDeneb);
 impl_store_item!(ExecutionPayloadElectra);
 impl_store_item!(ExecutionPayloadFulu);
-impl_store_item!(BlobSidecarList);
 
 /// This fork-agnostic implementation should be only used for writing.
 ///

beacon_node/store/src/lib.rs

@@ -7,6 +7,7 @@
 //!
 //! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
 //! tests for implementation examples.
+pub mod blob_sidecar_list_from_root;
 pub mod chunked_iter;
 pub mod chunked_vector;
 pub mod config;
@@ -28,6 +29,7 @@ pub mod state_cache;
 
 pub mod iter;
 
+pub use self::blob_sidecar_list_from_root::BlobSidecarListFromRoot;
 pub use self::config::StoreConfig;
 pub use self::consensus_context::OnDiskConsensusContext;
 pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split};