Implement Overflow LRU Cache for Pending Blobs (#4203)

* All Necessary Objects Implement Encode/Decode

* Major Components for LRUOverflowCache Implemented

* Finish Database Code

* Add Maintenance Methods

* Added Maintenance Service

* Persist Blobs on Shutdown / Reload on Startup

* Address Clippy Complaints

* Add (emum_behaviour = "tag") to ssz_derive

* Convert Encode/Decode Implementations to "tag"

* Started Adding Tests

* Added a ton of tests

* 1 character fix

* Feature Guard Minimal Spec Tests

* Update beacon_node/beacon_chain/src/data_availability_checker.rs

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

* Address Sean's Comments

* Add iter_raw_keys method

* Remove TODOs

---------

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

beacon_node/store/src/leveldb_store.rs

@@ -198,6 +198,36 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
         )
     }
 
+    fn iter_raw_entries(&self, column: DBColumn, prefix: &[u8]) -> RawEntryIter {
+        let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix));
+
+        let iter = self.db.iter(self.read_options());
+        iter.seek(&start_key);
+
+        Box::new(
+            iter.take_while(move |(key, _)| key.key.starts_with(start_key.key.as_slice()))
+                .map(move |(bytes_key, value)| {
+                    let subkey = &bytes_key.key[column.as_bytes().len()..];
+                    Ok((Vec::from(subkey), value))
+                }),
+        )
+    }
+
+    fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter {
+        let start_key = BytesKey::from_vec(get_key_for_col(column.into(), prefix));
+
+        let iter = self.db.keys_iter(self.read_options());
+        iter.seek(&start_key);
+
+        Box::new(
+            iter.take_while(move |key| key.key.starts_with(start_key.key.as_slice()))
+                .map(move |bytes_key| {
+                    let subkey = &bytes_key.key[column.as_bytes().len()..];
+                    Ok(Vec::from(subkey))
+                }),
+        )
+    }
+
     /// Iterate through all keys and values in a particular column.
     fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter {
         let start_key =

beacon_node/store/src/lib.rs

@@ -49,6 +49,9 @@ pub use types::*;
 pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
 pub type ColumnKeyIter<'a> = Box<dyn Iterator<Item = Result<Hash256, Error>> + 'a>;
 
+pub type RawEntryIter<'a> = Box<dyn Iterator<Item = Result<(Vec<u8>, Vec<u8>), Error>> + 'a>;
+pub type RawKeyIter<'a> = Box<dyn Iterator<Item = Result<Vec<u8>, Error>> + 'a>;
+
 pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
     /// Retrieve some bytes in `column` with `key`.
     fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
@@ -88,6 +91,14 @@ pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
         Box::new(std::iter::empty())
     }
 
+    fn iter_raw_entries(&self, _column: DBColumn, _prefix: &[u8]) -> RawEntryIter {
+        Box::new(std::iter::empty())
+    }
+
+    fn iter_raw_keys(&self, _column: DBColumn, _prefix: &[u8]) -> RawKeyIter {
+        Box::new(std::iter::empty())
+    }
+
     /// Iterate through all keys in a particular column.
     fn iter_column_keys(&self, _column: DBColumn) -> ColumnKeyIter {
         // Default impl for non LevelDB databases
@@ -227,6 +238,8 @@ pub enum DBColumn {
     OptimisticTransitionBlock,
     #[strum(serialize = "bhs")]
     BeaconHistoricalSummaries,
+    #[strum(serialize = "olc")]
+    OverflowLRUCache,
 }
 
 /// A block from the database, which might have an execution payload or not.