Modularize beacon node backend (#4718)

#4669


  Modularize the beacon node backend to make it easier to add new database implementations

beacon_node/store/src/memory_store.rs

@@ -1,9 +1,9 @@
 use crate::{
-    get_key_for_col, leveldb_store::BytesKey, ColumnIter, ColumnKeyIter, DBColumn, Error,
-    ItemStore, Key, KeyValueStore, KeyValueStoreOp, RawKeyIter,
+    errors::Error as DBError, get_key_for_col, hot_cold_store::BytesKey, ColumnIter, ColumnKeyIter,
+    DBColumn, Error, ItemStore, Key, KeyValueStore, KeyValueStoreOp,
 };
 use parking_lot::{Mutex, MutexGuard, RwLock};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashSet};
 use std::marker::PhantomData;
 use types::*;
 
@@ -29,19 +29,19 @@ impl<E: EthSpec> MemoryStore<E> {
 
 impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     /// Get the value of some key from the database. Returns `None` if the key does not exist.
-    fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
+    fn get_bytes(&self, col: DBColumn, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
         let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         Ok(self.db.read().get(&column_key).cloned())
     }
 
     /// Puts a key in the database.
-    fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
+    fn put_bytes(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> {
         let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         self.db.write().insert(column_key, val.to_vec());
         Ok(())
     }
 
-    fn put_bytes_sync(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
+    fn put_bytes_sync(&self, col: DBColumn, key: &[u8], val: &[u8]) -> Result<(), Error> {
         self.put_bytes(col, key, val)
     }
 
@@ -51,13 +51,13 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     }
 
     /// Return true if some key exists in some column.
-    fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
+    fn key_exists(&self, col: DBColumn, key: &[u8]) -> Result<bool, Error> {
         let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         Ok(self.db.read().contains_key(&column_key))
     }
 
     /// Delete some key from the database.
-    fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
+    fn key_delete(&self, col: DBColumn, key: &[u8]) -> Result<(), Error> {
         let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         self.db.write().remove(&column_key);
         Ok(())
@@ -66,12 +66,16 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     fn do_atomically(&self, batch: Vec<KeyValueStoreOp>) -> Result<(), Error> {
         for op in batch {
             match op {
-                KeyValueStoreOp::PutKeyValue(key, value) => {
-                    self.db.write().insert(BytesKey::from_vec(key), value);
+                KeyValueStoreOp::PutKeyValue(col, key, value) => {
+                    let column_key = get_key_for_col(col, &key);
+                    self.db
+                        .write()
+                        .insert(BytesKey::from_vec(column_key), value);
                 }
 
-                KeyValueStoreOp::DeleteKey(key) => {
-                    self.db.write().remove(&BytesKey::from_vec(key));
+                KeyValueStoreOp::DeleteKey(col, key) => {
+                    let column_key = get_key_for_col(col, &key);
+                    self.db.write().remove(&BytesKey::from_vec(column_key));
                 }
             }
         }
@@ -82,8 +86,7 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
         // We use this awkward pattern because we can't lock the `self.db` field *and* maintain a
         // reference to the lock guard across calls to `.next()`. This would be require a
         // struct with a field (the iterator) which references another field (the lock guard).
-        let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), from));
-        let col = column.as_str();
+        let start_key = BytesKey::from_vec(get_key_for_col(column, from));
         let keys = self
             .db
             .read()
@@ -92,7 +95,7 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
             .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec()))
             .collect::<Vec<_>>();
         Box::new(keys.into_iter().filter_map(move |key| {
-            self.get_bytes(col, &key).transpose().map(|res| {
+            self.get_bytes(column, &key).transpose().map(|res| {
                 let k = K::from_bytes(&key)?;
                 let v = res?;
                 Ok((k, v))
@@ -100,18 +103,6 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
         }))
     }
 
-    fn iter_raw_keys(&self, column: DBColumn, prefix: &[u8]) -> RawKeyIter {
-        let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), prefix));
-        let keys = self
-            .db
-            .read()
-            .range(start_key.clone()..)
-            .take_while(|(k, _)| k.starts_with(&start_key))
-            .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec()))
-            .collect::<Vec<_>>();
-        Box::new(keys.into_iter().map(Ok))
-    }
-
     fn iter_column_keys<K: Key>(&self, column: DBColumn) -> ColumnKeyIter<K> {
         Box::new(self.iter_column(column).map(|res| res.map(|(k, _)| k)))
     }
@@ -123,6 +114,44 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     fn compact_column(&self, _column: DBColumn) -> Result<(), Error> {
         Ok(())
     }
+
+    fn iter_column_keys_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnKeyIter<K> {
+        // We use this awkward pattern because we can't lock the `self.db` field *and* maintain a
+        // reference to the lock guard across calls to `.next()`. This would be require a
+        // struct with a field (the iterator) which references another field (the lock guard).
+        let start_key = BytesKey::from_vec(get_key_for_col(column, from));
+        let keys = self
+            .db
+            .read()
+            .range(start_key..)
+            .take_while(|(k, _)| k.remove_column_variable(column).is_some())
+            .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec()))
+            .collect::<Vec<_>>();
+        Box::new(keys.into_iter().map(move |key| K::from_bytes(&key)))
+    }
+
+    fn delete_batch(&self, col: DBColumn, ops: HashSet<&[u8]>) -> Result<(), DBError> {
+        for op in ops {
+            let column_key = get_key_for_col(col, op);
+            self.db.write().remove(&BytesKey::from_vec(column_key));
+        }
+        Ok(())
+    }
+
+    fn delete_if(
+        &self,
+        column: DBColumn,
+        mut f: impl FnMut(&[u8]) -> Result<bool, Error>,
+    ) -> Result<(), Error> {
+        self.db.write().retain(|key, value| {
+            if key.remove_column_variable(column).is_some() {
+                !f(value).unwrap_or(false)
+            } else {
+                true
+            }
+        });
+        Ok(())
+    }
 }
 
 impl<E: EthSpec> ItemStore<E> for MemoryStore<E> {}