Fix failing tests (#4423)

* Get tests passing

* Get benchmarks compiling

* Fix EF withdrawals test

* Remove unused deps

* Fix tree_hash panic in tests

* Fix slasher compilation

* Fix ssz_generic test

* Get more tests passing

* Fix EF tests for real

* Fix local testnet scripts

beacon_node/store/src/memory_store.rs

@@ -1,16 +1,17 @@
-use crate::{ColumnIter, DBColumn, Error, ItemStore, Key, KeyValueStore, KeyValueStoreOp};
+use crate::{
+    get_key_for_col, leveldb_store::BytesKey, ColumnIter, ColumnKeyIter, DBColumn, Error,
+    ItemStore, Key, KeyValueStore, KeyValueStoreOp,
+};
 use parking_lot::{Mutex, MutexGuard, RwLock};
-use std::collections::{HashMap, HashSet};
+use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use types::*;
 
-type DBHashMap = HashMap<Vec<u8>, Vec<u8>>;
-type DBKeyMap = HashMap<Vec<u8>, HashSet<Vec<u8>>>;
+type DBMap = BTreeMap<BytesKey, Vec<u8>>;
 
-/// A thread-safe `HashMap` wrapper.
+/// A thread-safe `BTreeMap` wrapper.
 pub struct MemoryStore<E: EthSpec> {
-    db: RwLock<DBHashMap>,
-    col_keys: RwLock<DBKeyMap>,
+    db: RwLock<DBMap>,
     transaction_mutex: Mutex<()>,
     _phantom: PhantomData<E>,
 }
@@ -19,36 +20,24 @@ impl<E: EthSpec> MemoryStore<E> {
     /// Create a new, empty database.
     pub fn open() -> Self {
         Self {
-            db: RwLock::new(HashMap::new()),
-            col_keys: RwLock::new(HashMap::new()),
+            db: RwLock::new(BTreeMap::new()),
             transaction_mutex: Mutex::new(()),
             _phantom: PhantomData,
         }
     }
-
-    fn get_key_for_col(col: &str, key: &[u8]) -> Vec<u8> {
-        let mut col = col.as_bytes().to_vec();
-        col.append(&mut key.to_vec());
-        col
-    }
 }
 
 impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     /// Get the value of some key from the database. Returns `None` if the key does not exist.
     fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
-        let column_key = Self::get_key_for_col(col, key);
+        let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         Ok(self.db.read().get(&column_key).cloned())
     }
 
     /// Puts a key in the database.
     fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
-        let column_key = Self::get_key_for_col(col, key);
+        let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         self.db.write().insert(column_key, val.to_vec());
-        self.col_keys
-            .write()
-            .entry(col.as_bytes().to_vec())
-            .or_insert_with(HashSet::new)
-            .insert(key.to_vec());
         Ok(())
     }
 
@@ -63,18 +52,14 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
 
     /// Return true if some key exists in some column.
     fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
-        let column_key = Self::get_key_for_col(col, key);
+        let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         Ok(self.db.read().contains_key(&column_key))
     }
 
     /// Delete some key from the database.
     fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
-        let column_key = Self::get_key_for_col(col, key);
+        let column_key = BytesKey::from_vec(get_key_for_col(col, key));
         self.db.write().remove(&column_key);
-        self.col_keys
-            .write()
-            .get_mut(&col.as_bytes().to_vec())
-            .map(|set| set.remove(key));
         Ok(())
     }
 
@@ -82,35 +67,41 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
         for op in batch {
             match op {
                 KeyValueStoreOp::PutKeyValue(key, value) => {
-                    self.db.write().insert(key, value);
+                    self.db.write().insert(BytesKey::from_vec(key), value);
                 }
 
-                KeyValueStoreOp::DeleteKey(hash) => {
-                    self.db.write().remove(&hash);
+                KeyValueStoreOp::DeleteKey(key) => {
+                    self.db.write().remove(&BytesKey::from_vec(key));
                 }
             }
         }
         Ok(())
     }
 
-    fn iter_column<K: Key>(&self, column: DBColumn) -> ColumnIter<K> {
+    fn iter_column_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnIter<K> {
+        // We use this awkward pattern because we can't lock the `self.db` field *and* maintain a
+        // reference to the lock guard across calls to `.next()`. This would be require a
+        // struct with a field (the iterator) which references another field (the lock guard).
+        let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), from));
         let col = column.as_str();
-        if let Some(keys) = self
-            .col_keys
+        let keys = self
+            .db
             .read()
-            .get(col.as_bytes())
-            .map(|set| set.iter().cloned().collect::<Vec<_>>())
-        {
-            Box::new(keys.into_iter().filter_map(move |key| {
-                self.get_bytes(col, &key).transpose().map(|res| {
-                    let k = K::from_bytes(&key)?;
-                    let v = res?;
-                    Ok((k, v))
-                })
-            }))
-        } else {
-            Box::new(std::iter::empty())
-        }
+            .range(start_key..)
+            .take_while(|(k, _)| k.remove_column_variable(column).is_some())
+            .filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec()))
+            .collect::<Vec<_>>();
+        Box::new(keys.into_iter().filter_map(move |key| {
+            self.get_bytes(col, &key).transpose().map(|res| {
+                let k = K::from_bytes(&key)?;
+                let v = res?;
+                Ok((k, v))
+            })
+        }))
+    }
+
+    fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter {
+        Box::new(self.iter_column(column).map(|res| res.map(|(k, _)| k)))
     }
 
     fn begin_rw_transaction(&self) -> MutexGuard<()> {