Merge remote-tracking branch 'origin/unstable' into tree-states

beacon_node/store/src/hot_cold_store.rs

@@ -636,7 +636,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         for op in batch {
             match op {
                 StoreOp::PutBlock(block_root, block) => {
-                    self.block_as_kv_store_ops(&block_root, *block, &mut key_value_batch)?;
+                    self.block_as_kv_store_ops(
+                        &block_root,
+                        block.as_ref().clone(),
+                        &mut key_value_batch,
+                    )?;
                 }
 
                 StoreOp::PutState(state_root, state) => {
@@ -1522,7 +1526,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     }
 
     /// Load a frozen state's slot, given its root.
-    fn load_cold_state_slot(&self, state_root: &Hash256) -> Result<Option<Slot>, Error> {
+    pub fn load_cold_state_slot(&self, state_root: &Hash256) -> Result<Option<Slot>, Error> {
         Ok(self
             .cold_db
             .get(state_root)?
@@ -1808,6 +1812,7 @@ impl StoreItem for Split {
 /// Struct for summarising a state in the hot database.
 ///
 /// Allows full reconstruction by replaying blocks.
+// FIXME(sproul): change to V20
 #[superstruct(
     variants(V1, V10),
     variant_attributes(derive(Debug, Clone, Copy, Default, Encode, Decode)),

beacon_node/store/src/iter.rs

@@ -212,7 +212,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
             (Err(BeaconStateError::SlotOutOfBounds), Err(BeaconStateError::SlotOutOfBounds)) => {
                 // Read a `BeaconState` from the store that has access to prior historical roots.
                 if let Some(beacon_state) =
-                    next_historical_root_backtrack_state(&*self.store, &self.beacon_state)
+                    next_historical_root_backtrack_state(self.store, &self.beacon_state)
                         .handle_unavailable()?
                 {
                     self.beacon_state = Cow::Owned(beacon_state);

beacon_node/store/src/lib.rs

@@ -42,6 +42,7 @@ pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer;
 pub use metadata::AnchorInfo;
 pub use metrics::scrape_for_metrics;
 use parking_lot::MutexGuard;
+use std::sync::Arc;
 use strum::{EnumString, IntoStaticStr};
 pub use types::*;
 
@@ -155,7 +156,7 @@ pub trait ItemStore<E: EthSpec>: KeyValueStore<E> + Sync + Send + Sized + 'stati
 /// Reified key-value storage operation.  Helps in modifying the storage atomically.
 /// See also https://github.com/sigp/lighthouse/issues/692
 pub enum StoreOp<'a, E: EthSpec> {
-    PutBlock(Hash256, Box<SignedBeaconBlock<E>>),
+    PutBlock(Hash256, Arc<SignedBeaconBlock<E>>),
     PutState(Hash256, &'a BeaconState<E>),
     PutStateTemporaryFlag(Hash256),
     DeleteStateTemporaryFlag(Hash256),
@@ -213,6 +214,9 @@ pub enum DBColumn {
     BeaconRandaoMixes,
     #[strum(serialize = "dht")]
     DhtEnrs,
+    /// For Optimistically Imported Merge Transition Blocks
+    #[strum(serialize = "otb")]
+    OptimisticTransitionBlock,
 }
 
 /// A block from the database, which might have an execution payload or not.

beacon_node/store/src/memory_store.rs

@@ -1,14 +1,17 @@
 use super::{Error, ItemStore, KeyValueStore, KeyValueStoreOp};
+use crate::{ColumnIter, DBColumn};
 use parking_lot::{Mutex, MutexGuard, RwLock};
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::marker::PhantomData;
 use types::*;
 
 type DBHashMap = HashMap<Vec<u8>, Vec<u8>>;
+type DBKeyMap = HashMap<Vec<u8>, HashSet<Vec<u8>>>;
 
 /// A thread-safe `HashMap` wrapper.
 pub struct MemoryStore<E: EthSpec> {
     db: RwLock<DBHashMap>,
+    col_keys: RwLock<DBKeyMap>,
     transaction_mutex: Mutex<()>,
     _phantom: PhantomData<E>,
 }
@@ -18,6 +21,7 @@ impl<E: EthSpec> MemoryStore<E> {
     pub fn open() -> Self {
         Self {
             db: RwLock::new(HashMap::new()),
+            col_keys: RwLock::new(HashMap::new()),
             transaction_mutex: Mutex::new(()),
             _phantom: PhantomData,
         }
@@ -41,6 +45,11 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
         let column_key = Self::get_key_for_col(col, key);
         self.db.write().insert(column_key, val.to_vec());
+        self.col_keys
+            .write()
+            .entry(col.as_bytes().to_vec())
+            .or_insert_with(HashSet::new)
+            .insert(key.to_vec());
         Ok(())
     }
 
@@ -63,6 +72,10 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
     fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
         let column_key = Self::get_key_for_col(col, key);
         self.db.write().remove(&column_key);
+        self.col_keys
+            .write()
+            .get_mut(&col.as_bytes().to_vec())
+            .map(|set| set.remove(key));
         Ok(())
     }
 
@@ -81,6 +94,26 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
         Ok(())
     }
 
+    // pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
+    fn iter_column(&self, column: DBColumn) -> ColumnIter {
+        let col = column.as_str();
+        if let Some(keys) = self
+            .col_keys
+            .read()
+            .get(col.as_bytes())
+            .map(|set| set.iter().cloned().collect::<Vec<_>>())
+        {
+            Box::new(keys.into_iter().filter_map(move |key| {
+                let hash = Hash256::from_slice(&key);
+                self.get_bytes(col, &key)
+                    .transpose()
+                    .map(|res| res.map(|bytes| (hash, bytes)))
+            }))
+        } else {
+            Box::new(std::iter::empty())
+        }
+    }
+
     fn begin_rw_transaction(&self) -> MutexGuard<()> {
         self.transaction_mutex.lock()
     }