Implement freezer database (#508)

* Implement freezer database for state vectors

* Improve BeaconState safe accessors

And fix a bug in the compact committees accessor.

* Banish dodgy type bounds back to gRPC

* Clean up

* Switch to exclusive end points in chunked vec

* Cleaning up and start of tests

* Randao fix, more tests

* Fix unsightly hack

* Resolve test FIXMEs

* Config file support

* More clean-ups, migrator beginnings

* Finish migrator, integrate into BeaconChain

* Fixups

* Fix store tests

* Fix BeaconChain tests

* Fix LMD GHOST tests

* Address review comments, delete 'static bounds

* Cargo format

* Address review comments

* Fix LMD ghost tests

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

* Update for new builder paradigm, spec changes

* Freezer DB integration into BeaconNode

* Cleaning up

* This works, clean it up

* Cleanups

* Fix and improve store tests

* Refine store test

* Delete unused beacon_chain_builder.rs

* Fix CLI

* Store state at split slot in hot database

* Make fork choice lookup fast again

* Store freezer DB split slot in the database

* Handle potential div by 0 in chunked_vector

* Exclude committee caches from freezer DB

* Remove FIXME about long-running test

beacon_node/store/src/lib.rs

@@ -11,16 +11,25 @@
 extern crate lazy_static;
 
 mod block_at_slot;
+pub mod chunked_vector;
 mod errors;
+mod hot_cold_store;
 mod impls;
 mod leveldb_store;
 mod memory_store;
 mod metrics;
+mod partial_beacon_state;
 
 pub mod iter;
+pub mod migrate;
 
-pub use self::leveldb_store::LevelDB as DiskStore;
+use std::sync::Arc;
+
+pub use self::hot_cold_store::HotColdDB as DiskStore;
+pub use self::leveldb_store::LevelDB as SimpleDiskStore;
 pub use self::memory_store::MemoryStore;
+pub use self::migrate::Migrate;
+pub use self::partial_beacon_state::PartialBeaconState;
 pub use errors::Error;
 pub use metrics::scrape_for_metrics;
 pub use types::*;
@@ -30,9 +39,21 @@ pub use types::*;
 /// A `Store` is fundamentally backed by a key-value database, however it provides support for
 /// columns. A simple column implementation might involve prefixing a key with some bytes unique to
 /// each column.
-pub trait Store: Sync + Send + Sized {
+pub trait Store: Sync + Send + Sized + 'static {
+    /// Retrieve some bytes in `column` with `key`.
+    fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
+
+    /// Store some `value` in `column`, indexed with `key`.
+    fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>;
+
+    /// Return `true` if `key` exists in `column`.
+    fn key_exists(&self, column: &str, key: &[u8]) -> Result<bool, Error>;
+
+    /// Removes `key` from `column`.
+    fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>;
+
     /// Store an item in `Self`.
-    fn put(&self, key: &Hash256, item: &impl StoreItem) -> Result<(), Error> {
+    fn put<I: StoreItem>(&self, key: &Hash256, item: &I) -> Result<(), Error> {
         item.db_put(self, key)
     }
 
@@ -51,6 +72,20 @@ pub trait Store: Sync + Send + Sized {
         I::db_delete(self, key)
     }
 
+    /// Store a state in the store.
+    fn put_state<E: EthSpec>(
+        &self,
+        state_root: &Hash256,
+        state: &BeaconState<E>,
+    ) -> Result<(), Error>;
+
+    /// Fetch a state from the store.
+    fn get_state<E: EthSpec>(
+        &self,
+        state_root: &Hash256,
+        slot: Option<Slot>,
+    ) -> Result<Option<BeaconState<E>>, Error>;
+
     /// Given the root of an existing block in the store (`start_block_root`), return a parent
     /// block with the specified `slot`.
     ///
@@ -64,42 +99,48 @@ pub trait Store: Sync + Send + Sized {
         block_at_slot::get_block_at_preceeding_slot::<_, E>(self, slot, start_block_root)
     }
 
-    /// Retrieve some bytes in `column` with `key`.
-    fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
-
-    /// Store some `value` in `column`, indexed with `key`.
-    fn put_bytes(&self, column: &str, key: &[u8], value: &[u8]) -> Result<(), Error>;
-
-    /// Return `true` if `key` exists in `column`.
-    fn key_exists(&self, column: &str, key: &[u8]) -> Result<bool, Error>;
-
-    /// Removes `key` from `column`.
-    fn key_delete(&self, column: &str, key: &[u8]) -> Result<(), Error>;
+    /// (Optionally) Move all data before the frozen slot to the freezer database.
+    fn freeze_to_state<E: EthSpec>(
+        _store: Arc<Self>,
+        _frozen_head_root: Hash256,
+        _frozen_head: &BeaconState<E>,
+    ) -> Result<(), Error> {
+        Ok(())
+    }
 }
 
 /// A unique column identifier.
+#[derive(Debug, Clone, Copy, PartialEq)]
 pub enum DBColumn {
+    /// For data related to the database itself.
+    BeaconMeta,
     BeaconBlock,
     BeaconState,
     BeaconChain,
+    BeaconBlockRoots,
+    BeaconStateRoots,
+    BeaconHistoricalRoots,
+    BeaconRandaoMixes,
 }
 
-impl<'a> Into<&'a str> for DBColumn {
+impl Into<&'static str> for DBColumn {
     /// Returns a `&str` that can be used for keying a key-value data base.
-    fn into(self) -> &'a str {
+    fn into(self) -> &'static str {
         match self {
-            DBColumn::BeaconBlock => &"blk",
-            DBColumn::BeaconState => &"ste",
-            DBColumn::BeaconChain => &"bch",
+            DBColumn::BeaconMeta => "bma",
+            DBColumn::BeaconBlock => "blk",
+            DBColumn::BeaconState => "ste",
+            DBColumn::BeaconChain => "bch",
+            DBColumn::BeaconBlockRoots => "bbr",
+            DBColumn::BeaconStateRoots => "bsr",
+            DBColumn::BeaconHistoricalRoots => "bhr",
+            DBColumn::BeaconRandaoMixes => "brm",
         }
     }
 }
 
-/// An item that may be stored in a `Store`.
-///
-/// Provides default methods that are suitable for most applications, however when overridden they
-/// provide full customizability of `Store` operations.
-pub trait StoreItem: Sized {
+/// An item that may stored in a `Store` by serializing and deserializing from bytes.
+pub trait SimpleStoreItem: Sized {
     /// Identifies which column this item should be placed in.
     fn db_column() -> DBColumn;
 
@@ -107,10 +148,32 @@ pub trait StoreItem: Sized {
     fn as_store_bytes(&self) -> Vec<u8>;
 
     /// De-serialize `self` from bytes.
-    fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error>;
+    ///
+    /// Return an instance of the type and the number of bytes that were read.
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error>;
+}
 
+/// An item that may be stored in a `Store`.
+pub trait StoreItem: Sized {
     /// Store `self`.
-    fn db_put(&self, store: &impl Store, key: &Hash256) -> Result<(), Error> {
+    fn db_put<S: Store>(&self, store: &S, key: &Hash256) -> Result<(), Error>;
+
+    /// Retrieve an instance of `Self` from `store`.
+    fn db_get<S: Store>(store: &S, key: &Hash256) -> Result<Option<Self>, Error>;
+
+    /// Return `true` if an instance of `Self` exists in `store`.
+    fn db_exists<S: Store>(store: &S, key: &Hash256) -> Result<bool, Error>;
+
+    /// Delete an instance of `Self` from `store`.
+    fn db_delete<S: Store>(store: &S, key: &Hash256) -> Result<(), Error>;
+}
+
+impl<T> StoreItem for T
+where
+    T: SimpleStoreItem,
+{
+    /// Store `self`.
+    fn db_put<S: Store>(&self, store: &S, key: &Hash256) -> Result<(), Error> {
         let column = Self::db_column().into();
         let key = key.as_bytes();
 
@@ -120,18 +183,18 @@ pub trait StoreItem: Sized {
     }
 
     /// Retrieve an instance of `Self`.
-    fn db_get(store: &impl Store, key: &Hash256) -> Result<Option<Self>, Error> {
+    fn db_get<S: Store>(store: &S, key: &Hash256) -> Result<Option<Self>, Error> {
         let column = Self::db_column().into();
         let key = key.as_bytes();
 
         match store.get_bytes(column, key)? {
-            Some(mut bytes) => Ok(Some(Self::from_store_bytes(&mut bytes[..])?)),
+            Some(bytes) => Ok(Some(Self::from_store_bytes(&bytes[..])?)),
             None => Ok(None),
         }
     }
 
     /// Return `true` if an instance of `Self` exists in `Store`.
-    fn db_exists(store: &impl Store, key: &Hash256) -> Result<bool, Error> {
+    fn db_exists<S: Store>(store: &S, key: &Hash256) -> Result<bool, Error> {
         let column = Self::db_column().into();
         let key = key.as_bytes();
 
@@ -139,7 +202,7 @@ pub trait StoreItem: Sized {
     }
 
     /// Delete `self` from the `Store`.
-    fn db_delete(store: &impl Store, key: &Hash256) -> Result<(), Error> {
+    fn db_delete<S: Store>(store: &S, key: &Hash256) -> Result<(), Error> {
         let column = Self::db_column().into();
         let key = key.as_bytes();
 
@@ -160,7 +223,7 @@ mod tests {
         b: u64,
     }
 
-    impl StoreItem for StorableThing {
+    impl SimpleStoreItem for StorableThing {
         fn db_column() -> DBColumn {
             DBColumn::BeaconBlock
         }
@@ -169,7 +232,7 @@ mod tests {
             self.as_ssz_bytes()
         }
 
-        fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error> {
+        fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
             Self::from_ssz_bytes(bytes).map_err(Into::into)
         }
     }
@@ -196,9 +259,22 @@ mod tests {
 
     #[test]
     fn diskdb() {
+        use sloggers::{null::NullLoggerBuilder, Build};
+
+        let hot_dir = tempdir().unwrap();
+        let cold_dir = tempdir().unwrap();
+        let spec = MinimalEthSpec::default_spec();
+        let log = NullLoggerBuilder.build().unwrap();
+        let store = DiskStore::open(&hot_dir.path(), &cold_dir.path(), spec, log).unwrap();
+
+        test_impl(store);
+    }
+
+    #[test]
+    fn simplediskdb() {
         let dir = tempdir().unwrap();
         let path = dir.path();
-        let store = DiskStore::open(&path).unwrap();
+        let store = SimpleDiskStore::open(&path).unwrap();
 
         test_impl(store);
     }