Implement freezer database (#508)

* Implement freezer database for state vectors

* Improve BeaconState safe accessors

And fix a bug in the compact committees accessor.

* Banish dodgy type bounds back to gRPC

* Clean up

* Switch to exclusive end points in chunked vec

* Cleaning up and start of tests

* Randao fix, more tests

* Fix unsightly hack

* Resolve test FIXMEs

* Config file support

* More clean-ups, migrator beginnings

* Finish migrator, integrate into BeaconChain

* Fixups

* Fix store tests

* Fix BeaconChain tests

* Fix LMD GHOST tests

* Address review comments, delete 'static bounds

* Cargo format

* Address review comments

* Fix LMD ghost tests

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

* Update for new builder paradigm, spec changes

* Freezer DB integration into BeaconNode

* Cleaning up

* This works, clean it up

* Cleanups

* Fix and improve store tests

* Refine store test

* Delete unused beacon_chain_builder.rs

* Fix CLI

* Store state at split slot in hot database

* Make fork choice lookup fast again

* Store freezer DB split slot in the database

* Handle potential div by 0 in chunked_vector

* Exclude committee caches from freezer DB

* Remove FIXME about long-running test

beacon_node/store/src/impls/beacon_state.rs

@@ -4,6 +4,43 @@ use ssz_derive::{Decode, Encode};
 use std::convert::TryInto;
 use types::beacon_state::{BeaconTreeHashCache, CommitteeCache, CACHED_EPOCHS};
 
+pub fn store_full_state<S: Store, E: EthSpec>(
+    store: &S,
+    state_root: &Hash256,
+    state: &BeaconState<E>,
+) -> Result<(), Error> {
+    let timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES);
+
+    let bytes = StorageContainer::new(state).as_ssz_bytes();
+    let result = store.put_bytes(DBColumn::BeaconState.into(), state_root.as_bytes(), &bytes);
+
+    metrics::stop_timer(timer);
+    metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT);
+    metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64);
+
+    result
+}
+
+pub fn get_full_state<S: Store, E: EthSpec>(
+    store: &S,
+    state_root: &Hash256,
+) -> Result<Option<BeaconState<E>>, Error> {
+    let timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES);
+
+    match store.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())? {
+        Some(bytes) => {
+            let container = StorageContainer::from_ssz_bytes(&bytes)?;
+
+            metrics::stop_timer(timer);
+            metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT);
+            metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, bytes.len() as i64);
+
+            Ok(Some(container.try_into()?))
+        }
+        None => Ok(None),
+    }
+}
+
 /// A container for storing `BeaconState` components.
 // TODO: would be more space efficient with the caches stored separately and referenced by hash
 #[derive(Encode, Decode)]
@@ -53,36 +90,3 @@ impl<T: EthSpec> TryInto<BeaconState<T>> for StorageContainer {
         Ok(state)
     }
 }
-
-impl<T: EthSpec> StoreItem for BeaconState<T> {
-    fn db_column() -> DBColumn {
-        DBColumn::BeaconState
-    }
-
-    fn as_store_bytes(&self) -> Vec<u8> {
-        let timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES);
-
-        let container = StorageContainer::new(self);
-        let bytes = container.as_ssz_bytes();
-
-        metrics::stop_timer(timer);
-        metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT);
-        metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64);
-
-        bytes
-    }
-
-    fn from_store_bytes(bytes: &mut [u8]) -> Result<Self, Error> {
-        let timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES);
-
-        let len = bytes.len();
-        let container = StorageContainer::from_ssz_bytes(bytes)?;
-        let result = container.try_into();
-
-        metrics::stop_timer(timer);
-        metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT);
-        metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, len as i64);
-
-        result
-    }
-}

beacon_node/store/src/impls/partial_beacon_state.rs

@@ -0,0 +1,16 @@
+use crate::*;
+use ssz::{Decode, Encode};
+
+impl<T: EthSpec> SimpleStoreItem for PartialBeaconState<T> {
+    fn db_column() -> DBColumn {
+        DBColumn::BeaconState
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
+        Ok(Self::from_ssz_bytes(bytes)?)
+    }
+}