Use the database to persist the pubkey cache (#2234)

## Issue Addressed

Closes #1787

## Proposed Changes

* Abstract the `ValidatorPubkeyCache` over a "backing" which is either a file (legacy), or the database.
* Implement a migration from schema v2 to schema v3, whereby the contents of the cache file are copied to the DB, and then the file is deleted. The next release to include this change must be a minor version bump, and we will need to warn users of the inability to downgrade (this is our first DB schema change since mainnet genesis).
* Move the schema migration code from the `store` crate into the `beacon_chain` crate so that it can access the datadir and the `ValidatorPubkeyCache`, etc. It gets injected back into the `store` via a closure (similar to what we do in fork choice).

beacon_node/store/src/errors.rs

@@ -19,6 +19,7 @@ pub enum Error {
     NoContinuationData,
     SplitPointModified(Slot, Slot),
     ConfigError(StoreConfigError),
+    SchemaMigrationError(String),
 }
 
 impl From<DecodeError> for Error {

beacon_node/store/src/hot_cold_store.rs

@@ -135,16 +135,20 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
     /// Open a new or existing database, with the given paths to the hot and cold DBs.
     ///
     /// The `slots_per_restore_point` parameter must be a divisor of `SLOTS_PER_HISTORICAL_ROOT`.
+    ///
+    /// The `migrate_schema` function is passed in so that the parent `BeaconChain` can provide
+    /// context and access `BeaconChain`-level code without creating a circular dependency.
     pub fn open(
         hot_path: &Path,
         cold_path: &Path,
+        migrate_schema: impl FnOnce(Arc<Self>, SchemaVersion, SchemaVersion) -> Result<(), Error>,
         config: StoreConfig,
         spec: ChainSpec,
         log: Logger,
-    ) -> Result<HotColdDB<E, LevelDB<E>, LevelDB<E>>, Error> {
+    ) -> Result<Arc<Self>, Error> {
         Self::verify_slots_per_restore_point(config.slots_per_restore_point)?;
 
-        let db = HotColdDB {
+        let db = Arc::new(HotColdDB {
             split: RwLock::new(Split::default()),
             cold_db: LevelDB::open(cold_path)?,
             hot_db: LevelDB::open(hot_path)?,
@@ -153,7 +157,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
             spec,
             log,
             _phantom: PhantomData,
-        };
+        });
 
         // Ensure that the schema version of the on-disk database matches the software.
         // If the version is mismatched, an automatic migration will be attempted.
@@ -164,7 +168,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
                 "from_version" => schema_version.as_u64(),
                 "to_version" => CURRENT_SCHEMA_VERSION.as_u64(),
             );
-            db.migrate_schema(schema_version, CURRENT_SCHEMA_VERSION)?;
+            migrate_schema(db.clone(), schema_version, CURRENT_SCHEMA_VERSION)?;
         } else {
             db.store_schema_version(CURRENT_SCHEMA_VERSION)?;
         }
@@ -830,7 +834,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     }
 
     /// Store the database schema version.
-    pub(crate) fn store_schema_version(&self, schema_version: SchemaVersion) -> Result<(), Error> {
+    pub fn store_schema_version(&self, schema_version: SchemaVersion) -> Result<(), Error> {
         self.hot_db.put(&SCHEMA_VERSION_KEY, &schema_version)
     }
 

beacon_node/store/src/lib.rs

@@ -20,10 +20,9 @@ pub mod hot_cold_store;
 mod impls;
 mod leveldb_store;
 mod memory_store;
-mod metadata;
+pub mod metadata;
 mod metrics;
 mod partial_beacon_state;
-mod schema_change;
 
 pub mod iter;
 
@@ -153,6 +152,7 @@ pub enum DBColumn {
     OpPool,
     Eth1Cache,
     ForkChoice,
+    PubkeyCache,
     /// For the table mapping restore point numbers to state roots.
     BeaconRestorePoint,
     /// For the mapping from state roots to their slots or summaries.
@@ -178,6 +178,7 @@ impl Into<&'static str> for DBColumn {
             DBColumn::OpPool => "opo",
             DBColumn::Eth1Cache => "etc",
             DBColumn::ForkChoice => "frk",
+            DBColumn::PubkeyCache => "pkc",
             DBColumn::BeaconRestorePoint => "brp",
             DBColumn::BeaconStateSummary => "bss",
             DBColumn::BeaconStateTemporary => "bst",

beacon_node/store/src/metadata.rs

@@ -2,7 +2,7 @@ use crate::{DBColumn, Error, StoreItem};
 use ssz::{Decode, Encode};
 use types::{Checkpoint, Hash256};
 
-pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(2);
+pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(3);
 
 // All the keys that get stored under the `BeaconMeta` column.
 //

beacon_node/store/src/schema_change.rs

@@ -1,32 +0,0 @@
-//! Utilities for managing database schema changes.
-use crate::hot_cold_store::{HotColdDB, HotColdDBError};
-use crate::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
-use crate::{Error, ItemStore};
-use types::EthSpec;
-
-impl<E, Hot, Cold> HotColdDB<E, Hot, Cold>
-where
-    E: EthSpec,
-    Hot: ItemStore<E>,
-    Cold: ItemStore<E>,
-{
-    /// Migrate the database from one schema version to another, applying all requisite mutations.
-    pub fn migrate_schema(&self, from: SchemaVersion, to: SchemaVersion) -> Result<(), Error> {
-        match (from, to) {
-            // Migration from v0.3.0 to v0.3.x, adding the temporary states column.
-            // Nothing actually needs to be done, but once a DB uses v2 it shouldn't go back.
-            (SchemaVersion(1), SchemaVersion(2)) => {
-                self.store_schema_version(to)?;
-                Ok(())
-            }
-            // Migrating from the current schema version to iself is always OK, a no-op.
-            (_, _) if from == to && to == CURRENT_SCHEMA_VERSION => Ok(()),
-            // Anything else is an error.
-            (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion {
-                target_version: to,
-                current_version: from,
-            }
-            .into()),
-        }
-    }
-}