Implement freezer database (#508)

* Implement freezer database for state vectors * Improve BeaconState safe accessors And fix a bug in the compact committees accessor. * Banish dodgy type bounds back to gRPC * Clean up * Switch to exclusive end points in chunked vec * Cleaning up and start of tests * Randao fix, more tests * Fix unsightly hack * Resolve test FIXMEs * Config file support * More clean-ups, migrator beginnings * Finish migrator, integrate into BeaconChain * Fixups * Fix store tests * Fix BeaconChain tests * Fix LMD GHOST tests * Address review comments, delete 'static bounds * Cargo format * Address review comments * Fix LMD ghost tests * Update to spec v0.9.0 * Update to v0.9.1 * Bump spec tags for v0.9.1 * Formatting, fix CI failures * Resolve accidental KeyPair merge conflict * Document new BeaconState functions * Fix incorrect cache drops in `advance_caches` * Update fork choice for v0.9.1 * Clean up some FIXMEs * Fix a few docs/logs * Update for new builder paradigm, spec changes * Freezer DB integration into BeaconNode * Cleaning up * This works, clean it up * Cleanups * Fix and improve store tests * Refine store test * Delete unused beacon_chain_builder.rs * Fix CLI * Store state at split slot in hot database * Make fork choice lookup fast again * Store freezer DB split slot in the database * Handle potential div by 0 in chunked_vector * Exclude committee caches from freezer DB * Remove FIXME about long-running test
2026-03-09 03:31:45 +00:00 · 2019-11-27 10:54:46 +11:00
parent a514968155
commit bf2eeae3f2
36 changed files with 2486 additions and 580 deletions
--- a/beacon_node/client/src/builder.rs
+++ b/beacon_node/client/src/builder.rs
@@ -5,7 +5,10 @@ use beacon_chain::{
    eth1_chain::CachingEth1Backend,
    lmd_ghost::ThreadSafeReducedTree,
    slot_clock::{SlotClock, SystemTimeSlotClock},
-    store::{DiskStore, MemoryStore, Store},
+    store::{
+        migrate::{BackgroundMigrator, Migrate, NullMigrator},
+        DiskStore, MemoryStore, SimpleDiskStore, Store,
+    },
    BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler,
 };
 use environment::RuntimeContext;
@@ -52,6 +55,7 @@ pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 500;
 pub struct ClientBuilder<T: BeaconChainTypes> {
    slot_clock: Option<T::SlotClock>,
    store: Option<Arc<T::Store>>,
+    store_migrator: Option<T::StoreMigrator>,
    runtime_context: Option<RuntimeContext<T::EthSpec>>,
    chain_spec: Option<ChainSpec>,
    beacon_chain_builder: Option<BeaconChainBuilder<T>>,
@@ -66,10 +70,21 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
    eth_spec_instance: T::EthSpec,
 }

-impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
-    ClientBuilder<Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>>
+impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+    ClientBuilder<
+        Witness<
+            TStore,
+            TStoreMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
+    >
 where
    TStore: Store + 'static,
+    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + Clone + 'static,
    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -83,6 +98,7 @@ where
        Self {
            slot_clock: None,
            store: None,
+            store_migrator: None,
            runtime_context: None,
            chain_spec: None,
            beacon_chain_builder: None,
@@ -118,6 +134,7 @@ where
        config: Eth1Config,
    ) -> impl Future<Item = Self, Error = String> {
        let store = self.store.clone();
+        let store_migrator = self.store_migrator.take();
        let chain_spec = self.chain_spec.clone();
        let runtime_context = self.runtime_context.clone();
        let eth_spec_instance = self.eth_spec_instance.clone();
@@ -126,6 +143,9 @@ where
            .and_then(move |()| {
                let store = store
                    .ok_or_else(|| "beacon_chain_start_method requires a store".to_string())?;
+                let store_migrator = store_migrator.ok_or_else(|| {
+                    "beacon_chain_start_method requires a store migrator".to_string()
+                })?;
                let context = runtime_context
                    .ok_or_else(|| "beacon_chain_start_method requires a log".to_string())?
                    .service_context("beacon");
@@ -135,6 +155,7 @@ where
                let builder = BeaconChainBuilder::new(eth_spec_instance)
                    .logger(context.log.clone())
                    .store(store.clone())
+                    .store_migrator(store_migrator)
                    .custom_spec(spec.clone());

                Ok((builder, spec, context))
@@ -300,7 +321,9 @@ where
            &context.executor,
            beacon_chain.clone(),
            network_info,
-            client_config.db_path().expect("unable to read datadir"),
+            client_config
+                .create_db_path()
+                .expect("unable to read datadir"),
            eth2_config.clone(),
            context.log,
        )
@@ -420,7 +443,17 @@ where
    /// If type inference errors are being raised, see the comment on the definition of `Self`.
    pub fn build(
        self,
-    ) -> Client<Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>> {
+    ) -> Client<
+        Witness<
+            TStore,
+            TStoreMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
+    > {
        Client {
            beacon_chain: self.beacon_chain,
            libp2p_network: self.libp2p_network,
@@ -431,10 +464,11 @@ where
    }
 }

-impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            TStore,
+            TStoreMigrator,
            TSlotClock,
            ThreadSafeReducedTree<TStore, TEthSpec>,
            TEth1Backend,
@@ -444,6 +478,7 @@ impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
    >
 where
    TStore: Store + 'static,
+    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + Clone + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
    TEthSpec: EthSpec + 'static,
@@ -476,12 +511,21 @@ where
    }
 }

-impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
+impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
    ClientBuilder<
-        Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, WebSocketSender<TEthSpec>>,
+        Witness<
+            TStore,
+            TStoreMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            WebSocketSender<TEthSpec>,
+        >,
    >
 where
    TStore: Store + 'static,
+    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + 'static,
    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -517,18 +561,68 @@ where
    }
 }

-impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
-    ClientBuilder<Witness<DiskStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>>
+impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+    ClientBuilder<
+        Witness<
+            DiskStore,
+            TStoreMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
+    >
 where
    TSlotClock: SlotClock + 'static,
+    TStoreMigrator: store::Migrate<DiskStore, TEthSpec> + 'static,
    TLmdGhost: LmdGhost<DiskStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
    TEthSpec: EthSpec + 'static,
    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
    /// Specifies that the `Client` should use a `DiskStore` database.
-    pub fn disk_store(mut self, path: &Path) -> Result<Self, String> {
-        let store = DiskStore::open(path)
+    pub fn disk_store(mut self, hot_path: &Path, cold_path: &Path) -> Result<Self, String> {
+        let context = self
+            .runtime_context
+            .as_ref()
+            .ok_or_else(|| "disk_store requires a log".to_string())?
+            .service_context("freezer_db");
+        let spec = self
+            .chain_spec
+            .clone()
+            .ok_or_else(|| "disk_store requires a chain spec".to_string())?;
+
+        let store = DiskStore::open(hot_path, cold_path, spec, context.log)
+            .map_err(|e| format!("Unable to open database: {:?}", e).to_string())?;
+        self.store = Some(Arc::new(store));
+        Ok(self)
+    }
+}
+
+impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+    ClientBuilder<
+        Witness<
+            SimpleDiskStore,
+            TStoreMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
+    >
+where
+    TSlotClock: SlotClock + 'static,
+    TStoreMigrator: store::Migrate<SimpleDiskStore, TEthSpec> + 'static,
+    TLmdGhost: LmdGhost<SimpleDiskStore, TEthSpec> + 'static,
+    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
+    TEthSpec: EthSpec + 'static,
+    TEventHandler: EventHandler<TEthSpec> + 'static,
+{
+    /// Specifies that the `Client` should use a `DiskStore` database.
+    pub fn simple_disk_store(mut self, path: &Path) -> Result<Self, String> {
+        let store = SimpleDiskStore::open(path)
            .map_err(|e| format!("Unable to open database: {:?}", e).to_string())?;
        self.store = Some(Arc::new(store));
        Ok(self)
@@ -537,7 +631,15 @@ where

 impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
-        Witness<MemoryStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
+        Witness<
+            MemoryStore,
+            NullMigrator,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
    >
 where
    TSlotClock: SlotClock + 'static,
@@ -547,17 +649,49 @@ where
    TEventHandler: EventHandler<TEthSpec> + 'static,
 {
    /// Specifies that the `Client` should use a `MemoryStore` database.
+    ///
+    /// Also sets the `store_migrator` to the `NullMigrator`, as that's the only viable choice.
    pub fn memory_store(mut self) -> Self {
        let store = MemoryStore::open();
        self.store = Some(Arc::new(store));
+        self.store_migrator = Some(NullMigrator);
        self
    }
 }

-impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
+impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+    ClientBuilder<
+        Witness<
+            DiskStore,
+            BackgroundMigrator<TEthSpec>,
+            TSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
+    >
+where
+    TSlotClock: SlotClock + 'static,
+    TLmdGhost: LmdGhost<DiskStore, TEthSpec> + 'static,
+    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
+    TEthSpec: EthSpec + 'static,
+    TEventHandler: EventHandler<TEthSpec> + 'static,
+{
+    pub fn background_migrator(mut self) -> Result<Self, String> {
+        let store = self.store.clone().ok_or_else(|| {
+            "background_migrator requires the store to be initialized".to_string()
+        })?;
+        self.store_migrator = Some(BackgroundMigrator::new(store));
+        Ok(self)
+    }
+}
+
+impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
    ClientBuilder<
        Witness<
            TStore,
+            TStoreMigrator,
            TSlotClock,
            TLmdGhost,
            CachingEth1Backend<TEthSpec, TStore>,
@@ -567,6 +701,7 @@ impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
    >
 where
    TStore: Store + 'static,
+    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TSlotClock: SlotClock + 'static,
    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEthSpec: EthSpec + 'static,
@@ -643,12 +778,21 @@ where
    }
 }

-impl<TStore, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
+impl<TStore, TStoreMigrator, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
    ClientBuilder<
-        Witness<TStore, SystemTimeSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
+        Witness<
+            TStore,
+            TStoreMigrator,
+            SystemTimeSlotClock,
+            TLmdGhost,
+            TEth1Backend,
+            TEthSpec,
+            TEventHandler,
+        >,
    >
 where
    TStore: Store + 'static,
+    TStoreMigrator: store::Migrate<TStore, TEthSpec>,
    TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
    TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
    TEthSpec: EthSpec + 'static,
--- a/beacon_node/client/src/config.rs
+++ b/beacon_node/client/src/config.rs
@@ -7,6 +7,9 @@ use std::path::PathBuf;
 /// The number initial validators when starting the `Minimal`.
 const TESTNET_SPEC_CONSTANTS: &str = "minimal";

+/// Default directory name for the freezer database under the top-level data dir.
+const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db";
+
 /// Defines how the client should initialize the `BeaconChain` and other components.
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub enum ClientGenesis {
@@ -39,6 +42,7 @@ pub struct Config {
    pub data_dir: PathBuf,
    pub db_type: String,
    db_name: String,
+    freezer_db_path: Option<PathBuf>,
    pub log_file: PathBuf,
    pub spec_constants: String,
    /// If true, the node will use co-ordinated junk for eth1 values.
@@ -63,6 +67,7 @@ impl Default for Config {
            log_file: PathBuf::from(""),
            db_type: "disk".to_string(),
            db_name: "chain_db".to_string(),
+            freezer_db_path: None,
            genesis: <_>::default(),
            network: NetworkConfig::new(),
            rest_api: <_>::default(),
@@ -76,19 +81,59 @@ impl Default for Config {
 }

 impl Config {
-    /// Returns the path to which the client may initialize an on-disk database.
-    pub fn db_path(&self) -> Option<PathBuf> {
-        self.data_dir()
-            .and_then(|path| Some(path.join(&self.db_name)))
+    /// Get the database path without initialising it.
+    pub fn get_db_path(&self) -> Option<PathBuf> {
+        self.get_data_dir()
+            .map(|data_dir| data_dir.join(&self.db_name))
+    }
+
+    /// Get the database path, creating it if necessary.
+    pub fn create_db_path(&self) -> Result<PathBuf, String> {
+        let db_path = self
+            .get_db_path()
+            .ok_or_else(|| "Unable to locate user home directory")?;
+        ensure_dir_exists(db_path)
+    }
+
+    /// Fetch default path to use for the freezer database.
+    fn default_freezer_db_path(&self) -> Option<PathBuf> {
+        self.get_data_dir()
+            .map(|data_dir| data_dir.join(DEFAULT_FREEZER_DB_DIR))
+    }
+
+    /// Returns the path to which the client may initialize the on-disk freezer database.
+    ///
+    /// Will attempt to use the user-supplied path from e.g. the CLI, or will default
+    /// to a directory in the data_dir if no path is provided.
+    pub fn get_freezer_db_path(&self) -> Option<PathBuf> {
+        self.freezer_db_path
+            .clone()
+            .or_else(|| self.default_freezer_db_path())
+    }
+
+    /// Get the freezer DB path, creating it if necessary.
+    pub fn create_freezer_db_path(&self) -> Result<PathBuf, String> {
+        let freezer_db_path = self
+            .get_freezer_db_path()
+            .ok_or_else(|| "Unable to locate user home directory")?;
+        ensure_dir_exists(freezer_db_path)
+    }
+
+    /// Returns the core path for the client.
+    ///
+    /// Will not create any directories.
+    pub fn get_data_dir(&self) -> Option<PathBuf> {
+        dirs::home_dir().map(|home_dir| home_dir.join(&self.data_dir))
    }

    /// Returns the core path for the client.
    ///
    /// Creates the directory if it does not exist.
-    pub fn data_dir(&self) -> Option<PathBuf> {
-        let path = dirs::home_dir()?.join(&self.data_dir);
-        fs::create_dir_all(&path).ok()?;
-        Some(path)
+    pub fn create_data_dir(&self) -> Result<PathBuf, String> {
+        let path = self
+            .get_data_dir()
+            .ok_or_else(|| "Unable to locate user home directory".to_string())?;
+        ensure_dir_exists(path)
    }

    /// Apply the following arguments to `self`, replacing values if they are specified in `args`.
@@ -100,9 +145,9 @@ impl Config {
            self.data_dir = PathBuf::from(dir);
        };

-        if let Some(dir) = args.value_of("db") {
-            self.db_type = dir.to_string();
-        };
+        if let Some(freezer_dir) = args.value_of("freezer-dir") {
+            self.freezer_db_path = Some(PathBuf::from(freezer_dir));
+        }

        self.network.apply_cli_args(args)?;
        self.rest_api.apply_cli_args(args)?;
@@ -112,6 +157,12 @@ impl Config {
    }
 }

+/// Ensure that the directory at `path` exists, by creating it and all parents if necessary.
+fn ensure_dir_exists(path: PathBuf) -> Result<PathBuf, String> {
+    fs::create_dir_all(&path).map_err(|e| format!("Unable to create {}: {}", path.display(), e))?;
+    Ok(path)
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;