Implement tree states & hierarchical state DB

beacon_node/src/cli.rs

@@ -638,7 +638,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true)
         )
         /*
-         * Database purging and compaction.
+         * Database.
          */
         .arg(
             Arg::with_name("purge-db")
@@ -658,6 +658,21 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true)
                 .default_value("true")
         )
+        .arg(
+            Arg::with_name("state-cache-size")
+                .long("state-cache-size")
+                .value_name("SIZE")
+                .help("Specifies how many states the database should cache in memory [default: 128]")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("compression-level")
+                .long("compression-level")
+                .value_name("LEVEL")
+                .help("Compression level (-99 to 22) for zstd compression applied to states on disk \
+                       [default: 1]. You may change the compression level freely without re-syncing.")
+                .takes_value(true)
+        )
         .arg(
             Arg::with_name("prune-payloads")
                 .long("prune-payloads")
@@ -667,7 +682,26 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                 .takes_value(true)
                 .default_value("true")
         )
-
+        .arg(
+            Arg::with_name("db-migration-period")
+                .long("db-migration-period")
+                .value_name("EPOCHS")
+                .help("Specifies the number of epochs to wait between applying each finalization \
+                       migration to the database. Applying migrations less frequently can lead to \
+                       less total disk writes.")
+                .default_value("4")
+                .takes_value(true)
+        )
+        .arg(
+            Arg::with_name("epochs-per-state-diff")
+                .long("epochs-per-state-diff")
+                .value_name("EPOCHS")
+                .help("Number of epochs between state diffs stored in the database. Lower values \
+                       result in more writes and more data stored, while higher values result in \
+                       more block replaying and longer load times in case of cache miss.")
+                .default_value("4")
+                .takes_value(true)
+        )
         /*
          * Misc.
          */
@@ -1117,4 +1151,10 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     developers. This directory is not pruned, users should be careful to avoid \
                     filling up their disks.")
         )
+        .arg(
+            Arg::with_name("unsafe-and-dangerous-mode")
+            .long("unsafe-and-dangerous-mode")
+            .help("Don't use this flag unless you know what you're doing. Go back and download a \
+                   stable Lighthouse release")
+        )
 }

beacon_node/src/config.rs

@@ -13,7 +13,7 @@ use http_api::TlsConfig;
 use lighthouse_network::ListenAddress;
 use lighthouse_network::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
 use sensitive_url::SensitiveUrl;
-use slog::{info, warn, Logger};
+use slog::{crit, info, warn, Logger};
 use std::cmp;
 use std::cmp::max;
 use std::fmt::Debug;
@@ -373,14 +373,25 @@ pub fn get_config<E: EthSpec>(
         client_config.freezer_db_path = Some(PathBuf::from(freezer_dir));
     }
 
-    let (sprp, sprp_explicit) = get_slots_per_restore_point::<E>(cli_args)?;
-    client_config.store.slots_per_restore_point = sprp;
-    client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit;
+    if !cli_args.is_present("unsafe-and-dangerous-mode") {
+        crit!(
+            log,
+            "This is an EXPERIMENTAL build of Lighthouse. If you are seeing this message you may \
+            have downloaded the wrong version by mistake. If so, go back and download the latest \
+            stable release. If you are certain that you want to continue, read the docs for the \
+            latest experimental release and continue at your own risk."
+        );
+        return Err("FATAL ERROR, YOU HAVE THE WRONG LIGHTHOUSE BINARY".into());
+    }
 
-    if let Some(block_cache_size) = cli_args.value_of("block-cache-size") {
-        client_config.store.block_cache_size = block_cache_size
-            .parse()
-            .map_err(|_| "block-cache-size is not a valid integer".to_string())?;
+    if let Some(block_cache_size) = clap_utils::parse_optional(cli_args, "block-cache-size")? {
+        client_config.store.block_cache_size = block_cache_size;
+    }
+    if let Some(state_cache_size) = clap_utils::parse_optional(cli_args, "state-cache-size")? {
+        client_config.store.state_cache_size = state_cache_size;
+    }
+    if let Some(compression_level) = clap_utils::parse_optional(cli_args, "compression-level")? {
+        client_config.store.compression_level = compression_level;
     }
 
     if let Some(historic_state_cache_size) = cli_args.value_of("historic-state-cache-size") {
@@ -400,6 +411,17 @@ pub fn get_config<E: EthSpec>(
         client_config.store.prune_payloads = prune_payloads;
     }
 
+    if let Some(epochs_per_migration) = clap_utils::parse_optional(cli_args, "db-migration-period")?
+    {
+        client_config.store_migrator.epochs_per_run = epochs_per_migration;
+    }
+
+    if let Some(epochs_per_state_diff) =
+        clap_utils::parse_optional(cli_args, "epochs-per-state-diff")?
+    {
+        client_config.store.epochs_per_state_diff = epochs_per_state_diff;
+    }
+
     /*
      * Zero-ports
      *
@@ -1294,25 +1316,6 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
         .unwrap_or_else(|| PathBuf::from("."))
 }
 
-/// Get the `slots_per_restore_point` value to use for the database.
-///
-/// Return `(sprp, set_explicitly)` where `set_explicitly` is `true` if the user provided the value.
-pub fn get_slots_per_restore_point<E: EthSpec>(
-    cli_args: &ArgMatches,
-) -> Result<(u64, bool), String> {
-    if let Some(slots_per_restore_point) =
-        clap_utils::parse_optional(cli_args, "slots-per-restore-point")?
-    {
-        Ok((slots_per_restore_point, true))
-    } else {
-        let default = std::cmp::min(
-            E::slots_per_historical_root() as u64,
-            store::config::DEFAULT_SLOTS_PER_RESTORE_POINT,
-        );
-        Ok((default, false))
-    }
-}
-
 /// Parses the `cli_value` as a comma-separated string of values to be parsed with `parser`.
 ///
 /// If there is more than one value, log a warning. If there are no values, return an error.

beacon_node/src/lib.rs

@@ -13,7 +13,7 @@ use beacon_chain::{
 use clap::ArgMatches;
 pub use cli::cli_app;
 pub use client::{Client, ClientBuilder, ClientConfig, ClientGenesis};
-pub use config::{get_config, get_data_dir, get_slots_per_restore_point, set_network_config};
+pub use config::{get_config, get_data_dir, set_network_config};
 use environment::RuntimeContext;
 pub use eth2_config::Eth2Config;
 use slasher::{DatabaseBackendOverride, Slasher};