Directory restructure (#1532)

Closes #1487
Closes #1427

Directory restructure in accordance with #1487. Also has temporary migration code to move the old directories into new structure.
Also extracts all default directory names and utility functions into a `directory` crate to avoid repetitio.

~Since `validator_definition.yaml` stores absolute paths, users will have to manually change the keystore paths or delete the file to get the validators picked up by the vc.~. `validator_definition.yaml` is migrated as well from the default directories.

Co-authored-by: realbigsean <seananderson33@gmail.com>
Co-authored-by: Paul Hauner <paul@paulhauner.com>

beacon_node/Cargo.toml

@@ -30,6 +30,7 @@ tokio = { version = "0.2.21", features = ["time"] }
 exit-future = "0.2.0"
 dirs = "2.0.2"
 logging = { path = "../common/logging" }
+directory = {path = "../common/directory"}
 futures = "0.3.5"
 environment = { path = "../lighthouse/environment" }
 genesis = { path = "genesis" }

beacon_node/client/Cargo.toml

@@ -41,3 +41,4 @@ lazy_static = "1.4.0"
 lighthouse_metrics = { path = "../../common/lighthouse_metrics" }
 time = "0.2.16"
 bus = "2.2.3"
+directory = {path = "../../common/directory"}

beacon_node/client/src/config.rs

@@ -1,11 +1,10 @@
+use directory::DEFAULT_ROOT_DIR;
 use network::NetworkConfig;
 use serde_derive::{Deserialize, Serialize};
 use std::fs;
 use std::path::PathBuf;
 use types::Graffiti;
 
-pub const DEFAULT_DATADIR: &str = ".lighthouse";
-
 /// The number initial validators when starting the `Minimal`.
 const TESTNET_SPEC_CONSTANTS: &str = "minimal";
 
@@ -72,7 +71,7 @@ pub struct Config {
 impl Default for Config {
     fn default() -> Self {
         Self {
-            data_dir: PathBuf::from(DEFAULT_DATADIR),
+            data_dir: PathBuf::from(DEFAULT_ROOT_DIR),
             db_name: "chain_db".to_string(),
             freezer_db_path: None,
             log_file: PathBuf::from(""),

beacon_node/eth2_libp2p/Cargo.toml

@@ -36,6 +36,7 @@ discv5 = { version = "0.1.0-alpha.12", features = ["libp2p"] }
 tiny-keccak = "2.0.2"
 environment = { path = "../../lighthouse/environment" }
 rand = "0.7.3"
+directory = { path = "../../common/directory" }
 regex = "1.3.9"
 
 [dependencies.libp2p]

beacon_node/eth2_libp2p/src/config.rs

@@ -1,5 +1,8 @@
 use crate::types::GossipKind;
 use crate::{Enr, PeerIdSerialized};
+use directory::{
+    DEFAULT_BEACON_NODE_DIR, DEFAULT_HARDCODED_TESTNET, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR,
+};
 use discv5::{Discv5Config, Discv5ConfigBuilder};
 use libp2p::gossipsub::{
     GossipsubConfig, GossipsubConfigBuilder, GossipsubMessage, MessageId, ValidationMode,
@@ -74,9 +77,14 @@ pub struct Config {
 impl Default for Config {
     /// Generate a default network configuration.
     fn default() -> Self {
-        let mut network_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
-        network_dir.push(".lighthouse");
-        network_dir.push("network");
+        // WARNING: this directory default should be always overrided with parameters
+        // from cli for specific networks.
+        let network_dir = dirs::home_dir()
+            .unwrap_or_else(|| PathBuf::from("."))
+            .join(DEFAULT_ROOT_DIR)
+            .join(DEFAULT_HARDCODED_TESTNET)
+            .join(DEFAULT_BEACON_NODE_DIR)
+            .join(DEFAULT_NETWORK_DIR);
 
         // The function used to generate a gossipsub message id
         // We use the first 8 bytes of SHA256(data) for content addressing

beacon_node/src/config.rs

@@ -1,7 +1,8 @@
 use beacon_chain::builder::PUBKEY_CACHE_FILENAME;
 use clap::ArgMatches;
 use clap_utils::BAD_TESTNET_DIR_MESSAGE;
-use client::{config::DEFAULT_DATADIR, ClientConfig, ClientGenesis};
+use client::{ClientConfig, ClientGenesis};
+use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR};
 use eth2_libp2p::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized};
 use eth2_testnet_config::Eth2TestnetConfig;
 use slog::{crit, info, warn, Logger};
@@ -13,9 +14,6 @@ use std::net::{TcpListener, UdpSocket};
 use std::path::PathBuf;
 use types::{ChainSpec, EthSpec, GRAFFITI_BYTES_LEN};
 
-pub const BEACON_NODE_DIR: &str = "beacon";
-pub const NETWORK_DIR: &str = "network";
-
 /// Gets the fully-initialized global client.
 ///
 /// The top-level `clap` arguments should be provided as `cli_args`.
@@ -295,7 +293,7 @@ pub fn set_network_config(
     if let Some(dir) = cli_args.value_of("network-dir") {
         config.network_dir = PathBuf::from(dir);
     } else {
-        config.network_dir = data_dir.join(NETWORK_DIR);
+        config.network_dir = data_dir.join(DEFAULT_NETWORK_DIR);
     };
 
     if let Some(listen_address_str) = cli_args.value_of("listen-address") {
@@ -456,11 +454,18 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf {
     // Read the `--datadir` flag.
     //
     // If it's not present, try and find the home directory (`~`) and push the default data
-    // directory onto it.
+    // directory and the testnet name onto it.
+
     cli_args
         .value_of("datadir")
-        .map(|path| PathBuf::from(path).join(BEACON_NODE_DIR))
-        .or_else(|| dirs::home_dir().map(|home| home.join(DEFAULT_DATADIR).join(BEACON_NODE_DIR)))
+        .map(|path| PathBuf::from(path).join(DEFAULT_BEACON_NODE_DIR))
+        .or_else(|| {
+            dirs::home_dir().map(|home| {
+                home.join(DEFAULT_ROOT_DIR)
+                    .join(directory::get_testnet_name(cli_args))
+                    .join(DEFAULT_BEACON_NODE_DIR)
+            })
+        })
         .unwrap_or_else(|| PathBuf::from("."))
 }