Misc. dependency cleanup (#6810)

* remove ensure_dir_exists (2 deps saved)

* group UNHANDLED_ERRORs into a generic (2 deps saved)

* Introduce separate `health_metrics` crate

* separate health_metrics crate

* remove metrics from warp_utils

* move ProcessHealth::observe and SystemHealth::observe to health_metrics

* fix errors

* nitpick `Cargo.toml`s

---------

Co-authored-by: Daniel Knopik <daniel@dknopik.de>
# Conflicts:
#	Cargo.toml

common/account_utils/Cargo.toml

@@ -6,7 +6,6 @@ edition = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-directory = { workspace = true }
 eth2_keystore = { workspace = true }
 eth2_wallet = { workspace = true }
 filesystem = { workspace = true }

common/account_utils/src/validator_definitions.rs

@@ -4,13 +4,12 @@
 //! attempt) to load into the `crate::intialized_validators::InitializedValidators` struct.
 
 use crate::{default_keystore_password_path, read_password_string, write_file_via_temporary};
-use directory::ensure_dir_exists;
 use eth2_keystore::Keystore;
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 use slog::{error, Logger};
 use std::collections::HashSet;
-use std::fs::{self, File};
+use std::fs::{self, create_dir_all, File};
 use std::io;
 use std::path::{Path, PathBuf};
 use types::{graffiti::GraffitiString, Address, PublicKey};
@@ -229,7 +228,7 @@ impl From<Vec<ValidatorDefinition>> for ValidatorDefinitions {
 impl ValidatorDefinitions {
     /// Open an existing file or create a new, empty one if it does not exist.
     pub fn open_or_create<P: AsRef<Path>>(validators_dir: P) -> Result<Self, Error> {
-        ensure_dir_exists(validators_dir.as_ref()).map_err(|_| {
+        create_dir_all(validators_dir.as_ref()).map_err(|_| {
             Error::UnableToCreateValidatorDir(PathBuf::from(validators_dir.as_ref()))
         })?;
         let config_path = validators_dir.as_ref().join(CONFIG_FILENAME);

common/directory/src/lib.rs

@@ -1,6 +1,6 @@
 use clap::ArgMatches;
 pub use eth2_network_config::DEFAULT_HARDCODED_NETWORK;
-use std::fs::{self, create_dir_all};
+use std::fs;
 use std::path::{Path, PathBuf};
 
 /// Names for the default directories.
@@ -30,17 +30,6 @@ pub fn get_network_dir(matches: &ArgMatches) -> String {
     }
 }
 
-/// Checks if a directory exists in the given path and creates a directory if it does not exist.
-pub fn ensure_dir_exists<P: AsRef<Path>>(path: P) -> Result<(), String> {
-    let path = path.as_ref();
-
-    if !path.exists() {
-        create_dir_all(path).map_err(|e| format!("Unable to create {:?}: {:?}", path, e))?;
-    }
-
-    Ok(())
-}
-
 /// If `arg` is in `matches`, parses the value as a path.
 ///
 /// Otherwise, attempts to find the default directory for the `testnet` from the `matches`.

common/eth2/Cargo.toml

@@ -31,10 +31,6 @@ zeroize = { workspace = true }
 [dev-dependencies]
 tokio = { workspace = true }
 
-[target.'cfg(target_os = "linux")'.dependencies]
-psutil = { version = "3.3.0", optional = true }
-procfs = { version = "0.15.1", optional = true }
-
 [features]
 default = ["lighthouse"]
-lighthouse = ["psutil", "procfs"]
+lighthouse = []

common/eth2/src/lighthouse.rs

@@ -88,12 +88,6 @@ pub struct ValidatorInclusionData {
     pub is_previous_epoch_head_attester: bool,
 }
 
-#[cfg(target_os = "linux")]
-use {
-    psutil::cpu::os::linux::CpuTimesExt, psutil::memory::os::linux::VirtualMemoryExt,
-    psutil::process::Process,
-};
-
 /// Reports on the health of the Lighthouse instance.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct Health {
@@ -164,69 +158,6 @@ pub struct SystemHealth {
     pub misc_os: String,
 }
 
-impl SystemHealth {
-    #[cfg(not(target_os = "linux"))]
-    pub fn observe() -> Result<Self, String> {
-        Err("Health is only available on Linux".into())
-    }
-
-    #[cfg(target_os = "linux")]
-    pub fn observe() -> Result<Self, String> {
-        let vm = psutil::memory::virtual_memory()
-            .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?;
-        let loadavg =
-            psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?;
-
-        let cpu =
-            psutil::cpu::cpu_times().map_err(|e| format!("Unable to get cpu times: {:?}", e))?;
-
-        let disk_usage = psutil::disk::disk_usage("/")
-            .map_err(|e| format!("Unable to disk usage info: {:?}", e))?;
-
-        let disk = psutil::disk::DiskIoCountersCollector::default()
-            .disk_io_counters()
-            .map_err(|e| format!("Unable to get disk counters: {:?}", e))?;
-
-        let net = psutil::network::NetIoCountersCollector::default()
-            .net_io_counters()
-            .map_err(|e| format!("Unable to get network io counters: {:?}", e))?;
-
-        let boot_time = psutil::host::boot_time()
-            .map_err(|e| format!("Unable to get system boot time: {:?}", e))?
-            .duration_since(std::time::UNIX_EPOCH)
-            .map_err(|e| format!("Boot time is lower than unix epoch: {}", e))?
-            .as_secs();
-
-        Ok(Self {
-            sys_virt_mem_total: vm.total(),
-            sys_virt_mem_available: vm.available(),
-            sys_virt_mem_used: vm.used(),
-            sys_virt_mem_free: vm.free(),
-            sys_virt_mem_cached: vm.cached(),
-            sys_virt_mem_buffers: vm.buffers(),
-            sys_virt_mem_percent: vm.percent(),
-            sys_loadavg_1: loadavg.one,
-            sys_loadavg_5: loadavg.five,
-            sys_loadavg_15: loadavg.fifteen,
-            cpu_cores: psutil::cpu::cpu_count_physical(),
-            cpu_threads: psutil::cpu::cpu_count(),
-            system_seconds_total: cpu.system().as_secs(),
-            cpu_time_total: cpu.total().as_secs(),
-            user_seconds_total: cpu.user().as_secs(),
-            iowait_seconds_total: cpu.iowait().as_secs(),
-            idle_seconds_total: cpu.idle().as_secs(),
-            disk_node_bytes_total: disk_usage.total(),
-            disk_node_bytes_free: disk_usage.free(),
-            disk_node_reads_total: disk.read_count(),
-            disk_node_writes_total: disk.write_count(),
-            network_node_bytes_total_received: net.bytes_recv(),
-            network_node_bytes_total_transmit: net.bytes_sent(),
-            misc_node_boot_ts_seconds: boot_time,
-            misc_os: std::env::consts::OS.to_string(),
-        })
-    }
-}
-
 /// Process specific health
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct ProcessHealth {
@@ -244,59 +175,6 @@ pub struct ProcessHealth {
     pub pid_process_seconds_total: u64,
 }
 
-impl ProcessHealth {
-    #[cfg(not(target_os = "linux"))]
-    pub fn observe() -> Result<Self, String> {
-        Err("Health is only available on Linux".into())
-    }
-
-    #[cfg(target_os = "linux")]
-    pub fn observe() -> Result<Self, String> {
-        let process =
-            Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?;
-
-        let process_mem = process
-            .memory_info()
-            .map_err(|e| format!("Unable to get process memory info: {:?}", e))?;
-
-        let me = procfs::process::Process::myself()
-            .map_err(|e| format!("Unable to get process: {:?}", e))?;
-        let stat = me
-            .stat()
-            .map_err(|e| format!("Unable to get stat: {:?}", e))?;
-
-        let process_times = process
-            .cpu_times()
-            .map_err(|e| format!("Unable to get process cpu times : {:?}", e))?;
-
-        Ok(Self {
-            pid: process.pid(),
-            pid_num_threads: stat.num_threads,
-            pid_mem_resident_set_size: process_mem.rss(),
-            pid_mem_virtual_memory_size: process_mem.vms(),
-            pid_mem_shared_memory_size: process_mem.shared(),
-            pid_process_seconds_total: process_times.busy().as_secs()
-                + process_times.children_system().as_secs()
-                + process_times.children_system().as_secs(),
-        })
-    }
-}
-
-impl Health {
-    #[cfg(not(target_os = "linux"))]
-    pub fn observe() -> Result<Self, String> {
-        Err("Health is only available on Linux".into())
-    }
-
-    #[cfg(target_os = "linux")]
-    pub fn observe() -> Result<Self, String> {
-        Ok(Self {
-            process: ProcessHealth::observe()?,
-            system: SystemHealth::observe()?,
-        })
-    }
-}
-
 /// Indicates how up-to-date the Eth1 caches are.
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct Eth1SyncStatusData {

common/health_metrics/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "health_metrics"
+version = "0.1.0"
+edition = { workspace = true }
+
+[dependencies]
+eth2 = { workspace = true }
+metrics = { workspace = true }
+
+[target.'cfg(target_os = "linux")'.dependencies]
+psutil = "3.3.0"
+procfs = "0.15.1"

common/health_metrics/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod metrics;
+pub mod observe;

common/health_metrics/src/metrics.rs

@@ -1,3 +1,4 @@
+use crate::observe::Observe;
 use eth2::lighthouse::{ProcessHealth, SystemHealth};
 use metrics::*;
 use std::sync::LazyLock;

common/health_metrics/src/observe.rs

@@ -0,0 +1,127 @@
+use eth2::lighthouse::{Health, ProcessHealth, SystemHealth};
+
+#[cfg(target_os = "linux")]
+use {
+    psutil::cpu::os::linux::CpuTimesExt, psutil::memory::os::linux::VirtualMemoryExt,
+    psutil::process::Process,
+};
+
+pub trait Observe: Sized {
+    fn observe() -> Result<Self, String>;
+}
+
+impl Observe for Health {
+    #[cfg(not(target_os = "linux"))]
+    fn observe() -> Result<Self, String> {
+        Err("Health is only available on Linux".into())
+    }
+
+    #[cfg(target_os = "linux")]
+    fn observe() -> Result<Self, String> {
+        Ok(Self {
+            process: ProcessHealth::observe()?,
+            system: SystemHealth::observe()?,
+        })
+    }
+}
+
+impl Observe for SystemHealth {
+    #[cfg(not(target_os = "linux"))]
+    fn observe() -> Result<Self, String> {
+        Err("Health is only available on Linux".into())
+    }
+
+    #[cfg(target_os = "linux")]
+    fn observe() -> Result<Self, String> {
+        let vm = psutil::memory::virtual_memory()
+            .map_err(|e| format!("Unable to get virtual memory: {:?}", e))?;
+        let loadavg =
+            psutil::host::loadavg().map_err(|e| format!("Unable to get loadavg: {:?}", e))?;
+
+        let cpu =
+            psutil::cpu::cpu_times().map_err(|e| format!("Unable to get cpu times: {:?}", e))?;
+
+        let disk_usage = psutil::disk::disk_usage("/")
+            .map_err(|e| format!("Unable to disk usage info: {:?}", e))?;
+
+        let disk = psutil::disk::DiskIoCountersCollector::default()
+            .disk_io_counters()
+            .map_err(|e| format!("Unable to get disk counters: {:?}", e))?;
+
+        let net = psutil::network::NetIoCountersCollector::default()
+            .net_io_counters()
+            .map_err(|e| format!("Unable to get network io counters: {:?}", e))?;
+
+        let boot_time = psutil::host::boot_time()
+            .map_err(|e| format!("Unable to get system boot time: {:?}", e))?
+            .duration_since(std::time::UNIX_EPOCH)
+            .map_err(|e| format!("Boot time is lower than unix epoch: {}", e))?
+            .as_secs();
+
+        Ok(Self {
+            sys_virt_mem_total: vm.total(),
+            sys_virt_mem_available: vm.available(),
+            sys_virt_mem_used: vm.used(),
+            sys_virt_mem_free: vm.free(),
+            sys_virt_mem_cached: vm.cached(),
+            sys_virt_mem_buffers: vm.buffers(),
+            sys_virt_mem_percent: vm.percent(),
+            sys_loadavg_1: loadavg.one,
+            sys_loadavg_5: loadavg.five,
+            sys_loadavg_15: loadavg.fifteen,
+            cpu_cores: psutil::cpu::cpu_count_physical(),
+            cpu_threads: psutil::cpu::cpu_count(),
+            system_seconds_total: cpu.system().as_secs(),
+            cpu_time_total: cpu.total().as_secs(),
+            user_seconds_total: cpu.user().as_secs(),
+            iowait_seconds_total: cpu.iowait().as_secs(),
+            idle_seconds_total: cpu.idle().as_secs(),
+            disk_node_bytes_total: disk_usage.total(),
+            disk_node_bytes_free: disk_usage.free(),
+            disk_node_reads_total: disk.read_count(),
+            disk_node_writes_total: disk.write_count(),
+            network_node_bytes_total_received: net.bytes_recv(),
+            network_node_bytes_total_transmit: net.bytes_sent(),
+            misc_node_boot_ts_seconds: boot_time,
+            misc_os: std::env::consts::OS.to_string(),
+        })
+    }
+}
+
+impl Observe for ProcessHealth {
+    #[cfg(not(target_os = "linux"))]
+    fn observe() -> Result<Self, String> {
+        Err("Health is only available on Linux".into())
+    }
+
+    #[cfg(target_os = "linux")]
+    fn observe() -> Result<Self, String> {
+        let process =
+            Process::current().map_err(|e| format!("Unable to get current process: {:?}", e))?;
+
+        let process_mem = process
+            .memory_info()
+            .map_err(|e| format!("Unable to get process memory info: {:?}", e))?;
+
+        let me = procfs::process::Process::myself()
+            .map_err(|e| format!("Unable to get process: {:?}", e))?;
+        let stat = me
+            .stat()
+            .map_err(|e| format!("Unable to get stat: {:?}", e))?;
+
+        let process_times = process
+            .cpu_times()
+            .map_err(|e| format!("Unable to get process cpu times : {:?}", e))?;
+
+        Ok(Self {
+            pid: process.pid(),
+            pid_num_threads: stat.num_threads,
+            pid_mem_resident_set_size: process_mem.rss(),
+            pid_mem_virtual_memory_size: process_mem.vms(),
+            pid_mem_shared_memory_size: process_mem.shared(),
+            pid_process_seconds_total: process_times.busy().as_secs()
+                + process_times.children_system().as_secs()
+                + process_times.children_system().as_secs(),
+        })
+    }
+}

common/monitoring_api/Cargo.toml

@@ -7,6 +7,7 @@ edition = { workspace = true }
 
 [dependencies]
 eth2 = { workspace = true }
+health_metrics = { workspace = true }
 lighthouse_version = { workspace = true }
 metrics = { workspace = true }
 regex = { workspace = true }

common/monitoring_api/src/gather.rs

@@ -1,4 +1,5 @@
 use super::types::{BeaconProcessMetrics, ValidatorProcessMetrics};
+use health_metrics::observe::Observe;
 use metrics::{MetricFamily, MetricType};
 use serde_json::json;
 use std::collections::HashMap;

common/monitoring_api/src/lib.rs

@@ -4,6 +4,7 @@ use std::{path::PathBuf, time::Duration};
 
 use eth2::lighthouse::SystemHealth;
 use gather::{gather_beacon_metrics, gather_validator_metrics};
+use health_metrics::observe::Observe;
 use reqwest::{IntoUrl, Response};
 pub use reqwest::{StatusCode, Url};
 use sensitive_url::SensitiveUrl;

common/validator_dir/Cargo.toml

@@ -12,7 +12,6 @@ insecure_keys = []
 bls = { workspace = true }
 deposit_contract = { workspace = true }
 derivative = { workspace = true }
-directory = { workspace = true }
 eth2_keystore = { workspace = true }
 filesystem = { workspace = true }
 hex = { workspace = true }

common/validator_dir/src/builder.rs

@@ -1,7 +1,6 @@
 use crate::{Error as DirError, ValidatorDir};
 use bls::get_withdrawal_credentials;
 use deposit_contract::{encode_eth1_tx_data, Error as DepositError};
-use directory::ensure_dir_exists;
 use eth2_keystore::{Error as KeystoreError, Keystore, KeystoreBuilder, PlainText};
 use filesystem::create_with_600_perms;
 use rand::{distributions::Alphanumeric, Rng};
@@ -42,7 +41,7 @@ pub enum Error {
     #[cfg(feature = "insecure_keys")]
     InsecureKeysError(String),
     MissingPasswordDir,
-    UnableToCreatePasswordDir(String),
+    UnableToCreatePasswordDir(io::Error),
 }
 
 impl From<KeystoreError> for Error {
@@ -163,7 +162,7 @@ impl<'a> Builder<'a> {
         }
 
         if let Some(password_dir) = &self.password_dir {
-            ensure_dir_exists(password_dir).map_err(Error::UnableToCreatePasswordDir)?;
+            create_dir_all(password_dir).map_err(Error::UnableToCreatePasswordDir)?;
         }
 
         // The withdrawal keystore must be initialized in order to store it or create an eth1

common/warp_utils/Cargo.toml

@@ -6,7 +6,6 @@ edition = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-beacon_chain = { workspace = true }
 bytes = { workspace = true }
 eth2 = { workspace = true }
 headers = "0.3.2"
@@ -15,7 +14,6 @@ safe_arith = { workspace = true }
 serde = { workspace = true }
 serde_array_query = "0.1.0"
 serde_json = { workspace = true }
-state_processing = { workspace = true }
 tokio = { workspace = true }
 types = { workspace = true }
 warp = { workspace = true }

common/warp_utils/src/lib.rs

@@ -3,7 +3,6 @@
 
 pub mod cors;
 pub mod json;
-pub mod metrics;
 pub mod query;
 pub mod reject;
 pub mod task;

common/warp_utils/src/reject.rs

@@ -2,6 +2,7 @@ use eth2::types::{ErrorMessage, Failure, IndexedErrorMessage};
 use std::convert::Infallible;
 use std::error::Error;
 use std::fmt;
+use std::fmt::Debug;
 use warp::{http::StatusCode, reject::Reject, reply::Response, Reply};
 
 #[derive(Debug)]
@@ -19,15 +20,6 @@ pub fn server_sent_event_error(s: String) -> ServerSentEventError {
     ServerSentEventError(s)
 }
 
-#[derive(Debug)]
-pub struct BeaconChainError(pub beacon_chain::BeaconChainError);
-
-impl Reject for BeaconChainError {}
-
-pub fn beacon_chain_error(e: beacon_chain::BeaconChainError) -> warp::reject::Rejection {
-    warp::reject::custom(BeaconChainError(e))
-}
-
 #[derive(Debug)]
 pub struct BeaconStateError(pub types::BeaconStateError);
 
@@ -47,21 +39,12 @@ pub fn arith_error(e: safe_arith::ArithError) -> warp::reject::Rejection {
 }
 
 #[derive(Debug)]
-pub struct SlotProcessingError(pub state_processing::SlotProcessingError);
+pub struct UnhandledError(pub Box<dyn Debug + Send + Sync + 'static>);
 
-impl Reject for SlotProcessingError {}
+impl Reject for UnhandledError {}
 
-pub fn slot_processing_error(e: state_processing::SlotProcessingError) -> warp::reject::Rejection {
-    warp::reject::custom(SlotProcessingError(e))
-}
-
-#[derive(Debug)]
-pub struct BlockProductionError(pub beacon_chain::BlockProductionError);
-
-impl Reject for BlockProductionError {}
-
-pub fn block_production_error(e: beacon_chain::BlockProductionError) -> warp::reject::Rejection {
-    warp::reject::custom(BlockProductionError(e))
+pub fn unhandled_error<D: Debug + Send + Sync + 'static>(e: D) -> warp::reject::Rejection {
+    warp::reject::custom(UnhandledError(Box::new(e)))
 }
 
 #[derive(Debug)]
@@ -191,16 +174,7 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result<impl warp::Reply,
     } else if let Some(e) = err.find::<warp::reject::InvalidQuery>() {
         code = StatusCode::BAD_REQUEST;
         message = format!("BAD_REQUEST: invalid query: {}", e);
-    } else if let Some(e) = err.find::<crate::reject::BeaconChainError>() {
-        code = StatusCode::INTERNAL_SERVER_ERROR;
-        message = format!("UNHANDLED_ERROR: {:?}", e.0);
-    } else if let Some(e) = err.find::<crate::reject::BeaconStateError>() {
-        code = StatusCode::INTERNAL_SERVER_ERROR;
-        message = format!("UNHANDLED_ERROR: {:?}", e.0);
-    } else if let Some(e) = err.find::<crate::reject::SlotProcessingError>() {
-        code = StatusCode::INTERNAL_SERVER_ERROR;
-        message = format!("UNHANDLED_ERROR: {:?}", e.0);
-    } else if let Some(e) = err.find::<crate::reject::BlockProductionError>() {
+    } else if let Some(e) = err.find::<crate::reject::UnhandledError>() {
         code = StatusCode::INTERNAL_SERVER_ERROR;
         message = format!("UNHANDLED_ERROR: {:?}", e.0);
     } else if let Some(e) = err.find::<crate::reject::CustomNotFound>() {