Add /node/health endpoint (#1197)

* Start adding health endpoint

* Use psutil more

* Add get_health test

* Expose health to Prom

* Update comments

* Add /node/health to docs

* Update Prom naming

beacon_node/rest_api/src/metrics.rs

@@ -3,6 +3,7 @@ use crate::{ApiError, ApiResult};
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use hyper::{Body, Request};
 use lighthouse_metrics::{Encoder, TextEncoder};
+use rest_types::Health;
 use std::path::PathBuf;
 use std::sync::Arc;
 
@@ -36,6 +37,38 @@ lazy_static! {
             "http_server_validator_duties_get_request_duration_seconds",
             "Time taken to respond to GET /validator/duties"
         );
+    pub static ref PROCESS_NUM_THREADS: Result<IntGauge> = try_create_int_gauge(
+        "process_num_threads",
+        "Number of threads used by the current process"
+    );
+    pub static ref PROCESS_RES_MEM: Result<IntGauge> = try_create_int_gauge(
+        "process_resident_memory_bytes",
+        "Resident memory used by the current process"
+    );
+    pub static ref PROCESS_VIRT_MEM: Result<IntGauge> = try_create_int_gauge(
+        "process_virtual_memory_bytes",
+        "Virtual memory used by the current process"
+    );
+    pub static ref SYSTEM_VIRT_MEM_TOTAL: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_total_bytes", "Total system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_AVAILABLE: Result<IntGauge> = try_create_int_gauge(
+        "system_virt_mem_available_bytes",
+        "Available system virtual memory"
+    );
+    pub static ref SYSTEM_VIRT_MEM_USED: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_used_bytes", "Used system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_FREE: Result<IntGauge> =
+        try_create_int_gauge("system_virt_mem_free_bytes", "Free system virtual memory");
+    pub static ref SYSTEM_VIRT_MEM_PERCENTAGE: Result<Gauge> = try_create_float_gauge(
+        "system_virt_mem_percentage",
+        "Percentage of used virtual memory"
+    );
+    pub static ref SYSTEM_LOADAVG_1: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_1", "Loadavg over 1 minute");
+    pub static ref SYSTEM_LOADAVG_5: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_5", "Loadavg over 5 minutes");
+    pub static ref SYSTEM_LOADAVG_15: Result<Gauge> =
+        try_create_float_gauge("system_loadavg_15", "Loadavg over 15 minutes");
 }
 
 /// Returns the full set of Prometheus metrics for the Beacon Node application.
@@ -72,6 +105,26 @@ pub fn get_prometheus<T: BeaconChainTypes>(
     store::scrape_for_metrics(&db_path, &freezer_db_path);
     beacon_chain::scrape_for_metrics(&beacon_chain);
 
+    if let Ok(health) = Health::observe() {
+        set_gauge(&PROCESS_NUM_THREADS, health.pid_num_threads as i64);
+        set_gauge(&PROCESS_RES_MEM, health.pid_mem_resident_set_size as i64);
+        set_gauge(&PROCESS_VIRT_MEM, health.pid_mem_virtual_memory_size as i64);
+        set_gauge(&SYSTEM_VIRT_MEM_TOTAL, health.sys_virt_mem_total as i64);
+        set_gauge(
+            &SYSTEM_VIRT_MEM_AVAILABLE,
+            health.sys_virt_mem_available as i64,
+        );
+        set_gauge(&SYSTEM_VIRT_MEM_USED, health.sys_virt_mem_used as i64);
+        set_gauge(&SYSTEM_VIRT_MEM_FREE, health.sys_virt_mem_free as i64);
+        set_float_gauge(
+            &SYSTEM_VIRT_MEM_PERCENTAGE,
+            health.sys_virt_mem_percent as f64,
+        );
+        set_float_gauge(&SYSTEM_LOADAVG_1, health.sys_loadavg_1);
+        set_float_gauge(&SYSTEM_LOADAVG_5, health.sys_loadavg_5);
+        set_float_gauge(&SYSTEM_LOADAVG_15, health.sys_loadavg_15);
+    }
+
     encoder
         .encode(&lighthouse_metrics::gather(), &mut buffer)
         .unwrap();

beacon_node/rest_api/src/node.rs

@@ -1,8 +1,8 @@
 use crate::response_builder::ResponseBuilder;
-use crate::ApiResult;
+use crate::{ApiError, ApiResult};
 use eth2_libp2p::{types::SyncState, NetworkGlobals};
 use hyper::{Body, Request};
-use rest_types::{SyncingResponse, SyncingStatus};
+use rest_types::{Health, SyncingResponse, SyncingStatus};
 use std::sync::Arc;
 use types::{EthSpec, Slot};
 use version;
@@ -41,3 +41,9 @@ pub fn syncing<T: EthSpec>(
         sync_status,
     })
 }
+
+pub fn get_health(req: Request<Body>) -> ApiResult {
+    let health = Health::observe().map_err(|e| ApiError::ServerError(e))?;
+
+    ResponseBuilder::new(&req)?.body_no_ssz(&health)
+}

beacon_node/rest_api/src/router.rs

@@ -33,6 +33,7 @@ pub async fn route<T: BeaconChainTypes>(
     let log = local_log.clone();
     let request_result = match (req.method(), path.as_ref()) {
         // Methods for Client
+        (&Method::GET, "/node/health") => node::get_health(req),
         (&Method::GET, "/node/version") => node::get_version(req),
         (&Method::GET, "/node/syncing") => {
             // inform the current slot, or set to 0

beacon_node/rest_api/tests/test.rs

@@ -1252,3 +1252,15 @@ mod validator_attestation {
         );
     }
 }
+
+#[test]
+fn get_health() {
+    let mut env = build_env();
+
+    let node = build_node(&mut env, testing_client_config());
+    let remote_node = node.remote_node().expect("should produce remote node");
+
+    env.runtime()
+        .block_on(remote_node.http.node().get_health())
+        .unwrap();
+}