Cache validator balances and allow them to be served over the HTTP API (#3863)

## Issue Addressed

#3804

## Proposed Changes

- Add `total_balance` to the validator monitor and adjust the number of historical epochs which are cached. 
- Allow certain values in the cache to be served out via the HTTP API without requiring a state read.

## Usage
```
curl -X POST "http://localhost:5052/lighthouse/ui/validator_info" -d '{"indices": [0]}' -H "Content-Type: application/json" | jq
```

```
{
  "data": {
    "validators": {
      "0": {
        "info": [
          {
            "epoch": 172981,
            "total_balance": 36566388519
          },
         ...
          {
            "epoch": 172990,
            "total_balance": 36566496513
          }
        ]
      },
      "1": {
        "info": [
          {
            "epoch": 172981,
            "total_balance": 36355797968
          },
          ...
          {
            "epoch": 172990,
            "total_balance": 36355905962
          }
        ]
      }
    }
  }
}
```

## Additional Info

This requires no historical states to operate which mean it will still function on the freshly checkpoint synced node, however because of this, the values will populate each epoch (up to a maximum of 10 entries).

Another benefit of this method, is that we can easily cache any other values which would normally require a state read and serve them via the same endpoint. However, we would need be cautious about not overly increasing block processing time by caching values from complex computations.

This also caches some of the validator metrics directly, rather than pulling them from the Prometheus metrics when the API is called. This means when the validator count exceeds the individual monitor threshold, the cached values will still be available.

Co-authored-by: Paul Hauner <paul@paulhauner.com>

beacon_node/http_api/Cargo.toml

@@ -36,6 +36,7 @@ tree_hash = "0.4.1"
 sysinfo = "0.26.5"
 system_health = { path = "../../common/system_health" }
 directory = { path = "../../common/directory" }
+eth2_serde_utils = "0.1.1"
 
 [dev-dependencies]
 store = { path = "../store" }

beacon_node/http_api/src/lib.rs

@@ -3025,6 +3025,22 @@ pub fn serve<T: BeaconChainTypes>(
             },
         );
 
+    // POST lighthouse/ui/validator_info
+    let post_lighthouse_ui_validator_info = warp::path("lighthouse")
+        .and(warp::path("ui"))
+        .and(warp::path("validator_info"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(chain_filter.clone())
+        .and_then(
+            |request_data: ui::ValidatorInfoRequestData, chain: Arc<BeaconChain<T>>| {
+                blocking_json_task(move || {
+                    ui::get_validator_info(request_data, chain)
+                        .map(api_types::GenericResponse::from)
+                })
+            },
+        );
+
     // GET lighthouse/syncing
     let get_lighthouse_syncing = warp::path("lighthouse")
         .and(warp::path("syncing"))
@@ -3522,6 +3538,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(post_lighthouse_database_historical_blocks.boxed())
                 .or(post_lighthouse_block_rewards.boxed())
                 .or(post_lighthouse_ui_validator_metrics.boxed())
+                .or(post_lighthouse_ui_validator_info.boxed())
                 .recover(warp_utils::reject::handle_rejection),
         ))
         .recover(warp_utils::reject::handle_rejection)

beacon_node/http_api/src/ui.rs

@@ -1,5 +1,7 @@
-use beacon_chain::{metrics, BeaconChain, BeaconChainError, BeaconChainTypes};
-use eth2::types::ValidatorStatus;
+use beacon_chain::{
+    validator_monitor::HISTORIC_EPOCHS, BeaconChain, BeaconChainError, BeaconChainTypes,
+};
+use eth2::types::{Epoch, ValidatorStatus};
 use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::sync::Arc;
@@ -71,6 +73,82 @@ pub fn get_validator_count<T: BeaconChainTypes>(
     })
 }
 
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorInfoRequestData {
+    #[serde(with = "eth2_serde_utils::quoted_u64_vec")]
+    indices: Vec<u64>,
+}
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorInfoValues {
+    #[serde(with = "eth2_serde_utils::quoted_u64")]
+    epoch: u64,
+    #[serde(with = "eth2_serde_utils::quoted_u64")]
+    total_balance: u64,
+}
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorInfo {
+    info: Vec<ValidatorInfoValues>,
+}
+
+#[derive(PartialEq, Serialize, Deserialize)]
+pub struct ValidatorInfoResponse {
+    validators: HashMap<String, ValidatorInfo>,
+}
+
+pub fn get_validator_info<T: BeaconChainTypes>(
+    request_data: ValidatorInfoRequestData,
+    chain: Arc<BeaconChain<T>>,
+) -> Result<ValidatorInfoResponse, warp::Rejection> {
+    let current_epoch = chain.epoch().map_err(beacon_chain_error)?;
+
+    let epochs = current_epoch.saturating_sub(HISTORIC_EPOCHS).as_u64()..=current_epoch.as_u64();
+
+    let validator_ids = chain
+        .validator_monitor
+        .read()
+        .get_all_monitored_validators()
+        .iter()
+        .cloned()
+        .collect::<HashSet<String>>();
+
+    let indices = request_data
+        .indices
+        .iter()
+        .map(|index| index.to_string())
+        .collect::<HashSet<String>>();
+
+    let ids = validator_ids
+        .intersection(&indices)
+        .collect::<HashSet<&String>>();
+
+    let mut validators = HashMap::new();
+
+    for id in ids {
+        if let Ok(index) = id.parse::<u64>() {
+            if let Some(validator) = chain
+                .validator_monitor
+                .read()
+                .get_monitored_validator(index)
+            {
+                let mut info = vec![];
+                for epoch in epochs.clone() {
+                    if let Some(total_balance) = validator.get_total_balance(Epoch::new(epoch)) {
+                        info.push(ValidatorInfoValues {
+                            epoch,
+                            total_balance,
+                        });
+                    }
+                }
+                validators.insert(id.clone(), ValidatorInfo { info });
+            }
+        }
+    }
+
+    Ok(ValidatorInfoResponse { validators })
+}
+
 #[derive(PartialEq, Serialize, Deserialize)]
 pub struct ValidatorMetricsRequestData {
     indices: Vec<u64>,
@@ -119,76 +197,56 @@ pub fn post_validator_monitor_metrics<T: BeaconChainTypes>(
     let mut validators = HashMap::new();
 
     for id in ids {
-        let attestation_hits = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_HIT,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let attestation_misses = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_ATTESTER_MISS,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let attestations = attestation_hits + attestation_misses;
-        let attestation_hit_percentage: f64 = if attestations == 0 {
-            0.0
-        } else {
-            (100 * attestation_hits / attestations) as f64
-        };
+        if let Ok(index) = id.parse::<u64>() {
+            if let Some(validator) = chain
+                .validator_monitor
+                .read()
+                .get_monitored_validator(index)
+            {
+                let val_metrics = validator.metrics.read();
+                let attestation_hits = val_metrics.attestation_hits;
+                let attestation_misses = val_metrics.attestation_misses;
+                let attestation_head_hits = val_metrics.attestation_head_hits;
+                let attestation_head_misses = val_metrics.attestation_head_misses;
+                let attestation_target_hits = val_metrics.attestation_target_hits;
+                let attestation_target_misses = val_metrics.attestation_target_misses;
+                drop(val_metrics);
 
-        let attestation_head_hits = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_HIT,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let attestation_head_misses = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_HEAD_ATTESTER_MISS,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let head_attestations = attestation_head_hits + attestation_head_misses;
-        let attestation_head_hit_percentage: f64 = if head_attestations == 0 {
-            0.0
-        } else {
-            (100 * attestation_head_hits / head_attestations) as f64
-        };
+                let attestations = attestation_hits + attestation_misses;
+                let attestation_hit_percentage: f64 = if attestations == 0 {
+                    0.0
+                } else {
+                    (100 * attestation_hits / attestations) as f64
+                };
+                let head_attestations = attestation_head_hits + attestation_head_misses;
+                let attestation_head_hit_percentage: f64 = if head_attestations == 0 {
+                    0.0
+                } else {
+                    (100 * attestation_head_hits / head_attestations) as f64
+                };
 
-        let attestation_target_hits = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_HIT,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let attestation_target_misses = metrics::get_int_counter(
-            &metrics::VALIDATOR_MONITOR_PREV_EPOCH_ON_CHAIN_TARGET_ATTESTER_MISS,
-            &[id],
-        )
-        .map(|counter| counter.get())
-        .unwrap_or(0);
-        let target_attestations = attestation_target_hits + attestation_target_misses;
-        let attestation_target_hit_percentage: f64 = if target_attestations == 0 {
-            0.0
-        } else {
-            (100 * attestation_target_hits / target_attestations) as f64
-        };
+                let target_attestations = attestation_target_hits + attestation_target_misses;
+                let attestation_target_hit_percentage: f64 = if target_attestations == 0 {
+                    0.0
+                } else {
+                    (100 * attestation_target_hits / target_attestations) as f64
+                };
 
-        let metrics = ValidatorMetrics {
-            attestation_hits,
-            attestation_misses,
-            attestation_hit_percentage,
-            attestation_head_hits,
-            attestation_head_misses,
-            attestation_head_hit_percentage,
-            attestation_target_hits,
-            attestation_target_misses,
-            attestation_target_hit_percentage,
-        };
+                let metrics = ValidatorMetrics {
+                    attestation_hits,
+                    attestation_misses,
+                    attestation_hit_percentage,
+                    attestation_head_hits,
+                    attestation_head_misses,
+                    attestation_head_hit_percentage,
+                    attestation_target_hits,
+                    attestation_target_misses,
+                    attestation_target_hit_percentage,
+                };
 
-        validators.insert(id.clone(), metrics);
+                validators.insert(id.clone(), metrics);
+            }
+        }
     }
 
     Ok(ValidatorMetricsResponse { validators })