Detailed validator monitoring (#2151)

## Issue Addressed

- Resolves #2064

## Proposed Changes

Adds a `ValidatorMonitor` struct which provides additional logging and Grafana metrics for specific validators.

Use `lighthouse bn --validator-monitor` to automatically enable monitoring for any validator that hits the [subnet subscription](https://ethereum.github.io/eth2.0-APIs/#/Validator/prepareBeaconCommitteeSubnet) HTTP API endpoint.

Also, use `lighthouse bn --validator-monitor-pubkeys` to supply a list of validators which will always be monitored.

See the new docs included in this PR for more info.

## TODO

- [x] Track validator balance, `slashed` status, etc.
- [x] ~~Register slashings in current epoch, not offense epoch~~
- [ ] Publish Grafana dashboard, update TODO link in docs
- [x] ~~#2130 is merged into this branch, resolve that~~

common/lighthouse_metrics/src/lib.rs

@@ -282,6 +282,12 @@ pub fn inc_counter_by(counter: &Result<IntCounter>, value: u64) {
     }
 }
 
+pub fn set_gauge_vec(int_gauge_vec: &Result<IntGaugeVec>, name: &[&str], value: i64) {
+    if let Some(gauge) = get_int_gauge(int_gauge_vec, name) {
+        gauge.set(value);
+    }
+}
+
 pub fn set_gauge(gauge: &Result<IntGauge>, value: i64) {
     if let Ok(gauge) = gauge {
         gauge.set(value);

common/slot_clock/src/lib.rs

@@ -59,6 +59,9 @@ pub trait SlotClock: Send + Sync + Sized {
     /// Returns the duration until the first slot of the next epoch.
     fn duration_to_next_epoch(&self, slots_per_epoch: u64) -> Option<Duration>;
 
+    /// Returns the start time of the slot, as a duration since `UNIX_EPOCH`.
+    fn start_of(&self, slot: Slot) -> Option<Duration>;
+
     /// Returns the first slot to be returned at the genesis time.
     fn genesis_slot(&self) -> Slot;
 

common/slot_clock/src/manual_slot_clock.rs

@@ -45,18 +45,6 @@ impl ManualSlotClock {
         &self.genesis_duration
     }
 
-    /// Returns the duration between UNIX epoch and the start of `slot`.
-    pub fn start_of(&self, slot: Slot) -> Option<Duration> {
-        let slot = slot
-            .as_u64()
-            .checked_sub(self.genesis_slot.as_u64())?
-            .try_into()
-            .ok()?;
-        let unadjusted_slot_duration = self.slot_duration.checked_mul(slot)?;
-
-        self.genesis_duration.checked_add(unadjusted_slot_duration)
-    }
-
     /// Returns the duration from `now` until the start of `slot`.
     ///
     /// Will return `None` if `now` is later than the start of `slot`.
@@ -147,6 +135,18 @@ impl SlotClock for ManualSlotClock {
         self.duration_to_slot(slot, *self.current_time.read())
     }
 
+    /// Returns the duration between UNIX epoch and the start of `slot`.
+    fn start_of(&self, slot: Slot) -> Option<Duration> {
+        let slot = slot
+            .as_u64()
+            .checked_sub(self.genesis_slot.as_u64())?
+            .try_into()
+            .ok()?;
+        let unadjusted_slot_duration = self.slot_duration.checked_mul(slot)?;
+
+        self.genesis_duration.checked_add(unadjusted_slot_duration)
+    }
+
     fn genesis_slot(&self) -> Slot {
         self.genesis_slot
     }

common/slot_clock/src/system_time_slot_clock.rs

@@ -54,6 +54,10 @@ impl SlotClock for SystemTimeSlotClock {
         self.clock.duration_to_slot(slot, now)
     }
 
+    fn start_of(&self, slot: Slot) -> Option<Duration> {
+        self.clock.start_of(slot)
+    }
+
     fn genesis_slot(&self) -> Slot {
         self.clock.genesis_slot()
     }