Move notifier and latency service to validator_services (#7427)

We would like to reuse the `notifier` and `latency_service` in Anchor. To make this possible, this PR moves these from `validator_client` to `validator_services` and makes them use the new `ValidatorStore` trait is used so that the code can be reused in Anchor.

validator_client/validator_services/src/latency_service.rs

@@ -0,0 +1,60 @@
+use beacon_node_fallback::BeaconNodeFallback;
+use slot_clock::SlotClock;
+use std::sync::Arc;
+use task_executor::TaskExecutor;
+use tokio::time::sleep;
+use tracing::debug;
+
+/// The latency service will run 11/12ths of the way through the slot.
+pub const SLOT_DELAY_MULTIPLIER: u32 = 11;
+pub const SLOT_DELAY_DENOMINATOR: u32 = 12;
+
+/// Starts a service that periodically checks the latency between the VC and the
+/// candidate BNs.
+pub fn start_latency_service<T: SlotClock + 'static>(
+    executor: TaskExecutor,
+    slot_clock: T,
+    beacon_nodes: Arc<BeaconNodeFallback<T>>,
+) {
+    let future = async move {
+        loop {
+            let sleep_time = slot_clock
+                .duration_to_next_slot()
+                .map(|next_slot| {
+                    // This is 11/12ths through the next slot. On mainnet this
+                    // will happen in the 11th second of each slot, one second
+                    // before the next slot.
+                    next_slot + (next_slot / SLOT_DELAY_DENOMINATOR) * SLOT_DELAY_MULTIPLIER
+                })
+                // If we can't read the slot clock, just wait one slot. Running
+                // the measurement at a non-exact time is not a big issue.
+                .unwrap_or_else(|| slot_clock.slot_duration());
+
+            // Sleep until it's time to perform the measurement.
+            sleep(sleep_time).await;
+
+            for (i, measurement) in beacon_nodes.measure_latency().await.iter().enumerate() {
+                if let Some(latency) = measurement.latency {
+                    debug!(
+                        node = &measurement.beacon_node_id,
+                        latency = latency.as_millis(),
+                        "Measured BN latency"
+                    );
+                    validator_metrics::observe_timer_vec(
+                        &validator_metrics::VC_BEACON_NODE_LATENCY,
+                        &[&measurement.beacon_node_id],
+                        latency,
+                    );
+                    if i == 0 {
+                        validator_metrics::observe_duration(
+                            &validator_metrics::VC_BEACON_NODE_LATENCY_PRIMARY_ENDPOINT,
+                            latency,
+                        );
+                    }
+                }
+            }
+        }
+    };
+
+    executor.spawn(future, "latency");
+}