BN Fallback v2 (#2080)

## Issue Addressed

- Resolves #1883

## Proposed Changes

This follows on from @blacktemplar's work in #2018.

- Allows the VC to connect to multiple BN for redundancy.
  - Update the simulator so some nodes always need to rely on their fallback.
- Adds some extra deprecation warnings for `--eth1-endpoint`
- Pass `SignatureBytes` as a reference instead of by value.

## Additional Info

NA

Co-authored-by: blacktemplar <blacktemplar@a1.net>

validator_client/src/block_service.rs

@@ -1,6 +1,7 @@
+use crate::beacon_node_fallback::{BeaconNodeFallback, RequireSynced};
 use crate::{http_metrics::metrics, validator_store::ValidatorStore};
 use environment::RuntimeContext;
-use eth2::{types::Graffiti, BeaconNodeHttpClient};
+use eth2::types::Graffiti;
 use futures::channel::mpsc::Receiver;
 use futures::{StreamExt, TryFutureExt};
 use slog::{crit, debug, error, info, trace, warn};
@@ -13,7 +14,7 @@ use types::{EthSpec, PublicKey, Slot};
 pub struct BlockServiceBuilder<T, E: EthSpec> {
     validator_store: Option<ValidatorStore<T, E>>,
     slot_clock: Option<Arc<T>>,
-    beacon_node: Option<BeaconNodeHttpClient>,
+    beacon_nodes: Option<Arc<BeaconNodeFallback<T, E>>>,
     context: Option<RuntimeContext<E>>,
     graffiti: Option<Graffiti>,
 }
@@ -23,7 +24,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockServiceBuilder<T, E> {
         Self {
             validator_store: None,
             slot_clock: None,
-            beacon_node: None,
+            beacon_nodes: None,
             context: None,
             graffiti: None,
         }
@@ -39,8 +40,8 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockServiceBuilder<T, E> {
         self
     }
 
-    pub fn beacon_node(mut self, beacon_node: BeaconNodeHttpClient) -> Self {
-        self.beacon_node = Some(beacon_node);
+    pub fn beacon_nodes(mut self, beacon_nodes: Arc<BeaconNodeFallback<T, E>>) -> Self {
+        self.beacon_nodes = Some(beacon_nodes);
         self
     }
 
@@ -63,8 +64,8 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockServiceBuilder<T, E> {
                 slot_clock: self
                     .slot_clock
                     .ok_or("Cannot build BlockService without slot_clock")?,
-                beacon_node: self
-                    .beacon_node
+                beacon_nodes: self
+                    .beacon_nodes
                     .ok_or("Cannot build BlockService without beacon_node")?,
                 context: self
                     .context
@@ -79,7 +80,7 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockServiceBuilder<T, E> {
 pub struct Inner<T, E: EthSpec> {
     validator_store: ValidatorStore<T, E>,
     slot_clock: Arc<T>,
-    beacon_node: BeaconNodeHttpClient,
+    beacon_nodes: Arc<BeaconNodeFallback<T, E>>,
     context: RuntimeContext<E>,
     graffiti: Option<Graffiti>,
 }
@@ -222,24 +223,37 @@ impl<T: SlotClock + 'static, E: EthSpec> BlockService<T, E> {
         let randao_reveal = self
             .validator_store
             .randao_reveal(&validator_pubkey, slot.epoch(E::slots_per_epoch()))
-            .ok_or("Unable to produce randao reveal")?;
-
-        let block = self
-            .beacon_node
-            .get_validator_blocks(slot, randao_reveal.into(), self.graffiti.as_ref())
-            .await
-            .map_err(|e| format!("Error from beacon node when producing block: {:?}", e))?
-            .data;
+            .ok_or("Unable to produce randao reveal")?
+            .into();
 
+        let randao_reveal_ref = &randao_reveal;
+        let self_ref = &self;
+        let validator_pubkey_ref = &validator_pubkey;
         let signed_block = self
-            .validator_store
-            .sign_block(&validator_pubkey, block, current_slot)
-            .ok_or("Unable to sign block")?;
+            .beacon_nodes
+            .first_success(RequireSynced::No, |beacon_node| async move {
+                let block = beacon_node
+                    .get_validator_blocks(slot, randao_reveal_ref, self_ref.graffiti.as_ref())
+                    .await
+                    .map_err(|e| format!("Error from beacon node when producing block: {:?}", e))?
+                    .data;
 
-        self.beacon_node
-            .post_beacon_blocks(&signed_block)
+                let signed_block = self_ref
+                    .validator_store
+                    .sign_block(validator_pubkey_ref, block, current_slot)
+                    .ok_or("Unable to sign block")?;
+
+                beacon_node
+                    .post_beacon_blocks(&signed_block)
+                    .await
+                    .map_err(|e| {
+                        format!("Error from beacon node when publishing block: {:?}", e)
+                    })?;
+
+                Ok::<_, String>(signed_block)
+            })
             .await
-            .map_err(|e| format!("Error from beacon node when publishing block: {:?}", e))?;
+            .map_err(|e| e.to_string())?;
 
         info!(
             log,