Fallback nodes for eth1 access (#1918)

## Issue Addressed

part of  #1883

## Proposed Changes

Adds a new cli argument `--eth1-endpoints` that can be used instead of `--eth1-endpoint` to specify a comma-separated list of endpoints. If the first endpoint returns an error for some request the other endpoints are tried in the given order.

## Additional Info

Currently if the first endpoint fails the fallbacks are used silently (except for `try_fallback_test_endpoint` that is used in `do_update` which logs a `WARN` for each endpoint that is not reachable). A question is if we should add more logs so that the user gets warned if his main endpoint is for example just slow and sometimes hits timeouts.

testing/eth1_test_rig/src/ganache.rs

@@ -14,9 +14,6 @@ use web3::{
 /// How long we will wait for ganache to indicate that it is ready.
 const GANACHE_STARTUP_TIMEOUT_MILLIS: u64 = 10_000;
 
-const NETWORK_ID: u64 = 42;
-const CHAIN_ID: u64 = 42;
-
 /// Provides a dedicated `ganachi-cli` instance with a connected `Web3` instance.
 ///
 /// Requires that `ganachi-cli` is installed and available on `PATH`.
@@ -25,39 +22,17 @@ pub struct GanacheInstance {
     child: Child,
     _event_loop: Arc<EventLoopHandle>,
     pub web3: Web3<Http>,
+    network_id: u64,
+    chain_id: u64,
 }
 
 impl GanacheInstance {
-    /// Start a new `ganache-cli` process, waiting until it indicates that it is ready to accept
-    /// RPC connections.
-    pub fn new() -> Result<Self, String> {
-        let port = unused_port()?;
-
-        let mut child = Command::new("ganache-cli")
-            .stdout(Stdio::piped())
-            .arg("--defaultBalanceEther")
-            .arg("1000000000")
-            .arg("--gasLimit")
-            .arg("1000000000")
-            .arg("--accounts")
-            .arg("10")
-            .arg("--port")
-            .arg(format!("{}", port))
-            .arg("--mnemonic")
-            .arg("\"vast thought differ pull jewel broom cook wrist tribe word before omit\"")
-            .arg("--networkId")
-            .arg(format!("{}", NETWORK_ID))
-            .arg("--chainId")
-            .arg(format!("{}", CHAIN_ID))
-            .spawn()
-            .map_err(|e| {
-                format!(
-                    "Failed to start ganache-cli. \
-                     Is it ganache-cli installed and available on $PATH? Error: {:?}",
-                    e
-                )
-            })?;
-
+    fn new_from_child(
+        mut child: Child,
+        port: u16,
+        network_id: u64,
+        chain_id: u64,
+    ) -> Result<Self, String> {
         let stdout = child
             .stdout
             .ok_or_else(|| "Unable to get stdout for ganache child process")?;
@@ -96,9 +71,67 @@ impl GanacheInstance {
             port,
             _event_loop: Arc::new(event_loop),
             web3,
+            network_id,
+            chain_id,
         })
     }
 
+    /// Start a new `ganache-cli` process, waiting until it indicates that it is ready to accept
+    /// RPC connections.
+    pub fn new(network_id: u64, chain_id: u64) -> Result<Self, String> {
+        let port = unused_port()?;
+
+        let child = Command::new("ganache-cli")
+            .stdout(Stdio::piped())
+            .arg("--defaultBalanceEther")
+            .arg("1000000000")
+            .arg("--gasLimit")
+            .arg("1000000000")
+            .arg("--accounts")
+            .arg("10")
+            .arg("--port")
+            .arg(format!("{}", port))
+            .arg("--mnemonic")
+            .arg("\"vast thought differ pull jewel broom cook wrist tribe word before omit\"")
+            .arg("--networkId")
+            .arg(format!("{}", network_id))
+            .arg("--chainId")
+            .arg(format!("{}", chain_id))
+            .spawn()
+            .map_err(|e| {
+                format!(
+                    "Failed to start ganache-cli. \
+                     Is it ganache-cli installed and available on $PATH? Error: {:?}",
+                    e
+                )
+            })?;
+
+        Self::new_from_child(child, port, network_id, chain_id)
+    }
+
+    pub fn fork(&self) -> Result<Self, String> {
+        let port = unused_port()?;
+
+        let child = Command::new("ganache-cli")
+            .stdout(Stdio::piped())
+            .arg("--fork")
+            .arg(self.endpoint())
+            .arg("--port")
+            .arg(format!("{}", port))
+            .arg("--chainId")
+            .arg(format!("{}", self.chain_id))
+            .spawn()
+            .map_err(|e| {
+                format!(
+                    "Failed to start ganache-cli. \
+                     Is it ganache-cli installed and available on $PATH? Error: {:?}",
+                    e
+                )
+            })?;
+
+        Self::new_from_child(child, port, self.network_id, self.chain_id)
+    }
+
     /// Returns the endpoint that this instance is listening on.
     pub fn endpoint(&self) -> String {
         endpoint(self.port)
@@ -106,12 +139,12 @@ impl GanacheInstance {
 
     /// Returns the network id of the ganache instance
     pub fn network_id(&self) -> u64 {
-        NETWORK_ID
+        self.network_id
     }
 
     /// Returns the chain id of the ganache instance
     pub fn chain_id(&self) -> u64 {
-        CHAIN_ID
+        self.chain_id
     }
 
     /// Increase the timestamp on future blocks by `increase_by` seconds.

testing/eth1_test_rig/src/lib.rs

@@ -31,8 +31,8 @@ pub struct GanacheEth1Instance {
 }
 
 impl GanacheEth1Instance {
-    pub async fn new() -> Result<Self, String> {
-        let ganache = GanacheInstance::new()?;
+    pub async fn new(network_id: u64, chain_id: u64) -> Result<Self, String> {
+        let ganache = GanacheInstance::new(network_id, chain_id)?;
         DepositContract::deploy(ganache.web3.clone(), 0, None)
             .await
             .map(|deposit_contract| Self {

testing/simulator/src/eth1_sim.rs

@@ -1,6 +1,7 @@
 use crate::{checks, LocalNetwork, E};
 use clap::ArgMatches;
 use eth1::http::Eth1Id;
+use eth1::{DEFAULT_CHAIN_ID, DEFAULT_NETWORK_ID};
 use eth1_test_rig::GanacheEth1Instance;
 use futures::prelude::*;
 use node_test_rig::{
@@ -73,7 +74,8 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
          * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit
          * validators.
          */
-        let ganache_eth1_instance = GanacheEth1Instance::new().await?;
+        let ganache_eth1_instance =
+            GanacheEth1Instance::new(DEFAULT_NETWORK_ID.into(), DEFAULT_CHAIN_ID.into()).await?;
         let deposit_contract = ganache_eth1_instance.deposit_contract;
         let network_id = ganache_eth1_instance.ganache.network_id();
         let chain_id = ganache_eth1_instance.ganache.chain_id();
@@ -102,15 +104,16 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> {
         let mut beacon_config = testing_client_config();
 
         beacon_config.genesis = ClientGenesis::DepositContract;
-        beacon_config.eth1.endpoint = eth1_endpoint;
+        beacon_config.eth1.endpoints = vec![eth1_endpoint];
         beacon_config.eth1.deposit_contract_address = deposit_contract_address;
         beacon_config.eth1.deposit_contract_deploy_block = 0;
         beacon_config.eth1.lowest_cached_block_number = 0;
         beacon_config.eth1.follow_distance = 1;
+        beacon_config.eth1.node_far_behind_seconds = 20;
         beacon_config.dummy_eth1_backend = false;
         beacon_config.sync_eth1_chain = true;
-        beacon_config.eth1.network_id = Eth1Id::Custom(network_id);
-        beacon_config.eth1.chain_id = Eth1Id::Custom(chain_id);
+        beacon_config.eth1.network_id = Eth1Id::from(network_id);
+        beacon_config.eth1.chain_id = Eth1Id::from(chain_id);
 
         beacon_config.network.enr_address = Some(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)));