Improve eth1 block sync (#2008)

## Issue Addressed

NA

## Proposed Changes

- Log about eth1 whilst waiting for genesis.
- For the block and deposit caches, update them after each download instead of when *all* downloads are complete.
  - This prevents the case where a single timeout error can cause us to drop *all* previously download blocks/deposits.
- Set `max_log_requests_per_update` to avoid timeouts due to very large log counts in a response.
- Set `max_blocks_per_update` to prevent a single update of the block cache to download an unreasonable number of blocks.
  - This shouldn't have any affect in normal use, it's just a safe-guard against bugs.
- Increase the timeout for eth1 calls from 15s to 60s, as per @pawanjay176's experience with Infura.

## Additional Info

NA

beacon_node/client/src/notifier.rs

@@ -3,7 +3,7 @@ use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::NetworkGlobals;
 use futures::prelude::*;
 use parking_lot::Mutex;
-use slog::{debug, error, info, warn};
+use slog::{debug, error, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -56,6 +56,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                         "peers" => peer_count_pretty(network.connected_peers()),
                         "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)),
                     );
+                    eth1_logging(&beacon_chain, &log);
                     sleep(slot_duration).await;
                 }
                 _ => break,
@@ -172,37 +173,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
                 );
             }
 
-            // Perform some logging about the eth1 chain
-            if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
-                if let Some(status) =
-                    eth1_chain.sync_status(head_info.genesis_time, current_slot, &beacon_chain.spec)
-                {
-                    debug!(
-                        log,
-                        "Eth1 cache sync status";
-                        "eth1_head_block" => status.head_block_number,
-                        "latest_cached_block_number" => status.latest_cached_block_number,
-                        "latest_cached_timestamp" => status.latest_cached_block_timestamp,
-                        "voting_target_timestamp" => status.voting_target_timestamp,
-                        "ready" => status.lighthouse_is_cached_and_ready
-                    );
-
-                    if !status.lighthouse_is_cached_and_ready {
-                        warn!(
-                            log,
-                            "Syncing eth1 block cache";
-                            "target_timestamp" => status.voting_target_timestamp,
-                            "latest_timestamp" => status.latest_cached_block_timestamp,
-                            "msg" => "block production temporarily impaired"
-                        );
-                    }
-                } else {
-                    error!(
-                        log,
-                        "Unable to determine eth1 sync status";
-                    );
-                }
-            }
+            eth1_logging(&beacon_chain, &log);
         }
         Ok::<(), ()>(())
     };
@@ -213,6 +184,59 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
     Ok(())
 }
 
+fn eth1_logging<T: BeaconChainTypes>(beacon_chain: &BeaconChain<T>, log: &Logger) {
+    let current_slot_opt = beacon_chain.slot().ok();
+
+    if let Ok(head_info) = beacon_chain.head_info() {
+        // Perform some logging about the eth1 chain
+        if let Some(eth1_chain) = beacon_chain.eth1_chain.as_ref() {
+            if let Some(status) =
+                eth1_chain.sync_status(head_info.genesis_time, current_slot_opt, &beacon_chain.spec)
+            {
+                debug!(
+                    log,
+                    "Eth1 cache sync status";
+                    "eth1_head_block" => status.head_block_number,
+                    "latest_cached_block_number" => status.latest_cached_block_number,
+                    "latest_cached_timestamp" => status.latest_cached_block_timestamp,
+                    "voting_target_timestamp" => status.voting_target_timestamp,
+                    "ready" => status.lighthouse_is_cached_and_ready
+                );
+
+                if !status.lighthouse_is_cached_and_ready {
+                    let voting_target_timestamp = status.voting_target_timestamp;
+
+                    let distance = status
+                        .latest_cached_block_timestamp
+                        .map(|latest| {
+                            voting_target_timestamp.saturating_sub(latest)
+                                / beacon_chain.spec.seconds_per_eth1_block
+                        })
+                        .map(|distance| distance.to_string())
+                        .unwrap_or_else(|| "initializing deposits".to_string());
+
+                    warn!(
+                        log,
+                        "Syncing eth1 block cache";
+                        "msg" => "sync can take longer when using remote eth1 nodes",
+                        "est_blocks_remaining" => distance,
+                    );
+                }
+            } else {
+                error!(
+                    log,
+                    "Unable to determine eth1 sync status";
+                );
+            }
+        }
+    } else {
+        error!(
+            log,
+            "Unable to get head info";
+        );
+    }
+}
+
 /// Returns the peer count, returning something helpful if it's `usize::max_value` (effectively a
 /// `None` value).
 fn peer_count_pretty(peer_count: usize) -> String {