Add timeouts to canonical head rwlock (#759)

* Add TimeoutRwLock to BeaconChain

* Update network crate

* Update rest api

* Fix beacon chain tests

* Fix rest api tests

* Set test back to !debug_assertions

beacon_node/network/src/sync/manager.rs

@@ -229,7 +229,16 @@ impl<T: BeaconChainTypes> SyncManager<T> {
             }
         };
 
-        let local = PeerSyncInfo::from(&chain);
+        let local = match PeerSyncInfo::from_chain(&chain) {
+            Some(local) => local,
+            None => {
+                return error!(
+                    self.log,
+                    "Failed to get peer sync info";
+                    "msg" => "likely due to head lock contention"
+                )
+            }
+        };
 
         // If a peer is within SLOT_IMPORT_TOLERANCE from our head slot, ignore a batch/range sync,
         // consider it a fully-sync'd peer.

beacon_node/network/src/sync/network_context.rs

@@ -43,7 +43,9 @@ impl SyncNetworkContext {
             "peer" => format!("{:?}", peer_id)
         );
         if let Some(chain) = chain.upgrade() {
-            let _ = self.send_rpc_request(peer_id, RPCRequest::Status(status_message(&chain)));
+            if let Some(status_message) = status_message(&chain) {
+                let _ = self.send_rpc_request(peer_id, RPCRequest::Status(status_message));
+            }
         }
     }
 

beacon_node/network/src/sync/range_sync/chain_collection.rs

@@ -8,7 +8,7 @@ use crate::message_processor::PeerSyncInfo;
 use crate::sync::network_context::SyncNetworkContext;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::PeerId;
-use slog::{debug, warn};
+use slog::{debug, error, warn};
 use std::sync::Weak;
 use types::EthSpec;
 use types::{Hash256, Slot};
@@ -103,9 +103,22 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     /// updates the state of the collection.
     pub fn update_finalized(&mut self, network: &mut SyncNetworkContext, log: &slog::Logger) {
         let local_slot = match self.beacon_chain.upgrade() {
-            Some(chain) => PeerSyncInfo::from(&chain)
-                .finalized_epoch
-                .start_slot(T::EthSpec::slots_per_epoch()),
+            Some(chain) => {
+                let local = match PeerSyncInfo::from_chain(&chain) {
+                    Some(local) => local,
+                    None => {
+                        return error!(
+                            log,
+                            "Failed to get peer sync info";
+                            "msg" => "likely due to head lock contention"
+                        )
+                    }
+                };
+
+                local
+                    .finalized_epoch
+                    .start_slot(T::EthSpec::slots_per_epoch())
+            }
             None => {
                 warn!(log, "Beacon chain dropped. Chains not updated");
                 return;
@@ -113,7 +126,7 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
         };
 
         // Remove any outdated finalized chains
-        self.purge_outdated_chains(network);
+        self.purge_outdated_chains(network, log);
 
         // Check if any chains become the new syncing chain
         if let Some(index) = self.finalized_syncing_index() {
@@ -248,14 +261,23 @@ impl<T: BeaconChainTypes> ChainCollection<T> {
     ///
     /// This removes chains with no peers, or chains whose start block slot is less than our current
     /// finalized block slot.
-    pub fn purge_outdated_chains(&mut self, network: &mut SyncNetworkContext) {
+    pub fn purge_outdated_chains(&mut self, network: &mut SyncNetworkContext, log: &slog::Logger) {
         // Remove any chains that have no peers
         self.finalized_chains
             .retain(|chain| !chain.peer_pool.is_empty());
         self.head_chains.retain(|chain| !chain.peer_pool.is_empty());
 
         let local_info = match self.beacon_chain.upgrade() {
-            Some(chain) => PeerSyncInfo::from(&chain),
+            Some(chain) => match PeerSyncInfo::from_chain(&chain) {
+                Some(local) => local,
+                None => {
+                    return error!(
+                        log,
+                        "Failed to get peer sync info";
+                        "msg" => "likely due to head lock contention"
+                    )
+                }
+            },
             None => {
                 return;
             }

beacon_node/network/src/sync/range_sync/range.rs

@@ -46,7 +46,7 @@ use crate::sync::network_context::SyncNetworkContext;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use eth2_libp2p::rpc::RequestId;
 use eth2_libp2p::PeerId;
-use slog::{debug, trace, warn};
+use slog::{debug, error, trace, warn};
 use std::collections::HashSet;
 use std::sync::Weak;
 use types::{BeaconBlock, EthSpec};
@@ -106,7 +106,16 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         // determine if we need to run a sync to the nearest finalized state or simply sync to
         // its current head
         let local_info = match self.beacon_chain.upgrade() {
-            Some(chain) => PeerSyncInfo::from(&chain),
+            Some(chain) => match PeerSyncInfo::from_chain(&chain) {
+                Some(local) => local,
+                None => {
+                    return error!(
+                        self.log,
+                        "Failed to get peer sync info";
+                        "msg" => "likely due to head lock contention"
+                    )
+                }
+            },
             None => {
                 warn!(self.log,
                       "Beacon chain dropped. Peer not considered for sync";
@@ -127,7 +136,7 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         self.remove_peer(network, &peer_id);
 
         // remove any out-of-date chains
-        self.chains.purge_outdated_chains(network);
+        self.chains.purge_outdated_chains(network, &self.log);
 
         if remote_finalized_slot > local_info.head_slot {
             debug!(self.log, "Finalization sync peer joined"; "peer_id" => format!("{:?}", peer_id));