Re-add NoPeers error

beacon_node/network/src/sync/backfill_sync/mod.rs

@@ -301,6 +301,14 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
 
     pub fn peer_disconnected(&mut self, peer_id: &PeerId) {
         self.peers.write().remove(peer_id);
+
+        if self.peers.read().is_empty() {
+            info!(
+                "reason" = "insufficient_synced_peers",
+                "Backfill sync paused"
+            );
+            self.set_state(BackFillState::Paused);
+        }
     }
 
     /// An RPC error has occurred.
@@ -946,6 +954,15 @@ impl<T: BeaconChainTypes> BackFillSync<T> {
                     return Ok(());
                 }
                 Err(e) => match e {
+                    RpcRequestSendError::NoPeers => {
+                        // If we are here the chain has no more synced peers
+                        info!(
+                            "reason" = "insufficient_synced_peers",
+                            "Backfill sync paused"
+                        );
+                        self.set_state(BackFillState::Paused);
+                        return Err(BackFillError::Paused);
+                    }
                     RpcRequestSendError::InternalError(e) => {
                         // NOTE: under normal conditions this shouldn't happen but we handle it anyway
                         warn!(%batch_id, error = ?e, %batch,"Could not send batch request");

beacon_node/network/src/sync/network_context.rs

@@ -98,6 +98,7 @@ pub enum RpcRequestSendError {
     // If RpcRequestSendError has a single variant `InternalError` it's to signal to downstream
     // consumers that sends are expected to be infallible. If this assumption changes in the future,
     // add a new variant.
+    NoPeers,
 }
 
 #[derive(Debug, PartialEq, Eq)]

beacon_node/network/src/sync/network_context/block_components_by_range.rs

@@ -138,9 +138,7 @@ impl<T: BeaconChainTypes> BlockComponentsByRangeRequest<T> {
         else {
             // When a peer disconnects and is removed from the SyncingChain peer set, if the set
             // reaches zero the SyncingChain is removed.
-            return Err(RpcRequestSendError::InternalError(
+            return Err(RpcRequestSendError::NoPeers);
-                "A batch peer set should never be empty".to_string(),
-            ));
         };
 
         let blocks_req_id = cx.send_blocks_by_range_request(block_peer, request.clone(), id)?;
@@ -269,6 +267,10 @@ impl<T: BeaconChainTypes> BlockComponentsByRangeRequest<T> {
                                     RpcRequestSendError::InternalError(e) => {
                                         Error::InternalError(e)
                                     }
+                                    RpcRequestSendError::NoPeers => Error::InternalError(
+                                        "send_custody_by_range_request does not error with NoPeers"
+                                            .to_owned(),
+                                    ),
                                 })?;
 
                             *state = FuluEnabledState::CustodyRequest {

beacon_node/network/src/sync/range_sync/chain.rs

@@ -957,7 +957,7 @@ impl<T: BeaconChainTypes> SyncingChain<T> {
                     return Ok(KeepChain);
                 }
                 Err(e) => match e {
-                    RpcRequestSendError::InternalError(e) => {
+                    e @ (RpcRequestSendError::NoPeers | RpcRequestSendError::InternalError(_)) => {
                         // NOTE: under normal conditions this shouldn't happen but we handle it anyway
                         warn!(%batch_id, error = ?e, "batch_id" = %batch_id, %batch, "Could not send batch request");
                         // register the failed download and check if the batch can be retried