From panic to crit (#1726)

## Issue Addressed
Downgrade inconsistent chain segment states from `panic` to `crit`. I don't love this solution but since range can always bounce back from any of those, we don't panic.

Co-authored-by: Age Manning <Age@AgeManning.com>

beacon_node/network/src/sync/range_sync/batch.rs

@@ -1,6 +1,7 @@
 use crate::sync::RequestId;
 use eth2_libp2p::rpc::methods::BlocksByRangeRequest;
 use eth2_libp2p::PeerId;
+use slog::{crit, warn, Logger};
 use ssz::Encode;
 use std::collections::HashSet;
 use std::hash::{Hash, Hasher};
@@ -125,13 +126,17 @@ impl<T: EthSpec> BatchInfo<T> {
     }
 
     /// Adds a block to a downloading batch.
-    pub fn add_block(&mut self, block: SignedBeaconBlock<T>) {
+    pub fn add_block(&mut self, block: SignedBeaconBlock<T>, logger: &Logger) {
         match self.state.poison() {
             BatchState::Downloading(peer, mut blocks, req_id) => {
                 blocks.push(block);
                 self.state = BatchState::Downloading(peer, blocks, req_id)
             }
-            other => unreachable!("Add block for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Add block for batch in wrong state"; "state" => ?other);
+                self.state = other
+            }
         }
     }
 
@@ -140,14 +145,8 @@ impl<T: EthSpec> BatchInfo<T> {
     #[must_use = "Batch may have failed"]
     pub fn download_completed(
         &mut self,
-    ) -> Result<
-        usize, /* Received blocks */
-        (
-            Slot, /* expected slot */
-            Slot, /* received slot */
-            &BatchState<T>,
-        ),
-    > {
+        logger: &Logger,
+    ) -> Result<usize /* Received blocks */, &BatchState<T>> {
         match self.state.poison() {
             BatchState::Downloading(peer, blocks, _request_id) => {
                 // verify that blocks are in range
@@ -163,7 +162,7 @@ impl<T: EthSpec> BatchInfo<T> {
                         None
                     };
 
-                    if let Some(range) = failed_range {
+                    if let Some((expected, received)) = failed_range {
                         // this is a failed download, register the attempt and check if the batch
                         // can be tried again
                         self.failed_download_attempts.push(peer);
@@ -175,7 +174,9 @@ impl<T: EthSpec> BatchInfo<T> {
                             // drop the blocks
                             BatchState::AwaitingDownload
                         };
-                        return Err((range.0, range.1, &self.state));
+                        warn!(logger, "Batch received out of range blocks";
+                            &self, "expected" => expected, "received" => received);
+                        return Err(&self.state);
                     }
                 }
 
@@ -183,12 +184,17 @@ impl<T: EthSpec> BatchInfo<T> {
                 self.state = BatchState::AwaitingProcessing(peer, blocks);
                 Ok(received)
             }
-            other => unreachable!("Download completed for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Download completed for batch in wrong state"; "state" => ?other);
+                self.state = other;
+                Err(&self.state)
+            }
         }
     }
 
     #[must_use = "Batch may have failed"]
-    pub fn download_failed(&mut self) -> &BatchState<T> {
+    pub fn download_failed(&mut self, logger: &Logger) -> &BatchState<T> {
         match self.state.poison() {
             BatchState::Downloading(peer, _, _request_id) => {
                 // register the attempt and check if the batch can be tried again
@@ -203,31 +209,50 @@ impl<T: EthSpec> BatchInfo<T> {
                 };
                 &self.state
             }
-            other => unreachable!("Download failed for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Download failed for batch in wrong state"; "state" => ?other);
+                self.state = other;
+                &self.state
+            }
         }
     }
 
-    pub fn start_downloading_from_peer(&mut self, peer: PeerId, request_id: RequestId) {
+    pub fn start_downloading_from_peer(
+        &mut self,
+        peer: PeerId,
+        request_id: RequestId,
+        logger: &Logger,
+    ) {
         match self.state.poison() {
             BatchState::AwaitingDownload => {
                 self.state = BatchState::Downloading(peer, Vec::new(), request_id);
             }
-            other => unreachable!("Starting download for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Starting download for batch in wrong state"; "state" => ?other);
+                self.state = other
+            }
         }
     }
 
-    pub fn start_processing(&mut self) -> Vec<SignedBeaconBlock<T>> {
+    pub fn start_processing(&mut self, logger: &Logger) -> Vec<SignedBeaconBlock<T>> {
         match self.state.poison() {
             BatchState::AwaitingProcessing(peer, blocks) => {
                 self.state = BatchState::Processing(Attempt::new(peer, &blocks));
                 blocks
             }
-            other => unreachable!("Start processing for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Starting procesing batch in wrong state"; "state" => ?other);
+                self.state = other;
+                vec![]
+            }
         }
     }
 
     #[must_use = "Batch may have failed"]
-    pub fn processing_completed(&mut self, was_sucessful: bool) -> &BatchState<T> {
+    pub fn processing_completed(&mut self, was_sucessful: bool, logger: &Logger) -> &BatchState<T> {
         match self.state.poison() {
             BatchState::Processing(attempt) => {
                 self.state = if !was_sucessful {
@@ -247,12 +272,17 @@ impl<T: EthSpec> BatchInfo<T> {
                 };
                 &self.state
             }
-            other => unreachable!("Processing completed for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Procesing completed for batch in wrong state"; "state" => ?other);
+                self.state = other;
+                &self.state
+            }
         }
     }
 
     #[must_use = "Batch may have failed"]
-    pub fn validation_failed(&mut self) -> &BatchState<T> {
+    pub fn validation_failed(&mut self, logger: &Logger) -> &BatchState<T> {
         match self.state.poison() {
             BatchState::AwaitingValidation(attempt) => {
                 self.failed_processing_attempts.push(attempt);
@@ -267,7 +297,12 @@ impl<T: EthSpec> BatchInfo<T> {
                 };
                 &self.state
             }
-            other => unreachable!("Validation failed for batch in wrong state: {:?}", other),
+            BatchState::Poisoned => unreachable!("Poisoned batch"),
+            other => {
+                crit!(logger, "Validation failed for batch in wrong state"; "state" => ?other);
+                self.state = other;
+                &self.state
+            }
         }
     }
 }