Revert bad blocks on missed fork (#2529)

## Issue Addressed

Closes #2526

## Proposed Changes

If the head block fails to decode on start up, do two things:

1. Revert all blocks between the head and the most recent hard fork (to `fork_slot - 1`).
2. Reset fork choice so that it contains the new head, and all blocks back to the new head's finalized checkpoint.

## Additional Info

I tweaked some of the beacon chain test harness stuff in order to make it generic enough to test with a non-zero slot clock on start-up. In the process I consolidated all the various `new_` methods into a single generic one which will hopefully serve all future uses 🤞

beacon_node/store/src/hot_cold_store.rs

@@ -262,22 +262,40 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
             return Ok(Some(block.clone()));
         }
 
-        // Fetch from database.
-        match self
-            .hot_db
-            .get_bytes(DBColumn::BeaconBlock.into(), block_root.as_bytes())?
-        {
-            Some(block_bytes) => {
-                // Deserialize.
-                let block = SignedBeaconBlock::from_ssz_bytes(&block_bytes, &self.spec)?;
+        let block = self.get_block_with(block_root, |bytes| {
+            SignedBeaconBlock::from_ssz_bytes(bytes, &self.spec)
+        })?;
 
-                // Add to cache.
-                self.block_cache.lock().put(*block_root, block.clone());
-
-                Ok(Some(block))
-            }
-            None => Ok(None),
+        // Add to cache.
+        if let Some(ref block) = block {
+            self.block_cache.lock().put(*block_root, block.clone());
         }
+
+        Ok(block)
+    }
+
+    /// Fetch a block from the store, ignoring which fork variant it *should* be for.
+    pub fn get_block_any_variant(
+        &self,
+        block_root: &Hash256,
+    ) -> Result<Option<SignedBeaconBlock<E>>, Error> {
+        self.get_block_with(block_root, SignedBeaconBlock::any_from_ssz_bytes)
+    }
+
+    /// Fetch a block from the store using a custom decode function.
+    ///
+    /// This is useful for e.g. ignoring the slot-indicated fork to forcefully load a block as if it
+    /// were for a different fork.
+    pub fn get_block_with(
+        &self,
+        block_root: &Hash256,
+        decoder: impl FnOnce(&[u8]) -> Result<SignedBeaconBlock<E>, ssz::DecodeError>,
+    ) -> Result<Option<SignedBeaconBlock<E>>, Error> {
+        self.hot_db
+            .get_bytes(DBColumn::BeaconBlock.into(), block_root.as_bytes())?
+            .map(|block_bytes| decoder(&block_bytes))
+            .transpose()
+            .map_err(|e| e.into())
     }
 
     /// Determine whether a block exists in the database.
@@ -766,7 +784,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
     ///
     /// Blocks are returned in slot-ascending order, suitable for replaying on a state with slot
     /// equal to `start_slot`, to reach a state with slot equal to `end_slot`.
-    fn load_blocks_to_replay(
+    pub fn load_blocks_to_replay(
         &self,
         start_slot: Slot,
         end_slot: Slot,

beacon_node/store/src/iter.rs

@@ -232,6 +232,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> Iterator
 pub struct ParentRootBlockIterator<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
     store: &'a HotColdDB<E, Hot, Cold>,
     next_block_root: Hash256,
+    decode_any_variant: bool,
     _phantom: PhantomData<E>,
 }
 
@@ -242,6 +243,17 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
         Self {
             store,
             next_block_root: start_block_root,
+            decode_any_variant: false,
+            _phantom: PhantomData,
+        }
+    }
+
+    /// Block iterator that is tolerant of blocks that have the wrong fork for their slot.
+    pub fn fork_tolerant(store: &'a HotColdDB<E, Hot, Cold>, start_block_root: Hash256) -> Self {
+        Self {
+            store,
+            next_block_root: start_block_root,
+            decode_any_variant: true,
             _phantom: PhantomData,
         }
     }
@@ -253,10 +265,12 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
             Ok(None)
         } else {
             let block_root = self.next_block_root;
-            let block = self
-                .store
-                .get_block(&block_root)?
-                .ok_or(Error::BlockNotFound(block_root))?;
+            let block = if self.decode_any_variant {
+                self.store.get_block_any_variant(&block_root)
+            } else {
+                self.store.get_block(&block_root)
+            }?
+            .ok_or(Error::BlockNotFound(block_root))?;
             self.next_block_root = block.message().parent_root();
             Ok(Some((block_root, block)))
         }