Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

consensus/types/src/beacon_block.rs

@@ -197,6 +197,11 @@ impl<'a, T: EthSpec> BeaconBlockRef<'a, T> {
         }
     }
 
+    /// Returns the epoch corresponding to `self.slot()`.
+    pub fn epoch(&self) -> Epoch {
+        self.slot().epoch(T::slots_per_epoch())
+    }
+
     /// Returns a full `BeaconBlockHeader` of this block.
     pub fn block_header(&self) -> BeaconBlockHeader {
         BeaconBlockHeader {

consensus/types/src/beacon_state/tests.rs

@@ -1,15 +1,21 @@
 #![cfg(test)]
 use crate::test_utils::*;
+use crate::test_utils::{SeedableRng, XorShiftRng};
 use beacon_chain::store::config::StoreConfig;
-use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
+use beacon_chain::test_utils::{
+    interop_genesis_state, test_spec, BeaconChainHarness, EphemeralHarnessType,
+};
 use beacon_chain::types::{
     test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError,
     ChainSpec, CloneConfig, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec,
     MinimalEthSpec, RelativeEpoch, Slot,
 };
+use safe_arith::SafeArith;
 use ssz::{Decode, Encode};
+use state_processing::per_slot_processing;
 use std::ops::Mul;
 use swap_or_not_shuffle::compute_shuffled_index;
+use tree_hash::TreeHash;
 
 pub const MAX_VALIDATOR_COUNT: usize = 129;
 pub const SLOT_OFFSET: Slot = Slot::new(1);
@@ -489,9 +495,6 @@ fn decode_base_and_altair() {
 
 #[test]
 fn tree_hash_cache_linear_history() {
-    use crate::test_utils::{SeedableRng, XorShiftRng};
-    use tree_hash::TreeHash;
-
     let mut rng = XorShiftRng::from_seed([42; 16]);
 
     let mut state: BeaconState<MainnetEthSpec> =
@@ -545,3 +548,59 @@ fn tree_hash_cache_linear_history() {
     let root = state.update_tree_hash_cache().unwrap();
     assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
 }
+
+// Check how the cache behaves when there's a distance larger than `SLOTS_PER_HISTORICAL_ROOT`
+// since its last update.
+#[test]
+fn tree_hash_cache_linear_history_long_skip() {
+    let validator_count = 128;
+    let keypairs = generate_deterministic_keypairs(validator_count);
+
+    let spec = &test_spec::<MinimalEthSpec>();
+
+    // This state has a cache that advances normally each slot.
+    let mut state: BeaconState<MinimalEthSpec> = interop_genesis_state(&keypairs, 0, spec).unwrap();
+
+    state.update_tree_hash_cache().unwrap();
+
+    // This state retains its original cache until it is updated after a long skip.
+    let mut original_cache_state = state.clone();
+    assert!(original_cache_state.tree_hash_cache().is_initialized());
+
+    // Advance the states to a slot beyond the historical state root limit, using the state root
+    // from the first state to avoid touching the original state's cache.
+    let start_slot = state.slot();
+    let target_slot = start_slot
+        .safe_add(MinimalEthSpec::slots_per_historical_root() as u64 + 1)
+        .unwrap();
+
+    let mut prev_state_root;
+    while state.slot() < target_slot {
+        prev_state_root = state.update_tree_hash_cache().unwrap();
+        per_slot_processing(&mut state, None, spec).unwrap();
+        per_slot_processing(&mut original_cache_state, Some(prev_state_root), spec).unwrap();
+    }
+
+    // The state with the original cache should still be initialized at the starting slot.
+    assert_eq!(
+        original_cache_state
+            .tree_hash_cache()
+            .initialized_slot()
+            .unwrap(),
+        start_slot
+    );
+
+    // Updating the tree hash cache should be successful despite the long skip.
+    assert_eq!(
+        original_cache_state.update_tree_hash_cache().unwrap(),
+        state.update_tree_hash_cache().unwrap()
+    );
+
+    assert_eq!(
+        original_cache_state
+            .tree_hash_cache()
+            .initialized_slot()
+            .unwrap(),
+        target_slot
+    );
+}

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -118,6 +118,13 @@ impl<T: EthSpec> BeaconTreeHashCache<T> {
     pub fn uninitialize(&mut self) {
         self.inner = None;
     }
+
+    /// Return the slot at which the cache was last updated.
+    ///
+    /// This should probably only be used during testing.
+    pub fn initialized_slot(&self) -> Option<Slot> {
+        Some(self.inner.as_ref()?.previous_state?.1)
+    }
 }
 
 #[derive(Debug, PartialEq, Clone)]
@@ -206,7 +213,8 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
     /// Updates the cache and returns the tree hash root for the given `state`.
     ///
     /// The provided `state` should be a descendant of the last `state` given to this function, or
-    /// the `Self::new` function.
+    /// the `Self::new` function. If the state is more than `SLOTS_PER_HISTORICAL_ROOT` slots
+    /// after `self.previous_state` then the whole cache will be re-initialized.
     pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState<T>) -> Result<Hash256, Error> {
         // If this cache has previously produced a root, ensure that it is in the state root
         // history of this state.
@@ -224,10 +232,15 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             }
 
             // If the state is newer, the previous root must be in the history of the given state.
-            if previous_slot < state.slot()
-                && *state.get_state_root(previous_slot)? != previous_root
-            {
-                return Err(Error::NonLinearTreeHashCacheHistory);
+            // If the previous slot is out of range of the `state_roots` array (indicating a long
+            // gap between the cache's last use and the current state) then we re-initialize.
+            match state.get_state_root(previous_slot) {
+                Ok(state_previous_root) if *state_previous_root == previous_root => {}
+                Ok(_) => return Err(Error::NonLinearTreeHashCacheHistory),
+                Err(Error::SlotOutOfBounds) => {
+                    *self = Self::new(state);
+                }
+                Err(e) => return Err(e),
             }
         }