Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

beacon_node/network/src/sync/range_sync/range.rs

@@ -39,7 +39,7 @@
 //!  Each chain is downloaded in batches of blocks. The batched blocks are processed sequentially
 //!  and further batches are requested as current blocks are being processed.
 
-use super::chain::{ChainId, RemoveChain, SyncingChain};
+use super::chain::{BatchId, ChainId, RemoveChain, SyncingChain};
 use super::chain_collection::ChainCollection;
 use super::sync_type::RangeSyncType;
 use crate::beacon_processor::WorkEvent as BeaconWorkEvent;
@@ -194,34 +194,29 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         peer_id: PeerId,
+        chain_id: ChainId,
+        batch_id: BatchId,
         request_id: RequestId,
         beacon_block: Option<SignedBeaconBlock<T::EthSpec>>,
     ) {
-        // get the chain and batch for which this response belongs
-        if let Some((chain_id, batch_id)) =
-            network.blocks_by_range_response(request_id, beacon_block.is_none())
-        {
-            // check if this chunk removes the chain
-            match self.chains.call_by_id(chain_id, |chain| {
-                chain.on_block_response(network, batch_id, &peer_id, request_id, beacon_block)
-            }) {
-                Ok((removed_chain, sync_type)) => {
-                    if let Some((removed_chain, remove_reason)) = removed_chain {
-                        self.on_chain_removed(
-                            removed_chain,
-                            sync_type,
-                            remove_reason,
-                            network,
-                            "block response",
-                        );
-                    }
-                }
-                Err(_) => {
-                    trace!(self.log, "BlocksByRange response for removed chain"; "chain" => chain_id)
+        // check if this chunk removes the chain
+        match self.chains.call_by_id(chain_id, |chain| {
+            chain.on_block_response(network, batch_id, &peer_id, request_id, beacon_block)
+        }) {
+            Ok((removed_chain, sync_type)) => {
+                if let Some((removed_chain, remove_reason)) = removed_chain {
+                    self.on_chain_removed(
+                        removed_chain,
+                        sync_type,
+                        remove_reason,
+                        network,
+                        "block response",
+                    );
                 }
             }
-        } else {
-            trace!(self.log, "Response/Error for non registered request"; "request_id" => request_id)
+            Err(_) => {
+                trace!(self.log, "BlocksByRange response for removed chain"; "chain" => chain_id)
+            }
         }
     }
 
@@ -298,31 +293,28 @@ impl<T: BeaconChainTypes> RangeSync<T> {
         &mut self,
         network: &mut SyncNetworkContext<T::EthSpec>,
         peer_id: PeerId,
+        batch_id: BatchId,
+        chain_id: ChainId,
         request_id: RequestId,
     ) {
-        // get the chain and batch for which this response belongs
-        if let Some((chain_id, batch_id)) = network.blocks_by_range_response(request_id, true) {
-            // check that this request is pending
-            match self.chains.call_by_id(chain_id, |chain| {
-                chain.inject_error(network, batch_id, &peer_id, request_id)
-            }) {
-                Ok((removed_chain, sync_type)) => {
-                    if let Some((removed_chain, remove_reason)) = removed_chain {
-                        self.on_chain_removed(
-                            removed_chain,
-                            sync_type,
-                            remove_reason,
-                            network,
-                            "RPC error",
-                        );
-                    }
-                }
-                Err(_) => {
-                    trace!(self.log, "BlocksByRange response for removed chain"; "chain" => chain_id)
+        // check that this request is pending
+        match self.chains.call_by_id(chain_id, |chain| {
+            chain.inject_error(network, batch_id, &peer_id, request_id)
+        }) {
+            Ok((removed_chain, sync_type)) => {
+                if let Some((removed_chain, remove_reason)) = removed_chain {
+                    self.on_chain_removed(
+                        removed_chain,
+                        sync_type,
+                        remove_reason,
+                        network,
+                        "RPC error",
+                    );
                 }
             }
-        } else {
-            trace!(self.log, "Response/Error for non registered request"; "request_id" => request_id)
+            Err(_) => {
+                trace!(self.log, "BlocksByRange response for removed chain"; "chain" => chain_id)
+            }
         }
     }