Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

beacon_node/beacon_chain/tests/store_tests.rs

@@ -1,11 +1,16 @@
 #![cfg(not(debug_assertions))]
 
 use beacon_chain::attestation_verification::Error as AttnError;
+use beacon_chain::builder::BeaconChainBuilder;
 use beacon_chain::test_utils::{
     test_logger, test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy,
     DiskHarnessType, HARNESS_SLOT_TIME,
 };
-use beacon_chain::{BeaconChain, BeaconChainTypes, BeaconSnapshot, ChainConfig};
+use beacon_chain::{
+    historical_blocks::HistoricalBlockError, migrate::MigratorConfig, BeaconChain,
+    BeaconChainError, BeaconChainTypes, BeaconSnapshot, ChainConfig, ServerSentEventHandler,
+    WhenSlotSkipped,
+};
 use lazy_static::lazy_static;
 use maplit::hashset;
 use rand::Rng;
@@ -558,7 +563,7 @@ fn multiple_attestations_per_block() {
     let harness = get_harness(store, HIGH_VALIDATOR_COUNT);
 
     harness.extend_chain(
-        MainnetEthSpec::slots_per_epoch() as usize * 3,
+        E::slots_per_epoch() as usize * 3,
         BlockStrategy::OnCanonicalHead,
         AttestationStrategy::AllValidators,
     );
@@ -1741,6 +1746,173 @@ fn garbage_collect_temp_states_from_failed_block() {
     assert_eq!(store.iter_temporary_state_roots().count(), 0);
 }
 
+#[test]
+fn weak_subjectivity_sync() {
+    // Build an initial chain on one harness, representing a synced node with full history.
+    let num_initial_blocks = E::slots_per_epoch() * 11;
+    let num_final_blocks = E::slots_per_epoch() * 2;
+
+    let temp1 = tempdir().unwrap();
+    let full_store = get_store(&temp1);
+    let harness = get_harness(full_store.clone(), LOW_VALIDATOR_COUNT);
+
+    harness.extend_chain(
+        num_initial_blocks as usize,
+        BlockStrategy::OnCanonicalHead,
+        AttestationStrategy::AllValidators,
+    );
+
+    let genesis_state = full_store
+        .get_state(&harness.chain.genesis_state_root, Some(Slot::new(0)))
+        .unwrap()
+        .unwrap();
+    let wss_checkpoint = harness.chain.head_info().unwrap().finalized_checkpoint;
+    let wss_block = harness.get_block(wss_checkpoint.root.into()).unwrap();
+    let wss_state = full_store
+        .get_state(&wss_block.state_root(), None)
+        .unwrap()
+        .unwrap();
+    let wss_slot = wss_block.slot();
+
+    // Add more blocks that advance finalization further.
+    harness.advance_slot();
+    harness.extend_chain(
+        num_final_blocks as usize,
+        BlockStrategy::OnCanonicalHead,
+        AttestationStrategy::AllValidators,
+    );
+
+    let (shutdown_tx, _shutdown_rx) = futures::channel::mpsc::channel(1);
+    let log = test_logger();
+    let temp2 = tempdir().unwrap();
+    let store = get_store(&temp2);
+
+    // Initialise a new beacon chain from the finalized checkpoint
+    let beacon_chain = BeaconChainBuilder::new(MinimalEthSpec)
+        .store(store.clone())
+        .custom_spec(test_spec::<E>())
+        .weak_subjectivity_state(wss_state, wss_block.clone(), genesis_state)
+        .unwrap()
+        .logger(log.clone())
+        .store_migrator_config(MigratorConfig::default().blocking())
+        .dummy_eth1_backend()
+        .expect("should build dummy backend")
+        .testing_slot_clock(HARNESS_SLOT_TIME)
+        .expect("should configure testing slot clock")
+        .shutdown_sender(shutdown_tx)
+        .chain_config(ChainConfig::default())
+        .event_handler(Some(ServerSentEventHandler::new_with_capacity(
+            log.clone(),
+            1,
+        )))
+        .monitor_validators(true, vec![], log)
+        .build()
+        .expect("should build");
+
+    // Apply blocks forward to reach head.
+    let chain_dump = harness.chain.chain_dump().unwrap();
+    let new_blocks = &chain_dump[wss_slot.as_usize() + 1..];
+
+    assert_eq!(new_blocks[0].beacon_block.slot(), wss_slot + 1);
+
+    for snapshot in new_blocks {
+        let block = &snapshot.beacon_block;
+        beacon_chain.slot_clock.set_slot(block.slot().as_u64());
+        beacon_chain.process_block(block.clone()).unwrap();
+        beacon_chain.fork_choice().unwrap();
+
+        // Check that the new block's state can be loaded correctly.
+        let state_root = block.state_root();
+        let mut state = beacon_chain
+            .store
+            .get_state(&state_root, Some(block.slot()))
+            .unwrap()
+            .unwrap();
+        assert_eq!(state.update_tree_hash_cache().unwrap(), state_root);
+    }
+
+    // Forwards iterator from 0 should fail as we lack blocks.
+    assert!(matches!(
+        beacon_chain.forwards_iter_block_roots(Slot::new(0)),
+        Err(BeaconChainError::HistoricalBlockError(
+            HistoricalBlockError::BlockOutOfRange { .. }
+        ))
+    ));
+
+    // Simulate processing of a `StatusMessage` with an older finalized epoch by calling
+    // `block_root_at_slot` with an old slot for which we don't know the block root. It should
+    // return `None` rather than erroring.
+    assert_eq!(
+        beacon_chain
+            .block_root_at_slot(Slot::new(1), WhenSlotSkipped::None)
+            .unwrap(),
+        None
+    );
+
+    // Simulate querying the API for a historic state that is unknown. It should also return
+    // `None` rather than erroring.
+    assert_eq!(beacon_chain.state_root_at_slot(Slot::new(1)).unwrap(), None);
+
+    // Supply blocks backwards to reach genesis. Omit the genesis block to check genesis handling.
+    let historical_blocks = chain_dump[..wss_block.slot().as_usize()]
+        .iter()
+        .filter(|s| s.beacon_block.slot() != 0)
+        .map(|s| s.beacon_block.clone())
+        .collect::<Vec<_>>();
+    beacon_chain
+        .import_historical_block_batch(&historical_blocks)
+        .unwrap();
+    assert_eq!(beacon_chain.store.get_oldest_block_slot(), 0);
+
+    // Resupplying the blocks should not fail, they can be safely ignored.
+    beacon_chain
+        .import_historical_block_batch(&historical_blocks)
+        .unwrap();
+
+    // The forwards iterator should now match the original chain
+    let forwards = beacon_chain
+        .forwards_iter_block_roots(Slot::new(0))
+        .unwrap()
+        .map(Result::unwrap)
+        .collect::<Vec<_>>();
+    let expected = harness
+        .chain
+        .forwards_iter_block_roots(Slot::new(0))
+        .unwrap()
+        .map(Result::unwrap)
+        .collect::<Vec<_>>();
+    assert_eq!(forwards, expected);
+
+    // All blocks can be loaded.
+    for (block_root, slot) in beacon_chain
+        .forwards_iter_block_roots(Slot::new(0))
+        .unwrap()
+        .map(Result::unwrap)
+    {
+        let block = store.get_block(&block_root).unwrap().unwrap();
+        assert_eq!(block.slot(), slot);
+    }
+
+    // All states from the oldest state slot can be loaded.
+    let (_, oldest_state_slot) = store.get_historic_state_limits();
+    for (state_root, slot) in beacon_chain
+        .forwards_iter_state_roots(oldest_state_slot)
+        .unwrap()
+        .map(Result::unwrap)
+    {
+        let state = store.get_state(&state_root, Some(slot)).unwrap().unwrap();
+        assert_eq!(state.slot(), slot);
+        assert_eq!(state.canonical_root(), state_root);
+    }
+
+    // Anchor slot is still set to the starting slot.
+    assert_eq!(store.get_anchor_slot(), Some(wss_slot));
+
+    // Reconstruct states.
+    store.clone().reconstruct_historic_states().unwrap();
+    assert_eq!(store.get_anchor_slot(), None);
+}
+
 #[test]
 fn finalizes_after_resuming_from_db() {
     let validator_count = 16;