Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

beacon_node/beacon_chain/src/schema_change.rs

@@ -2,12 +2,15 @@
 use crate::beacon_chain::{BeaconChainTypes, OP_POOL_DB_KEY};
 use crate::validator_pubkey_cache::ValidatorPubkeyCache;
 use operation_pool::{PersistedOperationPool, PersistedOperationPoolBase};
+use ssz::{Decode, Encode};
+use ssz_derive::{Decode, Encode};
 use std::fs;
 use std::path::Path;
 use std::sync::Arc;
+use store::config::OnDiskStoreConfig;
 use store::hot_cold_store::{HotColdDB, HotColdDBError};
-use store::metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION};
-use store::Error as StoreError;
+use store::metadata::{SchemaVersion, CONFIG_KEY, CURRENT_SCHEMA_VERSION};
+use store::{DBColumn, Error as StoreError, ItemStore, StoreItem};
 
 const PUBKEY_CACHE_FILENAME: &str = "pubkey_cache.ssz";
 
@@ -73,6 +76,23 @@ pub fn migrate_schema<T: BeaconChainTypes>(
 
             Ok(())
         }
+        // Migration for weak subjectivity sync support and clean up of `OnDiskStoreConfig` (#1784).
+        (SchemaVersion(4), SchemaVersion(5)) => {
+            if let Some(OnDiskStoreConfigV4 {
+                slots_per_restore_point,
+                ..
+            }) = db.hot_db.get(&CONFIG_KEY)?
+            {
+                let new_config = OnDiskStoreConfig {
+                    slots_per_restore_point,
+                };
+                db.hot_db.put(&CONFIG_KEY, &new_config)?;
+            }
+
+            db.store_schema_version(to)?;
+
+            Ok(())
+        }
         // Anything else is an error.
         (_, _) => Err(HotColdDBError::UnsupportedSchemaVersion {
             target_version: to,
@@ -81,3 +101,24 @@ pub fn migrate_schema<T: BeaconChainTypes>(
         .into()),
     }
 }
+
+// Store config used in v4 schema and earlier.
+#[derive(Debug, Clone, PartialEq, Eq, Encode, Decode)]
+pub struct OnDiskStoreConfigV4 {
+    pub slots_per_restore_point: u64,
+    pub _block_cache_size: usize,
+}
+
+impl StoreItem for OnDiskStoreConfigV4 {
+    fn db_column() -> DBColumn {
+        DBColumn::BeaconMeta
+    }
+
+    fn as_store_bytes(&self) -> Vec<u8> {
+        self.as_ssz_bytes()
+    }
+
+    fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
+        Ok(Self::from_ssz_bytes(bytes)?)
+    }
+}