Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

beacon_node/http_api/src/database.rs

@@ -0,0 +1,33 @@
+use beacon_chain::store::{metadata::CURRENT_SCHEMA_VERSION, AnchorInfo};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use eth2::lighthouse::DatabaseInfo;
+use std::sync::Arc;
+use types::SignedBeaconBlock;
+
+pub fn info<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+) -> Result<DatabaseInfo, warp::Rejection> {
+    let store = &chain.store;
+    let split = store.get_split_info();
+    let anchor = store.get_anchor_info();
+
+    Ok(DatabaseInfo {
+        schema_version: CURRENT_SCHEMA_VERSION.as_u64(),
+        split,
+        anchor,
+    })
+}
+
+pub fn historical_blocks<T: BeaconChainTypes>(
+    chain: Arc<BeaconChain<T>>,
+    blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+) -> Result<AnchorInfo, warp::Rejection> {
+    chain
+        .import_historical_block_batch(&blocks)
+        .map_err(warp_utils::reject::beacon_chain_error)?;
+
+    let anchor = chain.store.get_anchor_info().ok_or_else(|| {
+        warp_utils::reject::custom_bad_request("node is not checkpoint synced".to_string())
+    })?;
+    Ok(anchor)
+}

beacon_node/http_api/src/lib.rs

@@ -7,6 +7,7 @@
 
 mod attester_duties;
 mod block_id;
+mod database;
 mod metrics;
 mod proposer_duties;
 mod state_id;
@@ -349,7 +350,7 @@ pub fn serve<T: BeaconChainTypes>(
                             )))
                         }
                     }
-                    SyncState::SyncingHead { .. } | SyncState::SyncTransition => Ok(()),
+                    SyncState::SyncingHead { .. } | SyncState::SyncTransition | SyncState::BackFillSyncing { .. }  => Ok(()),
                     SyncState::Synced => Ok(()),
                     SyncState::Stalled => Err(warp_utils::reject::not_synced(
                         "sync is stalled".to_string(),
@@ -1579,7 +1580,8 @@ pub fn serve<T: BeaconChainTypes>(
             blocking_task(move || match *network_globals.sync_state.read() {
                 SyncState::SyncingFinalized { .. }
                 | SyncState::SyncingHead { .. }
-                | SyncState::SyncTransition => Ok(warp::reply::with_status(
+                | SyncState::SyncTransition
+                | SyncState::BackFillSyncing { .. } => Ok(warp::reply::with_status(
                     warp::reply(),
                     warp::http::StatusCode::PARTIAL_CONTENT,
                 )),
@@ -2040,7 +2042,7 @@ pub fn serve<T: BeaconChainTypes>(
         .and(warp::path("validator"))
         .and(warp::path("contribution_and_proofs"))
         .and(warp::path::end())
-        .and(not_while_syncing_filter)
+        .and(not_while_syncing_filter.clone())
         .and(chain_filter.clone())
         .and(warp::body::json())
         .and(network_tx_filter.clone())
@@ -2399,6 +2401,49 @@ pub fn serve<T: BeaconChainTypes>(
             })
         });
 
+    let database_path = warp::path("lighthouse").and(warp::path("database"));
+
+    // GET lighthouse/database/info
+    let get_lighthouse_database_info = database_path
+        .and(warp::path("info"))
+        .and(warp::path::end())
+        .and(chain_filter.clone())
+        .and_then(|chain: Arc<BeaconChain<T>>| blocking_json_task(move || database::info(chain)));
+
+    // POST lighthouse/database/reconstruct
+    let post_lighthouse_database_reconstruct = database_path
+        .and(warp::path("reconstruct"))
+        .and(warp::path::end())
+        .and(not_while_syncing_filter)
+        .and(chain_filter.clone())
+        .and_then(|chain: Arc<BeaconChain<T>>| {
+            blocking_json_task(move || {
+                chain.store_migrator.process_reconstruction();
+                Ok("success")
+            })
+        });
+
+    // POST lighthouse/database/historical_blocks
+    let post_lighthouse_database_historical_blocks = database_path
+        .and(warp::path("historical_blocks"))
+        .and(warp::path::end())
+        .and(warp::body::json())
+        .and(chain_filter.clone())
+        .and(log_filter.clone())
+        .and_then(
+            |blocks: Vec<SignedBeaconBlock<T::EthSpec>>,
+             chain: Arc<BeaconChain<T>>,
+             log: Logger| {
+                info!(
+                    log,
+                    "Importing historical blocks";
+                    "count" => blocks.len(),
+                    "source" => "http_api"
+                );
+                blocking_json_task(move || database::historical_blocks(chain, blocks))
+            },
+        );
+
     let get_events = eth1_v1
         .and(warp::path("events"))
         .and(warp::path::end())
@@ -2510,6 +2555,7 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(get_lighthouse_eth1_deposit_cache.boxed())
                 .or(get_lighthouse_beacon_states_ssz.boxed())
                 .or(get_lighthouse_staking.boxed())
+                .or(get_lighthouse_database_info.boxed())
                 .or(get_events.boxed()),
         )
         .or(warp::post().and(
@@ -2526,7 +2572,9 @@ pub fn serve<T: BeaconChainTypes>(
                 .or(post_validator_contribution_and_proofs.boxed())
                 .or(post_validator_beacon_committee_subscriptions.boxed())
                 .or(post_validator_sync_committee_subscriptions.boxed())
-                .or(post_lighthouse_liveness.boxed()),
+                .or(post_lighthouse_liveness.boxed())
+                .or(post_lighthouse_database_reconstruct.boxed())
+                .or(post_lighthouse_database_historical_blocks.boxed()),
         ))
         .recover(warp_utils::reject::handle_rejection)
         .with(slog_logging(log.clone()))

beacon_node/http_api/tests/tests.rs

@@ -2101,6 +2101,29 @@ impl ApiTester {
         self
     }
 
+    pub async fn test_get_lighthouse_database_info(self) -> Self {
+        let info = self.client.get_lighthouse_database_info().await.unwrap();
+
+        assert_eq!(info.anchor, self.chain.store.get_anchor_info());
+        assert_eq!(info.split, self.chain.store.get_split_info());
+        assert_eq!(
+            info.schema_version,
+            store::metadata::CURRENT_SCHEMA_VERSION.as_u64()
+        );
+
+        self
+    }
+
+    pub async fn test_post_lighthouse_database_reconstruct(self) -> Self {
+        let response = self
+            .client
+            .post_lighthouse_database_reconstruct()
+            .await
+            .unwrap();
+        assert_eq!(response, "success");
+        self
+    }
+
     pub async fn test_post_lighthouse_liveness(self) -> Self {
         let epoch = self.chain.epoch().unwrap();
         let head_state = self.chain.head_beacon_state().unwrap();
@@ -2653,6 +2676,10 @@ async fn lighthouse_endpoints() {
         .await
         .test_get_lighthouse_staking()
         .await
+        .test_get_lighthouse_database_info()
+        .await
+        .test_post_lighthouse_database_reconstruct()
+        .await
         .test_post_lighthouse_liveness()
         .await;
 }