Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

common/eth2/Cargo.toml

@@ -26,6 +26,7 @@ eth2_ssz = "0.3.0"
 eth2_ssz_derive = "0.2.1"
 futures-util = "0.3.8"
 futures = "0.3.8"
+store = { path = "../../beacon_node/store", optional = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 psutil = { version = "3.2.0", optional = true }
@@ -33,4 +34,4 @@ procinfo = { version = "0.4.2", optional = true }
 
 [features]
 default = ["lighthouse"]
-lighthouse = ["proto_array", "psutil", "procinfo"]
+lighthouse = ["proto_array", "psutil", "procinfo", "store"]

common/eth2/src/lib.rs

@@ -262,17 +262,23 @@ impl BeaconNodeHttpClient {
 
     /// Perform a HTTP POST request.
     async fn post<T: Serialize, U: IntoUrl>(&self, url: U, body: &T) -> Result<(), Error> {
-        let response = self
-            .client
-            .post(url)
-            .json(body)
-            .send()
-            .await
-            .map_err(Error::Reqwest)?;
-        ok_or_error(response).await?;
+        self.post_generic(url, body, None).await?;
         Ok(())
     }
 
+    /// Perform a HTTP POST request, returning a JSON response.
+    async fn post_with_response<T: Serialize, U: IntoUrl, R: DeserializeOwned>(
+        &self,
+        url: U,
+        body: &T,
+    ) -> Result<R, Error> {
+        self.post_generic(url, body, None)
+            .await?
+            .json()
+            .await
+            .map_err(Error::Reqwest)
+    }
+
     /// Perform a HTTP POST request with a custom timeout.
     async fn post_with_timeout<T: Serialize, U: IntoUrl>(
         &self,
@@ -280,15 +286,7 @@ impl BeaconNodeHttpClient {
         body: &T,
         timeout: Duration,
     ) -> Result<(), Error> {
-        let response = self
-            .client
-            .post(url)
-            .timeout(timeout)
-            .json(body)
-            .send()
-            .await
-            .map_err(Error::Reqwest)?;
-        ok_or_error(response).await?;
+        self.post_generic(url, body, Some(timeout)).await?;
         Ok(())
     }
 
@@ -299,21 +297,28 @@ impl BeaconNodeHttpClient {
         body: &V,
         timeout: Duration,
     ) -> Result<T, Error> {
-        let response = self
-            .client
-            .post(url)
-            .timeout(timeout)
-            .json(body)
-            .send()
-            .await
-            .map_err(Error::Reqwest)?;
-        ok_or_error(response)
+        self.post_generic(url, body, Some(timeout))
             .await?
             .json()
             .await
             .map_err(Error::Reqwest)
     }
 
+    /// Generic POST function supporting arbitrary responses and timeouts.
+    async fn post_generic<T: Serialize, U: IntoUrl>(
+        &self,
+        url: U,
+        body: &T,
+        timeout: Option<Duration>,
+    ) -> Result<Response, Error> {
+        let mut builder = self.client.post(url);
+        if let Some(timeout) = timeout {
+            builder = builder.timeout(timeout);
+        }
+        let response = builder.json(body).send().await.map_err(Error::Reqwest)?;
+        ok_or_error(response).await
+    }
+
     /// `GET beacon/genesis`
     ///
     /// ## Errors

common/eth2/src/lighthouse.rs

@@ -9,6 +9,7 @@ use proto_array::core::ProtoArray;
 use reqwest::IntoUrl;
 use serde::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode};
+use store::{AnchorInfo, Split};
 
 pub use eth2_libp2p::{types::SyncState, PeerInfo};
 
@@ -311,6 +312,13 @@ impl Eth1Block {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub struct DatabaseInfo {
+    pub schema_version: u64,
+    pub split: Split,
+    pub anchor: Option<AnchorInfo>,
+}
+
 impl BeaconNodeHttpClient {
     /// Perform a HTTP GET request, returning `None` on a 404 error.
     async fn get_bytes_opt<U: IntoUrl>(&self, url: U) -> Result<Option<Vec<u8>>, Error> {
@@ -490,4 +498,30 @@ impl BeaconNodeHttpClient {
 
         self.get_opt::<(), _>(path).await.map(|opt| opt.is_some())
     }
+
+    /// `GET lighthouse/database/info`
+    pub async fn get_lighthouse_database_info(&self) -> Result<DatabaseInfo, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("database")
+            .push("info");
+
+        self.get(path).await
+    }
+
+    /// `POST lighthouse/database/reconstruct`
+    pub async fn post_lighthouse_database_reconstruct(&self) -> Result<String, Error> {
+        let mut path = self.server.full.clone();
+
+        path.path_segments_mut()
+            .map_err(|()| Error::InvalidUrl(self.server.clone()))?
+            .push("lighthouse")
+            .push("database")
+            .push("reconstruct");
+
+        self.post_with_response(path, &()).await
+    }
 }