Implement checkpoint sync (#2244)

## Issue Addressed

Closes #1891
Closes #1784

## Proposed Changes

Implement checkpoint sync for Lighthouse, enabling it to start from a weak subjectivity checkpoint.

## Additional Info

- [x] Return unavailable status for out-of-range blocks requested by peers (#2561)
- [x] Implement sync daemon for fetching historical blocks (#2561)
- [x] Verify chain hashes (either in `historical_blocks.rs` or the calling module)
- [x] Consistency check for initial block + state
- [x] Fetch the initial state and block from a beacon node HTTP endpoint
- [x] Don't crash fetching beacon states by slot from the API
- [x] Background service for state reconstruction, triggered by CLI flag or API call.

Considered out of scope for this PR:

- Drop the requirement to provide the `--checkpoint-block` (this would require some pretty heavy refactoring of block verification)


Co-authored-by: Diva M <divma@protonmail.com>

beacon_node/src/cli.rs

@@ -54,7 +54,8 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
         .arg(
             Arg::with_name("shutdown-after-sync")
                 .long("shutdown-after-sync")
-                .help("Shutdown beacon node as soon as sync is completed")
+                .help("Shutdown beacon node as soon as sync is completed. Backfill sync will \
+                       not be performed before shutdown.")
                 .takes_value(false),
         )
         .arg(
@@ -479,12 +480,46 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
             Arg::with_name("wss-checkpoint")
                 .long("wss-checkpoint")
                 .help(
-                    "Used to input a Weak Subjectivity State Checkpoint in `block_root:epoch_number` format,\
-                     where block_root is an '0x' prefixed 32-byte hex string and epoch_number is an integer."
+                    "Specify a weak subjectivity checkpoint in `block_root:epoch` format to verify \
+                     the node's sync against. The block root should be 0x-prefixed. Note that this \
+                     flag is for verification only, to perform a checkpoint sync from a recent \
+                     state use --checkpoint-sync-url."
                 )
                 .value_name("WSS_CHECKPOINT")
                 .takes_value(true)
         )
+        .arg(
+            Arg::with_name("checkpoint-state")
+                .long("checkpoint-state")
+                .help("Set a checkpoint state to start syncing from. Must be aligned and match \
+                       --checkpoint-block. Using --checkpoint-sync-url instead is recommended.")
+                .value_name("STATE_SSZ")
+                .takes_value(true)
+                .requires("checkpoint-block")
+        )
+        .arg(
+            Arg::with_name("checkpoint-block")
+                .long("checkpoint-block")
+                .help("Set a checkpoint block to start syncing from. Must be aligned and match \
+                       --checkpoint-state. Using --checkpoint-sync-url instead is recommended.")
+                .value_name("BLOCK_SSZ")
+                .takes_value(true)
+                .requires("checkpoint-state")
+        )
+        .arg(
+            Arg::with_name("checkpoint-sync-url")
+                .long("checkpoint-sync-url")
+                .help("Set the remote beacon node HTTP endpoint to use for checkpoint sync.")
+                .value_name("BEACON_NODE")
+                .takes_value(true)
+                .conflicts_with("checkpoint-state")
+        )
+        .arg(
+            Arg::with_name("reconstruct-historic-states")
+                .long("reconstruct-historic-states")
+                .help("After a checkpoint sync, reconstruct historic states in the database.")
+                .takes_value(false)
+        )
         .arg(
             Arg::with_name("validator-monitor-auto")
                 .long("validator-monitor-auto")

beacon_node/src/config.rs

@@ -292,15 +292,62 @@ pub fn get_config<E: EthSpec>(
         }
     }
 
-    if let Some(genesis_state_bytes) = eth2_network_config.genesis_state_bytes {
-        // Note: re-serializing the genesis state is not so efficient, however it avoids adding
-        // trait bounds to the `ClientGenesis` enum. This would have significant flow-on
-        // effects.
-        client_config.genesis = ClientGenesis::SszBytes {
-            genesis_state_bytes,
-        };
+    client_config.genesis = if let Some(genesis_state_bytes) =
+        eth2_network_config.genesis_state_bytes
+    {
+        // Set up weak subjectivity sync, or start from the hardcoded genesis state.
+        if let (Some(initial_state_path), Some(initial_block_path)) = (
+            cli_args.value_of("checkpoint-state"),
+            cli_args.value_of("checkpoint-block"),
+        ) {
+            let read = |path: &str| {
+                use std::fs::File;
+                use std::io::Read;
+                File::open(Path::new(path))
+                    .and_then(|mut f| {
+                        let mut buffer = vec![];
+                        f.read_to_end(&mut buffer)?;
+                        Ok(buffer)
+                    })
+                    .map_err(|e| format!("Unable to open {}: {:?}", path, e))
+            };
+
+            let anchor_state_bytes = read(initial_state_path)?;
+            let anchor_block_bytes = read(initial_block_path)?;
+
+            ClientGenesis::WeakSubjSszBytes {
+                genesis_state_bytes,
+                anchor_state_bytes,
+                anchor_block_bytes,
+            }
+        } else if let Some(remote_bn_url) = cli_args.value_of("checkpoint-sync-url") {
+            let url = SensitiveUrl::parse(remote_bn_url)
+                .map_err(|e| format!("Invalid checkpoint sync URL: {:?}", e))?;
+
+            ClientGenesis::CheckpointSyncUrl {
+                genesis_state_bytes,
+                url,
+            }
+        } else {
+            // Note: re-serializing the genesis state is not so efficient, however it avoids adding
+            // trait bounds to the `ClientGenesis` enum. This would have significant flow-on
+            // effects.
+            ClientGenesis::SszBytes {
+                genesis_state_bytes,
+            }
+        }
     } else {
-        client_config.genesis = ClientGenesis::DepositContract;
+        if cli_args.is_present("checkpoint-state") || cli_args.is_present("checkpoint-sync-url") {
+            return Err(
+                "Checkpoint sync is not available for this network as no genesis state is known"
+                    .to_string(),
+            );
+        }
+        ClientGenesis::DepositContract
+    };
+
+    if cli_args.is_present("reconstruct-historic-states") {
+        client_config.chain.reconstruct_historic_states = true;
     }
 
     let raw_graffiti = if let Some(graffiti) = cli_args.value_of("graffiti") {