Merge unstable 20230911 into deneb-free-blobs.

common/eth2_network_config/src/lib.rs

@@ -14,11 +14,20 @@
 use discv5::enr::{CombinedKey, Enr};
 use eth2_config::{instantiate_hardcoded_nets, HardcodedNet};
 use kzg::{KzgPreset, KzgPresetId, TrustedSetup};
+use pretty_reqwest_error::PrettyReqwestError;
+use reqwest::blocking::Client;
+use sensitive_url::SensitiveUrl;
+use sha2::{Digest, Sha256};
+use slog::{info, warn, Logger};
 use std::fs::{create_dir_all, File};
 use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::str::FromStr;
-use types::{BeaconState, ChainSpec, Config, Epoch, EthSpec, EthSpecId};
+use std::time::Duration;
+use types::{BeaconState, ChainSpec, Config, Epoch, EthSpec, EthSpecId, Hash256};
+use url::Url;
+
+pub use eth2_config::GenesisStateSource;
 
 pub const DEPLOY_BLOCK_FILE: &str = "deploy_block.txt";
 pub const BOOT_ENR_FILE: &str = "boot_enr.yaml";
@@ -70,6 +79,35 @@ fn get_trusted_setup_from_config(config: &Config) -> Result<Option<TrustedSetup>
         .transpose()
 }
 
+/// A simple slice-or-vec enum to avoid cloning the beacon state bytes in the
+/// binary whilst also supporting loading them from a file at runtime.
+#[derive(Clone, PartialEq, Debug)]
+pub enum GenesisStateBytes {
+    Slice(&'static [u8]),
+    Vec(Vec<u8>),
+}
+
+impl AsRef<[u8]> for GenesisStateBytes {
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            GenesisStateBytes::Slice(slice) => slice,
+            GenesisStateBytes::Vec(vec) => vec.as_ref(),
+        }
+    }
+}
+
+impl From<&'static [u8]> for GenesisStateBytes {
+    fn from(slice: &'static [u8]) -> Self {
+        GenesisStateBytes::Slice(slice)
+    }
+}
+
+impl From<Vec<u8>> for GenesisStateBytes {
+    fn from(vec: Vec<u8>) -> Self {
+        GenesisStateBytes::Vec(vec)
+    }
+}
+
 /// Specifies an Eth2 network.
 ///
 /// See the crate-level documentation for more details.
@@ -79,7 +117,8 @@ pub struct Eth2NetworkConfig {
     /// value to be the block number where the first deposit occurs.
     pub deposit_contract_deploy_block: u64,
     pub boot_enr: Option<Vec<Enr<CombinedKey>>>,
-    pub genesis_state_bytes: Option<Vec<u8>>,
+    pub genesis_state_source: GenesisStateSource,
+    pub genesis_state_bytes: Option<GenesisStateBytes>,
     pub config: Config,
     pub kzg_trusted_setup: Option<TrustedSetup>,
 }
@@ -107,8 +146,10 @@ impl Eth2NetworkConfig {
                 serde_yaml::from_reader(net.boot_enr)
                     .map_err(|e| format!("Unable to parse boot enr: {:?}", e))?,
             ),
-            genesis_state_bytes: Some(net.genesis_state_bytes.to_vec())
-                .filter(|bytes| !bytes.is_empty()),
+            genesis_state_source: net.genesis_state_source,
+            genesis_state_bytes: Some(net.genesis_state_bytes)
+                .filter(|bytes| !bytes.is_empty())
+                .map(Into::into),
             config,
             kzg_trusted_setup,
         })
@@ -123,8 +164,37 @@ impl Eth2NetworkConfig {
     }
 
     /// Returns `true` if this configuration contains a `BeaconState`.
-    pub fn beacon_state_is_known(&self) -> bool {
-        self.genesis_state_bytes.is_some()
+    pub fn genesis_state_is_known(&self) -> bool {
+        self.genesis_state_source != GenesisStateSource::Unknown
+    }
+
+    /// The `genesis_validators_root` of the genesis state. May download the
+    /// genesis state if the value is not already available.
+    pub fn genesis_validators_root<E: EthSpec>(
+        &self,
+        genesis_state_url: Option<&str>,
+        timeout: Duration,
+        log: &Logger,
+    ) -> Result<Option<Hash256>, String> {
+        if let GenesisStateSource::Url {
+            genesis_validators_root,
+            ..
+        } = self.genesis_state_source
+        {
+            Hash256::from_str(genesis_validators_root)
+                .map(Option::Some)
+                .map_err(|e| {
+                    format!(
+                        "Unable to parse genesis state genesis_validators_root: {:?}",
+                        e
+                    )
+                })
+        } else {
+            self.genesis_state::<E>(genesis_state_url, timeout, log)?
+                .map(|state| state.genesis_validators_root())
+                .map(Result::Ok)
+                .transpose()
+        }
     }
 
     /// Construct a consolidated `ChainSpec` from the YAML config.
@@ -138,15 +208,65 @@ impl Eth2NetworkConfig {
     }
 
     /// Attempts to deserialize `self.beacon_state`, returning an error if it's missing or invalid.
-    pub fn beacon_state<E: EthSpec>(&self) -> Result<BeaconState<E>, String> {
+    ///
+    /// If the genesis state is configured to be downloaded from a URL, then the
+    /// `genesis_state_url` will override the built-in list of download URLs.
+    pub fn genesis_state<E: EthSpec>(
+        &self,
+        genesis_state_url: Option<&str>,
+        timeout: Duration,
+        log: &Logger,
+    ) -> Result<Option<BeaconState<E>>, String> {
         let spec = self.chain_spec::<E>()?;
-        let genesis_state_bytes = self
-            .genesis_state_bytes
-            .as_ref()
-            .ok_or("Genesis state is unknown")?;
+        match &self.genesis_state_source {
+            GenesisStateSource::Unknown => Ok(None),
+            GenesisStateSource::IncludedBytes => {
+                let state = self
+                    .genesis_state_bytes
+                    .as_ref()
+                    .map(|bytes| {
+                        BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| {
+                            format!("Built-in genesis state SSZ bytes are invalid: {:?}", e)
+                        })
+                    })
+                    .ok_or("Genesis state bytes missing from Eth2NetworkConfig")??;
+                Ok(Some(state))
+            }
+            GenesisStateSource::Url {
+                urls: built_in_urls,
+                checksum,
+                genesis_validators_root,
+            } => {
+                let checksum = Hash256::from_str(checksum).map_err(|e| {
+                    format!("Unable to parse genesis state bytes checksum: {:?}", e)
+                })?;
+                let bytes = if let Some(specified_url) = genesis_state_url {
+                    download_genesis_state(&[specified_url], timeout, checksum, log)
+                } else {
+                    download_genesis_state(built_in_urls, timeout, checksum, log)
+                }?;
+                let state = BeaconState::from_ssz_bytes(bytes.as_ref(), &spec).map_err(|e| {
+                    format!("Downloaded genesis state SSZ bytes are invalid: {:?}", e)
+                })?;
 
-        BeaconState::from_ssz_bytes(genesis_state_bytes, &spec)
-            .map_err(|e| format!("Genesis state SSZ bytes are invalid: {:?}", e))
+                let genesis_validators_root =
+                    Hash256::from_str(genesis_validators_root).map_err(|e| {
+                        format!(
+                            "Unable to parse genesis state genesis_validators_root: {:?}",
+                            e
+                        )
+                    })?;
+                if state.genesis_validators_root() != genesis_validators_root {
+                    return Err(format!(
+                        "Downloaded genesis validators root {:?} does not match expected {:?}",
+                        state.genesis_validators_root(),
+                        genesis_validators_root
+                    ));
+                }
+
+                Ok(Some(state))
+            }
+        }
     }
 
     /// Write the files to the directory.
@@ -204,7 +324,7 @@ impl Eth2NetworkConfig {
             File::create(&file)
                 .map_err(|e| format!("Unable to create {:?}: {:?}", file, e))
                 .and_then(|mut file| {
-                    file.write_all(genesis_state_bytes)
+                    file.write_all(genesis_state_bytes.as_ref())
                         .map_err(|e| format!("Unable to write {:?}: {:?}", file, e))
                 })?;
         }
@@ -240,7 +360,7 @@ impl Eth2NetworkConfig {
 
         // The genesis state is a special case because it uses SSZ, not YAML.
         let genesis_file_path = base_dir.join(GENESIS_STATE_FILE);
-        let genesis_state_bytes = if genesis_file_path.exists() {
+        let (genesis_state_bytes, genesis_state_source) = if genesis_file_path.exists() {
             let mut bytes = vec![];
             File::open(&genesis_file_path)
                 .map_err(|e| format!("Unable to open {:?}: {:?}", genesis_file_path, e))
@@ -249,9 +369,15 @@ impl Eth2NetworkConfig {
                         .map_err(|e| format!("Unable to read {:?}: {:?}", file, e))
                 })?;
 
-            Some(bytes).filter(|bytes| !bytes.is_empty())
+            let state = Some(bytes).filter(|bytes| !bytes.is_empty());
+            let genesis_state_source = if state.is_some() {
+                GenesisStateSource::IncludedBytes
+            } else {
+                GenesisStateSource::Unknown
+            };
+            (state, genesis_state_source)
         } else {
-            None
+            (None, GenesisStateSource::Unknown)
         };
 
         let kzg_trusted_setup = get_trusted_setup_from_config(&config)?;
@@ -259,13 +385,92 @@ impl Eth2NetworkConfig {
         Ok(Self {
             deposit_contract_deploy_block,
             boot_enr,
-            genesis_state_bytes,
+            genesis_state_source,
+            genesis_state_bytes: genesis_state_bytes.map(Into::into),
             config,
             kzg_trusted_setup,
         })
     }
 }
 
+/// Try to download a genesis state from each of the `urls` in the order they
+/// are defined. Return `Ok` if any url returns a response that matches the
+/// given `checksum`.
+fn download_genesis_state(
+    urls: &[&str],
+    timeout: Duration,
+    checksum: Hash256,
+    log: &Logger,
+) -> Result<Vec<u8>, String> {
+    if urls.is_empty() {
+        return Err(
+            "The genesis state is not present in the binary and there are no known download URLs. \
+            Please use --checkpoint-sync-url or --genesis-state-url."
+                .to_string(),
+        );
+    }
+
+    let mut errors = vec![];
+    for url in urls {
+        // URLs are always expected to be the base URL of a server that supports
+        // the beacon-API.
+        let url = parse_state_download_url(url)?;
+        let redacted_url = SensitiveUrl::new(url.clone())
+            .map(|url| url.to_string())
+            .unwrap_or_else(|_| "<REDACTED>".to_string());
+
+        info!(
+            log,
+            "Downloading genesis state";
+            "server" => &redacted_url,
+            "timeout" => ?timeout,
+            "info" => "this may take some time on testnets with large validator counts"
+        );
+
+        let client = Client::new();
+        let response = client
+            .get(url)
+            .header("Accept", "application/octet-stream")
+            .timeout(timeout)
+            .send()
+            .and_then(|r| r.error_for_status().and_then(|r| r.bytes()));
+
+        match response {
+            Ok(bytes) => {
+                // Check the server response against our local checksum.
+                if Sha256::digest(bytes.as_ref())[..] == checksum[..] {
+                    return Ok(bytes.into());
+                } else {
+                    warn!(
+                        log,
+                        "Genesis state download failed";
+                        "server" => &redacted_url,
+                        "timeout" => ?timeout,
+                    );
+                    errors.push(format!(
+                        "Response from {} did not match local checksum",
+                        redacted_url
+                    ))
+                }
+            }
+            Err(e) => errors.push(PrettyReqwestError::from(e).to_string()),
+        }
+    }
+    Err(format!(
+        "Unable to download a genesis state from {} source(s): {}",
+        errors.len(),
+        errors.join(",")
+    ))
+}
+
+/// Parses the `url` and joins the necessary state download path.
+fn parse_state_download_url(url: &str) -> Result<Url, String> {
+    Url::parse(url)
+        .map_err(|e| format!("Invalid genesis state URL: {:?}", e))?
+        .join("eth/v2/debug/beacon/states/genesis")
+        .map_err(|e| format!("Failed to append genesis state path to URL: {:?}", e))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -305,7 +510,9 @@ mod tests {
     #[test]
     fn mainnet_genesis_state() {
         let config = Eth2NetworkConfig::from_hardcoded_net(&MAINNET).unwrap();
-        config.beacon_state::<E>().expect("beacon state can decode");
+        config
+            .genesis_state::<E>(None, Duration::from_secs(1), &logging::test_logger())
+            .expect("beacon state can decode");
     }
 
     #[test]
@@ -319,10 +526,10 @@ mod tests {
     fn hard_coded_nets_work() {
         for net in HARDCODED_NETS {
             let config = Eth2NetworkConfig::from_hardcoded_net(net)
-                .unwrap_or_else(|_| panic!("{:?}", net.name));
+                .unwrap_or_else(|e| panic!("{:?}: {:?}", net.name, e));
 
             // Ensure we can parse the YAML config to a chain spec.
-            if net.name == types::GNOSIS {
+            if config.config.preset_base == types::GNOSIS {
                 config.chain_spec::<GnosisEthSpec>().unwrap();
             } else {
                 config.chain_spec::<MainnetEthSpec>().unwrap();
@@ -330,10 +537,25 @@ mod tests {
 
             assert_eq!(
                 config.genesis_state_bytes.is_some(),
-                net.genesis_is_known,
+                net.genesis_state_source == GenesisStateSource::IncludedBytes,
                 "{:?}",
                 net.name
             );
+
+            if let GenesisStateSource::Url {
+                urls,
+                checksum,
+                genesis_validators_root,
+            } = net.genesis_state_source
+            {
+                Hash256::from_str(checksum).expect("the checksum must be a valid 32-byte value");
+                Hash256::from_str(genesis_validators_root)
+                    .expect("the GVR must be a valid 32-byte value");
+                for url in urls {
+                    parse_state_download_url(url).expect("url must be valid");
+                }
+            }
+
             assert_eq!(config.config.config_name, Some(net.config_dir.to_string()));
         }
     }
@@ -369,10 +591,20 @@ mod tests {
         let base_dir = temp_dir.path().join("my_testnet");
         let deposit_contract_deploy_block = 42;
 
+        let genesis_state_source = if genesis_state.is_some() {
+            GenesisStateSource::IncludedBytes
+        } else {
+            GenesisStateSource::Unknown
+        };
+
         let testnet = Eth2NetworkConfig {
             deposit_contract_deploy_block,
             boot_enr,
-            genesis_state_bytes: genesis_state.as_ref().map(Encode::as_ssz_bytes),
+            genesis_state_source,
+            genesis_state_bytes: genesis_state
+                .as_ref()
+                .map(Encode::as_ssz_bytes)
+                .map(Into::into),
             config,
             kzg_trusted_setup: None,
         };