Merge in latest master

beacon_node/eth2-libp2p/Cargo.toml

@@ -9,8 +9,8 @@ clap = "2.33.0"
 hex = "0.3"
 # rust-libp2p is presently being sourced from a Sigma Prime fork of the
 # `libp2p/rust-libp2p` repository.
-libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "b13ec466ce1661d88ea95be7e1fcd7bfdfa22ca8" }
+libp2p =  { git = "https://github.com/SigP/rust-libp2p", rev = "1c1b3ba402eefbd31ad40c561545554ef66b58a7" }
-enr =  { git = "https://github.com/SigP/rust-libp2p/", rev = "b13ec466ce1661d88ea95be7e1fcd7bfdfa22ca8", features = ["serde"] }
+enr =  { git = "https://github.com/SigP/rust-libp2p/", rev = "1c1b3ba402eefbd31ad40c561545554ef66b58a7", features = ["serde"] }
 types = { path =  "../../eth2/types" }
 serde = "1.0.102"
 serde_derive = "1.0.102"

beacon_node/eth2-libp2p/src/lib.rs

@@ -22,7 +22,6 @@ pub use libp2p::enr::Enr;
 pub use libp2p::gossipsub::{Topic, TopicHash};
 pub use libp2p::multiaddr;
 pub use libp2p::Multiaddr;
-pub use libp2p::{core::ConnectedPoint, swarm::NetworkBehaviour};
 pub use libp2p::{
     gossipsub::{GossipsubConfig, GossipsubConfigBuilder},
     PeerId, Swarm,

beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs

@@ -1,7 +1,9 @@
 use crate::rpc::methods::*;
 use crate::rpc::{
     codec::base::OutboundCodec,
-    protocol::{ProtocolId, RPCError},
+    protocol::{
+        ProtocolId, RPCError, RPC_BLOCKS_BY_RANGE, RPC_BLOCKS_BY_ROOT, RPC_GOODBYE, RPC_STATUS,
+    },
 };
 use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse};
 use libp2p::bytes::{BufMut, Bytes, BytesMut};
@@ -43,7 +45,6 @@ impl Encoder for SSZInboundCodec {
                     RPCResponse::Status(res) => res.as_ssz_bytes(),
                     RPCResponse::BlocksByRange(res) => res, // already raw bytes
                     RPCResponse::BlocksByRoot(res) => res,  // already raw bytes
-                    RPCResponse::Goodbye => unreachable!("Never encode or decode this message"),
                 }
             }
             RPCErrorResponse::InvalidRequest(err) => err.as_ssz_bytes(),
@@ -76,25 +77,25 @@ impl Decoder for SSZInboundCodec {
     fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
         match self.inner.decode(src).map_err(RPCError::from) {
             Ok(Some(packet)) => match self.protocol.message_name.as_str() {
-                "status" => match self.protocol.version.as_str() {
+                RPC_STATUS => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
                         &packet,
                     )?))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
-                "goodbye" => match self.protocol.version.as_str() {
+                RPC_GOODBYE => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCRequest::Goodbye(GoodbyeReason::from_ssz_bytes(
                         &packet,
                     )?))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
-                "blocks_by_range" => match self.protocol.version.as_str() {
+                RPC_BLOCKS_BY_RANGE => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCRequest::BlocksByRange(
                         BlocksByRangeRequest::from_ssz_bytes(&packet)?,
                     ))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
-                "blocks_by_root" => match self.protocol.version.as_str() {
+                RPC_BLOCKS_BY_ROOT => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
                         block_roots: Vec::from_ssz_bytes(&packet)?,
                     }))),
@@ -164,18 +165,18 @@ impl Decoder for SSZOutboundCodec {
             // clear the buffer and return an empty object
             src.clear();
             match self.protocol.message_name.as_str() {
-                "status" => match self.protocol.version.as_str() {
+                RPC_STATUS => match self.protocol.version.as_str() {
                     "1" => Err(RPCError::Custom(
                         "Status stream terminated unexpectedly".into(),
                     )), // cannot have an empty HELLO message. The stream has terminated unexpectedly
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
-                "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
+                RPC_GOODBYE => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
-                "blocks_by_range" => match self.protocol.version.as_str() {
+                RPC_BLOCKS_BY_RANGE => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCResponse::BlocksByRange(Vec::new()))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
-                "blocks_by_root" => match self.protocol.version.as_str() {
+                RPC_BLOCKS_BY_ROOT => match self.protocol.version.as_str() {
                     "1" => Ok(Some(RPCResponse::BlocksByRoot(Vec::new()))),
                     _ => unreachable!("Cannot negotiate an unknown version"),
                 },
@@ -188,20 +189,20 @@ impl Decoder for SSZOutboundCodec {
                     let raw_bytes = packet.take();
 
                     match self.protocol.message_name.as_str() {
-                        "status" => match self.protocol.version.as_str() {
+                        RPC_STATUS => match self.protocol.version.as_str() {
                             "1" => Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
                                 &raw_bytes,
                             )?))),
                             _ => unreachable!("Cannot negotiate an unknown version"),
                         },
-                        "goodbye" => {
+                        RPC_GOODBYE => {
                             Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response"))
                         }
-                        "blocks_by_range" => match self.protocol.version.as_str() {
+                        RPC_BLOCKS_BY_RANGE => match self.protocol.version.as_str() {
                             "1" => Ok(Some(RPCResponse::BlocksByRange(raw_bytes.to_vec()))),
                             _ => unreachable!("Cannot negotiate an unknown version"),
                         },
-                        "blocks_by_root" => match self.protocol.version.as_str() {
+                        RPC_BLOCKS_BY_ROOT => match self.protocol.version.as_str() {
                             "1" => Ok(Some(RPCResponse::BlocksByRoot(raw_bytes.to_vec()))),
                             _ => unreachable!("Cannot negotiate an unknown version"),
                         },

beacon_node/eth2-libp2p/src/rpc/handler.rs

@@ -1,7 +1,7 @@
 #![allow(clippy::type_complexity)]
 #![allow(clippy::cognitive_complexity)]
 
-use super::methods::{RPCErrorResponse, RPCResponse, RequestId};
+use super::methods::{RPCErrorResponse, RequestId};
 use super::protocol::{RPCError, RPCProtocol, RPCRequest};
 use super::RPCEvent;
 use crate::rpc::protocol::{InboundFramed, OutboundFramed};
@@ -208,7 +208,6 @@ where
         // drop the stream and return a 0 id for goodbye "requests"
         if let r @ RPCRequest::Goodbye(_) = req {
             self.events_out.push(RPCEvent::Request(0, r));
-            warn!(self.log, "Goodbye Received");
             return;
         }
 
@@ -244,14 +243,6 @@ where
 
         // add the stream to substreams if we expect a response, otherwise drop the stream.
         match rpc_event {
-            RPCEvent::Request(id, RPCRequest::Goodbye(_)) => {
-                // notify the application layer, that a goodbye has been sent, so the application can
-                // drop and remove the peer
-                self.events_out.push(RPCEvent::Response(
-                    id,
-                    RPCErrorResponse::Success(RPCResponse::Goodbye),
-                ));
-            }
             RPCEvent::Request(id, request) if request.expect_response() => {
                 // new outbound request. Store the stream and tag the output.
                 let delay_key = self

beacon_node/eth2-libp2p/src/rpc/methods.rs

@@ -139,9 +139,6 @@ pub enum RPCResponse {
 
     /// A response to a get BLOCKS_BY_ROOT request.
     BlocksByRoot(Vec<u8>),
-
-    /// A Goodbye message has been sent
-    Goodbye,
 }
 
 /// Indicates which response is being terminated by a stream termination response.
@@ -208,7 +205,6 @@ impl RPCErrorResponse {
                 RPCResponse::Status(_) => false,
                 RPCResponse::BlocksByRange(_) => true,
                 RPCResponse::BlocksByRoot(_) => true,
-                RPCResponse::Goodbye => false,
             },
             RPCErrorResponse::InvalidRequest(_) => true,
             RPCErrorResponse::ServerError(_) => true,
@@ -252,7 +248,6 @@ impl std::fmt::Display for RPCResponse {
             RPCResponse::Status(status) => write!(f, "{}", status),
             RPCResponse::BlocksByRange(_) => write!(f, "<BlocksByRange>"),
             RPCResponse::BlocksByRoot(_) => write!(f, "<BlocksByRoot>"),
-            RPCResponse::Goodbye => write!(f, "Goodbye Sent"),
         }
     }
 }

beacon_node/eth2-libp2p/src/rpc/protocol.rs

@@ -33,6 +33,16 @@ const TTFB_TIMEOUT: u64 = 5;
 /// established before the stream is terminated.
 const REQUEST_TIMEOUT: u64 = 15;
 
+/// Protocol names to be used.
+/// The Status protocol name.
+pub const RPC_STATUS: &str = "status";
+/// The Goodbye protocol name.
+pub const RPC_GOODBYE: &str = "goodbye";
+/// The `BlocksByRange` protocol name.
+pub const RPC_BLOCKS_BY_RANGE: &str = "beacon_blocks_by_range";
+/// The `BlocksByRoot` protocol name.
+pub const RPC_BLOCKS_BY_ROOT: &str = "beacon_blocks_by_root";
+
 #[derive(Debug, Clone)]
 pub struct RPCProtocol;
 
@@ -42,10 +52,10 @@ impl UpgradeInfo for RPCProtocol {
 
     fn protocol_info(&self) -> Self::InfoIter {
         vec![
-            ProtocolId::new("status", "1", "ssz"),
+            ProtocolId::new(RPC_STATUS, "1", "ssz"),
-            ProtocolId::new("goodbye", "1", "ssz"),
+            ProtocolId::new(RPC_GOODBYE, "1", "ssz"),
-            ProtocolId::new("blocks_by_range", "1", "ssz"),
+            ProtocolId::new(RPC_BLOCKS_BY_RANGE, "1", "ssz"),
-            ProtocolId::new("blocks_by_root", "1", "ssz"),
+            ProtocolId::new(RPC_BLOCKS_BY_ROOT, "1", "ssz"),
         ]
     }
 }
@@ -173,10 +183,10 @@ impl RPCRequest {
     pub fn supported_protocols(&self) -> Vec<ProtocolId> {
         match self {
             // add more protocols when versions/encodings are supported
-            RPCRequest::Status(_) => vec![ProtocolId::new("status", "1", "ssz")],
+            RPCRequest::Status(_) => vec![ProtocolId::new(RPC_STATUS, "1", "ssz")],
-            RPCRequest::Goodbye(_) => vec![ProtocolId::new("goodbye", "1", "ssz")],
+            RPCRequest::Goodbye(_) => vec![ProtocolId::new(RPC_GOODBYE, "1", "ssz")],
-            RPCRequest::BlocksByRange(_) => vec![ProtocolId::new("blocks_by_range", "1", "ssz")],
+            RPCRequest::BlocksByRange(_) => vec![ProtocolId::new(RPC_BLOCKS_BY_RANGE, "1", "ssz")],
-            RPCRequest::BlocksByRoot(_) => vec![ProtocolId::new("blocks_by_root", "1", "ssz")],
+            RPCRequest::BlocksByRoot(_) => vec![ProtocolId::new(RPC_BLOCKS_BY_ROOT, "1", "ssz")],
         }
     }
 

beacon_node/eth2-libp2p/src/service.rs

@@ -9,19 +9,24 @@ use futures::prelude::*;
 use futures::Stream;
 use libp2p::core::{
     identity::Keypair, multiaddr::Multiaddr, muxing::StreamMuxerBox, nodes::Substream,
-    transport::boxed::Boxed,
+    transport::boxed::Boxed, ConnectedPoint,
 };
-use libp2p::{core, secio, PeerId, Swarm, Transport};
+use libp2p::{core, secio, swarm::NetworkBehaviour, PeerId, Swarm, Transport};
 use slog::{crit, debug, info, trace, warn};
+use smallvec::SmallVec;
 use std::fs::File;
 use std::io::prelude::*;
 use std::io::{Error, ErrorKind};
 use std::time::Duration;
+use std::time::Instant;
 
 type Libp2pStream = Boxed<(PeerId, StreamMuxerBox), Error>;
 type Libp2pBehaviour = Behaviour<Substream<StreamMuxerBox>>;
 
 const NETWORK_KEY_FILENAME: &str = "key";
+/// The time in milliseconds to wait before banning a peer. This allows for any Goodbye messages to be
+/// flushed and protocols to be negotiated.
+const BAN_PEER_TIMEOUT: u64 = 200;
 
 /// The configuration and state of the libp2p components for the beacon node.
 pub struct Service {
@@ -32,8 +37,11 @@ pub struct Service {
     /// This node's PeerId.
     pub local_peer_id: PeerId,
 
+    /// A current list of peers to ban after a given timeout.
+    peers_to_ban: SmallVec<[(PeerId, Instant); 4]>,
+
     /// Indicates if the listening address have been verified and compared to the expected ENR.
-    pub verified_listen_address: bool,
+    verified_listen_address: bool,
 
     /// The libp2p logger handle.
     pub log: slog::Logger,
@@ -149,10 +157,19 @@ impl Service {
         Ok(Service {
             local_peer_id,
             swarm,
+            peers_to_ban: SmallVec::new(),
             verified_listen_address: false,
             log,
         })
     }
+
+    /// Adds a peer to be banned after a timeout period.
+    pub fn disconnect_and_ban_peer(&mut self, peer_id: PeerId) {
+        self.peers_to_ban.push((
+            peer_id,
+            Instant::now() + Duration::from_millis(BAN_PEER_TIMEOUT),
+        ));
+    }
 }
 
 impl Stream for Service {
@@ -193,7 +210,11 @@ impl Stream for Service {
                     }
                 },
                 Ok(Async::Ready(None)) => unreachable!("Swarm stream shouldn't end"),
-                Ok(Async::NotReady) => {
+                Ok(Async::NotReady) => break,
+                _ => break,
+            }
+        }
+        // swarm is not ready
         // check to see if the address is different to the config. If so, update our ENR
         if !self.verified_listen_address {
             let multiaddr = Swarm::listeners(&self.swarm).next();
@@ -204,11 +225,28 @@ impl Stream for Service {
             }
         }
 
+        // check if there are peers to ban
+        while !self.peers_to_ban.is_empty() {
+            if self.peers_to_ban[0].1 < Instant::now() {
+                let (peer_id, _) = self.peers_to_ban.remove(0);
+                warn!(self.log, "Disconnecting and banning peer"; "peer_id" => format!("{:?}", peer_id));
+                Swarm::ban_peer_id(&mut self.swarm, peer_id.clone());
+                // TODO: Correctly notify protocols of the disconnect
+                // TODO: Also remove peer from the DHT: https://github.com/sigp/lighthouse/issues/629
+                let dummy_connected_point = ConnectedPoint::Dialer {
+                    address: "/ip4/0.0.0.0"
+                        .parse::<Multiaddr>()
+                        .expect("valid multiaddr"),
+                };
+                self.swarm
+                    .inject_disconnected(&peer_id, dummy_connected_point);
+                // inform the behaviour that the peer has been banned
+                self.swarm.peer_banned(peer_id);
+            } else {
                 break;
             }
-                _ => break,
-            }
         }
+
         Ok(Async::NotReady)
     }
 }

beacon_node/network/src/message_handler.rs

@@ -195,9 +195,6 @@ impl<T: BeaconChainTypes> MessageHandler<T> {
                             }
                         }
                     }
-                    RPCResponse::Goodbye => {
-                        // A goodbye was successfully sent, ignore it
-                    }
                 }
             }
             RPCErrorResponse::StreamTermination(response_type) => {

beacon_node/network/src/service.rs

@@ -4,15 +4,12 @@ use crate::NetworkConfig;
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use core::marker::PhantomData;
 use eth2_libp2p::Service as LibP2PService;
-use eth2_libp2p::{
+use eth2_libp2p::{rpc::RPCRequest, Enr, Libp2pEvent, Multiaddr, PeerId, Swarm, Topic};
-    rpc::{RPCErrorResponse, RPCRequest, RPCResponse},
-    ConnectedPoint, Enr, Libp2pEvent, Multiaddr, NetworkBehaviour, PeerId, Swarm, Topic,
-};
 use eth2_libp2p::{PubsubMessage, RPCEvent};
 use futures::prelude::*;
 use futures::Stream;
-use parking_lot::{Mutex, MutexGuard};
+use parking_lot::Mutex;
-use slog::{debug, info, trace, warn};
+use slog::{debug, info, trace};
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
 use tokio::sync::{mpsc, oneshot};
@@ -158,9 +155,6 @@ fn network_service(
     propagation_percentage: Option<u8>,
 ) -> impl futures::Future<Item = (), Error = eth2_libp2p::error::Error> {
     futures::future::poll_fn(move || -> Result<_, eth2_libp2p::error::Error> {
-        // keep a list of peers to disconnect, once all channels are processed, remove the peers.
-        let mut peers_to_ban = Vec::new();
-
         // processes the network channel before processing the libp2p swarm
         loop {
             // poll the network channel
@@ -218,7 +212,7 @@ fn network_service(
                         }
                     }
                     NetworkMessage::Disconnect { peer_id } => {
-                        peers_to_ban.push(peer_id);
+                        libp2p_service.lock().disconnect_and_ban_peer(peer_id);
                     }
                 },
                 Ok(Async::NotReady) => break,
@@ -233,21 +227,15 @@ fn network_service(
 
         loop {
             // poll the swarm
-            match libp2p_service.lock().poll() {
+            let mut locked_service = libp2p_service.lock();
+            match locked_service.poll() {
                 Ok(Async::Ready(Some(event))) => match event {
                     Libp2pEvent::RPC(peer_id, rpc_event) => {
                         trace!(log, "Received RPC"; "rpc" => format!("{}", rpc_event));
 
-                        // if we received or sent a Goodbye message, drop and ban the peer
+                        // if we received a Goodbye message, drop and ban the peer
-                        match rpc_event {
+                        if let RPCEvent::Request(_, RPCRequest::Goodbye(_)) = rpc_event {
-                            RPCEvent::Request(_, RPCRequest::Goodbye(_))
+                            locked_service.disconnect_and_ban_peer(peer_id.clone());
-                            | RPCEvent::Response(
-                                _,
-                                RPCErrorResponse::Success(RPCResponse::Goodbye),
-                            ) => {
-                                peers_to_ban.push(peer_id.clone());
-                            }
-                            _ => {}
                         };
                         message_handler_send
                             .try_send(HandlerMessage::RPC(peer_id, rpc_event))
@@ -283,32 +271,10 @@ fn network_service(
             }
         }
 
-        while !peers_to_ban.is_empty() {
-            let service = libp2p_service.lock();
-            disconnect_peer(service, peers_to_ban.pop().expect("element exists"), &log);
-        }
-
         Ok(Async::NotReady)
     })
 }
 
-fn disconnect_peer(mut service: MutexGuard<LibP2PService>, peer_id: PeerId, log: &slog::Logger) {
-    warn!(log, "Disconnecting and banning peer"; "peer_id" => format!("{:?}", peer_id));
-    Swarm::ban_peer_id(&mut service.swarm, peer_id.clone());
-    // TODO: Correctly notify protocols of the disconnect
-    // TOOD: Also remove peer from the DHT: https://github.com/sigp/lighthouse/issues/629
-    let dummy_connected_point = ConnectedPoint::Dialer {
-        address: "/ip4/0.0.0.0"
-            .parse::<Multiaddr>()
-            .expect("valid multiaddr"),
-    };
-    service
-        .swarm
-        .inject_disconnected(&peer_id, dummy_connected_point);
-    // inform the behaviour that the peer has been banned
-    service.swarm.peer_banned(peer_id);
-}
-
 /// Types of messages that the network service can receive.
 #[derive(Debug)]
 pub enum NetworkMessage {

beacon_node/src/cli.rs

@@ -282,7 +282,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
                     .short("m")
                     .value_name("MINUTES")
                     .required(true)
-                    .default_value("0")
+                    .default_value("30")
                     .help("The maximum number of minutes that will have elapsed before genesis"))
             )
             /*

eth2/types/Cargo.toml

@@ -31,6 +31,8 @@ tree_hash = "0.1.0"
 tree_hash_derive = "0.2"
 rand_xorshift = "0.2.0"
 cached_tree_hash = { path = "../utils/cached_tree_hash" }
+serde_yaml = "0.8.11"
+tempfile = "3.1.0"
 
 [dev-dependencies]
 env_logger = "0.7.1"

eth2/types/src/chain_spec.rs

@@ -1,7 +1,7 @@
 use crate::*;
 use int_to_bytes::int_to_bytes4;
 use serde_derive::{Deserialize, Serialize};
-use utils::{u8_from_hex_str, u8_to_hex_str};
+use utils::{u32_from_hex_str, u32_to_hex_str, u8_from_hex_str, u8_to_hex_str};
 
 /// Each of the BLS signature domains.
 ///
@@ -253,10 +253,10 @@ impl ChainSpec {
             target_committee_size: 4,
             shuffle_round_count: 10,
             min_genesis_active_validator_count: 64,
-            milliseconds_per_slot: 6_000,
             network_id: 2, // lighthouse testnet network id
             boot_nodes,
             eth1_follow_distance: 16,
+            milliseconds_per_slot: 6_000,
             ..ChainSpec::mainnet()
         }
     }
@@ -317,3 +317,341 @@ mod tests {
         test_domain(Domain::VoluntaryExit, spec.domain_voluntary_exit, &spec);
     }
 }
+
+// Yaml Config is declared here in order to access domain fields of ChainSpec which are private fields.
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+#[serde(rename_all = "UPPERCASE")]
+#[serde(default)]
+#[serde(deny_unknown_fields)]
+/// Union of a ChainSpec struct and an EthSpec struct that holds constants used for the configs folder of the Ethereum 2 spec (https://github.com/ethereum/eth2.0-specs/tree/dev/configs)
+/// Spec v0.9.1
+pub struct YamlConfig {
+    // ChainSpec
+    far_future_epoch: u64,
+    base_rewards_per_epoch: u64,
+    deposit_contract_tree_depth: u64,
+    seconds_per_day: u64,
+    max_committees_per_slot: usize,
+    target_committee_size: usize,
+    min_per_epoch_churn_limit: u64,
+    churn_limit_quotient: u64,
+    shuffle_round_count: u8,
+    min_genesis_active_validator_count: u64,
+    min_genesis_time: u64,
+    min_deposit_amount: u64,
+    max_effective_balance: u64,
+    ejection_balance: u64,
+    effective_balance_increment: u64,
+    genesis_slot: u64,
+    #[serde(deserialize_with = "u8_from_hex_str", serialize_with = "u8_to_hex_str")]
+    bls_withdrawal_prefix: u8,
+    seconds_per_slot: u64,
+    min_attestation_inclusion_delay: u64,
+    min_seed_lookahead: u64,
+    min_validator_withdrawability_delay: u64,
+    persistent_committee_period: u64,
+    min_epochs_to_inactivity_penalty: u64,
+    base_reward_factor: u64,
+    whistleblower_reward_quotient: u64,
+    proposer_reward_quotient: u64,
+    inactivity_penalty_quotient: u64,
+    min_slashing_penalty_quotient: u64,
+    safe_slots_to_update_justified: u64,
+
+    #[serde(skip_serializing)]
+    genesis_fork: Fork,
+
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_beacon_proposer: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_beacon_attester: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_randao: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_deposit: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    domain_voluntary_exit: u32,
+    #[serde(
+        deserialize_with = "u32_from_hex_str",
+        serialize_with = "u32_to_hex_str"
+    )]
+    // EthSpec
+    justification_bits_length: u32,
+    max_validators_per_committee: u32,
+    genesis_epoch: Epoch,
+    slots_per_epoch: u64,
+    slots_per_eth1_voting_period: usize,
+    slots_per_historical_root: usize,
+    epochs_per_historical_vector: usize,
+    epochs_per_slashings_vector: usize,
+    historical_roots_limit: u64,
+    validator_registry_limit: u64,
+    max_proposer_slashings: u32,
+    max_attester_slashings: u32,
+    max_attestations: u32,
+    max_deposits: u32,
+    max_voluntary_exits: u32,
+
+    // Unused
+    #[serde(skip_serializing)]
+    early_derived_secret_penalty_max_future_epochs: u32,
+    #[serde(skip_serializing)]
+    max_seed_lookahead: u32,
+    #[serde(skip_serializing)]
+    deposit_contract_address: String,
+
+    // Phase 1
+    #[serde(skip_serializing)]
+    epochs_per_custody_period: u32,
+    #[serde(skip_serializing)]
+    custody_period_to_randao_padding: u32,
+    #[serde(skip_serializing)]
+    shard_slots_per_beacon_slot: u32,
+    #[serde(skip_serializing)]
+    epochs_per_shard_period: u32,
+    #[serde(skip_serializing)]
+    phase_1_fork_epoch: u32,
+    #[serde(skip_serializing)]
+    phase_1_fork_slot: u32,
+    #[serde(skip_serializing)]
+    domain_custody_bit_challenge: u32,
+    #[serde(skip_serializing)]
+    domain_shard_proposer: u32,
+    #[serde(skip_serializing)]
+    domain_shard_attester: u32,
+    #[serde(skip_serializing)]
+    max_epochs_per_crosslink: u64,
+}
+
+impl Default for YamlConfig {
+    fn default() -> Self {
+        let chain_spec = MainnetEthSpec::default_spec();
+        YamlConfig::from_spec::<MainnetEthSpec>(&chain_spec)
+    }
+}
+
+/// Spec v0.8.1
+impl YamlConfig {
+    pub fn from_spec<T: EthSpec>(spec: &ChainSpec) -> Self {
+        Self {
+            // ChainSpec
+            far_future_epoch: spec.far_future_epoch.into(),
+            base_rewards_per_epoch: spec.base_rewards_per_epoch,
+            deposit_contract_tree_depth: spec.deposit_contract_tree_depth,
+            seconds_per_day: spec.seconds_per_day,
+            max_committees_per_slot: spec.max_committees_per_slot,
+            target_committee_size: spec.target_committee_size,
+            min_per_epoch_churn_limit: spec.min_per_epoch_churn_limit,
+            churn_limit_quotient: spec.churn_limit_quotient,
+            shuffle_round_count: spec.shuffle_round_count,
+            min_genesis_active_validator_count: spec.min_genesis_active_validator_count,
+            min_genesis_time: spec.min_genesis_time,
+            min_deposit_amount: spec.min_deposit_amount,
+            max_effective_balance: spec.max_effective_balance,
+            ejection_balance: spec.ejection_balance,
+            effective_balance_increment: spec.effective_balance_increment,
+            genesis_slot: spec.genesis_slot.into(),
+            bls_withdrawal_prefix: spec.bls_withdrawal_prefix_byte,
+            seconds_per_slot: spec.milliseconds_per_slot / 1000,
+            min_attestation_inclusion_delay: spec.min_attestation_inclusion_delay,
+            min_seed_lookahead: spec.min_seed_lookahead.into(),
+            min_validator_withdrawability_delay: spec.min_validator_withdrawability_delay.into(),
+            persistent_committee_period: spec.persistent_committee_period,
+            min_epochs_to_inactivity_penalty: spec.min_epochs_to_inactivity_penalty,
+            base_reward_factor: spec.base_reward_factor,
+            whistleblower_reward_quotient: spec.whistleblower_reward_quotient,
+            proposer_reward_quotient: spec.proposer_reward_quotient,
+            inactivity_penalty_quotient: spec.inactivity_penalty_quotient,
+            min_slashing_penalty_quotient: spec.min_slashing_penalty_quotient,
+            genesis_fork: spec.genesis_fork.clone(),
+            safe_slots_to_update_justified: spec.safe_slots_to_update_justified,
+            domain_beacon_proposer: spec.domain_beacon_proposer,
+            domain_beacon_attester: spec.domain_beacon_attester,
+            domain_randao: spec.domain_randao,
+            domain_deposit: spec.domain_deposit,
+            domain_voluntary_exit: spec.domain_voluntary_exit,
+
+            // EthSpec
+            justification_bits_length: T::JustificationBitsLength::to_u32(),
+            max_validators_per_committee: T::MaxValidatorsPerCommittee::to_u32(),
+            genesis_epoch: T::genesis_epoch(),
+            slots_per_epoch: T::slots_per_epoch(),
+            slots_per_eth1_voting_period: T::slots_per_eth1_voting_period(),
+            slots_per_historical_root: T::slots_per_historical_root(),
+            epochs_per_historical_vector: T::epochs_per_historical_vector(),
+            epochs_per_slashings_vector: T::EpochsPerSlashingsVector::to_usize(),
+            historical_roots_limit: T::HistoricalRootsLimit::to_u64(),
+            validator_registry_limit: T::ValidatorRegistryLimit::to_u64(),
+            max_proposer_slashings: T::MaxProposerSlashings::to_u32(),
+            max_attester_slashings: T::MaxAttesterSlashings::to_u32(),
+            max_attestations: T::MaxAttestations::to_u32(),
+            max_deposits: T::MaxDeposits::to_u32(),
+            max_voluntary_exits: T::MaxVoluntaryExits::to_u32(),
+
+            // Unused
+            early_derived_secret_penalty_max_future_epochs: 0,
+            max_seed_lookahead: 0,
+            deposit_contract_address: String::new(),
+
+            // Phase 1
+            epochs_per_custody_period: 0,
+            custody_period_to_randao_padding: 0,
+            shard_slots_per_beacon_slot: 0,
+            epochs_per_shard_period: 0,
+            phase_1_fork_epoch: 0,
+            phase_1_fork_slot: 0,
+            domain_custody_bit_challenge: 0,
+            domain_shard_proposer: 0,
+            domain_shard_attester: 0,
+            max_epochs_per_crosslink: 0,
+        }
+    }
+
+    pub fn apply_to_chain_spec<T: EthSpec>(&self, chain_spec: &ChainSpec) -> Option<ChainSpec> {
+        // Checking for EthSpec constants
+        if self.justification_bits_length != T::JustificationBitsLength::to_u32()
+            || self.max_validators_per_committee != T::MaxValidatorsPerCommittee::to_u32()
+            || self.genesis_epoch != T::genesis_epoch()
+            || self.slots_per_epoch != T::slots_per_epoch()
+            || self.slots_per_eth1_voting_period != T::slots_per_eth1_voting_period()
+            || self.slots_per_historical_root != T::slots_per_historical_root()
+            || self.epochs_per_historical_vector != T::epochs_per_historical_vector()
+            || self.epochs_per_slashings_vector != T::EpochsPerSlashingsVector::to_usize()
+            || self.historical_roots_limit != T::HistoricalRootsLimit::to_u64()
+            || self.validator_registry_limit != T::ValidatorRegistryLimit::to_u64()
+            || self.max_proposer_slashings != T::MaxProposerSlashings::to_u32()
+            || self.max_attester_slashings != T::MaxAttesterSlashings::to_u32()
+            || self.max_attestations != T::MaxAttestations::to_u32()
+            || self.max_deposits != T::MaxDeposits::to_u32()
+            || self.max_voluntary_exits != T::MaxVoluntaryExits::to_u32()
+        {
+            return None;
+        }
+
+        // Create a ChainSpec from the yaml config
+        Some(ChainSpec {
+            far_future_epoch: Epoch::from(self.far_future_epoch),
+            base_rewards_per_epoch: self.base_rewards_per_epoch,
+            deposit_contract_tree_depth: self.deposit_contract_tree_depth,
+            seconds_per_day: self.seconds_per_day,
+            target_committee_size: self.target_committee_size,
+            min_per_epoch_churn_limit: self.min_per_epoch_churn_limit,
+            churn_limit_quotient: self.churn_limit_quotient,
+            shuffle_round_count: self.shuffle_round_count,
+            min_genesis_active_validator_count: self.min_genesis_active_validator_count,
+            min_genesis_time: self.min_genesis_time,
+            min_deposit_amount: self.min_deposit_amount,
+            max_effective_balance: self.max_effective_balance,
+            ejection_balance: self.ejection_balance,
+            effective_balance_increment: self.effective_balance_increment,
+            genesis_slot: Slot::from(self.genesis_slot),
+            bls_withdrawal_prefix_byte: self.bls_withdrawal_prefix,
+            milliseconds_per_slot: self.seconds_per_slot * 1000,
+            min_attestation_inclusion_delay: self.min_attestation_inclusion_delay,
+            min_seed_lookahead: Epoch::from(self.min_seed_lookahead),
+            min_validator_withdrawability_delay: Epoch::from(
+                self.min_validator_withdrawability_delay,
+            ),
+            persistent_committee_period: self.persistent_committee_period,
+            min_epochs_to_inactivity_penalty: self.min_epochs_to_inactivity_penalty,
+            base_reward_factor: self.base_reward_factor,
+            whistleblower_reward_quotient: self.whistleblower_reward_quotient,
+            proposer_reward_quotient: self.proposer_reward_quotient,
+            inactivity_penalty_quotient: self.inactivity_penalty_quotient,
+            min_slashing_penalty_quotient: self.min_slashing_penalty_quotient,
+            domain_beacon_proposer: self.domain_beacon_proposer,
+            domain_randao: self.domain_randao,
+            domain_deposit: self.domain_deposit,
+            domain_voluntary_exit: self.domain_voluntary_exit,
+            boot_nodes: chain_spec.boot_nodes.clone(),
+            genesis_fork: chain_spec.genesis_fork.clone(),
+            ..*chain_spec
+        })
+    }
+}
+
+#[cfg(test)]
+mod yaml_tests {
+    use super::*;
+    use std::fs::OpenOptions;
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn minimal_round_trip() {
+        // create temp file
+        let tmp_file = NamedTempFile::new().expect("failed to create temp file");
+        let writer = OpenOptions::new()
+            .read(false)
+            .write(true)
+            .open(tmp_file.as_ref())
+            .expect("error opening file");
+        let minimal_spec = ChainSpec::minimal();
+
+        let yamlconfig = YamlConfig::from_spec::<MinimalEthSpec>(&minimal_spec);
+        // write fresh minimal config to file
+        serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize");
+
+        let reader = OpenOptions::new()
+            .read(true)
+            .write(false)
+            .open(tmp_file.as_ref())
+            .expect("error while opening the file");
+        // deserialize minimal config from file
+        let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing");
+        assert_eq!(from, yamlconfig);
+    }
+
+    #[test]
+    fn mainnet_round_trip() {
+        let tmp_file = NamedTempFile::new().expect("failed to create temp file");
+        let writer = OpenOptions::new()
+            .read(false)
+            .write(true)
+            .open(tmp_file.as_ref())
+            .expect("error opening file");
+        let mainnet_spec = ChainSpec::mainnet();
+        let yamlconfig = YamlConfig::from_spec::<MainnetEthSpec>(&mainnet_spec);
+        serde_yaml::to_writer(writer, &yamlconfig).expect("failed to write or serialize");
+
+        let reader = OpenOptions::new()
+            .read(true)
+            .write(false)
+            .open(tmp_file.as_ref())
+            .expect("error while opening the file");
+        let from: YamlConfig = serde_yaml::from_reader(reader).expect("error while deserializing");
+        assert_eq!(from, yamlconfig);
+    }
+
+    #[test]
+    fn apply_to_spec() {
+        let mut spec = ChainSpec::minimal();
+        let yamlconfig = YamlConfig::from_spec::<MinimalEthSpec>(&spec);
+
+        // modifying the original spec
+        spec.deposit_contract_tree_depth += 1;
+        // Applying a yaml config with incorrect EthSpec should fail
+        let res = yamlconfig.apply_to_chain_spec::<MainnetEthSpec>(&spec);
+        assert_eq!(res, None);
+
+        // Applying a yaml config with correct EthSpec should NOT fail
+        let new_spec = yamlconfig
+            .apply_to_chain_spec::<MinimalEthSpec>(&spec)
+            .expect("should have applied spec");
+        assert_eq!(new_spec, ChainSpec::minimal());
+    }
+}

eth2/types/src/utils/serde_utils.rs

@@ -11,7 +11,11 @@ where
 {
     let s: String = Deserialize::deserialize(deserializer)?;
 
-    u8::from_str_radix(&s.as_str()[2..], 16).map_err(D::Error::custom)
+    let start = match s.as_str().get(2..) {
+        Some(start) => start,
+        None => return Err(D::Error::custom("string length too small")),
+    };
+    u8::from_str_radix(&start, 16).map_err(D::Error::custom)
 }
 
 #[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `byte` to be a ref.
@@ -25,13 +29,45 @@ where
     serializer.serialize_str(&hex)
 }
 
+pub fn u32_from_hex_str<'de, D>(deserializer: D) -> Result<u32, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let s: String = Deserialize::deserialize(deserializer)?;
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+
+    u32::from_str_radix(&start, 16)
+        .map_err(D::Error::custom)
+        .map(u32::from_be)
+}
+
+#[allow(clippy::trivially_copy_pass_by_ref)] // Serde requires the `num` to be a ref.
+pub fn u32_to_hex_str<S>(num: &u32, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut hex: String = "0x".to_string();
+    let bytes = num.to_le_bytes();
+    hex.push_str(&hex::encode(&bytes));
+
+    serializer.serialize_str(&hex)
+}
+
 pub fn fork_from_hex_str<'de, D>(deserializer: D) -> Result<[u8; FORK_BYTES_LEN], D::Error>
 where
     D: Deserializer<'de>,
 {
     let s: String = Deserialize::deserialize(deserializer)?;
     let mut array = [0 as u8; FORK_BYTES_LEN];
-    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+    let decoded: Vec<u8> = hex::decode(&start).map_err(D::Error::custom)?;
 
     if decoded.len() != FORK_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));
@@ -76,7 +112,12 @@ where
 {
     let s: String = Deserialize::deserialize(deserializer)?;
     let mut array = [0 as u8; GRAFFITI_BYTES_LEN];
-    let decoded: Vec<u8> = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?;
+
+    let start = s
+        .as_str()
+        .get(2..)
+        .ok_or_else(|| D::Error::custom("string length too small"))?;
+    let decoded: Vec<u8> = hex::decode(&start).map_err(D::Error::custom)?;
 
     if decoded.len() > GRAFFITI_BYTES_LEN {
         return Err(D::Error::custom("Fork length too long"));

validator_client/src/config.rs

@@ -36,8 +36,11 @@ pub struct Config {
 impl Default for Config {
     /// Build a new configuration from defaults.
     fn default() -> Self {
+        let mut data_dir = dirs::home_dir().unwrap_or_else(|| PathBuf::from("."));
+        data_dir.push(".lighthouse");
+        data_dir.push("validators");
         Self {
-            data_dir: PathBuf::from(".lighthouse/validators"),
+            data_dir,
             key_source: <_>::default(),
             http_server: DEFAULT_HTTP_SERVER.to_string(),
         }