diff --git a/.gitmodules b/.gitmodules index 1b0e150ce6..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "tests/ef_tests/eth2.0-spec-tests"] - path = tests/ef_tests/eth2.0-spec-tests - url = https://github.com/ethereum/eth2.0-spec-tests diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..d5517ed231 --- /dev/null +++ b/Makefile @@ -0,0 +1,31 @@ +TESTS_TAG := v0.8.3 +TESTS = general minimal mainnet + +TESTS_BASE_DIR := ./tests/ef_tests +REPO_NAME := eth2.0-spec-tests +OUTPUT_DIR := $(TESTS_BASE_DIR)/$(REPO_NAME) + +BASE_URL := https://github.com/ethereum/$(REPO_NAME)/releases/download/$(SPEC_VERSION) + +release: + cargo build --all --release + +clean_ef_tests: + rm -r $(OUTPUT_DIR) + +ef_tests: download_tests extract_tests + mkdir $(OUTPUT_DIR) + for test in $(TESTS); do \ + tar -C $(OUTPUT_DIR) -xvf $(TESTS_BASE_DIR)/$$test.tar ;\ + rm $(TESTS_BASE_DIR)/$$test.tar ;\ + done + +extract_tests: + for test in $(TESTS); do \ + gzip -df $(TESTS_BASE_DIR)/$$test.tar.gz ;\ + done + +download_tests: + for test in $(TESTS); do \ + wget -P $(TESTS_BASE_DIR) $(BASE_URL)/$$test.tar.gz; \ + done diff --git a/account_manager/src/main.rs b/account_manager/src/main.rs index b7448ddf25..ae3823049d 100644 --- a/account_manager/src/main.rs +++ b/account_manager/src/main.rs @@ -125,9 +125,13 @@ fn main() { } } } - _ => panic!( - "The account manager must be run with a subcommand. See help for more information." - ), + _ => { + crit!( + log, + "The account manager must be run with a subcommand. See help for more information." + ); + return; + } } } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 5409d37287..aa9332c01d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -178,7 +178,7 @@ impl BeaconChain { genesis_state.genesis_time, Duration::from_millis(spec.milliseconds_per_slot), ) - .ok_or_else(|| Error::SlotClockDidNotStart)?; + .map_err(|_| Error::SlotClockDidNotStart)?; info!(log, "Beacon chain initialized from genesis"; "validator_count" => genesis_state.validators.len(), @@ -220,7 +220,7 @@ impl BeaconChain { state.genesis_time, Duration::from_millis(spec.milliseconds_per_slot), ) - .ok_or_else(|| Error::SlotClockDidNotStart)?; + .map_err(|_| Error::SlotClockDidNotStart)?; let last_finalized_root = p.canonical_head.beacon_state.finalized_checkpoint.root; let last_finalized_block = &p.canonical_head.beacon_block; @@ -459,6 +459,15 @@ impl BeaconChain { None } + /// Returns the block canonical root of the current canonical chain at a given slot. + /// + /// Returns None if a block doesn't exist at the slot. + pub fn root_at_slot(&self, target_slot: Slot) -> Option { + self.rev_iter_block_roots() + .find(|(_root, slot)| *slot == target_slot) + .map(|(root, _slot)| root) + } + /// Reads the slot clock (see `self.read_slot_clock()` and returns the number of slots since /// genesis. pub fn slots_since_genesis(&self) -> Option { @@ -1017,7 +1026,7 @@ impl BeaconChain { }; // Load the parent blocks state from the database, returning an error if it is not found. - // It is an error because if know the parent block we should also know the parent state. + // It is an error because if we know the parent block we should also know the parent state. let parent_state_root = parent_block.state_root; let parent_state = self .store diff --git a/beacon_node/client/src/lib.rs b/beacon_node/client/src/lib.rs index 1d3cb40ecf..afcd538b58 100644 --- a/beacon_node/client/src/lib.rs +++ b/beacon_node/client/src/lib.rs @@ -16,7 +16,7 @@ use slog::{crit, error, info, o}; use slot_clock::SlotClock; use std::marker::PhantomData; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::runtime::TaskExecutor; use tokio::timer::Interval; use types::EthSpec; @@ -177,8 +177,18 @@ where .map_err(error::Error::from)?, ); - if beacon_chain.slot().is_err() { - panic!("Cannot start client before genesis!") + let since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| format!("Unable to read system time: {}", e))?; + let since_genesis = Duration::from_secs(beacon_chain.head().beacon_state.genesis_time); + + if since_genesis > since_epoch { + info!( + log, + "Starting node prior to genesis"; + "now" => since_epoch.as_secs(), + "genesis_seconds" => since_genesis.as_secs(), + ); } let network_config = &client_config.network; diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index d705637cba..343918d4d5 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -34,10 +34,10 @@ pub fn run(client: &Client, executor: TaskExecutor, exit // Panics if libp2p is poisoned. let connected_peer_count = libp2p.lock().swarm.connected_peers(); - debug!(log, "Libp2p connected peer status"; "peer_count" => connected_peer_count); + debug!(log, "Connected peer status"; "peer_count" => connected_peer_count); if connected_peer_count <= WARN_PEER_COUNT { - warn!(log, "Low libp2p peer count"; "peer_count" => connected_peer_count); + warn!(log, "Low peer count"; "peer_count" => connected_peer_count); } Ok(()) diff --git a/beacon_node/eth2-libp2p/Cargo.toml b/beacon_node/eth2-libp2p/Cargo.toml index caa5b28e43..59c7991053 100644 --- a/beacon_node/eth2-libp2p/Cargo.toml +++ b/beacon_node/eth2-libp2p/Cargo.toml @@ -7,8 +7,8 @@ edition = "2018" [dependencies] clap = "2.32.0" #SigP repository -libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "61036890d574f5b46573952b20def2baafd6a6e9" } -enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "61036890d574f5b46573952b20def2baafd6a6e9", features = ["serde"] } +libp2p = { git = "https://github.com/SigP/rust-libp2p", rev = "76f7475e4b7063e663ad03c7524cf091f9961968" } +enr = { git = "https://github.com/SigP/rust-libp2p/", rev = "76f7475e4b7063e663ad03c7524cf091f9961968", features = ["serde"] } types = { path = "../../eth2/types" } serde = "1.0" serde_derive = "1.0" diff --git a/beacon_node/eth2-libp2p/src/behaviour.rs b/beacon_node/eth2-libp2p/src/behaviour.rs index 2c574e46ad..a47d32ec2a 100644 --- a/beacon_node/eth2-libp2p/src/behaviour.rs +++ b/beacon_node/eth2-libp2p/src/behaviour.rs @@ -15,7 +15,7 @@ use libp2p::{ tokio_io::{AsyncRead, AsyncWrite}, NetworkBehaviour, PeerId, }; -use slog::{debug, o, trace}; +use slog::{debug, o}; use std::num::NonZeroU32; use std::time::Duration; @@ -90,13 +90,15 @@ impl NetworkBehaviourEventProcess { - trace!(self.log, "Received GossipEvent"); - + GossipsubEvent::Message(propagation_source, gs_msg) => { + let id = gs_msg.id(); let msg = PubsubMessage::from_topics(&gs_msg.topics, gs_msg.data); + // Note: We are keeping track here of the peer that sent us the message, not the + // peer that originally published the message. self.events.push(BehaviourEvent::GossipMessage { - source: gs_msg.source, + id, + source: propagation_source, topics: gs_msg.topics, message: msg, }); @@ -199,6 +201,13 @@ impl Behaviour { } } + /// Forwards a message that is waiting in gossipsub's mcache. Messages are only propagated + /// once validated by the beacon chain. + pub fn propagate_message(&mut self, propagation_source: &PeerId, message_id: String) { + self.gossipsub + .propagate_message(&message_id, propagation_source); + } + /* Eth2 RPC behaviour functions */ /// Sends an RPC Request/Response via the RPC protocol. @@ -214,12 +223,21 @@ impl Behaviour { /// The types of events than can be obtained from polling the behaviour. pub enum BehaviourEvent { + /// A received RPC event and the peer that it was received from. RPC(PeerId, RPCEvent), + /// We have completed an initial connection to a new peer. PeerDialed(PeerId), + /// A peer has disconnected. PeerDisconnected(PeerId), + /// A gossipsub message has been received. GossipMessage { + /// The gossipsub message id. Used when propagating blocks after validation. + id: String, + /// The peer from which we received this message, not the peer that published it. source: PeerId, + /// The topics that this message was sent on. topics: Vec, + /// The message itself. message: PubsubMessage, }, } diff --git a/beacon_node/eth2-libp2p/src/config.rs b/beacon_node/eth2-libp2p/src/config.rs index 7cb501c1f0..fd44b99af1 100644 --- a/beacon_node/eth2-libp2p/src/config.rs +++ b/beacon_node/eth2-libp2p/src/config.rs @@ -74,7 +74,8 @@ impl Default for Config { // parameter. gs_config: GossipsubConfigBuilder::new() .max_transmit_size(1_048_576) - .heartbeat_interval(Duration::from_secs(20)) + .heartbeat_interval(Duration::from_secs(20)) // TODO: Reduce for mainnet + .propagate_messages(false) // require validation before propagation .build(), boot_nodes: vec![], libp2p_nodes: vec![], diff --git a/beacon_node/eth2-libp2p/src/discovery.rs b/beacon_node/eth2-libp2p/src/discovery.rs index 4a8aba2b1b..69ca39ad79 100644 --- a/beacon_node/eth2-libp2p/src/discovery.rs +++ b/beacon_node/eth2-libp2p/src/discovery.rs @@ -114,7 +114,7 @@ impl Discovery { self.find_peers(); } - /// Add an Enr to the routing table of the discovery mechanism. + /// Add an ENR to the routing table of the discovery mechanism. pub fn add_enr(&mut self, enr: Enr) { self.discovery.add_enr(enr); } @@ -169,6 +169,7 @@ where fn inject_connected(&mut self, peer_id: PeerId, _endpoint: ConnectedPoint) { self.connected_peers.insert(peer_id); + // TODO: Drop peers if over max_peer limit metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); metrics::set_gauge(&metrics::PEERS_CONNECTED, self.connected_peers() as i64); diff --git a/beacon_node/eth2-libp2p/src/rpc/codec/base.rs b/beacon_node/eth2-libp2p/src/rpc/codec/base.rs index a8a2398673..973567473e 100644 --- a/beacon_node/eth2-libp2p/src/rpc/codec/base.rs +++ b/beacon_node/eth2-libp2p/src/rpc/codec/base.rs @@ -101,13 +101,15 @@ where type Error = ::Error; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + // if we have only received the response code, wait for more bytes + if src.len() == 1 { + return Ok(None); + } + // using the response code determine which kind of payload needs to be decoded. let response_code = { if let Some(resp_code) = self.response_code { resp_code } else { - // buffer should not be empty - debug_assert!(!src.is_empty()); - let resp_byte = src.split_to(1); let mut resp_code_byte = [0; 1]; resp_code_byte.copy_from_slice(&resp_byte); diff --git a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs index 260a00346c..d0e4d01cf3 100644 --- a/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs +++ b/beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs @@ -4,7 +4,7 @@ use crate::rpc::{ protocol::{ProtocolId, RPCError}, }; use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse}; -use bytes::{Bytes, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; use ssz::{Decode, Encode}; use tokio::codec::{Decoder, Encoder}; use unsigned_varint::codec::UviBytes; @@ -56,6 +56,10 @@ impl Encoder for SSZInboundCodec { .inner .encode(Bytes::from(bytes), dst) .map_err(RPCError::from); + } else { + // payload is empty, add a 0-byte length prefix + dst.reserve(1); + dst.put_u8(0); } Ok(()) } @@ -152,45 +156,49 @@ impl Decoder for SSZOutboundCodec { type Error = RPCError; fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { - match self.inner.decode(src).map_err(RPCError::from) { - Ok(Some(packet)) => match self.protocol.message_name.as_str() { + if src.len() == 1 && src[0] == 0_u8 { + // the object is empty. We return the empty object if this is the case + match self.protocol.message_name.as_str() { "hello" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes( - &packet, - )?))), + "1" => Err(RPCError::Custom( + "Hello stream terminated unexpectedly".into(), + )), // cannot have an empty HELLO message. The stream has terminated unexpectedly _ => unreachable!("Cannot negotiate an unknown version"), }, "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")), "beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))), + "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))), _ => unreachable!("Cannot negotiate an unknown version"), }, "recent_beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))), + "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))), _ => unreachable!("Cannot negotiate an unknown version"), }, _ => unreachable!("Cannot negotiate an unknown protocol"), - }, - Ok(None) => { - // the object sent could be a empty. We return the empty object if this is the case - match self.protocol.message_name.as_str() { + } + } else { + match self.inner.decode(src).map_err(RPCError::from) { + Ok(Some(packet)) => match self.protocol.message_name.as_str() { "hello" => match self.protocol.version.as_str() { - "1" => Ok(None), // cannot have an empty HELLO message. The stream has terminated unexpectedly + "1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes( + &packet, + )?))), _ => unreachable!("Cannot negotiate an unknown version"), }, "goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")), "beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))), + "1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))), _ => unreachable!("Cannot negotiate an unknown version"), }, "recent_beacon_blocks" => match self.protocol.version.as_str() { - "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))), + "1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))), _ => unreachable!("Cannot negotiate an unknown version"), }, _ => unreachable!("Cannot negotiate an unknown protocol"), - } + }, + Ok(None) => Ok(None), // waiting for more bytes + Err(e) => Err(e), } - Err(e) => Err(e), } } } diff --git a/beacon_node/eth2-libp2p/src/rpc/methods.rs b/beacon_node/eth2-libp2p/src/rpc/methods.rs index d912bcfa1e..ee8ad4860b 100644 --- a/beacon_node/eth2-libp2p/src/rpc/methods.rs +++ b/beacon_node/eth2-libp2p/src/rpc/methods.rs @@ -1,6 +1,5 @@ //!Available RPC methods types and ids. -use ssz::{impl_decode_via_from, impl_encode_via_from}; use ssz_derive::{Decode, Encode}; use types::{Epoch, Hash256, Slot}; @@ -66,8 +65,38 @@ impl Into for GoodbyeReason { } } -impl_encode_via_from!(GoodbyeReason, u64); -impl_decode_via_from!(GoodbyeReason, u64); +impl ssz::Encode for GoodbyeReason { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn ssz_bytes_len(&self) -> usize { + 0_u64.ssz_bytes_len() + } + + fn ssz_append(&self, buf: &mut Vec) { + let conv: u64 = self.clone().into(); + conv.ssz_append(buf) + } +} + +impl ssz::Decode for GoodbyeReason { + fn is_ssz_fixed_len() -> bool { + ::is_ssz_fixed_len() + } + + fn ssz_fixed_len() -> usize { + ::ssz_fixed_len() + } + + fn from_ssz_bytes(bytes: &[u8]) -> Result { + u64::from_ssz_bytes(bytes).and_then(|n| Ok(n.into())) + } +} /// Request a number of beacon block roots from a peer. #[derive(Encode, Decode, Clone, Debug, PartialEq)] @@ -157,3 +186,53 @@ impl ErrorMessage { String::from_utf8(self.error_message.clone()).unwrap_or_else(|_| "".into()) } } + +impl std::fmt::Display for HelloMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Hello Message: Fork Version: {:?}, Finalized Root: {}, Finalized Epoch: {}, Head Root: {}, Head Slot: {}", self.fork_version, self.finalized_root, self.finalized_epoch, self.head_root, self.head_slot) + } +} + +impl std::fmt::Display for RPCResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCResponse::Hello(hello) => write!(f, "{}", hello), + RPCResponse::BeaconBlocks(data) => write!(f, ", len: {}", data.len()), + RPCResponse::RecentBeaconBlocks(data) => { + write!(f, ", len: {}", data.len()) + } + } + } +} + +impl std::fmt::Display for RPCErrorResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCErrorResponse::Success(res) => write!(f, "{}", res), + RPCErrorResponse::InvalidRequest(err) => write!(f, "Invalid Request: {:?}", err), + RPCErrorResponse::ServerError(err) => write!(f, "Server Error: {:?}", err), + RPCErrorResponse::Unknown(err) => write!(f, "Unknown Error: {:?}", err), + } + } +} + +impl std::fmt::Display for GoodbyeReason { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + GoodbyeReason::ClientShutdown => write!(f, "Client Shutdown"), + GoodbyeReason::IrrelevantNetwork => write!(f, "Irrelevant Network"), + GoodbyeReason::Fault => write!(f, "Fault"), + GoodbyeReason::Unknown => write!(f, "Unknown Reason"), + } + } +} + +impl std::fmt::Display for BeaconBlocksRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Head Block Root: {}, Start Slot: {}, Count: {}, Step: {}", + self.head_block_root, self.start_slot, self.count, self.step + ) + } +} diff --git a/beacon_node/eth2-libp2p/src/rpc/mod.rs b/beacon_node/eth2-libp2p/src/rpc/mod.rs index 756a62e71b..2076615a9c 100644 --- a/beacon_node/eth2-libp2p/src/rpc/mod.rs +++ b/beacon_node/eth2-libp2p/src/rpc/mod.rs @@ -47,6 +47,16 @@ impl RPCEvent { } } +impl std::fmt::Display for RPCEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCEvent::Request(id, req) => write!(f, "RPC Request(Id: {}, {})", id, req), + RPCEvent::Response(id, res) => write!(f, "RPC Response(Id: {}, {})", id, res), + RPCEvent::Error(id, err) => write!(f, "RPC Request(Id: {}, Error: {:?})", id, err), + } + } +} + /// Implements the libp2p `NetworkBehaviour` trait and therefore manages network-level /// logic. pub struct RPC { diff --git a/beacon_node/eth2-libp2p/src/rpc/protocol.rs b/beacon_node/eth2-libp2p/src/rpc/protocol.rs index be1efdf5d4..401fa8b9ea 100644 --- a/beacon_node/eth2-libp2p/src/rpc/protocol.rs +++ b/beacon_node/eth2-libp2p/src/rpc/protocol.rs @@ -288,3 +288,14 @@ impl std::error::Error for RPCError { } } } + +impl std::fmt::Display for RPCRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RPCRequest::Hello(hello) => write!(f, "Hello Message: {}", hello), + RPCRequest::Goodbye(reason) => write!(f, "Goodbye: {}", reason), + RPCRequest::BeaconBlocks(req) => write!(f, "Beacon Blocks: {}", req), + RPCRequest::RecentBeaconBlocks(req) => write!(f, "Recent Beacon Blocks: {:?}", req), + } + } +} diff --git a/beacon_node/eth2-libp2p/src/service.rs b/beacon_node/eth2-libp2p/src/service.rs index 34781927ca..dac0117528 100644 --- a/beacon_node/eth2-libp2p/src/service.rs +++ b/beacon_node/eth2-libp2p/src/service.rs @@ -79,15 +79,32 @@ impl Service { } }; - // attempt to connect to user-input libp2p nodes - for multiaddr in config.libp2p_nodes { + // helper closure for dialing peers + let mut dial_addr = |multiaddr: Multiaddr| { match Swarm::dial_addr(&mut swarm, multiaddr.clone()) { Ok(()) => debug!(log, "Dialing libp2p peer"; "address" => format!("{}", multiaddr)), Err(err) => debug!( log, - "Could not connect to peer"; "address" => format!("{}", multiaddr), "Error" => format!("{:?}", err) + "Could not connect to peer"; "address" => format!("{}", multiaddr), "error" => format!("{:?}", err) ), }; + }; + + // attempt to connect to user-input libp2p nodes + for multiaddr in config.libp2p_nodes { + dial_addr(multiaddr); + } + + // attempt to connect to any specified boot-nodes + for bootnode_enr in config.boot_nodes { + for multiaddr in bootnode_enr.multiaddr() { + // ignore udp multiaddr if it exists + let components = multiaddr.iter().collect::>(); + if let Protocol::Udp(_) = components[1] { + continue; + } + dial_addr(multiaddr); + } } // subscribe to default gossipsub topics @@ -145,16 +162,16 @@ impl Stream for Service { fn poll(&mut self) -> Poll, Self::Error> { loop { match self.swarm.poll() { - //Behaviour events Ok(Async::Ready(Some(event))) => match event { - // TODO: Stub here for debugging BehaviourEvent::GossipMessage { + id, source, topics, message, } => { trace!(self.log, "Gossipsub message received"; "service" => "Swarm"); return Ok(Async::Ready(Some(Libp2pEvent::PubsubMessage { + id, source, topics, message, @@ -222,6 +239,7 @@ pub enum Libp2pEvent { PeerDisconnected(PeerId), /// Received pubsub message. PubsubMessage { + id: String, source: PeerId, topics: Vec, message: PubsubMessage, diff --git a/beacon_node/network/Cargo.toml b/beacon_node/network/Cargo.toml index dc08bd311c..06fc06ddea 100644 --- a/beacon_node/network/Cargo.toml +++ b/beacon_node/network/Cargo.toml @@ -19,3 +19,4 @@ futures = "0.1.25" error-chain = "0.12.0" tokio = "0.1.16" parking_lot = "0.9.0" +smallvec = "0.6.10" diff --git a/beacon_node/network/src/message_handler.rs b/beacon_node/network/src/message_handler.rs index c14fc970d7..782d2129ef 100644 --- a/beacon_node/network/src/message_handler.rs +++ b/beacon_node/network/src/message_handler.rs @@ -1,6 +1,6 @@ use crate::error; use crate::service::NetworkMessage; -use crate::sync::SimpleSync; +use crate::sync::MessageProcessor; use beacon_chain::{BeaconChain, BeaconChainTypes}; use eth2_libp2p::{ behaviour::PubsubMessage, @@ -9,18 +9,22 @@ use eth2_libp2p::{ }; use futures::future::Future; use futures::stream::Stream; -use slog::{debug, trace, warn}; +use slog::{debug, o, trace, warn}; use ssz::{Decode, DecodeError}; use std::sync::Arc; use tokio::sync::mpsc; use types::{Attestation, AttesterSlashing, BeaconBlock, ProposerSlashing, VoluntaryExit}; -/// Handles messages received from the network and client and organises syncing. +/// Handles messages received from the network and client and organises syncing. This +/// functionality of this struct is to validate an decode messages from the network before +/// passing them to the internal message processor. The message processor spawns a syncing thread +/// which manages which blocks need to be requested and processed. pub struct MessageHandler { - /// Currently loaded and initialised beacon chain. - _chain: Arc>, - /// The syncing framework. - sync: SimpleSync, + /// A channel to the network service to allow for gossip propagation. + network_send: mpsc::UnboundedSender, + /// Processes validated and decoded messages from the network. Has direct access to the + /// sync manager. + message_processor: MessageProcessor, /// The `MessageHandler` logger. log: slog::Logger, } @@ -34,8 +38,9 @@ pub enum HandlerMessage { PeerDisconnected(PeerId), /// An RPC response/request has been received. RPC(PeerId, RPCEvent), - /// A gossip message has been received. - PubsubMessage(PeerId, PubsubMessage), + /// A gossip message has been received. The fields are: message id, the peer that sent us this + /// message and the message itself. + PubsubMessage(String, PeerId, PubsubMessage), } impl MessageHandler { @@ -46,17 +51,20 @@ impl MessageHandler { executor: &tokio::runtime::TaskExecutor, log: slog::Logger, ) -> error::Result> { - trace!(log, "Service starting"); + let message_handler_log = log.new(o!("Service"=> "Message Handler")); + trace!(message_handler_log, "Service starting"); let (handler_send, handler_recv) = mpsc::unbounded_channel(); - // Initialise sync and begin processing in thread - let sync = SimpleSync::new(beacon_chain.clone(), network_send, &log); + + // Initialise a message instance, which itself spawns the syncing thread. + let message_processor = + MessageProcessor::new(executor, beacon_chain, network_send.clone(), &log); // generate the Message handler let mut handler = MessageHandler { - _chain: beacon_chain.clone(), - sync, - log: log.clone(), + network_send, + message_processor, + log: message_handler_log, }; // spawn handler task and move the message handler instance into the spawned thread @@ -65,7 +73,11 @@ impl MessageHandler { .for_each(move |msg| Ok(handler.handle_message(msg))) .map_err(move |_| { debug!(log, "Network message handler terminated."); - }), + }), /* + .then(move |_| { + debug!(log.clone(), "Message handler shutdown"); + }), + */ ); Ok(handler_send) @@ -76,19 +88,19 @@ impl MessageHandler { match message { // we have initiated a connection to a peer HandlerMessage::PeerDialed(peer_id) => { - self.sync.on_connect(peer_id); + self.message_processor.on_connect(peer_id); } // A peer has disconnected HandlerMessage::PeerDisconnected(peer_id) => { - self.sync.on_disconnect(peer_id); + self.message_processor.on_disconnect(peer_id); } // An RPC message request/response has been received HandlerMessage::RPC(peer_id, rpc_event) => { self.handle_rpc_message(peer_id, rpc_event); } // An RPC message request/response has been received - HandlerMessage::PubsubMessage(peer_id, gossip) => { - self.handle_gossip(peer_id, gossip); + HandlerMessage::PubsubMessage(id, peer_id, gossip) => { + self.handle_gossip(id, peer_id, gossip); } } } @@ -108,7 +120,7 @@ impl MessageHandler { fn handle_rpc_request(&mut self, peer_id: PeerId, request_id: RequestId, request: RPCRequest) { match request { RPCRequest::Hello(hello_message) => { - self.sync + self.message_processor .on_hello_request(peer_id, request_id, hello_message) } RPCRequest::Goodbye(goodbye_reason) => { @@ -117,13 +129,13 @@ impl MessageHandler { "peer" => format!("{:?}", peer_id), "reason" => format!("{:?}", goodbye_reason), ); - self.sync.on_disconnect(peer_id); + self.message_processor.on_disconnect(peer_id); } RPCRequest::BeaconBlocks(request) => self - .sync + .message_processor .on_beacon_blocks_request(peer_id, request_id, request), RPCRequest::RecentBeaconBlocks(request) => self - .sync + .message_processor .on_recent_beacon_blocks_request(peer_id, request_id, request), } } @@ -150,12 +162,13 @@ impl MessageHandler { RPCErrorResponse::Success(response) => { match response { RPCResponse::Hello(hello_message) => { - self.sync.on_hello_response(peer_id, hello_message); + self.message_processor + .on_hello_response(peer_id, hello_message); } RPCResponse::BeaconBlocks(response) => { match self.decode_beacon_blocks(&response) { Ok(beacon_blocks) => { - self.sync.on_beacon_blocks_response( + self.message_processor.on_beacon_blocks_response( peer_id, request_id, beacon_blocks, @@ -170,7 +183,7 @@ impl MessageHandler { RPCResponse::RecentBeaconBlocks(response) => { match self.decode_beacon_blocks(&response) { Ok(beacon_blocks) => { - self.sync.on_recent_beacon_blocks_response( + self.message_processor.on_recent_beacon_blocks_response( peer_id, request_id, beacon_blocks, @@ -194,24 +207,37 @@ impl MessageHandler { } /// Handle RPC messages - fn handle_gossip(&mut self, peer_id: PeerId, gossip_message: PubsubMessage) { + fn handle_gossip(&mut self, id: String, peer_id: PeerId, gossip_message: PubsubMessage) { match gossip_message { PubsubMessage::Block(message) => match self.decode_gossip_block(message) { Ok(block) => { - let _should_forward_on = self.sync.on_block_gossip(peer_id, block); + let should_forward_on = self + .message_processor + .on_block_gossip(peer_id.clone(), block); + // TODO: Apply more sophisticated validation and decoding logic + if should_forward_on { + self.propagate_message(id, peer_id.clone()); + } } Err(e) => { debug!(self.log, "Invalid gossiped beacon block"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e)); } }, PubsubMessage::Attestation(message) => match self.decode_gossip_attestation(message) { - Ok(attestation) => self.sync.on_attestation_gossip(peer_id, attestation), + Ok(attestation) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); + self.message_processor + .on_attestation_gossip(peer_id, attestation); + } Err(e) => { debug!(self.log, "Invalid gossiped attestation"; "peer_id" => format!("{}", peer_id), "Error" => format!("{:?}", e)); } }, PubsubMessage::VoluntaryExit(message) => match self.decode_gossip_exit(message) { Ok(_exit) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle exits debug!(self.log, "Received a voluntary exit"; "peer_id" => format!("{}", peer_id) ); } @@ -222,6 +248,8 @@ impl MessageHandler { PubsubMessage::ProposerSlashing(message) => { match self.decode_gossip_proposer_slashing(message) { Ok(_slashing) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle proposer slashings debug!(self.log, "Received a proposer slashing"; "peer_id" => format!("{}", peer_id) ); } @@ -233,6 +261,8 @@ impl MessageHandler { PubsubMessage::AttesterSlashing(message) => { match self.decode_gossip_attestation_slashing(message) { Ok(_slashing) => { + // TODO: Apply more sophisticated validation and decoding logic + self.propagate_message(id, peer_id.clone()); // TODO: Handle attester slashings debug!(self.log, "Received an attester slashing"; "peer_id" => format!("{}", peer_id) ); } @@ -248,6 +278,21 @@ impl MessageHandler { } } + /// Informs the network service that the message should be forwarded to other peers. + fn propagate_message(&mut self, message_id: String, propagation_source: PeerId) { + self.network_send + .try_send(NetworkMessage::Propagate { + propagation_source, + message_id, + }) + .unwrap_or_else(|_| { + warn!( + self.log, + "Could not send propagation request to the network service" + ) + }); + } + /* Decoding of gossipsub objects from the network. * * The decoding is done in the message handler as it has access to to a `BeaconChain` and can diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index a8b3c74b6a..1357b54951 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -34,13 +34,8 @@ impl Service { // build the network channel let (network_send, network_recv) = mpsc::unbounded_channel::(); // launch message handler thread - let message_handler_log = log.new(o!("Service" => "MessageHandler")); - let message_handler_send = MessageHandler::spawn( - beacon_chain, - network_send.clone(), - executor, - message_handler_log, - )?; + let message_handler_send = + MessageHandler::spawn(beacon_chain, network_send.clone(), executor, log.clone())?; let network_log = log.new(o!("Service" => "Network")); // launch libp2p service @@ -159,12 +154,23 @@ fn network_service( // poll the network channel match network_recv.poll() { Ok(Async::Ready(Some(message))) => match message { - NetworkMessage::Send(peer_id, outgoing_message) => match outgoing_message { - OutgoingMessage::RPC(rpc_event) => { - trace!(log, "Sending RPC Event: {:?}", rpc_event); - libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event); - } - }, + NetworkMessage::RPC(peer_id, rpc_event) => { + trace!(log, "{}", rpc_event); + libp2p_service.lock().swarm.send_rpc(peer_id, rpc_event); + } + NetworkMessage::Propagate { + propagation_source, + message_id, + } => { + trace!(log, "Propagating gossipsub message"; + "propagation_peer" => format!("{:?}", propagation_source), + "message_id" => format!("{}", message_id), + ); + libp2p_service + .lock() + .swarm + .propagate_message(&propagation_source, message_id); + } NetworkMessage::Publish { topics, message } => { debug!(log, "Sending pubsub message"; "topics" => format!("{:?}",topics)); libp2p_service.lock().swarm.publish(&topics, message); @@ -185,7 +191,7 @@ fn network_service( match libp2p_service.lock().poll() { Ok(Async::Ready(Some(event))) => match event { Libp2pEvent::RPC(peer_id, rpc_event) => { - trace!(log, "RPC Event: RPC message received: {:?}", rpc_event); + trace!(log, "{}", rpc_event); message_handler_send .try_send(HandlerMessage::RPC(peer_id, rpc_event)) .map_err(|_| "Failed to send RPC to handler")?; @@ -203,13 +209,14 @@ fn network_service( .map_err(|_| "Failed to send PeerDisconnected to handler")?; } Libp2pEvent::PubsubMessage { - source, message, .. + id, + source, + message, + .. } => { - //TODO: Decide if we need to propagate the topic upwards. (Potentially for - //attestations) message_handler_send - .try_send(HandlerMessage::PubsubMessage(source, message)) - .map_err(|_| " failed to send pubsub message to handler")?; + .try_send(HandlerMessage::PubsubMessage(id, source, message)) + .map_err(|_| "Failed to send pubsub message to handler")?; } }, Ok(Async::Ready(None)) => unreachable!("Stream never ends"), @@ -225,19 +232,16 @@ fn network_service( /// Types of messages that the network service can receive. #[derive(Debug)] pub enum NetworkMessage { - /// Send a message to libp2p service. - //TODO: Define typing for messages across the wire - Send(PeerId, OutgoingMessage), - /// Publish a message to pubsub mechanism. + /// Send an RPC message to the libp2p service. + RPC(PeerId, RPCEvent), + /// Publish a message to gossipsub. Publish { topics: Vec, message: PubsubMessage, }, -} - -/// Type of outgoing messages that can be sent through the network service. -#[derive(Debug)] -pub enum OutgoingMessage { - /// Send an RPC request/response. - RPC(RPCEvent), + /// Propagate a received gossipsub message + Propagate { + propagation_source: PeerId, + message_id: String, + }, } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 9cce6300df..171d0fdf0b 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -1,113 +1,309 @@ -use super::simple_sync::{PeerSyncInfo, FUTURE_SLOT_TOLERANCE}; +//! The `SyncManager` facilities the block syncing logic of lighthouse. The current networking +//! specification provides two methods from which to obtain blocks from peers. The `BeaconBlocks` +//! request and the `RecentBeaconBlocks` request. The former is used to obtain a large number of +//! blocks and the latter allows for searching for blocks given a block-hash. +//! +//! These two RPC methods are designed for two type of syncing. +//! - Long range (batch) sync, when a client is out of date and needs to the latest head. +//! - Parent lookup - when a peer provides us a block whose parent is unknown to us. +//! +//! Both of these syncing strategies are built into the `SyncManager`. +//! +//! +//! Currently the long-range (batch) syncing method functions by opportunistically downloading +//! batches blocks from all peers who know about a chain that we do not. When a new peer connects +//! which has a later head that is greater than `SLOT_IMPORT_TOLERANCE` from our current head slot, +//! the manager's state becomes `Syncing` and begins a batch syncing process with this peer. If +//! further peers connect, this process is run in parallel with those peers, until our head is +//! within `SLOT_IMPORT_TOLERANCE` of all connected peers. +//! +//! Batch Syncing +//! +//! This syncing process start by requesting `MAX_BLOCKS_PER_REQUEST` blocks from a peer with an +//! unknown chain (with a greater slot height) starting from our current head slot. If the earliest +//! block returned is known to us, then the group of blocks returned form part of a known chain, +//! and we process this batch of blocks, before requesting more batches forward and processing +//! those in turn until we reach the peer's chain's head. If the first batch doesn't contain a +//! block we know of, we must iteratively request blocks backwards (until our latest finalized head +//! slot) until we find a common ancestor before we can start processing the blocks. If no common +//! ancestor is found, the peer has a chain which is not part of our finalized head slot and we +//! drop the peer and the downloaded blocks. +//! Once we are fully synced with all known peers, the state of the manager becomes `Regular` which +//! then allows for parent lookups of propagated blocks. +//! +//! A schematic version of this logic with two chain variations looks like the following. +//! +//! |----------------------|---------------------------------| +//! ^finalized head ^current local head ^remotes head +//! +//! +//! An example of the remotes chain diverging before our current head. +//! |---------------------------| +//! ^---------------------------------------------| +//! ^chain diverges |initial batch| ^remotes head +//! +//! In this example, we cannot process the initial batch as it is not on a known chain. We must +//! then backwards sync until we reach a common chain to begin forwarding batch syncing. +//! +//! +//! Parent Lookup +//! +//! When a block with an unknown parent is received and we are in `Regular` sync mode, the block is +//! queued for lookup. A round-robin approach is used to request the parent from the known list of +//! fully sync'd peers. If `PARENT_FAIL_TOLERANCE` attempts at requesting the block fails, we +//! drop the propagated block and downvote the peer that sent it to us. + +use super::simple_sync::{hello_message, NetworkContext, PeerSyncInfo, FUTURE_SLOT_TOLERANCE}; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; -use eth2_libp2p::rpc::RequestId; +use eth2_libp2p::rpc::{RPCRequest, RequestId}; use eth2_libp2p::PeerId; +use futures::prelude::*; use slog::{debug, info, trace, warn, Logger}; +use smallvec::SmallVec; use std::collections::{HashMap, HashSet}; use std::ops::{Add, Sub}; -use std::sync::Arc; +use std::sync::Weak; +use tokio::sync::{mpsc, oneshot}; use types::{BeaconBlock, EthSpec, Hash256, Slot}; -const MAX_BLOCKS_PER_REQUEST: u64 = 10; +/// Blocks are downloaded in batches from peers. This constant specifies how many blocks per batch +/// is requested. Currently the value is small for testing. This will be incremented for +/// production. +const MAX_BLOCKS_PER_REQUEST: u64 = 50; -/// The number of slots that we can import blocks ahead of us, before going into full Sync mode. +/// The number of slots ahead of us that is allowed before requesting a long-range (batch) Sync +/// from a peer. If a peer is within this tolerance (forwards or backwards), it is treated as a +/// fully sync'd peer. const SLOT_IMPORT_TOLERANCE: usize = 10; +/// How many attempts we try to find a parent of a block before we give up trying . const PARENT_FAIL_TOLERANCE: usize = 3; +/// The maximum depth we will search for a parent block. In principle we should have sync'd any +/// canonical chain to its head once the peer connects. A chain should not appear where it's depth +/// is further back than the most recent head slot. const PARENT_DEPTH_TOLERANCE: usize = SLOT_IMPORT_TOLERANCE * 2; +/// The number of empty batches we tolerate before dropping the peer. This prevents endless +/// requests to peers who never return blocks. +const EMPTY_BATCH_TOLERANCE: usize = 100; + +#[derive(Debug)] +/// A message than can be sent to the sync manager thread. +pub enum SyncMessage { + /// A useful peer has been discovered. + AddPeer(PeerId, PeerSyncInfo), + /// A `BeaconBlocks` response has been received. + BeaconBlocksResponse { + peer_id: PeerId, + request_id: RequestId, + beacon_blocks: Vec>, + }, + /// A `RecentBeaconBlocks` response has been received. + RecentBeaconBlocksResponse { + peer_id: PeerId, + request_id: RequestId, + beacon_blocks: Vec>, + }, + /// A block with an unknown parent has been received. + UnknownBlock(PeerId, BeaconBlock), + /// A peer has disconnected. + Disconnect(PeerId), + /// An RPC Error has occurred on a request. + _RPCError(RequestId), +} #[derive(PartialEq)] +/// The current state of a block or batches lookup. enum BlockRequestsState { + /// The object is queued to be downloaded from a peer but has not yet been requested. Queued, + /// The batch or parent has been requested with the `RequestId` and we are awaiting a response. Pending(RequestId), - Complete, + /// The downloaded blocks are ready to be processed by the beacon chain. For a batch process + /// this means we have found a common chain. + ReadyToProcess, + /// A failure has occurred and we will drop and downvote the peer that caused the request. Failed, } +/// The state of batch requests. +enum SyncDirection { + /// The batch has just been initialised and we need to check to see if a backward sync is + /// required on first batch response. + Initial, + /// We are syncing forwards, the next batch should contain higher slot numbers than is + /// predecessor. + Forwards, + /// We are syncing backwards and looking for a common ancestor chain before we can start + /// processing the downloaded blocks. + Backwards, +} + +/// `BlockRequests` keep track of the long-range (batch) sync process per peer. struct BlockRequests { + /// The peer's head slot and the target of this batch download. target_head_slot: Slot, + /// The peer's head root, used to specify which chain of blocks we are downloading from the + /// blocks. target_head_root: Hash256, + /// The blocks that we have currently downloaded from the peer that are yet to be processed. downloaded_blocks: Vec>, + /// The number of blocks successfully processed in this request. + blocks_processed: usize, + /// The number of empty batches we have consecutively received. If a peer returns more than + /// EMPTY_BATCHES_TOLERANCE, they are dropped. + consecutive_empty_batches: usize, + /// The current state of this batch request. state: BlockRequestsState, - /// Specifies whether the current state is syncing forwards or backwards. - forward_sync: bool, + /// Specifies the current direction of this batch request. + sync_direction: SyncDirection, /// The current `start_slot` of the batched block request. current_start_slot: Slot, } +/// Maintains a sequential list of parents to lookup and the lookup's current state. struct ParentRequests { + /// The blocks that have currently been downloaded. downloaded_blocks: Vec>, + /// The number of failed attempts to retrieve a parent block. If too many attempts occur, this + /// lookup is failed and rejected. failed_attempts: usize, - last_submitted_peer: PeerId, // to downvote the submitting peer. + /// The peer who last submitted a block. If the chain ends or fails, this is the peer that is + /// downvoted. + last_submitted_peer: PeerId, + /// The current state of the parent lookup. state: BlockRequestsState, } impl BlockRequests { - // gets the start slot for next batch - // last block slot downloaded plus 1 + /// Gets the next start slot for a batch and transitions the state to a Queued state. fn update_start_slot(&mut self) { - if self.forward_sync { - self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST); - } else { - self.current_start_slot -= Slot::from(MAX_BLOCKS_PER_REQUEST); + match self.sync_direction { + SyncDirection::Initial | SyncDirection::Forwards => { + self.current_start_slot += Slot::from(MAX_BLOCKS_PER_REQUEST); + } + SyncDirection::Backwards => { + self.current_start_slot -= Slot::from(MAX_BLOCKS_PER_REQUEST); + } } self.state = BlockRequestsState::Queued; } } #[derive(PartialEq, Debug, Clone)] +/// The current state of the `ImportManager`. enum ManagerState { + /// The manager is performing a long-range (batch) sync. In this mode, parent lookups are + /// disabled. Syncing, + /// The manager is up to date with all known peers and is connected to at least one + /// fully-syncing peer. In this state, parent lookups are enabled. Regular, + /// No useful peers are connected. Long-range sync's cannot proceed and we have no useful + /// peers to download parents for. More peers need to be connected before we can proceed. Stalled, } -pub(crate) enum ImportManagerOutcome { - Idle, - RequestBlocks { - peer_id: PeerId, - request_id: RequestId, - request: BeaconBlocksRequest, - }, - /// Updates information with peer via requesting another HELLO handshake. - Hello(PeerId), - RecentRequest(PeerId, RecentBeaconBlocksRequest), - DownvotePeer(PeerId), -} - -pub struct ImportManager { - /// A reference to the underlying beacon chain. - chain: Arc>, +/// The primary object for handling and driving all the current syncing logic. It maintains the +/// current state of the syncing process, the number of useful peers, downloaded blocks and +/// controls the logic behind both the long-range (batch) sync and the on-going potential parent +/// look-up of blocks. +pub struct SyncManager { + /// A weak reference to the underlying beacon chain. + chain: Weak>, + /// The current state of the import manager. state: ManagerState, + /// A receiving channel sent by the message processor thread. + input_channel: mpsc::UnboundedReceiver>, + /// A network context to contact the network service. + network: NetworkContext, + /// A collection of `BlockRequest` per peer that is currently being downloaded. Used in the + /// long-range (batch) sync process. import_queue: HashMap>, - parent_queue: Vec>, + /// A collection of parent block lookups. + parent_queue: SmallVec<[ParentRequests; 3]>, + /// The collection of known, connected, fully-sync'd peers. full_peers: HashSet, + /// The current request Id. This is used to keep track of responses to various outbound + /// requests. This is an internal accounting mechanism, request id's are never sent to any + /// peers. current_req_id: usize, + /// The logger for the import manager. log: Logger, } -impl ImportManager { - pub fn new(beacon_chain: Arc>, log: &slog::Logger) -> Self { - ImportManager { - chain: beacon_chain.clone(), - state: ManagerState::Regular, - import_queue: HashMap::new(), - parent_queue: Vec::new(), - full_peers: HashSet::new(), - current_req_id: 0, - log: log.clone(), - } - } +/// Spawns a new `SyncManager` thread which has a weak reference to underlying beacon +/// chain. This allows the chain to be +/// dropped during the syncing process which will gracefully end the `SyncManager`. +pub fn spawn( + executor: &tokio::runtime::TaskExecutor, + beacon_chain: Weak>, + network: NetworkContext, + log: slog::Logger, +) -> ( + mpsc::UnboundedSender>, + oneshot::Sender<()>, +) { + // generate the exit channel + let (sync_exit, exit_rx) = tokio::sync::oneshot::channel(); + // generate the message channel + let (sync_send, sync_recv) = mpsc::unbounded_channel::>(); + // create an instance of the SyncManager + let sync_manager = SyncManager { + chain: beacon_chain, + state: ManagerState::Stalled, + input_channel: sync_recv, + network, + import_queue: HashMap::new(), + parent_queue: SmallVec::new(), + full_peers: HashSet::new(), + current_req_id: 0, + log: log.clone(), + }; + + // spawn the sync manager thread + debug!(log, "Sync Manager started"); + executor.spawn( + sync_manager + .select(exit_rx.then(|_| Ok(()))) + .then(move |_| { + info!(log.clone(), "Sync Manager shutdown"); + Ok(()) + }), + ); + (sync_send, sync_exit) +} + +impl SyncManager { + /* Input Handling Functions */ + + /// A peer has connected which has blocks that are unknown to us. + /// + /// This function handles the logic associated with the connection of a new peer. If the peer + /// is sufficiently ahead of our current head, a long-range (batch) sync is started and + /// batches of blocks are queued to download from the peer. Batched blocks begin at our + /// current head. If the resulting downloaded blocks are part of our current chain, we + /// continue with a forward sync. If not, we download blocks (in batches) backwards until we + /// reach a common ancestor. Batches are then processed and downloaded sequentially forwards. + /// + /// If the peer is within the `SLOT_IMPORT_TOLERANCE`, then it's head is sufficiently close to + /// ours that we consider it fully sync'd with respect to our current chain. pub fn add_peer(&mut self, peer_id: PeerId, remote: PeerSyncInfo) { - // TODO: Improve comments. - // initially try to download blocks from our current head - // then backwards search all the way back to our finalized epoch until we match on a chain - // has to be done sequentially to find next slot to start the batch from + // ensure the beacon chain still exists + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + warn!(self.log, + "Beacon chain dropped. Peer not considered for sync"; + "peer_id" => format!("{:?}", peer_id)); + return; + } + }; - let local = PeerSyncInfo::from(&self.chain); + let local = PeerSyncInfo::from(&chain); - // If a peer is within SLOT_IMPORT_TOLERANCE from our head slot, ignore a batch sync + // If a peer is within SLOT_IMPORT_TOLERANCE from our head slot, ignore a batch sync, + // consider it a fully-sync'd peer. if remote.head_slot.sub(local.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { trace!(self.log, "Ignoring full sync with peer"; "peer" => format!("{:?}", peer_id), @@ -116,34 +312,64 @@ impl ImportManager { ); // remove the peer from the queue if it exists self.import_queue.remove(&peer_id); + self.add_full_peer(peer_id); + // return; } + // Check if the peer is significantly behind us. If within `SLOT_IMPORT_TOLERANCE` + // treat them as a fully synced peer. If not, ignore them in the sync process + if local.head_slot.sub(remote.head_slot).as_usize() < SLOT_IMPORT_TOLERANCE { + self.add_full_peer(peer_id.clone()); + } else { + debug!( + self.log, + "Out of sync peer connected"; + "peer" => format!("{:?}", peer_id), + ); + return; + } + + // Check if we are already downloading blocks from this peer, if so update, if not set up + // a new request structure if let Some(block_requests) = self.import_queue.get_mut(&peer_id) { // update the target head slot if remote.head_slot > block_requests.target_head_slot { block_requests.target_head_slot = remote.head_slot; } } else { + // not already downloading blocks from this peer let block_requests = BlockRequests { target_head_slot: remote.head_slot, // this should be larger than the current head. It is checked in the SyncManager before add_peer is called target_head_root: remote.head_root, + consecutive_empty_batches: 0, downloaded_blocks: Vec::new(), + blocks_processed: 0, state: BlockRequestsState::Queued, - forward_sync: true, - current_start_slot: self.chain.best_slot(), + sync_direction: SyncDirection::Initial, + current_start_slot: chain.best_slot(), }; self.import_queue.insert(peer_id, block_requests); } } + /// A `BeaconBlocks` request has received a response. This function process the response. pub fn beacon_blocks_response( &mut self, peer_id: PeerId, request_id: RequestId, mut blocks: Vec>, ) { - // find the request + // ensure the underlying chain still exists + let chain = match self.chain.upgrade() { + Some(chain) => chain, + None => { + trace!(self.log, "Chain dropped. Sync terminating"); + return; + } + }; + + // find the request associated with this response let block_requests = match self .import_queue .get_mut(&peer_id) @@ -167,10 +393,25 @@ impl ImportManager { if blocks.is_empty() { debug!(self.log, "BeaconBlocks response was empty"; "request_id" => request_id); - block_requests.update_start_slot(); + block_requests.consecutive_empty_batches += 1; + if block_requests.consecutive_empty_batches >= EMPTY_BATCH_TOLERANCE { + warn!(self.log, "Peer returned too many empty block batches"; + "peer" => format!("{:?}", peer_id)); + block_requests.state = BlockRequestsState::Failed; + } else if block_requests.current_start_slot + MAX_BLOCKS_PER_REQUEST + >= block_requests.target_head_slot + { + warn!(self.log, "Peer did not return blocks it claimed to possess"; + "peer" => format!("{:?}", peer_id)); + block_requests.state = BlockRequestsState::Failed; + } else { + block_requests.update_start_slot(); + } return; } + block_requests.consecutive_empty_batches = 0; + // verify the range of received blocks // Note that the order of blocks is verified in block processing let last_sent_slot = blocks[blocks.len() - 1].slot; @@ -180,90 +421,96 @@ impl ImportManager { .add(MAX_BLOCKS_PER_REQUEST) < last_sent_slot { - //TODO: Downvote peer - add a reason to failed - dbg!(&blocks); warn!(self.log, "BeaconBlocks response returned out of range blocks"; "request_id" => request_id, "response_initial_slot" => blocks[0].slot, "requested_initial_slot" => block_requests.current_start_slot); + downvote_peer(&mut self.network, &self.log, peer_id); // consider this sync failed block_requests.state = BlockRequestsState::Failed; return; } // Determine if more blocks need to be downloaded. There are a few cases: - // - We have downloaded a batch from our head_slot, which has not reached the remotes head - // (target head). Therefore we need to download another sequential batch. - // - The latest batch includes blocks that greater than or equal to the target_head slot, - // which means we have caught up to their head. We then check to see if the first - // block downloaded matches our head. If so, we are on the same chain and can process - // the blocks. If not we need to sync back further until we are on the same chain. So - // request more blocks. - // - We are syncing backwards (from our head slot) and need to check if we are on the same - // chain. If so, process the blocks, if not, request more blocks all the way up to - // our last finalized slot. + // - We are in initial sync mode - We have requested blocks and need to determine if this + // is part of a known chain to determine the whether to start syncing backwards or continue + // syncing forwards. + // - We are syncing backwards and need to verify if we have found a common ancestor in + // order to start processing the downloaded blocks. + // - We are syncing forwards. We mark this as complete and check if any further blocks are + // required to download when processing the batch. - if block_requests.forward_sync { - // append blocks if syncing forward - block_requests.downloaded_blocks.append(&mut blocks); - } else { - // prepend blocks if syncing backwards - block_requests.downloaded_blocks.splice(..0, blocks); - } + match block_requests.sync_direction { + SyncDirection::Initial => { + block_requests.downloaded_blocks.append(&mut blocks); - // does the batch contain the target_head_slot - let last_element_index = block_requests.downloaded_blocks.len() - 1; - if block_requests.downloaded_blocks[last_element_index].slot - >= block_requests.target_head_slot - || !block_requests.forward_sync - { - // if the batch is on our chain, this is complete and we can then process. - // Otherwise start backwards syncing until we reach a common chain. - let earliest_slot = block_requests.downloaded_blocks[0].slot; - //TODO: Decide which is faster. Reading block from db and comparing or calculating - //the hash tree root and comparing. - if Some(block_requests.downloaded_blocks[0].canonical_root()) - == root_at_slot(&self.chain, earliest_slot) - { - block_requests.state = BlockRequestsState::Complete; - return; + // this batch is the first batch downloaded. Check if we can process or if we need + // to backwards search. + + //TODO: Decide which is faster. Reading block from db and comparing or calculating + //the hash tree root and comparing. + let earliest_slot = block_requests.downloaded_blocks[0].slot; + if Some(block_requests.downloaded_blocks[0].canonical_root()) + == chain.root_at_slot(earliest_slot) + { + // we have a common head, start processing and begin a forwards sync + block_requests.sync_direction = SyncDirection::Forwards; + block_requests.state = BlockRequestsState::ReadyToProcess; + return; + } + // no common head, begin a backwards search + block_requests.sync_direction = SyncDirection::Backwards; + block_requests.current_start_slot = + std::cmp::min(chain.best_slot(), block_requests.downloaded_blocks[0].slot); + block_requests.update_start_slot(); } - - // not on the same chain, request blocks backwards - let state = &self.chain.head().beacon_state; - let local_finalized_slot = state - .finalized_checkpoint - .epoch - .start_slot(T::EthSpec::slots_per_epoch()); - - // check that the request hasn't failed by having no common chain - if local_finalized_slot >= block_requests.current_start_slot { - warn!(self.log, "Peer returned an unknown chain."; "request_id" => request_id); - block_requests.state = BlockRequestsState::Failed; - return; + SyncDirection::Forwards => { + // continue processing all blocks forwards, verify the end in the processing + block_requests.downloaded_blocks.append(&mut blocks); + block_requests.state = BlockRequestsState::ReadyToProcess; } + SyncDirection::Backwards => { + block_requests.downloaded_blocks.splice(..0, blocks); - // if this is a forward sync, then we have reached the head without a common chain - // and we need to start syncing backwards. - if block_requests.forward_sync { - // Start a backwards sync by requesting earlier blocks - block_requests.forward_sync = false; - block_requests.current_start_slot = std::cmp::min( - self.chain.best_slot(), - block_requests.downloaded_blocks[0].slot, - ); + // verify the request hasn't failed by having no common ancestor chain + // get our local finalized_slot + let local_finalized_slot = { + let state = &chain.head().beacon_state; + state + .finalized_checkpoint + .epoch + .start_slot(T::EthSpec::slots_per_epoch()) + }; + + if local_finalized_slot >= block_requests.current_start_slot { + warn!(self.log, "Peer returned an unknown chain."; "request_id" => request_id); + block_requests.state = BlockRequestsState::Failed; + return; + } + + // check if we have reached a common chain ancestor + let earliest_slot = block_requests.downloaded_blocks[0].slot; + if Some(block_requests.downloaded_blocks[0].canonical_root()) + == chain.root_at_slot(earliest_slot) + { + // we have a common head, start processing and begin a forwards sync + block_requests.sync_direction = SyncDirection::Forwards; + block_requests.state = BlockRequestsState::ReadyToProcess; + return; + } + + // no common chain, haven't passed last_finalized_head, so continue backwards + // search + block_requests.update_start_slot(); } } - - // update the start slot and re-queue the batch - block_requests.update_start_slot(); } pub fn recent_blocks_response( &mut self, peer_id: PeerId, request_id: RequestId, - blocks: Vec>, + mut blocks: Vec>, ) { // find the request let parent_request = match self @@ -298,32 +545,18 @@ impl ImportManager { return; } + // add the block to response + parent_request + .downloaded_blocks + .push(blocks.pop().expect("must exist")); + // queue for processing - parent_request.state = BlockRequestsState::Complete; + parent_request.state = BlockRequestsState::ReadyToProcess; } - pub fn _inject_error(_peer_id: PeerId, _id: RequestId) { - //TODO: Remove block state from pending - } - - pub fn peer_disconnect(&mut self, peer_id: &PeerId) { - self.import_queue.remove(peer_id); - self.full_peers.remove(peer_id); - self.update_state(); - } - - pub fn add_full_peer(&mut self, peer_id: PeerId) { - debug!( - self.log, "Fully synced peer added"; - "peer" => format!("{:?}", peer_id), - ); - self.full_peers.insert(peer_id); - self.update_state(); - } - - pub fn add_unknown_block(&mut self, block: BeaconBlock, peer_id: PeerId) { + fn add_unknown_block(&mut self, peer_id: PeerId, block: BeaconBlock) { // if we are not in regular sync mode, ignore this block - if let ManagerState::Regular = self.state { + if self.state != ManagerState::Regular { return; } @@ -350,38 +583,28 @@ impl ImportManager { self.parent_queue.push(req); } - pub(crate) fn poll(&mut self) -> ImportManagerOutcome { - loop { - // update the state of the manager - self.update_state(); - - // process potential block requests - if let Some(outcome) = self.process_potential_block_requests() { - return outcome; - } - - // process any complete long-range batches - if let Some(outcome) = self.process_complete_batches() { - return outcome; - } - - // process any parent block lookup-requests - if let Some(outcome) = self.process_parent_requests() { - return outcome; - } - - // process any complete parent lookups - let (re_run, outcome) = self.process_complete_parent_requests(); - if let Some(outcome) = outcome { - return outcome; - } else if !re_run { - break; - } - } - - return ImportManagerOutcome::Idle; + fn inject_error(&mut self, _id: RequestId) { + //TODO: Remove block state from pending } + fn peer_disconnect(&mut self, peer_id: &PeerId) { + self.import_queue.remove(peer_id); + self.full_peers.remove(peer_id); + self.update_state(); + } + + fn add_full_peer(&mut self, peer_id: PeerId) { + debug!( + self.log, "Fully synced peer added"; + "peer" => format!("{:?}", peer_id), + ); + self.full_peers.insert(peer_id); + } + + /* Processing State Functions */ + // These functions are called in the main poll function to transition the state of the sync + // manager + fn update_state(&mut self) { let previous_state = self.state.clone(); self.state = { @@ -401,20 +624,22 @@ impl ImportManager { } } - fn process_potential_block_requests(&mut self) -> Option { + fn process_potential_block_requests(&mut self) { // check if an outbound request is required // Managing a fixed number of outbound requests is maintained at the RPC protocol libp2p - // layer and not needed here. - // If any in queued state we submit a request. + // layer and not needed here. Therefore we create many outbound requests and let the RPC + // handle the number of simultaneous requests. Request all queued objects. // remove any failed batches let debug_log = &self.log; + let full_peer_ref = &mut self.full_peers; self.import_queue.retain(|peer_id, block_request| { if let BlockRequestsState::Failed = block_request.state { debug!(debug_log, "Block import from peer failed"; "peer_id" => format!("{:?}", peer_id), - "downloaded_blocks" => block_request.downloaded_blocks.len() + "downloaded_blocks" => block_request.blocks_processed ); + full_peer_ref.remove(peer_id); false } else { true @@ -422,71 +647,101 @@ impl ImportManager { }); // process queued block requests - for (peer_id, block_requests) in self - .import_queue - .iter_mut() - .find(|(_peer_id, req)| req.state == BlockRequestsState::Queued) - { - let request_id = self.current_req_id; - block_requests.state = BlockRequestsState::Pending(request_id); - self.current_req_id += 1; + for (peer_id, block_requests) in self.import_queue.iter_mut() { + { + if block_requests.state == BlockRequestsState::Queued { + let request_id = self.current_req_id; + block_requests.state = BlockRequestsState::Pending(request_id); + self.current_req_id += 1; - let request = BeaconBlocksRequest { - head_block_root: block_requests.target_head_root, - start_slot: block_requests.current_start_slot.as_u64(), - count: MAX_BLOCKS_PER_REQUEST, - step: 0, - }; - return Some(ImportManagerOutcome::RequestBlocks { - peer_id: peer_id.clone(), - request, - request_id, - }); - } - - None - } - - fn process_complete_batches(&mut self) -> Option { - let completed_batches = self - .import_queue - .iter() - .filter(|(_peer, block_requests)| block_requests.state == BlockRequestsState::Complete) - .map(|(peer, _)| peer) - .cloned() - .collect::>(); - for peer_id in completed_batches { - let block_requests = self.import_queue.remove(&peer_id).expect("key exists"); - match self.process_blocks(block_requests.downloaded_blocks.clone()) { - Ok(()) => { - //TODO: Verify it's impossible to have empty downloaded_blocks - let last_element = block_requests.downloaded_blocks.len() - 1; - debug!(self.log, "Blocks processed successfully"; - "peer" => format!("{:?}", peer_id), - "start_slot" => block_requests.downloaded_blocks[0].slot, - "end_slot" => block_requests.downloaded_blocks[last_element].slot, - "no_blocks" => last_element + 1, + let request = BeaconBlocksRequest { + head_block_root: block_requests.target_head_root, + start_slot: block_requests.current_start_slot.as_u64(), + count: MAX_BLOCKS_PER_REQUEST, + step: 0, + }; + request_blocks( + &mut self.network, + &self.log, + peer_id.clone(), + request_id, + request, ); - // Re-HELLO to ensure we are up to the latest head - return Some(ImportManagerOutcome::Hello(peer_id)); - } - Err(e) => { - let last_element = block_requests.downloaded_blocks.len() - 1; - warn!(self.log, "Block processing failed"; - "peer" => format!("{:?}", peer_id), - "start_slot" => block_requests.downloaded_blocks[0].slot, - "end_slot" => block_requests.downloaded_blocks[last_element].slot, - "no_blocks" => last_element + 1, - "error" => format!("{:?}", e), - ); - return Some(ImportManagerOutcome::DownvotePeer(peer_id)); } } } - None } - fn process_parent_requests(&mut self) -> Option { + fn process_complete_batches(&mut self) -> bool { + // This function can queue extra blocks and the main poll loop will need to be re-executed + // to process these. This flag indicates that the main poll loop has to continue. + let mut re_run_poll = false; + + // create reference variables to be moved into subsequent closure + let chain_ref = self.chain.clone(); + let log_ref = &self.log; + let network_ref = &mut self.network; + + self.import_queue.retain(|peer_id, block_requests| { + if block_requests.state == BlockRequestsState::ReadyToProcess { + let downloaded_blocks = + std::mem::replace(&mut block_requests.downloaded_blocks, Vec::new()); + let last_element = downloaded_blocks.len() - 1; + let start_slot = downloaded_blocks[0].slot; + let end_slot = downloaded_blocks[last_element].slot; + + match process_blocks(chain_ref.clone(), downloaded_blocks, log_ref) { + Ok(()) => { + debug!(log_ref, "Blocks processed successfully"; + "peer" => format!("{:?}", peer_id), + "start_slot" => start_slot, + "end_slot" => end_slot, + "no_blocks" => last_element + 1, + ); + block_requests.blocks_processed += last_element + 1; + + // check if the batch is complete, by verifying if we have reached the + // target head + if end_slot >= block_requests.target_head_slot { + // Completed, re-hello the peer to ensure we are up to the latest head + hello_peer(network_ref, log_ref, chain_ref.clone(), peer_id.clone()); + // remove the request + false + } else { + // have not reached the end, queue another batch + block_requests.update_start_slot(); + re_run_poll = true; + // keep the batch + true + } + } + Err(e) => { + warn!(log_ref, "Block processing failed"; + "peer" => format!("{:?}", peer_id), + "start_slot" => start_slot, + "end_slot" => end_slot, + "no_blocks" => last_element + 1, + "error" => format!("{:?}", e), + ); + downvote_peer(network_ref, log_ref, peer_id.clone()); + false + } + } + } else { + // not ready to process + true + } + }); + + re_run_poll + } + + fn process_parent_requests(&mut self) { + // check to make sure there are peers to search for the parent from + if self.full_peers.is_empty() { + return; + } + // remove any failed requests let debug_log = &self.log; self.parent_queue.retain(|parent_request| { @@ -501,11 +756,6 @@ impl ImportManager { } }); - // check to make sure there are peers to search for the parent from - if self.full_peers.is_empty() { - return None; - } - // check if parents need to be searched for for parent_request in self.parent_queue.iter_mut() { if parent_request.failed_attempts >= PARENT_FAIL_TOLERANCE { @@ -518,34 +768,38 @@ impl ImportManager { continue; } - parent_request.state = BlockRequestsState::Pending(self.current_req_id); + let request_id = self.current_req_id; + parent_request.state = BlockRequestsState::Pending(request_id); self.current_req_id += 1; let last_element_index = parent_request.downloaded_blocks.len() - 1; let parent_hash = parent_request.downloaded_blocks[last_element_index].parent_root; - let req = RecentBeaconBlocksRequest { + let request = RecentBeaconBlocksRequest { block_roots: vec![parent_hash], }; // select a random fully synced peer to attempt to download the parent block let peer_id = self.full_peers.iter().next().expect("List is not empty"); - return Some(ImportManagerOutcome::RecentRequest(peer_id.clone(), req)); + recent_blocks_request( + &mut self.network, + &self.log, + peer_id.clone(), + request_id, + request, + ); } } - - None } - fn process_complete_parent_requests(&mut self) -> (bool, Option) { - // flag to determine if there is more process to drive or if the manager can be switched to - // an idle state - let mut re_run = false; + fn process_complete_parent_requests(&mut self) -> bool { + // returned value indicating whether the manager can be switched to idle or not + let mut re_run_poll = false; // Find any parent_requests ready to be processed for completed_request in self .parent_queue .iter_mut() - .filter(|req| req.state == BlockRequestsState::Complete) + .filter(|req| req.state == BlockRequestsState::ReadyToProcess) { // verify the last added block is the parent of the last requested block let last_index = completed_request.downloaded_blocks.len() - 1; @@ -563,7 +817,8 @@ impl ImportManager { "received_block" => format!("{}", block_hash), "expected_parent" => format!("{}", expected_hash), ); - return (true, Some(ImportManagerOutcome::DownvotePeer(peer))); + re_run_poll = true; + downvote_peer(&mut self.network, &self.log, peer); } // try and process the list of blocks up to the requested block @@ -572,72 +827,158 @@ impl ImportManager { .downloaded_blocks .pop() .expect("Block must exist exist"); - match self.chain.process_block(block.clone()) { - Ok(BlockProcessingOutcome::ParentUnknown { parent: _ }) => { - // need to keep looking for parents - completed_request.downloaded_blocks.push(block); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - break; - } - Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {} - Ok(outcome) => { - // it's a future slot or an invalid block, remove it and try again - completed_request.failed_attempts += 1; - trace!( - self.log, "Invalid parent block"; - "outcome" => format!("{:?}", outcome), - "peer" => format!("{:?}", completed_request.last_submitted_peer), - ); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - return ( - re_run, - Some(ImportManagerOutcome::DownvotePeer( + + // check if the chain exists + if let Some(chain) = self.chain.upgrade() { + match chain.process_block(block.clone()) { + Ok(BlockProcessingOutcome::ParentUnknown { parent: _ }) => { + // need to keep looking for parents + completed_request.downloaded_blocks.push(block); + completed_request.state = BlockRequestsState::Queued; + re_run_poll = true; + break; + } + Ok(BlockProcessingOutcome::Processed { block_root: _ }) => {} + Ok(outcome) => { + // it's a future slot or an invalid block, remove it and try again + completed_request.failed_attempts += 1; + trace!( + self.log, "Invalid parent block"; + "outcome" => format!("{:?}", outcome), + "peer" => format!("{:?}", completed_request.last_submitted_peer), + ); + completed_request.state = BlockRequestsState::Queued; + re_run_poll = true; + downvote_peer( + &mut self.network, + &self.log, completed_request.last_submitted_peer.clone(), - )), - ); - } - Err(e) => { - completed_request.failed_attempts += 1; - warn!( - self.log, "Parent processing error"; - "error" => format!("{:?}", e) - ); - completed_request.state = BlockRequestsState::Queued; - re_run = true; - return ( - re_run, - Some(ImportManagerOutcome::DownvotePeer( + ); + return re_run_poll; + } + Err(e) => { + completed_request.failed_attempts += 1; + warn!( + self.log, "Parent processing error"; + "error" => format!("{:?}", e) + ); + completed_request.state = BlockRequestsState::Queued; + re_run_poll = true; + downvote_peer( + &mut self.network, + &self.log, completed_request.last_submitted_peer.clone(), - )), - ); + ); + return re_run_poll; + } } + } else { + // chain doesn't exist - clear the event queue and return + return false; } } } - // remove any full completed and processed parent chains + // remove any fully processed parent chains self.parent_queue.retain(|req| { - if req.state == BlockRequestsState::Complete { + if req.state == BlockRequestsState::ReadyToProcess { false } else { true } }); - (re_run, None) + re_run_poll } +} - fn process_blocks(&mut self, blocks: Vec>) -> Result<(), String> { - for block in blocks { - let processing_result = self.chain.process_block(block.clone()); +/* Network Context Helper Functions */ + +fn hello_peer( + network: &mut NetworkContext, + log: &slog::Logger, + chain: Weak>, + peer_id: PeerId, +) { + trace!( + log, + "RPC Request"; + "method" => "HELLO", + "peer" => format!("{:?}", peer_id) + ); + if let Some(chain) = chain.upgrade() { + network.send_rpc_request(None, peer_id, RPCRequest::Hello(hello_message(&chain))); + } +} + +fn request_blocks( + network: &mut NetworkContext, + log: &slog::Logger, + peer_id: PeerId, + request_id: RequestId, + request: BeaconBlocksRequest, +) { + trace!( + log, + "RPC Request"; + "method" => "BeaconBlocks", + "id" => request_id, + "count" => request.count, + "peer" => format!("{:?}", peer_id) + ); + network.send_rpc_request( + Some(request_id), + peer_id.clone(), + RPCRequest::BeaconBlocks(request), + ); +} + +fn recent_blocks_request( + network: &mut NetworkContext, + log: &slog::Logger, + peer_id: PeerId, + request_id: RequestId, + request: RecentBeaconBlocksRequest, +) { + trace!( + log, + "RPC Request"; + "method" => "RecentBeaconBlocks", + "count" => request.block_roots.len(), + "peer" => format!("{:?}", peer_id) + ); + network.send_rpc_request( + Some(request_id), + peer_id.clone(), + RPCRequest::RecentBeaconBlocks(request), + ); +} + +fn downvote_peer(network: &mut NetworkContext, log: &slog::Logger, peer_id: PeerId) { + trace!( + log, + "Peer downvoted"; + "peer" => format!("{:?}", peer_id) + ); + // TODO: Implement reputation + network.disconnect(peer_id.clone(), GoodbyeReason::Fault); +} + +// Helper function to process blocks which only consumes the chain and blocks to process +fn process_blocks( + weak_chain: Weak>, + blocks: Vec>, + log: &Logger, +) -> Result<(), String> { + for block in blocks { + if let Some(chain) = weak_chain.upgrade() { + let processing_result = chain.process_block(block.clone()); if let Ok(outcome) = processing_result { match outcome { BlockProcessingOutcome::Processed { block_root } => { // The block was valid and we processed it successfully. trace!( - self.log, "Imported block from network"; + log, "Imported block from network"; "slot" => block.slot, "block_root" => format!("{}", block_root), ); @@ -645,7 +986,7 @@ impl ImportManager { BlockProcessingOutcome::ParentUnknown { parent } => { // blocks should be sequential and all parents should exist trace!( - self.log, "ParentBlockUnknown"; + log, "Parent block is unknown"; "parent_root" => format!("{}", parent), "baby_block_slot" => block.slot, ); @@ -654,6 +995,13 @@ impl ImportManager { block.slot )); } + BlockProcessingOutcome::BlockIsAlreadyKnown => { + // this block is already known to us, move to the next + debug!( + log, "Imported a block that is already known"; + "block_slot" => block.slot, + ); + } BlockProcessingOutcome::FutureSlot { present_slot, block_slot, @@ -661,7 +1009,7 @@ impl ImportManager { if present_slot + FUTURE_SLOT_TOLERANCE >= block_slot { // The block is too far in the future, drop it. trace!( - self.log, "FutureBlock"; + log, "Block is ahead of our slot clock"; "msg" => "block for future slot rejected, check your time", "present_slot" => present_slot, "block_slot" => block_slot, @@ -674,8 +1022,7 @@ impl ImportManager { } else { // The block is in the future, but not too far. trace!( - self.log, "QueuedFutureBlock"; - "msg" => "queuing future block, check your time", + log, "Block is slightly ahead of our slot clock, ignoring."; "present_slot" => present_slot, "block_slot" => block_slot, "FUTURE_SLOT_TOLERANCE" => FUTURE_SLOT_TOLERANCE, @@ -684,20 +1031,20 @@ impl ImportManager { } BlockProcessingOutcome::WouldRevertFinalizedSlot { .. } => { trace!( - self.log, "Finalized or earlier block processed"; + log, "Finalized or earlier block processed"; "outcome" => format!("{:?}", outcome), ); // block reached our finalized slot or was earlier, move to the next block } BlockProcessingOutcome::GenesisBlock => { trace!( - self.log, "Genesis block was processed"; + log, "Genesis block was processed"; "outcome" => format!("{:?}", outcome), ); } _ => { - trace!( - self.log, "InvalidBlock"; + warn!( + log, "Invalid block received"; "msg" => "peer sent invalid block", "outcome" => format!("{:?}", outcome), ); @@ -705,8 +1052,8 @@ impl ImportManager { } } } else { - trace!( - self.log, "BlockProcessingFailure"; + warn!( + log, "BlockProcessingFailure"; "msg" => "unexpected condition in processing block.", "outcome" => format!("{:?}", processing_result) ); @@ -715,17 +1062,96 @@ impl ImportManager { processing_result )); } + } else { + return Ok(()); // terminate early due to dropped beacon chain } - Ok(()) } + + Ok(()) } -fn root_at_slot( - chain: &Arc>, - target_slot: Slot, -) -> Option { - chain - .rev_iter_block_roots() - .find(|(_root, slot)| *slot == target_slot) - .map(|(root, _slot)| root) +impl Future for SyncManager { + type Item = (); + type Error = String; + + fn poll(&mut self) -> Result, Self::Error> { + // process any inbound messages + loop { + match self.input_channel.poll() { + Ok(Async::Ready(Some(message))) => match message { + SyncMessage::AddPeer(peer_id, info) => { + self.add_peer(peer_id, info); + } + SyncMessage::BeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + } => { + self.beacon_blocks_response(peer_id, request_id, beacon_blocks); + } + SyncMessage::RecentBeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + } => { + self.recent_blocks_response(peer_id, request_id, beacon_blocks); + } + SyncMessage::UnknownBlock(peer_id, block) => { + self.add_unknown_block(peer_id, block); + } + SyncMessage::Disconnect(peer_id) => { + self.peer_disconnect(&peer_id); + } + SyncMessage::_RPCError(request_id) => { + self.inject_error(request_id); + } + }, + Ok(Async::NotReady) => break, + Ok(Async::Ready(None)) => { + return Err("Sync manager channel closed".into()); + } + Err(e) => { + return Err(format!("Sync Manager channel error: {:?}", e)); + } + } + } + + loop { + //TODO: Optimize the lookups. Potentially keep state of whether each of these functions + //need to be called. + let mut re_run = false; + + // only process batch requests if there are any + if !self.import_queue.is_empty() { + // process potential block requests + self.process_potential_block_requests(); + + // process any complete long-range batches + re_run = re_run || self.process_complete_batches(); + } + + // only process parent objects if we are in regular sync + if !self.parent_queue.is_empty() { + // process any parent block lookup-requests + self.process_parent_requests(); + + // process any complete parent lookups + re_run = re_run || self.process_complete_parent_requests(); + } + + // Shutdown the thread if the chain has termined + if let None = self.chain.upgrade() { + return Ok(Async::Ready(())); + } + + if !re_run { + break; + } + } + + // update the state of the manager + self.update_state(); + + return Ok(Async::NotReady); + } } diff --git a/beacon_node/network/src/sync/mod.rs b/beacon_node/network/src/sync/mod.rs index b26d78c147..58ec386aac 100644 --- a/beacon_node/network/src/sync/mod.rs +++ b/beacon_node/network/src/sync/mod.rs @@ -4,7 +4,7 @@ mod manager; /// Stores the various syncing methods for the beacon chain. mod simple_sync; -pub use simple_sync::SimpleSync; +pub use simple_sync::MessageProcessor; /// Currently implemented sync methods. pub enum SyncMethod { diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 6745ceb628..c54c481c73 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -1,23 +1,23 @@ -use super::manager::{ImportManager, ImportManagerOutcome}; -use crate::service::{NetworkMessage, OutgoingMessage}; +use super::manager::SyncMessage; +use crate::service::NetworkMessage; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessingOutcome}; use eth2_libp2p::rpc::methods::*; use eth2_libp2p::rpc::{RPCEvent, RPCRequest, RPCResponse, RequestId}; use eth2_libp2p::PeerId; use slog::{debug, info, o, trace, warn}; use ssz::Encode; -use std::ops::Sub; use std::sync::Arc; use store::Store; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, oneshot}; use types::{Attestation, BeaconBlock, Epoch, EthSpec, Hash256, Slot}; +//TODO: Put a maximum limit on the number of block that can be requested. +//TODO: Rate limit requests + /// If a block is more than `FUTURE_SLOT_TOLERANCE` slots ahead of our slot clock, we drop it. /// Otherwise we queue it. pub(crate) const FUTURE_SLOT_TOLERANCE: u64 = 1; -/// The number of slots behind our head that we still treat a peer as a fully synced peer. -const FULL_PEER_TOLERANCE: u64 = 10; const SHOULD_FORWARD_GOSSIP_BLOCK: bool = true; const SHOULD_NOT_FORWARD_GOSSIP_BLOCK: bool = false; @@ -49,45 +49,63 @@ impl From<&Arc>> for PeerSyncInfo { } } -/// The current syncing state. -#[derive(PartialEq)] -pub enum SyncState { - _Idle, - _Downloading, - _Stopped, -} - -/// Simple Syncing protocol. -pub struct SimpleSync { +/// Processes validated messages from the network. It relays necessary data to the syncing thread +/// and processes blocks from the pubsub network. +pub struct MessageProcessor { /// A reference to the underlying beacon chain. chain: Arc>, - manager: ImportManager, + /// A channel to the syncing thread. + sync_send: mpsc::UnboundedSender>, + /// A oneshot channel for destroying the sync thread. + _sync_exit: oneshot::Sender<()>, + /// A nextwork context to return and handle RPC requests. network: NetworkContext, + /// The `RPCHandler` logger. log: slog::Logger, } -impl SimpleSync { - /// Instantiate a `SimpleSync` instance, with no peers and an empty queue. +impl MessageProcessor { + /// Instantiate a `MessageProcessor` instance pub fn new( + executor: &tokio::runtime::TaskExecutor, beacon_chain: Arc>, network_send: mpsc::UnboundedSender, log: &slog::Logger, ) -> Self { let sync_logger = log.new(o!("Service"=> "Sync")); + let sync_network_context = NetworkContext::new(network_send.clone(), sync_logger.clone()); - SimpleSync { - chain: beacon_chain.clone(), - manager: ImportManager::new(beacon_chain, log), + // spawn the sync thread + let (sync_send, _sync_exit) = super::manager::spawn( + executor, + Arc::downgrade(&beacon_chain), + sync_network_context, + sync_logger, + ); + + MessageProcessor { + chain: beacon_chain, + sync_send, + _sync_exit, network: NetworkContext::new(network_send, log.clone()), - log: sync_logger, + log: log.clone(), } } + fn send_to_sync(&mut self, message: SyncMessage) { + self.sync_send.try_send(message).unwrap_or_else(|_| { + warn!( + self.log, + "Could not send message to the sync service"; + ) + }); + } + /// Handle a peer disconnect. /// /// Removes the peer from the manager. pub fn on_disconnect(&mut self, peer_id: PeerId) { - self.manager.peer_disconnect(&peer_id); + self.send_to_sync(SyncMessage::Disconnect(peer_id)); } /// Handle the connection of a new peer. @@ -107,6 +125,7 @@ impl SimpleSync { request_id: RequestId, hello: HelloMessage, ) { + // ignore hello responses if we are shutting down trace!(self.log, "HelloRequest"; "peer" => format!("{:?}", peer_id)); // Say hello back. @@ -149,7 +168,7 @@ impl SimpleSync { } else if remote.finalized_epoch <= local.finalized_epoch && remote.finalized_root != Hash256::zero() && local.finalized_root != Hash256::zero() - && (self.root_at_slot(start_slot(remote.finalized_epoch)) + && (self.chain.root_at_slot(start_slot(remote.finalized_epoch)) != Some(remote.finalized_root)) { // The remotes finalized epoch is less than or greater than ours, but the block root is @@ -189,18 +208,16 @@ impl SimpleSync { .exists::>(&remote.head_root) .unwrap_or_else(|_| false) { + trace!( + self.log, "Peer with known chain found"; + "peer" => format!("{:?}", peer_id), + "remote_head_slot" => remote.head_slot, + "remote_latest_finalized_epoch" => remote.finalized_epoch, + ); + // If the node's best-block is already known to us and they are close to our current // head, treat them as a fully sync'd peer. - if self.chain.best_slot().sub(remote.head_slot).as_u64() < FULL_PEER_TOLERANCE { - self.manager.add_full_peer(peer_id); - self.process_sync(); - } else { - debug!( - self.log, - "Out of sync peer connected"; - "peer" => format!("{:?}", peer_id), - ); - } + self.send_to_sync(SyncMessage::AddPeer(peer_id, remote)); } else { // The remote node has an equal or great finalized epoch and we don't know it's head. // @@ -212,87 +229,10 @@ impl SimpleSync { "local_finalized_epoch" => local.finalized_epoch, "remote_latest_finalized_epoch" => remote.finalized_epoch, ); - - self.manager.add_peer(peer_id, remote); - self.process_sync(); + self.send_to_sync(SyncMessage::AddPeer(peer_id, remote)); } } - fn process_sync(&mut self) { - loop { - match self.manager.poll() { - ImportManagerOutcome::Hello(peer_id) => { - trace!( - self.log, - "RPC Request"; - "method" => "HELLO", - "peer" => format!("{:?}", peer_id) - ); - self.network.send_rpc_request( - None, - peer_id, - RPCRequest::Hello(hello_message(&self.chain)), - ); - } - ImportManagerOutcome::RequestBlocks { - peer_id, - request_id, - request, - } => { - trace!( - self.log, - "RPC Request"; - "method" => "BeaconBlocks", - "id" => request_id, - "count" => request.count, - "peer" => format!("{:?}", peer_id) - ); - self.network.send_rpc_request( - Some(request_id), - peer_id.clone(), - RPCRequest::BeaconBlocks(request), - ); - } - ImportManagerOutcome::RecentRequest(peer_id, req) => { - trace!( - self.log, - "RPC Request"; - "method" => "RecentBeaconBlocks", - "count" => req.block_roots.len(), - "peer" => format!("{:?}", peer_id) - ); - self.network.send_rpc_request( - None, - peer_id.clone(), - RPCRequest::RecentBeaconBlocks(req), - ); - } - ImportManagerOutcome::DownvotePeer(peer_id) => { - trace!( - self.log, - "Peer downvoted"; - "peer" => format!("{:?}", peer_id) - ); - // TODO: Implement reputation - self.network - .disconnect(peer_id.clone(), GoodbyeReason::Fault); - } - ImportManagerOutcome::Idle => { - // nothing to do - return; - } - } - } - } - - //TODO: Move to beacon chain - fn root_at_slot(&self, target_slot: Slot) -> Option { - self.chain - .rev_iter_block_roots() - .find(|(_root, slot)| *slot == target_slot) - .map(|(root, _slot)| root) - } - /// Handle a `RecentBeaconBlocks` request from the peer. pub fn on_recent_beacon_blocks_request( &mut self, @@ -321,7 +261,7 @@ impl SimpleSync { debug!( self.log, - "BlockBodiesRequest"; + "RecentBeaconBlocksRequest"; "peer" => format!("{:?}", peer_id), "requested" => request.block_roots.len(), "returned" => blocks.len(), @@ -380,18 +320,16 @@ impl SimpleSync { blocks.reverse(); blocks.dedup_by_key(|brs| brs.slot); - if blocks.len() as u64 != req.count { - debug!( - self.log, - "BeaconBlocksRequest response"; - "peer" => format!("{:?}", peer_id), - "msg" => "Failed to return all requested hashes", - "start_slot" => req.start_slot, - "current_slot" => format!("{:?}", self.chain.slot()), - "requested" => req.count, - "returned" => blocks.len(), - ); - } + debug!( + self.log, + "BeaconBlocksRequest response"; + "peer" => format!("{:?}", peer_id), + "msg" => "Failed to return all requested hashes", + "start_slot" => req.start_slot, + "current_slot" => self.chain.slot().unwrap_or_else(|_| Slot::from(0_u64)).as_u64(), + "requested" => req.count, + "returned" => blocks.len(), + ); self.network.send_rpc_response( peer_id, @@ -414,10 +352,11 @@ impl SimpleSync { "count" => beacon_blocks.len(), ); - self.manager - .beacon_blocks_response(peer_id, request_id, beacon_blocks); - - self.process_sync(); + self.send_to_sync(SyncMessage::BeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + }); } /// Handle a `RecentBeaconBlocks` response from the peer. @@ -429,15 +368,16 @@ impl SimpleSync { ) { debug!( self.log, - "BeaconBlocksResponse"; + "RecentBeaconBlocksResponse"; "peer" => format!("{:?}", peer_id), "count" => beacon_blocks.len(), ); - self.manager - .recent_blocks_response(peer_id, request_id, beacon_blocks); - - self.process_sync(); + self.send_to_sync(SyncMessage::RecentBeaconBlocksResponse { + peer_id, + request_id, + beacon_blocks, + }); } /// Process a gossip message declaring a new block. @@ -455,9 +395,9 @@ impl SimpleSync { } BlockProcessingOutcome::ParentUnknown { parent: _ } => { // Inform the sync manager to find parents for this block - trace!(self.log, "Unknown parent gossip"; + trace!(self.log, "Block with unknown parent received"; "peer_id" => format!("{:?}",peer_id)); - self.manager.add_unknown_block(block.clone(), peer_id); + self.send_to_sync(SyncMessage::UnknownBlock(peer_id, block.clone())); SHOULD_FORWARD_GOSSIP_BLOCK } BlockProcessingOutcome::FutureSlot { @@ -468,7 +408,7 @@ impl SimpleSync { SHOULD_FORWARD_GOSSIP_BLOCK } BlockProcessingOutcome::BlockIsAlreadyKnown => SHOULD_FORWARD_GOSSIP_BLOCK, - _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, + _ => SHOULD_NOT_FORWARD_GOSSIP_BLOCK, //TODO: Decide if we want to forward these } } else { SHOULD_NOT_FORWARD_GOSSIP_BLOCK @@ -491,15 +431,10 @@ impl SimpleSync { } } } - - /// Generates our current state in the form of a HELLO RPC message. - pub fn generate_hello(&self) -> HelloMessage { - hello_message(&self.chain) - } } /// Build a `HelloMessage` representing the state of the given `beacon_chain`. -fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { +pub(crate) fn hello_message(beacon_chain: &BeaconChain) -> HelloMessage { let state = &beacon_chain.head().beacon_state; HelloMessage { @@ -527,7 +462,7 @@ impl NetworkContext { pub fn disconnect(&mut self, peer_id: PeerId, reason: GoodbyeReason) { warn!( &self.log, - "Disconnecting peer"; + "Disconnecting peer (RPC)"; "reason" => format!("{:?}", reason), "peer_id" => format!("{:?}", peer_id), ); @@ -560,12 +495,8 @@ impl NetworkContext { } fn send_rpc_event(&mut self, peer_id: PeerId, rpc_event: RPCEvent) { - self.send(peer_id, OutgoingMessage::RPC(rpc_event)) - } - - fn send(&mut self, peer_id: PeerId, outgoing_message: OutgoingMessage) { self.network_send - .try_send(NetworkMessage::Send(peer_id, outgoing_message)) + .try_send(NetworkMessage::RPC(peer_id, rpc_event)) .unwrap_or_else(|_| { warn!( self.log, diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 6a13a9aaec..cf56169384 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -137,6 +137,7 @@ fn process_testnet_subcommand( .and_then(|s| s.parse::().ok()); builder.import_bootstrap_libp2p_address(server, port)?; + builder.import_bootstrap_enr_address(server)?; builder.import_bootstrap_eth2_config(server)?; builder.set_beacon_chain_start_method(BeaconChainStartMethod::HttpBootstrap { @@ -301,7 +302,7 @@ impl<'a> ConfigBuilder<'a> { self.client_config.eth1_backend_method = method; } - /// Import the libp2p address for `server` into the list of bootnodes in `self`. + /// Import the libp2p address for `server` into the list of libp2p nodes to connect with. /// /// If `port` is `Some`, it is used as the port for the `Multiaddr`. If `port` is `None`, /// attempts to connect to the `server` via HTTP and retrieve it's libp2p listen port. @@ -333,6 +334,28 @@ impl<'a> ConfigBuilder<'a> { Ok(()) } + /// Import the enr address for `server` into the list of initial enrs (boot nodes). + pub fn import_bootstrap_enr_address(&mut self, server: &str) -> Result<()> { + let bootstrapper = Bootstrapper::connect(server.to_string(), &self.log)?; + + if let Ok(enr) = bootstrapper.enr() { + info!( + self.log, + "Loaded bootstrapper libp2p address"; + "enr" => format!("{:?}", enr) + ); + + self.client_config.network.boot_nodes.push(enr); + } else { + warn!( + self.log, + "Unable to estimate a bootstrapper enr address, this node may not find any peers." + ); + }; + + Ok(()) + } + /// Set the config data_dir to be an random directory. /// /// Useful for easily spinning up ephemeral testnets. diff --git a/beacon_node/src/main.rs b/beacon_node/src/main.rs index fab75ea4ea..5d23887852 100644 --- a/beacon_node/src/main.rs +++ b/beacon_node/src/main.rs @@ -33,14 +33,14 @@ fn main() { .arg( Arg::with_name("logfile") .long("logfile") - .value_name("logfile") + .value_name("FILE") .help("File path where output will be written.") .takes_value(true), ) .arg( Arg::with_name("network-dir") .long("network-dir") - .value_name("NETWORK-DIR") + .value_name("DIR") .help("Data directory for network keys.") .takes_value(true) .global(true) @@ -83,7 +83,7 @@ fn main() { Arg::with_name("boot-nodes") .long("boot-nodes") .allow_hyphen_values(true) - .value_name("BOOTNODES") + .value_name("ENR-LIST") .help("One or more comma-delimited base64-encoded ENR's to bootstrap the p2p network.") .takes_value(true), ) @@ -128,13 +128,14 @@ fn main() { .arg( Arg::with_name("rpc-address") .long("rpc-address") - .value_name("Address") + .value_name("ADDRESS") .help("Listen address for RPC endpoint.") .takes_value(true), ) .arg( Arg::with_name("rpc-port") .long("rpc-port") + .value_name("PORT") .help("Listen port for RPC endpoint.") .conflicts_with("port-bump") .takes_value(true), @@ -149,14 +150,14 @@ fn main() { .arg( Arg::with_name("api-address") .long("api-address") - .value_name("APIADDRESS") + .value_name("ADDRESS") .help("Set the listen address for the RESTful HTTP API server.") .takes_value(true), ) .arg( Arg::with_name("api-port") .long("api-port") - .value_name("APIPORT") + .value_name("PORT") .help("Set the listen TCP port for the RESTful HTTP API server.") .conflicts_with("port-bump") .takes_value(true), @@ -196,13 +197,6 @@ fn main() { .possible_values(&["info", "debug", "trace", "warn", "error", "crit"]) .default_value("trace"), ) - .arg( - Arg::with_name("verbosity") - .short("v") - .multiple(true) - .help("Sets the verbosity level") - .takes_value(true), - ) /* * The "testnet" sub-command. * diff --git a/book/src/interop-cheat-sheet.md b/book/src/interop-cheat-sheet.md index ea7794c338..7fea539ead 100644 --- a/book/src/interop-cheat-sheet.md +++ b/book/src/interop-cheat-sheet.md @@ -9,6 +9,7 @@ interop testing. - [Avoid port clashes when starting multiple nodes](#port-bump) - [Specify a custom slot time](#slot-time) - Using the beacon node HTTP API: + - [Pretty-print the genesis state and state root](#http-state) - [Curl a node's ENR](#http-enr) - [Curl a node's connected peers](#http-peer-ids) - [Curl a node's local peer id](#http-peer-id) @@ -82,6 +83,15 @@ $ ./beacon_node testnet -t 500 recent 8 Examples assume there is a Lighthouse node exposing a HTTP API on `localhost:5052`. Responses are JSON. + +### Pretty-print the genesis state and state root + +Returns the genesis state and state root in your terminal, in YAML. + +``` +$ curl --header "Content-Type: application/yaml" "localhost:5052/beacon/state?slot=0" +``` + ### Get the node's ENR diff --git a/book/src/interop-scenarios.md b/book/src/interop-scenarios.md index dc87893622..5e44d822a9 100644 --- a/book/src/interop-scenarios.md +++ b/book/src/interop-scenarios.md @@ -25,8 +25,11 @@ cheat-sheet](./interop-cheat-sheet.md). To start a brand-new beacon node (with no history) use: ``` -$ ./beacon_node testnet -f quick 8 1567222226 +$ ./beacon_node testnet -f quick 8 ``` + +Where `GENESIS_TIME` is in [unix time](https://duckduckgo.com/?q=unix+time&t=ffab&ia=answer). + > Notes: > > - This method conforms the ["Quick-start diff --git a/book/src/interop-tips.md b/book/src/interop-tips.md deleted file mode 100644 index 0d52e896ac..0000000000 --- a/book/src/interop-tips.md +++ /dev/null @@ -1 +0,0 @@ -# Interop Tips & Tricks diff --git a/book/src/setup.md b/book/src/setup.md index e53ca93d83..532de3fc04 100644 --- a/book/src/setup.md +++ b/book/src/setup.md @@ -9,11 +9,8 @@ See the [Quick instructions](#quick-instructions) for a summary or the 1. Install Rust + Cargo with [rustup](https://rustup.rs/). 1. Install build dependencies using your package manager. - - `$ clang protobuf libssl-dev cmake git-lfs` - - Ensure [git-lfs](https://git-lfs.github.com/) is installed with `git lfs - install`. -1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse), ensuring to - **initialize submodules**. + - `$ clang protobuf libssl-dev cmake` +1. Clone the [sigp/lighthouse](https://github.com/sigp/lighthouse). 1. In the root of the repo, run the tests with `cargo test --all --release`. 1. Then, build the binaries with `cargo build --all --release`. 1. Lighthouse is now fully built and tested. @@ -37,13 +34,8 @@ steps: - `protobuf`: required for protobuf serialization (gRPC) - `libssl-dev`: also gRPC - `cmake`: required for building protobuf - - `git-lfs`: The Git extension for [Large File - Support](https://git-lfs.github.com/) (required for Ethereum Foundation - test vectors). - 1. Clone the repository with submodules: `git clone --recursive - https://github.com/sigp/lighthouse`. If you're already cloned the repo, - ensure testing submodules are present: `$ git submodule init; git - submodule update` + 1. Clone the repository with submodules: `git clone + https://github.com/sigp/lighthouse`. 1. Change directory to the root of the repository. 1. Run the test suite with `cargo test --all --release`. The build and test process can take several minutes. If you experience any failures on @@ -63,3 +55,27 @@ Perl](http://strawberryperl.com/), or alternatively use a choco install command Additionally, the dependency `protoc-grpcio v0.3.1` is reported to have issues compiling in Windows. You can specify a known working version by editing version in `protos/Cargo.toml` section to `protoc-grpcio = "<=0.3.0"`. + +## eth2.0-spec-tests + +The +[ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/) +repository contains a large set of tests that verify Lighthouse behaviour +against the Ethereum Foundation specifications. + +The `tests/ef_tests` crate runs these tests and it has some interesting +behaviours: + +- If the `tests/ef_tests/eth2.0-spec-tests` directory is not present, all tests + indicate a `pass` when they did not actually run. +- If that directory _is_ present, the tests are executed faithfully, failing if + a discrepancy is found. + +The `tests/ef_tests/eth2.0-spec-tests` directory is not present by default. To +obtain it, use the Makefile in the root of the repository: + +``` +make ef_tests +``` + +_Note: this will download 100+ MB of test files from the [ethereum/eth2.0-spec-tests](https://github.com/ethereum/eth2.0-spec-tests/)._ diff --git a/eth2/state_processing/src/common/get_attesting_indices.rs b/eth2/state_processing/src/common/get_attesting_indices.rs index f558909f6e..adb71801a4 100644 --- a/eth2/state_processing/src/common/get_attesting_indices.rs +++ b/eth2/state_processing/src/common/get_attesting_indices.rs @@ -17,11 +17,9 @@ pub fn get_attesting_indices( target_relative_epoch, )?; - /* TODO(freeze): re-enable this? - if bitlist.len() > committee.committee.len() { + if bitlist.len() != committee.committee.len() { return Err(BeaconStateError::InvalidBitfield); } - */ Ok(committee .committee diff --git a/eth2/state_processing/src/common/get_compact_committees_root.rs b/eth2/state_processing/src/common/get_compact_committees_root.rs index 75edb3549e..b8ab4345fd 100644 --- a/eth2/state_processing/src/common/get_compact_committees_root.rs +++ b/eth2/state_processing/src/common/get_compact_committees_root.rs @@ -3,7 +3,7 @@ use types::*; /// Return the compact committee root at `relative_epoch`. /// -/// Spec v0.8.0 +/// Spec v0.8.3 pub fn get_compact_committees_root( state: &BeaconState, relative_epoch: RelativeEpoch, @@ -11,28 +11,13 @@ pub fn get_compact_committees_root( ) -> Result { let mut committees = FixedVector::<_, T::ShardCount>::from_elem(CompactCommittee::::default()); - // FIXME: this is a spec bug, whereby the start shard for the epoch after the next epoch - // is mistakenly used. The start shard from the cache SHOULD work. - // Waiting on a release to fix https://github.com/ethereum/eth2.0-specs/issues/1315 - let start_shard = if relative_epoch == RelativeEpoch::Next { - state.next_epoch_start_shard(spec)? - } else { - state.get_epoch_start_shard(relative_epoch)? - }; + let start_shard = state.get_epoch_start_shard(relative_epoch)?; for committee_number in 0..state.get_committee_count(relative_epoch)? { let shard = (start_shard + committee_number) % T::ShardCount::to_u64(); - // FIXME: this is a partial workaround for the above, but it only works in the case - // where there's a committee for every shard in every epoch. It works for the minimal - // tests but not the mainnet ones. - let fake_shard = if relative_epoch == RelativeEpoch::Next { - (shard + 1) % T::ShardCount::to_u64() - } else { - shard - }; for &index in state - .get_crosslink_committee_for_shard(fake_shard, relative_epoch)? + .get_crosslink_committee_for_shard(shard, relative_epoch)? .committee { let validator = state diff --git a/eth2/state_processing/src/common/get_indexed_attestation.rs b/eth2/state_processing/src/common/get_indexed_attestation.rs index 2507c76f2d..5e9362331b 100644 --- a/eth2/state_processing/src/common/get_indexed_attestation.rs +++ b/eth2/state_processing/src/common/get_indexed_attestation.rs @@ -11,6 +11,8 @@ pub fn get_indexed_attestation( state: &BeaconState, attestation: &Attestation, ) -> Result> { + // Note: we rely on both calls to `get_attesting_indices` to check the bitfield lengths + // against the committee length let attesting_indices = get_attesting_indices(state, &attestation.data, &attestation.aggregation_bits)?; diff --git a/eth2/state_processing/src/per_epoch_processing.rs b/eth2/state_processing/src/per_epoch_processing.rs index 8d6153aeac..f66ce4ea28 100644 --- a/eth2/state_processing/src/per_epoch_processing.rs +++ b/eth2/state_processing/src/per_epoch_processing.rs @@ -1,8 +1,5 @@ use crate::common::get_compact_committees_root; -use apply_rewards::process_rewards_and_penalties; use errors::EpochProcessingError as Error; -use process_slashings::process_slashings; -use registry_updates::process_registry_updates; use std::collections::HashMap; use tree_hash::TreeHash; use types::*; @@ -17,6 +14,10 @@ pub mod tests; pub mod validator_statuses; pub mod winning_root; +pub use apply_rewards::process_rewards_and_penalties; +pub use process_slashings::process_slashings; +pub use registry_updates::process_registry_updates; + /// Maps a shard to a winning root. /// /// It is generated during crosslink processing and later used to reward/penalize validators. @@ -218,45 +219,29 @@ pub fn process_final_updates( } } - // Update start shard. - state.start_shard = state.next_epoch_start_shard(spec)?; - - // This is a hack to allow us to update index roots and slashed balances for the next epoch. - // - // The indentation here is to make it obvious where the weird stuff happens. - { - state.slot += 1; - - // Set active index root - let index_epoch = next_epoch + spec.activation_exit_delay; - let indices_list = VariableList::::from( - state.get_active_validator_indices(index_epoch), - ); - state.set_active_index_root( - index_epoch, - Hash256::from_slice(&indices_list.tree_hash_root()), - spec, - )?; - - // Reset slashings - state.set_slashings(next_epoch, 0)?; - - // Set randao mix - state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; - - state.slot -= 1; - } + // Set active index root + let index_epoch = next_epoch + spec.activation_exit_delay; + let indices_list = VariableList::::from( + state.get_active_validator_indices(index_epoch), + ); + state.set_active_index_root( + index_epoch, + Hash256::from_slice(&indices_list.tree_hash_root()), + spec, + )?; // Set committees root - // Note: we do this out-of-order w.r.t. to the spec, because we don't want the slot to be - // incremented. It's safe because the updates to slashings and the RANDAO mix (above) don't - // affect this. state.set_compact_committee_root( next_epoch, get_compact_committees_root(state, RelativeEpoch::Next, spec)?, - spec, )?; + // Reset slashings + state.set_slashings(next_epoch, 0)?; + + // Set randao mix + state.set_randao_mix(next_epoch, *state.get_randao_mix(current_epoch)?)?; + // Set historical root accumulator if next_epoch.as_u64() % (T::SlotsPerHistoricalRoot::to_u64() / T::slots_per_epoch()) == 0 { let historical_batch = state.historical_batch(); @@ -265,6 +250,9 @@ pub fn process_final_updates( .push(Hash256::from_slice(&historical_batch.tree_hash_root()))?; } + // Update start shard. + state.start_shard = state.get_epoch_start_shard(RelativeEpoch::Next)?; + // Rotate current/previous epoch attestations state.previous_epoch_attestations = std::mem::replace(&mut state.current_epoch_attestations, VariableList::empty()); diff --git a/eth2/types/src/attestation_data.rs b/eth2/types/src/attestation_data.rs index f2e63598f5..4d82ce1261 100644 --- a/eth2/types/src/attestation_data.rs +++ b/eth2/types/src/attestation_data.rs @@ -4,25 +4,13 @@ use crate::{Checkpoint, Crosslink, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// The data upon which an attestation is based. /// /// Spec v0.8.0 #[derive( - Debug, - Clone, - PartialEq, - Eq, - Serialize, - Deserialize, - Hash, - Encode, - Decode, - TreeHash, - TestRandom, - SignedRoot, + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash, Encode, Decode, TreeHash, TestRandom, )] pub struct AttestationData { // LMD GHOST vote diff --git a/eth2/types/src/beacon_state.rs b/eth2/types/src/beacon_state.rs index fb923fc06a..6b2b44d0ee 100644 --- a/eth2/types/src/beacon_state.rs +++ b/eth2/types/src/beacon_state.rs @@ -60,6 +60,22 @@ pub enum Error { SszTypesError(ssz_types::Error), } +/// Control whether an epoch-indexed field can be indexed at the next epoch or not. +#[derive(Debug, PartialEq, Clone, Copy)] +enum AllowNextEpoch { + True, + False, +} + +impl AllowNextEpoch { + fn upper_bound_of(self, current_epoch: Epoch) -> Epoch { + match self { + AllowNextEpoch::True => current_epoch + 1, + AllowNextEpoch::False => current_epoch, + } + } +} + /// The state of the `BeaconChain` at some slot. /// /// Spec v0.8.0 @@ -108,12 +124,12 @@ where pub start_shard: u64, pub randao_mixes: FixedVector, #[compare_fields(as_slice)] - active_index_roots: FixedVector, + pub active_index_roots: FixedVector, #[compare_fields(as_slice)] - compact_committees_roots: FixedVector, + pub compact_committees_roots: FixedVector, // Slashings - slashings: FixedVector, + pub slashings: FixedVector, // Attestations pub previous_epoch_attestations: VariableList, T::MaxPendingAttestations>, @@ -282,14 +298,6 @@ impl BeaconState { Ok(cache.epoch_start_shard()) } - pub fn next_epoch_start_shard(&self, spec: &ChainSpec) -> Result { - let cache = self.cache(RelativeEpoch::Current)?; - let active_validator_count = cache.active_validator_count(); - let shard_delta = T::get_shard_delta(active_validator_count, spec.target_committee_size); - - Ok((self.start_shard + shard_delta) % T::ShardCount::to_u64()) - } - /// Get the slot of an attestation. /// /// Note: Utilizes the cache and will fail if the appropriate cache is not initialized. @@ -463,12 +471,16 @@ impl BeaconState { /// Safely obtains the index for `randao_mixes` /// - /// Spec v0.8.0 - fn get_randao_mix_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_randao_mix_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let len = T::EpochsPerHistoricalVector::to_u64(); - if epoch + len > current_epoch && epoch <= current_epoch { + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) @@ -496,7 +508,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_randao_mix(&self, epoch: Epoch) -> Result<&Hash256, Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::False)?; Ok(&self.randao_mixes[i]) } @@ -504,21 +516,29 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn set_randao_mix(&mut self, epoch: Epoch, mix: Hash256) -> Result<(), Error> { - let i = self.get_randao_mix_index(epoch)?; + let i = self.get_randao_mix_index(epoch, AllowNextEpoch::True)?; self.randao_mixes[i] = mix; Ok(()) } /// Safely obtains the index for `active_index_roots`, given some `epoch`. /// + /// If `allow_next_epoch` is `True`, then we allow an _extra_ one epoch of lookahead. + /// /// Spec v0.8.1 - fn get_active_index_root_index(&self, epoch: Epoch, spec: &ChainSpec) -> Result { + fn get_active_index_root_index( + &self, + epoch: Epoch, + spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, + ) -> Result { let current_epoch = self.current_epoch(); let lookahead = spec.activation_exit_delay; let lookback = self.active_index_roots.len() as u64 - lookahead; + let epoch_upper_bound = allow_next_epoch.upper_bound_of(current_epoch) + lookahead; - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { + if current_epoch < epoch + lookback && epoch <= epoch_upper_bound { Ok(epoch.as_usize() % self.active_index_roots.len()) } else { Err(Error::EpochOutOfBounds) @@ -529,7 +549,7 @@ impl BeaconState { /// /// Spec v0.8.1 pub fn get_active_index_root(&self, epoch: Epoch, spec: &ChainSpec) -> Result { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::False)?; Ok(self.active_index_roots[i]) } @@ -542,7 +562,7 @@ impl BeaconState { index_root: Hash256, spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_active_index_root_index(epoch, spec)?; + let i = self.get_active_index_root_index(epoch, spec, AllowNextEpoch::True)?; self.active_index_roots[i] = index_root; Ok(()) } @@ -556,19 +576,17 @@ impl BeaconState { /// Safely obtains the index for `compact_committees_roots`, given some `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 fn get_compact_committee_root_index( &self, epoch: Epoch, - spec: &ChainSpec, + allow_next_epoch: AllowNextEpoch, ) -> Result { let current_epoch = self.current_epoch(); + let len = T::EpochsPerHistoricalVector::to_u64(); - let lookahead = spec.activation_exit_delay; - let lookback = self.compact_committees_roots.len() as u64 - lookahead; - - if epoch + lookback > current_epoch && current_epoch + lookahead >= epoch { - Ok(epoch.as_usize() % self.compact_committees_roots.len()) + if current_epoch < epoch + len && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { + Ok(epoch.as_usize() % len as usize) } else { Err(Error::EpochOutOfBounds) } @@ -576,26 +594,21 @@ impl BeaconState { /// Return the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 - pub fn get_compact_committee_root( - &self, - epoch: Epoch, - spec: &ChainSpec, - ) -> Result { - let i = self.get_compact_committee_root_index(epoch, spec)?; + /// Spec v0.8.1 + pub fn get_compact_committee_root(&self, epoch: Epoch) -> Result { + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::False)?; Ok(self.compact_committees_roots[i]) } /// Set the `compact_committee_root` at a recent `epoch`. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_compact_committee_root( &mut self, epoch: Epoch, index_root: Hash256, - spec: &ChainSpec, ) -> Result<(), Error> { - let i = self.get_compact_committee_root_index(epoch, spec)?; + let i = self.get_compact_committee_root_index(epoch, AllowNextEpoch::True)?; self.compact_committees_roots[i] = index_root; Ok(()) } @@ -646,14 +659,19 @@ impl BeaconState { /// Safely obtain the index for `slashings`, given some `epoch`. /// - /// Spec v0.8.0 - fn get_slashings_index(&self, epoch: Epoch) -> Result { + /// Spec v0.8.1 + fn get_slashings_index( + &self, + epoch: Epoch, + allow_next_epoch: AllowNextEpoch, + ) -> Result { // We allow the slashings vector to be accessed at any cached epoch at or before - // the current epoch. - if epoch <= self.current_epoch() - && epoch + T::EpochsPerSlashingsVector::to_u64() >= self.current_epoch() + 1 + // the current epoch, or the next epoch if `AllowNextEpoch::True` is passed. + let current_epoch = self.current_epoch(); + if current_epoch < epoch + T::EpochsPerSlashingsVector::to_u64() + && epoch <= allow_next_epoch.upper_bound_of(current_epoch) { - Ok((epoch.as_u64() % T::EpochsPerSlashingsVector::to_u64()) as usize) + Ok(epoch.as_usize() % T::EpochsPerSlashingsVector::to_usize()) } else { Err(Error::EpochOutOfBounds) } @@ -668,17 +686,17 @@ impl BeaconState { /// Get the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn get_slashings(&self, epoch: Epoch) -> Result { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::False)?; Ok(self.slashings[i]) } /// Set the total slashed balances for some epoch. /// - /// Spec v0.8.0 + /// Spec v0.8.1 pub fn set_slashings(&mut self, epoch: Epoch, value: u64) -> Result<(), Error> { - let i = self.get_slashings_index(epoch)?; + let i = self.get_slashings_index(epoch, AllowNextEpoch::True)?; self.slashings[i] = value; Ok(()) } diff --git a/eth2/types/src/beacon_state/tests.rs b/eth2/types/src/beacon_state/tests.rs index 67adccdda8..0363e5848d 100644 --- a/eth2/types/src/beacon_state/tests.rs +++ b/eth2/types/src/beacon_state/tests.rs @@ -90,11 +90,11 @@ fn test_active_index(state_slot: Slot) { // Test the start and end of the range. assert_eq!( - state.get_active_index_root_index(*range.start(), &spec), + state.get_active_index_root_index(*range.start(), &spec, AllowNextEpoch::False), Ok(modulo(*range.start())) ); assert_eq!( - state.get_active_index_root_index(*range.end(), &spec), + state.get_active_index_root_index(*range.end(), &spec, AllowNextEpoch::False), Ok(modulo(*range.end())) ); @@ -102,12 +102,12 @@ fn test_active_index(state_slot: Slot) { if state.current_epoch() > 0 { // Test is invalid on epoch zero, cannot subtract from zero. assert_eq!( - state.get_active_index_root_index(*range.start() - 1, &spec), + state.get_active_index_root_index(*range.start() - 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } assert_eq!( - state.get_active_index_root_index(*range.end() + 1, &spec), + state.get_active_index_root_index(*range.end() + 1, &spec, AllowNextEpoch::False), Err(Error::EpochOutOfBounds) ); } diff --git a/eth2/types/src/checkpoint.rs b/eth2/types/src/checkpoint.rs index dc40b336fe..d5d40fa674 100644 --- a/eth2/types/src/checkpoint.rs +++ b/eth2/types/src/checkpoint.rs @@ -3,8 +3,7 @@ use crate::{Epoch, Hash256}; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use test_random_derive::TestRandom; -use tree_hash::TreeHash; -use tree_hash_derive::{SignedRoot, TreeHash}; +use tree_hash_derive::TreeHash; /// Casper FFG checkpoint, used in attestations. /// @@ -22,7 +21,6 @@ use tree_hash_derive::{SignedRoot, TreeHash}; Decode, TreeHash, TestRandom, - SignedRoot, )] pub struct Checkpoint { pub epoch: Epoch, diff --git a/eth2/types/src/lib.rs b/eth2/types/src/lib.rs index 3edf8b36ba..d1eaa393f8 100644 --- a/eth2/types/src/lib.rs +++ b/eth2/types/src/lib.rs @@ -86,5 +86,8 @@ pub type AttesterMap = HashMap<(u64, u64), Vec>; /// Maps a slot to a block proposer. pub type ProposerMap = HashMap; -pub use bls::{AggregatePublicKey, AggregateSignature, Keypair, PublicKey, SecretKey, Signature}; +pub use bls::{ + AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, + Signature, SignatureBytes, +}; pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList}; diff --git a/eth2/types/src/slot_epoch_macros.rs b/eth2/types/src/slot_epoch_macros.rs index 62ca6b3af3..3bd54ee2da 100644 --- a/eth2/types/src/slot_epoch_macros.rs +++ b/eth2/types/src/slot_epoch_macros.rs @@ -201,6 +201,10 @@ macro_rules! impl_ssz { ::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + 0_u64.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { self.0.ssz_append(buf) } diff --git a/eth2/utils/bls/Cargo.toml b/eth2/utils/bls/Cargo.toml index 4f499ad37d..349e08f545 100644 --- a/eth2/utils/bls/Cargo.toml +++ b/eth2/utils/bls/Cargo.toml @@ -5,7 +5,8 @@ authors = ["Paul Hauner "] edition = "2018" [dependencies] -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.10.0" } +# FIXME: update sigp repo +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" } eth2_hashing = { path = "../eth2_hashing" } hex = "0.3" rand = "^0.5" diff --git a/eth2/utils/bls/src/macros.rs b/eth2/utils/bls/src/macros.rs index 09838b73ed..e8bd3dd048 100644 --- a/eth2/utils/bls/src/macros.rs +++ b/eth2/utils/bls/src/macros.rs @@ -9,6 +9,10 @@ macro_rules! impl_ssz { $byte_size } + fn ssz_bytes_len(&self) -> usize { + $byte_size + } + fn ssz_append(&self, buf: &mut Vec) { buf.append(&mut self.as_bytes()) } diff --git a/eth2/utils/eth2_interop_keypairs/Cargo.toml b/eth2/utils/eth2_interop_keypairs/Cargo.toml index 31f9718cd4..d8a1118553 100644 --- a/eth2/utils/eth2_interop_keypairs/Cargo.toml +++ b/eth2/utils/eth2_interop_keypairs/Cargo.toml @@ -10,7 +10,7 @@ edition = "2018" lazy_static = "1.4" num-bigint = "0.2" eth2_hashing = "0.1" -milagro_bls = { git = "https://github.com/sigp/milagro_bls", tag = "v0.10.0" } +milagro_bls = { git = "https://github.com/michaelsproul/milagro_bls", branch = "little-endian-v0.10" } [dev-dependencies] base64 = "0.10" diff --git a/eth2/utils/slot_clock/src/lib.rs b/eth2/utils/slot_clock/src/lib.rs index fd3bf029be..6192d1b6f1 100644 --- a/eth2/utils/slot_clock/src/lib.rs +++ b/eth2/utils/slot_clock/src/lib.rs @@ -5,7 +5,7 @@ mod metrics; mod system_time_slot_clock; mod testing_slot_clock; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, SystemTimeError, UNIX_EPOCH}; pub use crate::system_time_slot_clock::SystemTimeSlotClock; pub use crate::testing_slot_clock::TestingSlotClock; @@ -17,18 +17,21 @@ pub trait SlotClock: Send + Sync + Sized { genesis_slot: Slot, genesis_seconds: u64, slot_duration: Duration, - ) -> Option { - let duration_between_now_and_unix_epoch = - SystemTime::now().duration_since(UNIX_EPOCH).ok()?; + ) -> Result { + let duration_between_now_and_unix_epoch = SystemTime::now().duration_since(UNIX_EPOCH)?; let duration_between_unix_epoch_and_genesis = Duration::from_secs(genesis_seconds); - if duration_between_now_and_unix_epoch < duration_between_unix_epoch_and_genesis { - None + let genesis_instant = if duration_between_now_and_unix_epoch + < duration_between_unix_epoch_and_genesis + { + Instant::now() + + (duration_between_unix_epoch_and_genesis - duration_between_now_and_unix_epoch) } else { - let genesis_instant = Instant::now() - - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis); - Some(Self::new(genesis_slot, genesis_instant, slot_duration)) - } + Instant::now() + - (duration_between_now_and_unix_epoch - duration_between_unix_epoch_and_genesis) + }; + + Ok(Self::new(genesis_slot, genesis_instant, slot_duration)) } fn new(genesis_slot: Slot, genesis: Instant, slot_duration: Duration) -> Self; diff --git a/eth2/utils/slot_clock/src/system_time_slot_clock.rs b/eth2/utils/slot_clock/src/system_time_slot_clock.rs index 0d4a52ef64..aae12c18c4 100644 --- a/eth2/utils/slot_clock/src/system_time_slot_clock.rs +++ b/eth2/utils/slot_clock/src/system_time_slot_clock.rs @@ -42,7 +42,7 @@ impl SlotClock for SystemTimeSlotClock { fn duration_to_next_slot(&self) -> Option { let now = Instant::now(); if now < self.genesis { - None + Some(self.genesis - now) } else { let duration_since_genesis = now - self.genesis; let millis_since_genesis = duration_since_genesis.as_millis(); diff --git a/eth2/utils/ssz/examples/struct_definition.rs b/eth2/utils/ssz/examples/struct_definition.rs index fa3ed2a642..0971e21da9 100644 --- a/eth2/utils/ssz/examples/struct_definition.rs +++ b/eth2/utils/ssz/examples/struct_definition.rs @@ -12,6 +12,13 @@ impl Encode for Foo { ::is_ssz_fixed_len() && as Encode>::is_ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + ::ssz_fixed_len() + + ssz::BYTES_PER_LENGTH_OFFSET + + ::ssz_fixed_len() + + self.b.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { let offset = ::ssz_fixed_len() + as Encode>::ssz_fixed_len() diff --git a/eth2/utils/ssz/src/encode.rs b/eth2/utils/ssz/src/encode.rs index 6ceb08debc..5113fb71a1 100644 --- a/eth2/utils/ssz/src/encode.rs +++ b/eth2/utils/ssz/src/encode.rs @@ -27,6 +27,8 @@ pub trait Encode { BYTES_PER_LENGTH_OFFSET } + fn ssz_bytes_len(&self) -> usize; + /// Returns the full-form encoding of this object. /// /// The default implementation of this method should suffice for most cases. diff --git a/eth2/utils/ssz/src/encode/impls.rs b/eth2/utils/ssz/src/encode/impls.rs index 3d68d8911a..d25e79370e 100644 --- a/eth2/utils/ssz/src/encode/impls.rs +++ b/eth2/utils/ssz/src/encode/impls.rs @@ -13,6 +13,10 @@ macro_rules! impl_encodable_for_uint { $bit_size / 8 } + fn ssz_bytes_len(&self) -> usize { + $bit_size / 8 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&self.to_le_bytes()); } @@ -58,6 +62,23 @@ macro_rules! impl_encode_for_tuples { } } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + let mut len = 0; + $( + len += if <$T as Encode>::is_ssz_fixed_len() { + <$T as Encode>::ssz_fixed_len() + } else { + BYTES_PER_LENGTH_OFFSET + + self.$idx.ssz_bytes_len() + }; + )* + len + } + } + fn ssz_append(&self, buf: &mut Vec) { let offset = $( <$T as Encode>::ssz_fixed_len() + @@ -185,6 +206,19 @@ impl Encode for Option { false } + fn ssz_bytes_len(&self) -> usize { + if let Some(some) = self { + let len = if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + some.ssz_bytes_len() + }; + len + BYTES_PER_LENGTH_OFFSET + } else { + BYTES_PER_LENGTH_OFFSET + } + } + fn ssz_append(&self, buf: &mut Vec) { match self { None => buf.append(&mut encode_union_index(0)), @@ -201,6 +235,16 @@ impl Encode for Vec { false } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() * self.len() + } else { + let mut len = self.into_iter().map(|item| item.ssz_bytes_len()).sum(); + len += BYTES_PER_LENGTH_OFFSET * self.len(); + len + } + } + fn ssz_append(&self, buf: &mut Vec) { if T::is_ssz_fixed_len() { buf.reserve(T::ssz_fixed_len() * self.len()); @@ -229,6 +273,10 @@ impl Encode for bool { 1 } + fn ssz_bytes_len(&self) -> usize { + 1 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&(*self as u8).to_le_bytes()); } @@ -243,6 +291,10 @@ impl Encode for NonZeroUsize { ::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + std::mem::size_of::() + } + fn ssz_append(&self, buf: &mut Vec) { self.get().ssz_append(buf) } @@ -257,6 +309,10 @@ impl Encode for H256 { 32 } + fn ssz_bytes_len(&self) -> usize { + 32 + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(self.as_bytes()); } @@ -271,6 +327,10 @@ impl Encode for U256 { 32 } + fn ssz_bytes_len(&self) -> usize { + 32 + } + fn ssz_append(&self, buf: &mut Vec) { let n = ::ssz_fixed_len(); let s = buf.len(); @@ -289,6 +349,10 @@ impl Encode for U128 { 16 } + fn ssz_bytes_len(&self) -> usize { + 16 + } + fn ssz_append(&self, buf: &mut Vec) { let n = ::ssz_fixed_len(); let s = buf.len(); @@ -309,6 +373,10 @@ macro_rules! impl_encodable_for_u8_array { $len } + fn ssz_bytes_len(&self) -> usize { + $len + } + fn ssz_append(&self, buf: &mut Vec) { buf.extend_from_slice(&self[..]); } diff --git a/eth2/utils/ssz/src/lib.rs b/eth2/utils/ssz/src/lib.rs index 696d36cbf7..115633889e 100644 --- a/eth2/utils/ssz/src/lib.rs +++ b/eth2/utils/ssz/src/lib.rs @@ -36,7 +36,6 @@ mod decode; mod encode; -mod macros; pub use decode::{ impls::decode_list_of_variable_length_items, Decode, DecodeError, SszDecoder, SszDecoderBuilder, diff --git a/eth2/utils/ssz/src/macros.rs b/eth2/utils/ssz/src/macros.rs index 04147a8058..8b13789179 100644 --- a/eth2/utils/ssz/src/macros.rs +++ b/eth2/utils/ssz/src/macros.rs @@ -1,96 +1 @@ -/// Implements `Encode` for `$impl_type` using an implementation of `From<$impl_type> for -/// $from_type`. -/// -/// In effect, this allows for easy implementation of `Encode` for some type that implements a -/// `From` conversion into another type that already has `Encode` implemented. -#[macro_export] -macro_rules! impl_encode_via_from { - ($impl_type: ty, $from_type: ty) => { - impl ssz::Encode for $impl_type { - fn is_ssz_fixed_len() -> bool { - <$from_type as ssz::Encode>::is_ssz_fixed_len() - } - fn ssz_fixed_len() -> usize { - <$from_type as ssz::Encode>::ssz_fixed_len() - } - - fn ssz_append(&self, buf: &mut Vec) { - let conv: $from_type = self.clone().into(); - - conv.ssz_append(buf) - } - } - }; -} - -/// Implements `Decode` for `$impl_type` using an implementation of `From<$impl_type> for -/// $from_type`. -/// -/// In effect, this allows for easy implementation of `Decode` for some type that implements a -/// `From` conversion into another type that already has `Decode` implemented. -#[macro_export] -macro_rules! impl_decode_via_from { - ($impl_type: ty, $from_type: tt) => { - impl ssz::Decode for $impl_type { - fn is_ssz_fixed_len() -> bool { - <$from_type as ssz::Decode>::is_ssz_fixed_len() - } - - fn ssz_fixed_len() -> usize { - <$from_type as ssz::Decode>::ssz_fixed_len() - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - $from_type::from_ssz_bytes(bytes).and_then(|dec| Ok(dec.into())) - } - } - }; -} - -#[cfg(test)] -mod tests { - use self::ssz::{Decode, Encode}; - use crate as ssz; - - #[derive(PartialEq, Debug, Clone, Copy)] - struct Wrapper(u64); - - impl From for Wrapper { - fn from(x: u64) -> Wrapper { - Wrapper(x) - } - } - - impl From for u64 { - fn from(x: Wrapper) -> u64 { - x.0 - } - } - - impl_encode_via_from!(Wrapper, u64); - impl_decode_via_from!(Wrapper, u64); - - #[test] - fn impl_encode_via_from() { - let check_encode = |a: u64, b: Wrapper| assert_eq!(a.as_ssz_bytes(), b.as_ssz_bytes()); - - check_encode(0, Wrapper(0)); - check_encode(1, Wrapper(1)); - check_encode(42, Wrapper(42)); - } - - #[test] - fn impl_decode_via_from() { - let check_decode = |bytes: Vec| { - let a = u64::from_ssz_bytes(&bytes).unwrap(); - let b = Wrapper::from_ssz_bytes(&bytes).unwrap(); - - assert_eq!(a, b.into()) - }; - - check_decode(vec![0, 0, 0, 0, 0, 0, 0, 0]); - check_decode(vec![1, 0, 0, 0, 0, 0, 0, 0]); - check_decode(vec![1, 0, 0, 0, 2, 0, 0, 0]); - } -} diff --git a/eth2/utils/ssz/tests/tests.rs b/eth2/utils/ssz/tests/tests.rs index c19e366622..26f2f53efe 100644 --- a/eth2/utils/ssz/tests/tests.rs +++ b/eth2/utils/ssz/tests/tests.rs @@ -8,6 +8,7 @@ mod round_trip { fn round_trip(items: Vec) { for item in items { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } } diff --git a/eth2/utils/ssz_derive/src/lib.rs b/eth2/utils/ssz_derive/src/lib.rs index 47d96859e0..5bdb9ca9dd 100644 --- a/eth2/utils/ssz_derive/src/lib.rs +++ b/eth2/utils/ssz_derive/src/lib.rs @@ -81,9 +81,12 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { }; let field_idents = get_serializable_named_field_idents(&struct_data); + let field_idents_a = get_serializable_named_field_idents(&struct_data); let field_types_a = get_serializable_field_types(&struct_data); let field_types_b = field_types_a.clone(); - let field_types_c = field_types_a.clone(); + let field_types_d = field_types_a.clone(); + let field_types_e = field_types_a.clone(); + let field_types_f = field_types_a.clone(); let output = quote! { impl #impl_generics ssz::Encode for #name #ty_generics #where_clause { @@ -105,9 +108,27 @@ pub fn ssz_encode_derive(input: TokenStream) -> TokenStream { } } + fn ssz_bytes_len(&self) -> usize { + if ::is_ssz_fixed_len() { + ::ssz_fixed_len() + } else { + let mut len = 0; + #( + if <#field_types_d as ssz::Encode>::is_ssz_fixed_len() { + len += <#field_types_e as ssz::Encode>::ssz_fixed_len(); + } else { + len += ssz::BYTES_PER_LENGTH_OFFSET; + len += self.#field_idents_a.ssz_bytes_len(); + } + )* + + len + } + } + fn ssz_append(&self, buf: &mut Vec) { let offset = #( - <#field_types_c as ssz::Encode>::ssz_fixed_len() + + <#field_types_f as ssz::Encode>::ssz_fixed_len() + )* 0; diff --git a/eth2/utils/ssz_types/src/bitfield.rs b/eth2/utils/ssz_types/src/bitfield.rs index 197426046c..dbe1addbea 100644 --- a/eth2/utils/ssz_types/src/bitfield.rs +++ b/eth2/utils/ssz_types/src/bitfield.rs @@ -476,6 +476,12 @@ impl Encode for Bitfield> { false } + fn ssz_bytes_len(&self) -> usize { + // We could likely do better than turning this into bytes and reading the length, however + // it is kept this way for simplicity. + self.clone().into_bytes().len() + } + fn ssz_append(&self, buf: &mut Vec) { buf.append(&mut self.clone().into_bytes()) } @@ -498,6 +504,10 @@ impl Encode for Bitfield> { true } + fn ssz_bytes_len(&self) -> usize { + self.as_slice().len() + } + fn ssz_fixed_len() -> usize { bytes_for_bit_len(N::to_usize()) } @@ -616,6 +626,7 @@ mod bitvector { pub type BitVector4 = BitVector; pub type BitVector8 = BitVector; pub type BitVector16 = BitVector; + pub type BitVector64 = BitVector; #[test] fn ssz_encode() { @@ -706,6 +717,18 @@ mod bitvector { fn assert_round_trip(t: T) { assert_eq!(T::from_ssz_bytes(&t.as_ssz_bytes()).unwrap(), t); } + + #[test] + fn ssz_bytes_len() { + for i in 0..64 { + let mut bitfield = BitVector64::new(); + for j in 0..i { + bitfield.set(j, true).expect("should set bit in bounds"); + } + let bytes = bitfield.as_ssz_bytes(); + assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i); + } + } } #[cfg(test)] @@ -1152,4 +1175,16 @@ mod bitlist { vec![false, false, true, false, false, false, false, false, true] ); } + + #[test] + fn ssz_bytes_len() { + for i in 1..64 { + let mut bitfield = BitList1024::with_capacity(i).unwrap(); + for j in 0..i { + bitfield.set(j, true).expect("should set bit in bounds"); + } + let bytes = bitfield.as_ssz_bytes(); + assert_eq!(bitfield.ssz_bytes_len(), bytes.len(), "i = {}", i); + } + } } diff --git a/eth2/utils/ssz_types/src/fixed_vector.rs b/eth2/utils/ssz_types/src/fixed_vector.rs index edac77f0d0..f9c8963313 100644 --- a/eth2/utils/ssz_types/src/fixed_vector.rs +++ b/eth2/utils/ssz_types/src/fixed_vector.rs @@ -172,7 +172,7 @@ where T: ssz::Encode, { fn is_ssz_fixed_len() -> bool { - true + T::is_ssz_fixed_len() } fn ssz_fixed_len() -> usize { @@ -183,6 +183,10 @@ where } } + fn ssz_bytes_len(&self) -> usize { + self.vec.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { if T::is_ssz_fixed_len() { buf.reserve(T::ssz_fixed_len() * self.len()); @@ -220,13 +224,26 @@ where fn from_ssz_bytes(bytes: &[u8]) -> Result { if bytes.is_empty() { - Ok(FixedVector::from(vec![])) + Err(ssz::DecodeError::InvalidByteLength { + len: 0, + expected: 1, + }) } else if T::is_ssz_fixed_len() { bytes .chunks(T::ssz_fixed_len()) .map(|chunk| T::from_ssz_bytes(chunk)) .collect::, _>>() - .and_then(|vec| Ok(vec.into())) + .and_then(|vec| { + if vec.len() == N::to_usize() { + Ok(vec.into()) + } else { + Err(ssz::DecodeError::BytesInvalid(format!( + "wrong number of vec elements, got: {}, expected: {}", + vec.len(), + N::to_usize() + ))) + } + }) } else { ssz::decode_list_of_variable_length_items(bytes).and_then(|vec| Ok(vec.into())) } @@ -305,6 +322,7 @@ mod test { fn ssz_round_trip(item: T) { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } diff --git a/eth2/utils/ssz_types/src/variable_list.rs b/eth2/utils/ssz_types/src/variable_list.rs index beb7e6a938..feb656745b 100644 --- a/eth2/utils/ssz_types/src/variable_list.rs +++ b/eth2/utils/ssz_types/src/variable_list.rs @@ -208,6 +208,10 @@ where >::ssz_fixed_len() } + fn ssz_bytes_len(&self) -> usize { + self.vec.ssz_bytes_len() + } + fn ssz_append(&self, buf: &mut Vec) { self.vec.ssz_append(buf) } @@ -304,6 +308,7 @@ mod test { fn round_trip(item: T) { let encoded = &item.as_ssz_bytes(); + assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!(T::from_ssz_bytes(&encoded), Ok(item)); } diff --git a/eth2/utils/tree_hash/src/impls.rs b/eth2/utils/tree_hash/src/impls.rs index 88293196e4..9f09f50ce7 100644 --- a/eth2/utils/tree_hash/src/impls.rs +++ b/eth2/utils/tree_hash/src/impls.rs @@ -1,5 +1,5 @@ use super::*; -use ethereum_types::H256; +use ethereum_types::{H256, U128, U256}; macro_rules! impl_for_bitsize { ($type: ident, $bit_size: expr) => { @@ -73,6 +73,46 @@ macro_rules! impl_for_u8_array { impl_for_u8_array!(4); impl_for_u8_array!(32); +impl TreeHash for U128 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 16]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 2 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + +impl TreeHash for U256 { + fn tree_hash_type() -> TreeHashType { + TreeHashType::Basic + } + + fn tree_hash_packed_encoding(&self) -> Vec { + let mut result = vec![0; 32]; + self.to_little_endian(&mut result); + result + } + + fn tree_hash_packing_factor() -> usize { + 1 + } + + fn tree_hash_root(&self) -> Vec { + merkle_root(&self.tree_hash_packed_encoding(), 0) + } +} + impl TreeHash for H256 { fn tree_hash_type() -> TreeHashType { TreeHashType::Vector diff --git a/tests/ef_tests/.gitignore b/tests/ef_tests/.gitignore new file mode 100644 index 0000000000..a83c5aa961 --- /dev/null +++ b/tests/ef_tests/.gitignore @@ -0,0 +1 @@ +/eth2.0-spec-tests diff --git a/tests/ef_tests/Cargo.toml b/tests/ef_tests/Cargo.toml index ba6aca259d..2f1dea11d1 100644 --- a/tests/ef_tests/Cargo.toml +++ b/tests/ef_tests/Cargo.toml @@ -18,7 +18,9 @@ serde_derive = "1.0" serde_repr = "0.1" serde_yaml = "0.8" eth2_ssz = "0.1" +eth2_ssz_derive = "0.1" tree_hash = "0.1" +tree_hash_derive = "0.2" state_processing = { path = "../../eth2/state_processing" } swap_or_not_shuffle = { path = "../../eth2/utils/swap_or_not_shuffle" } types = { path = "../../eth2/types" } diff --git a/tests/ef_tests/eth2.0-spec-tests b/tests/ef_tests/eth2.0-spec-tests deleted file mode 160000 index aaa1673f50..0000000000 --- a/tests/ef_tests/eth2.0-spec-tests +++ /dev/null @@ -1 +0,0 @@ -Subproject commit aaa1673f508103e11304833e0456e4149f880065 diff --git a/tests/ef_tests/src/bls_setting.rs b/tests/ef_tests/src/bls_setting.rs index 79990c8eec..add7d8b7bd 100644 --- a/tests/ef_tests/src/bls_setting.rs +++ b/tests/ef_tests/src/bls_setting.rs @@ -2,7 +2,6 @@ use self::BlsSetting::*; use crate::error::Error; use serde_repr::Deserialize_repr; -// TODO: use this in every test case #[derive(Deserialize_repr, Debug, Clone, Copy)] #[repr(u8)] pub enum BlsSetting { diff --git a/tests/ef_tests/src/case_result.rs b/tests/ef_tests/src/case_result.rs index 88fd353a14..9df60f402c 100644 --- a/tests/ef_tests/src/case_result.rs +++ b/tests/ef_tests/src/case_result.rs @@ -1,6 +1,7 @@ use super::*; use compare_fields::{CompareFields, Comparison, FieldComparison}; use std::fmt::Debug; +use std::path::{Path, PathBuf}; use types::BeaconState; pub const MAX_VALUE_STRING_LEN: usize = 500; @@ -9,14 +10,21 @@ pub const MAX_VALUE_STRING_LEN: usize = 500; pub struct CaseResult { pub case_index: usize, pub desc: String, + pub path: PathBuf, pub result: Result<(), Error>, } impl CaseResult { - pub fn new(case_index: usize, case: &impl Case, result: Result<(), Error>) -> Self { + pub fn new( + case_index: usize, + path: &Path, + case: &impl Case, + result: Result<(), Error>, + ) -> Self { CaseResult { case_index, desc: case.description(), + path: path.into(), result, } } diff --git a/tests/ef_tests/src/cases.rs b/tests/ef_tests/src/cases.rs index 1ae4ea1d8d..c5b0d8c4f1 100644 --- a/tests/ef_tests/src/cases.rs +++ b/tests/ef_tests/src/cases.rs @@ -1,5 +1,7 @@ use super::*; +use rayon::prelude::*; use std::fmt::Debug; +use std::path::{Path, PathBuf}; mod bls_aggregate_pubkeys; mod bls_aggregate_sigs; @@ -7,20 +9,11 @@ mod bls_g2_compressed; mod bls_g2_uncompressed; mod bls_priv_to_pub; mod bls_sign_msg; -mod epoch_processing_crosslinks; -mod epoch_processing_final_updates; -mod epoch_processing_justification_and_finalization; -mod epoch_processing_registry_updates; -mod epoch_processing_slashings; +mod common; +mod epoch_processing; mod genesis_initialization; mod genesis_validity; -mod operations_attestation; -mod operations_attester_slashing; -mod operations_block_header; -mod operations_deposit; -mod operations_exit; -mod operations_proposer_slashing; -mod operations_transfer; +mod operations; mod sanity_blocks; mod sanity_slots; mod shuffling; @@ -33,27 +26,23 @@ pub use bls_g2_compressed::*; pub use bls_g2_uncompressed::*; pub use bls_priv_to_pub::*; pub use bls_sign_msg::*; -pub use epoch_processing_crosslinks::*; -pub use epoch_processing_final_updates::*; -pub use epoch_processing_justification_and_finalization::*; -pub use epoch_processing_registry_updates::*; -pub use epoch_processing_slashings::*; +pub use common::SszStaticType; +pub use epoch_processing::*; pub use genesis_initialization::*; pub use genesis_validity::*; -pub use operations_attestation::*; -pub use operations_attester_slashing::*; -pub use operations_block_header::*; -pub use operations_deposit::*; -pub use operations_exit::*; -pub use operations_proposer_slashing::*; -pub use operations_transfer::*; +pub use operations::*; pub use sanity_blocks::*; pub use sanity_slots::*; pub use shuffling::*; pub use ssz_generic::*; pub use ssz_static::*; -pub trait Case: Debug { +pub trait LoadCase: Sized { + /// Load the test case from a test case directory. + fn load_from_dir(_path: &Path) -> Result; +} + +pub trait Case: Debug + Sync { /// An optional field for implementing a custom description. /// /// Defaults to "no description". @@ -70,51 +59,15 @@ pub trait Case: Debug { #[derive(Debug)] pub struct Cases { - pub test_cases: Vec, + pub test_cases: Vec<(PathBuf, T)>, } -impl EfTest for Cases -where - T: Case + Debug, -{ - fn test_results(&self) -> Vec { +impl Cases { + pub fn test_results(&self) -> Vec { self.test_cases - .iter() + .into_par_iter() .enumerate() - .map(|(i, tc)| CaseResult::new(i, tc, tc.result(i))) + .map(|(i, (ref path, ref tc))| CaseResult::new(i, path, tc, tc.result(i))) .collect() } } - -impl YamlDecode for Cases { - /// Decodes a YAML list of test cases - fn yaml_decode(yaml: &str) -> Result { - let mut p = 0; - let mut elems: Vec<&str> = yaml - .match_indices("\n- ") - // Skip the `\n` used for matching a new line - .map(|(i, _)| i + 1) - .map(|i| { - let yaml_element = &yaml[p..i]; - p = i; - - yaml_element - }) - .collect(); - - elems.push(&yaml[p..]); - - let test_cases = elems - .iter() - .map(|s| { - // Remove the `- ` prefix. - let s = &s[2..]; - // Remove a single level of indenting. - s.replace("\n ", "\n") - }) - .map(|s| T::yaml_decode(&s.to_string()).unwrap()) - .collect(); - - Ok(Self { test_cases }) - } -} diff --git a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs index 6e38743f2f..13c2fea179 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_pubkeys.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregatePublicKey, PublicKey}; use serde_derive::Deserialize; @@ -9,11 +10,7 @@ pub struct BlsAggregatePubkeys { pub output: String, } -impl YamlDecode for BlsAggregatePubkeys { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregatePubkeys {} impl Case for BlsAggregatePubkeys { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs index eeecab82cd..22fa197df6 100644 --- a/tests/ef_tests/src/cases/bls_aggregate_sigs.rs +++ b/tests/ef_tests/src/cases/bls_aggregate_sigs.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{AggregateSignature, Signature}; use serde_derive::Deserialize; @@ -9,11 +10,7 @@ pub struct BlsAggregateSigs { pub output: String, } -impl YamlDecode for BlsAggregateSigs { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsAggregateSigs {} impl Case for BlsAggregateSigs { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_g2_compressed.rs b/tests/ef_tests/src/cases/bls_g2_compressed.rs index 185cb58f35..1a9f1d5617 100644 --- a/tests/ef_tests/src/cases/bls_g2_compressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_compressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{compress_g2, hash_on_g2}; use serde_derive::Deserialize; @@ -15,11 +16,7 @@ pub struct BlsG2Compressed { pub output: Vec, } -impl YamlDecode for BlsG2Compressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsG2Compressed {} impl Case for BlsG2Compressed { fn result(&self, _case_index: usize) -> Result<(), Error> { @@ -45,14 +42,9 @@ impl Case for BlsG2Compressed { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } diff --git a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs index 962b6aac39..3eae29967b 100644 --- a/tests/ef_tests/src/cases/bls_g2_uncompressed.rs +++ b/tests/ef_tests/src/cases/bls_g2_uncompressed.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::hash_on_g2; use serde_derive::Deserialize; @@ -9,18 +10,14 @@ pub struct BlsG2UncompressedInput { pub domain: String, } +impl BlsCase for BlsG2UncompressedInput {} + #[derive(Debug, Clone, Deserialize)] pub struct BlsG2Uncompressed { pub input: BlsG2UncompressedInput, pub output: Vec>, } -impl YamlDecode for BlsG2Uncompressed { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - impl Case for BlsG2Uncompressed { fn result(&self, _case_index: usize) -> Result<(), Error> { // Convert message and domain to required types diff --git a/tests/ef_tests/src/cases/bls_priv_to_pub.rs b/tests/ef_tests/src/cases/bls_priv_to_pub.rs index d72a43bbbc..016e04dd14 100644 --- a/tests/ef_tests/src/cases/bls_priv_to_pub.rs +++ b/tests/ef_tests/src/cases/bls_priv_to_pub.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{PublicKey, SecretKey}; use serde_derive::Deserialize; @@ -9,11 +10,7 @@ pub struct BlsPrivToPub { pub output: String, } -impl YamlDecode for BlsPrivToPub { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsPrivToPub {} impl Case for BlsPrivToPub { fn result(&self, _case_index: usize) -> Result<(), Error> { diff --git a/tests/ef_tests/src/cases/bls_sign_msg.rs b/tests/ef_tests/src/cases/bls_sign_msg.rs index e62c3550fa..7ee109f812 100644 --- a/tests/ef_tests/src/cases/bls_sign_msg.rs +++ b/tests/ef_tests/src/cases/bls_sign_msg.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::BlsCase; use bls::{SecretKey, Signature}; use serde_derive::Deserialize; @@ -16,11 +17,7 @@ pub struct BlsSign { pub output: String, } -impl YamlDecode for BlsSign { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} +impl BlsCase for BlsSign {} impl Case for BlsSign { fn result(&self, _case_index: usize) -> Result<(), Error> { @@ -45,16 +42,11 @@ impl Case for BlsSign { } } -// Converts a vector to u64 (from big endian) +// Converts a vector to u64 (from little endian) fn bytes_to_u64(array: &[u8]) -> u64 { - let mut result: u64 = 0; - for (i, value) in array.iter().rev().enumerate() { - if i == 8 { - break; - } - result += u64::pow(2, i as u32 * 8) * u64::from(*value); - } - result + let mut bytes = [0u8; 8]; + bytes.copy_from_slice(array); + u64::from_le_bytes(bytes) } // Increase the size of an array to 48 bytes diff --git a/tests/ef_tests/src/cases/common.rs b/tests/ef_tests/src/cases/common.rs new file mode 100644 index 0000000000..8e787f157c --- /dev/null +++ b/tests/ef_tests/src/cases/common.rs @@ -0,0 +1,72 @@ +use crate::cases::LoadCase; +use crate::decode::yaml_decode_file; +use crate::error::Error; +use serde_derive::Deserialize; +use ssz::{Decode, Encode}; +use ssz_derive::{Decode, Encode}; +use std::convert::TryFrom; +use std::fmt::Debug; +use std::path::Path; +use tree_hash::TreeHash; + +/// Trait for all BLS cases to eliminate some boilerplate. +pub trait BlsCase: serde::de::DeserializeOwned {} + +impl LoadCase for T { + fn load_from_dir(path: &Path) -> Result { + yaml_decode_file(&path.join("data.yaml")) + } +} + +/// Macro to wrap U128 and U256 so they deserialize correctly. +macro_rules! uint_wrapper { + ($wrapper_name:ident, $wrapped_type:ty) => { + #[derive(Debug, Clone, Copy, Default, PartialEq, Decode, Encode, Deserialize)] + #[serde(try_from = "String")] + pub struct $wrapper_name { + pub x: $wrapped_type, + } + + impl TryFrom for $wrapper_name { + type Error = String; + + fn try_from(s: String) -> Result { + <$wrapped_type>::from_dec_str(&s) + .map(|x| Self { x }) + .map_err(|e| format!("{:?}", e)) + } + } + + impl tree_hash::TreeHash for $wrapper_name { + fn tree_hash_type() -> tree_hash::TreeHashType { + <$wrapped_type>::tree_hash_type() + } + + fn tree_hash_packed_encoding(&self) -> Vec { + self.x.tree_hash_packed_encoding() + } + + fn tree_hash_packing_factor() -> usize { + <$wrapped_type>::tree_hash_packing_factor() + } + + fn tree_hash_root(&self) -> Vec { + self.x.tree_hash_root() + } + } + }; +} + +uint_wrapper!(TestU128, ethereum_types::U128); +uint_wrapper!(TestU256, ethereum_types::U256); + +/// Trait alias for all deez bounds +pub trait SszStaticType: + serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} + +impl SszStaticType for T where + T: serde::de::DeserializeOwned + Decode + Encode + TreeHash + Clone + PartialEq + Debug + Sync +{ +} diff --git a/tests/ef_tests/src/cases/epoch_processing.rs b/tests/ef_tests/src/cases/epoch_processing.rs new file mode 100644 index 0000000000..ece69b3fe2 --- /dev/null +++ b/tests/ef_tests/src/cases/epoch_processing.rs @@ -0,0 +1,143 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; +use crate::type_name; +use crate::type_name::TypeName; +use serde_derive::Deserialize; +use state_processing::per_epoch_processing::{ + errors::EpochProcessingError, process_crosslinks, process_final_updates, + process_justification_and_finalization, process_registry_updates, process_slashings, + validator_statuses::ValidatorStatuses, +}; +use std::marker::PhantomData; +use std::path::{Path, PathBuf}; +use types::{BeaconState, ChainSpec, EthSpec}; + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct EpochProcessing> { + pub path: PathBuf, + pub metadata: Metadata, + pub pre: BeaconState, + pub post: Option>, + #[serde(skip_deserializing)] + _phantom: PhantomData, +} + +pub trait EpochTransition: TypeName + Debug + Sync { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError>; +} + +#[derive(Debug)] +pub struct JustificationAndFinalization; +#[derive(Debug)] +pub struct Crosslinks; +#[derive(Debug)] +pub struct RegistryUpdates; +#[derive(Debug)] +pub struct Slashings; +#[derive(Debug)] +pub struct FinalUpdates; + +type_name!( + JustificationAndFinalization, + "justification_and_finalization" +); +type_name!(Crosslinks, "crosslinks"); +type_name!(RegistryUpdates, "registry_updates"); +type_name!(Slashings, "slashings"); +type_name!(FinalUpdates, "final_updates"); + +impl EpochTransition for JustificationAndFinalization { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(state, spec)?; + validator_statuses.process_attestations(state, spec)?; + process_justification_and_finalization(state, &validator_statuses.total_balances) + } +} + +impl EpochTransition for Crosslinks { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_crosslinks(state, spec)?; + Ok(()) + } +} + +impl EpochTransition for RegistryUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_registry_updates(state, spec) + } +} + +impl EpochTransition for Slashings { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; + validator_statuses.process_attestations(&state, spec)?; + process_slashings(state, validator_statuses.total_balances.current_epoch, spec)?; + Ok(()) + } +} + +impl EpochTransition for FinalUpdates { + fn run(state: &mut BeaconState, spec: &ChainSpec) -> Result<(), EpochProcessingError> { + process_final_updates(state, spec) + } +} + +impl> LoadCase for EpochProcessing { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + path: path.into(), + metadata, + pre, + post, + _phantom: PhantomData, + }) + } +} + +impl> Case for EpochProcessing { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + let spec = &E::default_spec(); + + let mut result = (|| { + // Processing requires the epoch cache. + state.build_all_caches(spec)?; + + T::run(&mut state, spec).map(|_| state) + })(); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs b/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs deleted file mode 100644 index f2676d1220..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_crosslinks.rs +++ /dev/null @@ -1,37 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_crosslinks; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingCrosslinks { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingCrosslinks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingCrosslinks { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let mut result = process_crosslinks(&mut state, &E::default_spec()).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs b/tests/ef_tests/src/cases/epoch_processing_final_updates.rs deleted file mode 100644 index 69e6b8bd35..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_final_updates.rs +++ /dev/null @@ -1,41 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::process_final_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingFinalUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingFinalUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingFinalUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - process_final_updates(&mut state, spec).map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs b/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs deleted file mode 100644 index 7883010865..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_justification_and_finalization.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_justification_and_finalization, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingJustificationAndFinalization { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingJustificationAndFinalization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingJustificationAndFinalization { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = (|| { - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_justification_and_finalization(&mut state, &validator_statuses.total_balances) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs b/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs deleted file mode 100644 index a01f895fe1..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_registry_updates.rs +++ /dev/null @@ -1,38 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::registry_updates::process_registry_updates; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingRegistryUpdates { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingRegistryUpdates { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingRegistryUpdates { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - let spec = &E::default_spec(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = process_registry_updates(&mut state, spec).map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/epoch_processing_slashings.rs b/tests/ef_tests/src/cases/epoch_processing_slashings.rs deleted file mode 100644 index d2a988d92e..0000000000 --- a/tests/ef_tests/src/cases/epoch_processing_slashings.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::*; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_epoch_processing::{ - process_slashings::process_slashings, validator_statuses::ValidatorStatuses, -}; -use types::{BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct EpochProcessingSlashings { - pub description: String, - pub pre: BeaconState, - pub post: Option>, -} - -impl YamlDecode for EpochProcessingSlashings { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for EpochProcessingSlashings { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - let spec = &E::default_spec(); - - let mut result = (|| { - // Processing requires the epoch cache. - state.build_all_caches(spec)?; - - let mut validator_statuses = ValidatorStatuses::new(&state, spec)?; - validator_statuses.process_attestations(&state, spec)?; - process_slashings( - &mut state, - validator_statuses.total_balances.current_epoch, - spec, - ) - .map(|_| state) - })(); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/genesis_initialization.rs b/tests/ef_tests/src/cases/genesis_initialization.rs index 7ae8eef59d..0fb64ccb37 100644 --- a/tests/ef_tests/src/cases/genesis_initialization.rs +++ b/tests/ef_tests/src/cases/genesis_initialization.rs @@ -1,34 +1,51 @@ use super::*; -use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::initialize_beacon_state_from_eth1; +use std::path::PathBuf; use types::{BeaconState, Deposit, EthSpec, Hash256}; +#[derive(Debug, Clone, Deserialize)] +struct Metadata { + deposits_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisInitialization { - pub description: String, - pub bls_setting: Option, + pub path: PathBuf, pub eth1_block_hash: Hash256, pub eth1_timestamp: u64, pub deposits: Vec, pub state: Option>, } -impl YamlDecode for GenesisInitialization { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisInitialization { + fn load_from_dir(path: &Path) -> Result { + let eth1_block_hash = ssz_decode_file(&path.join("eth1_block_hash.ssz"))?; + let eth1_timestamp = yaml_decode_file(&path.join("eth1_timestamp.yaml"))?; + let meta: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let deposits: Vec = (0..meta.deposits_count) + .map(|i| { + let filename = format!("deposits_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let state = ssz_decode_file(&path.join("state.ssz"))?; + + Ok(Self { + path: path.into(), + eth1_block_hash, + eth1_timestamp, + deposits, + state: Some(state), + }) } } impl Case for GenesisInitialization { - fn description(&self) -> String { - self.description.clone() - } - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let mut result = initialize_beacon_state_from_eth1( diff --git a/tests/ef_tests/src/cases/genesis_validity.rs b/tests/ef_tests/src/cases/genesis_validity.rs index 7ddd3e8fd5..f72ac4c3e6 100644 --- a/tests/ef_tests/src/cases/genesis_validity.rs +++ b/tests/ef_tests/src/cases/genesis_validity.rs @@ -1,31 +1,28 @@ use super::*; -use crate::bls_setting::BlsSetting; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::is_valid_genesis_state; +use std::path::Path; use types::{BeaconState, EthSpec}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct GenesisValidity { - pub description: String, - pub bls_setting: Option, pub genesis: BeaconState, pub is_valid: bool, } -impl YamlDecode for GenesisValidity { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for GenesisValidity { + fn load_from_dir(path: &Path) -> Result { + let genesis = ssz_decode_file(&path.join("genesis.ssz"))?; + let is_valid = yaml_decode_file(&path.join("is_valid.yaml"))?; + + Ok(Self { genesis, is_valid }) } } impl Case for GenesisValidity { - fn description(&self) -> String { - self.description.clone() - } - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; let spec = &E::default_spec(); let is_valid = is_valid_genesis_state(&self.genesis, spec); diff --git a/tests/ef_tests/src/cases/operations.rs b/tests/ef_tests/src/cases/operations.rs new file mode 100644 index 0000000000..3da04d2a33 --- /dev/null +++ b/tests/ef_tests/src/cases/operations.rs @@ -0,0 +1,194 @@ +use super::*; +use crate::bls_setting::BlsSetting; +use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; +use crate::type_name::TypeName; +use serde_derive::Deserialize; +use ssz::Decode; +use state_processing::per_block_processing::{ + errors::BlockProcessingError, process_attestations, process_attester_slashings, + process_block_header, process_deposits, process_exits, process_proposer_slashings, + process_transfers, VerifySignatures, +}; +use std::fmt::Debug; +use std::path::Path; +use types::{ + Attestation, AttesterSlashing, BeaconBlock, BeaconState, ChainSpec, Deposit, EthSpec, + ProposerSlashing, Transfer, VoluntaryExit, +}; + +#[derive(Debug, Clone, Default, Deserialize)] +struct Metadata { + description: Option, + bls_setting: Option, +} + +#[derive(Debug, Clone)] +pub struct Operations> { + metadata: Metadata, + pub pre: BeaconState, + pub operation: O, + pub post: Option>, +} + +pub trait Operation: Decode + TypeName + Debug + Sync { + fn handler_name() -> String { + Self::name().to_lowercase() + } + + fn filename() -> String { + format!("{}.ssz", Self::handler_name()) + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError>; +} + +impl Operation for Attestation { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attestations(state, &[self.clone()], VerifySignatures::True, spec) + } +} + +impl Operation for AttesterSlashing { + fn handler_name() -> String { + "attester_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_attester_slashings(state, &[self.clone()], VerifySignatures::True, spec) + } +} + +impl Operation for Deposit { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_deposits(state, &[self.clone()], spec) + } +} + +impl Operation for ProposerSlashing { + fn handler_name() -> String { + "proposer_slashing".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_proposer_slashings(state, &[self.clone()], VerifySignatures::True, spec) + } +} + +impl Operation for Transfer { + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_transfers(state, &[self.clone()], VerifySignatures::True, spec) + } +} + +impl Operation for VoluntaryExit { + fn handler_name() -> String { + "voluntary_exit".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + process_exits(state, &[self.clone()], VerifySignatures::True, spec) + } +} + +impl Operation for BeaconBlock { + fn handler_name() -> String { + "block_header".into() + } + + fn filename() -> String { + "block.ssz".into() + } + + fn apply_to( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), BlockProcessingError> { + Ok(process_block_header( + state, + self, + None, + VerifySignatures::True, + spec, + )?) + } +} + +impl> LoadCase for Operations { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let operation = ssz_decode_file(&path.join(O::filename()))?; + let post_filename = path.join("post.ssz"); + let post = if post_filename.is_file() { + Some(ssz_decode_file(&post_filename)?) + } else { + None + }; + + Ok(Self { + metadata, + pre, + operation, + post, + }) + } +} + +impl> Case for Operations { + fn description(&self) -> String { + self.metadata + .description + .clone() + .unwrap_or_else(String::new) + } + + fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + + let spec = &E::default_spec(); + let mut state = self.pre.clone(); + let mut expected = self.post.clone(); + + // Processing requires the epoch cache. + state.build_all_caches(spec).unwrap(); + + let mut result = self.operation.apply_to(&mut state, spec).map(|()| state); + + compare_beacon_state_results_without_caches(&mut result, &mut expected) + } +} diff --git a/tests/ef_tests/src/cases/operations_attestation.rs b/tests/ef_tests/src/cases/operations_attestation.rs deleted file mode 100644 index ecd4835b89..0000000000 --- a/tests/ef_tests/src/cases/operations_attestation.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_attestations, VerifySignatures}; -use types::{Attestation, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsAttestation { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub attestation: Attestation, - pub post: Option>, -} - -impl YamlDecode for OperationsAttestation { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(&yaml).unwrap()) - } -} - -impl Case for OperationsAttestation { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attestation = self.attestation.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let result = process_attestations(&mut state, &[attestation], VerifySignatures::True, spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_attester_slashing.rs b/tests/ef_tests/src/cases/operations_attester_slashing.rs deleted file mode 100644 index 952443cee6..0000000000 --- a/tests/ef_tests/src/cases/operations_attester_slashing.rs +++ /dev/null @@ -1,52 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_attester_slashings, VerifySignatures}; -use types::{AttesterSlashing, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -pub struct OperationsAttesterSlashing { - pub description: String, - pub bls_setting: Option, - #[serde(bound = "E: EthSpec")] - pub pre: BeaconState, - #[serde(bound = "E: EthSpec")] - pub attester_slashing: AttesterSlashing, - #[serde(bound = "E: EthSpec")] - pub post: Option>, -} - -impl YamlDecode for OperationsAttesterSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsAttesterSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let attester_slashing = self.attester_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = process_attester_slashings( - &mut state, - &[attester_slashing], - VerifySignatures::True, - &E::default_spec(), - ); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_block_header.rs b/tests/ef_tests/src/cases/operations_block_header.rs deleted file mode 100644 index f9b9dab1db..0000000000 --- a/tests/ef_tests/src/cases/operations_block_header.rs +++ /dev/null @@ -1,46 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_block_header, VerifySignatures}; -use types::{BeaconBlock, BeaconState, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsBlockHeader { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub block: BeaconBlock, - pub post: Option>, -} - -impl YamlDecode for OperationsBlockHeader { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsBlockHeader { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - let spec = &E::default_spec(); - - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(spec).unwrap(); - - let mut result = - process_block_header(&mut state, &self.block, None, VerifySignatures::True, spec) - .map(|_| state); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_deposit.rs b/tests/ef_tests/src/cases/operations_deposit.rs deleted file mode 100644 index 801c020298..0000000000 --- a/tests/ef_tests/src/cases/operations_deposit.rs +++ /dev/null @@ -1,42 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::process_deposits; -use types::{BeaconState, Deposit, EthSpec}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsDeposit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub deposit: Deposit, - pub post: Option>, -} - -impl YamlDecode for OperationsDeposit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsDeposit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let deposit = self.deposit.clone(); - let mut expected = self.post.clone(); - - let result = process_deposits(&mut state, &[deposit], &E::default_spec()); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_exit.rs b/tests/ef_tests/src/cases/operations_exit.rs deleted file mode 100644 index 6040e7ef34..0000000000 --- a/tests/ef_tests/src/cases/operations_exit.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_exits, VerifySignatures}; -use types::{BeaconState, EthSpec, VoluntaryExit}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsExit { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub voluntary_exit: VoluntaryExit, - pub post: Option>, -} - -impl YamlDecode for OperationsExit { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsExit { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let exit = self.voluntary_exit.clone(); - let mut expected = self.post.clone(); - - // Exit processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = process_exits( - &mut state, - &[exit], - VerifySignatures::True, - &E::default_spec(), - ); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_proposer_slashing.rs b/tests/ef_tests/src/cases/operations_proposer_slashing.rs deleted file mode 100644 index 282d932749..0000000000 --- a/tests/ef_tests/src/cases/operations_proposer_slashing.rs +++ /dev/null @@ -1,50 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_proposer_slashings, VerifySignatures}; -use types::{BeaconState, EthSpec, ProposerSlashing}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsProposerSlashing { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub proposer_slashing: ProposerSlashing, - pub post: Option>, -} - -impl YamlDecode for OperationsProposerSlashing { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsProposerSlashing { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let proposer_slashing = self.proposer_slashing.clone(); - let mut expected = self.post.clone(); - - // Processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let result = process_proposer_slashings( - &mut state, - &[proposer_slashing], - VerifySignatures::True, - &E::default_spec(), - ); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/operations_transfer.rs b/tests/ef_tests/src/cases/operations_transfer.rs deleted file mode 100644 index 77069b5cfc..0000000000 --- a/tests/ef_tests/src/cases/operations_transfer.rs +++ /dev/null @@ -1,47 +0,0 @@ -use super::*; -use crate::bls_setting::BlsSetting; -use crate::case_result::compare_beacon_state_results_without_caches; -use serde_derive::Deserialize; -use state_processing::per_block_processing::{process_transfers, VerifySignatures}; -use types::{BeaconState, EthSpec, Transfer}; - -#[derive(Debug, Clone, Deserialize)] -#[serde(bound = "E: EthSpec")] -pub struct OperationsTransfer { - pub description: String, - pub bls_setting: Option, - pub pre: BeaconState, - pub transfer: Transfer, - pub post: Option>, -} - -impl YamlDecode for OperationsTransfer { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) - } -} - -impl Case for OperationsTransfer { - fn description(&self) -> String { - self.description.clone() - } - - fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; - - let mut state = self.pre.clone(); - let transfer = self.transfer.clone(); - let mut expected = self.post.clone(); - - // Transfer processing requires the epoch cache. - state.build_all_caches(&E::default_spec()).unwrap(); - - let spec = E::default_spec(); - - let result = process_transfers(&mut state, &[transfer], VerifySignatures::True, &spec); - - let mut result = result.and_then(|_| Ok(state)); - - compare_beacon_state_results_without_caches(&mut result, &mut expected) - } -} diff --git a/tests/ef_tests/src/cases/sanity_blocks.rs b/tests/ef_tests/src/cases/sanity_blocks.rs index bc4d7b3de0..24ae6f81c3 100644 --- a/tests/ef_tests/src/cases/sanity_blocks.rs +++ b/tests/ef_tests/src/cases/sanity_blocks.rs @@ -1,35 +1,65 @@ use super::*; use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy, }; use types::{BeaconBlock, BeaconState, EthSpec, RelativeEpoch}; +#[derive(Debug, Clone, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, + pub blocks_count: usize, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanityBlocks { - pub description: String, - pub bls_setting: Option, + pub metadata: Metadata, pub pre: BeaconState, pub blocks: Vec>, pub post: Option>, } -impl YamlDecode for SanityBlocks { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanityBlocks { + fn load_from_dir(path: &Path) -> Result { + let metadata: Metadata = yaml_decode_file(&path.join("meta.yaml"))?; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let blocks: Vec> = (0..metadata.blocks_count) + .map(|i| { + let filename = format!("blocks_{}.ssz", i); + ssz_decode_file(&path.join(filename)) + }) + .collect::>()?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + metadata, + pre, + blocks, + post, + }) } } impl Case for SanityBlocks { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) } fn result(&self, _case_index: usize) -> Result<(), Error> { - self.bls_setting.unwrap_or_default().check()?; + self.metadata.bls_setting.unwrap_or_default().check()?; let mut state = self.pre.clone(); let mut expected = self.post.clone(); diff --git a/tests/ef_tests/src/cases/sanity_slots.rs b/tests/ef_tests/src/cases/sanity_slots.rs index fbce1a06af..e9b80a252d 100644 --- a/tests/ef_tests/src/cases/sanity_slots.rs +++ b/tests/ef_tests/src/cases/sanity_slots.rs @@ -1,30 +1,63 @@ use super::*; +use crate::bls_setting::BlsSetting; use crate::case_result::compare_beacon_state_results_without_caches; +use crate::decode::{ssz_decode_file, yaml_decode_file}; use serde_derive::Deserialize; use state_processing::per_slot_processing; use types::{BeaconState, EthSpec}; +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Metadata { + pub description: Option, + pub bls_setting: Option, +} + #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec")] pub struct SanitySlots { - pub description: String, + pub metadata: Metadata, pub pre: BeaconState, - pub slots: usize, + pub slots: u64, pub post: Option>, } -impl YamlDecode for SanitySlots { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for SanitySlots { + fn load_from_dir(path: &Path) -> Result { + let metadata_path = path.join("meta.yaml"); + let metadata: Metadata = if metadata_path.is_file() { + yaml_decode_file(&metadata_path)? + } else { + Metadata::default() + }; + let pre = ssz_decode_file(&path.join("pre.ssz"))?; + let slots: u64 = yaml_decode_file(&path.join("slots.yaml"))?; + let post_file = path.join("post.ssz"); + let post = if post_file.is_file() { + Some(ssz_decode_file(&post_file)?) + } else { + None + }; + + Ok(Self { + metadata, + pre, + slots, + post, + }) } } impl Case for SanitySlots { fn description(&self) -> String { - self.description.clone() + self.metadata + .description + .clone() + .unwrap_or_else(String::new) } fn result(&self, _case_index: usize) -> Result<(), Error> { + self.metadata.bls_setting.unwrap_or_default().check()?; + let mut state = self.pre.clone(); let mut expected = self.post.clone(); let spec = &E::default_spec(); diff --git a/tests/ef_tests/src/cases/shuffling.rs b/tests/ef_tests/src/cases/shuffling.rs index d7ff40e596..2fe632e84d 100644 --- a/tests/ef_tests/src/cases/shuffling.rs +++ b/tests/ef_tests/src/cases/shuffling.rs @@ -1,5 +1,6 @@ use super::*; use crate::case_result::compare_result; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; use std::marker::PhantomData; use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; @@ -8,21 +9,21 @@ use swap_or_not_shuffle::{get_permutated_index, shuffle_list}; pub struct Shuffling { pub seed: String, pub count: usize, - pub shuffled: Vec, + pub mapping: Vec, #[serde(skip)] _phantom: PhantomData, } -impl YamlDecode for Shuffling { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +impl LoadCase for Shuffling { + fn load_from_dir(path: &Path) -> Result { + yaml_decode_file(&path.join("mapping.yaml")) } } impl Case for Shuffling { fn result(&self, _case_index: usize) -> Result<(), Error> { if self.count == 0 { - compare_result::<_, Error>(&Ok(vec![]), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(vec![]), &Some(self.mapping.clone()))?; } else { let spec = T::default_spec(); let seed = hex::decode(&self.seed[2..]) @@ -34,12 +35,12 @@ impl Case for Shuffling { get_permutated_index(i, self.count, &seed, spec.shuffle_round_count).unwrap() }) .collect(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; // Test "shuffle_list" let input: Vec = (0..self.count).collect(); let shuffling = shuffle_list(input, spec.shuffle_round_count, &seed, false).unwrap(); - compare_result::<_, Error>(&Ok(shuffling), &Some(self.shuffled.clone()))?; + compare_result::<_, Error>(&Ok(shuffling), &Some(self.mapping.clone()))?; } Ok(()) diff --git a/tests/ef_tests/src/cases/ssz_generic.rs b/tests/ef_tests/src/cases/ssz_generic.rs index ca49d21060..fc62e66fc0 100644 --- a/tests/ef_tests/src/cases/ssz_generic.rs +++ b/tests/ef_tests/src/cases/ssz_generic.rs @@ -1,68 +1,302 @@ +#![allow(non_snake_case)] + use super::*; -use crate::case_result::compare_result; -use ethereum_types::{U128, U256}; +use crate::cases::common::{SszStaticType, TestU128, TestU256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash}; +use crate::decode::yaml_decode_file; +use serde::{de::Error as SerdeError, Deserializer}; use serde_derive::Deserialize; -use ssz::Decode; -use std::fmt::Debug; +use ssz_derive::{Decode, Encode}; +use std::fs; +use std::path::{Path, PathBuf}; +use tree_hash_derive::TreeHash; +use types::typenum::*; +use types::{BitList, BitVector, FixedVector, VariableList}; #[derive(Debug, Clone, Deserialize)] -pub struct SszGeneric { - #[serde(alias = "type")] - pub type_name: String, - pub valid: bool, - pub value: Option, - pub ssz: Option, +struct Metadata { + root: String, + signing_root: Option, } -impl YamlDecode for SszGeneric { - fn yaml_decode(yaml: &str) -> Result { - Ok(serde_yaml::from_str(yaml).unwrap()) +#[derive(Debug, Clone)] +pub struct SszGeneric { + path: PathBuf, + handler_name: String, + case_name: String, +} + +impl LoadCase for SszGeneric { + fn load_from_dir(path: &Path) -> Result { + let components = path + .components() + .map(|c| c.as_os_str().to_string_lossy().into_owned()) + .rev() + .collect::>(); + // Test case name is last + let case_name = components[0].clone(); + // Handler name is third last, before suite name and case name + let handler_name = components[2].clone(); + Ok(Self { + path: path.into(), + handler_name, + case_name, + }) + } +} + +macro_rules! type_dispatch { + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => primitive_type ] $($rest:tt)*) => { + match $value { + "bool" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* bool>, $($rest)*), + "uint8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u8>, $($rest)*), + "uint16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u16>, $($rest)*), + "uint32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u32>, $($rest)*), + "uint64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* u64>, $($rest)*), + "uint128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU128>, $($rest)*), + "uint256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* TestU256>, $($rest)*), + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => typenum ] $($rest:tt)*) => { + match $value { + // DO YOU LIKE NUMBERS? + "0" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U0>, $($rest)*), + "1" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1>, $($rest)*), + "2" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2>, $($rest)*), + "3" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U3>, $($rest)*), + "4" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4>, $($rest)*), + "5" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U5>, $($rest)*), + "6" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U6>, $($rest)*), + "7" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U7>, $($rest)*), + "8" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8>, $($rest)*), + "9" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U9>, $($rest)*), + "16" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U16>, $($rest)*), + "31" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U31>, $($rest)*), + "32" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U32>, $($rest)*), + "64" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U64>, $($rest)*), + "128" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U128>, $($rest)*), + "256" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U256>, $($rest)*), + "512" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U512>, $($rest)*), + "513" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U513>, $($rest)*), + "1024" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U1024>, $($rest)*), + "2048" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U2048>, $($rest)*), + "4096" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U4096>, $($rest)*), + "8192" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* U8192>, $($rest)*), + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), + } + }; + ($function:ident, + ($($arg:expr),*), + $base_ty:tt, + <$($param_ty:ty),*>, + [ $value:expr => test_container ] $($rest:tt)*) => { + match $value { + "SingleFieldTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SingleFieldTestStruct>, $($rest)*), + "SmallTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* SmallTestStruct>, $($rest)*), + "FixedTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* FixedTestStruct>, $($rest)*), + "VarTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* VarTestStruct>, $($rest)*), + "ComplexTestStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* ComplexTestStruct>, $($rest)*), + "BitsStruct" => type_dispatch!($function, ($($arg),*), $base_ty, <$($param_ty,)* BitsStruct>, $($rest)*), + _ => Err(Error::FailedToParseTest(format!("unsupported: {}", $value))), + } + }; + // No base type: apply type params to function + ($function:ident, ($($arg:expr),*), _, <$($param_ty:ty),*>,) => { + $function::<$($param_ty),*>($($arg),*) + }; + ($function:ident, ($($arg:expr),*), $base_type_name:ident, <$($param_ty:ty),*>,) => { + $function::<$base_type_name<$($param_ty),*>>($($arg),*) } } impl Case for SszGeneric { fn result(&self, _case_index: usize) -> Result<(), Error> { - if let Some(ssz) = &self.ssz { - match self.type_name.as_ref() { - "uint8" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint16" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint32" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint64" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint128" => ssz_generic_test::(self.valid, ssz, &self.value), - "uint256" => ssz_generic_test::(self.valid, ssz, &self.value), - _ => Err(Error::FailedToParseTest(format!( - "Unknown type: {}", - self.type_name - ))), + let parts = self.case_name.split('_').collect::>(); + + match self.handler_name.as_str() { + "basic_vector" => { + let elem_ty = parts[1]; + let length = parts[2]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + FixedVector, + <>, + [elem_ty => primitive_type] + [length => typenum] + )?; } - } else { - // Skip tests that do not have an ssz field. - // - // See: https://github.com/ethereum/eth2.0-specs/issues/1079 - Ok(()) + "bitlist" => { + let mut limit = parts[1]; + + // Test format is inconsistent, pretend the limit is 32 (arbitrary) + // https://github.com/ethereum/eth2.0-spec-tests + if limit == "no" { + limit = "32"; + } + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitList, + <>, + [limit => typenum] + )?; + } + "bitvector" => { + let length = parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + BitVector, + <>, + [length => typenum] + )?; + } + "boolean" => { + ssz_generic_test::(&self.path)?; + } + "uints" => { + let type_name = "uint".to_owned() + parts[1]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name.as_str() => primitive_type] + )?; + } + "containers" => { + let type_name = parts[0]; + + type_dispatch!( + ssz_generic_test, + (&self.path), + _, + <>, + [type_name => test_container] + )?; + } + _ => panic!("unsupported handler: {}", self.handler_name), } + Ok(()) } } -/// Execute a `ssz_generic` test case. -fn ssz_generic_test(should_be_ok: bool, ssz: &str, value: &Option) -> Result<(), Error> -where - T: Decode + YamlDecode + Debug + PartialEq, -{ - let ssz = hex::decode(&ssz[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - - // We do not cater for the scenario where the test is valid but we are not passed any SSZ. - if should_be_ok && value.is_none() { - panic!("Unexpected test input. Cannot pass without value.") - } - - let expected = if let Some(string) = value { - Some(T::yaml_decode(string)?) +fn ssz_generic_test(path: &Path) -> Result<(), Error> { + let meta_path = path.join("meta.yaml"); + let meta: Option = if meta_path.is_file() { + Some(yaml_decode_file(&meta_path)?) } else { None }; - let decoded = T::from_ssz_bytes(&ssz); + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); - compare_result(&decoded, &expected) + let value_path = path.join("value.yaml"); + let value: Option = if value_path.is_file() { + Some(yaml_decode_file(&value_path)?) + } else { + None + }; + + // Valid + // TODO: signing root (annoying because of traits) + if let Some(value) = value { + check_serialization(&value, &serialized)?; + + if let Some(ref meta) = meta { + check_tree_hash(&meta.root, value.tree_hash_root())?; + } + } + // Invalid + else { + if let Ok(decoded) = T::from_ssz_bytes(&serialized) { + return Err(Error::DidntFail(format!( + "Decoded invalid bytes into: {:?}", + decoded + ))); + } + } + + Ok(()) +} + +// Containers for SSZ generic tests +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SingleFieldTestStruct { + A: u8, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct SmallTestStruct { + A: u16, + B: u16, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct FixedTestStruct { + A: u8, + B: u64, + C: u32, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct VarTestStruct { + A: u16, + B: VariableList, + C: u8, +} + +#[derive(Debug, Clone, Default, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct ComplexTestStruct { + A: u16, + B: VariableList, + C: u8, + #[serde(deserialize_with = "byte_list_from_hex_str")] + D: VariableList, + E: VarTestStruct, + F: FixedVector, + G: FixedVector, +} + +#[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] +struct BitsStruct { + A: BitList, + B: BitVector, + C: BitVector, + D: BitList, + E: BitVector, +} + +fn byte_list_from_hex_str<'de, D, N: Unsigned>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: String = serde::de::Deserialize::deserialize(deserializer)?; + let decoded: Vec = hex::decode(&s.as_str()[2..]).map_err(D::Error::custom)?; + + if decoded.len() > N::to_usize() { + return Err(D::Error::custom(format!( + "Too many values for list, got: {}, limit: {}", + decoded.len(), + N::to_usize() + ))); + } else { + Ok(decoded.into()) + } } diff --git a/tests/ef_tests/src/cases/ssz_static.rs b/tests/ef_tests/src/cases/ssz_static.rs index 96ba38b6ad..62f285d580 100644 --- a/tests/ef_tests/src/cases/ssz_static.rs +++ b/tests/ef_tests/src/cases/ssz_static.rs @@ -1,127 +1,101 @@ use super::*; use crate::case_result::compare_result; +use crate::cases::common::SszStaticType; +use crate::decode::yaml_decode_file; use serde_derive::Deserialize; -use ssz::{Decode, Encode}; -use std::fmt::Debug; -use std::marker::PhantomData; -use tree_hash::TreeHash; -use types::{ - test_utils::TestRandom, Attestation, AttestationData, AttestationDataAndCustodyBit, - AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockHeader, BeaconState, Checkpoint, - CompactCommittee, Crosslink, Deposit, DepositData, Eth1Data, EthSpec, Fork, Hash256, - HistoricalBatch, IndexedAttestation, PendingAttestation, ProposerSlashing, Transfer, Validator, - VoluntaryExit, -}; - -// Enum variant names are used by Serde when deserializing the test YAML -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, Deserialize)] -pub enum SszStatic -where - E: EthSpec, -{ - Fork(SszStaticInner), - Crosslink(SszStaticInner), - Checkpoint(SszStaticInner), - CompactCommittee(SszStaticInner, E>), - Eth1Data(SszStaticInner), - AttestationData(SszStaticInner), - AttestationDataAndCustodyBit(SszStaticInner), - IndexedAttestation(SszStaticInner, E>), - DepositData(SszStaticInner), - BeaconBlockHeader(SszStaticInner), - Validator(SszStaticInner), - PendingAttestation(SszStaticInner, E>), - HistoricalBatch(SszStaticInner, E>), - ProposerSlashing(SszStaticInner), - AttesterSlashing(SszStaticInner, E>), - Attestation(SszStaticInner, E>), - Deposit(SszStaticInner), - VoluntaryExit(SszStaticInner), - Transfer(SszStaticInner), - BeaconBlockBody(SszStaticInner, E>), - BeaconBlock(SszStaticInner, E>), - BeaconState(SszStaticInner, E>), -} +use std::fs; +use tree_hash::SignedRoot; +use types::Hash256; #[derive(Debug, Clone, Deserialize)] -pub struct SszStaticInner -where - E: EthSpec, -{ - pub value: T, - pub serialized: String, - pub root: String, - #[serde(skip, default)] - _phantom: PhantomData, +struct SszStaticRoots { + root: String, + signing_root: Option, } -impl YamlDecode for SszStatic { - fn yaml_decode(yaml: &str) -> Result { - serde_yaml::from_str(yaml).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +#[derive(Debug, Clone)] +pub struct SszStatic { + roots: SszStaticRoots, + serialized: Vec, + value: T, +} + +#[derive(Debug, Clone)] +pub struct SszStaticSR { + roots: SszStaticRoots, + serialized: Vec, + value: T, +} + +fn load_from_dir(path: &Path) -> Result<(SszStaticRoots, Vec, T), Error> { + let roots = yaml_decode_file(&path.join("roots.yaml"))?; + let serialized = fs::read(&path.join("serialized.ssz")).expect("serialized.ssz exists"); + let value = yaml_decode_file(&path.join("value.yaml"))?; + + Ok((roots, serialized, value)) +} + +impl LoadCase for SszStatic { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) } } -impl Case for SszStatic { - fn result(&self, _case_index: usize) -> Result<(), Error> { - use self::SszStatic::*; - - match *self { - Fork(ref val) => ssz_static_test(val), - Crosslink(ref val) => ssz_static_test(val), - Checkpoint(ref val) => ssz_static_test(val), - CompactCommittee(ref val) => ssz_static_test(val), - Eth1Data(ref val) => ssz_static_test(val), - AttestationData(ref val) => ssz_static_test(val), - AttestationDataAndCustodyBit(ref val) => ssz_static_test(val), - IndexedAttestation(ref val) => ssz_static_test(val), - DepositData(ref val) => ssz_static_test(val), - BeaconBlockHeader(ref val) => ssz_static_test(val), - Validator(ref val) => ssz_static_test(val), - PendingAttestation(ref val) => ssz_static_test(val), - HistoricalBatch(ref val) => ssz_static_test(val), - ProposerSlashing(ref val) => ssz_static_test(val), - AttesterSlashing(ref val) => ssz_static_test(val), - Attestation(ref val) => ssz_static_test(val), - Deposit(ref val) => ssz_static_test(val), - VoluntaryExit(ref val) => ssz_static_test(val), - Transfer(ref val) => ssz_static_test(val), - BeaconBlockBody(ref val) => ssz_static_test(val), - BeaconBlock(ref val) => ssz_static_test(val), - BeaconState(ref val) => ssz_static_test(val), - } +impl LoadCase for SszStaticSR { + fn load_from_dir(path: &Path) -> Result { + load_from_dir(path).map(|(roots, serialized, value)| Self { + roots, + serialized, + value, + }) } } -fn ssz_static_test(tc: &SszStaticInner) -> Result<(), Error> -where - T: Clone - + Decode - + Debug - + Encode - + PartialEq - + serde::de::DeserializeOwned - + TreeHash - + TestRandom, -{ - // Verify we can decode SSZ in the same way we can decode YAML. - let ssz = hex::decode(&tc.serialized[2..]) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected = tc.value.clone(); - let decode_result = T::from_ssz_bytes(&ssz); - compare_result(&decode_result, &Some(expected))?; +pub fn check_serialization(value: &T, serialized: &[u8]) -> Result<(), Error> { + // Check serialization + let serialized_result = value.as_ssz_bytes(); + compare_result::(&Ok(value.ssz_bytes_len()), &Some(serialized.len()))?; + compare_result::, Error>(&Ok(serialized_result), &Some(serialized.to_vec()))?; - // Verify we can encode the result back into original ssz bytes - let decoded = decode_result.unwrap(); - let encoded_result = decoded.as_ssz_bytes(); - compare_result::, Error>(&Ok(encoded_result), &Some(ssz))?; - - // Verify the TreeHash root of the decoded struct matches the test. - let expected_root = - &hex::decode(&tc.root[2..]).map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; - let expected_root = Hash256::from_slice(&expected_root); - let tree_hash_root = Hash256::from_slice(&decoded.tree_hash_root()); - compare_result::(&Ok(tree_hash_root), &Some(expected_root))?; + // Check deserialization + let deserialized_result = T::from_ssz_bytes(serialized); + compare_result(&deserialized_result, &Some(value.clone()))?; Ok(()) } + +pub fn check_tree_hash(expected_str: &str, actual_root: Vec) -> Result<(), Error> { + let expected_root = hex::decode(&expected_str[2..]) + .map_err(|e| Error::FailedToParseTest(format!("{:?}", e)))?; + let expected_root = Hash256::from_slice(&expected_root); + let tree_hash_root = Hash256::from_slice(&actual_root); + compare_result::(&Ok(tree_hash_root), &Some(expected_root)) +} + +impl Case for SszStatic { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + Ok(()) + } +} + +impl Case for SszStaticSR { + fn result(&self, _case_index: usize) -> Result<(), Error> { + check_serialization(&self.value, &self.serialized)?; + check_tree_hash(&self.roots.root, self.value.tree_hash_root())?; + check_tree_hash( + &self + .roots + .signing_root + .as_ref() + .expect("signed root exists"), + self.value.signed_root(), + )?; + Ok(()) + } +} diff --git a/tests/ef_tests/src/decode.rs b/tests/ef_tests/src/decode.rs new file mode 100644 index 0000000000..c1ea6fb3b9 --- /dev/null +++ b/tests/ef_tests/src/decode.rs @@ -0,0 +1,31 @@ +use super::*; +use std::fs; +use std::path::Path; + +pub fn yaml_decode(string: &str) -> Result { + serde_yaml::from_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) +} + +pub fn yaml_decode_file(path: &Path) -> Result { + fs::read_to_string(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| yaml_decode(&s)) +} + +pub fn ssz_decode_file(path: &Path) -> Result { + fs::read(path) + .map_err(|e| { + Error::FailedToParseTest(format!("Unable to load {}: {:?}", path.display(), e)) + }) + .and_then(|s| { + T::from_ssz_bytes(&s).map_err(|e| { + Error::FailedToParseTest(format!( + "Unable to parse SSZ at {}: {:?}", + path.display(), + e + )) + }) + }) +} diff --git a/tests/ef_tests/src/doc.rs b/tests/ef_tests/src/doc.rs deleted file mode 100644 index 7dfe9954c1..0000000000 --- a/tests/ef_tests/src/doc.rs +++ /dev/null @@ -1,253 +0,0 @@ -use crate::case_result::CaseResult; -use crate::cases::*; -use crate::doc_header::DocHeader; -use crate::error::Error; -use crate::yaml_decode::{yaml_split_header_and_cases, YamlDecode}; -use crate::EfTest; -use serde_derive::Deserialize; -use std::{fs::File, io::prelude::*, path::PathBuf}; -use types::{MainnetEthSpec, MinimalEthSpec}; - -#[derive(Debug, Deserialize)] -pub struct Doc { - pub header_yaml: String, - pub cases_yaml: String, - pub path: PathBuf, -} - -impl Doc { - fn from_path(path: PathBuf) -> Self { - let mut file = File::open(path.clone()).unwrap(); - - let mut yaml = String::new(); - file.read_to_string(&mut yaml).unwrap(); - - let (header_yaml, cases_yaml) = yaml_split_header_and_cases(yaml.clone()); - - Self { - header_yaml, - cases_yaml, - path, - } - } - - pub fn test_results(&self) -> Vec { - let header: DocHeader = serde_yaml::from_str(&self.header_yaml.as_str()).unwrap(); - - match ( - header.runner.as_ref(), - header.handler.as_ref(), - header.config.as_ref(), - ) { - ("ssz", "uint", _) => run_test::(self), - ("ssz", "static", "minimal") => run_test::>(self), - ("ssz", "static", "mainnet") => run_test::>(self), - ("sanity", "slots", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "slots", "mainnet") => vec![], // run_test::>(self), - ("sanity", "blocks", "minimal") => run_test::>(self), - // FIXME: skipped due to compact committees issue - ("sanity", "blocks", "mainnet") => vec![], // run_test::>(self), - ("shuffling", "core", "minimal") => run_test::>(self), - ("shuffling", "core", "mainnet") => run_test::>(self), - ("bls", "aggregate_pubkeys", "mainnet") => run_test::(self), - ("bls", "aggregate_sigs", "mainnet") => run_test::(self), - ("bls", "msg_hash_compressed", "mainnet") => run_test::(self), - // Note this test fails due to a difference in our internal representations. It does - // not effect verification or external representation. - // - // It is skipped. - ("bls", "msg_hash_uncompressed", "mainnet") => vec![], - ("bls", "priv_to_pub", "mainnet") => run_test::(self), - ("bls", "sign_msg", "mainnet") => run_test::(self), - ("operations", "deposit", "mainnet") => { - run_test::>(self) - } - ("operations", "deposit", "minimal") => { - run_test::>(self) - } - ("operations", "transfer", "mainnet") => { - run_test::>(self) - } - ("operations", "transfer", "minimal") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "mainnet") => { - run_test::>(self) - } - ("operations", "voluntary_exit", "minimal") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "proposer_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attester_slashing", "mainnet") => { - run_test::>(self) - } - ("operations", "attester_slashing", "minimal") => { - run_test::>(self) - } - ("operations", "attestation", "mainnet") => { - run_test::>(self) - } - ("operations", "attestation", "minimal") => { - run_test::>(self) - } - ("operations", "block_header", "mainnet") => { - run_test::>(self) - } - ("operations", "block_header", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "crosslinks", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "registry_updates", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "justification_and_finalization", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "slashings", "mainnet") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "minimal") => { - run_test::>(self) - } - ("epoch_processing", "final_updates", "mainnet") => { - vec![] - // FIXME: skipped due to compact committees issue - // run_test::>(self) - } - ("genesis", "initialization", "minimal") => { - run_test::>(self) - } - ("genesis", "initialization", "mainnet") => { - run_test::>(self) - } - ("genesis", "validity", "minimal") => run_test::>(self), - ("genesis", "validity", "mainnet") => run_test::>(self), - (runner, handler, config) => panic!( - "No implementation for runner: \"{}\", handler: \"{}\", config: \"{}\"", - runner, handler, config - ), - } - } - - pub fn assert_tests_pass(path: PathBuf) { - let doc = Self::from_path(path); - let results = doc.test_results(); - - let (failed, skipped_bls, skipped_known_failures) = categorize_results(&results); - - if failed.len() + skipped_known_failures.len() > 0 { - print_results( - &doc, - &failed, - &skipped_bls, - &skipped_known_failures, - &results, - ); - if !failed.is_empty() { - panic!("Tests failed (see above)"); - } - } else { - println!("Passed {} tests in {:?}", results.len(), doc.path); - } - } -} - -pub fn run_test(doc: &Doc) -> Vec -where - Cases: EfTest + YamlDecode, -{ - // Pass only the "test_cases" YAML string to `yaml_decode`. - let test_cases: Cases = Cases::yaml_decode(&doc.cases_yaml).unwrap(); - - test_cases.test_results() -} - -pub fn categorize_results( - results: &[CaseResult], -) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { - let mut failed = vec![]; - let mut skipped_bls = vec![]; - let mut skipped_known_failures = vec![]; - - for case in results { - match case.result.as_ref().err() { - Some(Error::SkippedBls) => skipped_bls.push(case), - Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), - Some(_) => failed.push(case), - None => (), - } - } - - (failed, skipped_bls, skipped_known_failures) -} - -pub fn print_results( - doc: &Doc, - failed: &[&CaseResult], - skipped_bls: &[&CaseResult], - skipped_known_failures: &[&CaseResult], - results: &[CaseResult], -) { - let header: DocHeader = serde_yaml::from_str(&doc.header_yaml).unwrap(); - println!("--------------------------------------------------"); - println!( - "Test {}", - if failed.is_empty() { - "Result" - } else { - "Failure" - } - ); - println!("Title: {}", header.title); - println!("File: {:?}", doc.path); - println!( - "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", - results.len(), - failed.len(), - skipped_known_failures.len(), - skipped_bls.len(), - results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() - ); - println!(); - - for case in skipped_known_failures { - println!("-------"); - println!( - "case[{}] ({}) skipped because it's a known failure", - case.case_index, case.desc, - ); - } - for failure in failed { - let error = failure.result.clone().unwrap_err(); - - println!("-------"); - println!( - "case[{}] ({}) failed with {}:", - failure.case_index, - failure.desc, - error.name() - ); - println!("{}", error.message()); - } - println!(); -} diff --git a/tests/ef_tests/src/doc_header.rs b/tests/ef_tests/src/doc_header.rs deleted file mode 100644 index c0d6d3276f..0000000000 --- a/tests/ef_tests/src/doc_header.rs +++ /dev/null @@ -1,12 +0,0 @@ -use serde_derive::Deserialize; - -#[derive(Debug, Deserialize)] -pub struct DocHeader { - pub title: String, - pub summary: String, - pub forks_timeline: String, - pub forks: Vec, - pub config: String, - pub runner: String, - pub handler: String, -} diff --git a/tests/ef_tests/src/handler.rs b/tests/ef_tests/src/handler.rs new file mode 100644 index 0000000000..e8c83e1f84 --- /dev/null +++ b/tests/ef_tests/src/handler.rs @@ -0,0 +1,297 @@ +use crate::cases::{self, Case, Cases, EpochTransition, LoadCase, Operation}; +use crate::type_name; +use crate::type_name::TypeName; +use std::fs; +use std::marker::PhantomData; +use std::path::PathBuf; +use tree_hash::SignedRoot; +use types::EthSpec; + +pub trait Handler { + type Case: Case + LoadCase; + + fn config_name() -> &'static str { + "general" + } + + fn fork_name() -> &'static str { + "phase0" + } + + fn runner_name() -> &'static str; + + fn handler_name() -> String; + + fn run() { + let handler_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("eth2.0-spec-tests") + .join("tests") + .join(Self::config_name()) + .join(Self::fork_name()) + .join(Self::runner_name()) + .join(Self::handler_name()); + + // If the directory containing the tests does not exist, just let all tests pass. + if !handler_path.exists() { + return; + } + + // Iterate through test suites + let test_cases = fs::read_dir(&handler_path) + .expect("handler dir exists") + .flat_map(|entry| { + entry + .ok() + .filter(|e| e.file_type().map(|ty| ty.is_dir()).unwrap_or(false)) + }) + .flat_map(|suite| fs::read_dir(suite.path()).expect("suite dir exists")) + .flat_map(Result::ok) + .map(|test_case_dir| { + let path = test_case_dir.path(); + let case = Self::Case::load_from_dir(&path).expect("test should load"); + (path, case) + }) + .collect(); + + let results = Cases { test_cases }.test_results(); + + let name = format!("{}/{}", Self::runner_name(), Self::handler_name()); + crate::results::assert_tests_pass(&name, &handler_path, &results); + } +} + +macro_rules! bls_handler { + ($runner_name: ident, $case_name:ident, $handler_name:expr) => { + pub struct $runner_name; + + impl Handler for $runner_name { + type Case = cases::$case_name; + + fn runner_name() -> &'static str { + "bls" + } + + fn handler_name() -> String { + $handler_name.into() + } + } + }; +} + +bls_handler!( + BlsAggregatePubkeysHandler, + BlsAggregatePubkeys, + "aggregate_pubkeys" +); +bls_handler!(BlsAggregateSigsHandler, BlsAggregateSigs, "aggregate_sigs"); +bls_handler!( + BlsG2CompressedHandler, + BlsG2Compressed, + "msg_hash_compressed" +); +bls_handler!(BlsPrivToPubHandler, BlsPrivToPub, "priv_to_pub"); +bls_handler!(BlsSignMsgHandler, BlsSign, "sign_msg"); + +/// Handler for SSZ types that do not implement `SignedRoot`. +pub struct SszStaticHandler(PhantomData<(T, E)>); + +/// Handler for SSZ types that do implement `SignedRoot`. +pub struct SszStaticSRHandler(PhantomData<(T, E)>); + +impl Handler for SszStaticHandler +where + T: cases::SszStaticType + TypeName, + E: TypeName, +{ + type Case = cases::SszStatic; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> String { + T::name().into() + } +} + +impl Handler for SszStaticSRHandler +where + T: cases::SszStaticType + SignedRoot + TypeName, + E: TypeName, +{ + type Case = cases::SszStaticSR; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "ssz_static" + } + + fn handler_name() -> String { + T::name().into() + } +} + +pub struct ShufflingHandler(PhantomData); + +impl Handler for ShufflingHandler { + type Case = cases::Shuffling; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "shuffling" + } + + fn handler_name() -> String { + "core".into() + } +} + +pub struct SanityBlocksHandler(PhantomData); + +impl Handler for SanityBlocksHandler { + type Case = cases::SanityBlocks; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> String { + "blocks".into() + } +} + +pub struct SanitySlotsHandler(PhantomData); + +impl Handler for SanitySlotsHandler { + type Case = cases::SanitySlots; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "sanity" + } + + fn handler_name() -> String { + "slots".into() + } +} + +pub struct EpochProcessingHandler(PhantomData<(E, T)>); + +impl> Handler for EpochProcessingHandler { + type Case = cases::EpochProcessing; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "epoch_processing" + } + + fn handler_name() -> String { + T::name().into() + } +} + +pub struct GenesisValidityHandler(PhantomData); + +impl Handler for GenesisValidityHandler { + type Case = cases::GenesisValidity; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> String { + "validity".into() + } +} + +pub struct GenesisInitializationHandler(PhantomData); + +impl Handler for GenesisInitializationHandler { + type Case = cases::GenesisInitialization; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "genesis" + } + + fn handler_name() -> String { + "initialization".into() + } +} + +pub struct OperationsHandler(PhantomData<(E, O)>); + +impl> Handler for OperationsHandler { + type Case = cases::Operations; + + fn config_name() -> &'static str { + E::name() + } + + fn runner_name() -> &'static str { + "operations" + } + + fn handler_name() -> String { + O::handler_name() + } +} + +pub struct SszGenericHandler(PhantomData); + +impl Handler for SszGenericHandler { + type Case = cases::SszGeneric; + + fn config_name() -> &'static str { + "general" + } + + fn runner_name() -> &'static str { + "ssz_generic" + } + + fn handler_name() -> String { + H::name().into() + } +} + +// Supported SSZ generic handlers +pub struct BasicVector; +type_name!(BasicVector, "basic_vector"); +pub struct Bitlist; +type_name!(Bitlist, "bitlist"); +pub struct Bitvector; +type_name!(Bitvector, "bitvector"); +pub struct Boolean; +type_name!(Boolean, "boolean"); +pub struct Uints; +type_name!(Uints, "uints"); +pub struct Containers; +type_name!(Containers, "containers"); diff --git a/tests/ef_tests/src/lib.rs b/tests/ef_tests/src/lib.rs index fdd4e7b859..719bfc1aaf 100644 --- a/tests/ef_tests/src/lib.rs +++ b/tests/ef_tests/src/lib.rs @@ -2,21 +2,17 @@ use types::EthSpec; pub use case_result::CaseResult; pub use cases::Case; -pub use doc::Doc; +pub use cases::{ + Crosslinks, FinalUpdates, JustificationAndFinalization, RegistryUpdates, Slashings, +}; pub use error::Error; -pub use yaml_decode::YamlDecode; +pub use handler::*; mod bls_setting; mod case_result; mod cases; -mod doc; -mod doc_header; +mod decode; mod error; -mod yaml_decode; - -/// Defined where an object can return the results of some test(s) adhering to the Ethereum -/// Foundation testing format. -pub trait EfTest { - /// Returns the results of executing one or more tests. - fn test_results(&self) -> Vec; -} +mod handler; +mod results; +mod type_name; diff --git a/tests/ef_tests/src/results.rs b/tests/ef_tests/src/results.rs new file mode 100644 index 0000000000..4f5513a9ae --- /dev/null +++ b/tests/ef_tests/src/results.rs @@ -0,0 +1,92 @@ +use crate::case_result::CaseResult; +use crate::error::Error; +use std::path::Path; + +pub fn assert_tests_pass(handler_name: &str, path: &Path, results: &[CaseResult]) { + let (failed, skipped_bls, skipped_known_failures) = categorize_results(results); + + if failed.len() + skipped_known_failures.len() > 0 { + print_results( + handler_name, + &failed, + &skipped_bls, + &skipped_known_failures, + &results, + ); + if !failed.is_empty() { + panic!("Tests failed (see above)"); + } + } else { + println!("Passed {} tests in {}", results.len(), path.display()); + } +} + +pub fn categorize_results( + results: &[CaseResult], +) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) { + let mut failed = vec![]; + let mut skipped_bls = vec![]; + let mut skipped_known_failures = vec![]; + + for case in results { + match case.result.as_ref().err() { + Some(Error::SkippedBls) => skipped_bls.push(case), + Some(Error::SkippedKnownFailure) => skipped_known_failures.push(case), + Some(_) => failed.push(case), + None => (), + } + } + + (failed, skipped_bls, skipped_known_failures) +} + +pub fn print_results( + handler_name: &str, + failed: &[&CaseResult], + skipped_bls: &[&CaseResult], + skipped_known_failures: &[&CaseResult], + results: &[CaseResult], +) { + println!("--------------------------------------------------"); + println!( + "Test {}", + if failed.is_empty() { + "Result" + } else { + "Failure" + } + ); + println!("Title: {}", handler_name); + println!( + "{} tests, {} failed, {} skipped (known failure), {} skipped (bls), {} passed. (See below for errors)", + results.len(), + failed.len(), + skipped_known_failures.len(), + skipped_bls.len(), + results.len() - skipped_bls.len() - skipped_known_failures.len() - failed.len() + ); + println!(); + + for case in skipped_known_failures { + println!("-------"); + println!( + "case ({}) from {} skipped because it's a known failure", + case.desc, + case.path.display() + ); + } + for failure in failed { + let error = failure.result.clone().unwrap_err(); + + println!("-------"); + println!( + "case {} ({}) from {} failed with {}:", + failure.case_index, + failure.desc, + failure.path.display(), + error.name() + ); + println!("{}", error.message()); + } + println!(); +} diff --git a/tests/ef_tests/src/type_name.rs b/tests/ef_tests/src/type_name.rs new file mode 100644 index 0000000000..5af0c52565 --- /dev/null +++ b/tests/ef_tests/src/type_name.rs @@ -0,0 +1,60 @@ +//! Mapping from types to canonical string identifiers used in testing. +use types::*; + +pub trait TypeName { + fn name() -> &'static str; +} + +#[macro_export] +macro_rules! type_name { + ($typ:ident) => { + type_name!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { + impl TypeName for $typ { + fn name() -> &'static str { + $name + } + } + }; +} + +#[macro_export] +macro_rules! type_name_generic { + ($typ:ident) => { + type_name_generic!($typ, stringify!($typ)); + }; + ($typ:ident, $name:expr) => { + impl TypeName for $typ { + fn name() -> &'static str { + $name + } + } + }; +} + +type_name!(MinimalEthSpec, "minimal"); +type_name!(MainnetEthSpec, "mainnet"); + +type_name_generic!(Attestation); +type_name!(AttestationData); +type_name!(AttestationDataAndCustodyBit); +type_name_generic!(AttesterSlashing); +type_name_generic!(BeaconBlock); +type_name_generic!(BeaconBlockBody); +type_name!(BeaconBlockHeader); +type_name_generic!(BeaconState); +type_name!(Checkpoint); +type_name_generic!(CompactCommittee); +type_name!(Crosslink); +type_name!(Deposit); +type_name!(DepositData); +type_name!(Eth1Data); +type_name!(Fork); +type_name_generic!(HistoricalBatch); +type_name_generic!(IndexedAttestation); +type_name_generic!(PendingAttestation); +type_name!(ProposerSlashing); +type_name!(Transfer); +type_name!(Validator); +type_name!(VoluntaryExit); diff --git a/tests/ef_tests/src/yaml_decode.rs b/tests/ef_tests/src/yaml_decode.rs deleted file mode 100644 index c89dd92a9e..0000000000 --- a/tests/ef_tests/src/yaml_decode.rs +++ /dev/null @@ -1,59 +0,0 @@ -use super::*; -use ethereum_types::{U128, U256}; -use types::Fork; - -mod utils; - -pub use utils::*; - -pub trait YamlDecode: Sized { - /// Decode an object from the test specification YAML. - fn yaml_decode(string: &str) -> Result; -} - -/// Basic types can general be decoded with the `parse` fn if they implement `str::FromStr`. -macro_rules! impl_via_parse { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - string - .parse::() - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_parse!(u8); -impl_via_parse!(u16); -impl_via_parse!(u32); -impl_via_parse!(u64); - -/// Some `ethereum-types` methods have a `str::FromStr` implementation that expects `0x`-prefixed: -/// hex, so we use `from_dec_str` instead. -macro_rules! impl_via_from_dec_str { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - Self::from_dec_str(string).map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_from_dec_str!(U128); -impl_via_from_dec_str!(U256); - -/// Types that already implement `serde::Deserialize` can be decoded using `serde_yaml`. -macro_rules! impl_via_serde_yaml { - ($ty: ty) => { - impl YamlDecode for $ty { - fn yaml_decode(string: &str) -> Result { - serde_yaml::from_str(string) - .map_err(|e| Error::FailedToParseTest(format!("{:?}", e))) - } - } - }; -} - -impl_via_serde_yaml!(Fork); diff --git a/tests/ef_tests/src/yaml_decode/utils.rs b/tests/ef_tests/src/yaml_decode/utils.rs deleted file mode 100644 index 7b6caac728..0000000000 --- a/tests/ef_tests/src/yaml_decode/utils.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub fn yaml_split_header_and_cases(mut yaml: String) -> (String, String) { - let test_cases_start = yaml.find("\ntest_cases:\n").unwrap(); - // + 1 to skip the \n we used for matching. - let mut test_cases = yaml.split_off(test_cases_start + 1); - - let end_of_first_line = test_cases.find('\n').unwrap(); - let test_cases = test_cases.split_off(end_of_first_line + 1); - - (yaml, test_cases) -} diff --git a/tests/ef_tests/tests/tests.rs b/tests/ef_tests/tests/tests.rs index deb699e786..337c54b461 100644 --- a/tests/ef_tests/tests/tests.rs +++ b/tests/ef_tests/tests/tests.rs @@ -1,225 +1,214 @@ use ef_tests::*; -use rayon::prelude::*; -use std::path::{Path, PathBuf}; -use walkdir::WalkDir; - -fn yaml_files_in_test_dir(dir: &Path) -> Vec { - let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("eth2.0-spec-tests") - .join("tests") - .join(dir); - - assert!( - base_path.exists(), - format!( - "Unable to locate {:?}. Did you init git submodules?", - base_path - ) - ); - - let mut paths: Vec = WalkDir::new(base_path) - .into_iter() - .filter_map(|e| e.ok()) - .filter_map(|entry| { - if entry.file_type().is_file() { - match entry.file_name().to_str() { - Some(f) if f.ends_with(".yaml") => Some(entry.path().to_path_buf()), - Some(f) if f.ends_with(".yml") => Some(entry.path().to_path_buf()), - _ => None, - } - } else { - None - } - }) - .collect(); - - // Reverse the file order. Assuming files come in lexicographical order, executing tests in - // reverse means we get the "minimal" tests before the "mainnet" tests. This makes life easier - // for debugging. - paths.reverse(); - paths -} - -#[test] -#[cfg(feature = "fake_crypto")] -fn ssz_generic() { - yaml_files_in_test_dir(&Path::new("ssz_generic")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} - -#[test] -#[cfg(feature = "fake_crypto")] -fn ssz_static() { - yaml_files_in_test_dir(&Path::new("ssz_static")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); -} +use types::*; #[test] fn shuffling() { - yaml_files_in_test_dir(&Path::new("shuffling").join("core")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + ShufflingHandler::::run(); + ShufflingHandler::::run(); } #[test] fn operations_deposit() { - yaml_files_in_test_dir(&Path::new("operations").join("deposit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_transfer() { - yaml_files_in_test_dir(&Path::new("operations").join("transfer")) - .into_par_iter() - .rev() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + // Note: there are no transfer tests for mainnet } #[test] fn operations_exit() { - yaml_files_in_test_dir(&Path::new("operations").join("voluntary_exit")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_proposer_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("proposer_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::::run(); + OperationsHandler::::run(); } #[test] fn operations_attester_slashing() { - yaml_files_in_test_dir(&Path::new("operations").join("attester_slashing")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_attestation() { - yaml_files_in_test_dir(&Path::new("operations").join("attestation")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn operations_block_header() { - yaml_files_in_test_dir(&Path::new("operations").join("block_header")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + OperationsHandler::>::run(); + OperationsHandler::>::run(); } #[test] fn sanity_blocks() { - yaml_files_in_test_dir(&Path::new("sanity").join("blocks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanityBlocksHandler::::run(); + SanityBlocksHandler::::run(); } #[test] fn sanity_slots() { - yaml_files_in_test_dir(&Path::new("sanity").join("slots")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + SanitySlotsHandler::::run(); + SanitySlotsHandler::::run(); } #[test] #[cfg(not(feature = "fake_crypto"))] -fn bls() { - yaml_files_in_test_dir(&Path::new("bls")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); +fn bls_aggregate_pubkeys() { + BlsAggregatePubkeysHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_aggregate_sigs() { + BlsAggregateSigsHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_msg_hash_g2_compressed() { + BlsG2CompressedHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_priv_to_pub() { + BlsPrivToPubHandler::run(); +} + +#[test] +#[cfg(not(feature = "fake_crypto"))] +fn bls_sign_msg() { + BlsSignMsgHandler::run(); +} + +#[cfg(feature = "fake_crypto")] +macro_rules! ssz_static_test { + // Signed-root + ($test_name:ident, $typ:ident$(<$generics:tt>)?, SR) => { + ssz_static_test!($test_name, SszStaticSRHandler, $typ$(<$generics>)?); + }; + // Non-signed root + ($test_name:ident, $typ:ident$(<$generics:tt>)?) => { + ssz_static_test!($test_name, SszStaticHandler, $typ$(<$generics>)?); + }; + // Generic + ($test_name:ident, $handler:ident, $typ:ident<_>) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Non-generic + ($test_name:ident, $handler:ident, $typ:ident) => { + ssz_static_test!( + $test_name, $handler, { + ($typ, MinimalEthSpec), + ($typ, MainnetEthSpec) + } + ); + }; + // Base case + ($test_name:ident, $handler:ident, { $(($typ:ty, $spec:ident)),+ }) => { + #[test] + fn $test_name() { + $( + $handler::<$typ, $spec>::run(); + )+ + } + }; +} + +#[cfg(feature = "fake_crypto")] +mod ssz_static { + use ef_tests::{Handler, SszStaticHandler, SszStaticSRHandler}; + use types::*; + + ssz_static_test!(attestation, Attestation<_>, SR); + ssz_static_test!(attestation_data, AttestationData); + ssz_static_test!( + attestation_data_and_custody_bit, + AttestationDataAndCustodyBit + ); + ssz_static_test!(attester_slashing, AttesterSlashing<_>); + ssz_static_test!(beacon_block, BeaconBlock<_>, SR); + ssz_static_test!(beacon_block_body, BeaconBlockBody<_>); + ssz_static_test!(beacon_block_header, BeaconBlockHeader, SR); + ssz_static_test!(beacon_state, BeaconState<_>); + ssz_static_test!(checkpoint, Checkpoint); + ssz_static_test!(compact_committee, CompactCommittee<_>); + ssz_static_test!(crosslink, Crosslink); + ssz_static_test!(deposit, Deposit); + ssz_static_test!(deposit_data, DepositData, SR); + ssz_static_test!(eth1_data, Eth1Data); + ssz_static_test!(fork, Fork); + ssz_static_test!(historical_batch, HistoricalBatch<_>); + ssz_static_test!(indexed_attestation, IndexedAttestation<_>, SR); + ssz_static_test!(pending_attestation, PendingAttestation<_>); + ssz_static_test!(proposer_slashing, ProposerSlashing); + ssz_static_test!(transfer, Transfer, SR); + ssz_static_test!(validator, Validator); + ssz_static_test!(voluntary_exit, VoluntaryExit, SR); +} + +#[test] +fn ssz_generic() { + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); + SszGenericHandler::::run(); } #[test] fn epoch_processing_justification_and_finalization() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("justification_and_finalization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_crosslinks() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("crosslinks")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_registry_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("registry_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_slashings() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("slashings")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn epoch_processing_final_updates() { - yaml_files_in_test_dir(&Path::new("epoch_processing").join("final_updates")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + EpochProcessingHandler::::run(); + EpochProcessingHandler::::run(); } #[test] fn genesis_initialization() { - yaml_files_in_test_dir(&Path::new("genesis").join("initialization")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisInitializationHandler::::run(); } #[test] fn genesis_validity() { - yaml_files_in_test_dir(&Path::new("genesis").join("validity")) - .into_par_iter() - .for_each(|file| { - Doc::assert_tests_pass(file); - }); + GenesisValidityHandler::::run(); + // Note: there are no genesis validity tests for mainnet } diff --git a/validator_client/src/service.rs b/validator_client/src/service.rs index 8adc79b91e..5169f67f87 100644 --- a/validator_client/src/service.rs +++ b/validator_client/src/service.rs @@ -27,7 +27,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::marker::PhantomData; use std::sync::Arc; use std::sync::RwLock; -use std::time::{Duration, Instant, SystemTime}; +use std::time::{Duration, Instant}; use tokio::prelude::*; use tokio::runtime::Builder; use tokio::timer::Interval; @@ -46,8 +46,8 @@ pub struct Service, slots_per_epoch: u64, /// The chain specification for this clients instance. spec: Arc, @@ -100,19 +100,6 @@ impl Service { - // verify the node's genesis time - if SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs() - < info.genesis_time - { - error!( - log, - "Beacon Node's genesis time is in the future. No work to do.\n Exiting" - ); - return Err("Genesis time in the future".into()); - } // verify the node's network id if eth2_config.spec.network_id != info.network_id as u8 { error!( @@ -177,12 +164,8 @@ impl Service(|| { - "Unable to start slot clock. Genesis may not have occurred yet. Exiting.".into() - })?; - - let current_slot = slot_clock.now().ok_or_else::(|| { - "Genesis has not yet occurred. Exiting.".into() + .map_err::(|e| { + format!("Unable to start slot clock: {}.", e).into() })?; /* Generate the duties manager */ @@ -215,7 +198,7 @@ impl Service Service::initialize_service( client_config, eth2_config, - log, + log.clone(), )?; // we have connected to a node and established its parameters. Spin up the core service @@ -253,7 +236,7 @@ impl Service(|| { - "Genesis is not in the past. Exiting.".into() + "Unable to determine duration to next slot. Exiting.".into() })?; // set up the validator work interval - start at next slot and proceed every slot @@ -264,6 +247,19 @@ impl Service duration_to_next_slot.as_secs() + ); + /* kick off the core service */ runtime.block_on( interval @@ -298,27 +294,29 @@ impl Service error_chain::Result<()> { - let current_slot = self + let wall_clock_slot = self .slot_clock .now() .ok_or_else::(|| { "Genesis is not in the past. Exiting.".into() })?; - let current_epoch = current_slot.epoch(self.slots_per_epoch); + let wall_clock_epoch = wall_clock_slot.epoch(self.slots_per_epoch); // this is a non-fatal error. If the slot clock repeats, the node could // have been slow to process the previous slot and is now duplicating tasks. // We ignore duplicated but raise a critical error. - if current_slot <= self.current_slot { - crit!( - self.log, - "The validator tried to duplicate a slot. Likely missed the previous slot" - ); - return Err("Duplicate slot".into()); + if let Some(current_slot) = self.current_slot { + if wall_clock_slot <= current_slot { + crit!( + self.log, + "The validator tried to duplicate a slot. Likely missed the previous slot" + ); + return Err("Duplicate slot".into()); + } } - self.current_slot = current_slot; - info!(self.log, "Processing"; "slot" => current_slot.as_u64(), "epoch" => current_epoch.as_u64()); + self.current_slot = Some(wall_clock_slot); + info!(self.log, "Processing"; "slot" => wall_clock_slot.as_u64(), "epoch" => wall_clock_epoch.as_u64()); Ok(()) } @@ -326,7 +324,10 @@ impl Service Service