gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-15 19:02:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

Testnet compatible network upgrade (#587)

Browse Source 
* Create libp2p instance

* Change logger to stdlog

* test_connection initial commit

* Add gossipsub test

* Delete tests in network crate

* Add test module

* Clean tests

* Remove dependency on discovery

* Working publish between 2 nodes
TODO: Publish should be called just once

* Working 2 peer gossipsub test with additional events

* Cleanup test

* Add rpc test

* Star topology discovery WIP

* build_nodes builds and connects n nodes. Increase nodes in gossipsub test

* Add unsubscribe method and expose reference to gossipsub object for gossipsub tests

* Add gossipsub message forwarding test

* Fix gossipsub forward test

* Test improvements
* Remove discovery tests
* Simplify gossipsub forward test topology
* Add helper functions for topology building

* Clean up tests

* Update naming to new network spec

* Correct ssz encoding of protocol names

* Further additions to network upgrade

* Initial network spec update WIP

* Temp commit

* Builds one side of the streamed RPC responses

* Temporary commit

* Propagates streaming changes up into message handler

* Intermediate network update

* Partial update in upgrading to the new network spec

* Update dependencies, remove redundant deps

* Correct sync manager for block stream handling

* Re-write of RPC handler, improves efficiency and corrects bugs

* Stream termination update

* Completed refactor of rpc handler

* Remove crates

* Correct compile issues associated with test merge

* Build basic tests and testing structure for eth2-libp2p

* Enhance RPC tests and add logging

* Complete RPC testing framework and STATUS test

* Decoding bug fixes, log improvements, stream test

* Clean up RPC handler logging

* Decoder bug fix, empty block stream test

* Add BlocksByRoot RPC test

* Add Goodbye RPC test

* Syncing and stream handling bug fixes and performance improvements

* Applies discv5 bug fixes

* Adds DHT IP filtering for lighthouse - currently disabled

* Adds randomized network propagation as a CLI arg

* Add disconnect functionality

* Adds attestation handling and parent lookup

* Adds RPC error handling for the sync manager

* Allow parent's blocks to be already processed

* Update github workflow

* Adds reviewer suggestions
This commit is contained in:
Age Manning
2019-11-27 12:47:46 +11:00
committed by Paul Hauner
parent bf2eeae3f2
commit 97aa8b75b8
25 changed files with 2239 additions and 659 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
beacon_node/eth2-libp2p/src/rpc/codec/base.rs
View File

@@ -1,8 +1,8 @@
//! This handles the various supported encoding mechanism for the Eth 2.0 RPC.
use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse};
use bytes::BufMut;
use bytes::BytesMut;
use libp2p::bytes::BufMut;
use libp2p::bytes::BytesMut;
use tokio::codec::{Decoder, Encoder};
pub trait OutboundCodec: Encoder + Decoder {
@@ -14,6 +14,9 @@ pub trait OutboundCodec: Encoder + Decoder {
) -> Result<Option<Self::ErrorType>, <Self as Decoder>::Error>;
}
/* Global Inbound Codec */
// This deals with Decoding RPC Requests from other peers and encoding our responses
pub struct BaseInboundCodec<TCodec>
where
TCodec: Encoder + Decoder,
@@ -31,15 +34,16 @@ where
}
}
/* Global Outbound Codec */
// This deals with Decoding RPC Responses from other peers and encoding our requests
pub struct BaseOutboundCodec<TOutboundCodec>
where
TOutboundCodec: OutboundCodec,
{
/// Inner codec for handling various encodings
/// Inner codec for handling various encodings.
inner: TOutboundCodec,
/// Optimisation for decoding. True if the response code has been read and we are awaiting a
/// response.
response_code: Option<u8>,
/// Keeps track of the current response code for a chunk.
current_response_code: Option<u8>,
}
impl<TOutboundCodec> BaseOutboundCodec<TOutboundCodec>
@@ -49,11 +53,16 @@ where
pub fn new(codec: TOutboundCodec) -> Self {
BaseOutboundCodec {
inner: codec,
response_code: None,
current_response_code: None,
}
}
}
/* Implementation of the Encoding/Decoding for the global codecs */
/* Base Inbound Codec */
// This Encodes RPC Responses sent to external peers
impl<TCodec> Encoder for BaseInboundCodec<TCodec>
where
TCodec: Decoder + Encoder<Item = RPCErrorResponse>,
@@ -64,11 +73,15 @@ where
fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
dst.clear();
dst.reserve(1);
dst.put_u8(item.as_u8());
dst.put_u8(
item.as_u8()
.expect("Should never encode a stream termination"),
);
self.inner.encode(item, dst)
}
}
// This Decodes RPC Requests from external peers
impl<TCodec> Decoder for BaseInboundCodec<TCodec>
where
TCodec: Encoder + Decoder<Item = RPCRequest>,
@@ -81,6 +94,9 @@ where
}
}
/* Base Outbound Codec */
// This Encodes RPC Requests sent to external peers
impl<TCodec> Encoder for BaseOutboundCodec<TCodec>
where
TCodec: OutboundCodec + Encoder<Item = RPCRequest>,
@@ -93,6 +109,7 @@ where
}
}
// This decodes RPC Responses received from external peers
impl<TCodec> Decoder for BaseOutboundCodec<TCodec>
where
TCodec: OutboundCodec<ErrorType = ErrorMessage> + Decoder<Item = RPCResponse>,
@@ -102,34 +119,36 @@ where
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
// if we have only received the response code, wait for more bytes
if src.len() == 1 {
if src.len() <= 1 {
return Ok(None);
}
// using the response code determine which kind of payload needs to be decoded.
let response_code = {
if let Some(resp_code) = self.response_code {
resp_code
} else {
let resp_byte = src.split_to(1);
let mut resp_code_byte = [0; 1];
resp_code_byte.copy_from_slice(&resp_byte);
let response_code = self.current_response_code.unwrap_or_else(|| {
let resp_code = src.split_to(1)[0];
self.current_response_code = Some(resp_code);
resp_code
});
let resp_code = u8::from_be_bytes(resp_code_byte);
self.response_code = Some(resp_code);
resp_code
let inner_result = {
if RPCErrorResponse::is_response(response_code) {
// decode an actual response and mutates the buffer if enough bytes have been read
// returning the result.
self.inner
.decode(src)
.map(|r| r.map(RPCErrorResponse::Success))
} else {
// decode an error
self.inner
.decode_error(src)
.map(|r| r.map(|resp| RPCErrorResponse::from_error(response_code, resp)))
}
};
if RPCErrorResponse::is_response(response_code) {
// decode an actual response
self.inner
.decode(src)
.map(|r| r.map(RPCErrorResponse::Success))
} else {
// decode an error
self.inner
.decode_error(src)
.map(|r| r.map(|resp| RPCErrorResponse::from_error(response_code, resp)))
// if the inner decoder was capable of decoding a chunk, we need to reset the current
// response code for the next chunk
if let Ok(Some(_)) = inner_result {
self.current_response_code = None;
}
// return the result
inner_result
}
}

2
beacon_node/eth2-libp2p/src/rpc/codec/mod.rs
View File

@@ -5,7 +5,7 @@ use self::base::{BaseInboundCodec, BaseOutboundCodec};
use self::ssz::{SSZInboundCodec, SSZOutboundCodec};
use crate::rpc::protocol::RPCError;
use crate::rpc::{RPCErrorResponse, RPCRequest};
use bytes::BytesMut;
use libp2p::bytes::BytesMut;
use tokio::codec::{Decoder, Encoder};
// Known types of codecs

106
beacon_node/eth2-libp2p/src/rpc/codec/ssz.rs
View File

@@ -4,7 +4,7 @@ use crate::rpc::{
protocol::{ProtocolId, RPCError},
};
use crate::rpc::{ErrorMessage, RPCErrorResponse, RPCRequest, RPCResponse};
use bytes::{BufMut, Bytes, BytesMut};
use libp2p::bytes::{BufMut, Bytes, BytesMut};
use ssz::{Decode, Encode};
use tokio::codec::{Decoder, Encoder};
use unsigned_varint::codec::UviBytes;
@@ -31,7 +31,7 @@ impl SSZInboundCodec {
}
}
// Encoder for inbound
// Encoder for inbound streams: Encodes RPC Responses sent to peers.
impl Encoder for SSZInboundCodec {
type Item = RPCErrorResponse;
type Error = RPCError;
@@ -40,16 +40,19 @@ impl Encoder for SSZInboundCodec {
let bytes = match item {
RPCErrorResponse::Success(resp) => {
match resp {
RPCResponse::Hello(res) => res.as_ssz_bytes(),
RPCResponse::BeaconBlocks(res) => res, // already raw bytes
RPCResponse::RecentBeaconBlocks(res) => res, // already raw bytes
RPCResponse::Status(res) => res.as_ssz_bytes(),
RPCResponse::BlocksByRange(res) => res, // already raw bytes
RPCResponse::BlocksByRoot(res) => res, // already raw bytes
RPCResponse::Goodbye => unreachable!("Never encode or decode this message"),
}
}
RPCErrorResponse::InvalidRequest(err) => err.as_ssz_bytes(),
RPCErrorResponse::ServerError(err) => err.as_ssz_bytes(),
RPCErrorResponse::Unknown(err) => err.as_ssz_bytes(),
RPCErrorResponse::StreamTermination(_) => {
unreachable!("Code error - attempting to encode a stream termination")
}
};
if !bytes.is_empty() {
// length-prefix and return
return self
@@ -65,7 +68,7 @@ impl Encoder for SSZInboundCodec {
}
}
// Decoder for inbound
// Decoder for inbound streams: Decodes RPC requests from peers
impl Decoder for SSZInboundCodec {
type Item = RPCRequest;
type Error = RPCError;
@@ -73,8 +76,8 @@ impl Decoder for SSZInboundCodec {
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
match self.inner.decode(src).map_err(RPCError::from) {
Ok(Some(packet)) => match self.protocol.message_name.as_str() {
"hello" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::Hello(HelloMessage::from_ssz_bytes(
"status" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::Status(StatusMessage::from_ssz_bytes(
&packet,
)?))),
_ => unreachable!("Cannot negotiate an unknown version"),
@@ -85,16 +88,16 @@ impl Decoder for SSZInboundCodec {
)?))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::BeaconBlocks(
BeaconBlocksRequest::from_ssz_bytes(&packet)?,
"blocks_by_range" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::BlocksByRange(
BlocksByRangeRequest::from_ssz_bytes(&packet)?,
))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"recent_beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::RecentBeaconBlocks(
RecentBeaconBlocksRequest::from_ssz_bytes(&packet)?,
))),
"blocks_by_root" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCRequest::BlocksByRoot(BlocksByRootRequest {
block_roots: Vec::from_ssz_bytes(&packet)?,
}))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
_ => unreachable!("Cannot negotiate an unknown protocol"),
@@ -105,7 +108,7 @@ impl Decoder for SSZInboundCodec {
}
}
/* Outbound Codec */
/* Outbound Codec: Codec for initiating RPC requests */
pub struct SSZOutboundCodec {
inner: UviBytes,
@@ -127,26 +130,26 @@ impl SSZOutboundCodec {
}
}
// Encoder for outbound
// Encoder for outbound streams: Encodes RPC Requests to peers
impl Encoder for SSZOutboundCodec {
type Item = RPCRequest;
type Error = RPCError;
fn encode(&mut self, item: Self::Item, dst: &mut BytesMut) -> Result<(), Self::Error> {
let bytes = match item {
RPCRequest::Hello(req) => req.as_ssz_bytes(),
RPCRequest::Status(req) => req.as_ssz_bytes(),
RPCRequest::Goodbye(req) => req.as_ssz_bytes(),
RPCRequest::BeaconBlocks(req) => req.as_ssz_bytes(),
RPCRequest::RecentBeaconBlocks(req) => req.as_ssz_bytes(),
RPCRequest::BlocksByRange(req) => req.as_ssz_bytes(),
RPCRequest::BlocksByRoot(req) => req.block_roots.as_ssz_bytes(),
};
// length-prefix
self.inner
.encode(bytes::Bytes::from(bytes), dst)
.encode(libp2p::bytes::Bytes::from(bytes), dst)
.map_err(RPCError::from)
}
}
// Decoder for outbound streams
// Decoder for outbound streams: Decodes RPC responses from peers.
//
// The majority of the decoding has now been pushed upstream due to the changing specification.
// We prefer to decode blocks and attestations with extra knowledge about the chain to perform
@@ -158,44 +161,53 @@ impl Decoder for SSZOutboundCodec {
fn decode(&mut self, src: &mut BytesMut) -> Result<Option<Self::Item>, Self::Error> {
if src.len() == 1 && src[0] == 0_u8 {
// the object is empty. We return the empty object if this is the case
// clear the buffer and return an empty object
src.clear();
match self.protocol.message_name.as_str() {
"hello" => match self.protocol.version.as_str() {
"status" => match self.protocol.version.as_str() {
"1" => Err(RPCError::Custom(
"Hello stream terminated unexpectedly".into(),
"Status stream terminated unexpectedly".into(),
)), // cannot have an empty HELLO message. The stream has terminated unexpectedly
_ => unreachable!("Cannot negotiate an unknown version"),
},
"goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
"beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BeaconBlocks(Vec::new()))),
"blocks_by_range" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BlocksByRange(Vec::new()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"recent_beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::RecentBeaconBlocks(Vec::new()))),
"blocks_by_root" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BlocksByRoot(Vec::new()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
_ => unreachable!("Cannot negotiate an unknown protocol"),
}
} else {
match self.inner.decode(src).map_err(RPCError::from) {
Ok(Some(packet)) => match self.protocol.message_name.as_str() {
"hello" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::Hello(HelloMessage::from_ssz_bytes(
&packet,
)?))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"goodbye" => Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response")),
"beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BeaconBlocks(packet.to_vec()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"recent_beacon_blocks" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::RecentBeaconBlocks(packet.to_vec()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
_ => unreachable!("Cannot negotiate an unknown protocol"),
},
Ok(Some(mut packet)) => {
// take the bytes from the buffer
let raw_bytes = packet.take();
match self.protocol.message_name.as_str() {
"status" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::Status(StatusMessage::from_ssz_bytes(
&raw_bytes,
)?))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"goodbye" => {
Err(RPCError::InvalidProtocol("GOODBYE doesn't have a response"))
}
"blocks_by_range" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BlocksByRange(raw_bytes.to_vec()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
"blocks_by_root" => match self.protocol.version.as_str() {
"1" => Ok(Some(RPCResponse::BlocksByRoot(raw_bytes.to_vec()))),
_ => unreachable!("Cannot negotiate an unknown version"),
},
_ => unreachable!("Cannot negotiate an unknown protocol"),
}
}
Ok(None) => Ok(None), // waiting for more bytes
Err(e) => Err(e),
}