diff --git a/boolean-bitfield/Cargo.toml b/boolean-bitfield/Cargo.toml index 6e790ea23e..feacb844a5 100644 --- a/boolean-bitfield/Cargo.toml +++ b/boolean-bitfield/Cargo.toml @@ -4,3 +4,4 @@ version = "0.1.0" authors = ["Paul Hauner "] [dependencies] +ssz = { path = "../ssz" } diff --git a/boolean-bitfield/src/lib.rs b/boolean-bitfield/src/lib.rs index da789c476f..0a690bf3db 100644 --- a/boolean-bitfield/src/lib.rs +++ b/boolean-bitfield/src/lib.rs @@ -8,66 +8,78 @@ */ use std::cmp::max; -#[derive(Eq)] +#[derive(Eq,Clone)] pub struct BooleanBitfield{ len: usize, vec: Vec } impl BooleanBitfield { + /// Create a new bitfield with a length of zero. pub fn new() -> Self { Self { len: 0, vec: vec![] } } - + + /// Create a new bitfield of a certain capacity pub fn with_capacity(capacity: usize) -> Self { Self { len: 0, - vec: Vec::with_capacity(capacity) + vec: Vec::with_capacity(capacity / 8 + 1) } } - // Output the bitfield as a big-endian vec of u8. - pub fn to_be_vec(&self) -> Vec { - let mut o = self.vec.clone(); - o.reverse(); - o - } - + /// Read the value of a bit. + /// + /// Will return `true` if the bit has been set to `true` + /// without then being set to `False`. pub fn get_bit(&self, i: &usize) -> bool { - self.get_bit_on_byte(*i % 8, *i / 8) - } + let bit = |i: &usize| *i % 8; + let byte = |i: &usize| *i / 8; - fn get_bit_on_byte(&self, bit: usize, byte: usize) -> bool { - assert!(bit < 8); - if byte >= self.vec.len() { + if byte(i) >= self.vec.len() { false } else { - self.vec[byte] & (1 << (bit as u8)) != 0 + self.vec[byte(i)] & (1 << (bit(i) as u8)) != 0 } } - pub fn set_bit(&mut self, bit: &usize, to: &bool) { - self.len = max(self.len, *bit + 1); - self.set_bit_on_byte(*bit % 8, *bit / 8, to); - } + /// Set the value of a bit. + /// + /// If this bit is larger than the length of the underlying byte + /// array it will be extended. + pub fn set_bit(&mut self, i: &usize, to: &bool) { + let bit = |i: &usize| *i % 8; + let byte = |i: &usize| *i / 8; - fn set_bit_on_byte(&mut self, bit: usize, byte: usize, val: &bool) { - assert!(bit < 8); - if byte >= self.vec.len() { - self.vec.resize(byte + 1, 0); + self.len = max(self.len, i + 1); + + if byte(i) >= self.vec.len() { + self.vec.resize(byte(i) + 1, 0); } - match val { - true => self.vec[byte] = self.vec[byte] | (1 << (bit as u8)), - false => self.vec[byte] = self.vec[byte] & !(1 << (bit as u8)) + match to { + true => { + self.vec[byte(i)] = + self.vec[byte(i)] | (1 << (bit(i) as u8)) + } + false => { + self.vec[byte(i)] = + self.vec[byte(i)] & !(1 << (bit(i) as u8)) + } } } + /// Return the "length" of this bitfield. Length is defined as + /// the highest bit that has been set. + /// + /// Note: this is distinct from the length of the underlying + /// vector. pub fn len(&self) -> usize { self.len } - // Return the total number of bits set to true. + /// Iterate through the underlying vector and count the number of + /// true bits. pub fn num_true_bits(&self) -> u64 { let mut count: u64 = 0; for byte in &self.vec { @@ -79,6 +91,13 @@ impl BooleanBitfield { } count } + + /// Clone and return the underlying byte array (`Vec`). + pub fn to_be_vec(&self) -> Vec { + let mut o = self.vec.clone(); + o.reverse(); + o + } } impl PartialEq for BooleanBitfield { @@ -88,34 +107,24 @@ impl PartialEq for BooleanBitfield { } } -impl Clone for BooleanBitfield { - fn clone(&self) -> Self { - Self { - vec: self.vec.to_vec(), - ..*self - } - } -} - #[cfg(test)] mod tests { use super::*; - use super::rlp; #[test] fn test_bitfield_set() { let mut b = BooleanBitfield::new(); b.set_bit(&0, &false); assert_eq!(b.to_be_vec(), [0]); - + b = BooleanBitfield::new(); b.set_bit(&7, &true); assert_eq!(b.to_be_vec(), [128]); b.set_bit(&7, &false); assert_eq!(b.to_be_vec(), [0]); assert_eq!(b.len(), 8); - + b = BooleanBitfield::new(); b.set_bit(&7, &true); b.set_bit(&0, &true); @@ -123,21 +132,22 @@ mod tests { b.set_bit(&7, &false); assert_eq!(b.to_be_vec(), [1]); assert_eq!(b.len(), 8); - + b = BooleanBitfield::new(); b.set_bit(&8, &true); assert_eq!(b.to_be_vec(), [1, 0]); + assert_eq!(b.len(), 9); b.set_bit(&8, &false); assert_eq!(b.to_be_vec(), [0, 0]); assert_eq!(b.len(), 9); - + b = BooleanBitfield::new(); b.set_bit(&15, &true); assert_eq!(b.to_be_vec(), [128, 0]); b.set_bit(&15, &false); assert_eq!(b.to_be_vec(), [0, 0]); assert_eq!(b.len(), 16); - + b = BooleanBitfield::new(); b.set_bit(&8, &true); b.set_bit(&15, &true); @@ -156,9 +166,9 @@ mod tests { b.set_bit(&i, &true); assert_eq!(b.get_bit(&i), true); b.set_bit(&i, &true); - } + } } - + #[test] fn test_bitfield_num_true_bits() { let mut b = BooleanBitfield::new(); diff --git a/lighthouse/client/mod.rs b/lighthouse/client/mod.rs index 8474bf105d..f87f05738b 100644 --- a/lighthouse/client/mod.rs +++ b/lighthouse/client/mod.rs @@ -8,7 +8,7 @@ use super::futures::sync::mpsc::{ use super::network_libp2p::service::listen as network_listen; use super::network_libp2p::state::NetworkState; use super::slog::Logger; -use super::sync::start_sync; +use super::sync::run_sync_future; /// Represents the co-ordination of the /// networking, syncing and RPC (not-yet-implemented) threads. @@ -59,7 +59,7 @@ impl Client { let sync_log = log.new(o!()); let sync_db = Arc::clone(&db); let thread = thread::spawn(move || { - start_sync( + run_sync_future( sync_db, network_tx.clone(), network_rx, diff --git a/lighthouse/state/attestation_record.rs b/lighthouse/state/attestation_record.rs index e8725cfd7f..fc949b9e6d 100644 --- a/lighthouse/state/attestation_record.rs +++ b/lighthouse/state/attestation_record.rs @@ -1,5 +1,6 @@ use super::utils::types::{ Hash256, Bitfield }; use super::utils::bls::{ AggregateSignature }; +use super::ssz::{ Encodable, SszStream }; pub struct AttestationRecord { @@ -11,6 +12,17 @@ pub struct AttestationRecord { pub aggregate_sig: Option, } +impl Encodable for AttestationRecord { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.slot); + s.append(&self.shard_id); + s.append_vec(&self.oblique_parent_hashes); + s.append(&self.shard_block_hash); + s.append_vec(&self.attester_bitfield.to_be_vec()); + // TODO: add aggregate signature + } +} + impl AttestationRecord { pub fn zero() -> Self { Self { diff --git a/lighthouse/state/block.rs b/lighthouse/state/block.rs index 11f9192bb3..d9c2c05790 100644 --- a/lighthouse/state/block.rs +++ b/lighthouse/state/block.rs @@ -1,6 +1,6 @@ use super::utils::types::Hash256; use super::attestation_record::AttestationRecord; -use super::ssz; +use super::ssz::{ Encodable, SszStream }; const SSZ_BLOCK_LENGTH: usize = 192; @@ -12,7 +12,7 @@ pub struct Block { pub pow_chain_ref: Hash256, pub active_state_root: Hash256, pub crystallized_state_root: Hash256, -} +} impl Block { pub fn zero() -> Self { @@ -27,12 +27,12 @@ impl Block { } } - // Not sure if this will be useful, leaving it here for the - // time being. - pub fn ssz_encode_without_attestations(&self) + /// Return the bytes that should be signed in order to + /// attest for this block. + pub fn encode_for_signing(&self) -> [u8; SSZ_BLOCK_LENGTH] { - let mut s = ssz::SszStream::new(); + let mut s = SszStream::new(); s.append(&self.parent_hash); s.append(&self.slot_number); s.append(&self.randao_reveal); @@ -45,6 +45,18 @@ impl Block { } } +impl Encodable for Block { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.parent_hash); + s.append(&self.slot_number); + s.append(&self.randao_reveal); + s.append_vec(&self.attestations); + s.append(&self.pow_chain_ref); + s.append(&self.active_state_root); + s.append(&self.crystallized_state_root); + } +} + #[cfg(test)] mod tests { diff --git a/lighthouse/sync/messages.rs b/lighthouse/sync/messages.rs new file mode 100644 index 0000000000..9173f1c40d --- /dev/null +++ b/lighthouse/sync/messages.rs @@ -0,0 +1,12 @@ +pub enum SyncEventType { + Invalid, + PeerConnect, + PeerDrop, + ReceiveBlocks, + ReceiveAttestationRecords, +} + +pub struct SyncEvent { + event: SyncEventType, + data: Option> +} diff --git a/lighthouse/sync/mod.rs b/lighthouse/sync/mod.rs index a668093f4a..3d7d2d8d27 100644 --- a/lighthouse/sync/mod.rs +++ b/lighthouse/sync/mod.rs @@ -1,50 +1,13 @@ extern crate futures; extern crate slog; extern crate tokio; +extern crate network_libp2p; -use self::futures::sync::mpsc::{ - UnboundedReceiver, - UnboundedSender, -}; -use self::tokio::prelude::*; -use std::sync::{ RwLock, Arc }; -use super::network_libp2p::message::{ - NetworkEvent, - OutgoingMessage, -}; -use super::db::DB; -use slog::Logger; +pub mod messages; +pub mod network; +pub mod sync_future; +pub mod wire_protocol; -type NetworkSender = UnboundedSender; -type NetworkReceiver = UnboundedReceiver; +pub use self::sync_future::run_sync_future; -type SyncSender = UnboundedSender>; -type SyncReceiver = UnboundedReceiver>; - -/// Start a syncing tokio future. -/// -/// This is effectively a stub function being -/// used to test network functionality. -/// -/// Expect a full re-write. -pub fn start_sync( - _db: Arc>, - _network_tx: NetworkSender, - network_rx: NetworkReceiver, - _sync_tx: SyncSender, - _sync_rx: SyncReceiver, - log: Logger) { - let rx_future = network_rx - .for_each(move |event| { - debug!(&log, "Sync receive"; - "msg" => format!("{:?}", event)); - Ok(()) - }) - .map_err(|_| panic!("rx failed")); - - /* - * This is an unfinished stub function. - */ - - tokio::run(rx_future); -} +use super::db; diff --git a/lighthouse/sync/network.rs b/lighthouse/sync/network.rs new file mode 100644 index 0000000000..c2d1f1149c --- /dev/null +++ b/lighthouse/sync/network.rs @@ -0,0 +1,66 @@ +use std::sync::{ RwLock, Arc }; +use super::db::DB; +use slog::Logger; + +use super::network_libp2p::message::{ + NetworkEvent, + OutgoingMessage, + NetworkEventType, +}; + +use super::wire_protocol::{ WireMessageType, message_type }; + +use super::futures::sync::mpsc::{ + UnboundedSender, +}; + +/// Accept a network event and perform all required processing. +/// +/// This function should be called whenever an underlying network +/// (e.g., libp2p) has an event to push up to the sync process. +pub fn handle_network_event( + event: NetworkEvent, + db: Arc>, + network_tx: UnboundedSender, + log: Logger) + -> Result<(), ()> +{ + debug!(&log, ""; + "network_event" => format!("{:?}", &event)); + match event.event { + NetworkEventType::PeerConnect => Ok(()), + NetworkEventType::PeerDrop => Ok(()), + NetworkEventType::Message => { + if let Some(data) = event.data { + handle_network_message( + data, + db, + network_tx, + log) + } else { + Ok(()) + } + } + } +} + +/// Accept a message from the network and perform all required +/// processing. +/// +/// This function should be called whenever a peer from a network +/// (e.g., libp2p) has sent a message to us. +fn handle_network_message( + message: Vec, + _db: Arc>, + _network_tx: UnboundedSender, + _log: Logger) + -> Result<(), ()> +{ + match message_type(&message) { + Some(WireMessageType::Blocks) => { + // Do something with inbound blocks. + Ok(()) + } + _ => Ok(()) + } +} diff --git a/lighthouse/sync/sync_future.rs b/lighthouse/sync/sync_future.rs new file mode 100644 index 0000000000..c3b2355fb7 --- /dev/null +++ b/lighthouse/sync/sync_future.rs @@ -0,0 +1,48 @@ +use super::tokio; +use super::futures::{ Future, Stream }; +use super::futures::sync::mpsc::{ + UnboundedReceiver, + UnboundedSender, +}; +use super::network_libp2p::message::{ + NetworkEvent, + OutgoingMessage, +}; +use super::network::handle_network_event; +use std::sync::{ RwLock, Arc }; +use super::db::DB; +use slog::Logger; + +type NetworkSender = UnboundedSender; +type NetworkReceiver = UnboundedReceiver; + +type SyncSender = UnboundedSender>; +type SyncReceiver = UnboundedReceiver>; + +/// Start a syncing tokio future. +/// +/// Uses green-threading to process messages +/// from the network and the RPC and update +/// the state. +pub fn run_sync_future( + db: Arc>, + network_tx: NetworkSender, + network_rx: NetworkReceiver, + _sync_tx: SyncSender, + _sync_rx: SyncReceiver, + log: Logger) +{ + let network_future = { + network_rx + .for_each(move |event| { + handle_network_event( + event, + db.clone(), + network_tx.clone(), + log.clone()) + }) + .map_err(|_| panic!("rx failed")) + }; + + tokio::run(network_future); +} diff --git a/lighthouse/sync/wire_protocol.rs b/lighthouse/sync/wire_protocol.rs new file mode 100644 index 0000000000..5d4f772041 --- /dev/null +++ b/lighthouse/sync/wire_protocol.rs @@ -0,0 +1,24 @@ +pub enum WireMessageType { + Status, + NewBlockHashes, + GetBlockHashes, + BlockHashes, + GetBlocks, + Blocks, + NewBlock, +} + + +/// Determines the message type of some given +/// message. +/// +/// Does not check the validity of the message data, +/// it just reads the first byte. +pub fn message_type(message: &Vec) + -> Option +{ + match message.get(0) { + Some(0x06) => Some(WireMessageType::Blocks), + _ => None + } +} diff --git a/network-libp2p/src/service.rs b/network-libp2p/src/service.rs index 1189c54447..5994ef0d5b 100644 --- a/network-libp2p/src/service.rs +++ b/network-libp2p/src/service.rs @@ -18,7 +18,7 @@ use super::state::NetworkState; use super::message::{ NetworkEvent, NetworkEventType, OutgoingMessage }; use self::bigint::U512; use self::futures::{ Future, Stream, Poll }; -use self::futures::sync::mpsc::{ +use self::futures::sync::mpsc::{ UnboundedSender, UnboundedReceiver }; use self::libp2p_core::{ AddrComponent, Endpoint, Multiaddr, @@ -39,14 +39,14 @@ pub use self::libp2p_floodsub::Message; pub fn listen(state: NetworkState, events_to_app: UnboundedSender, raw_rx: UnboundedReceiver, - log: Logger) + log: Logger) { let peer_store = state.peer_store; let peer_id = state.peer_id; let listen_multiaddr = state.listen_multiaddr; let listened_addrs = Arc::new(RwLock::new(vec![])); let rx = ApplicationReciever{ inner: raw_rx }; - + // Build a tokio core let mut core = tokio_core::reactor::Core::new().expect("tokio failure."); // Build a base TCP libp2p transport @@ -65,10 +65,10 @@ pub fn listen(state: NetworkState, // is stored not the internal addr. .map(move |out, _, _| { if let(Some(ref observed), ref listen_multiaddr) = - (out.observed_addr, listen_multiaddr) + (out.observed_addr, listen_multiaddr) { - if let Some(viewed_from_outside) = - transport.nat_traversal(listen_multiaddr, observed) + if let Some(viewed_from_outside) = + transport.nat_traversal(listen_multiaddr, observed) { listened_addrs.write().unwrap() .push(viewed_from_outside); @@ -79,7 +79,7 @@ pub fn listen(state: NetworkState, }; // Configure and build a Kademlia upgrade to be applied - // to the base TCP transport. + // to the base TCP transport. let kad_config = libp2p_kad::KademliaConfig { parallelism: 3, record_store: (), @@ -91,10 +91,10 @@ pub fn listen(state: NetworkState, KademliaControllerPrototype::new(kad_config); let kad_upgrade = libp2p_kad:: KademliaUpgrade::from_prototype(&kad_ctl_proto); - + // Build a floodsub upgrade to allow pushing topic'ed // messages across the network. - let (floodsub_upgrade, floodsub_rx) = + let (floodsub_upgrade, floodsub_rx) = FloodSubUpgrade::new(peer_id.clone()); // Combine the Kademlia and Identify upgrades into a single @@ -104,7 +104,7 @@ pub fn listen(state: NetworkState, floodsub: floodsub_upgrade.clone(), identify: libp2p_identify::IdentifyProtocolConfig, }; - + // Build a Swarm to manage upgrading connections to peers. let swarm_listened_addrs = listened_addrs.clone(); let swarm_peer_id = peer_id.clone(); @@ -166,7 +166,7 @@ pub fn listen(state: NetworkState, for peer in peers { let peer_hash = U512::from(peer.hash()); let distance = 512 - (local_hash ^ peer_hash).leading_zeros(); - info!(kad_poll_log, "Discovered peer"; + info!(kad_poll_log, "Discovered peer"; "distance" => distance, "peer_id" => peer.to_base58()); let peer_addr = AddrComponent::P2P(peer.into_bytes()).into(); @@ -240,7 +240,7 @@ struct ConnectionUpgrader { } impl ConnectionUpgrade for ConnectionUpgrader -where +where C: AsyncRead + AsyncWrite + 'static, P: Deref + Clone + 'static, for<'r> &'r Pc: libp2p_peerstore::Peerstore, @@ -251,7 +251,7 @@ where type Output = FinalUpgrade; type Future = Box, Error = IoError>>; - #[inline] + #[inline] fn protocol_names(&self) -> Self::NamesIter { vec![ (Bytes::from("/ipfs/kad/1.0.0"), 0), diff --git a/ssz/src/decode.rs b/ssz/src/decode.rs new file mode 100644 index 0000000000..d726a58320 --- /dev/null +++ b/ssz/src/decode.rs @@ -0,0 +1,135 @@ +use super::{ + LENGTH_BYTES, +}; + +#[derive(Debug, PartialEq)] +pub enum DecodeError { + OutOfBounds, + TooShort, + TooLong, +} + +pub trait Decodable: Sized { + fn ssz_decode(bytes: &[u8]) -> Result; +} + +/// Decode the nth element of some ssz list. +/// +/// A single ssz encoded value can be considered a list of +/// one element, so this function will work on it too. +pub fn decode_ssz_list_element(ssz_bytes: &[u8], n: usize) + -> Result + where T: Decodable +{ + T::ssz_decode(nth_value(ssz_bytes, n)?) +} + +/// Return the nth value in some ssz encoded list. +/// +/// The four-byte length prefix is not included in the return. +/// +/// A single ssz encoded value can be considered a list of +/// one element, so this function will work on it too. +fn nth_value(ssz_bytes: &[u8], n: usize) + -> Result<&[u8], DecodeError> +{ + let mut c: usize = 0; + for i in 0..(n + 1) { + let length = decode_length(&ssz_bytes[c..], LENGTH_BYTES)?; + let next = c + LENGTH_BYTES + length; + + if i == n { + return Ok(&ssz_bytes[c + LENGTH_BYTES..next]); + } else { + if next >= ssz_bytes.len() { + return Err(DecodeError::OutOfBounds); + } else { + c = next; + } + } + } + Err(DecodeError::OutOfBounds) +} + +/// Given some number of bytes, interpret the first four +/// bytes as a 32-bit big-endian integer and return the +/// result. +fn decode_length(bytes: &[u8], length_bytes: usize) + -> Result +{ + if bytes.len() < length_bytes { + return Err(DecodeError::TooShort); + }; + let mut len: usize = 0; + for i in 0..length_bytes { + let offset = (length_bytes - i - 1) * 8; + len = ((bytes[i] as usize) << offset) | len; + }; + Ok(len) +} + +#[cfg(test)] +mod tests { + use super::*; + use super::super::encode::encode_length; + + #[test] + fn test_ssz_decode_length() { + let decoded = decode_length( + &vec![0, 0, 1], + LENGTH_BYTES); + assert_eq!(decoded.unwrap(), 1); + + let decoded = decode_length( + &vec![0, 1, 0], + LENGTH_BYTES); + assert_eq!(decoded.unwrap(), 256); + + let decoded = decode_length( + &vec![0, 1, 255], + LENGTH_BYTES); + assert_eq!(decoded.unwrap(), 511); + + let decoded = decode_length( + &vec![255, 255, 255], + LENGTH_BYTES); + assert_eq!(decoded.unwrap(), 16777215); + } + + #[test] + fn test_encode_decode_length() { + let params: Vec = vec![ + 0, 1, 2, 3, 7, 8, 16, + 2^8, 2^8 + 1, + 2^16, 2^16 + 1, + 2^24, 2^24 + 1, + 2^32, + ]; + for i in params { + let decoded = decode_length( + &encode_length(i, LENGTH_BYTES), + LENGTH_BYTES).unwrap(); + assert_eq!(i, decoded); + } + } + + #[test] + fn test_ssz_nth_value() { + let ssz = vec![0, 0, 1, 0]; + let result = nth_value(&ssz, 0).unwrap(); + assert_eq!(result, vec![0].as_slice()); + + let ssz = vec![0, 0, 4, 1, 2, 3, 4]; + let result = nth_value(&ssz, 0).unwrap(); + assert_eq!(result, vec![1, 2, 3, 4].as_slice()); + + let ssz = vec![0, 0, 1, 0, 0, 0, 1, 1]; + let result = nth_value(&ssz, 1).unwrap(); + assert_eq!(result, vec![1].as_slice()); + + let mut ssz = vec![0, 1, 255]; + ssz.append(&mut vec![42; 511]); + let result = nth_value(&ssz, 0).unwrap(); + assert_eq!(result, vec![42; 511].as_slice()); + } +} diff --git a/ssz/src/encode.rs b/ssz/src/encode.rs new file mode 100644 index 0000000000..7e1cb80eb2 --- /dev/null +++ b/ssz/src/encode.rs @@ -0,0 +1,166 @@ +use super::LENGTH_BYTES; + +pub trait Encodable { + fn ssz_append(&self, s: &mut SszStream); +} + +/// Provides a buffer for appending ssz-encodable values. +/// +/// Use the `append()` fn to add a value to a list, then use +/// the `drain()` method to consume the struct and return the +/// ssz encoded bytes. +pub struct SszStream { + buffer: Vec +} + +impl SszStream { + /// Create a new, empty stream for writing ssz values. + pub fn new() -> Self { + SszStream { + buffer: Vec::new() + } + } + + /// Append some ssz encodable value to the stream. + pub fn append(&mut self, value: &E) -> &mut Self + where E: Encodable + { + value.ssz_append(self); + self + } + + /// Append some ssz encoded bytes to the stream. + /// + /// The length of the supplied bytes will be concatenated + /// to the stream before the supplied bytes. + pub fn append_encoded_val(&mut self, vec: &Vec) { + self.buffer.extend_from_slice( + &encode_length(vec.len(), + LENGTH_BYTES)); + self.buffer.extend_from_slice(&vec); + } + + /// Append some vector (list) of encodable values to the stream. + /// + /// The length of the list will be concatenated to the stream, then + /// each item in the vector will be encoded and concatenated. + pub fn append_vec(&mut self, vec: &Vec) + where E: Encodable + { + self.buffer.extend_from_slice(&encode_length(vec.len(), LENGTH_BYTES)); + for v in vec { + v.ssz_append(self); + } + } + + /// Consume the stream and return the underlying bytes. + pub fn drain(self) -> Vec { + self.buffer + } +} + +/// Encode some length into a ssz size prefix. +/// +/// The ssz size prefix is 4 bytes, which is treated as a continuious +/// 32bit big-endian integer. +pub fn encode_length(len: usize, length_bytes: usize) -> Vec { + assert!(length_bytes > 0); // For sanity + assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8)); + let mut header: Vec = vec![0; length_bytes]; + for i in 0..length_bytes { + let offset = (length_bytes - i - 1) * 8; + header[i] = ((len >> offset) & 0xff) as u8; + }; + header +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic] + fn test_encode_length_0_bytes_panic() { + encode_length(0, 0); + } + + #[test] + fn test_encode_length_4_bytes() { + assert_eq!( + encode_length(0, LENGTH_BYTES), + vec![0; 3] + ); + assert_eq!( + encode_length(1, LENGTH_BYTES), + vec![0, 0, 1] + ); + assert_eq!( + encode_length(255, LENGTH_BYTES), + vec![0, 0, 255] + ); + assert_eq!( + encode_length(256, LENGTH_BYTES), + vec![0, 1, 0] + ); + assert_eq!( + encode_length(16777215, LENGTH_BYTES), // 2^(3*8) - 1 + vec![255, 255, 255] + ); + } + + #[test] + #[should_panic] + fn test_encode_length_4_bytes_panic() { + encode_length(16777216, LENGTH_BYTES); // 2^(3*8) + } + + /* + #[test] + fn test_encode_struct() { + pub struct TestStruct { + pub one: u32, + pub two: H256, + pub three: u64, + pub four: U256, + } + + impl Encodable for TestStruct { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.one); + s.append(&self.two); + s.append(&self.three); + s.append(&self.four); + } + } + + let t = TestStruct { + one: 1, + two: H256::zero(), + three: 100, + four: U256::zero(), + }; + + let mut s = SszStream::new(); + s.append(&t); + let e = s.drain(); + + let expected_len = { + 3 + 4 + + 3 + 32 + + 3 + 8 + + 3 + 32 + }; + + assert_eq!(e[0..4], [0, 0, 0, 4]); + assert_eq!(e[4..8], [0, 0, 0, 1]); + assert_eq!(e[8..12], [0, 0, 0, 32]); + assert_eq!(e[12..44], [0; 32]); + assert_eq!(e[44..48], [0, 0, 0, 8]); + assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]); + assert_eq!(e[56..60], [0, 0, 0, 32]); + assert_eq!(e[60..92], [0; 32]); + assert_eq!(e.len(), expected_len); + } + */ +} diff --git a/ssz/src/impl_decode.rs b/ssz/src/impl_decode.rs new file mode 100644 index 0000000000..55fdf31e2e --- /dev/null +++ b/ssz/src/impl_decode.rs @@ -0,0 +1,114 @@ +use super::{ + DecodeError, + Decodable, +}; +macro_rules! impl_decodable_for_uint { + ($type: ident, $bit_size: expr) => { + impl Decodable for $type { + fn ssz_decode(bytes: &[u8]) + -> Result + { + assert!((0 < $bit_size) & + ($bit_size <= 64) & + ($bit_size % 8 == 0)); + let max_bytes = $bit_size / 8; + if bytes.len() <= max_bytes { + let mut result: $type = 0; + for i in 0..bytes.len() { + let offset = (bytes.len() - i - 1) * 8; + result = ((bytes[i] as $type) << offset) | result; + }; + Ok(result) + } else { + Err(DecodeError::TooLong) + } + } + } + } +} + +impl_decodable_for_uint!(u16, 16); +impl_decodable_for_uint!(u32, 32); +impl_decodable_for_uint!(u64, 64); +impl_decodable_for_uint!(usize, 64); + + +#[cfg(test)] +mod tests { + use super::super::{ + DecodeError, + decode_ssz_list_element, + }; + + #[test] + fn test_ssz_decode_u16() { + let ssz = vec![0, 0, 1, 0]; + let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 0); + + let ssz = vec![0, 0, 1, 16]; + let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 16); + + let ssz = vec![0, 0, 2, 1, 0]; + let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 256); + + let ssz = vec![0, 0, 2, 255, 255]; + let result: u16 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 65535); + + let ssz = vec![0, 0, 3, 0, 0, 1]; + let result: Result = + decode_ssz_list_element(&ssz, 0); + assert_eq!(result, Err(DecodeError::TooLong)); + } + + #[test] + fn test_ssz_decode_u32() { + let ssz = vec![0, 0, 1, 0]; + let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 0); + + let ssz = vec![0, 0, 4, 255, 255, 255, 255]; + let result: u32 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 4294967295); + + let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + let result: Result = + decode_ssz_list_element(&ssz, 0); + assert_eq!(result, Err(DecodeError::TooLong)); + } + + #[test] + fn test_ssz_decode_u64() { + let ssz = vec![0, 0, 1, 0]; + let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 0); + + let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]; + let result: u64 = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 18446744073709551615); + + let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + let result: Result = + decode_ssz_list_element(&ssz, 0); + assert_eq!(result, Err(DecodeError::TooLong)); + } + + #[test] + fn test_ssz_decode_usize() { + let ssz = vec![0, 0, 1, 0]; + let result: usize = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 0); + + let ssz = vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]; + let result: usize = decode_ssz_list_element(&ssz, 0).unwrap(); + assert_eq!(result, 18446744073709551615); + + let ssz = vec![0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + let result: Result = + decode_ssz_list_element(&ssz, 0); + assert_eq!(result, Err(DecodeError::TooLong)); + } +} diff --git a/ssz/src/impl_encode.rs b/ssz/src/impl_encode.rs new file mode 100644 index 0000000000..88b4c1edc8 --- /dev/null +++ b/ssz/src/impl_encode.rs @@ -0,0 +1,186 @@ +use super::{ + Encodable, + SszStream +}; +use super::ethereum_types::{ H256, U256 }; + +/* + * Note: there is a "to_bytes" function for integers + * in Rust nightly. When it is in stable, we should + * use it instead. + */ +macro_rules! impl_encodable_for_uint { + ($type: ident) => { + impl Encodable for $type { + fn ssz_append(&self, s: &mut SszStream) + { + // Number of bits required to represent this integer. + // This could be optimised at the expense of complexity. + let num_bits = { + let mut n = *self; + let mut r: usize = 0; + while n > 0 { + n >>= 1; + r += 1; + } + if r == 0 { 1 } else { r } + }; + // Number of bytes required to represent this bit + let num_bytes = (num_bits + 8 - 1) / 8; + let mut ssz_val: Vec = Vec::with_capacity(num_bytes); + ssz_val.resize(num_bytes, 0); + for i in (0..num_bytes).rev() { + let offset = (num_bytes - i - 1) * 8; + ssz_val[i] = 0_u8 | (self >> offset) as u8 + } + s.append_encoded_val(&ssz_val); + } + } + } +} + +impl_encodable_for_uint!(u8); +impl_encodable_for_uint!(u16); +impl_encodable_for_uint!(u32); +impl_encodable_for_uint!(u64); +impl_encodable_for_uint!(usize); + +impl Encodable for H256 { + fn ssz_append(&self, s: &mut SszStream) { + s.append_encoded_val(&self.to_vec()); + } +} + +impl Encodable for U256 { + fn ssz_append(&self, s: &mut SszStream) { + let mut a = [0; 32]; + self.to_big_endian(&mut a); + s.append_encoded_val(&a.to_vec()); + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ssz_encode_u8() { + let x: u16 = 0; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 0]); + + let x: u16 = 1; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 1]); + + let x: u16 = 100; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 100]); + + let x: u16 = 255; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 255]); + } + + #[test] + fn test_ssz_encode_u16() { + let x: u16 = 1; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 1]); + + let x: u16 = 100; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 100]); + + let x: u16 = 1 << 8; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 2, 1, 0]); + + let x: u16 = 65535; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 2, 255, 255]); + } + + #[test] + fn test_ssz_encode_u32() { + let x: u32 = 1; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 1]); + + let x: u32 = 100; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 100]); + + let x: u32 = 1 << 16; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 3, 1, 0, 0]); + + let x: u32 = 1 << 24; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 4, 1, 0, 0, 0]); + + let x: u32 = !0; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 4, 255, 255, 255, 255]); + } + + #[test] + fn test_ssz_encode_u64() { + let x: u64 = 1; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 1]); + + let x: u64 = 100; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 100]); + + let x: u64 = 1 << 32; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]); + + let x: u64 = !0; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]); + } + + #[test] + fn test_ssz_encode_usize() { + let x: usize = 1; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 1]); + + let x: usize = 100; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 1, 100]); + + let x: usize = 1 << 32; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 5, 1, 0, 0, 0, 0]); + + let x: usize = !0; + let mut ssz = SszStream::new(); + ssz.append(&x); + assert_eq!(ssz.drain(), vec![0, 0, 8, 255, 255, 255, 255, 255, 255, 255, 255]); + } +} diff --git a/ssz/src/lib.rs b/ssz/src/lib.rs index 9b392a8758..b5091c1fb3 100644 --- a/ssz/src/lib.rs +++ b/ssz/src/lib.rs @@ -1,8 +1,8 @@ /* - * This is a WIP of implementing an alternative + * This is a WIP of implementing an alternative * serialization strategy. It attempts to follow Vitalik's - * "ssz" format here: - * https://github.com/ethereum/research/tree/master/py_ssz + * "simpleserialize" format here: + * https://github.com/ethereum/beacon_chain/blob/master/beacon_chain/utils/simpleserialize.py * * This implementation is not final and would almost certainly * have issues. @@ -10,172 +10,19 @@ extern crate bytes; extern crate ethereum_types; -use self::bytes::{ BytesMut, BufMut }; -use self::ethereum_types::{ H256, U256 }; +mod encode; +mod decode; +mod impl_encode; +mod impl_decode; -pub const LENGTH_BYTES: usize = 4; +pub use decode::{ + Decodable, + DecodeError, + decode_ssz_list_element, +}; +pub use encode::{ + Encodable, + SszStream, +}; -pub trait Encodable { - fn ssz_append(&self, s: &mut SszStream); -} - -pub struct SszStream { - buffer: Vec -} - -impl SszStream { - pub fn new() -> Self { - SszStream { - buffer: Vec::new() - } - } - - pub fn append(&mut self, value: &E) -> &mut Self - where E: Encodable - { - value.ssz_append(self); - self - } - - fn append_encoded_vec(&mut self, v: &mut Vec) { - self.buffer.append(&mut encode_length(v.len(), LENGTH_BYTES)); - self.buffer.append(v) ; - } - - fn append_encoded_array(&mut self, a: &mut [u8]) { - let len = a.len(); - self.buffer.append(&mut encode_length(len, LENGTH_BYTES)); - self.buffer.extend_from_slice(&a[0..len]); - } - - pub fn drain(self) -> Vec { - self.buffer - } -} - -pub fn encode(value: &E) -> Vec - where E: Encodable -{ - let mut stream = SszStream::new(); - stream.append(value); - stream.drain() -} - -fn encode_length(len: usize, length_bytes: usize) -> Vec { - assert!(length_bytes > 0); // For sanity - assert!((len as usize) < 2usize.pow(length_bytes as u32 * 8)); - let mut header: Vec = vec![0; length_bytes]; - for i in 0..length_bytes { - let offset = (length_bytes - i - 1) * 8; - header[i] = ((len >> offset) & 0xff) as u8; - }; - header -} - -/* - * Implementations for various types - */ - -impl Encodable for u32 { - fn ssz_append(&self, s: &mut SszStream) { - let mut buf = BytesMut::with_capacity(32/8); - buf.put_u32_be(*self); - s.append_encoded_vec(&mut buf.to_vec()); - } -} - -impl Encodable for u64 { - fn ssz_append(&self, s: &mut SszStream) { - let mut buf = BytesMut::with_capacity(64/8); - buf.put_u64_be(*self); - s.append_encoded_vec(&mut buf.to_vec()); - } -} - -impl Encodable for H256 { - fn ssz_append(&self, s: &mut SszStream) { - s.append_encoded_vec(&mut self.to_vec()); - } -} - -impl Encodable for U256 { - fn ssz_append(&self, s: &mut SszStream) { - let mut a = [0; 32]; - self.to_big_endian(&mut a); - s.append_encoded_array(&mut a); - } -} - - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - #[should_panic] - fn test_encode_length_0_bytes_panic() { - encode_length(0, 0); - } - - #[test] - fn test_encode_length_4_bytes() { - assert_eq!( - encode_length(0, 4), - vec![0; 4] - ); - assert_eq!( - encode_length(1, 4), - vec![0, 0, 0, 1] - ); - assert_eq!( - encode_length(255, 4), - vec![0, 0, 0, 255] - ); - assert_eq!( - encode_length(256, 4), - vec![0, 0, 1, 0] - ); - assert_eq!( - encode_length(4294967295, 4), // 2^(4*8) - 1 - vec![255, 255, 255, 255] - ); - } - - #[test] - #[should_panic] - fn test_encode_length_4_bytes_panic() { - encode_length(4294967296, 4); // 2^(4*8) - } - - #[test] - fn test_serialization() { - pub struct TestStruct { - pub one: u32, - pub two: H256, - pub three: u64, - } - - impl Encodable for TestStruct { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.one); - s.append(&self.two); - s.append(&self.three); - } - } - - let t = TestStruct { - one: 1, - two: H256::zero(), - three: 100 - }; - - let e = encode(&t); - assert_eq!(e[0..4], [0, 0, 0, 4]); - assert_eq!(e[4..8], [0, 0, 0, 1]); - assert_eq!(e[8..12], [0, 0, 0, 32]); - assert_eq!(e[12..44], [0; 32]); - assert_eq!(e[44..48], [0, 0, 0, 8]); - assert_eq!(e[48..56], [0, 0, 0, 0, 0, 0, 0, 100]); - assert_eq!(e.len(), 56); - } -} +pub const LENGTH_BYTES: usize = 3;