Merge branch 'master' into validator-record-update

beacon_chain/attestation_validation/src/signature.rs

@@ -27,7 +27,7 @@ where
                 .get_public_key_by_index(validator)?
                 .ok_or(Error::NoPublicKeyForValidator)?;
             // Aggregate the public key.
-            agg_pub_key.add(&pub_key);
+            agg_pub_key.add(&pub_key.as_raw());
         }
     }
 
@@ -129,7 +129,8 @@ mod tests {
             &agg_sig,
             &attestation_indices,
             &store,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(outcome, Outcome::Valid);
 
         /*
@@ -143,7 +144,8 @@ mod tests {
             &agg_sig,
             &attestation_indices,
             &store,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(outcome, Outcome::Invalid(Invalid::SignatureInvalid));
     }
 }

beacon_chain/chain/src/maps.rs

@@ -45,7 +45,7 @@ mod tests {
     use super::*;
 
     fn sac_generator(
-        shard_count: u16,
+        shard_count: u64,
         slot_count: usize,
         sac_per_slot: usize,
         committee_size: usize,

beacon_chain/types/src/beacon_state.rs

@@ -6,6 +6,9 @@ use super::shard_committee::ShardCommittee;
 use super::shard_reassignment_record::ShardReassignmentRecord;
 use super::validator_record::ValidatorRecord;
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
 
 #[derive(Debug, PartialEq, Clone, Default)]
 pub struct BeaconState {
@@ -52,3 +55,114 @@ impl BeaconState {
         Hash256::zero()
     }
 }
+
+impl Encodable for BeaconState {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.slot);
+        s.append(&self.genesis_time);
+        s.append(&self.fork_data);
+        s.append(&self.validator_registry);
+        s.append(&self.validator_registry_latest_change_slot);
+        s.append(&self.validator_registry_exit_count);
+        s.append(&self.validator_registry_delta_chain_tip);
+        s.append(&self.randao_mix);
+        s.append(&self.next_seed);
+        s.append(&self.shard_committees_at_slots);
+        s.append(&self.persistent_committees);
+        s.append(&self.persistent_committee_reassignments);
+        s.append(&self.previous_justified_slot);
+        s.append(&self.justified_slot);
+        s.append(&self.justification_bitfield);
+        s.append(&self.finalized_slot);
+        s.append(&self.latest_crosslinks);
+        s.append(&self.latest_block_roots);
+        s.append(&self.latest_penalized_exit_balances);
+        s.append(&self.latest_attestations);
+        s.append(&self.processed_pow_receipt_root);
+        s.append(&self.candidate_pow_receipt_roots);
+    }
+}
+
+impl Decodable for BeaconState {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (genesis_time, i) = <_>::ssz_decode(bytes, i)?;
+        let (fork_data, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_latest_change_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_exit_count, i) = <_>::ssz_decode(bytes, i)?;
+        let (validator_registry_delta_chain_tip, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_mix, i) = <_>::ssz_decode(bytes, i)?;
+        let (next_seed, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard_committees_at_slots, i) = <_>::ssz_decode(bytes, i)?;
+        let (persistent_committees, i) = <_>::ssz_decode(bytes, i)?;
+        let (persistent_committee_reassignments, i) = <_>::ssz_decode(bytes, i)?;
+        let (previous_justified_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (justified_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (justification_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (finalized_slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_crosslinks, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_block_roots, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_penalized_exit_balances, i) = <_>::ssz_decode(bytes, i)?;
+        let (latest_attestations, i) = <_>::ssz_decode(bytes, i)?;
+        let (processed_pow_receipt_root, i) = <_>::ssz_decode(bytes, i)?;
+        let (candidate_pow_receipt_roots, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                slot,
+                genesis_time,
+                fork_data,
+                validator_registry,
+                validator_registry_latest_change_slot,
+                validator_registry_exit_count,
+                validator_registry_delta_chain_tip,
+                randao_mix,
+                next_seed,
+                shard_committees_at_slots,
+                persistent_committees,
+                persistent_committee_reassignments,
+                previous_justified_slot,
+                justified_slot,
+                justification_bitfield,
+                finalized_slot,
+                latest_crosslinks,
+                latest_block_roots,
+                latest_penalized_exit_balances,
+                latest_attestations,
+                processed_pow_receipt_root,
+                candidate_pow_receipt_roots,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for BeaconState {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            slot: <_>::random_for_test(rng),
+            genesis_time: <_>::random_for_test(rng),
+            fork_data: <_>::random_for_test(rng),
+            validator_registry: <_>::random_for_test(rng),
+            validator_registry_latest_change_slot: <_>::random_for_test(rng),
+            validator_registry_exit_count: <_>::random_for_test(rng),
+            validator_registry_delta_chain_tip: <_>::random_for_test(rng),
+            randao_mix: <_>::random_for_test(rng),
+            next_seed: <_>::random_for_test(rng),
+            shard_committees_at_slots: <_>::random_for_test(rng),
+            persistent_committees: <_>::random_for_test(rng),
+            persistent_committee_reassignments: <_>::random_for_test(rng),
+            previous_justified_slot: <_>::random_for_test(rng),
+            justified_slot: <_>::random_for_test(rng),
+            justification_bitfield: <_>::random_for_test(rng),
+            finalized_slot: <_>::random_for_test(rng),
+            latest_crosslinks: <_>::random_for_test(rng),
+            latest_block_roots: <_>::random_for_test(rng),
+            latest_penalized_exit_balances: <_>::random_for_test(rng),
+            latest_attestations: <_>::random_for_test(rng),
+            processed_pow_receipt_root: <_>::random_for_test(rng),
+            candidate_pow_receipt_roots: <_>::random_for_test(rng),
+        }
+    }
+}

beacon_chain/types/src/candidate_pow_receipt_root_record.rs

@@ -1,7 +1,60 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Debug, PartialEq, Clone)]
 pub struct CandidatePoWReceiptRootRecord {
     pub candidate_pow_receipt_root: Hash256,
     pub votes: u64,
 }
+
+impl Encodable for CandidatePoWReceiptRootRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.candidate_pow_receipt_root);
+        s.append(&self.votes);
+    }
+}
+
+impl Decodable for CandidatePoWReceiptRootRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (candidate_pow_receipt_root, i) = <_>::ssz_decode(bytes, i)?;
+        let (votes, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                candidate_pow_receipt_root,
+                votes,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for CandidatePoWReceiptRootRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            candidate_pow_receipt_root: <_>::random_for_test(rng),
+            votes: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = CandidatePoWReceiptRootRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/common/maps.rs

@@ -1,7 +0,0 @@
-use std::collections::HashMap;
-
-/// Maps a (slot, shard_id) to attestation_indices.
-pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
-
-/// Maps a slot to a block proposer.
-pub type ProposerMap = HashMap<u64, usize>;

beacon_chain/types/src/common/mod.rs

@@ -1,9 +0,0 @@
-mod delegation;
-mod shuffling;
-
-pub mod maps;
-pub mod attestation_parent_hashes;
-
-use super::utils;
-use super::utils::types::Hash256;
-pub use self::shuffling::shuffle;

beacon_chain/types/src/crosslink_record.rs

@@ -1,4 +1,7 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct CrosslinkRecord {
@@ -15,3 +18,53 @@ impl CrosslinkRecord {
         }
     }
 }
+
+impl Encodable for CrosslinkRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.slot);
+        s.append(&self.shard_block_root);
+    }
+}
+
+impl Decodable for CrosslinkRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard_block_root, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                slot,
+                shard_block_root,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for CrosslinkRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            slot: <_>::random_for_test(rng),
+            shard_block_root: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = CrosslinkRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/deposit_input.rs

@@ -1,4 +1,4 @@
-use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
@@ -15,7 +15,7 @@ pub struct DepositInput {
 
 impl Encodable for DepositInput {
     fn ssz_append(&self, s: &mut SszStream) {
-        s.append_vec(&self.pubkey.as_bytes());
+        s.append(&self.pubkey);
         s.append(&self.withdrawal_credentials);
         s.append(&self.randao_commitment);
         s.append(&self.poc_commitment);
@@ -25,8 +25,7 @@ impl Encodable for DepositInput {
 
 impl Decodable for DepositInput {
     fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
-        let (pubkey_bytes, i) = decode_ssz_list(bytes, i)?;
-        let pubkey = PublicKey::from_bytes(&pubkey_bytes).map_err(|_| DecodeError::TooShort)?;
+        let (pubkey, i) = <_>::ssz_decode(bytes, i)?;
         let (withdrawal_credentials, i) = <_>::ssz_decode(bytes, i)?;
         let (randao_commitment, i) = <_>::ssz_decode(bytes, i)?;
         let (poc_commitment, i) = <_>::ssz_decode(bytes, i)?;

beacon_chain/types/src/fork_data.rs

@@ -1,6 +1,64 @@
-#[derive(Debug, Clone, PartialEq, Default)]
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
+#[derive(Debug, Clone, PartialEq)]
 pub struct ForkData {
     pub pre_fork_version: u64,
     pub post_fork_version: u64,
     pub fork_slot: u64,
 }
+
+impl Encodable for ForkData {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.pre_fork_version);
+        s.append(&self.post_fork_version);
+        s.append(&self.fork_slot);
+    }
+}
+
+impl Decodable for ForkData {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (pre_fork_version, i) = <_>::ssz_decode(bytes, i)?;
+        let (post_fork_version, i) = <_>::ssz_decode(bytes, i)?;
+        let (fork_slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                pre_fork_version,
+                post_fork_version,
+                fork_slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ForkData {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            pre_fork_version: <_>::random_for_test(rng),
+            post_fork_version: <_>::random_for_test(rng),
+            fork_slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ForkData::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/lib.rs

@@ -63,7 +63,7 @@ pub type Bitfield = boolean_bitfield::BooleanBitfield;
 pub type BitfieldError = boolean_bitfield::Error;
 
 /// Maps a (slot, shard_id) to attestation_indices.
-pub type AttesterMap = HashMap<(u64, u16), Vec<usize>>;
+pub type AttesterMap = HashMap<(u64, u64), Vec<usize>>;
 
 /// Maps a slot to a block proposer.
 pub type ProposerMap = HashMap<u64, usize>;

beacon_chain/types/src/pending_attestation_record.rs

@@ -1,4 +1,7 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
 use super::{AttestationData, Bitfield};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct PendingAttestationRecord {
@@ -7,3 +10,61 @@ pub struct PendingAttestationRecord {
     pub custody_bitfield: Bitfield,
     pub slot_included: u64,
 }
+
+impl Encodable for PendingAttestationRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.data);
+        s.append(&self.participation_bitfield);
+        s.append(&self.custody_bitfield);
+        s.append(&self.slot_included);
+    }
+}
+
+impl Decodable for PendingAttestationRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (data, i) = <_>::ssz_decode(bytes, i)?;
+        let (participation_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (custody_bitfield, i) = <_>::ssz_decode(bytes, i)?;
+        let (slot_included, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                data,
+                participation_bitfield,
+                custody_bitfield,
+                slot_included,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for PendingAttestationRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            data: <_>::random_for_test(rng),
+            participation_bitfield: <_>::random_for_test(rng),
+            custody_bitfield: <_>::random_for_test(rng),
+            slot_included: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = PendingAttestationRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/shard_committee.rs

@@ -1,28 +1,53 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
 #[derive(Clone, Debug, PartialEq)]
 pub struct ShardCommittee {
-    pub shard: u16,
+    pub shard: u64,
     pub committee: Vec<usize>,
 }
 
-impl ShardCommittee {
-    /// Returns a new instance where the `shard_id` is zero and the
-    /// committee is an empty vector.
-    pub fn zero() -> Self {
+impl Encodable for ShardCommittee {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.shard);
+        s.append(&self.committee);
+    }
+}
+
+impl Decodable for ShardCommittee {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (committee, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((Self { shard, committee }, i))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ShardCommittee {
+    fn random_for_test(rng: &mut T) -> Self {
         Self {
-            shard: 0,
-            committee: vec![],
+            shard: <_>::random_for_test(rng),
+            committee: <_>::random_for_test(rng),
         }
     }
 }
 
 #[cfg(test)]
 mod tests {
+    use super::super::ssz::ssz_encode;
     use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
 
     #[test]
-    fn test_shard_and_committee_zero() {
-        let s = ShardCommittee::zero();
-        assert_eq!(s.shard, 0);
-        assert_eq!(s.committee.len(), 0);
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ShardCommittee::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
     }
 }

beacon_chain/types/src/shard_reassignment_record.rs

@@ -1,6 +1,64 @@
+use super::ssz::{Decodable, DecodeError, Encodable, SszStream};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+
 #[derive(Debug, PartialEq, Clone)]
 pub struct ShardReassignmentRecord {
     pub validator_index: u64,
     pub shard: u64,
     pub slot: u64,
 }
+
+impl Encodable for ShardReassignmentRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.validator_index);
+        s.append(&self.shard);
+        s.append(&self.slot);
+    }
+}
+
+impl Decodable for ShardReassignmentRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (validator_index, i) = <_>::ssz_decode(bytes, i)?;
+        let (shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                validator_index,
+                shard,
+                slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ShardReassignmentRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            validator_index: <_>::random_for_test(rng),
+            shard: <_>::random_for_test(rng),
+            slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ShardReassignmentRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/types/src/test_utils/address.rs

@@ -0,0 +1,11 @@
+use super::TestRandom;
+use crate::Address;
+use rand::RngCore;
+
+impl<T: RngCore> TestRandom<T> for Address {
+    fn random_for_test(rng: &mut T) -> Self {
+        let mut key_bytes = vec![0; 20];
+        rng.fill_bytes(&mut key_bytes);
+        Address::from(&key_bytes[..])
+    }
+}

beacon_chain/types/src/test_utils/mod.rs

@@ -2,6 +2,7 @@ use rand::RngCore;
 
 pub use rand::{prng::XorShiftRng, SeedableRng};
 
+pub mod address;
 pub mod aggregate_signature;
 pub mod bitfield;
 pub mod hash256;
@@ -27,6 +28,12 @@ impl<T: RngCore> TestRandom<T> for u32 {
     }
 }
 
+impl<T: RngCore> TestRandom<T> for usize {
+    fn random_for_test(rng: &mut T) -> Self {
+        rng.next_u32() as usize
+    }
+}
+
 impl<T: RngCore, U> TestRandom<T> for Vec<U>
 where U: TestRandom<T>
 {

beacon_chain/types/src/validator_record.rs

@@ -1,5 +1,8 @@
-use super::bls::{Keypair, PublicKey};
-use super::{Hash256};
+use super::bls::PublicKey;
+use super::{Address, Hash256};
+use crate::test_utils::TestRandom;
+use rand::RngCore;
+use ssz::{Decodable, DecodeError, Encodable, SszStream};
 use std::convert;
 
 #[derive(Debug, PartialEq, Clone)]
@@ -41,47 +44,136 @@ pub struct ValidatorRecord {
 }
 
 impl ValidatorRecord {
-    /// Generates a new instance where the keypair is generated using
-    /// `rand::thread_rng` entropy and all other fields are set to zero.
-    ///
-    /// Returns the new instance and new keypair.
-    pub fn zero_with_thread_rand_keypair() -> (Self, Keypair) {
-        let keypair = Keypair::random();
-        let s = Self {
-            pubkey: keypair.pk.clone(),
-            withdrawal_credentials: Hash256::zero(),
-            randao_commitment: Hash256::zero(),
-            randao_layers: 0,
-            status: From::from(0),
-            latest_status_change_slot: 0,
-            exit_count: 0,
-            poc_commitment: Hash256::zero(),
-            last_poc_change_slot: 0,
-            second_last_poc_slot: 0
-        };
-        (s, keypair)
-    }
-
     pub fn status_is(&self, status: ValidatorStatus) -> bool {
         self.status == status
     }
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_validator_record_zero_rand_keypair() {
-        let (v, _kp) = ValidatorRecord::zero_with_thread_rand_keypair();
-        assert!(v.withdrawal_credentials.is_zero());
-        assert!(v.randao_commitment.is_zero());
-        assert_eq!(v.randao_layers, 0);
-        assert_eq!(v.status, From::from(0));
-        assert_eq!(v.latest_status_change_slot, 0);
-        assert_eq!(v.exit_count, 0);
-        assert!(v.poc_commitment.is_zero());
-        assert_eq!(v.last_poc_change_slot, 0);
-        assert_eq!(v.second_last_poc_slot, 0);
+impl Encodable for ValidatorStatus {
+    fn ssz_append(&self, s: &mut SszStream) {
+        let byte: u8 = match self {
+            ValidatorStatus::PendingActivation => 0,
+            ValidatorStatus::Active => 1,
+            ValidatorStatus::PendingExit => 2,
+            ValidatorStatus::PendingWithdraw => 3,
+            ValidatorStatus::Withdrawn => 5,
+            ValidatorStatus::Penalized => 127,
+        };
+        s.append(&byte);
+    }
+}
+
+impl Decodable for ValidatorStatus {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (byte, i) = u8::ssz_decode(bytes, i)?;
+        let status = match byte {
+            0 => ValidatorStatus::PendingActivation,
+            1 => ValidatorStatus::Active,
+            2 => ValidatorStatus::PendingExit,
+            3 => ValidatorStatus::PendingWithdraw,
+            5 => ValidatorStatus::Withdrawn,
+            127 => ValidatorStatus::Penalized,
+            _ => return Err(DecodeError::Invalid),
+        };
+        Ok((status, i))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ValidatorStatus {
+    fn random_for_test(rng: &mut T) -> Self {
+        let options = vec![
+            ValidatorStatus::PendingActivation,
+            ValidatorStatus::Active,
+            ValidatorStatus::PendingExit,
+            ValidatorStatus::PendingWithdraw,
+            ValidatorStatus::Withdrawn,
+            ValidatorStatus::Penalized,
+        ];
+        options[(rng.next_u32() as usize) % options.len()].clone()
+    }
+}
+
+impl Encodable for ValidatorRecord {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append(&self.pubkey);
+        s.append(&self.withdrawal_shard);
+        s.append(&self.withdrawal_address);
+        s.append(&self.randao_commitment);
+        s.append(&self.randao_last_change);
+        s.append(&self.balance);
+        s.append(&self.status);
+        s.append(&self.exit_slot);
+    }
+}
+
+impl Decodable for ValidatorRecord {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (pubkey, i) = <_>::ssz_decode(bytes, i)?;
+        let (withdrawal_shard, i) = <_>::ssz_decode(bytes, i)?;
+        let (withdrawal_address, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_commitment, i) = <_>::ssz_decode(bytes, i)?;
+        let (randao_last_change, i) = <_>::ssz_decode(bytes, i)?;
+        let (balance, i) = <_>::ssz_decode(bytes, i)?;
+        let (status, i) = <_>::ssz_decode(bytes, i)?;
+        let (exit_slot, i) = <_>::ssz_decode(bytes, i)?;
+
+        Ok((
+            Self {
+                pubkey,
+                withdrawal_shard,
+                withdrawal_address,
+                randao_commitment,
+                randao_last_change,
+                balance,
+                status,
+                exit_slot,
+            },
+            i,
+        ))
+    }
+}
+
+impl<T: RngCore> TestRandom<T> for ValidatorRecord {
+    fn random_for_test(rng: &mut T) -> Self {
+        Self {
+            pubkey: <_>::random_for_test(rng),
+            withdrawal_shard: <_>::random_for_test(rng),
+            withdrawal_address: <_>::random_for_test(rng),
+            randao_commitment: <_>::random_for_test(rng),
+            randao_last_change: <_>::random_for_test(rng),
+            balance: <_>::random_for_test(rng),
+            status: <_>::random_for_test(rng),
+            exit_slot: <_>::random_for_test(rng),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+    use crate::test_utils::TestRandom;
+    use rand::{prng::XorShiftRng, SeedableRng};
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ValidatorRecord::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+
+    #[test]
+    pub fn test_validator_status_ssz_round_trip() {
+        let mut rng = XorShiftRng::from_seed([42; 16]);
+        let original = ValidatorStatus::random_for_test(&mut rng);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
     }
 }

beacon_chain/types/src/validator_registration.rs

@@ -1,5 +1,5 @@
 use super::{Address, Hash256};
-use bls::{create_proof_of_possession, Keypair, PublicKey, Signature};
+use bls::{PublicKey, Signature};
 
 /// The information gathered from the PoW chain validator registration function.
 #[derive(Debug, Clone, PartialEq)]
@@ -10,17 +10,3 @@ pub struct ValidatorRegistration {
     pub randao_commitment: Hash256,
     pub proof_of_possession: Signature,
 }
-
-impl ValidatorRegistration {
-    pub fn random() -> Self {
-        let keypair = Keypair::random();
-
-        Self {
-            pubkey: keypair.pk.clone(),
-            withdrawal_shard: 0,
-            withdrawal_address: Address::random(),
-            randao_commitment: Hash256::random(),
-            proof_of_possession: create_proof_of_possession(&keypair),
-        }
-    }
-}

beacon_chain/utils/bls/src/keypair.rs

@@ -0,0 +1,16 @@
+use super::{PublicKey, SecretKey};
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct Keypair {
+    pub sk: SecretKey,
+    pub pk: PublicKey,
+}
+
+impl Keypair {
+    /// Instantiate a Keypair using SecretKey::random().
+    pub fn random() -> Self {
+        let sk = SecretKey::random();
+        let pk = PublicKey::from_secret_key(&sk);
+        Keypair { sk, pk }
+    }
+}

beacon_chain/utils/bls/src/lib.rs

@@ -3,20 +3,24 @@ extern crate hashing;
 extern crate ssz;
 
 mod aggregate_signature;
+mod keypair;
+mod public_key;
+mod secret_key;
 mod signature;
 
 pub use crate::aggregate_signature::AggregateSignature;
+pub use crate::keypair::Keypair;
+pub use crate::public_key::PublicKey;
+pub use crate::secret_key::SecretKey;
 pub use crate::signature::Signature;
 
 pub use self::bls_aggregates::AggregatePublicKey;
-pub use self::bls_aggregates::Keypair;
-pub use self::bls_aggregates::PublicKey;
-pub use self::bls_aggregates::SecretKey;
 
 pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97;
 
 use hashing::canonical_hash;
 use std::default::Default;
+use ssz::ssz_encode;
 
 fn extend_if_needed(hash: &mut Vec<u8>) {
     // NOTE: bls_aggregates crate demands 48 bytes, this may be removed as we get closer to production
@@ -26,13 +30,13 @@ fn extend_if_needed(hash: &mut Vec<u8>) {
 /// For some signature and public key, ensure that the signature message was the public key and it
 /// was signed by the secret key that corresponds to that public key.
 pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool {
-    let mut hash = canonical_hash(&pubkey.as_bytes());
+    let mut hash = canonical_hash(&ssz_encode(pubkey));
     extend_if_needed(&mut hash);
     sig.verify_hashed(&hash, &pubkey)
 }
 
 pub fn create_proof_of_possession(keypair: &Keypair) -> Signature {
-    let mut hash = canonical_hash(&keypair.pk.as_bytes());
+    let mut hash = canonical_hash(&ssz_encode(&keypair.pk));
     extend_if_needed(&mut hash);
     Signature::new_hashed(&hash, &keypair.sk)
 }

beacon_chain/utils/bls/src/public_key.rs

@@ -0,0 +1,52 @@
+use super::SecretKey;
+use bls_aggregates::PublicKey as RawPublicKey;
+use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+
+/// A single BLS signature.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, PartialEq, Clone)]
+pub struct PublicKey(RawPublicKey);
+
+impl PublicKey {
+    pub fn from_secret_key(secret_key: &SecretKey) -> Self {
+        PublicKey(RawPublicKey::from_secret_key(secret_key.as_raw()))
+    }
+
+    /// Returns the underlying signature.
+    pub fn as_raw(&self) -> &RawPublicKey {
+        &self.0
+    }
+}
+
+impl Encodable for PublicKey {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self.0.as_bytes());
+    }
+}
+
+impl Decodable for PublicKey {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (sig_bytes, i) = decode_ssz_list(bytes, i)?;
+        let raw_sig = RawPublicKey::from_bytes(&sig_bytes).map_err(|_| DecodeError::TooShort)?;
+        Ok((PublicKey(raw_sig), i))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let sk = SecretKey::random();
+        let original = PublicKey::from_secret_key(&sk);
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = PublicKey::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/utils/bls/src/secret_key.rs

@@ -0,0 +1,59 @@
+use bls_aggregates::{DecodeError as BlsDecodeError, SecretKey as RawSecretKey};
+use ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
+
+/// A single BLS signature.
+///
+/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
+/// serialization).
+#[derive(Debug, PartialEq, Clone)]
+pub struct SecretKey(RawSecretKey);
+
+impl SecretKey {
+    pub fn random() -> Self {
+        SecretKey(RawSecretKey::random())
+    }
+
+    /// Instantiate a SecretKey from existing bytes.
+    ///
+    /// Note: this is _not_ SSZ decoding.
+    pub fn from_bytes(bytes: &[u8]) -> Result<SecretKey, BlsDecodeError> {
+        Ok(SecretKey(RawSecretKey::from_bytes(bytes)?))
+    }
+
+    /// Returns the underlying secret key.
+    pub fn as_raw(&self) -> &RawSecretKey {
+        &self.0
+    }
+}
+
+impl Encodable for SecretKey {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self.0.as_bytes());
+    }
+}
+
+impl Decodable for SecretKey {
+    fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> {
+        let (sig_bytes, i) = decode_ssz_list(bytes, i)?;
+        let raw_sig = RawSecretKey::from_bytes(&sig_bytes).map_err(|_| DecodeError::TooShort)?;
+        Ok((SecretKey(raw_sig), i))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::super::ssz::ssz_encode;
+    use super::*;
+
+    #[test]
+    pub fn test_ssz_round_trip() {
+        let original =
+            SecretKey::from_bytes("jzjxxgjajfjrmgodszzsgqccmhnyvetcuxobhtynojtpdtbj".as_bytes())
+                .unwrap();
+
+        let bytes = ssz_encode(&original);
+        let (decoded, _) = SecretKey::ssz_decode(&bytes, 0).unwrap();
+
+        assert_eq!(original, decoded);
+    }
+}

beacon_chain/utils/bls/src/signature.rs

@@ -1,5 +1,6 @@
 use super::ssz::{decode_ssz_list, Decodable, DecodeError, Encodable, SszStream};
-use bls_aggregates::{PublicKey, SecretKey, Signature as RawSignature};
+use super::{PublicKey, SecretKey};
+use bls_aggregates::Signature as RawSignature;
 
 /// A single BLS signature.
 ///
@@ -11,23 +12,23 @@ pub struct Signature(RawSignature);
 impl Signature {
     /// Instantiate a new Signature from a message and a SecretKey.
     pub fn new(msg: &[u8], sk: &SecretKey) -> Self {
-        Signature(RawSignature::new(msg, sk))
+        Signature(RawSignature::new(msg, sk.as_raw()))
     }
 
     /// Instantiate a new Signature from a message and a SecretKey, where the message has already
     /// been hashed.
     pub fn new_hashed(msg_hashed: &[u8], sk: &SecretKey) -> Self {
-        Signature(RawSignature::new_hashed(msg_hashed, sk))
+        Signature(RawSignature::new_hashed(msg_hashed, sk.as_raw()))
     }
 
     /// Verify the Signature against a PublicKey.
     pub fn verify(&self, msg: &[u8], pk: &PublicKey) -> bool {
-        self.0.verify(msg, pk)
+        self.0.verify(msg, pk.as_raw())
     }
 
     /// Verify the Signature against a PublicKey, where the message has already been hashed.
     pub fn verify_hashed(&self, msg_hash: &[u8], pk: &PublicKey) -> bool {
-        self.0.verify_hashed(msg_hash, pk)
+        self.0.verify_hashed(msg_hash, pk.as_raw())
     }
 
     /// Returns the underlying signature.

beacon_chain/utils/ssz/src/decode.rs

@@ -4,6 +4,7 @@ use super::LENGTH_BYTES;
 pub enum DecodeError {
     TooShort,
     TooLong,
+    Invalid,
 }
 
 pub trait Decodable: Sized {
@@ -148,7 +149,8 @@ mod tests {
                 0, 0, 0, 16, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 0, 10,
             ],
             0,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 20);
 
@@ -160,7 +162,8 @@ mod tests {
                 10, 0, 0, 0, 0, 0, 0, 0, 10,
             ],
             0,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 36);
 
@@ -172,7 +175,8 @@ mod tests {
                 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 15,
             ],
             10,
-        ).unwrap();
+        )
+        .unwrap();
         assert_eq!(decoded.0, v);
         assert_eq!(decoded.1, 46);
 

beacon_chain/utils/ssz/src/impl_decode.rs

@@ -1,5 +1,5 @@
 use super::decode::decode_ssz_list;
-use super::ethereum_types::H256;
+use super::ethereum_types::{Address, H256};
 use super::{Decodable, DecodeError};
 
 macro_rules! impl_decodable_for_uint {
@@ -49,6 +49,16 @@ impl Decodable for H256 {
     }
 }
 
+impl Decodable for Address {
+    fn ssz_decode(bytes: &[u8], index: usize) -> Result<(Self, usize), DecodeError> {
+        if bytes.len() < 20 || bytes.len() - 20 < index {
+            Err(DecodeError::TooShort)
+        } else {
+            Ok((Address::from(&bytes[index..(index + 20)]), index + 20))
+        }
+    }
+}
+
 impl<T> Decodable for Vec<T>
 where
     T: Decodable,

beacon_chain/utils/ssz/src/impl_encode.rs

@@ -1,7 +1,7 @@
 extern crate bytes;
 
 use self::bytes::{BufMut, BytesMut};
-use super::ethereum_types::H256;
+use super::ethereum_types::{Address, H256};
 use super::{Encodable, SszStream};
 
 /*
@@ -52,6 +52,21 @@ impl Encodable for H256 {
     }
 }
 
+impl Encodable for Address {
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_encoded_raw(&self.to_vec());
+    }
+}
+
+impl<T> Encodable for Vec<T>
+where
+    T: Encodable,
+{
+    fn ssz_append(&self, s: &mut SszStream) {
+        s.append_vec(&self);
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

beacon_chain/validator_change/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "validator_change"
+version = "0.1.0"
+authors = ["Paul Hauner <paul@paulhauner.com>"]
+edition = "2018"
+
+[dependencies]
+bytes = "0.4.10"
+hashing = { path = "../utils/hashing" }
+ssz = { path = "../utils/ssz" }
+types = { path = "../types" }

beacon_chain/validator_change/src/lib.rs

@@ -0,0 +1,142 @@
+extern crate bytes;
+extern crate hashing;
+extern crate types;
+
+use bytes::{BufMut, BytesMut};
+use hashing::canonical_hash;
+use ssz::ssz_encode;
+use std::cmp::max;
+use types::{Hash256, ValidatorRecord, ValidatorStatus};
+
+pub enum UpdateValidatorSetError {
+    ArithmeticOverflow,
+}
+
+const VALIDATOR_FLAG_ENTRY: u8 = 0;
+const VALIDATOR_FLAG_EXIT: u8 = 1;
+
+pub fn update_validator_set(
+    validators: &mut Vec<ValidatorRecord>,
+    hash_chain: Hash256,
+    present_slot: u64,
+    deposit_size_gwei: u64,
+    max_validator_churn_quotient: u64,
+) -> Result<(), UpdateValidatorSetError> {
+    /*
+     * Total balance of all active validators.
+     *
+     * Return an error if an overflow occurs.
+     */
+    let total_balance = {
+        let mut bal: u64 = 0;
+        for v in validators.iter() {
+            if v.status_is(ValidatorStatus::Active) {
+                bal = bal
+                    .checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+            }
+        }
+        bal
+    };
+
+    /*
+     * Note: this is not the maximum allowable change, it can actually be higher.
+     */
+    let max_allowable_change = {
+        let double_deposit_size = deposit_size_gwei
+            .checked_mul(2)
+            .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+        max(
+            double_deposit_size,
+            total_balance / max_validator_churn_quotient,
+        )
+    };
+
+    let mut hasher = ValidatorChangeHashChain {
+        bytes: hash_chain.to_vec(),
+    };
+    let mut total_changed: u64 = 0;
+    for (i, v) in validators.iter_mut().enumerate() {
+        match v.status {
+            /*
+             * Validator is pending activiation.
+             */
+            ValidatorStatus::PendingActivation => {
+                let new_total_changed = total_changed
+                    .checked_add(deposit_size_gwei)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If entering this validator would not exceed the max balance delta,
+                 * activate the validator.
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::Active;
+                    hasher.extend(i, &ssz_encode(&v.pubkey), VALIDATOR_FLAG_ENTRY);
+                    total_changed = new_total_changed;
+                } else {
+                    // Entering the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            /*
+             * Validator is pending exit.
+             */
+            ValidatorStatus::PendingExit => {
+                let new_total_changed = total_changed
+                    .checked_add(v.balance)
+                    .ok_or(UpdateValidatorSetError::ArithmeticOverflow)?;
+                /*
+                 * If exiting this validator would not exceed the max balance delta,
+                 * exit the validator
+                 */
+                if new_total_changed <= max_allowable_change {
+                    v.status = ValidatorStatus::PendingWithdraw;
+                    v.exit_slot = present_slot;
+                    hasher.extend(i, &ssz_encode(&v.pubkey), VALIDATOR_FLAG_EXIT);
+                    total_changed = new_total_changed;
+                } else {
+                    // Exiting the validator would exceed the balance delta.
+                    break;
+                }
+            }
+            _ => (),
+        };
+        if total_changed >= max_allowable_change {
+            break;
+        }
+    }
+    Ok(())
+}
+
+pub struct ValidatorChangeHashChain {
+    bytes: Vec<u8>,
+}
+
+impl ValidatorChangeHashChain {
+    pub fn extend(&mut self, index: usize, pubkey: &Vec<u8>, flag: u8) {
+        let mut message = self.bytes.clone();
+        message.append(&mut serialize_validator_change_record(index, pubkey, flag));
+        self.bytes = canonical_hash(&message);
+    }
+}
+
+fn serialize_validator_change_record(index: usize, pubkey: &Vec<u8>, flag: u8) -> Vec<u8> {
+    let mut buf = BytesMut::with_capacity(68);
+    buf.put_u8(flag);
+    let index_bytes = {
+        let mut buf = BytesMut::with_capacity(8);
+        buf.put_u64_be(index as u64);
+        buf.take()[8 - 3..8].to_vec()
+    };
+    buf.put(index_bytes);
+    buf.put(pubkey);
+    buf.take().to_vec()
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn it_works() {
+        assert_eq!(2 + 2, 4);
+    }
+}

beacon_chain/validator_shuffling/src/shuffle.rs

@@ -95,7 +95,7 @@ fn generate_cycle(
                 .honey_badger_split(committees_per_slot)
                 .enumerate()
                 .map(|(j, shard_indices)| ShardCommittee {
-                    shard: ((shard_start + j) % shard_count) as u16,
+                    shard: ((shard_start + j) % shard_count) as u64,
                     committee: shard_indices.to_vec(),
                 })
                 .collect()