Marge fixes to test_harness, add serdehex crate

eth2/types/src/attestation.rs

@@ -1,7 +1,7 @@
 use super::{AggregateSignature, AttestationData, Bitfield};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -9,7 +9,18 @@ use test_random_derive::TestRandom;
 /// Details an attestation that can be slashable.
 ///
 /// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    Clone,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Attestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,

eth2/types/src/attestation_data.rs

@@ -1,7 +1,7 @@
 use crate::test_utils::TestRandom;
 use crate::{Crosslink, Epoch, Hash256, Slot};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -15,6 +15,7 @@ use test_random_derive::TestRandom;
     PartialEq,
     Default,
     Serialize,
+    Deserialize,
     Hash,
     Encode,
     Decode,

eth2/types/src/attester_slashing.rs

@@ -1,13 +1,13 @@
 use crate::{test_utils::TestRandom, SlashableAttestation};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Two conflicting attestations.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct AttesterSlashing {
     pub slashable_attestation_1: SlashableAttestation,
     pub slashable_attestation_2: SlashableAttestation,

eth2/types/src/beacon_block.rs

@@ -2,7 +2,7 @@ use crate::test_utils::TestRandom;
 use crate::{BeaconBlockBody, ChainSpec, Eth1Data, Hash256, Proposal, Slot};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::{SignedRoot, TreeHash};
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// A block of the `BeaconChain`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct BeaconBlock {
     pub slot: Slot,
     pub parent_root: Hash256,

eth2/types/src/beacon_block_body.rs

@@ -1,14 +1,16 @@
 use super::{Attestation, AttesterSlashing, Deposit, ProposerSlashing, Transfer, VoluntaryExit};
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// The body of a `BeaconChain` block, containing operations.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct BeaconBlockBody {
     pub proposer_slashings: Vec<ProposerSlashing>,
     pub attester_slashings: Vec<AttesterSlashing>,

eth2/types/src/beacon_state.rs

@@ -7,7 +7,7 @@ use int_to_bytes::int_to_bytes32;
 use log::{debug, error, trace};
 use pubkey_cache::PubkeyCache;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::{hash, Decodable, DecodeError, Encodable, SignedRoot, SszStream, TreeHash};
 use std::collections::HashMap;
 use swap_or_not_shuffle::shuffle_list;
@@ -72,7 +72,7 @@ macro_rules! safe_sub_assign {
     };
 }
 
-#[derive(Debug, PartialEq, Clone, Default, Serialize)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)]
 pub struct BeaconState {
     // Misc
     pub slot: Slot,
@@ -114,7 +114,9 @@ pub struct BeaconState {
 
     // Caching (not in the spec)
     pub cache_index_offset: usize,
+    #[serde(default)]
     pub caches: Vec<EpochCache>,
+    #[serde(default)]
     pub pubkey_cache: PubkeyCache,
 }
 
@@ -137,11 +139,7 @@ impl BeaconState {
              */
             slot: spec.genesis_slot,
             genesis_time,
-            fork: Fork {
-                previous_version: spec.genesis_fork_version,
-                current_version: spec.genesis_fork_version,
-                epoch: spec.genesis_epoch,
-            },
+            fork: Fork::genesis(spec),
 
             /*
              * Validator registry
@@ -193,8 +191,8 @@ impl BeaconState {
              * Caching (not in spec)
              */
             cache_index_offset: 0,
-            caches: vec![EpochCache::empty(); CACHED_EPOCHS],
-            pubkey_cache: PubkeyCache::empty(),
+            caches: vec![EpochCache::default(); CACHED_EPOCHS],
+            pubkey_cache: PubkeyCache::default(),
         }
     }
 
@@ -276,7 +274,7 @@ impl BeaconState {
     /// Removes the specified cache and sets it to uninitialized.
     pub fn drop_cache(&mut self, relative_epoch: RelativeEpoch) {
         let previous_cache_index = self.cache_index(relative_epoch);
-        self.caches[previous_cache_index] = EpochCache::empty();
+        self.caches[previous_cache_index] = EpochCache::default();
     }
 
     /// Returns the index of `self.caches` for some `RelativeEpoch`.
@@ -324,7 +322,7 @@ impl BeaconState {
 
     /// Completely drops the `pubkey_cache`, replacing it with a new, empty cache.
     pub fn drop_pubkey_cache(&mut self) {
-        self.pubkey_cache = PubkeyCache::empty()
+        self.pubkey_cache = PubkeyCache::default()
     }
 
     /// If a validator pubkey exists in the validator registry, returns `Some(i)`, otherwise
@@ -1227,8 +1225,8 @@ impl Decodable for BeaconState {
                 eth1_data_votes,
                 deposit_index,
                 cache_index_offset: 0,
-                caches: vec![EpochCache::empty(); CACHED_EPOCHS],
-                pubkey_cache: PubkeyCache::empty(),
+                caches: vec![EpochCache::default(); CACHED_EPOCHS],
+                pubkey_cache: PubkeyCache::default(),
             },
             i,
         ))
@@ -1298,8 +1296,8 @@ impl<T: RngCore> TestRandom<T> for BeaconState {
             eth1_data_votes: <_>::random_for_test(rng),
             deposit_index: <_>::random_for_test(rng),
             cache_index_offset: 0,
-            caches: vec![EpochCache::empty(); CACHED_EPOCHS],
-            pubkey_cache: PubkeyCache::empty(),
+            caches: vec![EpochCache::default(); CACHED_EPOCHS],
+            pubkey_cache: PubkeyCache::default(),
         }
     }
 }

eth2/types/src/beacon_state/epoch_cache.rs

@@ -1,8 +1,8 @@
 use super::{AttestationDuty, BeaconState, CrosslinkCommittees, Error};
 use crate::{ChainSpec, Epoch};
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 
-#[derive(Debug, PartialEq, Clone, Serialize)]
+#[derive(Debug, Default, PartialEq, Clone, Serialize, Deserialize)]
 pub struct EpochCache {
     /// True if this cache has been initialized.
     pub initialized: bool,
@@ -15,16 +15,6 @@ pub struct EpochCache {
 }
 
 impl EpochCache {
-    /// Return a new, completely empty cache.
-    pub fn empty() -> EpochCache {
-        EpochCache {
-            initialized: false,
-            committees: vec![],
-            attestation_duties: vec![],
-            shard_committee_indices: vec![],
-        }
-    }
-
     /// Return a new, fully initialized cache.
     pub fn initialized(
         state: &BeaconState,

eth2/types/src/beacon_state/pubkey_cache.rs

@@ -1,22 +1,15 @@
 use crate::*;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 type ValidatorIndex = usize;
 
-#[derive(Debug, PartialEq, Clone, Default, Serialize)]
+#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)]
 pub struct PubkeyCache {
     map: HashMap<PublicKey, ValidatorIndex>,
 }
 
 impl PubkeyCache {
-    /// Instantiates a new, empty cache.
-    pub fn empty() -> Self {
-        Self {
-            map: HashMap::new(),
-        }
-    }
-
     /// Returns the number of validator indices already in the map.
     pub fn len(&self) -> ValidatorIndex {
         self.map.len()

eth2/types/src/chain_spec.rs

@@ -1,5 +1,7 @@
 use crate::{Address, Epoch, Fork, Hash256, Slot};
 use bls::Signature;
+use int_to_bytes::int_to_bytes4;
+use serde_derive::Deserialize;
 
 const GWEI: u64 = 1_000_000_000;
 
@@ -15,7 +17,8 @@ pub enum Domain {
 /// Holds all the "constants" for a BeaconChain.
 ///
 /// Spec v0.4.0
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug, Clone, Deserialize)]
+#[serde(default)]
 pub struct ChainSpec {
     /*
      * Misc
@@ -45,7 +48,7 @@ pub struct ChainSpec {
     /*
      * Initial Values
      */
-    pub genesis_fork_version: u64,
+    pub genesis_fork_version: u32,
     pub genesis_slot: Slot,
     pub genesis_epoch: Epoch,
     pub genesis_start_shard: u64,
@@ -100,12 +103,12 @@ pub struct ChainSpec {
      *
      * Use `ChainSpec::get_domain(..)` to access these values.
      */
-    domain_deposit: u64,
-    domain_attestation: u64,
-    domain_proposal: u64,
-    domain_exit: u64,
-    domain_randao: u64,
-    domain_transfer: u64,
+    domain_deposit: u32,
+    domain_attestation: u32,
+    domain_proposal: u32,
+    domain_exit: u32,
+    domain_randao: u32,
+    domain_transfer: u32,
 }
 
 impl ChainSpec {
@@ -135,8 +138,11 @@ impl ChainSpec {
             Domain::Transfer => self.domain_transfer,
         };
 
-        let fork_version = fork.get_fork_version(epoch);
-        fork_version * u64::pow(2, 32) + domain_constant
+        let mut fork_and_domain = [0; 8];
+        fork_and_domain.copy_from_slice(&fork.get_fork_version(epoch));
+        fork_and_domain.copy_from_slice(&int_to_bytes4(domain_constant));
+
+        u64::from_le_bytes(fork_and_domain)
     }
 
     /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification.
@@ -254,6 +260,12 @@ impl ChainSpec {
     }
 }
 
+impl Default for ChainSpec {
+    fn default() -> Self {
+        Self::foundation()
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

eth2/types/src/crosslink.rs

@@ -1,7 +1,7 @@
 use crate::test_utils::TestRandom;
 use crate::{Epoch, Hash256};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
@@ -9,7 +9,17 @@ use test_random_derive::TestRandom;
 ///
 /// Spec v0.4.0
 #[derive(
-    Debug, Clone, PartialEq, Default, Serialize, Hash, Encode, Decode, TreeHash, TestRandom,
+    Debug,
+    Clone,
+    PartialEq,
+    Default,
+    Serialize,
+    Deserialize,
+    Hash,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
 )]
 pub struct Crosslink {
     pub epoch: Epoch,

eth2/types/src/eth1_data.rs

@@ -1,14 +1,16 @@
 use super::Hash256;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Contains data obtained from the Eth1 chain.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Eth1Data {
     pub deposit_root: Hash256,
     pub block_hash: Hash256,

eth2/types/src/eth1_data_vote.rs

@@ -1,14 +1,16 @@
 use super::Eth1Data;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// A summation of votes for some `Eth1Data`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(
+    Debug, PartialEq, Clone, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Eth1DataVote {
     pub eth1_data: Eth1Data,
     pub vote_count: u64,

eth2/types/src/fork.rs

@@ -1,24 +1,41 @@
-use crate::{test_utils::TestRandom, Epoch};
+use crate::{test_utils::TestRandom, ChainSpec, Epoch};
+use int_to_bytes::int_to_bytes4;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Specifies a fork of the `BeaconChain`, to prevent replay attacks.
 ///
-/// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Default, Serialize, Encode, Decode, TreeHash, TestRandom)]
+/// Spec v0.5.0
+#[derive(
+    Debug, Clone, PartialEq, Default, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom,
+)]
 pub struct Fork {
-    pub previous_version: u64,
-    pub current_version: u64,
+    pub previous_version: [u8; 4],
+    pub current_version: [u8; 4],
     pub epoch: Epoch,
 }
 
 impl Fork {
+    /// Initialize the `Fork` from the genesis parameters in the `spec`.
+    ///
+    /// Spec v0.5.0
+    pub fn genesis(spec: &ChainSpec) -> Self {
+        let mut current_version: [u8; 4] = [0; 4];
+        current_version.copy_from_slice(&int_to_bytes4(spec.genesis_fork_version));
+
+        Self {
+            previous_version: current_version,
+            current_version,
+            epoch: spec.genesis_epoch,
+        }
+    }
+
     /// Return the fork version of the given ``epoch``.
     ///
-    /// Spec v0.4.0
-    pub fn get_fork_version(&self, epoch: Epoch) -> u64 {
+    /// Spec v0.5.0
+    pub fn get_fork_version(&self, epoch: Epoch) -> [u8; 4] {
         if epoch < self.epoch {
             return self.previous_version;
         }

eth2/types/src/pending_attestation.rs

@@ -1,14 +1,14 @@
 use crate::test_utils::TestRandom;
 use crate::{AttestationData, Bitfield, Slot};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// An attestation that has been included in the state but not yet fully processed.
 ///
 /// Spec v0.4.0
-#[derive(Debug, Clone, PartialEq, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct PendingAttestation {
     pub aggregation_bitfield: Bitfield,
     pub data: AttestationData,

eth2/types/src/proposal.rs

@@ -2,7 +2,7 @@ use crate::test_utils::TestRandom;
 use crate::{Hash256, Slot};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// A proposal for some shard or beacon block.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Proposal {
     pub slot: Slot,
     /// Shard number (spec.beacon_chain_shard_number for beacon chain)

eth2/types/src/proposer_slashing.rs

@@ -1,14 +1,14 @@
 use super::Proposal;
 use crate::test_utils::TestRandom;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz_derive::{Decode, Encode, TreeHash};
 use test_random_derive::TestRandom;
 
 /// Two conflicting proposals from the same proposer (validator).
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
 pub struct ProposerSlashing {
     pub proposer_index: u64,
     pub proposal_1: Proposal,

eth2/types/src/slashable_attestation.rs

@@ -1,6 +1,6 @@
 use crate::{test_utils::TestRandom, AggregateSignature, AttestationData, Bitfield, ChainSpec};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// To be included in an `AttesterSlashing`.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct SlashableAttestation {
     /// Lists validator registry indices, not committee indices.
     pub validator_indices: Vec<u64>,

eth2/types/src/test_utils/test_random.rs

@@ -51,3 +51,17 @@ where
         ]
     }
 }
+
+macro_rules! impl_test_random_for_u8_array {
+    ($len: expr) => {
+        impl<T: RngCore> TestRandom<T> for [u8; $len] {
+            fn random_for_test(rng: &mut T) -> Self {
+                let mut bytes = [0; $len];
+                rng.fill_bytes(&mut bytes);
+                bytes
+            }
+        }
+    };
+}
+
+impl_test_random_for_u8_array!(4);

eth2/types/src/transfer.rs

@@ -2,7 +2,7 @@ use super::Slot;
 use crate::test_utils::TestRandom;
 use bls::{PublicKey, Signature};
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -10,7 +10,18 @@ use test_random_derive::TestRandom;
 /// The data submitted to the deposit contract.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct Transfer {
     pub from: u64,
     pub to: u64,

eth2/types/src/voluntary_exit.rs

@@ -1,7 +1,7 @@
 use crate::{test_utils::TestRandom, Epoch};
 use bls::Signature;
 use rand::RngCore;
-use serde_derive::Serialize;
+use serde_derive::{Deserialize, Serialize};
 use ssz::TreeHash;
 use ssz_derive::{Decode, Encode, SignedRoot, TreeHash};
 use test_random_derive::TestRandom;
@@ -9,7 +9,18 @@ use test_random_derive::TestRandom;
 /// An exit voluntarily submitted a validator who wishes to withdraw.
 ///
 /// Spec v0.4.0
-#[derive(Debug, PartialEq, Clone, Serialize, Encode, Decode, TreeHash, TestRandom, SignedRoot)]
+#[derive(
+    Debug,
+    PartialEq,
+    Clone,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    SignedRoot,
+)]
 pub struct VoluntaryExit {
     pub epoch: Epoch,
     pub validator_index: u64,