Automate merkle proofs with metastruct

Cargo.lock

@@ -4021,6 +4021,29 @@ dependencies = [
  "safe_arith",
 ]
 
+[[package]]
+name = "metastruct"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "734788dec2091fe9afa39530ca2ea7994f4a2c9aff3dbfebb63f2c1945c6f10b"
+dependencies = [
+ "metastruct_macro",
+]
+
+[[package]]
+name = "metastruct_macro"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42ded15e7570c2a507a23e6c3a1c8d74507b779476e43afe93ddfc261d44173d"
+dependencies = [
+ "darling",
+ "itertools",
+ "proc-macro2",
+ "quote",
+ "smallvec",
+ "syn",
+]
+
 [[package]]
 name = "mev-build-rs"
 version = "0.2.1"
@@ -6453,9 +6476,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
 
 [[package]]
 name = "superstruct"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95a99807a055ff4ff5d249bb84c80d9eabb55ca3c452187daae43fd5b51ef695"
+checksum = "6f4e1f478a7728f8855d7e620e9a152cf8932c6614f86564c886f9b8141f3201"
 dependencies = [
  "darling",
  "itertools",
@@ -7235,6 +7258,7 @@ dependencies = [
  "log",
  "maplit",
  "merkle_proof",
+ "metastruct",
  "milhouse",
  "parking_lot 0.12.1",
  "rand 0.8.5",

beacon_node/beacon_chain/Cargo.toml

@@ -59,7 +59,7 @@ strum = { version = "0.24.0", features = ["derive"] }
 logging = { path = "../../common/logging" }
 execution_layer = { path = "../execution_layer" }
 sensitive_url = { path = "../../common/sensitive_url" }
-superstruct = "0.5.0"
+superstruct = "0.7.0"
 hex = "0.4.2"
 exit-future = "0.2.0"
 unused_port = {path = "../../common/unused_port"}

beacon_node/builder_client/Cargo.toml

@@ -9,4 +9,4 @@ reqwest = { version = "0.11.0", features = ["json","stream"] }
 sensitive_url = { path = "../../common/sensitive_url" }
 eth2 = { path = "../../common/eth2" }
 serde = { version = "1.0.116", features = ["derive"] }
-serde_json = "1.0.58"
+serde_json = "1.0.58"

beacon_node/eth1/Cargo.toml

@@ -25,7 +25,7 @@ eth2_ssz_derive = "0.3.0"
 tree_hash = "0.4.1"
 parking_lot = "0.12.0"
 slog = "2.5.2"
-superstruct = "0.5.0"
+superstruct = "0.7.0"
 tokio = { version = "1.14.0", features = ["full"] }
 state_processing = { path = "../../consensus/state_processing" }
 lighthouse_metrics = { path = "../../common/lighthouse_metrics"}

beacon_node/lighthouse_network/Cargo.toml

@@ -36,7 +36,7 @@ rand = "0.8.5"
 directory = { path = "../../common/directory" }
 regex = "1.5.5"
 strum = { version = "0.24.0", features = ["derive"] }
-superstruct = "0.5.0"
+superstruct = "0.7.0"
 prometheus-client = "0.18.0"
 unused_port = { path = "../../common/unused_port" }
 delay_map = "0.1.1"

consensus/types/Cargo.toml

@@ -44,7 +44,8 @@ regex = "1.5.5"
 lazy_static = "1.4.0"
 parking_lot = "0.12.0"
 itertools = "0.10.0"
-superstruct = "0.5.0"
+superstruct = "0.7.0"
+metastruct = "0.1.0"
 serde_json = "1.0.74"
 smallvec = "1.8.0"
 milhouse = { git = "https://github.com/sigp/milhouse", branch = "main" }

consensus/types/examples/clone_state.rs

@@ -1,51 +0,0 @@
-//! These examples only really exist so we can use them for flamegraph. If they get annoying to
-//! maintain, feel free to delete.
-
-use types::{
-    test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256,
-    MinimalEthSpec, Validator,
-};
-
-type E = MinimalEthSpec;
-
-fn get_state(validator_count: usize) -> BeaconState<E> {
-    let spec = &E::default_spec();
-    let eth1_data = Eth1Data {
-        deposit_root: Hash256::zero(),
-        deposit_count: 0,
-        block_hash: Hash256::zero(),
-    };
-
-    let mut state = BeaconState::new(0, eth1_data, spec);
-
-    for i in 0..validator_count {
-        state
-            .balances_mut()
-            .push(i as u64)
-            .expect("should add balance");
-        state
-            .validators_mut()
-            .push(Validator {
-                pubkey: generate_deterministic_keypair(i).pk.into(),
-                withdrawal_credentials: Hash256::from_low_u64_le(i as u64),
-                effective_balance: i as u64,
-                slashed: i % 2 == 0,
-                activation_eligibility_epoch: i.into(),
-                activation_epoch: i.into(),
-                exit_epoch: i.into(),
-                withdrawable_epoch: i.into(),
-            })
-            .expect("should add validator");
-    }
-
-    state
-}
-
-fn main() {
-    let validator_count = 1_024;
-    let state = get_state(validator_count);
-
-    for _ in 0..100_000 {
-        let _ = state.clone();
-    }
-}

consensus/types/examples/ssz_encode_state.rs

@@ -1,54 +0,0 @@
-//! These examples only really exist so we can use them for flamegraph. If they get annoying to
-//! maintain, feel free to delete.
-
-use ssz::Encode;
-use types::{
-    test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256,
-    MinimalEthSpec, Validator,
-};
-
-type E = MinimalEthSpec;
-
-fn get_state(validator_count: usize) -> BeaconState<E> {
-    let spec = &E::default_spec();
-    let eth1_data = Eth1Data {
-        deposit_root: Hash256::zero(),
-        deposit_count: 0,
-        block_hash: Hash256::zero(),
-    };
-
-    let mut state = BeaconState::new(0, eth1_data, spec);
-
-    for i in 0..validator_count {
-        state
-            .balances_mut()
-            .push(i as u64)
-            .expect("should add balance");
-        state
-            .validators_mut()
-            .push(Validator {
-                pubkey: generate_deterministic_keypair(i).pk.into(),
-                withdrawal_credentials: Hash256::from_low_u64_le(i as u64),
-                effective_balance: i as u64,
-                slashed: i % 2 == 0,
-                activation_eligibility_epoch: i.into(),
-                activation_epoch: i.into(),
-                exit_epoch: i.into(),
-                withdrawable_epoch: i.into(),
-            })
-            .expect("should add validator");
-    }
-
-    state
-}
-
-fn main() {
-    let validator_count = 1_024;
-    let state = get_state(validator_count);
-
-    for _ in 0..1_024 {
-        let state_bytes = state.as_ssz_bytes();
-        let _: BeaconState<E> =
-            BeaconState::from_ssz_bytes(&state_bytes, &E::default_spec()).expect("should decode");
-    }
-}

consensus/types/examples/tree_hash_state.rs

@@ -1,56 +0,0 @@
-//! These examples only really exist so we can use them for flamegraph. If they get annoying to
-//! maintain, feel free to delete.
-
-use types::{
-    test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256,
-    MinimalEthSpec, Validator,
-};
-
-type E = MinimalEthSpec;
-
-fn get_state(validator_count: usize) -> BeaconState<E> {
-    let spec = &E::default_spec();
-    let eth1_data = Eth1Data {
-        deposit_root: Hash256::zero(),
-        deposit_count: 0,
-        block_hash: Hash256::zero(),
-    };
-
-    let mut state = BeaconState::new(0, eth1_data, spec);
-
-    for i in 0..validator_count {
-        state
-            .balances_mut()
-            .push(i as u64)
-            .expect("should add balance");
-        state
-            .validators_mut()
-            .push(Validator {
-                pubkey: generate_deterministic_keypair(i).pk.into(),
-                withdrawal_credentials: Hash256::from_low_u64_le(i as u64),
-                effective_balance: i as u64,
-                slashed: i % 2 == 0,
-                activation_eligibility_epoch: i.into(),
-                activation_epoch: i.into(),
-                exit_epoch: i.into(),
-                withdrawable_epoch: i.into(),
-            })
-            .expect("should add validator");
-    }
-
-    state
-}
-
-fn main() {
-    let validator_count = 1_024;
-    let mut state = get_state(validator_count);
-    state.update_tree_hash_cache().expect("should update cache");
-
-    actual_thing::<E>(&mut state);
-}
-
-fn actual_thing<T: EthSpec>(state: &mut BeaconState<T>) {
-    for _ in 0..200_024 {
-        let _ = state.update_tree_hash_cache().expect("should update cache");
-    }
-}

consensus/types/src/beacon_state.rs

@@ -7,6 +7,7 @@ use compare_fields::CompareFields;
 use compare_fields_derive::CompareFields;
 use eth2_hashing::hash;
 use int_to_bytes::{int_to_bytes4, int_to_bytes8};
+use metastruct::{metastruct, NumFields};
 pub use pubkey_cache::PubkeyCache;
 use safe_arith::{ArithError, SafeArith};
 use serde_derive::{Deserialize, Serialize};
@@ -208,6 +209,29 @@ impl From<BeaconStateHash> for Hash256 {
         serde(bound = "T: EthSpec", deny_unknown_fields),
         cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))
     ),
+    specific_variant_attributes(
+        Base(metastruct(
+            mappings(
+                map_beacon_state_base_fields(),
+                map_beacon_state_base_tree_list_fields(mutable, fallible, groups(tree_lists)),
+            ),
+            num_fields(all()),
+        )),
+        Altair(metastruct(
+            mappings(
+                map_beacon_state_altair_fields(),
+                map_beacon_state_altair_tree_list_fields(mutable, fallible, groups(tree_lists)),
+            ),
+            num_fields(all()),
+        )),
+        Merge(metastruct(
+            mappings(
+                map_beacon_state_bellatrix_fields(),
+                map_beacon_state_bellatrix_tree_list_fields(mutable, fallible, groups(tree_lists)),
+            ),
+            num_fields(all()),
+        )),
+    ),
     cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
     partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
 )]
@@ -223,16 +247,21 @@ where
 {
     // Versioning
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     #[serde(with = "eth2_serde_utils::quoted_u64")]
     pub genesis_time: u64,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub genesis_validators_root: Hash256,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub slot: Slot,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub fork: Fork,
 
     // History
+    #[metastruct(exclude_from(tree_lists))]
     pub latest_block_header: BeaconBlockHeader,
     #[test_random(default)]
     pub block_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
@@ -242,10 +271,12 @@ where
     pub historical_roots: VList<Hash256, T::HistoricalRootsLimit>,
 
     // Ethereum 1.0 chain data
+    #[metastruct(exclude_from(tree_lists))]
     pub eth1_data: Eth1Data,
     #[test_random(default)]
     pub eth1_data_votes: VList<Eth1Data, T::SlotsPerEth1VotingPeriod>,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     #[serde(with = "eth2_serde_utils::quoted_u64")]
     pub eth1_deposit_index: u64,
 
@@ -285,12 +316,16 @@ where
 
     // Finality
     #[test_random(default)]
+    #[metastruct(exclude_from(tree_lists))]
     pub justification_bits: BitVector<T::JustificationBitsLength>,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub previous_justified_checkpoint: Checkpoint,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub current_justified_checkpoint: Checkpoint,
     #[superstruct(getter(copy))]
+    #[metastruct(exclude_from(tree_lists))]
     pub finalized_checkpoint: Checkpoint,
 
     // Inactivity
@@ -302,12 +337,15 @@ where
 
     // Light-client sync committees
     #[superstruct(only(Altair, Merge))]
+    #[metastruct(exclude_from(tree_lists))]
     pub current_sync_committee: Arc<SyncCommittee<T>>,
     #[superstruct(only(Altair, Merge))]
+    #[metastruct(exclude_from(tree_lists))]
     pub next_sync_committee: Arc<SyncCommittee<T>>,
 
     // Execution
     #[superstruct(only(Merge))]
+    #[metastruct(exclude_from(tree_lists))]
     pub latest_execution_payload_header: ExecutionPayloadHeader<T>,
 
     // Caching (not in the spec)
@@ -315,21 +353,25 @@ where
     #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
+    #[metastruct(exclude)]
     pub total_active_balance: Option<(Epoch, u64)>,
     #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
+    #[metastruct(exclude)]
     pub committee_caches: [Arc<CommitteeCache>; CACHED_EPOCHS],
     #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
+    #[metastruct(exclude)]
     pub pubkey_cache: PubkeyCache,
     #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing, skip_deserializing)]
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
+    #[metastruct(exclude)]
     pub exit_cache: ExitCache,
 }
 
@@ -1583,35 +1625,6 @@ impl<T: EthSpec> BeaconState<T> {
                 .map_or(false, VList::has_pending_updates)
     }
 
-    // FIXME(sproul): automate this somehow
-    pub fn apply_pending_mutations(&mut self) -> Result<(), Error> {
-        self.block_roots_mut().apply_updates()?;
-        self.state_roots_mut().apply_updates()?;
-        self.historical_roots_mut().apply_updates()?;
-        self.eth1_data_votes_mut().apply_updates()?;
-        self.validators_mut().apply_updates()?;
-        self.balances_mut().apply_updates()?;
-        self.randao_mixes_mut().apply_updates()?;
-        self.slashings_mut().apply_updates()?;
-
-        if let Ok(previous_epoch_attestations) = self.previous_epoch_attestations_mut() {
-            previous_epoch_attestations.apply_updates()?;
-        }
-        if let Ok(current_epoch_attestations) = self.current_epoch_attestations_mut() {
-            current_epoch_attestations.apply_updates()?;
-        }
-        if let Ok(inactivity_scores) = self.inactivity_scores_mut() {
-            inactivity_scores.apply_updates()?;
-        }
-        if let Ok(previous_epoch_participation) = self.previous_epoch_participation_mut() {
-            previous_epoch_participation.apply_updates()?;
-        }
-        if let Ok(current_epoch_participation) = self.current_epoch_participation_mut() {
-            current_epoch_participation.apply_updates()?;
-        }
-        Ok(())
-    }
-
     /// Compute the tree hash root of the state using the tree hash cache.
     ///
     /// Initialize the tree hash cache if it isn't already initialized.
@@ -1662,63 +1675,17 @@ impl<T: EthSpec> BeaconState<T> {
         };
         Ok(sync_committee)
     }
-
-    pub fn compute_merkle_proof(
-        &mut self,
-        generalized_index: usize,
-    ) -> Result<Vec<Hash256>, Error> {
-        /* FIXME(sproul): re-enable merkle proofs
-        // 1. Convert generalized index to field index.
-        let field_index = match generalized_index {
-            light_client_update::CURRENT_SYNC_COMMITTEE_INDEX
-            | light_client_update::NEXT_SYNC_COMMITTEE_INDEX => {
-                // Sync committees are top-level fields, subtract off the generalized indices
-                // for the internal nodes. Result should be 22 or 23, the field offset of the committee
-                // in the `BeaconState`:
-                // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
-                generalized_index
-                    .checked_sub(tree_hash_cache::NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES)
-                    .ok_or(Error::IndexNotSupported(generalized_index))?
-            }
-            light_client_update::FINALIZED_ROOT_INDEX => {
-                // Finalized root is the right child of `finalized_checkpoint`, divide by two to get
-                // the generalized index of `state.finalized_checkpoint`.
-                let finalized_checkpoint_generalized_index = generalized_index / 2;
-                // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches
-                // position of `finalized_checkpoint` in `BeaconState`.
-                finalized_checkpoint_generalized_index
-                    .checked_sub(tree_hash_cache::NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES)
-                    .ok_or(Error::IndexNotSupported(generalized_index))?
-            }
-            _ => return Err(Error::IndexNotSupported(generalized_index)),
-        };
-
-        // 2. Get all `BeaconState` leaves.
-        let mut cache = self
-            .tree_hash_cache_mut()
-            .take()
-            .ok_or(Error::TreeHashCacheNotInitialized)?;
-        let leaves = cache.recalculate_tree_hash_leaves(self)?;
-        self.tree_hash_cache_mut().restore(cache);
-
-        // 3. Make deposit tree.
-        // Use the depth of the `BeaconState` fields (i.e. `log2(32) = 5`).
-        let depth = light_client_update::CURRENT_SYNC_COMMITTEE_PROOF_LEN;
-        let tree = merkle_proof::MerkleTree::create(&leaves, depth);
-        let (_, mut proof) = tree.generate_proof(field_index, depth)?;
-
-        // 4. If we're proving the finalized root, patch in the finalized epoch to complete the proof.
-        if generalized_index == light_client_update::FINALIZED_ROOT_INDEX {
-            proof.insert(0, self.finalized_checkpoint().epoch.tree_hash_root());
-        }
-
-        Ok(proof)
-        */
-        unimplemented!()
-    }
 }
 
-impl<T: EthSpec, V: ValidatorTrait> BeaconState<T, V> {
+impl<T: EthSpec, GenericValidator: ValidatorTrait> BeaconState<T, GenericValidator> {
+    /// The number of fields of the `BeaconState` rounded up to the nearest power of two.
+    ///
+    /// This is relevant to tree-hashing of the `BeaconState`.
+    ///
+    /// We assume this value is stable across forks. This assumption is checked in the
+    /// `check_num_fields_pow2` test.
+    pub const NUM_FIELDS_POW2: usize = BeaconStateMerge::<T>::NUM_FIELDS.next_power_of_two();
+
     /// Specialised deserialisation method that uses the `ChainSpec` as context.
     #[allow(clippy::integer_arithmetic)]
     pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
@@ -1742,6 +1709,81 @@ impl<T: EthSpec, V: ValidatorTrait> BeaconState<T, V> {
             <_>::from_ssz_bytes(bytes)?
         ))
     }
+
+    pub fn apply_pending_mutations(&mut self) -> Result<(), Error> {
+        match self {
+            Self::Base(inner) => {
+                map_beacon_state_base_tree_list_fields!(inner, |_, x| { x.apply_updates() })
+            }
+            Self::Altair(inner) => {
+                map_beacon_state_altair_tree_list_fields!(inner, |_, x| { x.apply_updates() })
+            }
+            Self::Merge(inner) => {
+                map_beacon_state_bellatrix_tree_list_fields!(inner, |_, x| { x.apply_updates() })
+            }
+        }
+        Ok(())
+    }
+
+    pub fn compute_merkle_proof(&self, generalized_index: usize) -> Result<Vec<Hash256>, Error> {
+        // 1. Convert generalized index to field index.
+        let field_index = match generalized_index {
+            light_client_update::CURRENT_SYNC_COMMITTEE_INDEX
+            | light_client_update::NEXT_SYNC_COMMITTEE_INDEX => {
+                // Sync committees are top-level fields, subtract off the generalized indices
+                // for the internal nodes. Result should be 22 or 23, the field offset of the committee
+                // in the `BeaconState`:
+                // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate
+                generalized_index
+                    .checked_sub(Self::NUM_FIELDS_POW2)
+                    .ok_or(Error::IndexNotSupported(generalized_index))?
+            }
+            light_client_update::FINALIZED_ROOT_INDEX => {
+                // Finalized root is the right child of `finalized_checkpoint`, divide by two to get
+                // the generalized index of `state.finalized_checkpoint`.
+                let finalized_checkpoint_generalized_index = generalized_index / 2;
+                // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches
+                // position of `finalized_checkpoint` in `BeaconState`.
+                finalized_checkpoint_generalized_index
+                    .checked_sub(Self::NUM_FIELDS_POW2)
+                    .ok_or(Error::IndexNotSupported(generalized_index))?
+            }
+            _ => return Err(Error::IndexNotSupported(generalized_index)),
+        };
+
+        // 2. Get all `BeaconState` leaves.
+        let mut leaves = vec![];
+        match self {
+            BeaconState::Base(state) => {
+                map_beacon_state_base_fields!(state, |_, field| {
+                    leaves.push(field.tree_hash_root());
+                });
+            }
+            BeaconState::Altair(state) => {
+                map_beacon_state_altair_fields!(state, |_, field| {
+                    leaves.push(field.tree_hash_root());
+                });
+            }
+            BeaconState::Merge(state) => {
+                map_beacon_state_bellatrix_fields!(state, |_, field| {
+                    leaves.push(field.tree_hash_root());
+                });
+            }
+        };
+
+        // 3. Make deposit tree.
+        // Use the depth of the `BeaconState` fields (i.e. `log2(32) = 5`).
+        let depth = light_client_update::CURRENT_SYNC_COMMITTEE_PROOF_LEN;
+        let tree = merkle_proof::MerkleTree::create(&leaves, depth);
+        let (_, mut proof) = tree.generate_proof(field_index, depth)?;
+
+        // 4. If we're proving the finalized root, patch in the finalized epoch to complete the proof.
+        if generalized_index == light_client_update::FINALIZED_ROOT_INDEX {
+            proof.insert(0, self.finalized_checkpoint().epoch.tree_hash_root());
+        }
+
+        Ok(proof)
+    }
 }
 
 impl From<RelativeEpochError> for Error {

consensus/types/src/beacon_state/tests.rs

@@ -1,11 +1,10 @@
 #![cfg(test)]
-use crate::test_utils::*;
+use crate::{test_utils::*, ForkName};
-use crate::test_utils::{SeedableRng, XorShiftRng};
 use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
 use beacon_chain::types::{
     test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError,
-    ChainSpec, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec,
+    BeaconStateMerge, ChainSpec, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair,
-    MinimalEthSpec, RelativeEpoch, Slot,
+    MainnetEthSpec, MinimalEthSpec, RelativeEpoch, Slot,
 };
 use safe_arith::SafeArith;
 use ssz::{Decode, Encode};
@@ -103,6 +102,7 @@ async fn test_beacon_proposer_index<T: EthSpec>() {
         .validators_mut()
         .get_mut(slot0_candidate0)
         .unwrap()
+        .mutable
         .effective_balance = 0;
     test(&state, Slot::new(0), 1);
     for i in 1..T::slots_per_epoch() {
@@ -419,57 +419,19 @@ fn decode_base_and_altair() {
 }
 
 #[test]
-fn tree_hash_cache_linear_history() {
+fn check_num_fields_pow2() {
-    let mut rng = XorShiftRng::from_seed([42; 16]);
+    use metastruct::NumFields;
+    pub type E = MainnetEthSpec;
 
-    let mut state: BeaconState<MainnetEthSpec> =
+    for fork_name in ForkName::list_all() {
-        BeaconState::Base(BeaconStateBase::random_for_test(&mut rng));
+        let num_fields = match fork_name {
-
+            ForkName::Base => BeaconStateBase::<E>::NUM_FIELDS,
-    let root = state.update_tree_hash_cache().unwrap();
+            ForkName::Altair => BeaconStateAltair::<E>::NUM_FIELDS,
-
+            ForkName::Merge => BeaconStateMerge::<E>::NUM_FIELDS,
-    assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
+        };
-
+        assert_eq!(
-    /*
+            num_fields.next_power_of_two(),
-     * A cache should hash twice without updating the slot.
+            BeaconState::<E>::NUM_FIELDS_POW2
-     */
+        );
-
+    }
-    assert_eq!(
-        state.update_tree_hash_cache().unwrap(),
-        root,
-        "tree hash result should be identical on the same slot"
-    );
-
-    /*
-     * A cache should not hash after updating the slot but not updating the state roots.
-     */
-
-    // The tree hash cache needs to be rebuilt since it was dropped when it failed.
-    state
-        .update_tree_hash_cache()
-        .expect("should rebuild cache");
-
-    *state.slot_mut() += 1;
-
-    assert_eq!(
-        state.update_tree_hash_cache(),
-        Err(BeaconStateError::NonLinearTreeHashCacheHistory),
-        "should not build hash without updating the state root"
-    );
-
-    /*
-     * The cache should update if the slot and state root are updated.
-     */
-
-    // The tree hash cache needs to be rebuilt since it was dropped when it failed.
-    let root = state
-        .update_tree_hash_cache()
-        .expect("should rebuild cache");
-
-    *state.slot_mut() += 1;
-    state
-        .set_state_root(state.slot() - 1, root)
-        .expect("should set state root");
-
-    let root = state.update_tree_hash_cache().unwrap();
-    assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
 }

consensus/types/src/validator.rs

@@ -12,7 +12,7 @@ const NUM_FIELDS: usize = 8;
 
 /// Information about a `BeaconChain` validator.
 #[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, Default)]
 #[serde(deny_unknown_fields)]
 pub struct Validator {
     #[serde(flatten)]
@@ -176,22 +176,25 @@ impl Validator {
     }
 }
 
-impl Default for Validator {
+/// Yields a "default" `Validator`. Primarily used for testing.
-    /// Yields a "default" `Validator`. Primarily used for testing.
+impl Default for ValidatorImmutable {
     fn default() -> Self {
-        Self {
+        ValidatorImmutable {
-            immutable: Arc::new(ValidatorImmutable {
+            pubkey: PublicKeyBytes::empty(),
-                pubkey: PublicKeyBytes::empty(),
+            withdrawal_credentials: Hash256::default(),
-                withdrawal_credentials: Hash256::default(),
+        }
-            }),
+    }
-            mutable: ValidatorMutable {
+}
-                activation_eligibility_epoch: Epoch::from(std::u64::MAX),
+
-                activation_epoch: Epoch::from(std::u64::MAX),
+impl Default for ValidatorMutable {
-                exit_epoch: Epoch::from(std::u64::MAX),
+    fn default() -> Self {
-                withdrawable_epoch: Epoch::from(std::u64::MAX),
+        ValidatorMutable {
-                slashed: false,
+            activation_eligibility_epoch: Epoch::from(std::u64::MAX),
-                effective_balance: std::u64::MAX,
+            activation_epoch: Epoch::from(std::u64::MAX),
-            },
+            exit_epoch: Epoch::from(std::u64::MAX),
+            withdrawable_epoch: Epoch::from(std::u64::MAX),
+            slashed: false,
+            effective_balance: std::u64::MAX,
         }
     }
 }
@@ -236,7 +239,10 @@ mod tests {
         let epoch = Epoch::new(10);
 
         let v = Validator {
-            activation_epoch: epoch,
+            mutable: ValidatorMutable {
+                activation_epoch: epoch,
+                ..Default::default()
+            },
             ..Validator::default()
         };
 
@@ -250,7 +256,10 @@ mod tests {
         let epoch = Epoch::new(10);
 
         let v = Validator {
-            exit_epoch: epoch,
+            mutable: ValidatorMutable {
+                exit_epoch: epoch,
+                ..ValidatorMutable::default()
+            },
             ..Validator::default()
         };
 
@@ -264,7 +273,10 @@ mod tests {
         let epoch = Epoch::new(10);
 
         let v = Validator {
-            withdrawable_epoch: epoch,
+            mutable: ValidatorMutable {
+                withdrawable_epoch: epoch,
+                ..ValidatorMutable::default()
+            },
             ..Validator::default()
         };
 

testing/ef_tests/src/cases/merkle_proof_validity.rs

@@ -49,9 +49,8 @@ impl<E: EthSpec> LoadCase for MerkleProofValidity<E> {
 
 impl<E: EthSpec> Case for MerkleProofValidity<E> {
     fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> {
-        // FIXME(sproul): re-enable merkle proofs
-        /*
         let mut state = self.state.clone();
+        state.update_tree_hash_cache().unwrap();
         let proof = match state.compute_merkle_proof(self.merkle_proof.leaf_index) {
             Ok(proof) => proof,
             Err(_) => {
@@ -80,10 +79,6 @@ impl<E: EthSpec> Case for MerkleProofValidity<E> {
             }
         }
 
-        // Tree hash cache should still be initialized (not dropped).
-        assert!(state.tree_hash_cache().is_initialized());
-        */
-
         Ok(())
     }
 }

testing/node_test_rig/Cargo.toml

@@ -13,4 +13,4 @@ eth2 = { path = "../../common/eth2" }
 validator_client = { path = "../../validator_client" }
 validator_dir = { path = "../../common/validator_dir", features = ["insecure_keys"] }
 sensitive_url = { path = "../../common/sensitive_url" }
-execution_layer = { path = "../../beacon_node/execution_layer" }
+execution_layer = { path = "../../beacon_node/execution_layer" }