merge upstream, fix compile errors

consensus/types/src/beacon_state.rs

@@ -14,6 +14,7 @@ use ssz::{ssz_encode, Decode, DecodeError, Encode};
 use ssz_derive::{Decode, Encode};
 use ssz_types::{typenum::Unsigned, BitVector, FixedVector};
 use std::convert::TryInto;
+use std::hash::Hash;
 use std::{fmt, mem, sync::Arc};
 use superstruct::superstruct;
 use swap_or_not_shuffle::compute_shuffled_index;
@@ -25,6 +26,7 @@ pub use self::committee_cache::{
     compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
     CommitteeCache,
 };
+use crate::historical_summary::HistoricalSummary;
 pub use clone_config::CloneConfig;
 pub use eth_spec::*;
 pub use iter::BlockRootsIter;
@@ -223,6 +225,7 @@ where
     pub block_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
     #[compare_fields(as_slice)]
     pub state_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
+    // Frozen in Capella, replaced by historical_summaries
     pub historical_roots: VariableList<Hash256, T::HistoricalRootsLimit>,
 
     // Ethereum 1.0 chain data
@@ -296,11 +299,14 @@ where
     )]
     pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844<T>,
 
-    // Withdrawals
+    // Capella
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_index: u64,
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_validator_index: u64,
+    // Deep history valid from Capella onwards.
+    #[superstruct(only(Capella, Eip4844))]
+    pub historical_summaries: VariableList<HistoricalSummary, T::HistoricalRootsLimit>,
 
     // Caching (not in the spec)
     #[serde(skip_serializing, skip_deserializing)]
@@ -504,7 +510,7 @@ impl<T: EthSpec> BeaconState<T> {
     /// Spec v0.12.1
     pub fn get_committee_count_at_slot(&self, slot: Slot) -> Result<u64, Error> {
         let cache = self.committee_cache_at_slot(slot)?;
-        Ok(cache.committees_per_slot() as u64)
+        Ok(cache.committees_per_slot())
     }
 
     /// Compute the number of committees in an entire epoch.

consensus/types/src/beacon_state/committee_cache.rs

@@ -144,7 +144,7 @@ impl CommitteeCache {
             self.committees_per_slot as usize,
             index as usize,
         );
-        let committee = self.compute_committee(committee_index as usize)?;
+        let committee = self.compute_committee(committee_index)?;
 
         Some(BeaconCommittee {
             slot,

consensus/types/src/beacon_state/tests.rs

@@ -344,12 +344,7 @@ mod committees {
 
         let cache_epoch = cache_epoch.into_epoch(state_epoch);
 
-        execute_committee_consistency_test(
-            new_head_state,
-            cache_epoch,
-            validator_count as usize,
-            spec,
-        );
+        execute_committee_consistency_test(new_head_state, cache_epoch, validator_count, spec);
     }
 
     async fn committee_consistency_test_suite<T: EthSpec>(cached_epoch: RelativeEpoch) {
@@ -361,18 +356,13 @@ mod committees {
             .mul(spec.target_committee_size)
             .add(1);
 
-        committee_consistency_test::<T>(validator_count as usize, Epoch::new(0), cached_epoch)
+        committee_consistency_test::<T>(validator_count, Epoch::new(0), cached_epoch).await;
+
+        committee_consistency_test::<T>(validator_count, T::genesis_epoch() + 4, cached_epoch)
             .await;
 
         committee_consistency_test::<T>(
-            validator_count as usize,
-            T::genesis_epoch() + 4,
-            cached_epoch,
-        )
-        .await;
-
-        committee_consistency_test::<T>(
-            validator_count as usize,
+            validator_count,
             T::genesis_epoch()
                 + (T::slots_per_historical_root() as u64)
                     .mul(T::slots_per_epoch())

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -3,6 +3,7 @@
 #![allow(clippy::indexing_slicing)]
 
 use super::Error;
+use crate::historical_summary::HistoricalSummaryCache;
 use crate::{BeaconState, EthSpec, Hash256, ParticipationList, Slot, Unsigned, Validator};
 use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
 use rayon::prelude::*;
@@ -142,6 +143,7 @@ pub struct BeaconTreeHashCacheInner<T: EthSpec> {
     block_roots: TreeHashCache,
     state_roots: TreeHashCache,
     historical_roots: TreeHashCache,
+    historical_summaries: OptionalTreeHashCache,
     balances: TreeHashCache,
     randao_mixes: TreeHashCache,
     slashings: TreeHashCache,
@@ -164,6 +166,14 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         let historical_roots = state
             .historical_roots()
             .new_tree_hash_cache(&mut fixed_arena);
+        let historical_summaries = OptionalTreeHashCache::new(
+            state
+                .historical_summaries()
+                .ok()
+                .map(HistoricalSummaryCache::new)
+                .as_ref(),
+        );
+
         let randao_mixes = state.randao_mixes().new_tree_hash_cache(&mut fixed_arena);
 
         let validators = ValidatorsListTreeHashCache::new::<T>(state.validators());
@@ -200,6 +210,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             block_roots,
             state_roots,
             historical_roots,
+            historical_summaries,
             balances,
             randao_mixes,
             slashings,
@@ -249,6 +260,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
                 .slashings()
                 .recalculate_tree_hash_root(&mut self.slashings_arena, &mut self.slashings)?,
         ];
+
         // Participation
         if let BeaconState::Base(state) = state {
             leaves.push(state.previous_epoch_attestations.tree_hash_root());
@@ -291,6 +303,24 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         if let Ok(payload_header) = state.latest_execution_payload_header() {
             leaves.push(payload_header.tree_hash_root());
         }
+
+        // Withdrawal indices (Capella and later).
+        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
+            leaves.push(next_withdrawal_index.tree_hash_root());
+        }
+        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
+            leaves.push(next_withdrawal_validator_index.tree_hash_root());
+        }
+
+        // Historical roots/summaries (Capella and later).
+        if let Ok(historical_summaries) = state.historical_summaries() {
+            leaves.push(
+                self.historical_summaries.recalculate_tree_hash_root(
+                    &HistoricalSummaryCache::new(historical_summaries),
+                )?,
+            );
+        }
+
         Ok(leaves)
     }
 
@@ -335,14 +365,6 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             hasher.write(leaf.as_bytes())?;
         }
 
-        // Withdrawal indices (Capella and later).
-        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
-            hasher.write(next_withdrawal_index.tree_hash_root().as_bytes())?;
-        }
-        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
-            hasher.write(next_withdrawal_validator_index.tree_hash_root().as_bytes())?;
-        }
-
         let root = hasher.finish()?;
 
         self.previous_state = Some((root, state.slot()));

consensus/types/src/eth_spec.rs

@@ -253,11 +253,6 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq +
     fn max_blobs_per_block() -> usize {
         Self::MaxBlobsPerBlock::to_usize()
     }
-
-    /// Returns the `BYTES_PER_BLOB` constant for the specification.
-    fn bytes_per_blob() -> usize {
-        Self::BytesPerBlob::to_usize()
-    }
 }
 
 /// Macro to inherit some type values from another EthSpec.

consensus/types/src/historical_summary.rs

@@ -0,0 +1,88 @@
+use crate::test_utils::TestRandom;
+use crate::Unsigned;
+use crate::{BeaconState, EthSpec, Hash256};
+use cached_tree_hash::Error;
+use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
+use compare_fields_derive::CompareFields;
+use serde_derive::{Deserialize, Serialize};
+use ssz_derive::{Decode, Encode};
+use ssz_types::VariableList;
+use test_random_derive::TestRandom;
+use tree_hash::{mix_in_length, TreeHash, BYTES_PER_CHUNK};
+use tree_hash_derive::TreeHash;
+
+/// `HistoricalSummary` matches the components of the phase0 `HistoricalBatch`
+/// making the two hash_tree_root-compatible. This struct is introduced into the beacon state
+/// in the Capella hard fork.
+///
+/// https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#historicalsummary
+#[derive(
+    Debug,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    CompareFields,
+    Clone,
+    Copy,
+    Default,
+)]
+pub struct HistoricalSummary {
+    block_summary_root: Hash256,
+    state_summary_root: Hash256,
+}
+
+impl HistoricalSummary {
+    pub fn new<T: EthSpec>(state: &BeaconState<T>) -> Self {
+        Self {
+            block_summary_root: state.block_roots().tree_hash_root(),
+            state_summary_root: state.state_roots().tree_hash_root(),
+        }
+    }
+}
+
+/// Wrapper type allowing the implementation of `CachedTreeHash`.
+#[derive(Debug)]
+pub struct HistoricalSummaryCache<'a, N: Unsigned> {
+    pub inner: &'a VariableList<HistoricalSummary, N>,
+}
+
+impl<'a, N: Unsigned> HistoricalSummaryCache<'a, N> {
+    pub fn new(inner: &'a VariableList<HistoricalSummary, N>) -> Self {
+        Self { inner }
+    }
+
+    #[allow(clippy::len_without_is_empty)]
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+}
+
+impl<'a, N: Unsigned> CachedTreeHash<TreeHashCache> for HistoricalSummaryCache<'a, N> {
+    fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
+        TreeHashCache::new(arena, int_log(N::to_usize()), self.len())
+    }
+
+    fn recalculate_tree_hash_root(
+        &self,
+        arena: &mut CacheArena,
+        cache: &mut TreeHashCache,
+    ) -> Result<Hash256, Error> {
+        Ok(mix_in_length(
+            &cache.recalculate_merkle_root(arena, leaf_iter(self.inner))?,
+            self.len(),
+        ))
+    }
+}
+
+pub fn leaf_iter(
+    values: &[HistoricalSummary],
+) -> impl Iterator<Item = [u8; BYTES_PER_CHUNK]> + ExactSizeIterator + '_ {
+    values
+        .iter()
+        .map(|value| value.tree_hash_root())
+        .map(Hash256::to_fixed_bytes)
+}

consensus/types/src/lib.rs

@@ -49,6 +49,7 @@ pub mod fork_name;
 pub mod free_attestation;
 pub mod graffiti;
 pub mod historical_batch;
+pub mod historical_summary;
 pub mod indexed_attestation;
 pub mod light_client_bootstrap;
 pub mod light_client_finality_update;

consensus/types/src/payload.rs

@@ -92,7 +92,7 @@ pub trait AbstractExecPayload<T: EthSpec>:
         + From<ExecutionPayloadEip4844<T>>
         + TryFrom<ExecutionPayloadHeaderEip4844<T>>;
 
-    fn default_at_fork(fork_name: ForkName) -> Self;
+    fn default_at_fork(fork_name: ForkName) -> Result<Self, Error>;
 }
 
 #[superstruct(
@@ -372,13 +372,12 @@ impl<T: EthSpec> AbstractExecPayload<T> for FullPayload<T> {
     type Capella = FullPayloadCapella<T>;
     type Eip4844 = FullPayloadEip4844<T>;
 
-    fn default_at_fork(fork_name: ForkName) -> Self {
+    fn default_at_fork(fork_name: ForkName) -> Result<Self, Error> {
         match fork_name {
-            //FIXME(sean) error handling
-            ForkName::Base | ForkName::Altair => panic!(),
-            ForkName::Merge => FullPayloadMerge::default().into(),
-            ForkName::Capella => FullPayloadCapella::default().into(),
-            ForkName::Eip4844 => FullPayloadEip4844::default().into(),
+            ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant),
+            ForkName::Merge => Ok(FullPayloadMerge::default().into()),
+            ForkName::Capella => Ok(FullPayloadCapella::default().into()),
+            ForkName::Eip4844 => Ok(FullPayloadEip4844::default().into()),
         }
     }
 }
@@ -882,13 +881,12 @@ impl<T: EthSpec> AbstractExecPayload<T> for BlindedPayload<T> {
     type Capella = BlindedPayloadCapella<T>;
     type Eip4844 = BlindedPayloadEip4844<T>;
 
-    fn default_at_fork(fork_name: ForkName) -> Self {
+    fn default_at_fork(fork_name: ForkName) -> Result<Self, Error> {
         match fork_name {
-            //FIXME(sean) error handling
-            ForkName::Base | ForkName::Altair => panic!(),
-            ForkName::Merge => BlindedPayloadMerge::default().into(),
-            ForkName::Capella => BlindedPayloadCapella::default().into(),
-            ForkName::Eip4844 => BlindedPayloadEip4844::default().into(),
+            ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant),
+            ForkName::Merge => Ok(BlindedPayloadMerge::default().into()),
+            ForkName::Capella => Ok(BlindedPayloadCapella::default().into()),
+            ForkName::Eip4844 => Ok(BlindedPayloadEip4844::default().into()),
         }
     }
 }

consensus/types/src/preset.rs

@@ -223,7 +223,7 @@ mod test {
     }
 
     fn preset_from_file<T: DeserializeOwned>(preset_name: &str, filename: &str) -> T {
-        let f = File::open(&presets_base_path().join(preset_name).join(filename))
+        let f = File::open(presets_base_path().join(preset_name).join(filename))
             .expect("preset file exists");
         serde_yaml::from_reader(f).unwrap()
     }