add historical summaries (#3865)

* add historical summaries

* fix tree hash caching, disable the sanity slots test with fake crypto

* add ssz static HistoricalSummary

* only store historical summaries after capella

* Teach `UpdatePattern` about Capella

* Tidy EF tests

* Clippy

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

consensus/state_processing/src/per_epoch_processing.rs

@@ -3,14 +3,16 @@
 pub use epoch_processing_summary::EpochProcessingSummary;
 use errors::EpochProcessingError as Error;
 pub use justification_and_finalization_state::JustificationAndFinalizationState;
-pub use registry_updates::process_registry_updates;
 use safe_arith::SafeArith;
-pub use slashings::process_slashings;
 use types::{BeaconState, ChainSpec, EthSpec};
+
+pub use registry_updates::process_registry_updates;
+pub use slashings::process_slashings;
 pub use weigh_justification_and_finalization::weigh_justification_and_finalization;
 
 pub mod altair;
 pub mod base;
+pub mod capella;
 pub mod effective_balance_updates;
 pub mod epoch_processing_summary;
 pub mod errors;
@@ -37,10 +39,8 @@ pub fn process_epoch<T: EthSpec>(
 
     match state {
         BeaconState::Base(_) => base::process_epoch(state, spec),
-        BeaconState::Altair(_)
-        | BeaconState::Merge(_)
-        | BeaconState::Capella(_)
-        | BeaconState::Eip4844(_) => altair::process_epoch(state, spec),
+        BeaconState::Altair(_) | BeaconState::Merge(_) => altair::process_epoch(state, spec),
+        BeaconState::Capella(_) | BeaconState::Eip4844(_) => capella::process_epoch(state, spec),
     }
 }
 

consensus/state_processing/src/per_epoch_processing/capella.rs

@@ -0,0 +1,78 @@
+use super::altair::inactivity_updates::process_inactivity_updates;
+use super::altair::justification_and_finalization::process_justification_and_finalization;
+use super::altair::participation_cache::ParticipationCache;
+use super::altair::participation_flag_updates::process_participation_flag_updates;
+use super::altair::rewards_and_penalties::process_rewards_and_penalties;
+use super::altair::sync_committee_updates::process_sync_committee_updates;
+use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error};
+use crate::per_epoch_processing::{
+    effective_balance_updates::process_effective_balance_updates,
+    resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset},
+};
+use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch};
+
+pub use historical_summaries_update::process_historical_summaries_update;
+
+mod historical_summaries_update;
+
+pub fn process_epoch<T: EthSpec>(
+    state: &mut BeaconState<T>,
+    spec: &ChainSpec,
+) -> Result<EpochProcessingSummary<T>, Error> {
+    // Ensure the committee caches are built.
+    state.build_committee_cache(RelativeEpoch::Previous, spec)?;
+    state.build_committee_cache(RelativeEpoch::Current, spec)?;
+    state.build_committee_cache(RelativeEpoch::Next, spec)?;
+
+    // Pre-compute participating indices and total balances.
+    let participation_cache = ParticipationCache::new(state, spec)?;
+    let sync_committee = state.current_sync_committee()?.clone();
+
+    // Justification and finalization.
+    let justification_and_finalization_state =
+        process_justification_and_finalization(state, &participation_cache)?;
+    justification_and_finalization_state.apply_changes_to_state(state);
+
+    process_inactivity_updates(state, &participation_cache, spec)?;
+
+    // Rewards and Penalties.
+    process_rewards_and_penalties(state, &participation_cache, spec)?;
+
+    // Registry Updates.
+    process_registry_updates(state, spec)?;
+
+    // Slashings.
+    process_slashings(
+        state,
+        participation_cache.current_epoch_total_active_balance(),
+        spec,
+    )?;
+
+    // Reset eth1 data votes.
+    process_eth1_data_reset(state)?;
+
+    // Update effective balances with hysteresis (lag).
+    process_effective_balance_updates(state, spec)?;
+
+    // Reset slashings
+    process_slashings_reset(state)?;
+
+    // Set randao mix
+    process_randao_mixes_reset(state)?;
+
+    // Set historical summaries accumulator
+    process_historical_summaries_update(state)?;
+
+    // Rotate current/previous epoch participation
+    process_participation_flag_updates(state)?;
+
+    process_sync_committee_updates(state, spec)?;
+
+    // Rotate the epoch caches to suit the epoch transition.
+    state.advance_caches(spec)?;
+
+    Ok(EpochProcessingSummary::Altair {
+        participation_cache,
+        sync_committee,
+    })
+}

consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs

@@ -0,0 +1,23 @@
+use crate::EpochProcessingError;
+use safe_arith::SafeArith;
+use types::historical_summary::HistoricalSummary;
+use types::{BeaconState, EthSpec};
+
+pub fn process_historical_summaries_update<T: EthSpec>(
+    state: &mut BeaconState<T>,
+) -> Result<(), EpochProcessingError> {
+    // Set historical block root accumulator.
+    let next_epoch = state.next_epoch()?;
+    if next_epoch
+        .as_u64()
+        .safe_rem((T::slots_per_historical_root() as u64).safe_div(T::slots_per_epoch())?)?
+        == 0
+    {
+        let summary = HistoricalSummary::new(state);
+        return state
+            .historical_summaries_mut()?
+            .push(summary)
+            .map_err(Into::into);
+    }
+    Ok(())
+}

consensus/state_processing/src/upgrade/capella.rs

@@ -1,3 +1,4 @@
+use ssz_types::VariableList;
 use std::mem;
 use types::{BeaconState, BeaconStateCapella, BeaconStateError as Error, ChainSpec, EthSpec, Fork};
 
@@ -55,9 +56,10 @@ pub fn upgrade_to_capella<E: EthSpec>(
         next_sync_committee: pre.next_sync_committee.clone(),
         // Execution
         latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_capella(),
-        // Withdrawals
+        // Capella
         next_withdrawal_index: 0,
         next_withdrawal_validator_index: 0,
+        historical_summaries: VariableList::default(),
         // Caches
         total_active_balance: pre.total_active_balance,
         committee_caches: mem::take(&mut pre.committee_caches),

consensus/state_processing/src/upgrade/eip4844.rs

@@ -57,9 +57,10 @@ pub fn upgrade_to_eip4844<E: EthSpec>(
         next_sync_committee: pre.next_sync_committee.clone(),
         // Execution
         latest_execution_payload_header: pre.latest_execution_payload_header.upgrade_to_eip4844(),
-        // Withdrawals
+        // Capella
         next_withdrawal_index: pre.next_withdrawal_index,
         next_withdrawal_validator_index: pre.next_withdrawal_validator_index,
+        historical_summaries: pre.historical_summaries.clone(),
         // Caches
         total_active_balance: pre.total_active_balance,
         committee_caches: mem::take(&mut pre.committee_caches),

consensus/types/src/beacon_state.rs

@@ -14,6 +14,7 @@ use ssz::{ssz_encode, Decode, DecodeError, Encode};
 use ssz_derive::{Decode, Encode};
 use ssz_types::{typenum::Unsigned, BitVector, FixedVector};
 use std::convert::TryInto;
+use std::hash::Hash;
 use std::{fmt, mem, sync::Arc};
 use superstruct::superstruct;
 use swap_or_not_shuffle::compute_shuffled_index;
@@ -25,6 +26,7 @@ pub use self::committee_cache::{
     compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
     CommitteeCache,
 };
+use crate::historical_summary::HistoricalSummary;
 pub use clone_config::CloneConfig;
 pub use eth_spec::*;
 pub use iter::BlockRootsIter;
@@ -223,6 +225,7 @@ where
     pub block_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
     #[compare_fields(as_slice)]
     pub state_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
+    // Frozen in Capella, replaced by historical_summaries
     pub historical_roots: VariableList<Hash256, T::HistoricalRootsLimit>,
 
     // Ethereum 1.0 chain data
@@ -296,11 +299,14 @@ where
     )]
     pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844<T>,
 
-    // Withdrawals
+    // Capella
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_index: u64,
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_validator_index: u64,
+    // Deep history valid from Capella onwards.
+    #[superstruct(only(Capella, Eip4844))]
+    pub historical_summaries: VariableList<HistoricalSummary, T::HistoricalRootsLimit>,
 
     // Caching (not in the spec)
     #[serde(skip_serializing, skip_deserializing)]

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -3,6 +3,7 @@
 #![allow(clippy::indexing_slicing)]
 
 use super::Error;
+use crate::historical_summary::HistoricalSummaryCache;
 use crate::{BeaconState, EthSpec, Hash256, ParticipationList, Slot, Unsigned, Validator};
 use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
 use rayon::prelude::*;
@@ -142,6 +143,7 @@ pub struct BeaconTreeHashCacheInner<T: EthSpec> {
     block_roots: TreeHashCache,
     state_roots: TreeHashCache,
     historical_roots: TreeHashCache,
+    historical_summaries: OptionalTreeHashCache,
     balances: TreeHashCache,
     randao_mixes: TreeHashCache,
     slashings: TreeHashCache,
@@ -164,6 +166,14 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         let historical_roots = state
             .historical_roots()
             .new_tree_hash_cache(&mut fixed_arena);
+        let historical_summaries = OptionalTreeHashCache::new(
+            state
+                .historical_summaries()
+                .ok()
+                .map(HistoricalSummaryCache::new)
+                .as_ref(),
+        );
+
         let randao_mixes = state.randao_mixes().new_tree_hash_cache(&mut fixed_arena);
 
         let validators = ValidatorsListTreeHashCache::new::<T>(state.validators());
@@ -200,6 +210,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             block_roots,
             state_roots,
             historical_roots,
+            historical_summaries,
             balances,
             randao_mixes,
             slashings,
@@ -249,6 +260,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
                 .slashings()
                 .recalculate_tree_hash_root(&mut self.slashings_arena, &mut self.slashings)?,
         ];
+
         // Participation
         if let BeaconState::Base(state) = state {
             leaves.push(state.previous_epoch_attestations.tree_hash_root());
@@ -291,6 +303,24 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         if let Ok(payload_header) = state.latest_execution_payload_header() {
             leaves.push(payload_header.tree_hash_root());
         }
+
+        // Withdrawal indices (Capella and later).
+        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
+            leaves.push(next_withdrawal_index.tree_hash_root());
+        }
+        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
+            leaves.push(next_withdrawal_validator_index.tree_hash_root());
+        }
+
+        // Historical roots/summaries (Capella and later).
+        if let Ok(historical_summaries) = state.historical_summaries() {
+            leaves.push(
+                self.historical_summaries.recalculate_tree_hash_root(
+                    &HistoricalSummaryCache::new(historical_summaries),
+                )?,
+            );
+        }
+
         Ok(leaves)
     }
 
@@ -335,14 +365,6 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             hasher.write(leaf.as_bytes())?;
         }
 
-        // Withdrawal indices (Capella and later).
-        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
-            hasher.write(next_withdrawal_index.tree_hash_root().as_bytes())?;
-        }
-        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
-            hasher.write(next_withdrawal_validator_index.tree_hash_root().as_bytes())?;
-        }
-
         let root = hasher.finish()?;
 
         self.previous_state = Some((root, state.slot()));

consensus/types/src/historical_summary.rs

@@ -0,0 +1,88 @@
+use crate::test_utils::TestRandom;
+use crate::Unsigned;
+use crate::{BeaconState, EthSpec, Hash256};
+use cached_tree_hash::Error;
+use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
+use compare_fields_derive::CompareFields;
+use serde_derive::{Deserialize, Serialize};
+use ssz_derive::{Decode, Encode};
+use ssz_types::VariableList;
+use test_random_derive::TestRandom;
+use tree_hash::{mix_in_length, TreeHash, BYTES_PER_CHUNK};
+use tree_hash_derive::TreeHash;
+
+/// `HistoricalSummary` matches the components of the phase0 `HistoricalBatch`
+/// making the two hash_tree_root-compatible. This struct is introduced into the beacon state
+/// in the Capella hard fork.
+///
+/// https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#historicalsummary
+#[derive(
+    Debug,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    CompareFields,
+    Clone,
+    Copy,
+    Default,
+)]
+pub struct HistoricalSummary {
+    block_summary_root: Hash256,
+    state_summary_root: Hash256,
+}
+
+impl HistoricalSummary {
+    pub fn new<T: EthSpec>(state: &BeaconState<T>) -> Self {
+        Self {
+            block_summary_root: state.block_roots().tree_hash_root(),
+            state_summary_root: state.state_roots().tree_hash_root(),
+        }
+    }
+}
+
+/// Wrapper type allowing the implementation of `CachedTreeHash`.
+#[derive(Debug)]
+pub struct HistoricalSummaryCache<'a, N: Unsigned> {
+    pub inner: &'a VariableList<HistoricalSummary, N>,
+}
+
+impl<'a, N: Unsigned> HistoricalSummaryCache<'a, N> {
+    pub fn new(inner: &'a VariableList<HistoricalSummary, N>) -> Self {
+        Self { inner }
+    }
+
+    #[allow(clippy::len_without_is_empty)]
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+}
+
+impl<'a, N: Unsigned> CachedTreeHash<TreeHashCache> for HistoricalSummaryCache<'a, N> {
+    fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
+        TreeHashCache::new(arena, int_log(N::to_usize()), self.len())
+    }
+
+    fn recalculate_tree_hash_root(
+        &self,
+        arena: &mut CacheArena,
+        cache: &mut TreeHashCache,
+    ) -> Result<Hash256, Error> {
+        Ok(mix_in_length(
+            &cache.recalculate_merkle_root(arena, leaf_iter(self.inner))?,
+            self.len(),
+        ))
+    }
+}
+
+pub fn leaf_iter(
+    values: &[HistoricalSummary],
+) -> impl Iterator<Item = [u8; BYTES_PER_CHUNK]> + ExactSizeIterator + '_ {
+    values
+        .iter()
+        .map(|value| value.tree_hash_root())
+        .map(Hash256::to_fixed_bytes)
+}

consensus/types/src/lib.rs

@@ -49,6 +49,7 @@ pub mod fork_name;
 pub mod free_attestation;
 pub mod graffiti;
 pub mod historical_batch;
+pub mod historical_summary;
 pub mod indexed_attestation;
 pub mod light_client_bootstrap;
 pub mod light_client_finality_update;