add historical summaries (#3865)

* add historical summaries

* fix tree hash caching, disable the sanity slots test with fake crypto

* add ssz static HistoricalSummary

* only store historical summaries after capella

* Teach `UpdatePattern` about Capella

* Tidy EF tests

* Clippy

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

consensus/types/src/beacon_state.rs

@@ -14,6 +14,7 @@ use ssz::{ssz_encode, Decode, DecodeError, Encode};
 use ssz_derive::{Decode, Encode};
 use ssz_types::{typenum::Unsigned, BitVector, FixedVector};
 use std::convert::TryInto;
+use std::hash::Hash;
 use std::{fmt, mem, sync::Arc};
 use superstruct::superstruct;
 use swap_or_not_shuffle::compute_shuffled_index;
@@ -25,6 +26,7 @@ pub use self::committee_cache::{
     compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
     CommitteeCache,
 };
+use crate::historical_summary::HistoricalSummary;
 pub use clone_config::CloneConfig;
 pub use eth_spec::*;
 pub use iter::BlockRootsIter;
@@ -223,6 +225,7 @@ where
     pub block_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
     #[compare_fields(as_slice)]
     pub state_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
+    // Frozen in Capella, replaced by historical_summaries
     pub historical_roots: VariableList<Hash256, T::HistoricalRootsLimit>,
 
     // Ethereum 1.0 chain data
@@ -296,11 +299,14 @@ where
     )]
     pub latest_execution_payload_header: ExecutionPayloadHeaderEip4844<T>,
 
-    // Withdrawals
+    // Capella
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_index: u64,
     #[superstruct(only(Capella, Eip4844), partial_getter(copy))]
     pub next_withdrawal_validator_index: u64,
+    // Deep history valid from Capella onwards.
+    #[superstruct(only(Capella, Eip4844))]
+    pub historical_summaries: VariableList<HistoricalSummary, T::HistoricalRootsLimit>,
 
     // Caching (not in the spec)
     #[serde(skip_serializing, skip_deserializing)]

consensus/types/src/beacon_state/tree_hash_cache.rs

@@ -3,6 +3,7 @@
 #![allow(clippy::indexing_slicing)]
 
 use super::Error;
+use crate::historical_summary::HistoricalSummaryCache;
 use crate::{BeaconState, EthSpec, Hash256, ParticipationList, Slot, Unsigned, Validator};
 use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
 use rayon::prelude::*;
@@ -142,6 +143,7 @@ pub struct BeaconTreeHashCacheInner<T: EthSpec> {
     block_roots: TreeHashCache,
     state_roots: TreeHashCache,
     historical_roots: TreeHashCache,
+    historical_summaries: OptionalTreeHashCache,
     balances: TreeHashCache,
     randao_mixes: TreeHashCache,
     slashings: TreeHashCache,
@@ -164,6 +166,14 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         let historical_roots = state
             .historical_roots()
             .new_tree_hash_cache(&mut fixed_arena);
+        let historical_summaries = OptionalTreeHashCache::new(
+            state
+                .historical_summaries()
+                .ok()
+                .map(HistoricalSummaryCache::new)
+                .as_ref(),
+        );
+
         let randao_mixes = state.randao_mixes().new_tree_hash_cache(&mut fixed_arena);
 
         let validators = ValidatorsListTreeHashCache::new::<T>(state.validators());
@@ -200,6 +210,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             block_roots,
             state_roots,
             historical_roots,
+            historical_summaries,
             balances,
             randao_mixes,
             slashings,
@@ -249,6 +260,7 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
                 .slashings()
                 .recalculate_tree_hash_root(&mut self.slashings_arena, &mut self.slashings)?,
         ];
+
         // Participation
         if let BeaconState::Base(state) = state {
             leaves.push(state.previous_epoch_attestations.tree_hash_root());
@@ -291,6 +303,24 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
         if let Ok(payload_header) = state.latest_execution_payload_header() {
             leaves.push(payload_header.tree_hash_root());
         }
+
+        // Withdrawal indices (Capella and later).
+        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
+            leaves.push(next_withdrawal_index.tree_hash_root());
+        }
+        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
+            leaves.push(next_withdrawal_validator_index.tree_hash_root());
+        }
+
+        // Historical roots/summaries (Capella and later).
+        if let Ok(historical_summaries) = state.historical_summaries() {
+            leaves.push(
+                self.historical_summaries.recalculate_tree_hash_root(
+                    &HistoricalSummaryCache::new(historical_summaries),
+                )?,
+            );
+        }
+
         Ok(leaves)
     }
 
@@ -335,14 +365,6 @@ impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
             hasher.write(leaf.as_bytes())?;
         }
 
-        // Withdrawal indices (Capella and later).
-        if let Ok(next_withdrawal_index) = state.next_withdrawal_index() {
-            hasher.write(next_withdrawal_index.tree_hash_root().as_bytes())?;
-        }
-        if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() {
-            hasher.write(next_withdrawal_validator_index.tree_hash_root().as_bytes())?;
-        }
-
         let root = hasher.finish()?;
 
         self.previous_state = Some((root, state.slot()));

consensus/types/src/historical_summary.rs

@@ -0,0 +1,88 @@
+use crate::test_utils::TestRandom;
+use crate::Unsigned;
+use crate::{BeaconState, EthSpec, Hash256};
+use cached_tree_hash::Error;
+use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
+use compare_fields_derive::CompareFields;
+use serde_derive::{Deserialize, Serialize};
+use ssz_derive::{Decode, Encode};
+use ssz_types::VariableList;
+use test_random_derive::TestRandom;
+use tree_hash::{mix_in_length, TreeHash, BYTES_PER_CHUNK};
+use tree_hash_derive::TreeHash;
+
+/// `HistoricalSummary` matches the components of the phase0 `HistoricalBatch`
+/// making the two hash_tree_root-compatible. This struct is introduced into the beacon state
+/// in the Capella hard fork.
+///
+/// https://github.com/ethereum/consensus-specs/blob/dev/specs/capella/beacon-chain.md#historicalsummary
+#[derive(
+    Debug,
+    PartialEq,
+    Serialize,
+    Deserialize,
+    Encode,
+    Decode,
+    TreeHash,
+    TestRandom,
+    CompareFields,
+    Clone,
+    Copy,
+    Default,
+)]
+pub struct HistoricalSummary {
+    block_summary_root: Hash256,
+    state_summary_root: Hash256,
+}
+
+impl HistoricalSummary {
+    pub fn new<T: EthSpec>(state: &BeaconState<T>) -> Self {
+        Self {
+            block_summary_root: state.block_roots().tree_hash_root(),
+            state_summary_root: state.state_roots().tree_hash_root(),
+        }
+    }
+}
+
+/// Wrapper type allowing the implementation of `CachedTreeHash`.
+#[derive(Debug)]
+pub struct HistoricalSummaryCache<'a, N: Unsigned> {
+    pub inner: &'a VariableList<HistoricalSummary, N>,
+}
+
+impl<'a, N: Unsigned> HistoricalSummaryCache<'a, N> {
+    pub fn new(inner: &'a VariableList<HistoricalSummary, N>) -> Self {
+        Self { inner }
+    }
+
+    #[allow(clippy::len_without_is_empty)]
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+}
+
+impl<'a, N: Unsigned> CachedTreeHash<TreeHashCache> for HistoricalSummaryCache<'a, N> {
+    fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache {
+        TreeHashCache::new(arena, int_log(N::to_usize()), self.len())
+    }
+
+    fn recalculate_tree_hash_root(
+        &self,
+        arena: &mut CacheArena,
+        cache: &mut TreeHashCache,
+    ) -> Result<Hash256, Error> {
+        Ok(mix_in_length(
+            &cache.recalculate_merkle_root(arena, leaf_iter(self.inner))?,
+            self.len(),
+        ))
+    }
+}
+
+pub fn leaf_iter(
+    values: &[HistoricalSummary],
+) -> impl Iterator<Item = [u8; BYTES_PER_CHUNK]> + ExactSizeIterator + '_ {
+    values
+        .iter()
+        .map(|value| value.tree_hash_root())
+        .map(Hash256::to_fixed_bytes)
+}

consensus/types/src/lib.rs

@@ -49,6 +49,7 @@ pub mod fork_name;
 pub mod free_attestation;
 pub mod graffiti;
 pub mod historical_batch;
+pub mod historical_summary;
 pub mod indexed_attestation;
 pub mod light_client_bootstrap;
 pub mod light_client_finality_update;