Merge remote-tracking branch 'origin/unstable' into tree-states

consensus/types/Cargo.toml

@@ -27,7 +27,7 @@ serde_derive = "1.0.116"
 slog = "2.5.2"
 ethereum_ssz = { version = "0.5.0", features = ["arbitrary"] }
 ethereum_ssz_derive = "0.5.0"
-ssz_types = { version = "0.5.0", features = ["arbitrary"] }
+ssz_types = { version = "0.5.3", features = ["arbitrary"] }
 swap_or_not_shuffle = { path = "../swap_or_not_shuffle", features = ["arbitrary"] }
 test_random_derive = { path = "../../common/test_random_derive" }
 tree_hash = { version = "0.5.0", features = ["arbitrary"] }
@@ -54,6 +54,7 @@ milhouse = { git = "https://github.com/sigp/milhouse", branch = "main" }
 rpds = "0.11.0"
 serde_with = "1.13.0"
 maplit = "1.0.2"
+strum = { version = "0.24.0", features = ["derive"] }
 
 [dev-dependencies]
 criterion = "0.3.3"

consensus/types/benches/benches.rs

@@ -56,7 +56,7 @@ fn all_benches(c: &mut Criterion) {
     let spec = Arc::new(MainnetEthSpec::default_spec());
 
     let mut state = get_state::<MainnetEthSpec>(validator_count);
-    state.build_all_caches(&spec).expect("should build caches");
+    state.build_caches(&spec).expect("should build caches");
     let state_bytes = state.as_ssz_bytes();
 
     let inner_state = state.clone();

consensus/types/src/beacon_block_body.rs

@@ -89,7 +89,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockBodyRef<'a, T,
     }
 }
 
-impl<'a, T: EthSpec> BeaconBlockBodyRef<'a, T> {
+impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockBodyRef<'a, T, Payload> {
     /// Get the fork_name of this object
     pub fn fork_name(self) -> ForkName {
         match self {

consensus/types/src/beacon_state.rs

@@ -1,6 +1,5 @@
 use self::committee_cache::get_active_validator_indices;
 use self::exit_cache::ExitCache;
-use crate::historical_summary::HistoricalSummary;
 use crate::test_utils::TestRandom;
 use crate::validator::ValidatorTrait;
 use crate::*;
@@ -29,16 +28,21 @@ pub use self::committee_cache::{
     compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
     CommitteeCache,
 };
+pub use crate::beacon_state::balance::Balance;
+pub use crate::beacon_state::progressive_balances_cache::*;
 use crate::epoch_cache::EpochCache;
+use crate::historical_summary::HistoricalSummary;
 pub use eth_spec::*;
 pub use iter::BlockRootsIter;
 pub use milhouse::{interface::Interface, List as VList, List, Vector as FixedVector};
 
 #[macro_use]
 mod committee_cache;
+mod balance;
 pub mod compact_state;
 mod exit_cache;
 mod iter;
+mod progressive_balances_cache;
 mod pubkey_cache;
 mod tests;
 
@@ -105,6 +109,9 @@ pub enum Error {
     SszTypesError(ssz_types::Error),
     TreeHashCacheNotInitialized,
     NonLinearTreeHashCacheHistory,
+    ParticipationCacheError(String),
+    ProgressiveBalancesCacheNotInitialized,
+    ProgressiveBalancesCacheInconsistent,
     TreeHashCacheSkippedSlot {
         cache: Slot,
         state: Slot,
@@ -429,6 +436,12 @@ where
     #[tree_hash(skip_hashing)]
     #[test_random(default)]
     #[metastruct(exclude)]
+    pub progressive_balances_cache: ProgressiveBalancesCache,
+    #[serde(skip_serializing, skip_deserializing)]
+    #[ssz(skip_serializing, skip_deserializing)]
+    #[tree_hash(skip_hashing)]
+    #[test_random(default)]
+    #[metastruct(exclude)]
     pub pubkey_cache: PubkeyCache,
     #[serde(skip_serializing, skip_deserializing)]
     #[ssz(skip_serializing, skip_deserializing)]
@@ -495,6 +508,7 @@ impl<T: EthSpec> BeaconState<T> {
 
             // Caching (not in spec)
             total_active_balance: None,
+            progressive_balances_cache: <_>::default(),
             committee_caches: [
                 default_committee_cache.clone(),
                 default_committee_cache.clone(),
@@ -860,7 +874,7 @@ impl<T: EthSpec> BeaconState<T> {
         Ok(signature_hash_int.safe_rem(modulo)? == 0)
     }
 
-    /// Returns the beacon proposer index for the `slot` in the given `relative_epoch`.
+    /// Returns the beacon proposer index for the `slot` in `self.current_epoch()`.
     ///
     /// Spec v0.12.1
     pub fn get_beacon_proposer_index(&self, slot: Slot, spec: &ChainSpec) -> Result<usize, Error> {
@@ -1256,12 +1270,34 @@ impl<T: EthSpec> BeaconState<T> {
     }
 
     /// Convenience accessor for validators and balances simultaneously.
-    pub fn validators_and_balances_mut(&mut self) -> (&mut Validators<T>, &mut Balances<T>) {
+    pub fn validators_and_balances_and_progressive_balances_mut(
+        &mut self,
+    ) -> (
+        &mut Validators<T>,
+        &mut Balances<T>,
+        &mut ProgressiveBalancesCache,
+    ) {
         match self {
-            BeaconState::Base(state) => (&mut state.validators, &mut state.balances),
-            BeaconState::Altair(state) => (&mut state.validators, &mut state.balances),
-            BeaconState::Merge(state) => (&mut state.validators, &mut state.balances),
-            BeaconState::Capella(state) => (&mut state.validators, &mut state.balances),
+            BeaconState::Base(state) => (
+                &mut state.validators,
+                &mut state.balances,
+                &mut state.progressive_balances_cache,
+            ),
+            BeaconState::Altair(state) => (
+                &mut state.validators,
+                &mut state.balances,
+                &mut state.progressive_balances_cache,
+            ),
+            BeaconState::Merge(state) => (
+                &mut state.validators,
+                &mut state.balances,
+                &mut state.progressive_balances_cache,
+            ),
+            BeaconState::Capella(state) => (
+                &mut state.validators,
+                &mut state.balances,
+                &mut state.progressive_balances_cache,
+            ),
         }
     }
 
@@ -1534,7 +1570,7 @@ impl<T: EthSpec> BeaconState<T> {
     }
 
     /// Build all caches (except the tree hash cache), if they need to be built.
-    pub fn build_all_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> {
+    pub fn build_caches(&mut self, spec: &ChainSpec) -> Result<(), Error> {
         self.build_all_committee_caches(spec)?;
         self.update_pubkey_cache()?;
         self.build_exit_cache(spec)?;
@@ -1565,6 +1601,7 @@ impl<T: EthSpec> BeaconState<T> {
         self.drop_committee_cache(RelativeEpoch::Current)?;
         self.drop_committee_cache(RelativeEpoch::Next)?;
         self.drop_pubkey_cache();
+        self.drop_progressive_balances_cache();
         *self.exit_cache_mut() = ExitCache::default();
         *self.epoch_cache_mut() = EpochCache::default();
         Ok(())
@@ -1747,6 +1784,11 @@ impl<T: EthSpec> BeaconState<T> {
                 .map_or(false, VList::has_pending_updates)
     }
 
+    /// Completely drops the `progressive_balances_cache` cache, replacing it with a new, empty cache.
+    fn drop_progressive_balances_cache(&mut self) {
+        *self.progressive_balances_cache_mut() = ProgressiveBalancesCache::default();
+    }
+
     /// Compute the tree hash root of the state using the tree hash cache.
     ///
     /// Initialize the tree hash cache if it isn't already initialized.

consensus/types/src/beacon_state/balance.rs

@@ -0,0 +1,33 @@
+use arbitrary::Arbitrary;
+use safe_arith::{ArithError, SafeArith};
+
+/// A balance which will never be below the specified `minimum`.
+///
+/// This is an effort to ensure the `EFFECTIVE_BALANCE_INCREMENT` minimum is always respected.
+#[derive(PartialEq, Debug, Clone, Copy, Arbitrary)]
+pub struct Balance {
+    raw: u64,
+    minimum: u64,
+}
+
+impl Balance {
+    /// Initialize the balance to `0`, or the given `minimum`.
+    pub fn zero(minimum: u64) -> Self {
+        Self { raw: 0, minimum }
+    }
+
+    /// Returns the balance with respect to the initialization `minimum`.
+    pub fn get(&self) -> u64 {
+        std::cmp::max(self.raw, self.minimum)
+    }
+
+    /// Add-assign to the balance.
+    pub fn safe_add_assign(&mut self, other: u64) -> Result<(), ArithError> {
+        self.raw.safe_add_assign(other)
+    }
+
+    /// Sub-assign to the balance.
+    pub fn safe_sub_assign(&mut self, other: u64) -> Result<(), ArithError> {
+        self.raw.safe_sub_assign(other)
+    }
+}

consensus/types/src/beacon_state/compact_state.rs

@@ -49,6 +49,7 @@ macro_rules! full_to_compact {
             // Caches.
             total_active_balance: $s.total_active_balance.clone(),
             committee_caches: $s.committee_caches.clone(),
+            progressive_balances_cache: $s.progressive_balances_cache.clone(),
             pubkey_cache: $s.pubkey_cache.clone(),
             exit_cache: $s.exit_cache.clone(),
             epoch_cache: $s.epoch_cache.clone(),
@@ -110,6 +111,7 @@ macro_rules! compact_to_full {
             // Caching
             total_active_balance: $inner.total_active_balance,
             committee_caches: $inner.committee_caches,
+            progressive_balances_cache: $inner.progressive_balances_cache,
             pubkey_cache: $inner.pubkey_cache,
             exit_cache: $inner.exit_cache,
             epoch_cache: $inner.epoch_cache,

consensus/types/src/beacon_state/progressive_balances_cache.rs

@@ -0,0 +1,184 @@
+use crate::beacon_state::balance::Balance;
+use crate::{BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec};
+use arbitrary::Arbitrary;
+use safe_arith::SafeArith;
+use serde_derive::{Deserialize, Serialize};
+use strum::{Display, EnumString, EnumVariantNames};
+
+/// This cache keeps track of the accumulated target attestation balance for the current & previous
+/// epochs. The cached values can be utilised by fork choice to calculate unrealized justification
+/// and finalization instead of converting epoch participation arrays to balances for each block we
+/// process.
+#[derive(Default, Debug, PartialEq, Arbitrary, Clone)]
+pub struct ProgressiveBalancesCache {
+    inner: Option<Inner>,
+}
+
+#[derive(Debug, PartialEq, Arbitrary, Clone)]
+struct Inner {
+    pub current_epoch: Epoch,
+    pub previous_epoch_target_attesting_balance: Balance,
+    pub current_epoch_target_attesting_balance: Balance,
+}
+
+impl ProgressiveBalancesCache {
+    pub fn initialize(
+        &mut self,
+        current_epoch: Epoch,
+        previous_epoch_target_attesting_balance: Balance,
+        current_epoch_target_attesting_balance: Balance,
+    ) {
+        self.inner = Some(Inner {
+            current_epoch,
+            previous_epoch_target_attesting_balance,
+            current_epoch_target_attesting_balance,
+        });
+    }
+
+    pub fn is_initialized(&self) -> bool {
+        self.inner.is_some()
+    }
+
+    /// When a new target attestation has been processed, we update the cached
+    /// `current_epoch_target_attesting_balance` to include the validator effective balance.
+    /// If the epoch is neither the current epoch nor the previous epoch, an error is returned.
+    pub fn on_new_target_attestation(
+        &mut self,
+        epoch: Epoch,
+        validator_effective_balance: u64,
+    ) -> Result<(), BeaconStateError> {
+        let cache = self.get_inner_mut()?;
+
+        if epoch == cache.current_epoch {
+            cache
+                .current_epoch_target_attesting_balance
+                .safe_add_assign(validator_effective_balance)?;
+        } else if epoch.safe_add(1)? == cache.current_epoch {
+            cache
+                .previous_epoch_target_attesting_balance
+                .safe_add_assign(validator_effective_balance)?;
+        } else {
+            return Err(BeaconStateError::ProgressiveBalancesCacheInconsistent);
+        }
+
+        Ok(())
+    }
+
+    /// When a validator is slashed, we reduce the `current_epoch_target_attesting_balance` by the
+    /// validator's effective balance to exclude the validator weight.
+    pub fn on_slashing(
+        &mut self,
+        is_previous_epoch_target_attester: bool,
+        is_current_epoch_target_attester: bool,
+        effective_balance: u64,
+    ) -> Result<(), BeaconStateError> {
+        let cache = self.get_inner_mut()?;
+        if is_previous_epoch_target_attester {
+            cache
+                .previous_epoch_target_attesting_balance
+                .safe_sub_assign(effective_balance)?;
+        }
+        if is_current_epoch_target_attester {
+            cache
+                .current_epoch_target_attesting_balance
+                .safe_sub_assign(effective_balance)?;
+        }
+        Ok(())
+    }
+
+    /// When a current epoch target attester has its effective balance changed, we adjust the
+    /// its share of the target attesting balance in the cache.
+    pub fn on_effective_balance_change(
+        &mut self,
+        is_current_epoch_target_attester: bool,
+        old_effective_balance: u64,
+        new_effective_balance: u64,
+    ) -> Result<(), BeaconStateError> {
+        let cache = self.get_inner_mut()?;
+        if is_current_epoch_target_attester {
+            if new_effective_balance > old_effective_balance {
+                cache
+                    .current_epoch_target_attesting_balance
+                    .safe_add_assign(new_effective_balance.safe_sub(old_effective_balance)?)?;
+            } else {
+                cache
+                    .current_epoch_target_attesting_balance
+                    .safe_sub_assign(old_effective_balance.safe_sub(new_effective_balance)?)?;
+            }
+        }
+        Ok(())
+    }
+
+    /// On epoch transition, the balance from current epoch is shifted to previous epoch, and the
+    /// current epoch balance is reset to 0.
+    pub fn on_epoch_transition(&mut self, spec: &ChainSpec) -> Result<(), BeaconStateError> {
+        let cache = self.get_inner_mut()?;
+        cache.current_epoch.safe_add_assign(1)?;
+        cache.previous_epoch_target_attesting_balance =
+            cache.current_epoch_target_attesting_balance;
+        cache.current_epoch_target_attesting_balance =
+            Balance::zero(spec.effective_balance_increment);
+        Ok(())
+    }
+
+    pub fn previous_epoch_target_attesting_balance(&self) -> Result<u64, BeaconStateError> {
+        Ok(self
+            .get_inner()?
+            .previous_epoch_target_attesting_balance
+            .get())
+    }
+
+    pub fn current_epoch_target_attesting_balance(&self) -> Result<u64, BeaconStateError> {
+        Ok(self
+            .get_inner()?
+            .current_epoch_target_attesting_balance
+            .get())
+    }
+
+    fn get_inner_mut(&mut self) -> Result<&mut Inner, BeaconStateError> {
+        self.inner
+            .as_mut()
+            .ok_or(BeaconStateError::ProgressiveBalancesCacheNotInitialized)
+    }
+
+    fn get_inner(&self) -> Result<&Inner, BeaconStateError> {
+        self.inner
+            .as_ref()
+            .ok_or(BeaconStateError::ProgressiveBalancesCacheNotInitialized)
+    }
+}
+
+#[derive(
+    Debug, PartialEq, Eq, Clone, Copy, Deserialize, Serialize, Display, EnumString, EnumVariantNames,
+)]
+#[strum(serialize_all = "lowercase")]
+pub enum ProgressiveBalancesMode {
+    /// Disable the usage of progressive cache, and use the existing `ParticipationCache` calculation.
+    Disabled,
+    /// Enable the usage of progressive cache, with checks against the `ParticipationCache` and falls
+    /// back to the existing calculation if there is a balance mismatch.
+    Checked,
+    /// Enable the usage of progressive cache, with checks against the `ParticipationCache`. Errors
+    /// if there is a balance mismatch. Used in testing only.
+    Strict,
+    /// Enable the usage of progressive cache, with no comparative checks against the
+    /// `ParticipationCache`. This is fast but an experimental mode, use with caution.
+    Fast,
+}
+
+impl ProgressiveBalancesMode {
+    pub fn perform_comparative_checks(&self) -> bool {
+        match self {
+            Self::Disabled | Self::Fast => false,
+            Self::Checked | Self::Strict => true,
+        }
+    }
+}
+
+/// `ProgressiveBalancesCache` is only enabled from `Altair` as it requires `ParticipationCache`.
+pub fn is_progressive_balances_enabled<E: EthSpec>(state: &BeaconState<E>) -> bool {
+    match state {
+        BeaconState::Base(_) => false,
+        BeaconState::Altair(_) | BeaconState::Merge(_) | BeaconState::Capella(_) => true,
+    }
+}

consensus/types/src/chain_spec.rs

@@ -835,7 +835,7 @@ impl ChainSpec {
              * Capella hard fork params
              */
             capella_fork_version: [0x03, 0x00, 0x00, 0x64],
-            capella_fork_epoch: None,
+            capella_fork_epoch: Some(Epoch::new(648704)),
             max_validators_per_withdrawals_sweep: 8192,
 
             /*

consensus/types/src/validator.rs

@@ -175,10 +175,10 @@ impl Validator {
         self.activation_epoch() == spec.far_future_epoch
         // Placement in queue could be finalized.
         //
-        // NOTE: it's +1 rather than +2 because we consider the activations that occur at the *end*
-        // of `epoch`, after `process_justification_and_finalization` has already updated the
-        // state's checkpoint.
-        && self.activation_eligibility_epoch() + 1 <= epoch
+        // NOTE: the epoch distance is 1 rather than 2 because we consider the activations that
+        // occur at the *end* of `epoch`, after `process_justification_and_finalization` has already
+        // updated the state's checkpoint.
+        && self.activation_eligibility_epoch() < epoch
     }
 
     fn tree_hash_root_internal(&self) -> Result<Hash256, tree_hash::Error> {