Cache target attester balances for unrealized FFG progression calculation (#4362)

## Issue Addressed

#4118 

## Proposed Changes

This PR introduces a "progressive balances" cache on the `BeaconState`, which keeps track of the accumulated target attestation balance for the current & previous epochs. The cached values are utilised by fork choice to calculate unrealized justification and finalization (instead of converting epoch participation arrays to balances for each block we receive).

This optimization will be rolled out gradually to allow for more testing. A new `--progressive-balances disabled|checked|strict|fast` flag is introduced to support this:
- `checked`: enabled with checks against participation cache, and falls back to the existing epoch processing calculation if there is a total target attester balance mismatch. There is no performance gain from this as the participation cache still needs to be computed. **This is the default mode for now.**
- `strict`: enabled with checks against participation cache, returns error if there is a mismatch. **Used for testing only**.
- `fast`: enabled with no comparative checks and without computing the participation cache. This mode gives us the performance gains from the optimization. This is still experimental and not currently recommended for production usage, but will become the default mode in a future release.
- `disabled`: disable the usage of progressive cache, and use the existing method for FFG progression calculation. This mode may be useful if we find a bug and want to stop the frequent error logs.

### Tasks

- [x] Initial cache implementation in `BeaconState`
- [x] Perform checks in fork choice to compare the progressive balances cache against results from `ParticipationCache`
- [x] Add CLI flag, and disable the optimization by default
- [x] Testing on Goerli & Benchmarking
- [x]  Move caching logic from state processing to the `ProgressiveBalancesCache` (see [this comment](https://github.com/sigp/lighthouse/pull/4362#discussion_r1230877001))
- [x] Add attesting balance metrics



Co-authored-by: Jimmy Chen <jimmy@sigmaprime.io>

consensus/state_processing/src/per_epoch_processing/altair.rs

@@ -1,4 +1,7 @@
 use super::{process_registry_updates, process_slashings, EpochProcessingSummary, Error};
+use crate::common::update_progressive_balances_cache::{
+    initialize_progressive_balances_cache, update_progressive_balances_on_epoch_transition,
+};
 use crate::per_epoch_processing::{
     effective_balance_updates::process_effective_balance_updates,
     historical_roots_update::process_historical_roots_update,
@@ -31,6 +34,7 @@ pub fn process_epoch<T: EthSpec>(
     // Pre-compute participating indices and total balances.
     let participation_cache = ParticipationCache::new(state, spec)?;
     let sync_committee = state.current_sync_committee()?.clone();
+    initialize_progressive_balances_cache::<T>(state, Some(&participation_cache), spec)?;
 
     // Justification and finalization.
     let justification_and_finalization_state =
@@ -56,7 +60,7 @@ pub fn process_epoch<T: EthSpec>(
     process_eth1_data_reset(state)?;
 
     // Update effective balances with hysteresis (lag).
-    process_effective_balance_updates(state, spec)?;
+    process_effective_balance_updates(state, Some(&participation_cache), spec)?;
 
     // Reset slashings
     process_slashings_reset(state)?;
@@ -75,6 +79,8 @@ pub fn process_epoch<T: EthSpec>(
     // Rotate the epoch caches to suit the epoch transition.
     state.advance_caches(spec)?;
 
+    update_progressive_balances_on_epoch_transition(state, spec)?;
+
     Ok(EpochProcessingSummary::Altair {
         participation_cache,
         sync_committee,

consensus/state_processing/src/per_epoch_processing/altair/participation_cache.rs

@@ -11,49 +11,23 @@
 //! Additionally, this cache is returned from the `altair::process_epoch` function and can be used
 //! to get useful summaries about the validator participation in an epoch.
 
-use safe_arith::{ArithError, SafeArith};
 use types::{
     consts::altair::{
         NUM_FLAG_INDICES, TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX,
         TIMELY_TARGET_FLAG_INDEX,
     },
-    BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ParticipationFlags, RelativeEpoch,
+    Balance, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ParticipationFlags,
+    RelativeEpoch,
 };
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Clone)]
 pub enum Error {
     InvalidFlagIndex(usize),
     InvalidValidatorIndex(usize),
 }
 
-/// A balance which will never be below the specified `minimum`.
-///
-/// This is an effort to ensure the `EFFECTIVE_BALANCE_INCREMENT` minimum is always respected.
-#[derive(PartialEq, Debug, Clone, Copy)]
-struct Balance {
-    raw: u64,
-    minimum: u64,
-}
-
-impl Balance {
-    /// Initialize the balance to `0`, or the given `minimum`.
-    pub fn zero(minimum: u64) -> Self {
-        Self { raw: 0, minimum }
-    }
-
-    /// Returns the balance with respect to the initialization `minimum`.
-    pub fn get(&self) -> u64 {
-        std::cmp::max(self.raw, self.minimum)
-    }
-
-    /// Add-assign to the balance.
-    pub fn safe_add_assign(&mut self, other: u64) -> Result<(), ArithError> {
-        self.raw.safe_add_assign(other)
-    }
-}
-
 /// Caches the participation values for one epoch (either the previous or current).
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone)]
 struct SingleEpochParticipationCache {
     /// Maps an active validator index to their participation flags.
     ///
@@ -95,6 +69,14 @@ impl SingleEpochParticipationCache {
             .ok_or(Error::InvalidFlagIndex(flag_index))
     }
 
+    /// Returns the raw total balance of attesters who have `flag_index` set.
+    fn total_flag_balance_raw(&self, flag_index: usize) -> Result<Balance, Error> {
+        self.total_flag_balances
+            .get(flag_index)
+            .copied()
+            .ok_or(Error::InvalidFlagIndex(flag_index))
+    }
+
     /// Returns `true` if `val_index` is active, unslashed and has `flag_index` set.
     ///
     /// ## Errors
@@ -173,7 +155,7 @@ impl SingleEpochParticipationCache {
 }
 
 /// Maintains a cache to be used during `altair::process_epoch`.
-#[derive(PartialEq, Debug)]
+#[derive(PartialEq, Debug, Clone)]
 pub struct ParticipationCache {
     current_epoch: Epoch,
     /// Caches information about active validators pertaining to `self.current_epoch`.
@@ -291,6 +273,11 @@ impl ParticipationCache {
             .total_flag_balance(TIMELY_TARGET_FLAG_INDEX)
     }
 
+    pub fn current_epoch_target_attesting_balance_raw(&self) -> Result<Balance, Error> {
+        self.current_epoch_participation
+            .total_flag_balance_raw(TIMELY_TARGET_FLAG_INDEX)
+    }
+
     pub fn previous_epoch_total_active_balance(&self) -> u64 {
         self.previous_epoch_participation.total_active_balance.get()
     }
@@ -300,6 +287,11 @@ impl ParticipationCache {
             .total_flag_balance(TIMELY_TARGET_FLAG_INDEX)
     }
 
+    pub fn previous_epoch_target_attesting_balance_raw(&self) -> Result<Balance, Error> {
+        self.previous_epoch_participation
+            .total_flag_balance_raw(TIMELY_TARGET_FLAG_INDEX)
+    }
+
     pub fn previous_epoch_source_attesting_balance(&self) -> Result<u64, Error> {
         self.previous_epoch_participation
             .total_flag_balance(TIMELY_SOURCE_FLAG_INDEX)

consensus/state_processing/src/per_epoch_processing/base.rs

@@ -52,7 +52,7 @@ pub fn process_epoch<T: EthSpec>(
     process_eth1_data_reset(state)?;
 
     // Update effective balances with hysteresis (lag).
-    process_effective_balance_updates(state, spec)?;
+    process_effective_balance_updates(state, None, spec)?;
 
     // Reset slashings
     process_slashings_reset(state)?;

consensus/state_processing/src/per_epoch_processing/capella.rs

@@ -11,6 +11,9 @@ use crate::per_epoch_processing::{
 };
 use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch};
 
+use crate::common::update_progressive_balances_cache::{
+    initialize_progressive_balances_cache, update_progressive_balances_on_epoch_transition,
+};
 pub use historical_summaries_update::process_historical_summaries_update;
 
 mod historical_summaries_update;
@@ -27,6 +30,7 @@ pub fn process_epoch<T: EthSpec>(
     // Pre-compute participating indices and total balances.
     let participation_cache = ParticipationCache::new(state, spec)?;
     let sync_committee = state.current_sync_committee()?.clone();
+    initialize_progressive_balances_cache(state, Some(&participation_cache), spec)?;
 
     // Justification and finalization.
     let justification_and_finalization_state =
@@ -52,7 +56,7 @@ pub fn process_epoch<T: EthSpec>(
     process_eth1_data_reset(state)?;
 
     // Update effective balances with hysteresis (lag).
-    process_effective_balance_updates(state, spec)?;
+    process_effective_balance_updates(state, Some(&participation_cache), spec)?;
 
     // Reset slashings
     process_slashings_reset(state)?;
@@ -71,6 +75,8 @@ pub fn process_epoch<T: EthSpec>(
     // Rotate the epoch caches to suit the epoch transition.
     state.advance_caches(spec)?;
 
+    update_progressive_balances_on_epoch_transition(state, spec)?;
+
     Ok(EpochProcessingSummary::Altair {
         participation_cache,
         sync_committee,

consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs

@@ -1,11 +1,13 @@
 use super::errors::EpochProcessingError;
+use crate::per_epoch_processing::altair::ParticipationCache;
 use safe_arith::SafeArith;
 use types::beacon_state::BeaconState;
 use types::chain_spec::ChainSpec;
-use types::{BeaconStateError, EthSpec};
+use types::{BeaconStateError, EthSpec, ProgressiveBalancesCache};
 
 pub fn process_effective_balance_updates<T: EthSpec>(
     state: &mut BeaconState<T>,
+    maybe_participation_cache: Option<&ParticipationCache>,
     spec: &ChainSpec,
 ) -> Result<(), EpochProcessingError> {
     let hysteresis_increment = spec
@@ -13,7 +15,8 @@ pub fn process_effective_balance_updates<T: EthSpec>(
         .safe_div(spec.hysteresis_quotient)?;
     let downward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_downward_multiplier)?;
     let upward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_upward_multiplier)?;
-    let (validators, balances) = state.validators_and_balances_mut();
+    let (validators, balances, progressive_balances_cache) =
+        state.validators_and_balances_and_progressive_balances_mut();
     for (index, validator) in validators.iter_mut().enumerate() {
         let balance = balances
             .get(index)
@@ -23,11 +26,43 @@ pub fn process_effective_balance_updates<T: EthSpec>(
         if balance.safe_add(downward_threshold)? < validator.effective_balance
             || validator.effective_balance.safe_add(upward_threshold)? < balance
         {
-            validator.effective_balance = std::cmp::min(
+            let old_effective_balance = validator.effective_balance;
+            let new_effective_balance = std::cmp::min(
                 balance.safe_sub(balance.safe_rem(spec.effective_balance_increment)?)?,
                 spec.max_effective_balance,
             );
+
+            if let Some(participation_cache) = maybe_participation_cache {
+                update_progressive_balances(
+                    participation_cache,
+                    progressive_balances_cache,
+                    index,
+                    old_effective_balance,
+                    new_effective_balance,
+                )?;
+            }
+
+            validator.effective_balance = new_effective_balance;
         }
     }
     Ok(())
 }
+
+fn update_progressive_balances(
+    participation_cache: &ParticipationCache,
+    progressive_balances_cache: &mut ProgressiveBalancesCache,
+    index: usize,
+    old_effective_balance: u64,
+    new_effective_balance: u64,
+) -> Result<(), EpochProcessingError> {
+    if old_effective_balance != new_effective_balance {
+        let is_current_epoch_target_attester =
+            participation_cache.is_current_epoch_timely_target_attester(index)?;
+        progressive_balances_cache.on_effective_balance_change(
+            is_current_epoch_target_attester,
+            old_effective_balance,
+            new_effective_balance,
+        )?;
+    }
+    Ok(())
+}

consensus/state_processing/src/per_epoch_processing/slashings.rs

@@ -16,7 +16,7 @@ pub fn process_slashings<T: EthSpec>(
         total_balance,
     );
 
-    let (validators, balances) = state.validators_and_balances_mut();
+    let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut();
     for (index, validator) in validators.iter().enumerate() {
         if validator.slashed
             && epoch.safe_add(T::EpochsPerSlashingsVector::to_u64().safe_div(2)?)?