In-memory tree states (#5533)

* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state

consensus/state_processing/src/upgrade/altair.rs

@@ -4,13 +4,13 @@ use std::mem;
 use std::sync::Arc;
 use types::{
     BeaconState, BeaconStateAltair, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec,
-    Fork, ParticipationFlags, PendingAttestation, RelativeEpoch, SyncCommittee, VariableList,
+    Fork, List, ParticipationFlags, PendingAttestation, RelativeEpoch, SyncCommittee,
 };
 
 /// Translate the participation information from the epoch prior to the fork into Altair's format.
 pub fn translate_participation<E: EthSpec>(
     state: &mut BeaconState<E>,
-    pending_attestations: &VariableList<PendingAttestation<E>, E::MaxPendingAttestations>,
+    pending_attestations: &List<PendingAttestation<E>, E::MaxPendingAttestations>,
     spec: &ChainSpec,
 ) -> Result<(), Error> {
     // Previous epoch committee cache is required for `get_attesting_indices`.
@@ -51,8 +51,8 @@ pub fn upgrade_to_altair<E: EthSpec>(
     let pre = pre_state.as_base_mut()?;
 
     let default_epoch_participation =
-        VariableList::new(vec![ParticipationFlags::default(); pre.validators.len()])?;
-    let inactivity_scores = VariableList::new(vec![0; pre.validators.len()])?;
+        List::new(vec![ParticipationFlags::default(); pre.validators.len()])?;
+    let inactivity_scores = List::new(vec![0; pre.validators.len()])?;
 
     let temp_sync_committee = Arc::new(SyncCommittee::temporary());
 
@@ -108,7 +108,6 @@ pub fn upgrade_to_altair<E: EthSpec>(
         exit_cache: mem::take(&mut pre.exit_cache),
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
-        tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
     });
 
     // Fill in previous epoch participation from the pre state's pending attestations.

consensus/state_processing/src/upgrade/capella.rs

@@ -1,7 +1,7 @@
 use std::mem;
 use types::{
     BeaconState, BeaconStateCapella, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec,
-    Fork, VariableList,
+    Fork, List,
 };
 
 /// Transform a `Merge` state into an `Capella` state.
@@ -61,7 +61,7 @@ pub fn upgrade_to_capella<E: EthSpec>(
         // Capella
         next_withdrawal_index: 0,
         next_withdrawal_validator_index: 0,
-        historical_summaries: VariableList::default(),
+        historical_summaries: List::default(),
         // Caches
         total_active_balance: pre.total_active_balance,
         progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache),
@@ -70,7 +70,6 @@ pub fn upgrade_to_capella<E: EthSpec>(
         exit_cache: mem::take(&mut pre.exit_cache),
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
-        tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
     });
 
     *pre_state = post;

consensus/state_processing/src/upgrade/deneb.rs

@@ -71,7 +71,6 @@ pub fn upgrade_to_deneb<E: EthSpec>(
         exit_cache: mem::take(&mut pre.exit_cache),
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
-        tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
     });
 
     *pre_state = post;

consensus/state_processing/src/upgrade/electra.rs

@@ -70,7 +70,6 @@ pub fn upgrade_to_electra<E: EthSpec>(
         exit_cache: mem::take(&mut pre.exit_cache),
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
-        tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
     });
 
     *pre_state = post;

consensus/state_processing/src/upgrade/merge.rs

@@ -66,7 +66,6 @@ pub fn upgrade_to_bellatrix<E: EthSpec>(
         exit_cache: mem::take(&mut pre.exit_cache),
         slashings_cache: mem::take(&mut pre.slashings_cache),
         epoch_cache: EpochCache::default(),
-        tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
     });
 
     *pre_state = post;