In-memory tree states (#5533)

* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state

lcli/src/new_testnet.rs

@@ -264,7 +264,7 @@ fn initialize_state_with_validators<E: EthSpec>(
     let mut state = BeaconState::new(genesis_time, eth1_data, spec);
 
     // Seed RANDAO with Eth1 entropy
-    state.fill_randao_mixes_with(eth1_block_hash);
+    state.fill_randao_mixes_with(eth1_block_hash).unwrap();
 
     for keypair in keypairs.iter() {
         let withdrawal_credentials = |pubkey: &PublicKey| {

lcli/src/replace_state_pubkeys.rs

@@ -42,7 +42,8 @@ pub fn run<E: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(),
 
     let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH);
     let mut deposit_root = Hash256::zero();
-    for (index, validator) in state.validators_mut().iter_mut().enumerate() {
+    let validators = state.validators_mut();
+    for index in 0..validators.len() {
         let (secret, _) =
             recover_validator_secret_from_mnemonic(seed.as_bytes(), index as u32, KeyType::Voting)
                 .map_err(|e| format!("Unable to generate validator key: {:?}", e))?;
@@ -52,11 +53,11 @@ pub fn run<E: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(),
 
         eprintln!("{}: {}", index, keypair.pk);
 
-        validator.pubkey = keypair.pk.into();
+        validators.get_mut(index).unwrap().pubkey = keypair.pk.into();
 
         // Update the deposit tree.
         let mut deposit_data = DepositData {
-            pubkey: validator.pubkey,
+            pubkey: validators.get(index).unwrap().pubkey,
             // Set this to a junk value since it's very time consuming to generate the withdrawal
             // keys and it's not useful for the time being.
             withdrawal_credentials: Hash256::zero(),

lcli/src/skip_slots.rs

@@ -57,7 +57,7 @@ use std::fs::File;
 use std::io::prelude::*;
 use std::path::PathBuf;
 use std::time::{Duration, Instant};
-use types::{BeaconState, CloneConfig, EthSpec, Hash256};
+use types::{BeaconState, EthSpec, Hash256};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 
@@ -128,7 +128,7 @@ pub fn run<E: EthSpec>(
     };
 
     for i in 0..runs {
-        let mut state = state.clone_with(CloneConfig::all());
+        let mut state = state.clone();
 
         let start = Instant::now();
 

lcli/src/transition_blocks.rs

@@ -76,7 +76,7 @@ use ssz::Encode;
 use state_processing::state_advance::complete_state_advance;
 use state_processing::{
     block_signature_verifier::BlockSignatureVerifier, per_block_processing, AllCaches,
-    BlockSignatureStrategy, ConsensusContext, StateProcessingStrategy, VerifyBlockRoot,
+    BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot,
 };
 use std::borrow::Cow;
 use std::fs::File;
@@ -85,7 +85,7 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use store::HotColdDB;
-use types::{BeaconState, ChainSpec, CloneConfig, EthSpec, Hash256, SignedBeaconBlock};
+use types::{BeaconState, ChainSpec, EthSpec, Hash256, SignedBeaconBlock};
 
 const HTTP_TIMEOUT: Duration = Duration::from_secs(10);
 
@@ -234,7 +234,7 @@ pub fn run<E: EthSpec>(
     let mut output_post_state = None;
     let mut saved_ctxt = None;
     for i in 0..runs {
-        let pre_state = pre_state.clone_with(CloneConfig::all());
+        let pre_state = pre_state.clone();
         let block = block.clone();
 
         let start = Instant::now();
@@ -399,7 +399,6 @@ fn do_transition<E: EthSpec>(
         &mut pre_state,
         &block,
         BlockSignatureStrategy::NoVerification,
-        StateProcessingStrategy::Accurate,
         VerifyBlockRoot::True,
         &mut ctxt,
         spec,