In-memory tree states (#5533)

* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state

consensus/types/benches/benches.rs

@@ -2,6 +2,7 @@
 
 use criterion::Criterion;
 use criterion::{black_box, criterion_group, criterion_main, Benchmark};
+use milhouse::List;
 use rayon::prelude::*;
 use ssz::Encode;
 use std::sync::Arc;
@@ -27,21 +28,23 @@ fn get_state<E: EthSpec>(validator_count: usize) -> BeaconState<E> {
             .expect("should add balance");
     }
 
-    *state.validators_mut() = (0..validator_count)
-        .collect::<Vec<_>>()
-        .par_iter()
-        .map(|&i| Validator {
-            pubkey: generate_deterministic_keypair(i).pk.into(),
-            withdrawal_credentials: Hash256::from_low_u64_le(i as u64),
-            effective_balance: spec.max_effective_balance,
-            slashed: false,
-            activation_eligibility_epoch: Epoch::new(0),
-            activation_epoch: Epoch::new(0),
-            exit_epoch: Epoch::from(u64::max_value()),
-            withdrawable_epoch: Epoch::from(u64::max_value()),
-        })
-        .collect::<Vec<_>>()
-        .into();
+    *state.validators_mut() = List::new(
+        (0..validator_count)
+            .collect::<Vec<_>>()
+            .par_iter()
+            .map(|&i| Validator {
+                pubkey: generate_deterministic_keypair(i).pk.compress(),
+                withdrawal_credentials: Hash256::from_low_u64_le(i as u64),
+                effective_balance: spec.max_effective_balance,
+                slashed: false,
+                activation_eligibility_epoch: Epoch::new(0),
+                activation_epoch: Epoch::new(0),
+                exit_epoch: Epoch::from(u64::max_value()),
+                withdrawable_epoch: Epoch::from(u64::max_value()),
+            })
+            .collect(),
+    )
+    .unwrap();
 
     state
 }
@@ -96,19 +99,6 @@ fn all_benches(c: &mut Criterion) {
         .sample_size(10),
     );
 
-    let inner_state = state.clone();
-    c.bench(
-        &format!("{}_validators", validator_count),
-        Benchmark::new("clone/tree_hash_cache", move |b| {
-            b.iter_batched_ref(
-                || inner_state.clone(),
-                |state| black_box(state.tree_hash_cache().clone()),
-                criterion::BatchSize::SmallInput,
-            )
-        })
-        .sample_size(10),
-    );
-
     let inner_state = state.clone();
     c.bench(
         &format!("{}_validators", validator_count),