gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-20 22:38:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

In-memory tree states (#5533)

Browse Source 
* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state
This commit is contained in:
Michael Sproul
2024-04-24 11:22:36 +10:00
committed by GitHub
parent 4cad1fcbbe
commit 61962898e2
108 changed files with 2038 additions and 2762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
beacon_node/beacon_chain/tests/block_verification.rs
View File

@@ -15,8 +15,7 @@ use slasher::{Config as SlasherConfig, Slasher};
use state_processing::{
common::get_indexed_attestation,
per_block_processing::{per_block_processing, BlockSignatureStrategy},
per_slot_processing, BlockProcessingError, ConsensusContext, StateProcessingStrategy,
VerifyBlockRoot,
per_slot_processing, BlockProcessingError, ConsensusContext, VerifyBlockRoot,
};
use std::marker::PhantomData;
use std::sync::Arc;
@@ -1309,7 +1308,6 @@ async fn add_base_block_to_altair_chain() {
&mut state,
&base_block,
BlockSignatureStrategy::NoVerification,
StateProcessingStrategy::Accurate,
VerifyBlockRoot::True,
&mut ctxt,
&harness.chain.spec,
@@ -1445,7 +1443,6 @@ async fn add_altair_block_to_base_chain() {
&mut state,
&altair_block,
BlockSignatureStrategy::NoVerification,
StateProcessingStrategy::Accurate,
VerifyBlockRoot::True,
&mut ctxt,
&harness.chain.spec,

4
beacon_node/beacon_chain/tests/payload_invalidation.rs
View File

@@ -223,7 +223,7 @@ impl InvalidPayloadRig {
let mock_execution_layer = self.harness.mock_execution_layer.as_ref().unwrap();
let head = self.harness.chain.head_snapshot();
let state = head.beacon_state.clone_with_only_committee_caches();
let state = head.beacon_state.clone();
let slot = slot_override.unwrap_or(state.slot() + 1);
let ((block, blobs), post_state) = self.harness.make_block(state, slot).await;
let block_root = block.canonical_root();
@@ -2048,7 +2048,7 @@ async fn weights_after_resetting_optimistic_status() {
.fork_choice_read_lock()
.get_block_weight(&head.head_block_root())
.unwrap(),
head.snapshot.beacon_state.validators()[0].effective_balance,
head.snapshot.beacon_state.validators().get(0).unwrap().effective_balance,
"proposer boost should be removed from the head block and the vote of a single validator applied"
);

28
beacon_node/beacon_chain/tests/rewards.rs
View File

@@ -105,8 +105,8 @@ async fn test_sync_committee_rewards() {
.get_validator_index(&validator.pubkey)
.unwrap()
.unwrap();
let pre_state_balance = parent_state.balances()[validator_index];
let post_state_balance = state.balances()[validator_index];
let pre_state_balance = *parent_state.balances().get(validator_index).unwrap();
let post_state_balance = *state.balances().get(validator_index).unwrap();
let sync_committee_reward = rewards.get(&(validator_index as u64)).unwrap_or(&0);
if validator_index == proposer_index {
@@ -141,7 +141,7 @@ async fn test_verify_attestation_rewards_base() {
)
.await;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
// extend slots to beginning of epoch N + 2
harness.extend_slots(E::slots_per_epoch() as usize).await;
@@ -163,7 +163,7 @@ async fn test_verify_attestation_rewards_base() {
let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}
@@ -185,7 +185,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak() {
AttestationStrategy::SomeValidators(half_validators.clone()),
)
.await;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
// extend slots to beginning of epoch N + 2
harness.advance_slot();
@@ -215,7 +215,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak() {
let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}
@@ -241,7 +241,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak_justification_epoc
// advance to create first justification epoch and get initial balances
harness.extend_slots(E::slots_per_epoch() as usize).await;
target_epoch += 1;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
//assert previous_justified_checkpoint matches 0 as we were in inactivity leak from beginning
assert_eq!(
@@ -284,7 +284,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak_justification_epoc
let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}
@@ -298,7 +298,7 @@ async fn test_verify_attestation_rewards_altair() {
harness
.extend_slots((E::slots_per_epoch() * (target_epoch + 1)) as usize)
.await;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
// advance until epoch N + 2 and build proposal rewards map
let mut proposal_rewards_map: HashMap<u64, u64> = HashMap::new();
@@ -364,7 +364,7 @@ async fn test_verify_attestation_rewards_altair() {
apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}
@@ -386,7 +386,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak() {
half_validators.clone(),
)
.await;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
// advance until epoch N + 2 and build proposal rewards map
let mut proposal_rewards_map: HashMap<u64, u64> = HashMap::new();
@@ -458,7 +458,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak() {
apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}
@@ -492,7 +492,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak_justification_ep
// advance for first justification epoch and get balances
harness.extend_slots(E::slots_per_epoch() as usize).await;
target_epoch += 1;
let initial_balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let initial_balances: Vec<u64> = harness.get_current_state().balances().to_vec();
// advance until epoch N + 2 and build proposal rewards map
let mut proposal_rewards_map: HashMap<u64, u64> = HashMap::new();
@@ -568,7 +568,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak_justification_ep
apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances);
// verify expected balances against actual balances
let balances: Vec<u64> = harness.get_current_state().balances().clone().into();
let balances: Vec<u64> = harness.get_current_state().balances().to_vec();
assert_eq!(expected_balances, balances);
}

72
beacon_node/beacon_chain/tests/store_tests.rs
View File

@@ -719,52 +719,6 @@ async fn forwards_iter_block_and_state_roots_until() {
test_range(Slot::new(0), head_state.slot());
}
#[tokio::test]
async fn block_replay_with_inaccurate_state_roots() {
let num_blocks_produced = E::slots_per_epoch() * 3 + 31;
let db_path = tempdir().unwrap();
let store = get_store(&db_path);
let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT);
let chain = &harness.chain;
harness
.extend_chain(
num_blocks_produced as usize,
BlockStrategy::OnCanonicalHead,
AttestationStrategy::AllValidators,
)
.await;
// Slot must not be 0 mod 32 or else no blocks will be replayed.
let (mut head_state, head_state_root) = harness.get_current_state_and_root();
let head_block_root = harness.head_block_root();
assert_ne!(head_state.slot() % 32, 0);
let (_, mut fast_head_state) = store
.get_inconsistent_state_for_attestation_verification_only(
&head_block_root,
head_state.slot(),
head_state_root,
)
.unwrap()
.unwrap();
assert_eq!(head_state.validators(), fast_head_state.validators());
head_state.build_all_committee_caches(&chain.spec).unwrap();
fast_head_state
.build_all_committee_caches(&chain.spec)
.unwrap();
assert_eq!(
head_state
.get_cached_active_validator_indices(RelativeEpoch::Current)
.unwrap(),
fast_head_state
.get_cached_active_validator_indices(RelativeEpoch::Current)
.unwrap()
);
}
#[tokio::test]
async fn block_replayer_hooks() {
let db_path = tempdir().unwrap();
@@ -795,7 +749,7 @@ async fn block_replayer_hooks() {
let mut post_block_slots = vec![];
let mut replay_state = BlockReplayer::<MinimalEthSpec>::new(state, &chain.spec)
.pre_slot_hook(Box::new(|state| {
.pre_slot_hook(Box::new(|_, state| {
pre_slots.push(state.slot());
Ok(())
}))
@@ -834,6 +788,8 @@ async fn block_replayer_hooks() {
assert_eq!(post_block_slots, block_slots);
// States match.
end_state.apply_pending_mutations().unwrap();
replay_state.apply_pending_mutations().unwrap();
end_state.drop_all_caches().unwrap();
replay_state.drop_all_caches().unwrap();
assert_eq!(end_state, replay_state);
@@ -1219,9 +1175,17 @@ fn check_shuffling_compatible(
|committee_cache, _| {
let state_cache = head_state.committee_cache(RelativeEpoch::Current).unwrap();
if current_epoch_shuffling_is_compatible {
assert_eq!(committee_cache, state_cache, "block at slot {slot}");
assert_eq!(
committee_cache,
state_cache.as_ref(),
"block at slot {slot}"
);
} else {
assert_ne!(committee_cache, state_cache, "block at slot {slot}");
assert_ne!(
committee_cache,
state_cache.as_ref(),
"block at slot {slot}"
);
}
Ok(())
},
@@ -1251,9 +1215,9 @@ fn check_shuffling_compatible(
|committee_cache, _| {
let state_cache = head_state.committee_cache(RelativeEpoch::Previous).unwrap();
if previous_epoch_shuffling_is_compatible {
assert_eq!(committee_cache, state_cache);
assert_eq!(committee_cache, state_cache.as_ref());
} else {
assert_ne!(committee_cache, state_cache);
assert_ne!(committee_cache, state_cache.as_ref());
}
Ok(())
},
@@ -3605,16 +3569,16 @@ fn check_split_slot(harness: &TestHarness, store: Arc<HotColdDB<E, LevelDB<E>, L
/// Check that all the states in a chain dump have the correct tree hash.
fn check_chain_dump(harness: &TestHarness, expected_len: u64) {
let chain_dump = harness.chain.chain_dump().unwrap();
let mut chain_dump = harness.chain.chain_dump().unwrap();
let split_slot = harness.chain.store.get_split_slot();
assert_eq!(chain_dump.len() as u64, expected_len);
for checkpoint in &chain_dump {
for checkpoint in &mut chain_dump {
// Check that the tree hash of the stored state is as expected
assert_eq!(
checkpoint.beacon_state_root(),
checkpoint.beacon_state.tree_hash_root(),
checkpoint.beacon_state.update_tree_hash_cache().unwrap(),
"tree hash of stored state is incorrect"
);