gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-15 02:42:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

In-memory tree states (#5533)

Browse Source 
* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state
This commit is contained in:
Michael Sproul
2024-04-24 11:22:36 +10:00
committed by GitHub
parent 4cad1fcbbe
commit 61962898e2
108 changed files with 2038 additions and 2762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs
View File

@@ -2,17 +2,14 @@ use crate::EpochProcessingError;
use types::beacon_state::BeaconState;
use types::eth_spec::EthSpec;
use types::participation_flags::ParticipationFlags;
use types::VariableList;
use types::List;
pub fn process_participation_flag_updates<E: EthSpec>(
state: &mut BeaconState<E>,
) -> Result<(), EpochProcessingError> {
*state.previous_epoch_participation_mut()? =
std::mem::take(state.current_epoch_participation_mut()?);
*state.current_epoch_participation_mut()? = VariableList::new(vec![
ParticipationFlags::default(
);
state.validators().len()
])?;
*state.current_epoch_participation_mut()? =
List::repeat(ParticipationFlags::default(), state.validators().len())?;
Ok(())
}

3
consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs
View File

@@ -13,6 +13,9 @@ pub fn process_historical_summaries_update<E: EthSpec>(
.safe_rem((E::slots_per_historical_root() as u64).safe_div(E::slots_per_epoch())?)?
== 0
{
// We need to flush any pending mutations before hashing.
state.block_roots_mut().apply_updates()?;
state.state_roots_mut().apply_updates()?;
let summary = HistoricalSummary::new(state);
return state
.historical_summaries_mut()?

6
consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs
View File

@@ -21,7 +21,9 @@ pub fn process_effective_balance_updates<E: EthSpec>(
let downward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_downward_multiplier)?;
let upward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_upward_multiplier)?;
let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut();
for (index, validator) in validators.iter_mut().enumerate() {
let mut validators_iter = validators.iter_cow();
while let Some((index, validator)) = validators_iter.next_cow() {
let balance = balances
.get(index)
.copied()
@@ -44,7 +46,7 @@ pub fn process_effective_balance_updates<E: EthSpec>(
}
if new_effective_balance != validator.effective_balance {
validator.effective_balance = new_effective_balance;
validator.into_mut()?.effective_balance = new_effective_balance;
}
}

16
consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs
View File

@@ -3,8 +3,8 @@ use crate::metrics;
use std::sync::Arc;
use types::{
consts::altair::{TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX},
BeaconStateError, Epoch, EthSpec, ParticipationFlags, ProgressiveBalancesCache, SyncCommittee,
Validator, VariableList,
BeaconStateError, Epoch, EthSpec, List, ParticipationFlags, ProgressiveBalancesCache,
SyncCommittee, Validator,
};
/// Provides a summary of validator participation during the epoch.
@@ -25,20 +25,20 @@ pub enum EpochProcessingSummary<E: EthSpec> {
#[derive(PartialEq, Debug)]
pub struct ParticipationEpochSummary<E: EthSpec> {
/// Copy of the validator registry prior to mutation.
validators: VariableList<Validator, E::ValidatorRegistryLimit>,
validators: List<Validator, E::ValidatorRegistryLimit>,
/// Copy of the participation flags for the previous epoch.
previous_epoch_participation: VariableList<ParticipationFlags, E::ValidatorRegistryLimit>,
previous_epoch_participation: List<ParticipationFlags, E::ValidatorRegistryLimit>,
/// Copy of the participation flags for the current epoch.
current_epoch_participation: VariableList<ParticipationFlags, E::ValidatorRegistryLimit>,
current_epoch_participation: List<ParticipationFlags, E::ValidatorRegistryLimit>,
previous_epoch: Epoch,
current_epoch: Epoch,
}
impl<E: EthSpec> ParticipationEpochSummary<E> {
pub fn new(
validators: VariableList<Validator, E::ValidatorRegistryLimit>,
previous_epoch_participation: VariableList<ParticipationFlags, E::ValidatorRegistryLimit>,
current_epoch_participation: VariableList<ParticipationFlags, E::ValidatorRegistryLimit>,
validators: List<Validator, E::ValidatorRegistryLimit>,
previous_epoch_participation: List<ParticipationFlags, E::ValidatorRegistryLimit>,
current_epoch_participation: List<ParticipationFlags, E::ValidatorRegistryLimit>,
previous_epoch: Epoch,
current_epoch: Epoch,
) -> Self {

9
consensus/state_processing/src/per_epoch_processing/errors.rs
View File

@@ -1,4 +1,4 @@
use types::{BeaconStateError, EpochCacheError, InconsistentFork};
use types::{milhouse, BeaconStateError, EpochCacheError, InconsistentFork};
#[derive(Debug, PartialEq)]
pub enum EpochProcessingError {
@@ -23,6 +23,7 @@ pub enum EpochProcessingError {
InconsistentStateFork(InconsistentFork),
InvalidJustificationBit(ssz_types::Error),
InvalidFlagIndex(usize),
MilhouseError(milhouse::Error),
EpochCache(EpochCacheError),
}
@@ -50,6 +51,12 @@ impl From<safe_arith::ArithError> for EpochProcessingError {
}
}
impl From<milhouse::Error> for EpochProcessingError {
fn from(e: milhouse::Error) -> Self {
Self::MilhouseError(e)
}
}
impl From<EpochCacheError> for EpochProcessingError {
fn from(e: EpochCacheError) -> Self {
EpochProcessingError::EpochCache(e)

2
consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs
View File

@@ -14,7 +14,7 @@ pub fn process_historical_roots_update<E: EthSpec>(
.safe_rem(E::SlotsPerHistoricalRoot::to_u64().safe_div(E::slots_per_epoch())?)?
== 0
{
let historical_batch = state.historical_batch();
let historical_batch = state.historical_batch()?;
state
.historical_roots_mut()
.push(historical_batch.tree_hash_root())?;

4
consensus/state_processing/src/per_epoch_processing/resets.rs
View File

@@ -2,7 +2,7 @@ use super::errors::EpochProcessingError;
use safe_arith::SafeArith;
use types::beacon_state::BeaconState;
use types::eth_spec::EthSpec;
use types::{Unsigned, VariableList};
use types::{List, Unsigned};
pub fn process_eth1_data_reset<E: EthSpec>(
state: &mut BeaconState<E>,
@@ -13,7 +13,7 @@ pub fn process_eth1_data_reset<E: EthSpec>(
.safe_rem(E::SlotsPerEth1VotingPeriod::to_u64())?
== 0
{
*state.eth1_data_votes_mut() = VariableList::empty();
*state.eth1_data_votes_mut() = List::empty();
}
Ok(())
}

66
consensus/state_processing/src/per_epoch_processing/single_pass.rs
View File

@@ -12,6 +12,7 @@ use types::{
NUM_FLAG_INDICES, PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX,
TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR,
},
milhouse::Cow,
ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExitCache, ForkName,
ParticipationFlags, ProgressiveBalancesCache, RelativeEpoch, Unsigned, Validator,
};
@@ -173,9 +174,9 @@ pub fn process_epoch_single_pass<E: EthSpec>(
let effective_balances_ctxt = &EffectiveBalancesContext::new(spec)?;
// Iterate over the validators and related fields in one pass.
let mut validators_iter = validators.iter_mut();
let mut balances_iter = balances.iter_mut();
let mut inactivity_scores_iter = inactivity_scores.iter_mut();
let mut validators_iter = validators.iter_cow();
let mut balances_iter = balances.iter_cow();
let mut inactivity_scores_iter = inactivity_scores.iter_cow();
// Values computed for the next epoch transition.
let mut next_epoch_total_active_balance = 0;
@@ -186,14 +187,14 @@ pub fn process_epoch_single_pass<E: EthSpec>(
previous_epoch_participation.iter(),
current_epoch_participation.iter(),
) {
let validator = validators_iter
.next()
let (_, mut validator) = validators_iter
.next_cow()
.ok_or(BeaconStateError::UnknownValidator(index))?;
let balance = balances_iter
.next()
let (_, mut balance) = balances_iter
.next_cow()
.ok_or(BeaconStateError::UnknownValidator(index))?;
let inactivity_score = inactivity_scores_iter
.next()
let (_, mut inactivity_score) = inactivity_scores_iter
.next_cow()
.ok_or(BeaconStateError::UnknownValidator(index))?;
let is_active_current_epoch = validator.is_active_at(current_epoch);
@@ -223,7 +224,7 @@ pub fn process_epoch_single_pass<E: EthSpec>(
// `process_inactivity_updates`
if conf.inactivity_updates {
process_single_inactivity_update(
inactivity_score,
&mut inactivity_score,
validator_info,
state_ctxt,
spec,
@@ -233,8 +234,8 @@ pub fn process_epoch_single_pass<E: EthSpec>(
// `process_rewards_and_penalties`
if conf.rewards_and_penalties {
process_single_reward_and_penalty(
balance,
inactivity_score,
&mut balance,
&inactivity_score,
validator_info,
rewards_ctxt,
state_ctxt,
@@ -246,7 +247,7 @@ pub fn process_epoch_single_pass<E: EthSpec>(
// `process_registry_updates`
if conf.registry_updates {
process_single_registry_update(
validator,
&mut validator,
validator_info,
exit_cache,
activation_queue,
@@ -258,14 +259,14 @@ pub fn process_epoch_single_pass<E: EthSpec>(
// `process_slashings`
if conf.slashings {
process_single_slashing(balance, validator, slashings_ctxt, state_ctxt, spec)?;
process_single_slashing(&mut balance, &validator, slashings_ctxt, state_ctxt, spec)?;
}
// `process_effective_balance_updates`
if conf.effective_balance_updates {
process_single_effective_balance_update(
*balance,
validator,
&mut validator,
validator_info,
&mut next_epoch_total_active_balance,
&mut next_epoch_cache,
@@ -290,7 +291,7 @@ pub fn process_epoch_single_pass<E: EthSpec>(
}
fn process_single_inactivity_update(
inactivity_score: &mut u64,
inactivity_score: &mut Cow<u64>,
validator_info: &ValidatorInfo,
state_ctxt: &StateContext,
spec: &ChainSpec,
@@ -303,25 +304,27 @@ fn process_single_inactivity_update(
if validator_info.is_unslashed_participating_index(TIMELY_TARGET_FLAG_INDEX)? {
// Avoid mutating when the inactivity score is 0 and can't go any lower -- the common
// case.
if *inactivity_score == 0 {
if **inactivity_score == 0 {
return Ok(());
}
inactivity_score.safe_sub_assign(1)?;
inactivity_score.make_mut()?.safe_sub_assign(1)?;
} else {
inactivity_score.safe_add_assign(spec.inactivity_score_bias)?;
inactivity_score
.make_mut()?
.safe_add_assign(spec.inactivity_score_bias)?;
}
// Decrease the score of all validators for forgiveness when not during a leak
if !state_ctxt.is_in_inactivity_leak {
let deduction = min(spec.inactivity_score_recovery_rate, *inactivity_score);
inactivity_score.safe_sub_assign(deduction)?;
let deduction = min(spec.inactivity_score_recovery_rate, **inactivity_score);
inactivity_score.make_mut()?.safe_sub_assign(deduction)?;
}
Ok(())
}
fn process_single_reward_and_penalty(
balance: &mut u64,
balance: &mut Cow<u64>,
inactivity_score: &u64,
validator_info: &ValidatorInfo,
rewards_ctxt: &RewardsAndPenaltiesContext,
@@ -351,6 +354,7 @@ fn process_single_reward_and_penalty(
)?;
if delta.rewards != 0 || delta.penalties != 0 {
let balance = balance.make_mut()?;
balance.safe_add_assign(delta.rewards)?;
*balance = balance.saturating_sub(delta.penalties);
}
@@ -452,7 +456,7 @@ impl RewardsAndPenaltiesContext {
}
fn process_single_registry_update(
validator: &mut Validator,
validator: &mut Cow<Validator>,
validator_info: &ValidatorInfo,
exit_cache: &mut ExitCache,
activation_queue: &BTreeSet<usize>,
@@ -463,7 +467,7 @@ fn process_single_registry_update(
let current_epoch = state_ctxt.current_epoch;
if validator.is_eligible_for_activation_queue(spec) {
validator.activation_eligibility_epoch = current_epoch.safe_add(1)?;
validator.make_mut()?.activation_eligibility_epoch = current_epoch.safe_add(1)?;
}
if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance
@@ -472,7 +476,8 @@ fn process_single_registry_update(
}
if activation_queue.contains(&validator_info.index) {
validator.activation_epoch = spec.compute_activation_exit_epoch(current_epoch)?;
validator.make_mut()?.activation_epoch =
spec.compute_activation_exit_epoch(current_epoch)?;
}
// Caching: add to speculative activation queue for next epoch.
@@ -487,7 +492,7 @@ fn process_single_registry_update(
}
fn initiate_validator_exit(
validator: &mut Validator,
validator: &mut Cow<Validator>,
exit_cache: &mut ExitCache,
state_ctxt: &StateContext,
spec: &ChainSpec,
@@ -508,6 +513,7 @@ fn initiate_validator_exit(
exit_queue_epoch.safe_add_assign(1)?;
}
let validator = validator.make_mut()?;
validator.exit_epoch = exit_queue_epoch;
validator.withdrawable_epoch =
exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?;
@@ -540,7 +546,7 @@ impl SlashingsContext {
}
fn process_single_slashing(
balance: &mut u64,
balance: &mut Cow<u64>,
validator: &Validator,
slashings_ctxt: &SlashingsContext,
state_ctxt: &StateContext,
@@ -557,7 +563,7 @@ fn process_single_slashing(
.safe_div(state_ctxt.total_active_balance)?
.safe_mul(increment)?;
*balance = balance.saturating_sub(penalty);
*balance.make_mut()? = balance.saturating_sub(penalty);
}
Ok(())
}
@@ -581,7 +587,7 @@ impl EffectiveBalancesContext {
#[allow(clippy::too_many_arguments)]
fn process_single_effective_balance_update(
balance: u64,
validator: &mut Validator,
validator: &mut Cow<Validator>,
validator_info: &ValidatorInfo,
next_epoch_total_active_balance: &mut u64,
next_epoch_cache: &mut PreEpochCache,
@@ -611,7 +617,7 @@ fn process_single_effective_balance_update(
}
if new_effective_balance != old_effective_balance {
validator.effective_balance = new_effective_balance;
validator.make_mut()?.effective_balance = new_effective_balance;
// Update progressive balances cache for the *current* epoch, which will soon become the
// previous epoch once the epoch transition completes.