gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-03 00:31:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Single-pass epoch processing and optimised block processing (#5279)

Browse Source 
* Single-pass epoch processing (#4483, #4573)

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

* Delete unused epoch processing code (#5170)

* Delete unused epoch processing code

* Compare total deltas

* Remove unnecessary apply_pending

* cargo fmt

* Remove newline

* Use epoch cache in block packing (#5223)

* Remove progressive balances mode (#5224)

* inline inactivity_penalty_quotient_for_state

* drop previous_epoch_total_active_balance

* fc lint

* spec compliant process_sync_aggregate (#15)

* spec compliant process_sync_aggregate

* Update consensus/state_processing/src/per_block_processing/altair/sync_committee.rs

Co-authored-by: Michael Sproul <micsproul@gmail.com>

---------

Co-authored-by: Michael Sproul <micsproul@gmail.com>

* Delete the participation cache (#16)

* update help

* Fix op_pool tests

* Fix fork choice tests

* Merge remote-tracking branch 'sigp/unstable' into epoch-single-pass

* Simplify exit cache (#5280)

* Fix clippy on exit cache

* Clean up single-pass a bit (#5282)

* Address Mark's review of single-pass (#5386)

* Merge remote-tracking branch 'origin/unstable' into epoch-single-pass

* Address Sean's review comments (#5414)

* Address most of Sean's review comments

* Simplify total balance cache building

* Clean up unused junk

* Merge remote-tracking branch 'origin/unstable' into epoch-single-pass

* More self-review

* Merge remote-tracking branch 'origin/unstable' into epoch-single-pass

* Merge branch 'unstable' into epoch-single-pass

* Fix imports for beta compiler

* Fix tests, probably
This commit is contained in:
Michael Sproul
2024-04-05 00:14:36 +11:00
committed by GitHub
parent f4cdcea7b1
commit feb531f85b
81 changed files with 2545 additions and 1316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
testing/ef_tests/src/cases/epoch_processing.rs
View File

@@ -4,12 +4,17 @@ use crate::case_result::compare_beacon_state_results_without_caches;
use crate::decode::{ssz_decode_state, yaml_decode_file};
use crate::type_name;
use serde::Deserialize;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use state_processing::epoch_cache::initialize_epoch_cache;
use state_processing::per_epoch_processing::capella::process_historical_summaries_update;
use state_processing::per_epoch_processing::effective_balance_updates::process_effective_balance_updates;
use state_processing::per_epoch_processing::effective_balance_updates::{
process_effective_balance_updates, process_effective_balance_updates_slow,
};
use state_processing::per_epoch_processing::{
altair, base,
historical_roots_update::process_historical_roots_update,
process_registry_updates, process_slashings,
process_registry_updates, process_registry_updates_slow, process_slashings,
process_slashings_slow,
resets::{process_eth1_data_reset, process_randao_mixes_reset, process_slashings_reset},
};
use state_processing::EpochProcessingError;
@@ -104,11 +109,9 @@ impl<E: EthSpec> EpochTransition<E> for JustificationAndFinalization {
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => {
initialize_progressive_balances_cache(state, spec)?;
let justification_and_finalization_state =
altair::process_justification_and_finalization(
state,
&altair::ParticipationCache::new(state, spec).unwrap(),
)?;
altair::process_justification_and_finalization(state)?;
justification_and_finalization_state.apply_changes_to_state(state);
Ok(())
}
@@ -128,18 +131,20 @@ impl<E: EthSpec> EpochTransition<E> for RewardsAndPenalties {
| BeaconState::Merge(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => altair::process_rewards_and_penalties(
state,
&altair::ParticipationCache::new(state, spec).unwrap(),
spec,
),
| BeaconState::Electra(_) => altair::process_rewards_and_penalties_slow(state, spec),
}
}
}
impl<E: EthSpec> EpochTransition<E> for RegistryUpdates {
fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
process_registry_updates(state, spec)
initialize_epoch_cache(state, spec)?;
if let BeaconState::Base(_) = state {
process_registry_updates(state, spec)
} else {
process_registry_updates_slow(state, spec)
}
}
}
@@ -160,13 +165,7 @@ impl<E: EthSpec> EpochTransition<E> for Slashings {
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => {
process_slashings(
state,
altair::ParticipationCache::new(state, spec)
.unwrap()
.current_epoch_total_active_balance(),
spec,
)?;
process_slashings_slow(state, spec)?;
}
};
Ok(())
@@ -181,7 +180,11 @@ impl<E: EthSpec> EpochTransition<E> for Eth1DataReset {
impl<E: EthSpec> EpochTransition<E> for EffectiveBalanceUpdates {
fn run(state: &mut BeaconState<E>, spec: &ChainSpec) -> Result<(), EpochProcessingError> {
process_effective_balance_updates(state, None, spec)
if let BeaconState::Base(_) = state {
process_effective_balance_updates(state, spec)
} else {
process_effective_balance_updates_slow(state, spec)
}
}
}
@@ -250,11 +253,7 @@ impl<E: EthSpec> EpochTransition<E> for InactivityUpdates {
| BeaconState::Merge(_)
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => altair::process_inactivity_updates(
state,
&altair::ParticipationCache::new(state, spec).unwrap(),
spec,
),
| BeaconState::Electra(_) => altair::process_inactivity_updates_slow(state, spec),
}
}
}
@@ -328,17 +327,20 @@ impl<E: EthSpec, T: EpochTransition<E>> Case for EpochProcessing<E, T> {
fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
self.metadata.bls_setting.unwrap_or_default().check()?;
let mut state = self.pre.clone();
let spec = &testing_spec::<E>(fork_name);
let mut pre_state = self.pre.clone();
// Processing requires the committee caches.
pre_state.build_all_committee_caches(spec).unwrap();
let mut state = pre_state.clone();
let mut expected = self.post.clone();
let spec = &testing_spec::<E>(fork_name);
if let Some(post_state) = expected.as_mut() {
post_state.build_all_committee_caches(spec).unwrap();
}
let mut result = (|| {
// Processing requires the committee caches.
state.build_all_committee_caches(spec)?;
T::run(&mut state, spec).map(|_| state)
})();
let mut result = T::run(&mut state, spec).map(|_| state);
compare_beacon_state_results_without_caches(&mut result, &mut expected)
}

4
testing/ef_tests/src/cases/fork_choice.rs
View File

@@ -24,7 +24,7 @@ use std::sync::Arc;
use std::time::Duration;
use types::{
Attestation, AttesterSlashing, BeaconBlock, BeaconState, BlobSidecar, BlobsList, Checkpoint,
ExecutionBlockHash, Hash256, IndexedAttestation, KzgProof, ProgressiveBalancesMode,
EthSpec, ExecutionBlockHash, ForkName, Hash256, IndexedAttestation, KzgProof,
ProposerPreparationData, SignedBeaconBlock, Slot, Uint256,
};
@@ -557,9 +557,7 @@ impl<E: EthSpec> Tester<E> {
block_delay,
&state,
PayloadVerificationStatus::Irrelevant,
ProgressiveBalancesMode::Strict,
&self.harness.chain.spec,
self.harness.logger(),
);
if result.is_ok() {

22
testing/ef_tests/src/cases/operations.rs
View File

@@ -5,6 +5,7 @@ use crate::decode::{ssz_decode_file, ssz_decode_file_with, ssz_decode_state, yam
use serde::Deserialize;
use ssz::Decode;
use state_processing::common::update_progressive_balances_cache::initialize_progressive_balances_cache;
use state_processing::epoch_cache::initialize_epoch_cache;
use state_processing::{
per_block_processing::{
errors::BlockProcessingError,
@@ -87,6 +88,7 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
spec: &ChainSpec,
_: &Operations<E, Self>,
) -> Result<(), BlockProcessingError> {
initialize_epoch_cache(state, spec)?;
let mut ctxt = ConsensusContext::new(state.slot());
match state {
BeaconState::Base(_) => base::process_attestations(
@@ -101,7 +103,7 @@ impl<E: EthSpec> Operation<E> for Attestation<E> {
| BeaconState::Capella(_)
| BeaconState::Deneb(_)
| BeaconState::Electra(_) => {
initialize_progressive_balances_cache(state, None, spec)?;
initialize_progressive_balances_cache(state, spec)?;
altair_deneb::process_attestation(
state,
self,
@@ -131,7 +133,7 @@ impl<E: EthSpec> Operation<E> for AttesterSlashing<E> {
_: &Operations<E, Self>,
) -> Result<(), BlockProcessingError> {
let mut ctxt = ConsensusContext::new(state.slot());
initialize_progressive_balances_cache(state, None, spec)?;
initialize_progressive_balances_cache(state, spec)?;
process_attester_slashings(
state,
&[self.clone()],
@@ -182,7 +184,7 @@ impl<E: EthSpec> Operation<E> for ProposerSlashing {
_: &Operations<E, Self>,
) -> Result<(), BlockProcessingError> {
let mut ctxt = ConsensusContext::new(state.slot());
initialize_progressive_balances_cache(state, None, spec)?;
initialize_progressive_balances_cache(state, spec)?;
process_proposer_slashings(
state,
&[self.clone()],
@@ -484,14 +486,22 @@ impl<E: EthSpec, O: Operation<E>> Case for Operations<E, O> {
fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
let spec = &testing_spec::<E>(fork_name);
let mut state = self.pre.clone();
let mut expected = self.post.clone();
let mut pre_state = self.pre.clone();
// Processing requires the committee caches.
// NOTE: some of the withdrawals tests have 0 active validators, do not try
// to build the commitee cache in this case.
if O::handler_name() != "withdrawals" {
state.build_all_committee_caches(spec).unwrap();
pre_state.build_all_committee_caches(spec).unwrap();
}
let mut state = pre_state.clone();
let mut expected = self.post.clone();
if O::handler_name() != "withdrawals" {
if let Some(post_state) = expected.as_mut() {
post_state.build_all_committee_caches(spec).unwrap();
}
}
let mut result = self

148
testing/ef_tests/src/cases/rewards.rs
View File

@@ -7,16 +7,14 @@ use ssz::four_byte_option_impl;
use ssz_derive::{Decode, Encode};
use state_processing::{
per_epoch_processing::{
altair::{self, rewards_and_penalties::get_flag_index_deltas, ParticipationCache},
altair,
base::{self, rewards_and_penalties::AttestationDelta, ValidatorStatuses},
Delta,
},
EpochProcessingError,
};
use types::{
consts::altair::{TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX},
BeaconState,
};
use std::path::{Path, PathBuf};
use types::{BeaconState, EthSpec, ForkName};
#[derive(Debug, Clone, PartialEq, Decode, Encode, CompareFields)]
pub struct Deltas {
@@ -40,6 +38,11 @@ pub struct AllDeltas {
inactivity_penalty_deltas: Deltas,
}
#[derive(Debug, Clone, PartialEq, CompareFields)]
pub struct TotalDeltas {
deltas: Vec<i64>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct Metadata {
pub description: Option<String>,
@@ -109,11 +112,19 @@ impl<E: EthSpec> Case for RewardsTest<E> {
let mut state = self.pre.clone();
let spec = &testing_spec::<E>(fork_name);
let deltas: Result<AllDeltas, EpochProcessingError> = (|| {
// Processing requires the committee caches.
state.build_all_committee_caches(spec)?;
// Single-pass epoch processing doesn't compute rewards in the genesis epoch because that's
// what the spec for `process_rewards_and_penalties` says to do. We skip these tests for now.
//
// See: https://github.com/ethereum/consensus-specs/issues/3593
if fork_name != ForkName::Base && state.current_epoch() == 0 {
return Err(Error::SkippedKnownFailure);
}
if let BeaconState::Base(_) = state {
let deltas: Result<AllDeltas, EpochProcessingError> = (|| {
// Processing requires the committee caches.
state.build_all_committee_caches(spec)?;
if let BeaconState::Base(_) = state {
let mut validator_statuses = ValidatorStatuses::new(&state, spec)?;
validator_statuses.process_attestations(&state)?;
@@ -124,39 +135,19 @@ impl<E: EthSpec> Case for RewardsTest<E> {
)?;
Ok(convert_all_base_deltas(&deltas))
} else {
let total_active_balance = state.get_total_active_balance()?;
})();
compare_result_detailed(&deltas, &Some(self.deltas.clone()))?;
} else {
let deltas: Result<TotalDeltas, EpochProcessingError> = (|| {
// Processing requires the committee caches.
state.build_all_committee_caches(spec)?;
compute_altair_deltas(&mut state, spec)
})();
let source_deltas = compute_altair_flag_deltas(
&state,
TIMELY_SOURCE_FLAG_INDEX,
total_active_balance,
spec,
)?;
let target_deltas = compute_altair_flag_deltas(
&state,
TIMELY_TARGET_FLAG_INDEX,
total_active_balance,
spec,
)?;
let head_deltas = compute_altair_flag_deltas(
&state,
TIMELY_HEAD_FLAG_INDEX,
total_active_balance,
spec,
)?;
let inactivity_penalty_deltas = compute_altair_inactivity_deltas(&state, spec)?;
Ok(AllDeltas {
source_deltas,
target_deltas,
head_deltas,
inclusion_delay_deltas: None,
inactivity_penalty_deltas,
})
}
})();
let expected = all_deltas_to_total_deltas(&self.deltas);
compare_result_detailed(&deltas, &Some(self.deltas.clone()))?;
compare_result_detailed(&deltas, &Some(expected))?;
};
Ok(())
}
@@ -181,39 +172,54 @@ fn convert_base_deltas(attestation_deltas: &[AttestationDelta], accessor: Access
Deltas { rewards, penalties }
}
fn compute_altair_flag_deltas<E: EthSpec>(
state: &BeaconState<E>,
flag_index: usize,
total_active_balance: u64,
spec: &ChainSpec,
) -> Result<Deltas, EpochProcessingError> {
let mut deltas = vec![Delta::default(); state.validators().len()];
get_flag_index_deltas(
&mut deltas,
state,
flag_index,
total_active_balance,
&ParticipationCache::new(state, spec).unwrap(),
spec,
)?;
Ok(convert_altair_deltas(deltas))
fn deltas_to_total_deltas(d: &Deltas) -> impl Iterator<Item = i64> + '_ {
d.rewards
.iter()
.zip(&d.penalties)
.map(|(&reward, &penalty)| reward as i64 - penalty as i64)
}
fn compute_altair_inactivity_deltas<E: EthSpec>(
state: &BeaconState<E>,
spec: &ChainSpec,
) -> Result<Deltas, EpochProcessingError> {
let mut deltas = vec![Delta::default(); state.validators().len()];
altair::rewards_and_penalties::get_inactivity_penalty_deltas(
&mut deltas,
state,
&ParticipationCache::new(state, spec).unwrap(),
spec,
)?;
Ok(convert_altair_deltas(deltas))
fn optional_deltas_to_total_deltas(d: &Option<Deltas>, len: usize) -> TotalDeltas {
let deltas = if let Some(d) = d {
deltas_to_total_deltas(d).collect()
} else {
vec![0i64; len]
};
TotalDeltas { deltas }
}
fn convert_altair_deltas(deltas: Vec<Delta>) -> Deltas {
let (rewards, penalties) = deltas.into_iter().map(|d| (d.rewards, d.penalties)).unzip();
Deltas { rewards, penalties }
fn all_deltas_to_total_deltas(d: &AllDeltas) -> TotalDeltas {
let len = d.source_deltas.rewards.len();
let deltas = deltas_to_total_deltas(&d.source_deltas)
.zip(deltas_to_total_deltas(&d.target_deltas))
.zip(deltas_to_total_deltas(&d.head_deltas))
.zip(optional_deltas_to_total_deltas(&d.inclusion_delay_deltas, len).deltas)
.zip(deltas_to_total_deltas(&d.inactivity_penalty_deltas))
.map(
|((((source, target), head), inclusion_delay), inactivity_penalty)| {
source + target + head + inclusion_delay + inactivity_penalty
},
)
.collect::<Vec<i64>>();
TotalDeltas { deltas }
}
fn compute_altair_deltas<E: EthSpec>(
state: &mut BeaconState<E>,
spec: &ChainSpec,
) -> Result<TotalDeltas, EpochProcessingError> {
// Initialise deltas to pre-state balances.
let mut deltas = state
.balances()
.iter()
.map(|x| *x as i64)
.collect::<Vec<_>>();
altair::process_rewards_and_penalties_slow(state, spec)?;
for (delta, new_balance) in deltas.iter_mut().zip(state.balances()) {
let old_balance = *delta;
*delta = *new_balance as i64 - old_balance;
}
Ok(TotalDeltas { deltas })
}