From dbe474e1327b01579af79093fc2b53d85c347540 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 6 Jan 2026 14:08:02 +1100 Subject: [PATCH 01/18] Delete attester cache (#8469) Fixes attester cache write lock contention. Alternative to #8463. Co-Authored-By: Jimmy Chen --- .../beacon_chain/src/attester_cache.rs | 385 ------------------ beacon_node/beacon_chain/src/beacon_chain.rs | 83 ++-- beacon_node/beacon_chain/src/builder.rs | 11 - .../beacon_chain/src/canonical_head.rs | 3 - .../beacon_chain/src/early_attester_cache.rs | 78 +++- beacon_node/beacon_chain/src/errors.rs | 4 +- beacon_node/beacon_chain/src/lib.rs | 1 - beacon_node/beacon_chain/src/metrics.rs | 14 - .../beacon_chain/src/state_advance_timer.rs | 6 - .../tests/attestation_production.rs | 9 +- beacon_node/lighthouse_tracing/src/lib.rs | 3 + .../state_processing/src/state_advance.rs | 2 + 12 files changed, 106 insertions(+), 493 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/attester_cache.rs diff --git a/beacon_node/beacon_chain/src/attester_cache.rs b/beacon_node/beacon_chain/src/attester_cache.rs deleted file mode 100644 index 26a3389812..0000000000 --- a/beacon_node/beacon_chain/src/attester_cache.rs +++ /dev/null @@ -1,385 +0,0 @@ -//! This module provides the `AttesterCache`, a cache designed for reducing state-reads when -//! validators produce `AttestationData`. -//! -//! This cache is required *as well as* the `ShufflingCache` since the `ShufflingCache` does not -//! provide any information about the `state.current_justified_checkpoint`. It is not trivial to add -//! the justified checkpoint to the `ShufflingCache` since that cache is keyed by shuffling decision -//! root, which is not suitable for the justified checkpoint. Whilst we can know the shuffling for -//! epoch `n` during `n - 1`, we *cannot* know the justified checkpoint. Instead, we *must* perform -//! `per_epoch_processing` to transform the state from epoch `n - 1` to epoch `n` so that rewards -//! and penalties can be computed and the `state.current_justified_checkpoint` can be updated. - -use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; -use fixed_bytes::FixedBytesExtended; -use parking_lot::RwLock; -use state_processing::state_advance::{Error as StateAdvanceError, partial_state_advance}; -use std::collections::HashMap; -use std::ops::Range; -use types::{ - BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, RelativeEpoch, - Slot, - attestation::AttestationError, - beacon_state::{ - compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count, - }, -}; - -type JustifiedCheckpoint = Checkpoint; -type CommitteeLength = usize; -type CommitteeIndex = u64; -type CacheHashMap = HashMap; - -/// The maximum number of `AttesterCacheValues` to be kept in memory. -/// -/// Each `AttesterCacheValues` is very small (~16 bytes) and the cache will generally be kept small -/// by pruning on finality. -/// -/// The value provided here is much larger than will be used during ideal network conditions, -/// however we make it large since the values are so small. -const MAX_CACHE_LEN: usize = 1_024; - -#[derive(Debug)] -pub enum Error { - BeaconState(BeaconStateError), - // Boxed to avoid an infinite-size recursion issue. - BeaconChain(Box), - MissingBeaconState(Hash256), - FailedToTransitionState(StateAdvanceError), - CannotAttestToFutureState { - state_slot: Slot, - request_slot: Slot, - }, - /// Indicates a cache inconsistency. - WrongEpoch { - request_epoch: Epoch, - epoch: Epoch, - }, - InvalidCommitteeIndex { - committee_index: u64, - }, - /// Indicates an inconsistency with the beacon state committees. - InverseRange { - range: Range, - }, - AttestationError(AttestationError), -} - -impl From for Error { - fn from(e: BeaconStateError) -> Self { - Error::BeaconState(e) - } -} - -impl From for Error { - fn from(e: BeaconChainError) -> Self { - Error::BeaconChain(Box::new(e)) - } -} - -/// Stores the minimal amount of data required to compute the committee length for any committee at any -/// slot in a given `epoch`. -pub struct CommitteeLengths { - /// The `epoch` to which the lengths pertain. - epoch: Epoch, - /// The length of the shuffling in `self.epoch`. - active_validator_indices_len: usize, -} - -impl CommitteeLengths { - /// Instantiate `Self` using `state.current_epoch()`. - pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { - let active_validator_indices_len = if let Ok(committee_cache) = - state.committee_cache(RelativeEpoch::Current) - { - committee_cache.active_validator_indices().len() - } else { - // Building the cache like this avoids taking a mutable reference to `BeaconState`. - let committee_cache = state.initialize_committee_cache(state.current_epoch(), spec)?; - committee_cache.active_validator_indices().len() - }; - - Ok(Self { - epoch: state.current_epoch(), - active_validator_indices_len, - }) - } - - /// Get the count of committees per each slot of `self.epoch`. - pub fn get_committee_count_per_slot( - &self, - spec: &ChainSpec, - ) -> Result { - E::get_committee_count_per_slot(self.active_validator_indices_len, spec).map_err(Into::into) - } - - /// Get the length of the committee at the given `slot` and `committee_index`. - pub fn get_committee_length( - &self, - slot: Slot, - committee_index: CommitteeIndex, - spec: &ChainSpec, - ) -> Result { - let slots_per_epoch = E::slots_per_epoch(); - let request_epoch = slot.epoch(slots_per_epoch); - - // Sanity check. - if request_epoch != self.epoch { - return Err(Error::WrongEpoch { - request_epoch, - epoch: self.epoch, - }); - } - - let slots_per_epoch = slots_per_epoch as usize; - let committees_per_slot = self.get_committee_count_per_slot::(spec)?; - let index_in_epoch = compute_committee_index_in_epoch( - slot, - slots_per_epoch, - committees_per_slot, - committee_index as usize, - ); - let range = compute_committee_range_in_epoch( - epoch_committee_count(committees_per_slot, slots_per_epoch), - index_in_epoch, - self.active_validator_indices_len, - ) - .ok_or(Error::InvalidCommitteeIndex { committee_index })?; - - range - .end - .checked_sub(range.start) - .ok_or(Error::InverseRange { range }) - } -} - -/// Provides the following information for some epoch: -/// -/// - The `state.current_justified_checkpoint` value. -/// - The committee lengths for all indices and slots. -/// -/// These values are used during attestation production. -pub struct AttesterCacheValue { - current_justified_checkpoint: Checkpoint, - committee_lengths: CommitteeLengths, -} - -impl AttesterCacheValue { - /// Instantiate `Self` using `state.current_epoch()`. - pub fn new(state: &BeaconState, spec: &ChainSpec) -> Result { - let current_justified_checkpoint = state.current_justified_checkpoint(); - let committee_lengths = CommitteeLengths::new(state, spec)?; - Ok(Self { - current_justified_checkpoint, - committee_lengths, - }) - } - - /// Get the justified checkpoint and committee length for some `slot` and `committee_index`. - fn get( - &self, - slot: Slot, - committee_index: CommitteeIndex, - spec: &ChainSpec, - ) -> Result<(JustifiedCheckpoint, CommitteeLength), Error> { - self.committee_lengths - .get_committee_length::(slot, committee_index, spec) - .map(|committee_length| (self.current_justified_checkpoint, committee_length)) - } -} - -/// The `AttesterCacheKey` is fundamentally the same thing as the proposer shuffling decision root, -/// however here we use it as an identity for both of the following values: -/// -/// 1. The `state.current_justified_checkpoint`. -/// 2. The attester shuffling. -/// -/// This struct relies upon the premise that the `state.current_justified_checkpoint` in epoch `n` -/// is determined by the root of the latest block in epoch `n - 1`. Notably, this is identical to -/// how the proposer shuffling is keyed in `BeaconProposerCache`. -/// -/// It is also safe, but not maximally efficient, to key the attester shuffling with the same -/// strategy. For better shuffling keying strategies, see the `ShufflingCache`. -#[derive(Eq, PartialEq, Hash, Clone, Copy)] -pub struct AttesterCacheKey { - /// The epoch from which the justified checkpoint should be observed. - /// - /// Attestations which use `self.epoch` as `target.epoch` should use this key. - epoch: Epoch, - /// The root of the block at the last slot of `self.epoch - 1`. - decision_root: Hash256, -} - -impl AttesterCacheKey { - /// Instantiate `Self` to key `state.current_epoch()`. - /// - /// The `latest_block_root` should be the latest block that has been applied to `state`. This - /// parameter is required since the state does not store the block root for any block with the - /// same slot as `state.slot()`. - /// - /// ## Errors - /// - /// May error if `epoch` is out of the range of `state.block_roots`. - pub fn new( - epoch: Epoch, - state: &BeaconState, - latest_block_root: Hash256, - ) -> Result { - let slots_per_epoch = E::slots_per_epoch(); - let decision_slot = epoch.start_slot(slots_per_epoch).saturating_sub(1_u64); - - let decision_root = if decision_slot.epoch(slots_per_epoch) == epoch { - // This scenario is only possible during the genesis epoch. In this scenario, all-zeros - // is used as an alias to the genesis block. - Hash256::zero() - } else if epoch > state.current_epoch() { - // If the requested epoch is higher than the current epoch, the latest block will always - // be the decision root. - latest_block_root - } else { - *state.get_block_root(decision_slot)? - }; - - Ok(Self { - epoch, - decision_root, - }) - } -} - -/// Provides a cache for the justified checkpoint and committee length when producing an -/// attestation. -/// -/// See the module-level documentation for more information. -#[derive(Default)] -pub struct AttesterCache { - cache: RwLock, -} - -impl AttesterCache { - /// Get the justified checkpoint and committee length for the `slot` and `committee_index` in - /// the state identified by the cache `key`. - pub fn get( - &self, - key: &AttesterCacheKey, - slot: Slot, - committee_index: CommitteeIndex, - spec: &ChainSpec, - ) -> Result, Error> { - self.cache - .read() - .get(key) - .map(|cache_item| cache_item.get::(slot, committee_index, spec)) - .transpose() - } - - /// Cache the `state.current_epoch()` values if they are not already present in the state. - pub fn maybe_cache_state( - &self, - state: &BeaconState, - latest_block_root: Hash256, - spec: &ChainSpec, - ) -> Result<(), Error> { - let key = AttesterCacheKey::new(state.current_epoch(), state, latest_block_root)?; - let mut cache = self.cache.write(); - if !cache.contains_key(&key) { - let cache_item = AttesterCacheValue::new(state, spec)?; - Self::insert_respecting_max_len(&mut cache, key, cache_item); - } - Ok(()) - } - - /// Read the state identified by `state_root` from the database, advance it to the required - /// slot, use it to prime the cache and return the values for the provided `slot` and - /// `committee_index`. - /// - /// ## Notes - /// - /// This function takes a write-lock on the internal cache. Prefer attempting a `Self::get` call - /// before running this function as `Self::get` only takes a read-lock and is therefore less - /// likely to create contention. - pub fn load_and_cache_state( - &self, - state_root: Hash256, - key: AttesterCacheKey, - slot: Slot, - committee_index: CommitteeIndex, - chain: &BeaconChain, - ) -> Result<(JustifiedCheckpoint, CommitteeLength), Error> { - let spec = &chain.spec; - let slots_per_epoch = T::EthSpec::slots_per_epoch(); - let epoch = slot.epoch(slots_per_epoch); - - // Take a write-lock on the cache before starting the state read. - // - // Whilst holding the write-lock during the state read will create contention, it prevents - // the scenario where multiple requests from separate threads cause duplicate state reads. - let mut cache = self.cache.write(); - - // Try the cache to see if someone has already primed it between the time the function was - // called and when the cache write-lock was obtained. This avoids performing duplicate state - // reads. - if let Some(value) = cache - .get(&key) - .map(|cache_item| cache_item.get::(slot, committee_index, spec)) - .transpose()? - { - return Ok(value); - } - - // We use `cache_state = true` here because if we are attesting to the state it's likely - // to be recent and useful for other things. - let mut state: BeaconState = chain - .get_state(&state_root, None, true)? - .ok_or(Error::MissingBeaconState(state_root))?; - - if state.slot() > slot { - // This indicates an internal inconsistency. - return Err(Error::CannotAttestToFutureState { - state_slot: state.slot(), - request_slot: slot, - }); - } else if state.current_epoch() < epoch { - // Only perform a "partial" state advance since we do not require the state roots to be - // accurate. - partial_state_advance( - &mut state, - Some(state_root), - epoch.start_slot(slots_per_epoch), - spec, - ) - .map_err(Error::FailedToTransitionState)?; - state.build_committee_cache(RelativeEpoch::Current, spec)?; - } - - let cache_item = AttesterCacheValue::new(&state, spec)?; - let value = cache_item.get::(slot, committee_index, spec)?; - Self::insert_respecting_max_len(&mut cache, key, cache_item); - Ok(value) - } - - /// Insert a value to `cache`, ensuring it does not exceed the maximum length. - /// - /// If the cache is already full, the item with the lowest epoch will be removed. - fn insert_respecting_max_len( - cache: &mut CacheHashMap, - key: AttesterCacheKey, - value: AttesterCacheValue, - ) { - while cache.len() >= MAX_CACHE_LEN { - if let Some(oldest) = cache.keys().copied().min_by_key(|key| key.epoch) { - cache.remove(&oldest); - } else { - break; - } - } - - cache.insert(key, value); - } - - /// Remove all entries where the `key.epoch` is lower than the given `epoch`. - /// - /// Generally, the provided `epoch` should be the finalized epoch. - pub fn prune_below(&self, epoch: Epoch) { - self.cache.write().retain(|target, _| target.epoch >= epoch); - } -} diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 46ba14f596..a6be3dde8c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -3,7 +3,6 @@ use crate::attestation_verification::{ VerifiedUnaggregatedAttestation, batch_verify_aggregated_attestations, batch_verify_unaggregated_attestations, }; -use crate::attester_cache::{AttesterCache, AttesterCacheKey}; use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::beacon_proposer_cache::{ BeaconProposerCache, EpochBlockProposers, ensure_state_can_determine_proposers_for_epoch, @@ -92,6 +91,7 @@ use futures::channel::mpsc::Sender; use itertools::Itertools; use itertools::process_results; use kzg::Kzg; +use lighthouse_tracing::SPAN_PRODUCE_UNAGGREGATED_ATTESTATION; use logging::crit; use operation_pool::{ CompactAttestationRef, OperationPool, PersistedOperationPool, ReceivedPreCapella, @@ -455,8 +455,6 @@ pub struct BeaconChain { pub beacon_proposer_cache: Arc>, /// Caches a map of `validator_index -> validator_pubkey`. pub(crate) validator_pubkey_cache: RwLock>, - /// A cache used when producing attestations. - pub(crate) attester_cache: Arc, /// A cache used when producing attestations whilst the head block is still being imported. pub early_attester_cache: EarlyAttesterCache, /// A cache used to keep track of various block timings. @@ -1846,6 +1844,7 @@ impl BeaconChain { /// ## Errors /// /// May return an error if the `request_slot` is too far behind the head state. + #[instrument(name = SPAN_PRODUCE_UNAGGREGATED_ATTESTATION, skip_all, fields(%request_slot, %request_index), level = "debug")] pub fn produce_unaggregated_attestation( &self, request_slot: Slot, @@ -1889,19 +1888,17 @@ impl BeaconChain { * the head-lock is not desirable. */ - let head_state_slot; let beacon_block_root; let beacon_state_root; let target; let current_epoch_attesting_info: Option<(Checkpoint, usize)>; - let attester_cache_key; let head_timer = metrics::start_timer(&metrics::ATTESTATION_PRODUCTION_HEAD_SCRAPE_SECONDS); + let head_span = debug_span!("attestation_production_head_scrape").entered(); // The following braces are to prevent the `cached_head` Arc from being held for longer than // required. It also helps reduce the diff for a very large PR (#3244). { let head = self.head_snapshot(); let head_state = &head.beacon_state; - head_state_slot = head_state.slot(); // There is no value in producing an attestation to a block that is pre-finalization and // it is likely to cause expensive and pointless reads to the freezer database. Exit @@ -1969,12 +1966,8 @@ impl BeaconChain { // to determine the justified checkpoint and committee length. None }; - - // Determine the key for `self.attester_cache`, in case it is required later in this - // routine. - attester_cache_key = - AttesterCacheKey::new(request_epoch, head_state, beacon_block_root)?; } + drop(head_span); drop(head_timer); // Only attest to a block if it is fully verified (i.e. not optimistic or invalid). @@ -1997,47 +1990,38 @@ impl BeaconChain { * Phase 2/2: * * If the justified checkpoint and committee length from the head are suitable for this - * attestation, use them. If not, try the attester cache. If the cache misses, load a state - * from disk and prime the cache with it. + * attestation, use them. If not, use the database, which will hit the state cache. */ - - let cache_timer = - metrics::start_timer(&metrics::ATTESTATION_PRODUCTION_CACHE_INTERACTION_SECONDS); let (justified_checkpoint, committee_len) = if let Some((justified_checkpoint, committee_len)) = current_epoch_attesting_info { // The head state is in the same epoch as the attestation, so there is no more // required information. (justified_checkpoint, committee_len) - } else if let Some(cached_values) = self.attester_cache.get::( - &attester_cache_key, - request_slot, - request_index, - &self.spec, - )? { - // The suitable values were already cached. Return them. - cached_values } else { - debug!( - ?beacon_block_root, - %head_state_slot, - %request_slot, - "Attester cache miss" - ); + let (advanced_state_root, mut state) = self + .store + .get_advanced_hot_state(beacon_block_root, request_slot, beacon_state_root)? + .ok_or(Error::MissingBeaconState(beacon_state_root))?; + if state.current_epoch() < request_epoch { + partial_state_advance( + &mut state, + Some(advanced_state_root), + request_epoch.start_slot(T::EthSpec::slots_per_epoch()), + &self.spec, + ) + .map_err(Error::StateAdvanceError)?; - // Neither the head state, nor the attester cache was able to produce the required - // information to attest in this epoch. So, load a `BeaconState` from disk and use - // it to fulfil the request (and prime the cache to avoid this next time). - let _cache_build_timer = - metrics::start_timer(&metrics::ATTESTATION_PRODUCTION_CACHE_PRIME_SECONDS); - self.attester_cache.load_and_cache_state( - beacon_state_root, - attester_cache_key, - request_slot, - request_index, - self, - )? + state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + } + + ( + state.current_justified_checkpoint(), + state + .get_beacon_committee(request_slot, request_index)? + .committee + .len(), + ) }; - drop(cache_timer); Ok(Attestation::::empty_for_signing( request_index, @@ -3844,18 +3828,6 @@ impl BeaconChain { } }; - // Apply the state to the attester cache, only if it is from the previous epoch or later. - // - // In a perfect scenario there should be no need to add previous-epoch states to the cache. - // However, latency between the VC and the BN might cause the VC to produce attestations at - // a previous slot. - if state.current_epoch().saturating_add(1_u64) >= current_epoch { - let _attester_span = debug_span!("attester_cache_update").entered(); - self.attester_cache - .maybe_cache_state(&state, block_root, &self.spec) - .map_err(BeaconChainError::from)?; - } - // Take an upgradable read lock on fork choice so we can check if this block has already // been imported. We don't want to repeat work importing a block that is already imported. let fork_choice_reader = self.canonical_head.fork_choice_upgradable_read_lock(); @@ -3916,7 +3888,6 @@ impl BeaconChain { &signed_block, proto_block, &state, - &self.spec, ) { warn!( error = ?e, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 58dbf1c35e..b8770b8911 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1029,7 +1029,6 @@ where block_times_cache: <_>::default(), pre_finalization_block_cache: <_>::default(), validator_pubkey_cache: RwLock::new(validator_pubkey_cache), - attester_cache: <_>::default(), early_attester_cache: <_>::default(), light_client_server_cache: LightClientServerCache::new(), light_client_server_tx: self.light_client_server_tx, @@ -1061,16 +1060,6 @@ where let head = beacon_chain.head_snapshot(); - // Prime the attester cache with the head state. - beacon_chain - .attester_cache - .maybe_cache_state( - &head.beacon_state, - head.beacon_block_root, - &beacon_chain.spec, - ) - .map_err(|e| format!("Failed to prime attester cache: {:?}", e))?; - // Only perform the check if it was configured. if let Some(wss_checkpoint) = beacon_chain.config.weak_subjectivity_checkpoint && let Err(e) = beacon_chain.verify_weak_subjectivity_checkpoint( diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 7dd4c88c51..76c08c5e39 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -958,9 +958,6 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()), ); - self.attester_cache - .prune_below(new_view.finalized_checkpoint.epoch); - if let Some(event_handler) = self.event_handler.as_ref() && event_handler.has_finalized_subscribers() { diff --git a/beacon_node/beacon_chain/src/early_attester_cache.rs b/beacon_node/beacon_chain/src/early_attester_cache.rs index 5665ef3775..8b26cb2f58 100644 --- a/beacon_node/beacon_chain/src/early_attester_cache.rs +++ b/beacon_node/beacon_chain/src/early_attester_cache.rs @@ -1,13 +1,79 @@ use crate::data_availability_checker::{AvailableBlock, AvailableBlockData}; -use crate::{ - attester_cache::{CommitteeLengths, Error}, - metrics, -}; +use crate::{BeaconChainError as Error, metrics}; use parking_lot::RwLock; use proto_array::Block as ProtoBlock; use std::sync::Arc; +use tracing::instrument; use types::*; +/// Stores the minimal amount of data required to compute the committee length for any committee at any +/// slot in a given `epoch`. +pub struct CommitteeLengths { + /// The `epoch` to which the lengths pertain. + epoch: Epoch, + /// The length of the shuffling in `self.epoch`. + active_validator_indices_len: usize, +} + +impl CommitteeLengths { + /// Instantiate `Self` using `state.current_epoch()`. + pub fn new(state: &BeaconState) -> Result { + let active_validator_indices_len = state + .committee_cache(RelativeEpoch::Current)? + .active_validator_indices() + .len(); + + Ok(Self { + epoch: state.current_epoch(), + active_validator_indices_len, + }) + } + + /// Get the count of committees per each slot of `self.epoch`. + pub fn get_committee_count_per_slot( + &self, + spec: &ChainSpec, + ) -> Result { + E::get_committee_count_per_slot(self.active_validator_indices_len, spec).map_err(Into::into) + } + + /// Get the length of the committee at the given `slot` and `committee_index`. + pub fn get_committee_length( + &self, + slot: Slot, + committee_index: CommitteeIndex, + spec: &ChainSpec, + ) -> Result { + let slots_per_epoch = E::slots_per_epoch(); + let request_epoch = slot.epoch(slots_per_epoch); + + // Sanity check. + if request_epoch != self.epoch { + return Err(Error::EarlyAttesterCacheError); + } + + let slots_per_epoch = slots_per_epoch as usize; + let committees_per_slot = self.get_committee_count_per_slot::(spec)?; + let index_in_epoch = compute_committee_index_in_epoch( + slot, + slots_per_epoch, + committees_per_slot, + committee_index as usize, + ); + let range = compute_committee_range_in_epoch( + epoch_committee_count(committees_per_slot, slots_per_epoch), + index_in_epoch, + self.active_validator_indices_len, + ) + .ok_or(Error::EarlyAttesterCacheError)?; + + range + .end + .checked_sub(range.start) + .ok_or(Error::EarlyAttesterCacheError) + } +} + pub struct CacheItem { /* * Values used to create attestations. @@ -55,10 +121,9 @@ impl EarlyAttesterCache { block: &AvailableBlock, proto_block: ProtoBlock, state: &BeaconState, - spec: &ChainSpec, ) -> Result<(), Error> { let epoch = state.current_epoch(); - let committee_lengths = CommitteeLengths::new(state, spec)?; + let committee_lengths = CommitteeLengths::new(state)?; let source = state.current_justified_checkpoint(); let target_slot = epoch.start_slot(E::slots_per_epoch()); let target = Checkpoint { @@ -98,6 +163,7 @@ impl EarlyAttesterCache { /// - There is a cache `item` present. /// - If `request_slot` is in the same epoch as `item.epoch`. /// - If `request_index` does not exceed `item.committee_count`. + #[instrument(skip_all, fields(%request_slot, %request_index), level = "debug")] pub fn try_attest( &self, request_slot: Slot, diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index b021df2c33..7fa6a0b510 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -1,4 +1,3 @@ -use crate::attester_cache::Error as AttesterCacheError; use crate::beacon_block_streamer::Error as BlockStreamerError; use crate::beacon_chain::ForkChoiceError; use crate::beacon_fork_choice_store::Error as ForkChoiceStoreError; @@ -99,7 +98,7 @@ pub enum BeaconChainError { ObservedAttestersError(ObservedAttestersError), ObservedBlockProducersError(ObservedBlockProducersError), ObservedDataSidecarsError(ObservedDataSidecarsError), - AttesterCacheError(AttesterCacheError), + EarlyAttesterCacheError, PruningError(PruningError), ArithError(ArithError), InvalidShufflingId { @@ -266,7 +265,6 @@ easy_from_to!(ObservedAttestationsError, BeaconChainError); easy_from_to!(ObservedAttestersError, BeaconChainError); easy_from_to!(ObservedBlockProducersError, BeaconChainError); easy_from_to!(ObservedDataSidecarsError, BeaconChainError); -easy_from_to!(AttesterCacheError, BeaconChainError); easy_from_to!(BlockSignatureVerifierError, BeaconChainError); easy_from_to!(PruningError, BeaconChainError); easy_from_to!(ArithError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 4ac3e54742..f92030a671 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -1,7 +1,6 @@ pub mod attestation_rewards; pub mod attestation_simulator; pub mod attestation_verification; -mod attester_cache; pub mod beacon_block_reward; mod beacon_block_streamer; mod beacon_chain; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index e6557c7a27..cf295d9951 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -482,20 +482,6 @@ pub static ATTESTATION_PRODUCTION_HEAD_SCRAPE_SECONDS: LazyLock> = - LazyLock::new(|| { - try_create_histogram( - "attestation_production_cache_interaction_seconds", - "Time spent interacting with the attester cache", - ) - }); -pub static ATTESTATION_PRODUCTION_CACHE_PRIME_SECONDS: LazyLock> = - LazyLock::new(|| { - try_create_histogram( - "attestation_production_cache_prime_seconds", - "Time spent loading a new state from the disk due to a cache miss", - ) - }); /* * Fork Choice diff --git a/beacon_node/beacon_chain/src/state_advance_timer.rs b/beacon_node/beacon_chain/src/state_advance_timer.rs index a070dc350b..cb916cb514 100644 --- a/beacon_node/beacon_chain/src/state_advance_timer.rs +++ b/beacon_node/beacon_chain/src/state_advance_timer.rs @@ -409,12 +409,6 @@ fn advance_head(beacon_chain: &Arc>) -> Resu ); } - // Apply the state to the attester cache, if the cache deems it interesting. - beacon_chain - .attester_cache - .maybe_cache_state(&state, head_block_root, &beacon_chain.spec) - .map_err(BeaconChainError::from)?; - let final_slot = state.slot(); // If we have moved into the next slot whilst processing the state then this function is going diff --git a/beacon_node/beacon_chain/tests/attestation_production.rs b/beacon_node/beacon_chain/tests/attestation_production.rs index 017c249d10..a57c20211a 100644 --- a/beacon_node/beacon_chain/tests/attestation_production.rs +++ b/beacon_node/beacon_chain/tests/attestation_production.rs @@ -239,13 +239,7 @@ async fn produces_attestations() { .unwrap(); chain .early_attester_cache - .add_head_block( - block_root, - &available_block, - proto_block, - &state, - &chain.spec, - ) + .add_head_block(block_root, &available_block, proto_block, &state) .unwrap(); chain .early_attester_cache @@ -312,7 +306,6 @@ async fn early_attester_cache_old_request() { &available_block, head_proto_block, &head.beacon_state, - &harness.chain.spec, ) .unwrap(); diff --git a/beacon_node/lighthouse_tracing/src/lib.rs b/beacon_node/lighthouse_tracing/src/lib.rs index 56dccadaa9..094ad042fc 100644 --- a/beacon_node/lighthouse_tracing/src/lib.rs +++ b/beacon_node/lighthouse_tracing/src/lib.rs @@ -8,6 +8,9 @@ pub const SPAN_PRODUCE_BLOCK_V2: &str = "produce_block_v2"; pub const SPAN_PRODUCE_BLOCK_V3: &str = "produce_block_v3"; pub const SPAN_PUBLISH_BLOCK: &str = "publish_block"; +/// Root span names for attestation production +pub const SPAN_PRODUCE_UNAGGREGATED_ATTESTATION: &str = "produce_unaggregated_attestation"; + /// Data Availability checker span identifiers pub const SPAN_PENDING_COMPONENTS: &str = "pending_components"; diff --git a/consensus/state_processing/src/state_advance.rs b/consensus/state_processing/src/state_advance.rs index 19b21dad19..11a956bc2a 100644 --- a/consensus/state_processing/src/state_advance.rs +++ b/consensus/state_processing/src/state_advance.rs @@ -6,6 +6,7 @@ use crate::*; use fixed_bytes::FixedBytesExtended; +use tracing::instrument; use types::{BeaconState, ChainSpec, EthSpec, Hash256, Slot}; #[derive(Debug, PartialEq)] @@ -59,6 +60,7 @@ pub fn complete_state_advance( /// /// - If `state.slot > target_slot`, an error will be returned. /// - If `state_root_opt.is_none()` but the latest block header requires a state root. +#[instrument(skip_all, level = "debug")] pub fn partial_state_advance( state: &mut BeaconState, state_root_opt: Option, From 2fe59405bc261bb4bef363a9baa3785d66275b23 Mon Sep 17 00:00:00 2001 From: Shane K Moore <41407272+shane-moore@users.noreply.github.com> Date: Mon, 5 Jan 2026 19:28:44 -0800 Subject: [PATCH 02/18] Gloas add off protocol payment field to bid (#8596) Co-Authored-By: shane-moore --- consensus/state_processing/src/upgrade/gloas.rs | 5 ++++- consensus/types/src/execution/execution_payload_bid.rs | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/consensus/state_processing/src/upgrade/gloas.rs b/consensus/state_processing/src/upgrade/gloas.rs index d6c353cc2a..81c0fcfe63 100644 --- a/consensus/state_processing/src/upgrade/gloas.rs +++ b/consensus/state_processing/src/upgrade/gloas.rs @@ -70,7 +70,10 @@ pub fn upgrade_state_to_gloas( current_sync_committee: pre.current_sync_committee.clone(), next_sync_committee: pre.next_sync_committee.clone(), // Execution Bid - latest_execution_payload_bid: ExecutionPayloadBid::default(), + latest_execution_payload_bid: ExecutionPayloadBid { + block_hash: pre.latest_execution_payload_header.block_hash, + ..Default::default() + }, // Capella next_withdrawal_index: pre.next_withdrawal_index, next_withdrawal_validator_index: pre.next_withdrawal_validator_index, diff --git a/consensus/types/src/execution/execution_payload_bid.rs b/consensus/types/src/execution/execution_payload_bid.rs index 20e461334d..f0056463e9 100644 --- a/consensus/types/src/execution/execution_payload_bid.rs +++ b/consensus/types/src/execution/execution_payload_bid.rs @@ -18,6 +18,7 @@ pub struct ExecutionPayloadBid { pub parent_block_hash: ExecutionBlockHash, pub parent_block_root: Hash256, pub block_hash: ExecutionBlockHash, + pub prev_randao: Hash256, #[serde(with = "serde_utils::address_hex")] pub fee_recipient: Address, #[serde(with = "serde_utils::quoted_u64")] @@ -27,6 +28,8 @@ pub struct ExecutionPayloadBid { pub slot: Slot, #[serde(with = "serde_utils::quoted_u64")] pub value: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub execution_payment: u64, pub blob_kzg_commitments_root: Hash256, } From 3662e1ab7f782ada27a3f50ee70488dfa6259016 Mon Sep 17 00:00:00 2001 From: Mac L Date: Wed, 7 Jan 2026 23:38:31 +0400 Subject: [PATCH 03/18] Remove duplicated `crypto` dependencies (#8605) #8547 This unifies the following `crypto` dependencies to a single version each: - `sha2` - `hmac` - `pbkdf2` - `aes` - `cipher` - `ctr` - `scrypt` - `digest` Co-Authored-By: Mac L --- Cargo.lock | 275 +++++------------- Cargo.toml | 4 +- crypto/eth2_keystore/Cargo.toml | 10 +- .../src/json_keystore/kdf_module.rs | 5 +- crypto/eth2_keystore/src/keystore.rs | 17 +- crypto/eth2_wallet/Cargo.toml | 2 +- deny.toml | 6 +- 7 files changed, 105 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a054973ed..27f775b2d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -65,19 +65,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "aes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" -dependencies = [ - "cfg-if", - "cipher 0.3.0", - "cpufeatures", - "ctr 0.8.0", - "opaque-debug", -] - [[package]] name = "aes" version = "0.8.4" @@ -85,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures", ] @@ -96,9 +83,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", - "aes 0.8.4", - "cipher 0.4.4", - "ctr 0.9.2", + "aes", + "cipher", + "ctr", "ghash", "subtle", ] @@ -253,7 +240,7 @@ dependencies = [ "either", "serde", "serde_with", - "sha2 0.10.9", + "sha2", "thiserror 2.0.17", ] @@ -1284,7 +1271,7 @@ dependencies = [ "tree_hash_derive", "typenum", "types", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -1459,15 +1446,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -1637,26 +1615,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bzip2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" -dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "c-kzg" version = "2.1.5" @@ -1750,7 +1708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures", ] @@ -1762,7 +1720,7 @@ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", "chacha20", - "cipher 0.4.4", + "cipher", "poly1305", "zeroize", ] @@ -1808,15 +1766,6 @@ dependencies = [ "half", ] -[[package]] -name = "cipher" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" -dependencies = [ - "generic-array", -] - [[package]] name = "cipher" version = "0.4.4" @@ -2075,12 +2024,6 @@ dependencies = [ "unicode-xid", ] -[[package]] -name = "constant_time_eq" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" - [[package]] name = "context_deserialize" version = "0.2.0" @@ -2280,32 +2223,13 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" -dependencies = [ - "generic-array", - "subtle", -] - -[[package]] -name = "ctr" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" -dependencies = [ - "cipher 0.3.0", -] - [[package]] name = "ctr" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher 0.4.4", + "cipher", ] [[package]] @@ -2558,7 +2482,7 @@ dependencies = [ "hex", "reqwest", "serde_json", - "sha2 0.9.9", + "sha2", "tree_hash", "types", ] @@ -2668,7 +2592,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "const-oid", "crypto-common", "subtle", @@ -2709,11 +2633,11 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f170f4f6ed0e1df52bf43b403899f0081917ecf1500bfe312505cc3b515a8899" dependencies = [ - "aes 0.8.4", + "aes", "aes-gcm", "alloy-rlp", "arrayvec", - "ctr 0.9.2", + "ctr", "delay_map", "enr", "fnv", @@ -2837,7 +2761,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2 0.10.9", + "sha2", "subtle", "zeroize", ] @@ -2905,7 +2829,7 @@ dependencies = [ "itertools 0.14.0", "serde", "serde_json", - "sha2 0.10.9", + "sha2", ] [[package]] @@ -2970,7 +2894,7 @@ dependencies = [ "ekzg-bls12-381", "ekzg-maybe-rayon", "ekzg-polynomial", - "sha2 0.10.9", + "sha2", ] [[package]] @@ -3195,7 +3119,7 @@ dependencies = [ "hex", "num-bigint-dig", "ring", - "sha2 0.9.9", + "sha2", "zeroize", ] @@ -3203,18 +3127,20 @@ dependencies = [ name = "eth2_keystore" version = "0.1.0" dependencies = [ - "aes 0.7.5", + "aes", "bls", + "cipher", + "ctr", "eth2_key_derivation", "hex", - "hmac 0.11.0", - "pbkdf2 0.8.0", + "hmac", + "pbkdf2", "rand 0.9.2", "scrypt", "serde", "serde_json", "serde_repr", - "sha2 0.9.9", + "sha2", "tempfile", "unicode-normalization", "uuid 0.8.2", @@ -3235,7 +3161,7 @@ dependencies = [ "reqwest", "sensitive_url", "serde_yaml", - "sha2 0.9.9", + "sha2", "tempfile", "tokio", "tracing", @@ -3277,7 +3203,7 @@ checksum = "5aa93f58bb1eb3d1e556e4f408ef1dac130bad01ac37db4e7ade45de40d1c86a" dependencies = [ "cpufeatures", "ring", - "sha2 0.10.9", + "sha2", ] [[package]] @@ -3423,7 +3349,7 @@ dependencies = [ "sensitive_url", "serde", "serde_json", - "sha2 0.9.9", + "sha2", "slot_clock", "ssz_types", "state_processing", @@ -3567,6 +3493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", + "libz-rs-sys", "libz-sys", "miniz_oxide", ] @@ -4143,17 +4070,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac 0.12.1", -] - -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", + "hmac", ] [[package]] @@ -4856,7 +4773,7 @@ dependencies = [ "elliptic-curve", "once_cell", "serdect", - "sha2 0.10.9", + "sha2", "signature", ] @@ -5143,7 +5060,7 @@ dependencies = [ "quick-protobuf-codec", "rand 0.8.5", "regex", - "sha2 0.10.9", + "sha2", "tracing", "web-time", ] @@ -5183,7 +5100,7 @@ dependencies = [ "multihash", "quick-protobuf", "rand 0.8.5", - "sha2 0.10.9", + "sha2", "thiserror 2.0.17", "tracing", "zeroize", @@ -5423,6 +5340,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c" +dependencies = [ + "zlib-rs", +] + [[package]] name = "libz-sys" version = "1.1.23" @@ -5523,7 +5449,7 @@ dependencies = [ "rand 0.9.2", "regex", "serde", - "sha2 0.9.9", + "sha2", "smallvec", "snap", "ssz_types", @@ -6736,17 +6662,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "paste" version = "1.0.15" @@ -6755,23 +6670,12 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbkdf2" -version = "0.8.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" -dependencies = [ - "crypto-mac", -] - -[[package]] -name = "pbkdf2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", - "hmac 0.12.1", - "password-hash", - "sha2 0.10.9", + "hmac", ] [[package]] @@ -7586,7 +7490,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac 0.12.1", + "hmac", "subtle", ] @@ -7913,11 +7817,11 @@ checksum = "b147bb6111014916d3ef9d4c85173124a8e12193a67f6176d67244afd558d6c1" [[package]] name = "salsa20" -version = "0.8.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecbd2eb639fd7cab5804a0837fe373cc2172d15437e804c054a9fb885cb923b0" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" dependencies = [ - "cipher 0.3.0", + "cipher", ] [[package]] @@ -7985,14 +7889,13 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "scrypt" -version = "0.7.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879588d8f90906e73302547e20fffefdd240eb3e0e744e142321f5d49dea0518" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" dependencies = [ - "hmac 0.11.0", - "pbkdf2 0.8.0", + "pbkdf2", "salsa20", - "sha2 0.9.9", + "sha2", ] [[package]] @@ -8246,19 +8149,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.9" @@ -8505,7 +8395,7 @@ dependencies = [ "rand_core 0.6.4", "ring", "rustc_version 0.4.1", - "sha2 0.10.9", + "sha2", "subtle", ] @@ -8654,7 +8544,7 @@ dependencies = [ "typenum", "types", "xdelta3", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -9025,17 +8915,15 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "1.0.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62cc94d358b5a1e84a5cb9109f559aa3c4d634d2b1b4de3d0fa4adc7c78e2861" +checksum = "a30fd743a02bf35236f6faf99adb03089bb77e91c998dac2c2ad76bb424f668c" dependencies = [ - "anyhow", - "hmac 0.12.1", "once_cell", - "pbkdf2 0.11.0", + "pbkdf2", "rand 0.8.5", "rustc-hash 1.1.0", - "sha2 0.10.9", + "sha2", "thiserror 1.0.69", "unicode-normalization", "wasm-bindgen", @@ -10827,31 +10715,34 @@ dependencies = [ [[package]] name = "zip" -version = "0.6.6" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" dependencies = [ - "aes 0.8.4", - "byteorder", - "bzip2", - "constant_time_eq", + "arbitrary", "crc32fast", - "crossbeam-utils", "flate2", - "hmac 0.12.1", - "pbkdf2 0.11.0", - "sha1", - "time", - "zstd 0.11.2+zstd.1.5.2", + "indexmap 2.12.0", + "memchr", + "zopfli", ] [[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" +name = "zlib-rs" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", + "bumpalo", + "crc32fast", + "log", + "simd-adler32", ] [[package]] @@ -10860,17 +10751,7 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 7.2.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d5d1687c76..441490ee1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -225,7 +225,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_repr = "0.1" serde_yaml = "0.9" -sha2 = "0.9" +sha2 = "0.10" signing_method = { path = "validator_client/signing_method" } slasher = { path = "slasher", default-features = false } slashing_protection = { path = "validator_client/slashing_protection" } @@ -276,7 +276,7 @@ warp_utils = { path = "common/warp_utils" } workspace_members = { path = "common/workspace_members" } xdelta3 = { git = "https://github.com/sigp/xdelta3-rs", rev = "4db64086bb02e9febb584ba93b9d16bb2ae3825a" } zeroize = { version = "1", features = ["zeroize_derive", "serde"] } -zip = "0.6" +zip = { version = "6.0", default-features = false, features = ["deflate"] } zstd = "0.13" [profile.maxperf] diff --git a/crypto/eth2_keystore/Cargo.toml b/crypto/eth2_keystore/Cargo.toml index 290a10adc9..4fde662f5c 100644 --- a/crypto/eth2_keystore/Cargo.toml +++ b/crypto/eth2_keystore/Cargo.toml @@ -7,14 +7,16 @@ autotests = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -aes = { version = "0.7", features = ["ctr"] } +aes = "0.8" bls = { workspace = true } +cipher = "0.4" +ctr = "0.9" eth2_key_derivation = { workspace = true } hex = { workspace = true } -hmac = "0.11.0" -pbkdf2 = { version = "0.8.0", default-features = false } +hmac = "0.12.0" +pbkdf2 = { version = "0.12.0", default-features = false } rand = { workspace = true } -scrypt = { version = "0.7.0", default-features = false } +scrypt = { version = "0.11.0", default-features = false } serde = { workspace = true } serde_json = { workspace = true } serde_repr = { workspace = true } diff --git a/crypto/eth2_keystore/src/json_keystore/kdf_module.rs b/crypto/eth2_keystore/src/json_keystore/kdf_module.rs index 2086ac7b21..1702d8621c 100644 --- a/crypto/eth2_keystore/src/json_keystore/kdf_module.rs +++ b/crypto/eth2_keystore/src/json_keystore/kdf_module.rs @@ -5,9 +5,10 @@ use super::hex_bytes::HexBytes; use crate::DKLEN; -use hmac::{Hmac, Mac, NewMac}; +use hmac::{Hmac, Mac}; use serde::{Deserialize, Serialize}; use sha2::Sha256; +use sha2::digest::KeyInit; /// KDF module representation. #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] @@ -67,7 +68,7 @@ pub enum Prf { impl Prf { pub fn mac(&self, password: &[u8]) -> impl Mac { match &self { - Prf::HmacSha256 => Hmac::::new_from_slice(password) + Prf::HmacSha256 => as KeyInit>::new_from_slice(password) .expect("Could not derive HMAC using SHA256."), } } diff --git a/crypto/eth2_keystore/src/keystore.rs b/crypto/eth2_keystore/src/keystore.rs index b31e32eb4a..e6fc59d462 100644 --- a/crypto/eth2_keystore/src/keystore.rs +++ b/crypto/eth2_keystore/src/keystore.rs @@ -7,10 +7,11 @@ use crate::json_keystore::{ Aes128Ctr, ChecksumModule, Cipher, CipherModule, Crypto, EmptyMap, EmptyString, JsonKeystore, Kdf, KdfModule, Scrypt, Sha256Checksum, Version, }; -use aes::Aes128Ctr as AesCtr; -use aes::cipher::generic_array::GenericArray; -use aes::cipher::{NewCipher, StreamCipher}; +use aes::Aes128; use bls::{Keypair, PublicKey, SecretKey, ZeroizeHash}; +use cipher::generic_array::GenericArray; +use cipher::{KeyIvInit, StreamCipher}; +use ctr::Ctr64BE; use eth2_key_derivation::PlainText; use hmac::Hmac; use pbkdf2::pbkdf2; @@ -350,7 +351,7 @@ pub fn encrypt( // AES Encrypt let key = GenericArray::from_slice(&derived_key.as_bytes()[0..16]); let nonce = GenericArray::from_slice(params.iv.as_bytes()); - let mut cipher = AesCtr::new(key, nonce); + let mut cipher = Ctr64BE::::new(key, nonce); cipher.apply_keystream(&mut cipher_text); } }; @@ -396,7 +397,7 @@ pub fn decrypt(password: &[u8], crypto: &Crypto) -> Result { // AES Decrypt let key = GenericArray::from_slice(&derived_key.as_bytes()[0..16]); let nonce = GenericArray::from_slice(params.iv.as_bytes()); - let mut cipher = AesCtr::new(key, nonce); + let mut cipher = Ctr64BE::::new(key, nonce); cipher.apply_keystream(plain_text.as_mut_bytes()); } }; @@ -444,13 +445,15 @@ fn derive_key(password: &[u8], kdf: &Kdf) -> Result { params.salt.as_bytes(), params.c, dk.as_mut_bytes(), - ); + ) + // `pbkdf2` accepts keys of any size so this error should never occur in practice. + .map_err(|_| Error::InvalidPassword)?; } Kdf::Scrypt(params) => { scrypt( password, params.salt.as_bytes(), - &ScryptParams::new(log2_int(params.n) as u8, params.r, params.p) + &ScryptParams::new(log2_int(params.n) as u8, params.r, params.p, DKLEN as usize) .map_err(Error::ScryptInvalidParams)?, dk.as_mut_bytes(), ) diff --git a/crypto/eth2_wallet/Cargo.toml b/crypto/eth2_wallet/Cargo.toml index 0d454016a6..fcbcbe6ac8 100644 --- a/crypto/eth2_wallet/Cargo.toml +++ b/crypto/eth2_wallet/Cargo.toml @@ -13,7 +13,7 @@ rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_repr = { workspace = true } -tiny-bip39 = "1" +tiny-bip39 = "2" uuid = { workspace = true } [dev-dependencies] diff --git a/deny.toml b/deny.toml index 398a173dfa..54ede06429 100644 --- a/deny.toml +++ b/deny.toml @@ -11,7 +11,11 @@ deny = [ { crate = "derivative", reason = "use educe or derive_more instead" }, { crate = "ark-ff", reason = "present in Cargo.lock but not needed by Lighthouse" }, { crate = "strum", deny-multiple-versions = true, reason = "takes a long time to compile" }, - { crate = "reqwest", deny-multiple-versions = true, reason = "takes a long time to compile" } + { crate = "reqwest", deny-multiple-versions = true, reason = "takes a long time to compile" }, + { crate = "aes", deny-multiple-versions = true, reason = "takes a long time to compile" }, + { crate = "sha2", deny-multiple-versions = true, reason = "takes a long time to compile" }, + { crate = "pbkdf2", deny-multiple-versions = true, reason = "takes a long time to compile" }, + { crate = "scrypt", deny-multiple-versions = true, reason = "takes a long time to compile" }, ] [sources] From 79d314ddba3482b6c576505bb3f22b78b9168525 Mon Sep 17 00:00:00 2001 From: Akihito Nakano Date: Thu, 8 Jan 2026 04:42:16 +0900 Subject: [PATCH 04/18] Tweak a log message for mock-el (#8599) ```bash $ lcli mock-el .... ... ... Dec 15 11:52:06.002 INFO Metrics HTTP server started listen_address: "127.0.0.1:8551" ... ``` The log message "Metrics HTTP server" was misleading, as the server is actually a Mock Execution Client that provides a JSON-RPC API for testing purposes, not a metrics server. Co-Authored-By: ackintosh --- beacon_node/execution_layer/src/test_utils/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 8f12971560..2465a41d8b 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -754,7 +754,7 @@ pub fn serve( info!( listen_address = listening_socket.to_string(), - "Metrics HTTP server started" + "Mock execution client started" ); Ok((listening_socket, server)) From a39558f6e5fa8d626b34281ccf4719c5adcf43a7 Mon Sep 17 00:00:00 2001 From: chonghe <44791194+chong-he@users.noreply.github.com> Date: Thu, 8 Jan 2026 11:42:24 +0800 Subject: [PATCH 05/18] Remove Windows in the documentation (#8628) Co-Authored-By: Tan Chee Keong --- book/src/advanced_networking.md | 1 - book/src/installation.md | 4 +++- book/src/installation_binaries.md | 3 --- book/src/installation_source.md | 2 ++ book/src/validator_doppelganger.md | 1 - 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/book/src/advanced_networking.md b/book/src/advanced_networking.md index 0dc53bd42a..b14740dc18 100644 --- a/book/src/advanced_networking.md +++ b/book/src/advanced_networking.md @@ -69,7 +69,6 @@ The steps to do port forwarding depends on the router, but the general steps are - On Linux: open a terminal and run `ip route | grep default`, the result should look something similar to `default via 192.168.50.1 dev wlp2s0 proto dhcp metric 600`. The `192.168.50.1` is your router management default gateway IP. - On macOS: open a terminal and run `netstat -nr|grep default` and it should return the default gateway IP. - - On Windows: open a command prompt and run `ipconfig` and look for the `Default Gateway` which will show you the gateway IP. The default gateway IP usually looks like 192.168.X.X. Once you obtain the IP, enter it to a web browser and it will lead you to the router management page. diff --git a/book/src/installation.md b/book/src/installation.md index 95550e0807..15327400e9 100644 --- a/book/src/installation.md +++ b/book/src/installation.md @@ -1,6 +1,8 @@ # 📦 Installation -Lighthouse runs on Linux, macOS, and Windows. +Lighthouse runs on Linux, macOS, and Windows*. + +> \* Lighthouse does not officially support Windows platform. However, Lighthouse can still run natively on Windows until it does not at some point in the future. Windows users may also switch to using Docker, Windows Subsystem for Linux or other supported operating systems. There are three core methods to obtain the Lighthouse application: diff --git a/book/src/installation_binaries.md b/book/src/installation_binaries.md index 67a629e5c3..84e8bc04c4 100644 --- a/book/src/installation_binaries.md +++ b/book/src/installation_binaries.md @@ -11,7 +11,6 @@ Binaries are supplied for the following platforms: - `x86_64-unknown-linux-gnu`: AMD/Intel 64-bit processors (most desktops, laptops, servers) - `aarch64-unknown-linux-gnu`: 64-bit ARM processors (Raspberry Pi 4) - `aarch64-apple-darwin`: macOS with ARM chips -- `x86_64-windows`: Windows with 64-bit processors ## Usage @@ -32,5 +31,3 @@ a `x86_64` binary. 1. Test the binary with `./lighthouse --version` (it should print the version). 1. (Optional) Move the `lighthouse` binary to a location in your `PATH`, so the `lighthouse` command can be called from anywhere. For example, to copy `lighthouse` from the current directory to `usr/bin`, run `sudo cp lighthouse /usr/bin`. - -> Windows users will need to execute the commands in Step 2 from PowerShell. diff --git a/book/src/installation_source.md b/book/src/installation_source.md index f035d1d843..4a5c3d9922 100644 --- a/book/src/installation_source.md +++ b/book/src/installation_source.md @@ -69,6 +69,8 @@ After this, you are ready to [build Lighthouse](#build-lighthouse). ### Windows +> Note: Lighthouse no longer officially supports Windows binary. However, users will still be able to build the binary from source according to the instructions below. The instructions may no longer work at some point in the future. + 1. Install [Git](https://git-scm.com/download/win). 1. Install the [Chocolatey](https://chocolatey.org/install) package manager for Windows. > Tips: diff --git a/book/src/validator_doppelganger.md b/book/src/validator_doppelganger.md index 006df50bd9..6e199546f5 100644 --- a/book/src/validator_doppelganger.md +++ b/book/src/validator_doppelganger.md @@ -106,7 +106,6 @@ The steps to solving a doppelganger vary depending on the case, but some places 1. Is there another validator process running on this host? - Unix users can check by running the command `ps aux | grep lighthouse` - - Windows users can check the Task Manager. 1. Has this validator recently been moved from another host? Check to ensure it's not running. 1. Has this validator been delegated to a staking service? From 0706e62f528f6176b094deb4ab838e8e86deecf2 Mon Sep 17 00:00:00 2001 From: 0xMushow <105550256+0xMushow@users.noreply.github.com> Date: Thu, 8 Jan 2026 10:17:46 +0400 Subject: [PATCH 06/18] fix(peerdb): use start_slot instead of end_slot for safer actions (#8498) Which issue # does this PR address? None Discussed in private with @jimmygchen, Lighthouse's `earliest_available_slot` is guaranteed to always align with epoch boundaries, but as a safety implementation, we should use `start_slot` just in case other clients differ in their implementations. At least we agreed it would be safer for `synced_peers_for_epoch`, I also made the change in `has_good_custody_range_sync_peer`, but this is to be reviewed please. Co-Authored-By: Antoine James Co-Authored-By: Jimmy Chen --- beacon_node/lighthouse_network/src/peer_manager/peerdb.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index 87337cafcf..dc1686523f 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -259,10 +259,10 @@ impl PeerDB { info.is_connected() && match info.sync_status() { SyncStatus::Synced { info } => { - info.has_slot(epoch.end_slot(E::slots_per_epoch())) + info.has_slot(epoch.start_slot(E::slots_per_epoch())) } SyncStatus::Advanced { info } => { - info.has_slot(epoch.end_slot(E::slots_per_epoch())) + info.has_slot(epoch.start_slot(E::slots_per_epoch())) } SyncStatus::IrrelevantPeer | SyncStatus::Behind { .. } @@ -332,7 +332,7 @@ impl PeerDB { info.is_connected() && match info.sync_status() { SyncStatus::Synced { info } | SyncStatus::Advanced { info } => { - info.has_slot(epoch.end_slot(E::slots_per_epoch())) + info.has_slot(epoch.start_slot(E::slots_per_epoch())) } SyncStatus::IrrelevantPeer | SyncStatus::Behind { .. } From 6166ad2eb298a17e97c9f05638267175057c495e Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 8 Jan 2026 04:04:44 -0300 Subject: [PATCH 07/18] Replace tracing::debug! with debug! same for other levels (#8300) Just visual clean-up, making logging statements look uniform. There's no reason to use `tracing::debug` instead of `debug`. If we ever need to migrate our logging lib in the future it would make things easier too. Co-Authored-By: dapplion <35266934+dapplion@users.noreply.github.com> Co-Authored-By: Jimmy Chen Co-Authored-By: Michael Sproul --- .../lighthouse_network/src/types/globals.rs | 4 ++-- .../lighthouse_network/tests/rpc_tests.rs | 24 +++++++++---------- .../network/src/sync/backfill_sync/mod.rs | 2 +- .../src/sync/block_sidecar_coupling.rs | 6 ++--- .../network/src/sync/range_sync/chain.rs | 6 ++--- boot_node/src/config.rs | 5 ++-- lighthouse/src/main.rs | 3 ++- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index f46eb05ceb..3217f41f61 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -9,7 +9,7 @@ use network_utils::enr_ext::EnrExt; use parking_lot::RwLock; use std::collections::HashSet; use std::sync::Arc; -use tracing::error; +use tracing::{debug, error}; use types::data_column_custody_group::{compute_subnets_from_custody_group, get_custody_groups}; use types::{ChainSpec, ColumnIndex, DataColumnSubnetId, EthSpec}; @@ -79,7 +79,7 @@ impl NetworkGlobals { sampling_subnets.extend(subnets); } - tracing::debug!( + debug!( cgc = custody_group_count, ?sampling_subnets, "Starting node with custody params" diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 599fcd242b..2a17a04b90 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::runtime::Runtime; use tokio::time::sleep; -use tracing::{Instrument, debug, error, info_span, warn}; +use tracing::{Instrument, debug, error, info, info_span, warn}; use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockHeader, BlobSidecar, ChainSpec, DataColumnSidecar, DataColumnsByRootIdentifier, EmptyBlock, Epoch, @@ -1041,7 +1041,7 @@ fn test_tcp_columns_by_root_chunked_rpc() { loop { match sender.next_event().await { NetworkEvent::PeerConnectedOutgoing(peer_id) => { - tracing::info!("Sending RPC"); + info!("Sending RPC"); tokio::time::sleep(Duration::from_secs(1)).await; sender .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) @@ -1055,7 +1055,7 @@ fn test_tcp_columns_by_root_chunked_rpc() { Response::DataColumnsByRoot(Some(sidecar)) => { assert_eq!(sidecar, data_column.clone()); messages_received += 1; - tracing::info!("Chunk received"); + info!("Chunk received"); } Response::DataColumnsByRoot(None) => { // should be exactly messages_to_send @@ -1082,7 +1082,7 @@ fn test_tcp_columns_by_root_chunked_rpc() { } => { if request_type == rpc_request { // send the response - tracing::info!("Receiver got request"); + info!("Receiver got request"); for _ in 0..messages_to_send { receiver.send_response( @@ -1090,7 +1090,7 @@ fn test_tcp_columns_by_root_chunked_rpc() { inbound_request_id, rpc_response.clone(), ); - tracing::info!("Sending message"); + info!("Sending message"); } // send the stream termination receiver.send_response( @@ -1098,11 +1098,11 @@ fn test_tcp_columns_by_root_chunked_rpc() { inbound_request_id, Response::DataColumnsByRoot(None), ); - tracing::info!("Send stream term"); + info!("Send stream term"); } } e => { - tracing::info!(?e, "Got event"); + info!(?e, "Got event"); } // Ignore other events } } @@ -1186,7 +1186,7 @@ fn test_tcp_columns_by_range_chunked_rpc() { loop { match sender.next_event().await { NetworkEvent::PeerConnectedOutgoing(peer_id) => { - tracing::info!("Sending RPC"); + info!("Sending RPC"); sender .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); @@ -1199,7 +1199,7 @@ fn test_tcp_columns_by_range_chunked_rpc() { Response::DataColumnsByRange(Some(sidecar)) => { assert_eq!(sidecar, data_column.clone()); messages_received += 1; - tracing::info!("Chunk received"); + info!("Chunk received"); } Response::DataColumnsByRange(None) => { // should be exactly messages_to_send @@ -1226,7 +1226,7 @@ fn test_tcp_columns_by_range_chunked_rpc() { } => { if request_type == rpc_request { // send the response - tracing::info!("Receiver got request"); + info!("Receiver got request"); for _ in 0..messages_to_send { receiver.send_response( @@ -1234,7 +1234,7 @@ fn test_tcp_columns_by_range_chunked_rpc() { inbound_request_id, rpc_response.clone(), ); - tracing::info!("Sending message"); + info!("Sending message"); } // send the stream termination receiver.send_response( @@ -1242,7 +1242,7 @@ fn test_tcp_columns_by_range_chunked_rpc() { inbound_request_id, Response::DataColumnsByRange(None), ); - tracing::info!("Send stream term"); + info!("Send stream term"); } } _ => {} // Ignore other events diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 6c0cbd7e55..9802ec56a1 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -346,7 +346,7 @@ impl BackFillSync { } } CouplingError::BlobPeerFailure(msg) => { - tracing::debug!(?batch_id, msg, "Blob peer failure"); + debug!(?batch_id, msg, "Blob peer failure"); } CouplingError::InternalError(msg) => { error!(?batch_id, msg, "Block components coupling internal error"); diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index ed9a11a03d..6f563820f7 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -357,7 +357,7 @@ impl RangeBlockComponentsRequest { // we request the data from. // If there are duplicated indices, its likely a peer sending us the same index multiple times. // However we can still proceed even if there are extra columns, just log an error. - tracing::debug!(?block_root, ?index, "Repeated column for block_root"); + debug!(?block_root, ?index, "Repeated column for block_root"); continue; } } @@ -408,7 +408,7 @@ impl RangeBlockComponentsRequest { if !data_columns_by_index.is_empty() { let remaining_indices = data_columns_by_index.keys().collect::>(); // log the error but don't return an error, we can still progress with extra columns. - tracing::debug!( + debug!( ?block_root, ?remaining_indices, "Not all columns consumed for block" @@ -428,7 +428,7 @@ impl RangeBlockComponentsRequest { let remaining_roots = data_columns_by_block.keys().collect::>(); // log the error but don't return an error, we can still progress with responses. // this is most likely an internal error with overrequesting or a client bug. - tracing::debug!(?remaining_roots, "Not all columns consumed for block"); + debug!(?remaining_roots, "Not all columns consumed for block"); } Ok(rpc_blocks) diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 4ce10e23ca..a3f2b798b1 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -18,7 +18,7 @@ use std::collections::{BTreeMap, HashSet, btree_map::Entry}; use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use strum::IntoStaticStr; -use tracing::{Span, debug, instrument, warn}; +use tracing::{Span, debug, error, instrument, warn}; use types::{ColumnIndex, Epoch, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many epochs worth of @@ -942,10 +942,10 @@ impl SyncingChain { } } CouplingError::BlobPeerFailure(msg) => { - tracing::debug!(?batch_id, msg, "Blob peer failure"); + debug!(?batch_id, msg, "Blob peer failure"); } CouplingError::InternalError(msg) => { - tracing::error!(?batch_id, msg, "Block components coupling internal error"); + error!(?batch_id, msg, "Block components coupling internal error"); } } } diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index fb0daf5264..5b13b95c97 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -14,6 +14,7 @@ use ssz::Encode; use std::net::{SocketAddrV4, SocketAddrV6}; use std::time::Duration; use std::{marker::PhantomData, path::PathBuf}; +use tracing::{info, warn}; use types::EthSpec; /// A set of configuration parameters for the bootnode, established from CLI arguments. @@ -117,7 +118,7 @@ impl BootNodeConfig { let genesis_state_root = genesis_state .canonical_root() .map_err(|e| format!("Error hashing genesis state: {e:?}"))?; - tracing::info!(root = ?genesis_state_root, "Genesis state found"); + info!(root = ?genesis_state_root, "Genesis state found"); let enr_fork = spec.enr_fork_id::( types::Slot::from(0u64), genesis_state.genesis_validators_root(), @@ -125,7 +126,7 @@ impl BootNodeConfig { Some(enr_fork.as_ssz_bytes()) } else { - tracing::warn!("No genesis state provided. No Eth2 field added to the ENR"); + warn!("No genesis state provided. No Eth2 field added to the ENR"); None } }; diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index c93016a0f5..2592d2e604 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -730,7 +730,8 @@ fn run( #[cfg(all(feature = "modern", target_arch = "x86_64"))] if !std::is_x86_feature_detected!("adx") { - tracing::warn!( + use tracing::warn; + warn!( advice = "If you get a SIGILL, please try Lighthouse portable build", "CPU seems incompatible with optimized Lighthouse build" ); From b8c386d38d245e43b62bee7aeed05fcbdcccd777 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Sat, 10 Jan 2026 19:22:28 -0600 Subject: [PATCH 08/18] Move beacon processor work queue implementation to its own file (#8141) Co-Authored-By: Eitan Seri- Levi Co-Authored-By: Eitan Seri-Levi --- beacon_node/beacon_processor/src/lib.rs | 968 +++++++----------- .../beacon_processor/src/scheduler/mod.rs | 3 + .../src/scheduler/work_queue.rs | 388 +++++++ 3 files changed, 758 insertions(+), 601 deletions(-) create mode 100644 beacon_node/beacon_processor/src/scheduler/work_queue.rs diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 1cdf3693ff..0a2665ab11 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -38,20 +38,21 @@ //! checks the queues to see if there are more parcels of work that can be spawned in a new worker //! task. +pub use crate::scheduler::BeaconProcessorQueueLengths; +use crate::scheduler::work_queue::WorkQueues; use crate::work_reprocessing_queue::{ QueuedBackfillBatch, QueuedColumnReconstruction, QueuedGossipBlock, ReprocessQueueMessage, }; use futures::stream::{Stream, StreamExt}; use futures::task::Poll; use lighthouse_network::{MessageId, NetworkGlobals, PeerId}; -use logging::TimeLatch; use logging::crit; use parking_lot::Mutex; pub use scheduler::work_reprocessing_queue; use serde::{Deserialize, Serialize}; use slot_clock::SlotClock; use std::cmp; -use std::collections::{HashSet, VecDeque}; +use std::collections::HashSet; use std::fmt; use std::future::Future; use std::pin::Pin; @@ -63,10 +64,7 @@ use task_executor::{RayonPoolType, TaskExecutor}; use tokio::sync::mpsc; use tokio::sync::mpsc::error::TrySendError; use tracing::{debug, error, trace, warn}; -use types::{ - BeaconState, ChainSpec, EthSpec, Hash256, RelativeEpoch, SignedAggregateAndProof, - SingleAttestation, Slot, SubnetId, -}; +use types::{EthSpec, Hash256, SignedAggregateAndProof, SingleAttestation, Slot, SubnetId}; use work_reprocessing_queue::IgnoredRpcBlock; use work_reprocessing_queue::{ QueuedAggregate, QueuedLightClientUpdate, QueuedRpcBlock, QueuedUnaggregate, ReadyWork, @@ -90,124 +88,6 @@ const MAX_IDLE_QUEUE_LEN: usize = 16_384; /// The maximum size of the channel for re-processing work events. const DEFAULT_MAX_SCHEDULED_WORK_QUEUE_LEN: usize = 3 * DEFAULT_MAX_WORK_EVENT_QUEUE_LEN / 4; -/// Over-provision queues based on active validator count by some factor. The beacon chain has -/// strict churns that prevent the validator set size from changing rapidly. By over-provisioning -/// slightly, we don't need to adjust the queues during the lifetime of a process. -const ACTIVE_VALIDATOR_COUNT_OVERPROVISION_PERCENT: usize = 110; - -/// Minimum size of dynamically sized queues. Due to integer division we don't want 0 length queues -/// as the processor won't process that message type. 128 is an arbitrary value value >= 1 that -/// seems reasonable. -const MIN_QUEUE_LEN: usize = 128; - -/// Maximum number of queued items that will be stored before dropping them -pub struct BeaconProcessorQueueLengths { - aggregate_queue: usize, - attestation_queue: usize, - unknown_block_aggregate_queue: usize, - unknown_block_attestation_queue: usize, - sync_message_queue: usize, - sync_contribution_queue: usize, - gossip_voluntary_exit_queue: usize, - gossip_proposer_slashing_queue: usize, - gossip_attester_slashing_queue: usize, - unknown_light_client_update_queue: usize, - rpc_block_queue: usize, - rpc_blob_queue: usize, - rpc_custody_column_queue: usize, - column_reconstruction_queue: usize, - chain_segment_queue: usize, - backfill_chain_segment: usize, - gossip_block_queue: usize, - gossip_blob_queue: usize, - gossip_data_column_queue: usize, - delayed_block_queue: usize, - status_queue: usize, - block_brange_queue: usize, - block_broots_queue: usize, - blob_broots_queue: usize, - blob_brange_queue: usize, - dcbroots_queue: usize, - dcbrange_queue: usize, - gossip_bls_to_execution_change_queue: usize, - lc_gossip_finality_update_queue: usize, - lc_gossip_optimistic_update_queue: usize, - lc_bootstrap_queue: usize, - lc_rpc_optimistic_update_queue: usize, - lc_rpc_finality_update_queue: usize, - lc_update_range_queue: usize, - api_request_p0_queue: usize, - api_request_p1_queue: usize, -} - -impl BeaconProcessorQueueLengths { - pub fn from_state( - state: &BeaconState, - spec: &ChainSpec, - ) -> Result { - let active_validator_count = - match state.get_cached_active_validator_indices(RelativeEpoch::Current) { - Ok(indices) => indices.len(), - Err(_) => state - .get_active_validator_indices(state.current_epoch(), spec) - .map_err(|e| format!("Error computing active indices: {:?}", e))? - .len(), - }; - let active_validator_count = - (ACTIVE_VALIDATOR_COUNT_OVERPROVISION_PERCENT * active_validator_count) / 100; - let slots_per_epoch = E::slots_per_epoch() as usize; - - Ok(Self { - aggregate_queue: 4096, - unknown_block_aggregate_queue: 1024, - // Capacity for a full slot's worth of attestations if subscribed to all subnets - attestation_queue: std::cmp::max( - active_validator_count / slots_per_epoch, - MIN_QUEUE_LEN, - ), - // Capacity for a full slot's worth of attestations if subscribed to all subnets - unknown_block_attestation_queue: std::cmp::max( - active_validator_count / slots_per_epoch, - MIN_QUEUE_LEN, - ), - sync_message_queue: 2048, - sync_contribution_queue: 1024, - gossip_voluntary_exit_queue: 4096, - gossip_proposer_slashing_queue: 4096, - gossip_attester_slashing_queue: 4096, - unknown_light_client_update_queue: 128, - rpc_block_queue: 1024, - rpc_blob_queue: 1024, - // We don't request more than `PARENT_DEPTH_TOLERANCE` (32) lookups, so we can limit - // this queue size. With 48 max blobs per block, each column sidecar list could be up to 12MB. - rpc_custody_column_queue: 64, - column_reconstruction_queue: 1, - chain_segment_queue: 64, - backfill_chain_segment: 64, - gossip_block_queue: 1024, - gossip_blob_queue: 1024, - gossip_data_column_queue: 1024, - delayed_block_queue: 1024, - status_queue: 1024, - block_brange_queue: 1024, - block_broots_queue: 1024, - blob_broots_queue: 1024, - blob_brange_queue: 1024, - dcbroots_queue: 1024, - dcbrange_queue: 1024, - gossip_bls_to_execution_change_queue: 16384, - lc_gossip_finality_update_queue: 1024, - lc_gossip_optimistic_update_queue: 1024, - lc_bootstrap_queue: 1024, - lc_rpc_optimistic_update_queue: 512, - lc_rpc_finality_update_queue: 512, - lc_update_range_queue: 512, - api_request_p0_queue: 1024, - api_request_p1_queue: 1024, - }) - } -} - /// The name of the manager tokio task. const MANAGER_TASK_NAME: &str = "beacon_processor_manager"; @@ -279,89 +159,6 @@ impl Default for BeaconProcessorChannels { } } -/// A simple first-in-first-out queue with a maximum length. -struct FifoQueue { - queue: VecDeque, - max_length: usize, -} - -impl FifoQueue { - /// Create a new, empty queue with the given length. - pub fn new(max_length: usize) -> Self { - Self { - queue: VecDeque::default(), - max_length, - } - } - - /// Add a new item to the queue. - /// - /// Drops `item` if the queue is full. - pub fn push(&mut self, item: T, item_desc: &str) { - if self.queue.len() == self.max_length { - error!( - msg = "the system has insufficient resources for load", - queue_len = self.max_length, - queue = item_desc, - "Work queue is full" - ) - } else { - self.queue.push_back(item); - } - } - - /// Remove the next item from the queue. - pub fn pop(&mut self) -> Option { - self.queue.pop_front() - } - - /// Returns the current length of the queue. - pub fn len(&self) -> usize { - self.queue.len() - } -} - -/// A simple last-in-first-out queue with a maximum length. -struct LifoQueue { - queue: VecDeque, - max_length: usize, -} - -impl LifoQueue { - /// Create a new, empty queue with the given length. - pub fn new(max_length: usize) -> Self { - Self { - queue: VecDeque::default(), - max_length, - } - } - - /// Add a new item to the front of the queue. - /// - /// If the queue is full, the item at the back of the queue is dropped. - pub fn push(&mut self, item: T) { - if self.queue.len() == self.max_length { - self.queue.pop_back(); - } - self.queue.push_front(item); - } - - /// Remove the next item from the queue. - pub fn pop(&mut self) -> Option { - self.queue.pop_front() - } - - /// Returns `true` if the queue is full. - pub fn is_full(&self) -> bool { - self.queue.len() >= self.max_length - } - - /// Returns the current length of the queue. - pub fn len(&self) -> usize { - self.queue.len() - } -} - /// A handle that sends a message on the provided channel to a receiver when it gets dropped. /// /// The receiver task is responsible for removing the provided `entry` from the `DuplicateCache` @@ -834,74 +631,8 @@ impl BeaconProcessor { // Used by workers to communicate that they are finished a task. let (idle_tx, idle_rx) = mpsc::channel::(MAX_IDLE_QUEUE_LEN); - // Using LIFO queues for attestations since validator profits rely upon getting fresh - // attestations into blocks. Additionally, later attestations contain more information than - // earlier ones, so we consider them more valuable. - let mut aggregate_queue = LifoQueue::new(queue_lengths.aggregate_queue); - let mut aggregate_debounce = TimeLatch::default(); - let mut attestation_queue = LifoQueue::new(queue_lengths.attestation_queue); - let mut attestation_to_convert_queue = LifoQueue::new(queue_lengths.attestation_queue); - let mut attestation_debounce = TimeLatch::default(); - let mut unknown_block_aggregate_queue = - LifoQueue::new(queue_lengths.unknown_block_aggregate_queue); - let mut unknown_block_attestation_queue = - LifoQueue::new(queue_lengths.unknown_block_attestation_queue); - - let mut sync_message_queue = LifoQueue::new(queue_lengths.sync_message_queue); - let mut sync_contribution_queue = LifoQueue::new(queue_lengths.sync_contribution_queue); - - // Using a FIFO queue for voluntary exits since it prevents exit censoring. I don't have - // a strong feeling about queue type for exits. - let mut gossip_voluntary_exit_queue = - FifoQueue::new(queue_lengths.gossip_voluntary_exit_queue); - - // Using a FIFO queue for slashing to prevent people from flushing their slashings from the - // queues with lots of junk messages. - let mut gossip_proposer_slashing_queue = - FifoQueue::new(queue_lengths.gossip_proposer_slashing_queue); - let mut gossip_attester_slashing_queue = - FifoQueue::new(queue_lengths.gossip_attester_slashing_queue); - - // Using a FIFO queue since blocks need to be imported sequentially. - let mut rpc_block_queue = FifoQueue::new(queue_lengths.rpc_block_queue); - let mut rpc_blob_queue = FifoQueue::new(queue_lengths.rpc_blob_queue); - let mut rpc_custody_column_queue = FifoQueue::new(queue_lengths.rpc_custody_column_queue); - let mut column_reconstruction_queue = - LifoQueue::new(queue_lengths.column_reconstruction_queue); - let mut chain_segment_queue = FifoQueue::new(queue_lengths.chain_segment_queue); - let mut backfill_chain_segment = FifoQueue::new(queue_lengths.backfill_chain_segment); - let mut gossip_block_queue = FifoQueue::new(queue_lengths.gossip_block_queue); - let mut gossip_blob_queue = FifoQueue::new(queue_lengths.gossip_blob_queue); - let mut gossip_data_column_queue = FifoQueue::new(queue_lengths.gossip_data_column_queue); - let mut delayed_block_queue = FifoQueue::new(queue_lengths.delayed_block_queue); - - let mut status_queue = FifoQueue::new(queue_lengths.status_queue); - let mut block_brange_queue = FifoQueue::new(queue_lengths.block_brange_queue); - let mut block_broots_queue = FifoQueue::new(queue_lengths.block_broots_queue); - let mut blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); - let mut blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); - let mut dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); - let mut dcbrange_queue = FifoQueue::new(queue_lengths.dcbrange_queue); - - let mut gossip_bls_to_execution_change_queue = - FifoQueue::new(queue_lengths.gossip_bls_to_execution_change_queue); - - // Using FIFO queues for light client updates to maintain sequence order. - let mut lc_gossip_finality_update_queue = - FifoQueue::new(queue_lengths.lc_gossip_finality_update_queue); - let mut lc_gossip_optimistic_update_queue = - FifoQueue::new(queue_lengths.lc_gossip_optimistic_update_queue); - let mut unknown_light_client_update_queue = - FifoQueue::new(queue_lengths.unknown_light_client_update_queue); - let mut lc_bootstrap_queue = FifoQueue::new(queue_lengths.lc_bootstrap_queue); - let mut lc_rpc_optimistic_update_queue = - FifoQueue::new(queue_lengths.lc_rpc_optimistic_update_queue); - let mut lc_rpc_finality_update_queue = - FifoQueue::new(queue_lengths.lc_rpc_finality_update_queue); - let mut lc_update_range_queue = FifoQueue::new(queue_lengths.lc_update_range_queue); - - let mut api_request_p0_queue = FifoQueue::new(queue_lengths.api_request_p0_queue); - let mut api_request_p1_queue = FifoQueue::new(queue_lengths.api_request_p1_queue); + // Initialize the worker queues. + let mut work_queues: WorkQueues = WorkQueues::new(queue_lengths); // Channels for sending work to the re-process scheduler (`work_reprocessing_tx`) and to // receive them back once they are ready (`ready_work_rx`). @@ -1030,228 +761,238 @@ impl BeaconProcessor { None if can_spawn => { // Check for chain segments first, they're the most efficient way to get // blocks into the system. - let work_event: Option> = - if let Some(item) = chain_segment_queue.pop() { - Some(item) - // Check sync blocks before gossip blocks, since we've already explicitly - // requested these blocks. - } else if let Some(item) = rpc_block_queue.pop() { - Some(item) - } else if let Some(item) = rpc_blob_queue.pop() { - Some(item) - } else if let Some(item) = rpc_custody_column_queue.pop() { - Some(item) - } else if let Some(item) = rpc_custody_column_queue.pop() { - Some(item) - // Check delayed blocks before gossip blocks, the gossip blocks might rely - // on the delayed ones. - } else if let Some(item) = delayed_block_queue.pop() { - Some(item) - // Check gossip blocks before gossip attestations, since a block might be - // required to verify some attestations. - } else if let Some(item) = gossip_block_queue.pop() { - Some(item) - } else if let Some(item) = gossip_blob_queue.pop() { - Some(item) - } else if let Some(item) = gossip_data_column_queue.pop() { - Some(item) - } else if let Some(item) = column_reconstruction_queue.pop() { - Some(item) - // Check the priority 0 API requests after blocks and blobs, but before attestations. - } else if let Some(item) = api_request_p0_queue.pop() { - Some(item) - // Check the aggregates, *then* the unaggregates since we assume that - // aggregates are more valuable to local validators and effectively give us - // more information with less signature verification time. - } else if aggregate_queue.len() > 0 { - let batch_size = cmp::min( - aggregate_queue.len(), - self.config.max_gossip_aggregate_batch_size, - ); + let work_event: Option> = if let Some(item) = + work_queues.chain_segment_queue.pop() + { + Some(item) + // Check sync blocks before gossip blocks, since we've already explicitly + // requested these blocks. + } else if let Some(item) = work_queues.rpc_block_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.rpc_blob_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.rpc_custody_column_queue.pop() { + Some(item) + // Check delayed blocks before gossip blocks, the gossip blocks might rely + // on the delayed ones. + } else if let Some(item) = work_queues.delayed_block_queue.pop() { + Some(item) + // Check gossip blocks before gossip attestations, since a block might be + // required to verify some attestations. + } else if let Some(item) = work_queues.gossip_block_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.gossip_blob_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.gossip_data_column_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.column_reconstruction_queue.pop() { + Some(item) + // Check the priority 0 API requests after blocks and blobs, but before attestations. + } else if let Some(item) = work_queues.api_request_p0_queue.pop() { + Some(item) + // Check the aggregates, *then* the unaggregates since we assume that + // aggregates are more valuable to local validators and effectively give us + // more information with less signature verification time. + } else if !work_queues.aggregate_queue.is_empty() { + let batch_size = cmp::min( + work_queues.aggregate_queue.len(), + self.config.max_gossip_aggregate_batch_size, + ); - if batch_size < 2 { - // One single aggregate is in the queue, process it individually. - aggregate_queue.pop() - } else { - // Collect two or more aggregates into a batch, so they can take - // advantage of batch signature verification. - // - // Note: this will convert the `Work::GossipAggregate` item into a - // `Work::GossipAggregateBatch` item. - let mut aggregates = Vec::with_capacity(batch_size); - let mut process_batch_opt = None; - for _ in 0..batch_size { - if let Some(item) = aggregate_queue.pop() { - match item { - Work::GossipAggregate { - aggregate, - process_individual: _, - process_batch, - } => { - aggregates.push(*aggregate); - if process_batch_opt.is_none() { - process_batch_opt = Some(process_batch); - } - } - _ => { - error!("Invalid item in aggregate queue"); - } - } - } - } - - if let Some(process_batch) = process_batch_opt { - // Process all aggregates with a single worker. - Some(Work::GossipAggregateBatch { - aggregates, - process_batch, - }) - } else { - // There is no good reason for this to - // happen, it is a serious logic error. - // Since we only form batches when multiple - // work items exist, we should always have a - // work closure at this point. - crit!("Missing aggregate work"); - None - } - } - // Check the unaggregated attestation queue. - // - // Potentially use batching. - } else if attestation_queue.len() > 0 { - let batch_size = cmp::min( - attestation_queue.len(), - self.config.max_gossip_attestation_batch_size, - ); - - if batch_size < 2 { - // One single attestation is in the queue, process it individually. - attestation_queue.pop() - } else { - // Collect two or more attestations into a batch, so they can take - // advantage of batch signature verification. - // - // Note: this will convert the `Work::GossipAttestation` item into a - // `Work::GossipAttestationBatch` item. - let mut attestations = Vec::with_capacity(batch_size); - let mut process_batch_opt = None; - for _ in 0..batch_size { - if let Some(item) = attestation_queue.pop() { - match item { - Work::GossipAttestation { - attestation, - process_individual: _, - process_batch, - } => { - attestations.push(*attestation); - if process_batch_opt.is_none() { - process_batch_opt = Some(process_batch); - } - } - _ => error!("Invalid item in attestation queue"), - } - } - } - - if let Some(process_batch) = process_batch_opt { - // Process all attestations with a single worker. - Some(Work::GossipAttestationBatch { - attestations, - process_batch, - }) - } else { - // There is no good reason for this to - // happen, it is a serious logic error. - // Since we only form batches when multiple - // work items exist, we should always have a - // work closure at this point. - crit!("Missing attestations work"); - None - } - } - // Convert any gossip attestations that need to be converted. - } else if let Some(item) = attestation_to_convert_queue.pop() { - Some(item) - // Check sync committee messages after attestations as their rewards are lesser - // and they don't influence fork choice. - } else if let Some(item) = sync_contribution_queue.pop() { - Some(item) - } else if let Some(item) = sync_message_queue.pop() { - Some(item) - // Aggregates and unaggregates queued for re-processing are older and we - // care about fresher ones, so check those first. - } else if let Some(item) = unknown_block_aggregate_queue.pop() { - Some(item) - } else if let Some(item) = unknown_block_attestation_queue.pop() { - Some(item) - // Check RPC methods next. Status messages are needed for sync so - // prioritize them over syncing requests from other peers (BlocksByRange - // and BlocksByRoot) - } else if let Some(item) = status_queue.pop() { - Some(item) - } else if let Some(item) = block_brange_queue.pop() { - Some(item) - } else if let Some(item) = block_broots_queue.pop() { - Some(item) - } else if let Some(item) = blob_brange_queue.pop() { - Some(item) - } else if let Some(item) = blob_broots_queue.pop() { - Some(item) - } else if let Some(item) = dcbroots_queue.pop() { - Some(item) - } else if let Some(item) = dcbrange_queue.pop() { - Some(item) - // Check slashings after all other consensus messages so we prioritize - // following head. - // - // Check attester slashings before proposer slashings since they have the - // potential to slash multiple validators at once. - } else if let Some(item) = gossip_attester_slashing_queue.pop() { - Some(item) - } else if let Some(item) = gossip_proposer_slashing_queue.pop() { - Some(item) - // Check exits and address changes late since our validators don't get - // rewards from them. - } else if let Some(item) = gossip_voluntary_exit_queue.pop() { - Some(item) - } else if let Some(item) = gossip_bls_to_execution_change_queue.pop() { - Some(item) - // Check the priority 1 API requests after we've - // processed all the interesting things from the network - // and things required for us to stay in good repute - // with our P2P peers. - } else if let Some(item) = api_request_p1_queue.pop() { - Some(item) - // Handle backfill sync chain segments. - } else if let Some(item) = backfill_chain_segment.pop() { - Some(item) - // Handle light client requests. - } else if let Some(item) = lc_gossip_finality_update_queue.pop() { - Some(item) - } else if let Some(item) = lc_gossip_optimistic_update_queue.pop() { - Some(item) - } else if let Some(item) = unknown_light_client_update_queue.pop() { - Some(item) - } else if let Some(item) = lc_bootstrap_queue.pop() { - Some(item) - } else if let Some(item) = lc_rpc_optimistic_update_queue.pop() { - Some(item) - } else if let Some(item) = lc_rpc_finality_update_queue.pop() { - Some(item) - } else if let Some(item) = lc_update_range_queue.pop() { - Some(item) - // This statement should always be the final else statement. + if batch_size < 2 { + // One single aggregate is in the queue, process it individually. + work_queues.aggregate_queue.pop() } else { - // Let the journal know that a worker is freed and there's nothing else - // for it to do. - if let Some(work_journal_tx) = &work_journal_tx { - // We don't care if this message was successfully sent, we only use the journal - // during testing. - let _ = work_journal_tx.try_send(NOTHING_TO_DO); + // Collect two or more aggregates into a batch, so they can take + // advantage of batch signature verification. + // + // Note: this will convert the `Work::GossipAggregate` item into a + // `Work::GossipAggregateBatch` item. + let mut aggregates = Vec::with_capacity(batch_size); + let mut process_batch_opt = None; + for _ in 0..batch_size { + if let Some(item) = work_queues.aggregate_queue.pop() { + match item { + Work::GossipAggregate { + aggregate, + process_individual: _, + process_batch, + } => { + aggregates.push(*aggregate); + if process_batch_opt.is_none() { + process_batch_opt = Some(process_batch); + } + } + _ => { + error!("Invalid item in aggregate queue"); + } + } + } } - None - }; + + if let Some(process_batch) = process_batch_opt { + // Process all aggregates with a single worker. + Some(Work::GossipAggregateBatch { + aggregates, + process_batch, + }) + } else { + // There is no good reason for this to + // happen, it is a serious logic error. + // Since we only form batches when multiple + // work items exist, we should always have a + // work closure at this point. + crit!("Missing aggregate work"); + None + } + } + // Check the unaggregated attestation queue. + // + // Potentially use batching. + } else if !work_queues.attestation_queue.is_empty() { + let batch_size = cmp::min( + work_queues.attestation_queue.len(), + self.config.max_gossip_attestation_batch_size, + ); + + if batch_size < 2 { + // One single attestation is in the queue, process it individually. + work_queues.attestation_queue.pop() + } else { + // Collect two or more attestations into a batch, so they can take + // advantage of batch signature verification. + // + // Note: this will convert the `Work::GossipAttestation` item into a + // `Work::GossipAttestationBatch` item. + let mut attestations = Vec::with_capacity(batch_size); + let mut process_batch_opt = None; + for _ in 0..batch_size { + if let Some(item) = work_queues.attestation_queue.pop() { + match item { + Work::GossipAttestation { + attestation, + process_individual: _, + process_batch, + } => { + attestations.push(*attestation); + if process_batch_opt.is_none() { + process_batch_opt = Some(process_batch); + } + } + _ => error!("Invalid item in attestation queue"), + } + } + } + + if let Some(process_batch) = process_batch_opt { + // Process all attestations with a single worker. + Some(Work::GossipAttestationBatch { + attestations, + process_batch, + }) + } else { + // There is no good reason for this to + // happen, it is a serious logic error. + // Since we only form batches when multiple + // work items exist, we should always have a + // work closure at this point. + crit!("Missing attestations work"); + None + } + } + // Convert any gossip attestations that need to be converted. + } else if let Some(item) = work_queues.attestation_to_convert_queue.pop() { + Some(item) + // Check sync committee messages after attestations as their rewards are lesser + // and they don't influence fork choice. + } else if let Some(item) = work_queues.sync_contribution_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.sync_message_queue.pop() { + Some(item) + // Aggregates and unaggregates queued for re-processing are older and we + // care about fresher ones, so check those first. + } else if let Some(item) = work_queues.unknown_block_aggregate_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.unknown_block_attestation_queue.pop() + { + Some(item) + // Check RPC methods next. Status messages are needed for sync so + // prioritize them over syncing requests from other peers (BlocksByRange + // and BlocksByRoot) + } else if let Some(item) = work_queues.status_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.bbrange_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.bbroots_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.blbrange_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.blbroots_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.dcbroots_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.dcbrange_queue.pop() { + Some(item) + // Check slashings after all other consensus messages so we prioritize + // following head. + // + // Check attester slashings before proposer slashings since they have the + // potential to slash multiple validators at once. + } else if let Some(item) = work_queues.gossip_attester_slashing_queue.pop() + { + Some(item) + } else if let Some(item) = work_queues.gossip_proposer_slashing_queue.pop() + { + Some(item) + // Check exits and address changes late since our validators don't get + // rewards from them. + } else if let Some(item) = work_queues.gossip_voluntary_exit_queue.pop() { + Some(item) + } else if let Some(item) = + work_queues.gossip_bls_to_execution_change_queue.pop() + { + Some(item) + // Check the priority 1 API requests after we've + // processed all the interesting things from the network + // and things required for us to stay in good repute + // with our P2P peers. + } else if let Some(item) = work_queues.api_request_p1_queue.pop() { + Some(item) + // Handle backfill sync chain segments. + } else if let Some(item) = work_queues.backfill_chain_segment.pop() { + Some(item) + // Handle light client requests. + } else if let Some(item) = work_queues.lc_gossip_finality_update_queue.pop() + { + Some(item) + } else if let Some(item) = + work_queues.lc_gossip_optimistic_update_queue.pop() + { + Some(item) + } else if let Some(item) = + work_queues.unknown_light_client_update_queue.pop() + { + Some(item) + } else if let Some(item) = work_queues.lc_bootstrap_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.lc_rpc_optimistic_update_queue.pop() + { + Some(item) + } else if let Some(item) = work_queues.lc_rpc_finality_update_queue.pop() { + Some(item) + } else if let Some(item) = work_queues.lc_update_range_queue.pop() { + Some(item) + // This statement should always be the final else statement. + } else { + // Let the journal know that a worker is freed and there's nothing else + // for it to do. + if let Some(work_journal_tx) = &work_journal_tx { + // We don't care if this message was successfully sent, we only use the journal + // during testing. + let _ = work_journal_tx.try_send(NOTHING_TO_DO); + } + None + }; if let Some(work_event) = work_event { let work_type = work_event.to_type(); @@ -1304,14 +1045,16 @@ impl BeaconProcessor { } } _ if can_spawn => self.spawn_worker(work, created_timestamp, idle_tx), - Work::GossipAttestation { .. } => attestation_queue.push(work), + Work::GossipAttestation { .. } => { + work_queues.attestation_queue.push(work) + } // Attestation batches are formed internally within the // `BeaconProcessor`, they are not sent from external services. Work::GossipAttestationBatch { .. } => crit!( work_type = "GossipAttestationBatch", "Unsupported inbound event" ), - Work::GossipAggregate { .. } => aggregate_queue.push(work), + Work::GossipAggregate { .. } => work_queues.aggregate_queue.push(work), // Aggregate batches are formed internally within the `BeaconProcessor`, // they are not sent from external services. Work::GossipAggregateBatch { .. } => { @@ -1320,90 +1063,104 @@ impl BeaconProcessor { "Unsupported inbound event" ) } - Work::GossipBlock { .. } => gossip_block_queue.push(work, work_id), - Work::GossipBlobSidecar { .. } => gossip_blob_queue.push(work, work_id), + Work::GossipBlock { .. } => { + work_queues.gossip_block_queue.push(work, work_id) + } + Work::GossipBlobSidecar { .. } => { + work_queues.gossip_blob_queue.push(work, work_id) + } Work::GossipDataColumnSidecar { .. } => { - gossip_data_column_queue.push(work, work_id) + work_queues.gossip_data_column_queue.push(work, work_id) } Work::DelayedImportBlock { .. } => { - delayed_block_queue.push(work, work_id) + work_queues.delayed_block_queue.push(work, work_id) } Work::GossipVoluntaryExit { .. } => { - gossip_voluntary_exit_queue.push(work, work_id) + work_queues.gossip_voluntary_exit_queue.push(work, work_id) } - Work::GossipProposerSlashing { .. } => { - gossip_proposer_slashing_queue.push(work, work_id) + Work::GossipProposerSlashing { .. } => work_queues + .gossip_proposer_slashing_queue + .push(work, work_id), + Work::GossipAttesterSlashing { .. } => work_queues + .gossip_attester_slashing_queue + .push(work, work_id), + Work::GossipSyncSignature { .. } => { + work_queues.sync_message_queue.push(work) } - Work::GossipAttesterSlashing { .. } => { - gossip_attester_slashing_queue.push(work, work_id) - } - Work::GossipSyncSignature { .. } => sync_message_queue.push(work), Work::GossipSyncContribution { .. } => { - sync_contribution_queue.push(work) - } - Work::GossipLightClientFinalityUpdate { .. } => { - lc_gossip_finality_update_queue.push(work, work_id) - } - Work::GossipLightClientOptimisticUpdate { .. } => { - lc_gossip_optimistic_update_queue.push(work, work_id) + work_queues.sync_contribution_queue.push(work) } + Work::GossipLightClientFinalityUpdate { .. } => work_queues + .lc_gossip_finality_update_queue + .push(work, work_id), + Work::GossipLightClientOptimisticUpdate { .. } => work_queues + .lc_gossip_optimistic_update_queue + .push(work, work_id), Work::RpcBlock { .. } | Work::IgnoredRpcBlock { .. } => { - rpc_block_queue.push(work, work_id) + work_queues.rpc_block_queue.push(work, work_id) } - Work::RpcBlobs { .. } => rpc_blob_queue.push(work, work_id), + Work::RpcBlobs { .. } => work_queues.rpc_blob_queue.push(work, work_id), Work::RpcCustodyColumn { .. } => { - rpc_custody_column_queue.push(work, work_id) + work_queues.rpc_custody_column_queue.push(work, work_id) + } + Work::ColumnReconstruction(_) => { + work_queues.column_reconstruction_queue.push(work) + } + Work::ChainSegment { .. } => { + work_queues.chain_segment_queue.push(work, work_id) } - Work::ColumnReconstruction(_) => column_reconstruction_queue.push(work), - Work::ChainSegment { .. } => chain_segment_queue.push(work, work_id), Work::ChainSegmentBackfill { .. } => { - backfill_chain_segment.push(work, work_id) + work_queues.backfill_chain_segment.push(work, work_id) } - Work::Status { .. } => status_queue.push(work, work_id), + Work::Status { .. } => work_queues.status_queue.push(work, work_id), Work::BlocksByRangeRequest { .. } => { - block_brange_queue.push(work, work_id) + work_queues.bbrange_queue.push(work, work_id) } Work::BlocksByRootsRequest { .. } => { - block_broots_queue.push(work, work_id) + work_queues.bbroots_queue.push(work, work_id) } Work::BlobsByRangeRequest { .. } => { - blob_brange_queue.push(work, work_id) + work_queues.blbrange_queue.push(work, work_id) } Work::LightClientBootstrapRequest { .. } => { - lc_bootstrap_queue.push(work, work_id) - } - Work::LightClientOptimisticUpdateRequest { .. } => { - lc_rpc_optimistic_update_queue.push(work, work_id) + work_queues.lc_bootstrap_queue.push(work, work_id) } + Work::LightClientOptimisticUpdateRequest { .. } => work_queues + .lc_rpc_optimistic_update_queue + .push(work, work_id), Work::LightClientFinalityUpdateRequest { .. } => { - lc_rpc_finality_update_queue.push(work, work_id) + work_queues.lc_rpc_finality_update_queue.push(work, work_id) } Work::LightClientUpdatesByRangeRequest { .. } => { - lc_update_range_queue.push(work, work_id) + work_queues.lc_update_range_queue.push(work, work_id) } Work::UnknownBlockAttestation { .. } => { - unknown_block_attestation_queue.push(work) + work_queues.unknown_block_attestation_queue.push(work) } Work::UnknownBlockAggregate { .. } => { - unknown_block_aggregate_queue.push(work) - } - Work::GossipBlsToExecutionChange { .. } => { - gossip_bls_to_execution_change_queue.push(work, work_id) + work_queues.unknown_block_aggregate_queue.push(work) } + Work::GossipBlsToExecutionChange { .. } => work_queues + .gossip_bls_to_execution_change_queue + .push(work, work_id), Work::BlobsByRootsRequest { .. } => { - blob_broots_queue.push(work, work_id) + work_queues.blbroots_queue.push(work, work_id) } Work::DataColumnsByRootsRequest { .. } => { - dcbroots_queue.push(work, work_id) + work_queues.dcbroots_queue.push(work, work_id) } Work::DataColumnsByRangeRequest { .. } => { - dcbrange_queue.push(work, work_id) + work_queues.dcbrange_queue.push(work, work_id) } - Work::UnknownLightClientOptimisticUpdate { .. } => { - unknown_light_client_update_queue.push(work, work_id) + Work::UnknownLightClientOptimisticUpdate { .. } => work_queues + .unknown_light_client_update_queue + .push(work, work_id), + Work::ApiRequestP0 { .. } => { + work_queues.api_request_p0_queue.push(work, work_id) + } + Work::ApiRequestP1 { .. } => { + work_queues.api_request_p1_queue.push(work, work_id) } - Work::ApiRequestP0 { .. } => api_request_p0_queue.push(work, work_id), - Work::ApiRequestP1 { .. } => api_request_p1_queue.push(work, work_id), }; Some(work_type) } @@ -1411,57 +1168,81 @@ impl BeaconProcessor { if let Some(modified_queue_id) = modified_queue_id { let queue_len = match modified_queue_id { - WorkType::GossipAttestation => attestation_queue.len(), - WorkType::GossipAttestationToConvert => attestation_to_convert_queue.len(), - WorkType::UnknownBlockAttestation => unknown_block_attestation_queue.len(), + WorkType::GossipAttestation => work_queues.attestation_queue.len(), + WorkType::GossipAttestationToConvert => { + work_queues.attestation_to_convert_queue.len() + } + WorkType::UnknownBlockAttestation => { + work_queues.unknown_block_attestation_queue.len() + } WorkType::GossipAttestationBatch => 0, // No queue - WorkType::GossipAggregate => aggregate_queue.len(), - WorkType::UnknownBlockAggregate => unknown_block_aggregate_queue.len(), + WorkType::GossipAggregate => work_queues.aggregate_queue.len(), + WorkType::UnknownBlockAggregate => { + work_queues.unknown_block_aggregate_queue.len() + } WorkType::UnknownLightClientOptimisticUpdate => { - unknown_light_client_update_queue.len() + work_queues.unknown_light_client_update_queue.len() } WorkType::GossipAggregateBatch => 0, // No queue - WorkType::GossipBlock => gossip_block_queue.len(), - WorkType::GossipBlobSidecar => gossip_blob_queue.len(), - WorkType::GossipDataColumnSidecar => gossip_data_column_queue.len(), - WorkType::DelayedImportBlock => delayed_block_queue.len(), - WorkType::GossipVoluntaryExit => gossip_voluntary_exit_queue.len(), - WorkType::GossipProposerSlashing => gossip_proposer_slashing_queue.len(), - WorkType::GossipAttesterSlashing => gossip_attester_slashing_queue.len(), - WorkType::GossipSyncSignature => sync_message_queue.len(), - WorkType::GossipSyncContribution => sync_contribution_queue.len(), + WorkType::GossipBlock => work_queues.gossip_block_queue.len(), + WorkType::GossipBlobSidecar => work_queues.gossip_blob_queue.len(), + WorkType::GossipDataColumnSidecar => { + work_queues.gossip_data_column_queue.len() + } + WorkType::DelayedImportBlock => work_queues.delayed_block_queue.len(), + WorkType::GossipVoluntaryExit => { + work_queues.gossip_voluntary_exit_queue.len() + } + WorkType::GossipProposerSlashing => { + work_queues.gossip_proposer_slashing_queue.len() + } + WorkType::GossipAttesterSlashing => { + work_queues.gossip_attester_slashing_queue.len() + } + WorkType::GossipSyncSignature => work_queues.sync_message_queue.len(), + WorkType::GossipSyncContribution => { + work_queues.sync_contribution_queue.len() + } WorkType::GossipLightClientFinalityUpdate => { - lc_gossip_finality_update_queue.len() + work_queues.lc_gossip_finality_update_queue.len() } WorkType::GossipLightClientOptimisticUpdate => { - lc_gossip_optimistic_update_queue.len() + work_queues.lc_gossip_optimistic_update_queue.len() } - WorkType::RpcBlock => rpc_block_queue.len(), - WorkType::RpcBlobs | WorkType::IgnoredRpcBlock => rpc_blob_queue.len(), - WorkType::RpcCustodyColumn => rpc_custody_column_queue.len(), - WorkType::ColumnReconstruction => column_reconstruction_queue.len(), - WorkType::ChainSegment => chain_segment_queue.len(), - WorkType::ChainSegmentBackfill => backfill_chain_segment.len(), - WorkType::Status => status_queue.len(), - WorkType::BlocksByRangeRequest => block_brange_queue.len(), - WorkType::BlocksByRootsRequest => block_broots_queue.len(), - WorkType::BlobsByRangeRequest => blob_brange_queue.len(), - WorkType::BlobsByRootsRequest => blob_broots_queue.len(), - WorkType::DataColumnsByRootsRequest => dcbroots_queue.len(), - WorkType::DataColumnsByRangeRequest => dcbrange_queue.len(), + WorkType::RpcBlock => work_queues.rpc_block_queue.len(), + WorkType::RpcBlobs | WorkType::IgnoredRpcBlock => { + work_queues.rpc_blob_queue.len() + } + WorkType::RpcCustodyColumn => work_queues.rpc_custody_column_queue.len(), + WorkType::ColumnReconstruction => { + work_queues.column_reconstruction_queue.len() + } + WorkType::ChainSegment => work_queues.chain_segment_queue.len(), + WorkType::ChainSegmentBackfill => work_queues.backfill_chain_segment.len(), + WorkType::Status => work_queues.status_queue.len(), + WorkType::BlocksByRangeRequest => work_queues.blbrange_queue.len(), + WorkType::BlocksByRootsRequest => work_queues.blbroots_queue.len(), + WorkType::BlobsByRangeRequest => work_queues.bbrange_queue.len(), + WorkType::BlobsByRootsRequest => work_queues.bbroots_queue.len(), + WorkType::DataColumnsByRootsRequest => work_queues.dcbroots_queue.len(), + WorkType::DataColumnsByRangeRequest => work_queues.dcbrange_queue.len(), WorkType::GossipBlsToExecutionChange => { - gossip_bls_to_execution_change_queue.len() + work_queues.gossip_bls_to_execution_change_queue.len() + } + WorkType::LightClientBootstrapRequest => { + work_queues.lc_bootstrap_queue.len() } - WorkType::LightClientBootstrapRequest => lc_bootstrap_queue.len(), WorkType::LightClientOptimisticUpdateRequest => { - lc_rpc_optimistic_update_queue.len() + work_queues.lc_rpc_optimistic_update_queue.len() } WorkType::LightClientFinalityUpdateRequest => { - lc_rpc_finality_update_queue.len() + work_queues.lc_rpc_finality_update_queue.len() } - WorkType::LightClientUpdatesByRangeRequest => lc_update_range_queue.len(), - WorkType::ApiRequestP0 => api_request_p0_queue.len(), - WorkType::ApiRequestP1 => api_request_p1_queue.len(), + WorkType::LightClientUpdatesByRangeRequest => { + work_queues.lc_update_range_queue.len() + } + WorkType::ApiRequestP0 => work_queues.api_request_p0_queue.len(), + WorkType::ApiRequestP1 => work_queues.api_request_p1_queue.len(), WorkType::Reprocess => 0, }; metrics::observe_vec( @@ -1471,18 +1252,21 @@ impl BeaconProcessor { ); } - if aggregate_queue.is_full() && aggregate_debounce.elapsed() { + if work_queues.aggregate_queue.is_full() && work_queues.aggregate_debounce.elapsed() + { error!( msg = "the system has insufficient resources for load", - queue_len = aggregate_queue.max_length, + queue_len = work_queues.aggregate_queue.max_length, "Aggregate attestation queue full" ) } - if attestation_queue.is_full() && attestation_debounce.elapsed() { + if work_queues.attestation_queue.is_full() + && work_queues.attestation_debounce.elapsed() + { error!( msg = "the system has insufficient resources for load", - queue_len = attestation_queue.max_length, + queue_len = work_queues.attestation_queue.max_length, "Attestation queue full" ) } @@ -1730,21 +1514,3 @@ impl Drop for SendOnDrop { } } } - -#[cfg(test)] -mod tests { - use super::*; - use types::{BeaconState, ChainSpec, Eth1Data, ForkName, MainnetEthSpec}; - - #[test] - fn min_queue_len() { - // State with no validators. - let spec = ForkName::latest_stable().make_genesis_spec(ChainSpec::mainnet()); - let genesis_time = 0; - let state = BeaconState::::new(genesis_time, Eth1Data::default(), &spec); - assert_eq!(state.validators().len(), 0); - let queue_lengths = BeaconProcessorQueueLengths::from_state(&state, &spec).unwrap(); - assert_eq!(queue_lengths.attestation_queue, MIN_QUEUE_LEN); - assert_eq!(queue_lengths.unknown_block_attestation_queue, MIN_QUEUE_LEN); - } -} diff --git a/beacon_node/beacon_processor/src/scheduler/mod.rs b/beacon_node/beacon_processor/src/scheduler/mod.rs index e1a076a7c5..96393f9f3b 100644 --- a/beacon_node/beacon_processor/src/scheduler/mod.rs +++ b/beacon_node/beacon_processor/src/scheduler/mod.rs @@ -1 +1,4 @@ +pub mod work_queue; pub mod work_reprocessing_queue; + +pub use work_queue::BeaconProcessorQueueLengths; diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs new file mode 100644 index 0000000000..bfebbf1273 --- /dev/null +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -0,0 +1,388 @@ +use crate::Work; +use logging::TimeLatch; +use std::collections::VecDeque; +use tracing::error; +use types::{BeaconState, ChainSpec, EthSpec, RelativeEpoch}; + +/// Over-provision queues based on active validator count by some factor. The beacon chain has +/// strict churns that prevent the validator set size from changing rapidly. By over-provisioning +/// slightly, we don't need to adjust the queues during the lifetime of a process. +const ACTIVE_VALIDATOR_COUNT_OVERPROVISION_PERCENT: usize = 110; + +/// Minimum size of dynamically sized queues. Due to integer division we don't want 0 length queues +/// as the processor won't process that message type. 128 is an arbitrary value value >= 1 that +/// seems reasonable. +const MIN_QUEUE_LEN: usize = 128; + +/// A simple first-in-first-out queue with a maximum length. +pub struct FifoQueue { + queue: VecDeque, + max_length: usize, +} + +impl FifoQueue { + /// Create a new, empty queue with the given length. + pub fn new(max_length: usize) -> Self { + Self { + queue: VecDeque::default(), + max_length, + } + } + + /// Add a new item to the queue. + /// + /// Drops `item` if the queue is full. + pub fn push(&mut self, item: T, item_desc: &str) { + if self.queue.len() == self.max_length { + error!( + queue = item_desc, + queue_len = self.max_length, + msg = "the system has insufficient resources for load", + "Work queue is full", + ) + } else { + self.queue.push_back(item); + } + } + + /// Remove the next item from the queue. + pub fn pop(&mut self) -> Option { + self.queue.pop_front() + } + + /// Returns the current length of the queue. + pub fn len(&self) -> usize { + self.queue.len() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } +} + +/// A simple last-in-first-out queue with a maximum length. +pub struct LifoQueue { + queue: VecDeque, + pub max_length: usize, +} + +impl LifoQueue { + /// Create a new, empty queue with the given length. + pub fn new(max_length: usize) -> Self { + Self { + queue: VecDeque::default(), + max_length, + } + } + + /// Add a new item to the front of the queue. + /// + /// If the queue is full, the item at the back of the queue is dropped. + pub fn push(&mut self, item: T) { + if self.queue.len() == self.max_length { + self.queue.pop_back(); + } + self.queue.push_front(item); + } + + /// Remove the next item from the queue. + pub fn pop(&mut self) -> Option { + self.queue.pop_front() + } + + /// Returns `true` if the queue is full. + pub fn is_full(&self) -> bool { + self.queue.len() >= self.max_length + } + + /// Returns the current length of the queue. + pub fn len(&self) -> usize { + self.queue.len() + } + + pub fn is_empty(&self) -> bool { + self.queue.is_empty() + } +} + +/// Maximum number of queued items that will be stored before dropping them +pub struct BeaconProcessorQueueLengths { + aggregate_queue: usize, + attestation_queue: usize, + unknown_block_aggregate_queue: usize, + unknown_block_attestation_queue: usize, + sync_message_queue: usize, + sync_contribution_queue: usize, + gossip_voluntary_exit_queue: usize, + gossip_proposer_slashing_queue: usize, + gossip_attester_slashing_queue: usize, + unknown_light_client_update_queue: usize, + rpc_block_queue: usize, + rpc_blob_queue: usize, + rpc_custody_column_queue: usize, + column_reconstruction_queue: usize, + chain_segment_queue: usize, + backfill_chain_segment: usize, + gossip_block_queue: usize, + gossip_blob_queue: usize, + gossip_data_column_queue: usize, + delayed_block_queue: usize, + status_queue: usize, + bbrange_queue: usize, + bbroots_queue: usize, + blbroots_queue: usize, + blbrange_queue: usize, + dcbroots_queue: usize, + dcbrange_queue: usize, + gossip_bls_to_execution_change_queue: usize, + lc_bootstrap_queue: usize, + lc_rpc_optimistic_update_queue: usize, + lc_rpc_finality_update_queue: usize, + lc_gossip_finality_update_queue: usize, + lc_gossip_optimistic_update_queue: usize, + lc_update_range_queue: usize, + api_request_p0_queue: usize, + api_request_p1_queue: usize, +} + +impl BeaconProcessorQueueLengths { + pub fn from_state( + state: &BeaconState, + spec: &ChainSpec, + ) -> Result { + let active_validator_count = + match state.get_cached_active_validator_indices(RelativeEpoch::Current) { + Ok(indices) => indices.len(), + Err(_) => state + .get_active_validator_indices(state.current_epoch(), spec) + .map_err(|e| format!("Error computing active indices: {:?}", e))? + .len(), + }; + let active_validator_count = + (ACTIVE_VALIDATOR_COUNT_OVERPROVISION_PERCENT * active_validator_count) / 100; + let slots_per_epoch = E::slots_per_epoch() as usize; + + Ok(Self { + aggregate_queue: 4096, + unknown_block_aggregate_queue: 1024, + // Capacity for a full slot's worth of attestations if subscribed to all subnets + attestation_queue: std::cmp::max( + active_validator_count / slots_per_epoch, + MIN_QUEUE_LEN, + ), + // Capacity for a full slot's worth of attestations if subscribed to all subnets + unknown_block_attestation_queue: std::cmp::max( + active_validator_count / slots_per_epoch, + MIN_QUEUE_LEN, + ), + sync_message_queue: 2048, + sync_contribution_queue: 1024, + gossip_voluntary_exit_queue: 4096, + gossip_proposer_slashing_queue: 4096, + gossip_attester_slashing_queue: 4096, + unknown_light_client_update_queue: 128, + rpc_block_queue: 1024, + rpc_blob_queue: 1024, + // We don't request more than `PARENT_DEPTH_TOLERANCE` (32) lookups, so we can limit + // this queue size. With 48 max blobs per block, each column sidecar list could be up to 12MB. + rpc_custody_column_queue: 64, + column_reconstruction_queue: 1, + chain_segment_queue: 64, + backfill_chain_segment: 64, + gossip_block_queue: 1024, + gossip_blob_queue: 1024, + gossip_data_column_queue: 1024, + delayed_block_queue: 1024, + status_queue: 1024, + bbrange_queue: 1024, + bbroots_queue: 1024, + blbroots_queue: 1024, + blbrange_queue: 1024, + dcbroots_queue: 1024, + dcbrange_queue: 1024, + gossip_bls_to_execution_change_queue: 16384, + lc_gossip_finality_update_queue: 1024, + lc_gossip_optimistic_update_queue: 1024, + lc_bootstrap_queue: 1024, + lc_rpc_optimistic_update_queue: 512, + lc_rpc_finality_update_queue: 512, + lc_update_range_queue: 512, + api_request_p0_queue: 1024, + api_request_p1_queue: 1024, + }) + } +} + +pub struct WorkQueues { + pub aggregate_queue: LifoQueue>, + pub aggregate_debounce: TimeLatch, + pub attestation_queue: LifoQueue>, + pub attestation_to_convert_queue: LifoQueue>, + pub attestation_debounce: TimeLatch, + pub unknown_block_aggregate_queue: LifoQueue>, + pub unknown_block_attestation_queue: LifoQueue>, + pub sync_message_queue: LifoQueue>, + pub sync_contribution_queue: LifoQueue>, + pub gossip_voluntary_exit_queue: FifoQueue>, + pub gossip_proposer_slashing_queue: FifoQueue>, + pub gossip_attester_slashing_queue: FifoQueue>, + pub unknown_light_client_update_queue: FifoQueue>, + pub rpc_block_queue: FifoQueue>, + pub rpc_blob_queue: FifoQueue>, + pub rpc_custody_column_queue: FifoQueue>, + pub column_reconstruction_queue: LifoQueue>, + pub chain_segment_queue: FifoQueue>, + pub backfill_chain_segment: FifoQueue>, + pub gossip_block_queue: FifoQueue>, + pub gossip_blob_queue: FifoQueue>, + pub gossip_data_column_queue: FifoQueue>, + pub delayed_block_queue: FifoQueue>, + pub status_queue: FifoQueue>, + pub bbrange_queue: FifoQueue>, + pub bbroots_queue: FifoQueue>, + pub blbroots_queue: FifoQueue>, + pub blbrange_queue: FifoQueue>, + pub dcbroots_queue: FifoQueue>, + pub dcbrange_queue: FifoQueue>, + pub gossip_bls_to_execution_change_queue: FifoQueue>, + pub lc_gossip_finality_update_queue: FifoQueue>, + pub lc_gossip_optimistic_update_queue: FifoQueue>, + pub lc_bootstrap_queue: FifoQueue>, + pub lc_rpc_optimistic_update_queue: FifoQueue>, + pub lc_rpc_finality_update_queue: FifoQueue>, + pub lc_update_range_queue: FifoQueue>, + pub api_request_p0_queue: FifoQueue>, + pub api_request_p1_queue: FifoQueue>, +} + +impl WorkQueues { + pub fn new(queue_lengths: BeaconProcessorQueueLengths) -> Self { + // Using LIFO queues for attestations since validator profits rely upon getting fresh + // attestations into blocks. Additionally, later attestations contain more information than + // earlier ones, so we consider them more valuable. + let aggregate_queue = LifoQueue::new(queue_lengths.aggregate_queue); + let aggregate_debounce = TimeLatch::default(); + let attestation_queue = LifoQueue::new(queue_lengths.attestation_queue); + let attestation_to_convert_queue = LifoQueue::new(queue_lengths.attestation_queue); + let attestation_debounce = TimeLatch::default(); + let unknown_block_aggregate_queue = + LifoQueue::new(queue_lengths.unknown_block_aggregate_queue); + let unknown_block_attestation_queue = + LifoQueue::new(queue_lengths.unknown_block_attestation_queue); + + let sync_message_queue = LifoQueue::new(queue_lengths.sync_message_queue); + let sync_contribution_queue = LifoQueue::new(queue_lengths.sync_contribution_queue); + + // Using a FIFO queue for voluntary exits since it prevents exit censoring. I don't have + // a strong feeling about queue type for exits. + let gossip_voluntary_exit_queue = FifoQueue::new(queue_lengths.gossip_voluntary_exit_queue); + + // Using a FIFO queue for slashing to prevent people from flushing their slashings from the + // queues with lots of junk messages. + let gossip_proposer_slashing_queue = + FifoQueue::new(queue_lengths.gossip_proposer_slashing_queue); + let gossip_attester_slashing_queue = + FifoQueue::new(queue_lengths.gossip_attester_slashing_queue); + + // Using a FIFO queue for light client updates to maintain sequence order. + let unknown_light_client_update_queue = + FifoQueue::new(queue_lengths.unknown_light_client_update_queue); + // Using a FIFO queue since blocks need to be imported sequentially. + let rpc_block_queue = FifoQueue::new(queue_lengths.rpc_block_queue); + let rpc_blob_queue = FifoQueue::new(queue_lengths.rpc_blob_queue); + let rpc_custody_column_queue = FifoQueue::new(queue_lengths.rpc_custody_column_queue); + let column_reconstruction_queue = LifoQueue::new(queue_lengths.column_reconstruction_queue); + let chain_segment_queue = FifoQueue::new(queue_lengths.chain_segment_queue); + let backfill_chain_segment = FifoQueue::new(queue_lengths.backfill_chain_segment); + let gossip_block_queue = FifoQueue::new(queue_lengths.gossip_block_queue); + let gossip_blob_queue = FifoQueue::new(queue_lengths.gossip_blob_queue); + let gossip_data_column_queue = FifoQueue::new(queue_lengths.gossip_data_column_queue); + let delayed_block_queue = FifoQueue::new(queue_lengths.delayed_block_queue); + + let status_queue = FifoQueue::new(queue_lengths.status_queue); + let bbrange_queue = FifoQueue::new(queue_lengths.bbrange_queue); + let bbroots_queue = FifoQueue::new(queue_lengths.bbroots_queue); + let blbroots_queue = FifoQueue::new(queue_lengths.blbroots_queue); + let blbrange_queue = FifoQueue::new(queue_lengths.blbrange_queue); + let dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); + let dcbrange_queue = FifoQueue::new(queue_lengths.dcbrange_queue); + + let gossip_bls_to_execution_change_queue = + FifoQueue::new(queue_lengths.gossip_bls_to_execution_change_queue); + + let lc_gossip_optimistic_update_queue = + FifoQueue::new(queue_lengths.lc_gossip_optimistic_update_queue); + let lc_gossip_finality_update_queue = + FifoQueue::new(queue_lengths.lc_gossip_finality_update_queue); + let lc_bootstrap_queue = FifoQueue::new(queue_lengths.lc_bootstrap_queue); + let lc_rpc_optimistic_update_queue = + FifoQueue::new(queue_lengths.lc_rpc_optimistic_update_queue); + let lc_rpc_finality_update_queue = + FifoQueue::new(queue_lengths.lc_rpc_finality_update_queue); + let lc_update_range_queue: FifoQueue> = + FifoQueue::new(queue_lengths.lc_update_range_queue); + + let api_request_p0_queue = FifoQueue::new(queue_lengths.api_request_p0_queue); + let api_request_p1_queue = FifoQueue::new(queue_lengths.api_request_p1_queue); + + WorkQueues { + aggregate_queue, + aggregate_debounce, + attestation_queue, + attestation_to_convert_queue, + attestation_debounce, + unknown_block_aggregate_queue, + unknown_block_attestation_queue, + sync_message_queue, + sync_contribution_queue, + gossip_voluntary_exit_queue, + gossip_proposer_slashing_queue, + gossip_attester_slashing_queue, + unknown_light_client_update_queue, + rpc_block_queue, + rpc_blob_queue, + rpc_custody_column_queue, + chain_segment_queue, + column_reconstruction_queue, + backfill_chain_segment, + gossip_block_queue, + gossip_blob_queue, + gossip_data_column_queue, + delayed_block_queue, + status_queue, + bbrange_queue, + bbroots_queue, + blbroots_queue, + blbrange_queue, + dcbroots_queue, + dcbrange_queue, + gossip_bls_to_execution_change_queue, + lc_gossip_optimistic_update_queue, + lc_gossip_finality_update_queue, + lc_bootstrap_queue, + lc_rpc_optimistic_update_queue, + lc_rpc_finality_update_queue, + lc_update_range_queue, + api_request_p0_queue, + api_request_p1_queue, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use types::{BeaconState, ChainSpec, Eth1Data, ForkName, MainnetEthSpec}; + + #[test] + fn min_queue_len() { + // State with no validators. + let spec = ForkName::latest().make_genesis_spec(ChainSpec::mainnet()); + let genesis_time = 0; + let state = BeaconState::::new(genesis_time, Eth1Data::default(), &spec); + assert_eq!(state.validators().len(), 0); + let queue_lengths = BeaconProcessorQueueLengths::from_state(&state, &spec).unwrap(); + assert_eq!(queue_lengths.attestation_queue, MIN_QUEUE_LEN); + assert_eq!(queue_lengths.unknown_block_attestation_queue, MIN_QUEUE_LEN); + } +} From 3fac61e0c82b0679342295230c60699a0fc0133f Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 12 Jan 2026 17:36:01 -0600 Subject: [PATCH 09/18] Re-introduce clearer variable names in beacon processor work queue (#8649) #8648 Co-Authored-By: Eitan Seri-Levi --- beacon_node/beacon_processor/src/lib.rs | 24 +++++------ .../src/scheduler/work_queue.rs | 40 +++++++++---------- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 0a2665ab11..d9ae0e2345 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -921,13 +921,13 @@ impl BeaconProcessor { // and BlocksByRoot) } else if let Some(item) = work_queues.status_queue.pop() { Some(item) - } else if let Some(item) = work_queues.bbrange_queue.pop() { + } else if let Some(item) = work_queues.block_brange_queue.pop() { Some(item) - } else if let Some(item) = work_queues.bbroots_queue.pop() { + } else if let Some(item) = work_queues.block_broots_queue.pop() { Some(item) - } else if let Some(item) = work_queues.blbrange_queue.pop() { + } else if let Some(item) = work_queues.blob_brange_queue.pop() { Some(item) - } else if let Some(item) = work_queues.blbroots_queue.pop() { + } else if let Some(item) = work_queues.blob_broots_queue.pop() { Some(item) } else if let Some(item) = work_queues.dcbroots_queue.pop() { Some(item) @@ -1114,13 +1114,13 @@ impl BeaconProcessor { } Work::Status { .. } => work_queues.status_queue.push(work, work_id), Work::BlocksByRangeRequest { .. } => { - work_queues.bbrange_queue.push(work, work_id) + work_queues.block_brange_queue.push(work, work_id) } Work::BlocksByRootsRequest { .. } => { - work_queues.bbroots_queue.push(work, work_id) + work_queues.block_broots_queue.push(work, work_id) } Work::BlobsByRangeRequest { .. } => { - work_queues.blbrange_queue.push(work, work_id) + work_queues.blob_brange_queue.push(work, work_id) } Work::LightClientBootstrapRequest { .. } => { work_queues.lc_bootstrap_queue.push(work, work_id) @@ -1144,7 +1144,7 @@ impl BeaconProcessor { .gossip_bls_to_execution_change_queue .push(work, work_id), Work::BlobsByRootsRequest { .. } => { - work_queues.blbroots_queue.push(work, work_id) + work_queues.blob_broots_queue.push(work, work_id) } Work::DataColumnsByRootsRequest { .. } => { work_queues.dcbroots_queue.push(work, work_id) @@ -1220,10 +1220,10 @@ impl BeaconProcessor { WorkType::ChainSegment => work_queues.chain_segment_queue.len(), WorkType::ChainSegmentBackfill => work_queues.backfill_chain_segment.len(), WorkType::Status => work_queues.status_queue.len(), - WorkType::BlocksByRangeRequest => work_queues.blbrange_queue.len(), - WorkType::BlocksByRootsRequest => work_queues.blbroots_queue.len(), - WorkType::BlobsByRangeRequest => work_queues.bbrange_queue.len(), - WorkType::BlobsByRootsRequest => work_queues.bbroots_queue.len(), + WorkType::BlocksByRangeRequest => work_queues.block_brange_queue.len(), + WorkType::BlocksByRootsRequest => work_queues.block_broots_queue.len(), + WorkType::BlobsByRangeRequest => work_queues.blob_brange_queue.len(), + WorkType::BlobsByRootsRequest => work_queues.blob_broots_queue.len(), WorkType::DataColumnsByRootsRequest => work_queues.dcbroots_queue.len(), WorkType::DataColumnsByRangeRequest => work_queues.dcbrange_queue.len(), WorkType::GossipBlsToExecutionChange => { diff --git a/beacon_node/beacon_processor/src/scheduler/work_queue.rs b/beacon_node/beacon_processor/src/scheduler/work_queue.rs index bfebbf1273..c6f74961d1 100644 --- a/beacon_node/beacon_processor/src/scheduler/work_queue.rs +++ b/beacon_node/beacon_processor/src/scheduler/work_queue.rs @@ -128,10 +128,10 @@ pub struct BeaconProcessorQueueLengths { gossip_data_column_queue: usize, delayed_block_queue: usize, status_queue: usize, - bbrange_queue: usize, - bbroots_queue: usize, - blbroots_queue: usize, - blbrange_queue: usize, + block_brange_queue: usize, + block_broots_queue: usize, + blob_broots_queue: usize, + blob_brange_queue: usize, dcbroots_queue: usize, dcbrange_queue: usize, gossip_bls_to_execution_change_queue: usize, @@ -194,10 +194,10 @@ impl BeaconProcessorQueueLengths { gossip_data_column_queue: 1024, delayed_block_queue: 1024, status_queue: 1024, - bbrange_queue: 1024, - bbroots_queue: 1024, - blbroots_queue: 1024, - blbrange_queue: 1024, + block_brange_queue: 1024, + block_broots_queue: 1024, + blob_broots_queue: 1024, + blob_brange_queue: 1024, dcbroots_queue: 1024, dcbrange_queue: 1024, gossip_bls_to_execution_change_queue: 16384, @@ -238,10 +238,10 @@ pub struct WorkQueues { pub gossip_data_column_queue: FifoQueue>, pub delayed_block_queue: FifoQueue>, pub status_queue: FifoQueue>, - pub bbrange_queue: FifoQueue>, - pub bbroots_queue: FifoQueue>, - pub blbroots_queue: FifoQueue>, - pub blbrange_queue: FifoQueue>, + pub block_brange_queue: FifoQueue>, + pub block_broots_queue: FifoQueue>, + pub blob_broots_queue: FifoQueue>, + pub blob_brange_queue: FifoQueue>, pub dcbroots_queue: FifoQueue>, pub dcbrange_queue: FifoQueue>, pub gossip_bls_to_execution_change_queue: FifoQueue>, @@ -300,10 +300,10 @@ impl WorkQueues { let delayed_block_queue = FifoQueue::new(queue_lengths.delayed_block_queue); let status_queue = FifoQueue::new(queue_lengths.status_queue); - let bbrange_queue = FifoQueue::new(queue_lengths.bbrange_queue); - let bbroots_queue = FifoQueue::new(queue_lengths.bbroots_queue); - let blbroots_queue = FifoQueue::new(queue_lengths.blbroots_queue); - let blbrange_queue = FifoQueue::new(queue_lengths.blbrange_queue); + let block_brange_queue = FifoQueue::new(queue_lengths.block_brange_queue); + let block_broots_queue = FifoQueue::new(queue_lengths.block_broots_queue); + let blob_broots_queue = FifoQueue::new(queue_lengths.blob_broots_queue); + let blob_brange_queue = FifoQueue::new(queue_lengths.blob_brange_queue); let dcbroots_queue = FifoQueue::new(queue_lengths.dcbroots_queue); let dcbrange_queue = FifoQueue::new(queue_lengths.dcbrange_queue); @@ -350,10 +350,10 @@ impl WorkQueues { gossip_data_column_queue, delayed_block_queue, status_queue, - bbrange_queue, - bbroots_queue, - blbroots_queue, - blbrange_queue, + block_brange_queue, + block_broots_queue, + blob_broots_queue, + blob_brange_queue, dcbroots_queue, dcbrange_queue, gossip_bls_to_execution_change_queue, From 57bbc93d755a1fb0cd5b1db012114967258b971b Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 13 Jan 2026 11:28:34 +0530 Subject: [PATCH 10/18] Update buckets for metric (#8651) N/A The `beacon_data_column_sidecar_computation_seconds` used to record the full kzg proof generation times before we changed getBlobsV2 to just return the full proofs + cells. This metric should be taking way less time than 100ms which was the minimum bucket previously. Update the metric to use the default buckets for better granularity. Co-Authored-By: Pawan Dhananjay --- beacon_node/beacon_chain/src/metrics.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index cf295d9951..d6b76a6f94 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1613,10 +1613,9 @@ pub static BLOB_SIDECAR_INCLUSION_PROOF_COMPUTATION: LazyLock> ) }); pub static DATA_COLUMN_SIDECAR_COMPUTATION: LazyLock> = LazyLock::new(|| { - try_create_histogram_vec_with_buckets( + try_create_histogram_vec( "beacon_data_column_sidecar_computation_seconds", "Time taken to compute data column sidecar, including cells, proofs and inclusion proof", - Ok(vec![0.1, 0.15, 0.25, 0.35, 0.5, 0.7, 1.0, 2.5, 5.0, 10.0]), &["blob_count"], ) }); From c91345782a79d33408dc3c39698207a01116b99a Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 13 Jan 2026 13:20:40 +0530 Subject: [PATCH 11/18] Get blobs v2 metrics (#8641) N/A Add standardized metrics for getBlobsV2 from https://github.com/ethereum/beacon-metrics/pull/14. Co-Authored-By: Pawan Dhananjay --- .../beacon_chain/src/fetch_blobs/mod.rs | 11 ++++++++ beacon_node/beacon_chain/src/metrics.rs | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 4c6b2d10a9..3a1cf146ed 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -247,6 +247,12 @@ async fn fetch_and_process_blobs_v2( metrics::observe(&metrics::BLOBS_FROM_EL_EXPECTED, num_expected_blobs as f64); debug!(num_expected_blobs, "Fetching blobs from the EL"); + + // Track request count and duration for standardized metrics + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V2_REQUESTS_TOTAL); + let _timer = + metrics::start_timer(&metrics::BEACON_ENGINE_GET_BLOBS_V2_REQUEST_DURATION_SECONDS); + let response = chain_adapter .get_blobs_v2(versioned_hashes) .await @@ -254,6 +260,11 @@ async fn fetch_and_process_blobs_v2( inc_counter(&metrics::BLOBS_FROM_EL_ERROR_TOTAL); })?; + drop(_timer); + + // Track successful response + inc_counter(&metrics::BEACON_ENGINE_GET_BLOBS_V2_RESPONSES_TOTAL); + let Some(blobs_and_proofs) = response else { debug!(num_expected_blobs, "No blobs fetched from the EL"); inc_counter(&metrics::BLOBS_FROM_EL_MISS_TOTAL); diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index d6b76a6f94..6be07faa24 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1689,6 +1689,33 @@ pub static BLOBS_FROM_EL_RECEIVED: LazyLock> = LazyLock::new(| ) }); +/* + * Standardized getBlobs metrics across clients from https://github.com/ethereum/beacon-metrics + */ +pub static BEACON_ENGINE_GET_BLOBS_V2_REQUESTS_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_engine_getBlobsV2_requests_total", + "Total number of engine_getBlobsV2 requests made to the execution layer", + ) + }); + +pub static BEACON_ENGINE_GET_BLOBS_V2_RESPONSES_TOTAL: LazyLock> = + LazyLock::new(|| { + try_create_int_counter( + "beacon_engine_getBlobsV2_responses_total", + "Total number of successful engine_getBlobsV2 responses from the execution layer", + ) + }); + +pub static BEACON_ENGINE_GET_BLOBS_V2_REQUEST_DURATION_SECONDS: LazyLock> = + LazyLock::new(|| { + try_create_histogram( + "beacon_engine_getBlobsV2_request_duration_seconds", + "Duration of engine_getBlobsV2 requests to the execution layer in seconds", + ) + }); + /* * Light server message verification */ From f584521e85fb76658faa9f2f87c93dad945a5520 Mon Sep 17 00:00:00 2001 From: Jimmy Chu <898091+jimmychu0807@users.noreply.github.com> Date: Wed, 14 Jan 2026 14:25:07 +0800 Subject: [PATCH 12/18] chore(validator_client): Read genesis time and genesis validators root from eth2_network_config (#8638) #5019 If there is a known eth2_network_config, we read the genesis time and validators root from the config. Co-Authored-By: Jimmy Chu <898091+jimmychu0807@users.noreply.github.com> --- common/eth2_config/src/lib.rs | 10 ++++++++++ common/eth2_network_config/src/lib.rs | 10 ++++++++++ validator_client/src/lib.rs | 21 ++++++++++++++++----- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/common/eth2_config/src/lib.rs b/common/eth2_config/src/lib.rs index 544138f0fa..2c06fe146a 100644 --- a/common/eth2_config/src/lib.rs +++ b/common/eth2_config/src/lib.rs @@ -33,6 +33,8 @@ const HOLESKY_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url checksum: "0xd750639607c337bbb192b15c27f447732267bf72d1650180a0e44c2d93a80741", genesis_validators_root: "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1", genesis_state_root: "0x0ea3f6f9515823b59c863454675fefcd1d8b4f2dbe454db166206a41fda060a0", + // ref: https://github.com/eth-clients/holesky/blob/main/README.md - Launch Epoch time + genesis_time: 1695902400, }; const HOODI_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url { @@ -44,6 +46,8 @@ const HOODI_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url { checksum: "0x7f42257ef69e055496c964a753bb07e54001ccd57ab467ef72d67af086bcfce7", genesis_validators_root: "0x212f13fc4df078b6cb7db228f1c8307566dcecf900867401a92023d7ba99cb5f", genesis_state_root: "0x2683ebc120f91f740c7bed4c866672d01e1ba51b4cc360297138465ee5df40f0", + // ref: https://github.com/eth-clients/hoodi/blob/main/README.md - Launch Epoch time + genesis_time: 1742213400, }; const CHIADO_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url { @@ -52,6 +56,8 @@ const CHIADO_GENESIS_STATE_SOURCE: GenesisStateSource = GenesisStateSource::Url checksum: "0xd4a039454c7429f1dfaa7e11e397ef3d0f50d2d5e4c0e4dc04919d153aa13af1", genesis_validators_root: "0x9d642dac73058fbf39c0ae41ab1e34e4d889043cb199851ded7095bc99eb4c1e", genesis_state_root: "0xa48419160f8f146ecaa53d12a5d6e1e6af414a328afdc56b60d5002bb472a077", + // ref: https://github.com/gnosischain/configs/blob/main/chiado/genesis.ssz + genesis_time: 1665396300, }; /// The core configuration of a Lighthouse beacon node. @@ -117,6 +123,10 @@ pub enum GenesisStateSource { /// /// The format should be 0x-prefixed ASCII bytes. genesis_state_root: &'static str, + /// The genesis time. + /// + /// The format should be u64. + genesis_time: u64, }, } diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 16ee45e524..6fd8567bed 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -133,6 +133,16 @@ impl Eth2NetworkConfig { self.genesis_state_source != GenesisStateSource::Unknown } + /// The `genesis_time` of the genesis state. + pub fn genesis_time(&self) -> Result, String> { + if let GenesisStateSource::Url { genesis_time, .. } = self.genesis_state_source { + Ok(Some(genesis_time)) + } else { + self.get_genesis_state_from_bytes::() + .map(|state| Some(state.genesis_time())) + } + } + /// The `genesis_validators_root` of the genesis state. pub fn genesis_validators_root(&self) -> Result, String> { if let GenesisStateSource::Url { diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index fe6a93ae9d..b3cd3425f3 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -364,11 +364,22 @@ impl ProductionValidatorClient { context.eth2_config.spec.clone(), ); - // Perform some potentially long-running initialization tasks. - let (genesis_time, genesis_validators_root) = tokio::select! { - tuple = init_from_beacon_node::(&beacon_nodes, &proposer_nodes) => tuple?, - () = context.executor.exit() => return Err("Shutting down".to_string()) - }; + let (genesis_time, genesis_validators_root) = + if let Some(eth2_network_config) = context.eth2_network_config.as_ref() { + let time = eth2_network_config + .genesis_time::()? + .ok_or("no genesis time")?; + let root = eth2_network_config + .genesis_validators_root::()? + .ok_or("no genesis validators root")?; + (time, root) + } else { + // Perform some potentially long-running initialization tasks. + tokio::select! { + tuple = init_from_beacon_node::(&beacon_nodes, &proposer_nodes) => tuple?, + () = context.executor.exit() => return Err("Shutting down".to_string()), + } + }; // Update the metrics server. if let Some(ctx) = &validator_metrics_ctx { From 605ef8e8e6cbb9bc974b767bb8a35a445d5a1936 Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 15 Jan 2026 06:16:40 +0400 Subject: [PATCH 13/18] Remove `state` dependency from `core` module in `consensus/types` (#8653) #8652 - This removes instances of `BeaconStateError` from `eth_spec.rs`, and replaces them directly with `ArithError` which can be trivially converted back to `BeaconStateError` at the call site. - Also moves the state related methods on `ChainSpec` to be methods on `BeaconState` instead. I think this might be a more natural place for them to exist anyway. Co-Authored-By: Mac L --- .../beacon_chain/src/beacon_block_reward.rs | 4 +- beacon_node/beacon_chain/tests/rewards.rs | 8 ++-- .../src/common/slash_validator.rs | 7 ++- .../src/per_epoch_processing/single_pass.rs | 2 +- .../src/per_epoch_processing/slashings.rs | 2 +- consensus/types/src/core/chain_spec.rs | 46 ------------------- consensus/types/src/core/eth_spec.rs | 11 ++--- consensus/types/src/state/beacon_state.rs | 36 +++++++++++++++ consensus/types/src/state/committee_cache.rs | 3 +- 9 files changed, 53 insertions(+), 66 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_reward.rs b/beacon_node/beacon_chain/src/beacon_block_reward.rs index ac4ed2ab67..88686696ed 100644 --- a/beacon_node/beacon_chain/src/beacon_block_reward.rs +++ b/beacon_node/beacon_chain/src/beacon_block_reward.rs @@ -135,7 +135,7 @@ impl BeaconChain { state .get_validator(proposer_slashing.proposer_index() as usize)? .effective_balance - .safe_div(self.spec.whistleblower_reward_quotient_for_state(state))?, + .safe_div(state.get_whistleblower_reward_quotient(&self.spec))?, )?; } @@ -157,7 +157,7 @@ impl BeaconChain { state .get_validator(attester_index as usize)? .effective_balance - .safe_div(self.spec.whistleblower_reward_quotient_for_state(state))?, + .safe_div(state.get_whistleblower_reward_quotient(&self.spec))?, )?; } } diff --git a/beacon_node/beacon_chain/tests/rewards.rs b/beacon_node/beacon_chain/tests/rewards.rs index ee9cf511ea..94ad97c963 100644 --- a/beacon_node/beacon_chain/tests/rewards.rs +++ b/beacon_node/beacon_chain/tests/rewards.rs @@ -269,16 +269,16 @@ async fn test_rewards_electra_slashings() { harness.add_attester_slashing(vec![0]).unwrap(); let slashed_balance_1 = initial_balances.get_mut(0).unwrap(); let validator_1_effective_balance = state.get_effective_balance(0).unwrap(); - let delta_1 = validator_1_effective_balance - / harness.spec.min_slashing_penalty_quotient_for_state(&state); + let delta_1 = + validator_1_effective_balance / state.get_min_slashing_penalty_quotient(&harness.spec); *slashed_balance_1 -= delta_1; // add a proposer slashing and calculating slashing penalties harness.add_proposer_slashing(1).unwrap(); let slashed_balance_2 = initial_balances.get_mut(1).unwrap(); let validator_2_effective_balance = state.get_effective_balance(1).unwrap(); - let delta_2 = validator_2_effective_balance - / harness.spec.min_slashing_penalty_quotient_for_state(&state); + let delta_2 = + validator_2_effective_balance / state.get_min_slashing_penalty_quotient(&harness.spec); *slashed_balance_2 -= delta_2; check_all_electra_rewards(&harness, initial_balances).await; diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 01c1855fb1..ac7379abd9 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -42,8 +42,7 @@ pub fn slash_validator( decrease_balance( state, slashed_index, - validator_effective_balance - .safe_div(spec.min_slashing_penalty_quotient_for_state(state))?, + validator_effective_balance.safe_div(state.get_min_slashing_penalty_quotient(spec))?, )?; update_progressive_balances_on_slashing(state, slashed_index, validator_effective_balance)?; @@ -54,8 +53,8 @@ pub fn slash_validator( // Apply proposer and whistleblower rewards let proposer_index = ctxt.get_proposer_index(state, spec)? as usize; let whistleblower_index = opt_whistleblower_index.unwrap_or(proposer_index); - let whistleblower_reward = validator_effective_balance - .safe_div(spec.whistleblower_reward_quotient_for_state(state))?; + let whistleblower_reward = + validator_effective_balance.safe_div(state.get_whistleblower_reward_quotient(spec))?; let proposer_reward = if state.fork_name_unchecked().altair_enabled() { whistleblower_reward .safe_mul(PROPOSER_WEIGHT)? diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 914e025f2f..3e07803aa6 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -886,7 +886,7 @@ impl SlashingsContext { ) -> Result { let sum_slashings = state.get_all_slashings().iter().copied().safe_sum()?; let adjusted_total_slashing_balance = min( - sum_slashings.safe_mul(spec.proportional_slashing_multiplier_for_state(state))?, + sum_slashings.safe_mul(state.get_proportional_slashing_multiplier(spec))?, state_ctxt.total_active_balance, ); diff --git a/consensus/state_processing/src/per_epoch_processing/slashings.rs b/consensus/state_processing/src/per_epoch_processing/slashings.rs index 6008276d15..cf5e82663c 100644 --- a/consensus/state_processing/src/per_epoch_processing/slashings.rs +++ b/consensus/state_processing/src/per_epoch_processing/slashings.rs @@ -17,7 +17,7 @@ pub fn process_slashings( let sum_slashings = state.get_all_slashings().iter().copied().safe_sum()?; let adjusted_total_slashing_balance = std::cmp::min( - sum_slashings.safe_mul(spec.proportional_slashing_multiplier_for_state(state))?, + sum_slashings.safe_mul(state.get_proportional_slashing_multiplier(spec))?, total_balance, ); diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index da3f9b90cc..f47b4b4c91 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -19,7 +19,6 @@ use crate::{ data::{BlobIdentifier, DataColumnSubnetId, DataColumnsByRootIdentifier}, execution::ExecutionBlockHash, fork::{Fork, ForkData, ForkName}, - state::BeaconState, }; /// Each of the BLS signature domains. @@ -418,51 +417,6 @@ impl ChainSpec { } } - /// For a given `BeaconState`, return the proportional slashing multiplier associated with its variant. - pub fn proportional_slashing_multiplier_for_state( - &self, - state: &BeaconState, - ) -> u64 { - let fork_name = state.fork_name_unchecked(); - if fork_name >= ForkName::Bellatrix { - self.proportional_slashing_multiplier_bellatrix - } else if fork_name >= ForkName::Altair { - self.proportional_slashing_multiplier_altair - } else { - self.proportional_slashing_multiplier - } - } - - /// For a given `BeaconState`, return the minimum slashing penalty quotient associated with its variant. - pub fn min_slashing_penalty_quotient_for_state( - &self, - state: &BeaconState, - ) -> u64 { - let fork_name = state.fork_name_unchecked(); - if fork_name.electra_enabled() { - self.min_slashing_penalty_quotient_electra - } else if fork_name >= ForkName::Bellatrix { - self.min_slashing_penalty_quotient_bellatrix - } else if fork_name >= ForkName::Altair { - self.min_slashing_penalty_quotient_altair - } else { - self.min_slashing_penalty_quotient - } - } - - /// For a given `BeaconState`, return the whistleblower reward quotient associated with its variant. - pub fn whistleblower_reward_quotient_for_state( - &self, - state: &BeaconState, - ) -> u64 { - let fork_name = state.fork_name_unchecked(); - if fork_name.electra_enabled() { - self.whistleblower_reward_quotient_electra - } else { - self.whistleblower_reward_quotient - } - } - pub fn max_effective_balance_for_fork(&self, fork_name: ForkName) -> u64 { if fork_name.electra_enabled() { self.max_effective_balance_electra diff --git a/consensus/types/src/core/eth_spec.rs b/consensus/types/src/core/eth_spec.rs index 74795fdfc3..2983fc2c62 100644 --- a/consensus/types/src/core/eth_spec.rs +++ b/consensus/types/src/core/eth_spec.rs @@ -3,7 +3,7 @@ use std::{ str::FromStr, }; -use safe_arith::SafeArith; +use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use typenum::{ U0, U1, U2, U4, U8, U16, U17, U32, U64, U128, U256, U512, U625, U1024, U2048, U4096, U8192, @@ -11,10 +11,7 @@ use typenum::{ U1099511627776, UInt, Unsigned, bit::B0, }; -use crate::{ - core::{ChainSpec, Epoch}, - state::BeaconStateError, -}; +use crate::core::{ChainSpec, Epoch}; type U5000 = UInt, B0>, B0>; // 625 * 8 = 5000 @@ -196,7 +193,7 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + fn get_committee_count_per_slot( active_validator_count: usize, spec: &ChainSpec, - ) -> Result { + ) -> Result { Self::get_committee_count_per_slot_with( active_validator_count, spec.max_committees_per_slot, @@ -208,7 +205,7 @@ pub trait EthSpec: 'static + Default + Sync + Send + Clone + Debug + PartialEq + active_validator_count: usize, max_committees_per_slot: usize, target_committee_size: usize, - ) -> Result { + ) -> Result { let slots_per_epoch = Self::SlotsPerEpoch::to_usize(); Ok(std::cmp::max( diff --git a/consensus/types/src/state/beacon_state.rs b/consensus/types/src/state/beacon_state.rs index c1b6f0dc0c..f3e5ae411b 100644 --- a/consensus/types/src/state/beacon_state.rs +++ b/consensus/types/src/state/beacon_state.rs @@ -2528,6 +2528,42 @@ impl BeaconState { self.epoch_cache().get_base_reward(validator_index) } + /// Get the proportional slashing multiplier for the current fork. + pub fn get_proportional_slashing_multiplier(&self, spec: &ChainSpec) -> u64 { + let fork_name = self.fork_name_unchecked(); + if fork_name >= ForkName::Bellatrix { + spec.proportional_slashing_multiplier_bellatrix + } else if fork_name >= ForkName::Altair { + spec.proportional_slashing_multiplier_altair + } else { + spec.proportional_slashing_multiplier + } + } + + /// Get the minimum slashing penalty quotient for the current fork. + pub fn get_min_slashing_penalty_quotient(&self, spec: &ChainSpec) -> u64 { + let fork_name = self.fork_name_unchecked(); + if fork_name.electra_enabled() { + spec.min_slashing_penalty_quotient_electra + } else if fork_name >= ForkName::Bellatrix { + spec.min_slashing_penalty_quotient_bellatrix + } else if fork_name >= ForkName::Altair { + spec.min_slashing_penalty_quotient_altair + } else { + spec.min_slashing_penalty_quotient + } + } + + /// Get the whistleblower reward quotient for the current fork. + pub fn get_whistleblower_reward_quotient(&self, spec: &ChainSpec) -> u64 { + let fork_name = self.fork_name_unchecked(); + if fork_name.electra_enabled() { + spec.whistleblower_reward_quotient_electra + } else { + spec.whistleblower_reward_quotient + } + } + // ******* Electra accessors ******* /// Return the churn limit for the current epoch. diff --git a/consensus/types/src/state/committee_cache.rs b/consensus/types/src/state/committee_cache.rs index 15f6a4cd37..39e9011ef4 100644 --- a/consensus/types/src/state/committee_cache.rs +++ b/consensus/types/src/state/committee_cache.rs @@ -100,7 +100,8 @@ impl CommitteeCache { } let committees_per_slot = - E::get_committee_count_per_slot(active_validator_indices.len(), spec)? as u64; + E::get_committee_count_per_slot(active_validator_indices.len(), spec) + .map_err(BeaconStateError::ArithError)? as u64; let seed = state.get_seed(epoch, Domain::BeaconAttester, spec)?; From 1abc41e337c1f08656dac526c84f929cf9a5a130 Mon Sep 17 00:00:00 2001 From: Mac L Date: Thu, 15 Jan 2026 06:23:55 +0400 Subject: [PATCH 14/18] Cleanup `consensus/types` re-exports (#8643) Removes some of the temporary re-exports in `consensus/types`. I am doing this in multiple parts to keep each diff small. Co-Authored-By: Mac L --- beacon_node/beacon_chain/src/beacon_chain.rs | 2 +- beacon_node/beacon_chain/src/errors.rs | 4 ++-- .../beacon_chain/src/execution_payload.rs | 2 +- .../src/naive_aggregation_pool.rs | 2 +- .../beacon_chain/src/observed_aggregates.rs | 2 +- .../beacon_chain/src/observed_attesters.rs | 2 +- .../src/sync_committee_verification.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 2 +- beacon_node/execution_layer/src/lib.rs | 2 +- beacon_node/http_api/src/beacon/states.rs | 4 +--- beacon_node/http_api/src/produce_block.rs | 2 +- beacon_node/http_api/src/sync_committees.rs | 4 ++-- beacon_node/http_api/tests/tests.rs | 2 +- beacon_node/operation_pool/src/lib.rs | 4 ++-- .../state_processing/src/common/altair.rs | 2 +- .../src/common/get_attesting_indices.rs | 2 +- consensus/state_processing/src/genesis.rs | 4 ++-- .../src/per_block_processing.rs | 2 +- .../src/per_block_processing/deneb.rs | 2 +- consensus/types/src/lib.rs | 21 ------------------- .../src/test_rig.rs | 2 +- 21 files changed, 24 insertions(+), 47 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index a6be3dde8c..2f25c7728a 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -134,7 +134,7 @@ use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace use tree_hash::TreeHash; use types::blob_sidecar::FixedBlobSidecarList; use types::data_column_sidecar::ColumnIndex; -use types::payload::BlockProductionVersion; +use types::execution::BlockProductionVersion; use types::*; pub type ForkChoiceError = fork_choice::Error; diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 7fa6a0b510..816e75fd24 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -219,7 +219,7 @@ pub enum BeaconChainError { UnableToPublish, UnableToBuildColumnSidecar(String), AvailabilityCheckError(AvailabilityCheckError), - LightClientUpdateError(LightClientUpdateError), + LightClientError(LightClientError), LightClientBootstrapError(String), UnsupportedFork, MilhouseError(MilhouseError), @@ -274,7 +274,7 @@ easy_from_to!(BlockReplayError, BeaconChainError); easy_from_to!(InconsistentFork, BeaconChainError); easy_from_to!(AvailabilityCheckError, BeaconChainError); easy_from_to!(EpochCacheError, BeaconChainError); -easy_from_to!(LightClientUpdateError, BeaconChainError); +easy_from_to!(LightClientError, BeaconChainError); easy_from_to!(MilhouseError, BeaconChainError); easy_from_to!(AttestationError, BeaconChainError); diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index f0cab06ca3..9459b1acd7 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -26,7 +26,7 @@ use std::sync::Arc; use tokio::task::JoinHandle; use tracing::{Instrument, debug, debug_span, warn}; use tree_hash::TreeHash; -use types::payload::BlockProductionVersion; +use types::execution::BlockProductionVersion; use types::*; pub type PreparePayloadResult = Result, BlockProductionError>; diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index beefc2d678..e8f1108657 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -4,8 +4,8 @@ use itertools::Itertools; use smallvec::SmallVec; use std::collections::HashMap; use tree_hash::{MerkleHasher, TreeHash, TreeHashType}; +use types::SlotData; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; -use types::slot_data::SlotData; use types::sync_committee_contribution::SyncContributionData; use types::{ Attestation, AttestationData, AttestationRef, CommitteeIndex, EthSpec, Hash256, Slot, diff --git a/beacon_node/beacon_chain/src/observed_aggregates.rs b/beacon_node/beacon_chain/src/observed_aggregates.rs index b2c5cb4b38..7ecd581e85 100644 --- a/beacon_node/beacon_chain/src/observed_aggregates.rs +++ b/beacon_node/beacon_chain/src/observed_aggregates.rs @@ -7,10 +7,10 @@ use std::collections::HashMap; use std::marker::PhantomData; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use types::SlotData; use types::consts::altair::{ SYNC_COMMITTEE_SUBNET_COUNT, TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE, }; -use types::slot_data::SlotData; use types::{ Attestation, AttestationData, AttestationRef, EthSpec, Hash256, Slot, SyncCommitteeContribution, }; diff --git a/beacon_node/beacon_chain/src/observed_attesters.rs b/beacon_node/beacon_chain/src/observed_attesters.rs index d5433f49d1..277bf38ffc 100644 --- a/beacon_node/beacon_chain/src/observed_attesters.rs +++ b/beacon_node/beacon_chain/src/observed_attesters.rs @@ -20,7 +20,7 @@ use std::collections::{HashMap, HashSet}; use std::hash::Hash; use std::marker::PhantomData; use typenum::Unsigned; -use types::slot_data::SlotData; +use types::SlotData; use types::{Epoch, EthSpec, Hash256, Slot}; /// The maximum capacity of the `AutoPruningEpochContainer`. diff --git a/beacon_node/beacon_chain/src/sync_committee_verification.rs b/beacon_node/beacon_chain/src/sync_committee_verification.rs index e74e284e58..aacbb8cfec 100644 --- a/beacon_node/beacon_chain/src/sync_committee_verification.rs +++ b/beacon_node/beacon_chain/src/sync_committee_verification.rs @@ -48,8 +48,8 @@ use strum::AsRefStr; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::ChainSpec; +use types::SlotData; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; -use types::slot_data::SlotData; use types::sync_committee::SyncCommitteeError; use types::{ BeaconStateError, EthSpec, Hash256, SignedContributionAndProof, Slot, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 6d17d6d85c..0a226d0f76 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -68,8 +68,8 @@ use task_executor::{ShutdownReason, test_utils::TestRuntime}; use tree_hash::TreeHash; use typenum::U4294967296; use types::data_column_custody_group::CustodyIndex; +use types::execution::BlockProductionVersion; use types::indexed_attestation::IndexedAttestationBase; -use types::payload::BlockProductionVersion; use types::test_utils::TestRandom; pub use types::test_utils::generate_deterministic_keypairs; use types::*; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 554668dd8a..37b514f480 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -47,8 +47,8 @@ use tracing::{Instrument, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::builder_bid::BuilderBid; +use types::execution::BlockProductionVersion; use types::non_zero_usize::new_non_zero_usize; -use types::payload::BlockProductionVersion; use types::{ AbstractExecPayload, BlobsList, ExecutionPayloadDeneb, ExecutionRequests, KzgProofs, SignedBlindedBeaconBlock, diff --git a/beacon_node/http_api/src/beacon/states.rs b/beacon_node/http_api/src/beacon/states.rs index 6d06bcc77d..828efb86a7 100644 --- a/beacon_node/http_api/src/beacon/states.rs +++ b/beacon_node/http_api/src/beacon/states.rs @@ -12,9 +12,7 @@ use eth2::types::{ ValidatorsRequestBody, }; use std::sync::Arc; -use types::{ - AttestationShufflingId, CommitteeCache, Error as BeaconStateError, EthSpec, RelativeEpoch, -}; +use types::{AttestationShufflingId, BeaconStateError, CommitteeCache, EthSpec, RelativeEpoch}; use warp::filters::BoxedFilter; use warp::{Filter, Reply}; use warp_utils::query::multi_key_query; diff --git a/beacon_node/http_api/src/produce_block.rs b/beacon_node/http_api/src/produce_block.rs index 3bd0cec7e3..ee92702fb7 100644 --- a/beacon_node/http_api/src/produce_block.rs +++ b/beacon_node/http_api/src/produce_block.rs @@ -16,7 +16,7 @@ use lighthouse_tracing::{SPAN_PRODUCE_BLOCK_V2, SPAN_PRODUCE_BLOCK_V3}; use ssz::Encode; use std::sync::Arc; use tracing::instrument; -use types::{payload::BlockProductionVersion, *}; +use types::{execution::BlockProductionVersion, *}; use warp::{ Reply, hyper::{Body, Response}, diff --git a/beacon_node/http_api/src/sync_committees.rs b/beacon_node/http_api/src/sync_committees.rs index b9fa24ad6a..6e2f4c9585 100644 --- a/beacon_node/http_api/src/sync_committees.rs +++ b/beacon_node/http_api/src/sync_committees.rs @@ -17,8 +17,8 @@ use std::collections::HashMap; use tokio::sync::mpsc::UnboundedSender; use tracing::{debug, error, warn}; use types::{ - BeaconStateError, Epoch, EthSpec, SignedContributionAndProof, SyncCommitteeMessage, SyncDuty, - SyncSubnetId, slot_data::SlotData, + BeaconStateError, Epoch, EthSpec, SignedContributionAndProof, SlotData, SyncCommitteeMessage, + SyncDuty, SyncSubnetId, }; /// The struct that is returned to the requesting HTTP client. diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 72e2186959..8a79541363 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -44,7 +44,7 @@ use std::convert::TryInto; use std::sync::Arc; use tokio::time::Duration; use tree_hash::TreeHash; -use types::application_domain::ApplicationDomain; +use types::ApplicationDomain; use types::{ Domain, EthSpec, ExecutionBlockHash, Hash256, MainnetEthSpec, RelativeEpoch, SelectionProof, SignedRoot, SingleAttestation, Slot, attestation::AttestationBase, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 00361450a5..81423d6abd 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -39,8 +39,8 @@ use typenum::Unsigned; use types::{ AbstractExecPayload, Attestation, AttestationData, AttesterSlashing, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ProposerSlashing, SignedBeaconBlock, - SignedBlsToExecutionChange, SignedVoluntaryExit, Slot, SyncAggregate, - SyncCommitteeContribution, Validator, sync_aggregate::Error as SyncAggregateError, + SignedBlsToExecutionChange, SignedVoluntaryExit, Slot, SyncAggregate, SyncAggregateError, + SyncCommitteeContribution, Validator, }; type SyncContributions = RwLock>>>; diff --git a/consensus/state_processing/src/common/altair.rs b/consensus/state_processing/src/common/altair.rs index 4380154133..dae6b0302d 100644 --- a/consensus/state_processing/src/common/altair.rs +++ b/consensus/state_processing/src/common/altair.rs @@ -28,7 +28,7 @@ pub fn get_base_reward( validator_effective_balance: u64, base_reward_per_increment: BaseRewardPerIncrement, spec: &ChainSpec, -) -> Result { +) -> Result { validator_effective_balance .safe_div(spec.effective_balance_increment)? .safe_mul(base_reward_per_increment.as_u64()) diff --git a/consensus/state_processing/src/common/get_attesting_indices.rs b/consensus/state_processing/src/common/get_attesting_indices.rs index dc7be7c251..ff9f60c5ea 100644 --- a/consensus/state_processing/src/common/get_attesting_indices.rs +++ b/consensus/state_processing/src/common/get_attesting_indices.rs @@ -107,7 +107,7 @@ pub mod attesting_indices_electra { for committee_index in committee_indices { let beacon_committee = committees .get(committee_index as usize) - .ok_or(Error::NoCommitteeFound(committee_index))?; + .ok_or(BeaconStateError::NoCommitteeFound(committee_index))?; // This check is new to the spec's `process_attestation` in Electra. if committee_index >= committee_count_per_slot { diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 1575fce22f..861fccb374 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -195,7 +195,7 @@ pub fn is_valid_genesis_state(state: &BeaconState, spec: &ChainSp pub fn process_activations( state: &mut BeaconState, spec: &ChainSpec, -) -> Result<(), Error> { +) -> Result<(), BeaconStateError> { let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut(); let mut validators_iter = validators.iter_cow(); while let Some((index, validator)) = validators_iter.next_cow() { @@ -203,7 +203,7 @@ pub fn process_activations( let balance = balances .get(index) .copied() - .ok_or(Error::BalancesOutOfBounds(index))?; + .ok_or(BeaconStateError::BalancesOutOfBounds(index))?; validator.effective_balance = std::cmp::min( balance.safe_sub(balance.safe_rem(spec.effective_balance_increment)?)?, spec.max_effective_balance, diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index 07149ff2ee..cd1c1b9849 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -322,7 +322,7 @@ pub fn process_randao>( pub fn process_eth1_data( state: &mut BeaconState, eth1_data: &Eth1Data, -) -> Result<(), Error> { +) -> Result<(), BeaconStateError> { if let Some(new_eth1_data) = get_new_eth1_data(state, eth1_data)? { *state.eth1_data_mut() = new_eth1_data; } diff --git a/consensus/state_processing/src/per_block_processing/deneb.rs b/consensus/state_processing/src/per_block_processing/deneb.rs index a57c080c03..911e7e037e 100644 --- a/consensus/state_processing/src/per_block_processing/deneb.rs +++ b/consensus/state_processing/src/per_block_processing/deneb.rs @@ -1,5 +1,5 @@ use ethereum_hashing::hash_fixed; -use types::{KzgCommitment, VERSIONED_HASH_VERSION_KZG, VersionedHash}; +use types::{VersionedHash, kzg_ext::KzgCommitment, kzg_ext::consts::VERSIONED_HASH_VERSION_KZG}; pub fn kzg_commitment_to_versioned_hash(kzg_commitment: &KzgCommitment) -> VersionedHash { let mut hashed_commitment = hash_fixed(&kzg_commitment.0); diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 5a89fcb1d4..5ac1315f29 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -114,10 +114,6 @@ pub mod blob_sidecar { }; } -pub mod payload { - pub use crate::execution::BlockProductionVersion; -} - pub mod execution_requests { pub use crate::execution::{ ConsolidationRequests, DepositRequests, ExecutionRequests, RequestType, WithdrawalRequests, @@ -135,10 +131,6 @@ pub mod data_column_custody_group { }; } -pub mod sync_aggregate { - pub use crate::sync_committee::SyncAggregateError as Error; -} - pub mod light_client_update { pub use crate::light_client::consts::{ CURRENT_SYNC_COMMITTEE_INDEX, CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA, FINALIZED_ROOT_INDEX, @@ -153,10 +145,6 @@ pub mod sync_committee_contribution { }; } -pub mod slot_data { - pub use crate::core::SlotData; -} - pub mod signed_aggregate_and_proof { pub use crate::attestation::SignedAggregateAndProofRefMut; } @@ -166,12 +154,3 @@ pub mod payload_attestation { PayloadAttestation, PayloadAttestationData, PayloadAttestationMessage, }; } - -pub mod application_domain { - pub use crate::core::ApplicationDomain; -} - -// Temporary re-exports to maintain backwards compatibility for Lighthouse. -pub use crate::kzg_ext::consts::VERSIONED_HASH_VERSION_KZG; -pub use crate::light_client::LightClientError as LightClientUpdateError; -pub use crate::state::BeaconStateError as Error; diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index 8c64c7cf37..24d75f5a11 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use task_executor::TaskExecutor; use tokio::time::sleep; -use types::payload::BlockProductionVersion; +use types::execution::BlockProductionVersion; use types::{ Address, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ForkName, Hash256, MainnetEthSpec, Slot, Uint256, From acc746d9470c58bf525259ebfdbab82aa2dd811a Mon Sep 17 00:00:00 2001 From: Mr Ree <84216933+0xmrree@users.noreply.github.com> Date: Wed, 14 Jan 2026 22:35:57 -0600 Subject: [PATCH 15/18] Update chainspec p2p and etc (#8601) [Missing values in /eth/v1/config/spec #8571 ](https://github.com/sigp/lighthouse/issues/8571) - there will be follow up PR for the re org props 1. As per above issue from EF dev ops, I added ``` "EPOCHS_PER_SUBNET_SUBSCRIPTION": "256", "ATTESTATION_SUBNET_COUNT": "64", "ATTESTATION_SUBNET_EXTRA_BITS": "0", "UPDATE_TIMEOUT": "8192", "DOMAIN_BLS_TO_EXECUTION_CHANGE": "0x0a000000" ``` to `/eth/v1/config/spec` 2. Had to change the minimal config for UPDATE_TIMEOUT to get currents tests to pass. This is ok given UPDATE_TIMEOUT is not used in lighthouse as this config for light client spec from altair 3. ATTESTATION_SUBNET_PREFIX_BITS is now dynamically calculated and shimmed into the /eth/v1/config/spec output as advised by @michaelsproul Co-Authored-By: Joseph Patchen --- .../chiado/config.yaml | 1 + .../gnosis/config.yaml | 2 + .../holesky/config.yaml | 1 + .../hoodi/config.yaml | 1 + .../mainnet/config.yaml | 1 + .../sepolia/config.yaml | 1 + consensus/types/src/core/chain_spec.rs | 96 +++++++++++++++++-- consensus/types/src/core/config_and_preset.rs | 36 +++++++ consensus/types/src/core/preset.rs | 8 ++ .../environment/tests/testnet_dir/config.yaml | 1 + 10 files changed, 138 insertions(+), 10 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml index 6bc41113d6..eafe1ad38c 100644 --- a/common/eth2_network_config/built_in_network_configs/chiado/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/chiado/config.yaml @@ -125,6 +125,7 @@ SUBNETS_PER_NODE: 2 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 # Deneb diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index aa2dbb35d3..2beeb45b25 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -108,6 +108,8 @@ MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml index b1e9faea1d..fc5b53173b 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -127,6 +127,7 @@ SUBNETS_PER_NODE: 2 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 diff --git a/common/eth2_network_config/built_in_network_configs/hoodi/config.yaml b/common/eth2_network_config/built_in_network_configs/hoodi/config.yaml index 256957e119..947caa21f0 100644 --- a/common/eth2_network_config/built_in_network_configs/hoodi/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/hoodi/config.yaml @@ -133,6 +133,7 @@ SUBNETS_PER_NODE: 2 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 # Deneb diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 49168018cb..d4df1582f3 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -154,6 +154,7 @@ ATTESTATION_SUBNET_COUNT: 64 # 0 bits ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS (= 6 + 0) bits +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index b1a01933d7..6a655d8e84 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -133,6 +133,7 @@ SUBNETS_PER_NODE: 2 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +# computed at runtime ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index f47b4b4c91..c9a4f3d9ab 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -163,6 +163,7 @@ pub struct ChainSpec { pub inactivity_score_bias: u64, pub inactivity_score_recovery_rate: u64, pub min_sync_committee_participants: u64, + pub update_timeout: u64, pub(crate) domain_sync_committee: u32, pub(crate) domain_sync_committee_selection_proof: u32, pub(crate) domain_contribution_and_proof: u32, @@ -251,7 +252,9 @@ pub struct ChainSpec { pub message_domain_invalid_snappy: [u8; 4], pub message_domain_valid_snappy: [u8; 4], pub subnets_per_node: u8, + pub epochs_per_subnet_subscription: u64, pub attestation_subnet_count: u64, + pub attestation_subnet_extra_bits: u8, pub attestation_subnet_prefix_bits: u8, /* @@ -1014,6 +1017,7 @@ impl ChainSpec { inactivity_score_bias: 4, inactivity_score_recovery_rate: 16, min_sync_committee_participants: 1, + update_timeout: 8192, epochs_per_sync_committee_period: Epoch::new(256), domain_sync_committee: 7, domain_sync_committee_selection_proof: 8, @@ -1106,7 +1110,9 @@ impl ChainSpec { boot_nodes: vec![], network_id: 1, // mainnet network id attestation_propagation_slot_range: default_attestation_propagation_slot_range(), + epochs_per_subnet_subscription: 256, attestation_subnet_count: 64, + attestation_subnet_extra_bits: 0, subnets_per_node: 2, maximum_gossip_clock_disparity: default_maximum_gossip_clock_disparity(), target_aggregators_per_committee: 16, @@ -1116,7 +1122,10 @@ impl ChainSpec { resp_timeout: default_resp_timeout(), message_domain_invalid_snappy: default_message_domain_invalid_snappy(), message_domain_valid_snappy: default_message_domain_valid_snappy(), - attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(), + attestation_subnet_prefix_bits: compute_attestation_subnet_prefix_bits( + default_attestation_subnet_count(), + default_attestation_subnet_extra_bits(), + ), max_request_blocks: default_max_request_blocks(), /* @@ -1197,6 +1206,7 @@ impl ChainSpec { proportional_slashing_multiplier: 2, // Altair epochs_per_sync_committee_period: Epoch::new(8), + update_timeout: 64, altair_fork_version: [0x01, 0x00, 0x00, 0x01], altair_fork_epoch: None, // Bellatrix @@ -1380,6 +1390,7 @@ impl ChainSpec { inactivity_score_bias: 4, inactivity_score_recovery_rate: 16, min_sync_committee_participants: 1, + update_timeout: 8192, epochs_per_sync_committee_period: Epoch::new(512), domain_sync_committee: 7, domain_sync_committee_selection_proof: 8, @@ -1471,7 +1482,9 @@ impl ChainSpec { boot_nodes: vec![], network_id: 100, // Gnosis Chain network id attestation_propagation_slot_range: default_attestation_propagation_slot_range(), + epochs_per_subnet_subscription: 256, attestation_subnet_count: 64, + attestation_subnet_extra_bits: 0, subnets_per_node: 4, // Make this larger than usual to avoid network damage maximum_gossip_clock_disparity: default_maximum_gossip_clock_disparity(), target_aggregators_per_committee: 16, @@ -1482,7 +1495,10 @@ impl ChainSpec { message_domain_invalid_snappy: default_message_domain_invalid_snappy(), message_domain_valid_snappy: default_message_domain_valid_snappy(), max_request_blocks: default_max_request_blocks(), - attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(), + attestation_subnet_prefix_bits: compute_attestation_subnet_prefix_bits( + default_attestation_subnet_count(), + default_attestation_subnet_extra_bits(), + ), /* * Networking Deneb Specific @@ -1803,9 +1819,15 @@ pub struct Config { #[serde(default = "default_message_domain_valid_snappy")] #[serde(with = "serde_utils::bytes_4_hex")] message_domain_valid_snappy: [u8; 4], - #[serde(default = "default_attestation_subnet_prefix_bits")] + #[serde(default = "default_epochs_per_subnet_subscription")] + #[serde(with = "serde_utils::quoted_u64")] + epochs_per_subnet_subscription: u64, + #[serde(default = "default_attestation_subnet_count")] + #[serde(with = "serde_utils::quoted_u64")] + attestation_subnet_count: u64, + #[serde(default = "default_attestation_subnet_extra_bits")] #[serde(with = "serde_utils::quoted_u8")] - attestation_subnet_prefix_bits: u8, + attestation_subnet_extra_bits: u8, #[serde(default = "default_max_request_blocks_deneb")] #[serde(with = "serde_utils::quoted_u64")] max_request_blocks_deneb: u64, @@ -1920,8 +1942,36 @@ fn default_subnets_per_node() -> u8 { 2u8 } -fn default_attestation_subnet_prefix_bits() -> u8 { - 6 +const fn default_epochs_per_subnet_subscription() -> u64 { + 256 +} + +const fn default_attestation_subnet_count() -> u64 { + 64 +} + +const fn default_attestation_subnet_extra_bits() -> u8 { + 0 +} + +/// Compute attestation_subnet_prefix_bits dynamically as: +/// ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +fn compute_attestation_subnet_prefix_bits( + attestation_subnet_count: u64, + attestation_subnet_extra_bits: u8, +) -> u8 { + let default_attestation_subnet_prefix_bits = 6u8; + + // ceillog2() = next_power_of_two().ilog2() + // casting to u8 is fine given ilog2(u64::MAX) = 63 + let min_bits_needed = attestation_subnet_count + .checked_next_power_of_two() + .and_then(|x| x.checked_ilog2()) + .unwrap_or(default_attestation_subnet_prefix_bits as u32) as u8; + + min_bits_needed + .safe_add(attestation_subnet_extra_bits) + .unwrap_or(default_attestation_subnet_prefix_bits) } const fn default_max_per_epoch_activation_churn_limit() -> u64 { @@ -2206,7 +2256,9 @@ impl Config { shard_committee_period: spec.shard_committee_period, eth1_follow_distance: spec.eth1_follow_distance, subnets_per_node: spec.subnets_per_node, - attestation_subnet_prefix_bits: spec.attestation_subnet_prefix_bits, + epochs_per_subnet_subscription: spec.epochs_per_subnet_subscription, + attestation_subnet_count: spec.attestation_subnet_count, + attestation_subnet_extra_bits: spec.attestation_subnet_extra_bits, inactivity_score_bias: spec.inactivity_score_bias, inactivity_score_recovery_rate: spec.inactivity_score_recovery_rate, @@ -2297,7 +2349,9 @@ impl Config { shard_committee_period, eth1_follow_distance, subnets_per_node, - attestation_subnet_prefix_bits, + epochs_per_subnet_subscription, + attestation_subnet_count, + attestation_subnet_extra_bits, inactivity_score_bias, inactivity_score_recovery_rate, ejection_balance, @@ -2370,6 +2424,9 @@ impl Config { shard_committee_period, eth1_follow_distance, subnets_per_node, + epochs_per_subnet_subscription, + attestation_subnet_count, + attestation_subnet_extra_bits, inactivity_score_bias, inactivity_score_recovery_rate, ejection_balance, @@ -2390,7 +2447,11 @@ impl Config { resp_timeout, message_domain_invalid_snappy, message_domain_valid_snappy, - attestation_subnet_prefix_bits, + // Compute attestation_subnet_prefix_bits dynamically + attestation_subnet_prefix_bits: compute_attestation_subnet_prefix_bits( + attestation_subnet_count, + attestation_subnet_extra_bits, + ), max_request_blocks, attestation_propagation_slot_range, maximum_gossip_clock_disparity, @@ -2559,6 +2620,13 @@ mod tests { } } } + + #[test] + fn test_compute_min_bits_for_n_values_edge_cases() { + assert_eq!(compute_attestation_subnet_prefix_bits(64, 0), 6); + assert_eq!(compute_attestation_subnet_prefix_bits(65, 0), 7); + assert_eq!(compute_attestation_subnet_prefix_bits(0, 1), 1); + } } #[cfg(test)] @@ -2638,6 +2706,9 @@ mod yaml_tests { REORG_HEAD_WEIGHT_THRESHOLD: 20 REORG_PARENT_WEIGHT_THRESHOLD: 160 REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 + ATTESTATION_SUBNET_COUNT: 64 + ATTESTATION_SUBNET_EXTRA_BITS: 0 DEPOSIT_CHAIN_ID: 7042643276 DEPOSIT_NETWORK_ID: 7042643276 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa @@ -2787,6 +2858,9 @@ mod yaml_tests { REORG_HEAD_WEIGHT_THRESHOLD: 20 REORG_PARENT_WEIGHT_THRESHOLD: 160 REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 + ATTESTATION_SUBNET_COUNT: 64 + ATTESTATION_SUBNET_EXTRA_BITS: 0 DEPOSIT_CHAIN_ID: 7042643276 DEPOSIT_NETWORK_ID: 7042643276 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa @@ -2896,6 +2970,9 @@ mod yaml_tests { MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 CHURN_LIMIT_QUOTIENT: 65536 PROPOSER_SCORE_BOOST: 40 + EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 + ATTESTATION_SUBNET_COUNT: 64 + ATTESTATION_SUBNET_EXTRA_BITS: 0 DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa @@ -2928,7 +3005,6 @@ mod yaml_tests { check_default!(resp_timeout); check_default!(message_domain_invalid_snappy); check_default!(message_domain_valid_snappy); - check_default!(attestation_subnet_prefix_bits); assert_eq!(chain_spec.bellatrix_fork_epoch, None); } diff --git a/consensus/types/src/core/config_and_preset.rs b/consensus/types/src/core/config_and_preset.rs index 08141c7731..5b8b27b02e 100644 --- a/consensus/types/src/core/config_and_preset.rs +++ b/consensus/types/src/core/config_and_preset.rs @@ -114,6 +114,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { let u32_hex = |v: u32| hex_string(&v.to_le_bytes()); let u8_hex = |v: u8| hex_string(&v.to_le_bytes()); hashmap! { + "attestation_subnet_prefix_bits".to_uppercase() => spec.attestation_subnet_prefix_bits.to_string().into(), "bls_withdrawal_prefix".to_uppercase() => u8_hex(spec.bls_withdrawal_prefix_byte), "eth1_address_withdrawal_prefix".to_uppercase() => u8_hex(spec.eth1_address_withdrawal_prefix_byte), "domain_beacon_proposer".to_uppercase() => u32_hex(spec.domain_beacon_proposer), @@ -131,6 +132,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { "domain_sync_committee".to_uppercase() => u32_hex(spec.domain_sync_committee), "domain_sync_committee_selection_proof".to_uppercase() => u32_hex(spec.domain_sync_committee_selection_proof), + "domain_bls_to_execution_change".to_uppercase() => u32_hex(spec.domain_bls_to_execution_change), "sync_committee_subnet_count".to_uppercase() => consts::altair::SYNC_COMMITTEE_SUBNET_COUNT.to_string().into(), "target_aggregators_per_sync_subcommittee".to_uppercase() => @@ -183,4 +185,38 @@ mod test { serde_yaml::from_reader(reader).expect("error while deserializing"); assert_eq!(ConfigAndPreset::Gloas(from), yamlconfig); } + + #[test] + fn test_attestation_subnet_prefix_bits_in_extra_fields() { + let mainnet_spec = ChainSpec::mainnet(); + let config = ConfigAndPreset::from_chain_spec::(&mainnet_spec); + let extra_fields = config.extra_fields(); + assert!(extra_fields.contains_key("ATTESTATION_SUBNET_PREFIX_BITS")); + + // For mainnet: 64 subnets, 0 extra bits -> ceil(log2(64)) + 0 = 6 + assert_eq!( + extra_fields.get("ATTESTATION_SUBNET_PREFIX_BITS"), + Some(&Value::String("6".to_string())) + ); + } + + // This is not exhaustive, but it can be extended as new fields are added to the spec. + #[test] + fn test_required_spec_fields_exist() { + let mainnet_spec = ChainSpec::mainnet(); + let config = ConfigAndPreset::from_chain_spec::(&mainnet_spec); + let json = serde_json::to_value(&config).expect("should serialize"); + let obj = json.as_object().expect("should be an object"); + let required_fields = [ + "EPOCHS_PER_SUBNET_SUBSCRIPTION", + "ATTESTATION_SUBNET_COUNT", + "ATTESTATION_SUBNET_EXTRA_BITS", + "ATTESTATION_SUBNET_PREFIX_BITS", + "UPDATE_TIMEOUT", + "DOMAIN_BLS_TO_EXECUTION_CHANGE", + ]; + for field in required_fields { + assert!(obj.contains_key(field), "Missing required field: {}", field); + } + } } diff --git a/consensus/types/src/core/preset.rs b/consensus/types/src/core/preset.rs index 75d2d8df6b..5b1978f8e9 100644 --- a/consensus/types/src/core/preset.rs +++ b/consensus/types/src/core/preset.rs @@ -134,6 +134,9 @@ pub struct AltairPreset { pub epochs_per_sync_committee_period: Epoch, #[serde(with = "serde_utils::quoted_u64")] pub min_sync_committee_participants: u64, + #[serde(default = "default_update_timeout")] + #[serde(with = "serde_utils::quoted_u64")] + pub update_timeout: u64, } impl AltairPreset { @@ -145,10 +148,15 @@ impl AltairPreset { sync_committee_size: E::SyncCommitteeSize::to_u64(), epochs_per_sync_committee_period: spec.epochs_per_sync_committee_period, min_sync_committee_participants: spec.min_sync_committee_participants, + update_timeout: spec.update_timeout, } } } +const fn default_update_timeout() -> u64 { + 8192 +} + #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub struct BellatrixPreset { diff --git a/lighthouse/environment/tests/testnet_dir/config.yaml b/lighthouse/environment/tests/testnet_dir/config.yaml index 24c4a67225..7cfeb746f2 100644 --- a/lighthouse/environment/tests/testnet_dir/config.yaml +++ b/lighthouse/environment/tests/testnet_dir/config.yaml @@ -93,6 +93,7 @@ TTFB_TIMEOUT: 5 RESP_TIMEOUT: 10 MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 ATTESTATION_SUBNET_PREFIX_BITS: 6 From 3903e1c67feb5ffaa3aec6e45dbf3590e533b697 Mon Sep 17 00:00:00 2001 From: Mac L Date: Fri, 16 Jan 2026 08:43:05 +0400 Subject: [PATCH 16/18] More `consensus/types` re-export cleanup (#8665) Remove more of the temporary re-exports from `consensus/types` Co-Authored-By: Mac L --- beacon_node/beacon_chain/src/beacon_chain.rs | 3 +- .../beacon_chain/src/blob_verification.rs | 2 +- .../beacon_chain/src/block_verification.rs | 3 +- .../src/block_verification_types.rs | 2 +- beacon_node/beacon_chain/src/builder.rs | 2 +- .../src/data_availability_checker.rs | 4 +- .../overflow_lru_cache.rs | 2 +- .../src/data_column_verification.rs | 2 +- .../beacon_chain/src/fetch_blobs/mod.rs | 3 +- beacon_node/beacon_chain/src/kzg_utils.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 4 +- .../tests/attestation_verification.rs | 3 +- .../beacon_chain/tests/blob_verification.rs | 2 +- beacon_node/beacon_chain/tests/events.rs | 2 +- beacon_node/client/src/builder.rs | 2 +- .../src/engine_api/json_structures.rs | 6 +-- .../src/discovery/subnet_predicate.rs | 2 +- .../src/peer_manager/mod.rs | 4 +- .../src/peer_manager/peerdb.rs | 2 +- .../lighthouse_network/src/rpc/codec.rs | 3 +- .../lighthouse_network/src/rpc/methods.rs | 5 +- .../lighthouse_network/src/types/globals.rs | 2 +- .../src/network_beacon_processor/mod.rs | 2 +- .../network_beacon_processor/rpc_methods.rs | 2 +- .../network_beacon_processor/sync_methods.rs | 2 +- .../src/network_beacon_processor/tests.rs | 2 +- .../network/src/sync/block_lookups/common.rs | 2 +- .../sync/block_lookups/single_block_lookup.rs | 2 +- .../network/src/sync/network_context.rs | 2 +- .../src/sync/network_context/custody.rs | 2 +- .../network_context/requests/blobs_by_root.rs | 2 +- beacon_node/network/src/sync/tests/lookups.rs | 2 +- beacon_node/store/src/hot_cold_store.rs | 2 +- .../altair/participation_flag_updates.rs | 2 +- .../types/src/block/beacon_block_body.rs | 5 +- consensus/types/src/lib.rs | 48 ------------------- slasher/src/test_utils.rs | 2 +- .../compute_columns_for_custody_groups.rs | 2 +- .../ef_tests/src/cases/get_custody_groups.rs | 2 +- 39 files changed, 45 insertions(+), 100 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 2f25c7728a..e3de8d7324 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -132,8 +132,7 @@ use task_executor::{RayonPoolType, ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tracing::{Span, debug, debug_span, error, info, info_span, instrument, trace, warn}; use tree_hash::TreeHash; -use types::blob_sidecar::FixedBlobSidecarList; -use types::data_column_sidecar::ColumnIndex; +use types::data::{ColumnIndex, FixedBlobSidecarList}; use types::execution::BlockProductionVersion; use types::*; diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 874673b52e..45374f509b 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -15,7 +15,7 @@ use ssz_derive::{Decode, Encode}; use std::time::Duration; use tracing::{debug, instrument}; use tree_hash::TreeHash; -use types::blob_sidecar::BlobIdentifier; +use types::data::BlobIdentifier; use types::{ BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, }; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index bca8d2bc57..df8c49f8de 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -98,8 +98,7 @@ use tracing::{Instrument, Span, debug, debug_span, error, info_span, instrument} use types::{ BeaconBlockRef, BeaconState, BeaconStateError, BlobsList, ChainSpec, DataColumnSidecarList, Epoch, EthSpec, ExecutionBlockHash, FullPayload, Hash256, InconsistentFork, KzgProofs, - RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, - data_column_sidecar::DataColumnSidecarError, + RelativeEpoch, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, data::DataColumnSidecarError, }; pub const POS_PANDA_BANNER: &str = r#" diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index 5978e97c4d..f7831d5c77 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -7,7 +7,7 @@ use ssz_types::VariableList; use state_processing::ConsensusContext; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use types::blob_sidecar::BlobIdentifier; +use types::data::BlobIdentifier; use types::{ BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, SignedBeaconBlockHeader, Slot, diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index b8770b8911..dc38fc1c29 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -42,7 +42,7 @@ use std::time::Duration; use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp}; use task_executor::{ShutdownReason, TaskExecutor}; use tracing::{debug, error, info}; -use types::data_column_custody_group::CustodyIndex; +use types::data::CustodyIndex; use types::{ BeaconBlock, BeaconState, BlobSidecarList, ChainSpec, ColumnIndex, DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 3e859456b1..a4e10c8d62 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::time::Duration; use task_executor::TaskExecutor; use tracing::{debug, error, instrument}; -use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; +use types::data::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{ BlobSidecarList, BlockImportSource, ChainSpec, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot, @@ -876,7 +876,7 @@ mod test { use std::sync::Arc; use std::time::Duration; use store::HotColdDB; - use types::data_column_sidecar::DataColumn; + use types::data::DataColumn; use types::{ChainSpec, ColumnIndex, EthSpec, ForkName, MainnetEthSpec, Slot}; type E = MainnetEthSpec; diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 776fb50f61..c37bada8c1 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -18,7 +18,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{Span, debug, debug_span}; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::BlobIdentifier; +use types::data::BlobIdentifier; use types::{ BlobSidecar, BlockImportSource, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, diff --git a/beacon_node/beacon_chain/src/data_column_verification.rs b/beacon_node/beacon_chain/src/data_column_verification.rs index b998602566..7bb139756d 100644 --- a/beacon_node/beacon_chain/src/data_column_verification.rs +++ b/beacon_node/beacon_chain/src/data_column_verification.rs @@ -15,7 +15,7 @@ use std::iter; use std::marker::PhantomData; use std::sync::Arc; use tracing::{debug, instrument}; -use types::data_column_sidecar::ColumnIndex; +use types::data::ColumnIndex; use types::{ BeaconStateError, ChainSpec, DataColumnSidecar, DataColumnSubnetId, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, diff --git a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs index 3a1cf146ed..6559f24d23 100644 --- a/beacon_node/beacon_chain/src/fetch_blobs/mod.rs +++ b/beacon_node/beacon_chain/src/fetch_blobs/mod.rs @@ -33,8 +33,7 @@ use ssz_types::FixedVector; use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::sync::Arc; use tracing::{Span, debug, instrument, warn}; -use types::blob_sidecar::BlobSidecarError; -use types::data_column_sidecar::DataColumnSidecarError; +use types::data::{BlobSidecarError, DataColumnSidecarError}; use types::{ BeaconStateError, Blob, BlobSidecar, ColumnIndex, EthSpec, FullPayload, Hash256, KzgProofs, SignedBeaconBlock, SignedBeaconBlockHeader, VersionedHash, diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 334124419b..3e1119b2ae 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -7,7 +7,7 @@ use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; use types::beacon_block_body::KzgCommitments; -use types::data_column_sidecar::{Cell, DataColumn, DataColumnSidecarError}; +use types::data::{Cell, DataColumn, DataColumnSidecarError}; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 0a226d0f76..b6c235a4cb 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -67,9 +67,9 @@ use task_executor::TaskExecutor; use task_executor::{ShutdownReason, test_utils::TestRuntime}; use tree_hash::TreeHash; use typenum::U4294967296; -use types::data_column_custody_group::CustodyIndex; +use types::attestation::IndexedAttestationBase; +use types::data::CustodyIndex; use types::execution::BlockProductionVersion; -use types::indexed_attestation::IndexedAttestationBase; use types::test_utils::TestRandom; pub use types::test_utils::generate_deterministic_keypairs; use types::*; diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 7984ea4708..208798dfdf 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -24,8 +24,7 @@ use typenum::Unsigned; use types::{ Address, Attestation, AttestationRef, ChainSpec, Epoch, EthSpec, ForkName, Hash256, MainnetEthSpec, SelectionProof, SignedAggregateAndProof, SingleAttestation, Slot, SubnetId, - signed_aggregate_and_proof::SignedAggregateAndProofRefMut, - test_utils::generate_deterministic_keypair, + attestation::SignedAggregateAndProofRefMut, test_utils::generate_deterministic_keypair, }; pub type E = MainnetEthSpec; diff --git a/beacon_node/beacon_chain/tests/blob_verification.rs b/beacon_node/beacon_chain/tests/blob_verification.rs index d1a0d87adf..019736ca01 100644 --- a/beacon_node/beacon_chain/tests/blob_verification.rs +++ b/beacon_node/beacon_chain/tests/blob_verification.rs @@ -10,7 +10,7 @@ use beacon_chain::{ use bls::{Keypair, Signature}; use logging::create_test_tracing_subscriber; use std::sync::{Arc, LazyLock}; -use types::{blob_sidecar::FixedBlobSidecarList, *}; +use types::{data::FixedBlobSidecarList, *}; type E = MainnetEthSpec; diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index 86bdb03daf..35884541e1 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -7,7 +7,7 @@ use eth2::types::{EventKind, SseBlobSidecar, SseDataColumnSidecar}; use rand::SeedableRng; use rand::rngs::StdRng; use std::sync::Arc; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; use types::test_utils::TestRandom; use types::{BlobSidecar, DataColumnSidecar, EthSpec, MinimalEthSpec, Slot}; diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 694c6fb356..ba90cbd8be 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -43,7 +43,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use store::database::interface::BeaconNodeBackend; use timer::spawn_timer; use tracing::{debug, info, instrument, warn}; -use types::data_column_custody_group::compute_ordered_custody_column_indices; +use types::data::compute_ordered_custody_column_indices; use types::{ BeaconState, BlobSidecarList, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, SignedBeaconBlock, test_utils::generate_deterministic_keypairs, diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index fc8eae015b..c8488c94d6 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -6,10 +6,8 @@ use ssz_types::{FixedVector, VariableList, typenum::Unsigned}; use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::BlobsList; -use types::execution_requests::{ - ConsolidationRequests, DepositRequests, RequestType, WithdrawalRequests, -}; +use types::data::BlobsList; +use types::execution::{ConsolidationRequests, DepositRequests, RequestType, WithdrawalRequests}; use types::{Blob, KzgProof}; #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs index 6e841c25a5..757dbb5853 100644 --- a/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs +++ b/beacon_node/lighthouse_network/src/discovery/subnet_predicate.rs @@ -4,7 +4,7 @@ use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}; use std::ops::Deref; use tracing::trace; use types::ChainSpec; -use types::data_column_custody_group::compute_subnets_for_node; +use types::data::compute_subnets_for_node; /// Returns the predicate for a given subnet. pub fn subnet_predicate( diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 3cfe2b3c3b..43a44c85fc 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -33,9 +33,7 @@ pub use peerdb::sync_status::{SyncInfo, SyncStatus}; use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::net::IpAddr; use strum::IntoEnumIterator; -use types::data_column_custody_group::{ - CustodyIndex, compute_subnets_from_custody_group, get_custody_groups, -}; +use types::data::{CustodyIndex, compute_subnets_from_custody_group, get_custody_groups}; /// Unified peer subnet information structure for pruning logic. struct PeerSubnetInfo { diff --git a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs index dc1686523f..11ce785350 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/peerdb.rs @@ -15,7 +15,7 @@ use std::{ }; use sync_status::SyncStatus; use tracing::{debug, error, trace, warn}; -use types::data_column_custody_group::compute_subnets_for_node; +use types::data::compute_subnets_for_node; use types::{ChainSpec, DataColumnSubnetId, Epoch, EthSpec, Hash256, Slot}; pub mod client; diff --git a/beacon_node/lighthouse_network/src/rpc/codec.rs b/beacon_node/lighthouse_network/src/rpc/codec.rs index 48a29699c8..3611f02391 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec.rs @@ -913,7 +913,8 @@ mod tests { use types::{ BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockHeader, DataColumnsByRootIdentifier, EmptyBlock, Epoch, FullPayload, KzgCommitment, KzgProof, - SignedBeaconBlockHeader, Slot, blob_sidecar::BlobIdentifier, data_column_sidecar::Cell, + SignedBeaconBlockHeader, Slot, + data::{BlobIdentifier, Cell}, }; type Spec = types::MainnetEthSpec; diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index a9b4aa2fba..dbcf39ac6e 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -12,13 +12,12 @@ use std::ops::Deref; use std::sync::Arc; use strum::IntoStaticStr; use superstruct::superstruct; -use types::blob_sidecar::BlobIdentifier; +use types::data::BlobIdentifier; use types::light_client_update::MAX_REQUEST_LIGHT_CLIENT_UPDATES; use types::{ ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnsByRootIdentifier, Epoch, EthSpec, ForkContext, Hash256, LightClientBootstrap, LightClientFinalityUpdate, - LightClientOptimisticUpdate, LightClientUpdate, SignedBeaconBlock, Slot, - blob_sidecar::BlobSidecar, + LightClientOptimisticUpdate, LightClientUpdate, SignedBeaconBlock, Slot, data::BlobSidecar, }; /// Maximum length of error message. diff --git a/beacon_node/lighthouse_network/src/types/globals.rs b/beacon_node/lighthouse_network/src/types/globals.rs index 3217f41f61..df8dbdc559 100644 --- a/beacon_node/lighthouse_network/src/types/globals.rs +++ b/beacon_node/lighthouse_network/src/types/globals.rs @@ -10,7 +10,7 @@ use parking_lot::RwLock; use std::collections::HashSet; use std::sync::Arc; use tracing::{debug, error}; -use types::data_column_custody_group::{compute_subnets_from_custody_group, get_custody_groups}; +use types::data::{compute_subnets_from_custody_group, get_custody_groups}; use types::{ChainSpec, ColumnIndex, DataColumnSubnetId, EthSpec}; pub struct NetworkGlobals { diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index bebda36d71..fd9c2c1e55 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -31,7 +31,7 @@ use tracing::{debug, error, instrument, trace, warn}; use types::*; pub use sync_methods::ChainSegmentProcessId; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; pub type Error = TrySendError>; diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index ac24b648e0..e443eb78d8 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -23,7 +23,7 @@ use std::collections::{HashMap, HashSet, hash_map::Entry}; use std::sync::Arc; use tokio_stream::StreamExt; use tracing::{Span, debug, error, field, instrument, warn}; -use types::blob_sidecar::BlobIdentifier; +use types::data::BlobIdentifier; use types::{ColumnIndex, Epoch, EthSpec, Hash256, Slot}; impl NetworkBeaconProcessor { diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index e49ae134fe..c3a0cb2a86 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -32,7 +32,7 @@ use std::time::Duration; use store::KzgCommitment; use tracing::{debug, debug_span, error, info, instrument, warn}; use types::beacon_block_body::format_kzg_commitments; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; use types::{BlockImportSource, DataColumnSidecarList, Epoch, Hash256}; /// Id associated to a batch processing request, either a sync batch or a parent lookup. diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index ed04fe7bb9..49da522c9a 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -39,7 +39,7 @@ use std::iter::Iterator; use std::sync::Arc; use std::time::Duration; use tokio::sync::mpsc; -use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; +use types::data::{BlobIdentifier, FixedBlobSidecarList}; use types::{ AttesterSlashing, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecarList, DataColumnSubnetId, Epoch, EthSpec, Hash256, MainnetEthSpec, ProposerSlashing, diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index c6b0519087..edd99345b4 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -11,7 +11,7 @@ use lighthouse_network::service::api_types::Id; use parking_lot::RwLock; use std::collections::HashSet; use std::sync::Arc; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; use types::{DataColumnSidecarList, SignedBeaconBlock}; use super::SingleLookupId; diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 46897b2283..fea8794b2e 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -16,7 +16,7 @@ use std::time::{Duration, Instant}; use store::Hash256; use strum::IntoStaticStr; use tracing::{Span, debug_span}; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; use types::{DataColumnSidecarList, EthSpec, SignedBeaconBlock, Slot}; // Dedicated enum for LookupResult to force its usage diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 2e0c56db23..00d4bf865d 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -49,7 +49,7 @@ use std::time::Duration; use task_executor::TaskExecutor; use tokio::sync::mpsc; use tracing::{Span, debug, debug_span, error, warn}; -use types::blob_sidecar::FixedBlobSidecarList; +use types::data::FixedBlobSidecarList; use types::{ BlobSidecar, BlockImportSource, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, ForkContext, Hash256, SignedBeaconBlock, Slot, diff --git a/beacon_node/network/src/sync/network_context/custody.rs b/beacon_node/network/src/sync/network_context/custody.rs index 71e002cc42..fe0924fae8 100644 --- a/beacon_node/network/src/sync/network_context/custody.rs +++ b/beacon_node/network/src/sync/network_context/custody.rs @@ -13,7 +13,7 @@ use std::hash::{BuildHasher, RandomState}; use std::time::{Duration, Instant}; use std::{collections::HashMap, marker::PhantomData, sync::Arc}; use tracing::{Span, debug, debug_span, warn}; -use types::{DataColumnSidecar, Hash256, data_column_sidecar::ColumnIndex}; +use types::{DataColumnSidecar, Hash256, data::ColumnIndex}; use types::{DataColumnSidecarList, EthSpec}; use super::{LookupRequestResult, PeerGroup, RpcResponseResult, SyncNetworkContext}; diff --git a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs index 39886d814e..556985c2b4 100644 --- a/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs +++ b/beacon_node/network/src/sync/network_context/requests/blobs_by_root.rs @@ -1,6 +1,6 @@ use lighthouse_network::rpc::methods::BlobsByRootRequest; use std::sync::Arc; -use types::{BlobSidecar, EthSpec, ForkContext, Hash256, blob_sidecar::BlobIdentifier}; +use types::{BlobSidecar, EthSpec, ForkContext, Hash256, data::BlobIdentifier}; use super::{ActiveRequestItems, LookupVerifyError}; diff --git a/beacon_node/network/src/sync/tests/lookups.rs b/beacon_node/network/src/sync/tests/lookups.rs index ef52f89678..715928906e 100644 --- a/beacon_node/network/src/sync/tests/lookups.rs +++ b/beacon_node/network/src/sync/tests/lookups.rs @@ -43,7 +43,7 @@ use tracing::info; use types::{ BeaconState, BeaconStateBase, BlobSidecar, BlockImportSource, DataColumnSidecar, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec as E, SignedBeaconBlock, Slot, - data_column_sidecar::ColumnIndex, + data::ColumnIndex, test_utils::{SeedableRng, TestRandom, XorShiftRng}, }; diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index c413719174..8eec4d5ece 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -40,7 +40,7 @@ use std::sync::Arc; use std::time::Duration; use tracing::{debug, error, info, instrument, warn}; use typenum::Unsigned; -use types::data_column_sidecar::{ColumnIndex, DataColumnSidecar, DataColumnSidecarList}; +use types::data::{ColumnIndex, DataColumnSidecar, DataColumnSidecarList}; use types::*; use zstd::{Decoder, Encoder}; diff --git a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs index 5e177c5d2b..899d0dbb7d 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs @@ -1,8 +1,8 @@ use crate::EpochProcessingError; use milhouse::List; +use types::attestation::ParticipationFlags; use types::beacon_state::BeaconState; use types::eth_spec::EthSpec; -use types::participation_flags::ParticipationFlags; pub fn process_participation_flag_updates( state: &mut BeaconState, diff --git a/consensus/types/src/block/beacon_block_body.rs b/consensus/types/src/block/beacon_block_body.rs index 1a0b385900..a113f85fd3 100644 --- a/consensus/types/src/block/beacon_block_body.rs +++ b/consensus/types/src/block/beacon_block_body.rs @@ -13,10 +13,11 @@ use test_random_derive::TestRandom; use tree_hash::{BYTES_PER_CHUNK, TreeHash}; use tree_hash_derive::TreeHash; -use crate::payload_attestation::PayloadAttestation; use crate::{ SignedExecutionPayloadBid, - attestation::{AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut}, + attestation::{ + AttestationBase, AttestationElectra, AttestationRef, AttestationRefMut, PayloadAttestation, + }, core::{EthSpec, Graffiti, Hash256}, deposit::Deposit, execution::{ diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 5ac1315f29..0a02d100e6 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -77,18 +77,10 @@ pub mod graffiti { pub use crate::core::GraffitiString; } -pub mod indexed_attestation { - pub use crate::attestation::{IndexedAttestationBase, IndexedAttestationElectra}; -} - pub mod historical_summary { pub use crate::state::HistoricalSummary; } -pub mod participation_flags { - pub use crate::attestation::ParticipationFlags; -} - pub mod epoch_cache { pub use crate::state::{EpochCache, EpochCacheError, EpochCacheKey}; } @@ -97,40 +89,10 @@ pub mod non_zero_usize { pub use crate::core::new_non_zero_usize; } -pub mod data_column_sidecar { - pub use crate::data::{ - Cell, ColumnIndex, DataColumn, DataColumnSidecar, DataColumnSidecarError, - DataColumnSidecarList, - }; -} - pub mod builder_bid { pub use crate::builder::*; } -pub mod blob_sidecar { - pub use crate::data::{ - BlobIdentifier, BlobSidecar, BlobSidecarError, BlobsList, FixedBlobSidecarList, - }; -} - -pub mod execution_requests { - pub use crate::execution::{ - ConsolidationRequests, DepositRequests, ExecutionRequests, RequestType, WithdrawalRequests, - }; -} - -pub mod execution_payload_envelope { - pub use crate::execution::{ExecutionPayloadEnvelope, SignedExecutionPayloadEnvelope}; -} - -pub mod data_column_custody_group { - pub use crate::data::{ - CustodyIndex, compute_columns_for_custody_group, compute_ordered_custody_column_indices, - compute_subnets_for_node, compute_subnets_from_custody_group, get_custody_groups, - }; -} - pub mod light_client_update { pub use crate::light_client::consts::{ CURRENT_SYNC_COMMITTEE_INDEX, CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA, FINALIZED_ROOT_INDEX, @@ -144,13 +106,3 @@ pub mod sync_committee_contribution { SyncCommitteeContributionError as Error, SyncContributionData, }; } - -pub mod signed_aggregate_and_proof { - pub use crate::attestation::SignedAggregateAndProofRefMut; -} - -pub mod payload_attestation { - pub use crate::attestation::{ - PayloadAttestation, PayloadAttestationData, PayloadAttestationMessage, - }; -} diff --git a/slasher/src/test_utils.rs b/slasher/src/test_utils.rs index 20d1ee9217..82b7f885ec 100644 --- a/slasher/src/test_utils.rs +++ b/slasher/src/test_utils.rs @@ -6,7 +6,7 @@ use types::{ AttestationData, AttesterSlashing, AttesterSlashingBase, AttesterSlashingElectra, BeaconBlockHeader, ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, IndexedAttestation, MainnetEthSpec, SignedBeaconBlockHeader, Slot, - indexed_attestation::{IndexedAttestationBase, IndexedAttestationElectra}, + attestation::{IndexedAttestationBase, IndexedAttestationElectra}, }; pub type E = MainnetEthSpec; diff --git a/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs b/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs index 16d3eaf7af..e30f4c3ae1 100644 --- a/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs +++ b/testing/ef_tests/src/cases/compute_columns_for_custody_groups.rs @@ -1,7 +1,7 @@ use super::*; use serde::Deserialize; use std::marker::PhantomData; -use types::data_column_custody_group::{CustodyIndex, compute_columns_for_custody_group}; +use types::data::{CustodyIndex, compute_columns_for_custody_group}; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec", deny_unknown_fields)] diff --git a/testing/ef_tests/src/cases/get_custody_groups.rs b/testing/ef_tests/src/cases/get_custody_groups.rs index 1c1294305f..4221fa06e0 100644 --- a/testing/ef_tests/src/cases/get_custody_groups.rs +++ b/testing/ef_tests/src/cases/get_custody_groups.rs @@ -2,7 +2,7 @@ use super::*; use alloy_primitives::U256; use serde::Deserialize; use std::marker::PhantomData; -use types::data_column_custody_group::get_custody_groups; +use types::data::get_custody_groups; #[derive(Debug, Clone, Deserialize)] #[serde(bound = "E: EthSpec", deny_unknown_fields)] From d099ad56fb400186d684966be533e7aaa46e1385 Mon Sep 17 00:00:00 2001 From: Mac L Date: Fri, 16 Jan 2026 11:53:13 +0400 Subject: [PATCH 17/18] Remove `execution` dependency from `core` module in `consensus/types` (#8666) #8652 This moves the `ExecutionBlockHash` from the `execution` module to the `core` module. This allows `core` to not depend on the `execution` module, and the `ExecutionBlockHash` is a pretty core part of our types so I think it makes sense. Co-Authored-By: Mac L --- consensus/types/src/core/chain_spec.rs | 3 +-- .../types/src/{execution => core}/execution_block_hash.rs | 0 consensus/types/src/core/mod.rs | 2 ++ consensus/types/src/execution/execution_payload.rs | 3 +-- consensus/types/src/execution/execution_payload_header.rs | 7 +++---- consensus/types/src/execution/mod.rs | 2 -- consensus/types/src/execution/payload.rs | 4 ++-- 7 files changed, 9 insertions(+), 12 deletions(-) rename consensus/types/src/{execution => core}/execution_block_hash.rs (100%) diff --git a/consensus/types/src/core/chain_spec.rs b/consensus/types/src/core/chain_spec.rs index c9a4f3d9ab..1bdf6c2cb8 100644 --- a/consensus/types/src/core/chain_spec.rs +++ b/consensus/types/src/core/chain_spec.rs @@ -14,10 +14,9 @@ use tree_hash::TreeHash; use crate::{ core::{ APPLICATION_DOMAIN_BUILDER, Address, ApplicationDomain, EnrForkId, Epoch, EthSpec, - EthSpecId, Hash256, MainnetEthSpec, Slot, Uint256, + EthSpecId, ExecutionBlockHash, Hash256, MainnetEthSpec, Slot, Uint256, }, data::{BlobIdentifier, DataColumnSubnetId, DataColumnsByRootIdentifier}, - execution::ExecutionBlockHash, fork::{Fork, ForkData, ForkName}, }; diff --git a/consensus/types/src/execution/execution_block_hash.rs b/consensus/types/src/core/execution_block_hash.rs similarity index 100% rename from consensus/types/src/execution/execution_block_hash.rs rename to consensus/types/src/core/execution_block_hash.rs diff --git a/consensus/types/src/core/mod.rs b/consensus/types/src/core/mod.rs index bb50bb1856..33d5856dea 100644 --- a/consensus/types/src/core/mod.rs +++ b/consensus/types/src/core/mod.rs @@ -5,6 +5,7 @@ mod chain_spec; mod config_and_preset; mod enr_fork_id; mod eth_spec; +mod execution_block_hash; mod graffiti; mod non_zero_usize; mod preset; @@ -25,6 +26,7 @@ pub use config_and_preset::{ }; pub use enr_fork_id::EnrForkId; pub use eth_spec::{EthSpec, EthSpecId, GNOSIS, GnosisEthSpec, MainnetEthSpec, MinimalEthSpec}; +pub use execution_block_hash::ExecutionBlockHash; pub use graffiti::{GRAFFITI_BYTES_LEN, Graffiti, GraffitiString}; pub use non_zero_usize::new_non_zero_usize; pub use preset::{ diff --git a/consensus/types/src/execution/execution_payload.rs b/consensus/types/src/execution/execution_payload.rs index b2278c9166..d99b8785fa 100644 --- a/consensus/types/src/execution/execution_payload.rs +++ b/consensus/types/src/execution/execution_payload.rs @@ -10,8 +10,7 @@ use test_random_derive::TestRandom; use tree_hash_derive::TreeHash; use crate::{ - core::{Address, EthSpec, Hash256}, - execution::ExecutionBlockHash, + core::{Address, EthSpec, ExecutionBlockHash, Hash256}, fork::{ForkName, ForkVersionDecode}, state::BeaconStateError, test_utils::TestRandom, diff --git a/consensus/types/src/execution/execution_payload_header.rs b/consensus/types/src/execution/execution_payload_header.rs index cf78f7871b..0b8556634a 100644 --- a/consensus/types/src/execution/execution_payload_header.rs +++ b/consensus/types/src/execution/execution_payload_header.rs @@ -11,11 +11,10 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use crate::{ - core::{Address, EthSpec, Hash256, Uint256}, + core::{Address, EthSpec, ExecutionBlockHash, Hash256, Uint256}, execution::{ - ExecutionBlockHash, ExecutionPayloadBellatrix, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionPayloadRef, - Transactions, + ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionPayloadRef, Transactions, }, fork::ForkName, map_execution_payload_ref_into_execution_payload_header, diff --git a/consensus/types/src/execution/mod.rs b/consensus/types/src/execution/mod.rs index da6c860600..a3d4ed8730 100644 --- a/consensus/types/src/execution/mod.rs +++ b/consensus/types/src/execution/mod.rs @@ -1,5 +1,4 @@ mod eth1_data; -mod execution_block_hash; mod execution_block_header; #[macro_use] mod execution_payload; @@ -16,7 +15,6 @@ mod signed_execution_payload_envelope; pub use bls_to_execution_change::BlsToExecutionChange; pub use eth1_data::Eth1Data; -pub use execution_block_hash::ExecutionBlockHash; pub use execution_block_header::{EncodableExecutionBlockHeader, ExecutionBlockHeader}; pub use execution_payload::{ ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, diff --git a/consensus/types/src/execution/payload.rs b/consensus/types/src/execution/payload.rs index 703b082c18..c51369034c 100644 --- a/consensus/types/src/execution/payload.rs +++ b/consensus/types/src/execution/payload.rs @@ -11,9 +11,9 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use crate::{ - core::{Address, EthSpec, Hash256}, + core::{Address, EthSpec, ExecutionBlockHash, Hash256}, execution::{ - ExecutionBlockHash, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, + ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadFulu, ExecutionPayloadHeader, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderFulu, From 58b153cac5a078d849bf8478bd21baedfe98d9ee Mon Sep 17 00:00:00 2001 From: Mac L Date: Fri, 16 Jan 2026 23:51:29 +0400 Subject: [PATCH 18/18] Remove remaining facade module re-exports from `consensus/types` (#8672) Removes the remaining facade re-exports from `consensus/types`. I have left `graffiti` as I think it has some utility so am leaning towards keeping it in the final API design. Co-Authored-By: Mac L --- beacon_node/beacon_chain/benches/benches.rs | 2 +- .../beacon_chain/src/beacon_proposer_cache.rs | 2 +- .../src/data_availability_checker.rs | 2 +- .../overflow_lru_cache.rs | 4 +- beacon_node/beacon_chain/src/kzg_utils.rs | 5 +- .../src/light_client_server_cache.rs | 2 +- .../src/naive_aggregation_pool.rs | 2 +- .../src/pre_finalization_cache.rs | 2 +- .../beacon_chain/src/shuffling_cache.rs | 2 +- .../src/sync_committee_verification.rs | 2 +- beacon_node/builder_client/src/lib.rs | 4 +- .../src/engine_api/json_structures.rs | 2 +- beacon_node/execution_layer/src/engines.rs | 2 +- beacon_node/execution_layer/src/lib.rs | 8 +-- .../execution_layer/src/payload_cache.rs | 2 +- .../src/test_utils/mock_builder.rs | 2 +- beacon_node/http_api/src/block_rewards.rs | 4 +- .../lighthouse_network/src/discovery/mod.rs | 2 +- .../lighthouse_network/src/rpc/methods.rs | 2 +- .../gossip_methods.rs | 2 +- .../network_beacon_processor/sync_methods.rs | 2 +- beacon_node/operation_pool/src/attestation.rs | 2 +- beacon_node/store/src/config.rs | 2 +- beacon_node/store/src/hdiff.rs | 2 +- consensus/state_processing/src/epoch_cache.rs | 2 +- .../altair/inactivity_updates.rs | 5 +- .../altair/participation_flag_updates.rs | 4 +- .../altair/sync_committee_updates.rs | 5 +- .../base/participation_record_updates.rs | 4 +- .../capella/historical_summaries_update.rs | 2 +- .../effective_balance_updates.rs | 4 +- .../historical_roots_update.rs | 4 +- .../src/per_epoch_processing/resets.rs | 4 +- consensus/types/src/lib.rs | 56 +------------------ lighthouse/tests/beacon_node.rs | 2 +- slasher/src/config.rs | 2 +- .../src/cases/merkle_proof_validity.rs | 14 ++--- testing/ef_tests/src/type_name.rs | 2 +- testing/ef_tests/tests/tests.rs | 2 +- 39 files changed, 59 insertions(+), 116 deletions(-) diff --git a/beacon_node/beacon_chain/benches/benches.rs b/beacon_node/beacon_chain/benches/benches.rs index de3ced3be1..0d4040155d 100644 --- a/beacon_node/beacon_chain/benches/benches.rs +++ b/beacon_node/beacon_chain/benches/benches.rs @@ -8,7 +8,7 @@ use bls::Signature; use kzg::{KzgCommitment, KzgProof}; use types::{ BeaconBlock, BeaconBlockFulu, Blob, BlobsList, ChainSpec, EmptyBlock, EthSpec, KzgProofs, - MainnetEthSpec, SignedBeaconBlock, beacon_block_body::KzgCommitments, + MainnetEthSpec, SignedBeaconBlock, kzg_ext::KzgCommitments, }; fn create_test_block_and_blobs( diff --git a/beacon_node/beacon_chain/src/beacon_proposer_cache.rs b/beacon_node/beacon_chain/src/beacon_proposer_cache.rs index a923d657a8..912f7f3bad 100644 --- a/beacon_node/beacon_chain/src/beacon_proposer_cache.rs +++ b/beacon_node/beacon_chain/src/beacon_proposer_cache.rs @@ -19,7 +19,7 @@ use std::num::NonZeroUsize; use std::sync::Arc; use tracing::instrument; use typenum::Unsigned; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use types::{BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, Fork, Hash256, Slot}; /// The number of sets of proposer indices that should be cached. diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index a4e10c8d62..7aec24b8e5 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -39,7 +39,7 @@ use crate::metrics::{ }; use crate::observed_data_sidecars::ObservationStrategy; pub use error::{Error as AvailabilityCheckError, ErrorCategory as AvailabilityCheckErrorCategory}; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; /// The LRU Cache stores `PendingComponents`, which store block and its associated blob data: /// diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index c37bada8c1..c249af2d40 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -17,8 +17,8 @@ use std::cmp::Ordering; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{Span, debug, debug_span}; -use types::beacon_block_body::KzgCommitments; use types::data::BlobIdentifier; +use types::kzg_ext::KzgCommitments; use types::{ BlobSidecar, BlockImportSource, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, @@ -839,7 +839,7 @@ mod test { use store::{HotColdDB, ItemStore, StoreConfig, database::interface::BeaconNodeBackend}; use tempfile::{TempDir, tempdir}; use tracing::{debug_span, info}; - use types::non_zero_usize::new_non_zero_usize; + use types::new_non_zero_usize; use types::{ExecPayload, MinimalEthSpec}; const LOW_VALIDATOR_COUNT: usize = 32; diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 3e1119b2ae..a1c255e3b3 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -6,8 +6,8 @@ use rayon::prelude::*; use ssz_types::{FixedVector, VariableList}; use std::sync::Arc; use tracing::instrument; -use types::beacon_block_body::KzgCommitments; use types::data::{Cell, DataColumn, DataColumnSidecarError}; +use types::kzg_ext::KzgCommitments; use types::{ Blob, BlobSidecar, BlobSidecarList, ChainSpec, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, KzgCommitment, KzgProof, SignedBeaconBlock, SignedBeaconBlockHeader, @@ -448,8 +448,7 @@ mod test { use kzg::{Kzg, KzgCommitment, trusted_setup::get_trusted_setup}; use types::{ BeaconBlock, BeaconBlockFulu, BlobsList, ChainSpec, EmptyBlock, EthSpec, ForkName, - FullPayload, KzgProofs, MainnetEthSpec, SignedBeaconBlock, - beacon_block_body::KzgCommitments, + FullPayload, KzgProofs, MainnetEthSpec, SignedBeaconBlock, kzg_ext::KzgCommitments, }; type E = MainnetEthSpec; diff --git a/beacon_node/beacon_chain/src/light_client_server_cache.rs b/beacon_node/beacon_chain/src/light_client_server_cache.rs index 487ddfd3ec..5b405234e7 100644 --- a/beacon_node/beacon_chain/src/light_client_server_cache.rs +++ b/beacon_node/beacon_chain/src/light_client_server_cache.rs @@ -9,7 +9,7 @@ use store::DBColumn; use store::KeyValueStore; use tracing::debug; use tree_hash::TreeHash; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use types::{ BeaconBlockRef, BeaconState, ChainSpec, Checkpoint, EthSpec, ForkName, Hash256, LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, diff --git a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs index e8f1108657..72080b92da 100644 --- a/beacon_node/beacon_chain/src/naive_aggregation_pool.rs +++ b/beacon_node/beacon_chain/src/naive_aggregation_pool.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use tree_hash::{MerkleHasher, TreeHash, TreeHashType}; use types::SlotData; use types::consts::altair::SYNC_COMMITTEE_SUBNET_COUNT; -use types::sync_committee_contribution::SyncContributionData; +use types::sync_committee::SyncContributionData; use types::{ Attestation, AttestationData, AttestationRef, CommitteeIndex, EthSpec, Hash256, Slot, SyncCommitteeContribution, diff --git a/beacon_node/beacon_chain/src/pre_finalization_cache.rs b/beacon_node/beacon_chain/src/pre_finalization_cache.rs index 8996d6b874..54bd5c1940 100644 --- a/beacon_node/beacon_chain/src/pre_finalization_cache.rs +++ b/beacon_node/beacon_chain/src/pre_finalization_cache.rs @@ -6,7 +6,7 @@ use std::num::NonZeroUsize; use std::time::Duration; use tracing::debug; use types::Hash256; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; const BLOCK_ROOT_CACHE_LIMIT: NonZeroUsize = new_non_zero_usize(512); const LOOKUP_LIMIT: NonZeroUsize = new_non_zero_usize(8); diff --git a/beacon_node/beacon_chain/src/shuffling_cache.rs b/beacon_node/beacon_chain/src/shuffling_cache.rs index 618d459754..3d0fd80cf6 100644 --- a/beacon_node/beacon_chain/src/shuffling_cache.rs +++ b/beacon_node/beacon_chain/src/shuffling_cache.rs @@ -6,7 +6,7 @@ use oneshot_broadcast::{Receiver, Sender, oneshot}; use tracing::debug; use types::{ AttestationShufflingId, BeaconState, Epoch, EthSpec, Hash256, RelativeEpoch, - beacon_state::CommitteeCache, + state::CommitteeCache, }; use crate::{BeaconChainError, metrics}; diff --git a/beacon_node/beacon_chain/src/sync_committee_verification.rs b/beacon_node/beacon_chain/src/sync_committee_verification.rs index aacbb8cfec..2491e08eff 100644 --- a/beacon_node/beacon_chain/src/sync_committee_verification.rs +++ b/beacon_node/beacon_chain/src/sync_committee_verification.rs @@ -54,7 +54,7 @@ use types::sync_committee::SyncCommitteeError; use types::{ BeaconStateError, EthSpec, Hash256, SignedContributionAndProof, Slot, SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, - sync_committee_contribution::Error as ContributionError, + sync_committee::SyncCommitteeContributionError as ContributionError, }; /// Returned when a sync committee contribution was not successfully verified. It might not have been verified for diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 4fc6b3a379..b17a824fd7 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -2,7 +2,7 @@ use bls::PublicKeyBytes; use context_deserialize::ContextDeserialize; pub use eth2::Error; use eth2::types::beacon_response::EmptyMetadata; -use eth2::types::builder_bid::SignedBuilderBid; +use eth2::types::builder::SignedBuilderBid; use eth2::types::{ ContentType, EthSpec, ExecutionBlockHash, ForkName, ForkVersionDecode, ForkVersionedResponse, SignedValidatorRegistrationData, Slot, @@ -542,7 +542,7 @@ mod tests { use super::*; use bls::Signature; use eth2::types::MainnetEthSpec; - use eth2::types::builder_bid::{BuilderBid, BuilderBidFulu}; + use eth2::types::builder::{BuilderBid, BuilderBidFulu}; use eth2::types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; use mockito::{Matcher, Server, ServerGuard}; diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index c8488c94d6..97c8e8a625 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -5,9 +5,9 @@ use ssz::{Decode, TryFromIter}; use ssz_types::{FixedVector, VariableList, typenum::Unsigned}; use strum::EnumString; use superstruct::superstruct; -use types::beacon_block_body::KzgCommitments; use types::data::BlobsList; use types::execution::{ConsolidationRequests, DepositRequests, RequestType, WithdrawalRequests}; +use types::kzg_ext::KzgCommitments; use types::{Blob, KzgProof}; #[derive(Debug, PartialEq, Serialize, Deserialize)] diff --git a/beacon_node/execution_layer/src/engines.rs b/beacon_node/execution_layer/src/engines.rs index cc2bfcc7b6..3e6f78abbe 100644 --- a/beacon_node/execution_layer/src/engines.rs +++ b/beacon_node/execution_layer/src/engines.rs @@ -15,7 +15,7 @@ use tokio::sync::{Mutex, RwLock, watch}; use tokio_stream::wrappers::WatchStream; use tracing::{debug, error, info, warn}; use types::ExecutionBlockHash; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; /// The number of payload IDs that will be stored for each `Engine`. /// diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 37b514f480..33b83aab09 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -18,7 +18,7 @@ pub use engine_api::{http, http::HttpJsonRpc, http::deposit_methods}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; use eth2::types::{BlobsBundle, FullPayloadContents}; -use eth2::types::{ForkVersionedResponse, builder_bid::SignedBuilderBid}; +use eth2::types::{ForkVersionedResponse, builder::SignedBuilderBid}; use fixed_bytes::UintExtended; use fork_choice::ForkchoiceUpdateParameters; use logging::crit; @@ -45,10 +45,10 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tracing::{Instrument, debug, debug_span, error, info, instrument, warn}; use tree_hash::TreeHash; -use types::beacon_block_body::KzgCommitments; -use types::builder_bid::BuilderBid; +use types::builder::BuilderBid; use types::execution::BlockProductionVersion; -use types::non_zero_usize::new_non_zero_usize; +use types::kzg_ext::KzgCommitments; +use types::new_non_zero_usize; use types::{ AbstractExecPayload, BlobsList, ExecutionPayloadDeneb, ExecutionRequests, KzgProofs, SignedBlindedBeaconBlock, diff --git a/beacon_node/execution_layer/src/payload_cache.rs b/beacon_node/execution_layer/src/payload_cache.rs index 26ae89b5cb..ce65a53ef1 100644 --- a/beacon_node/execution_layer/src/payload_cache.rs +++ b/beacon_node/execution_layer/src/payload_cache.rs @@ -3,7 +3,7 @@ use lru::LruCache; use parking_lot::Mutex; use std::num::NonZeroUsize; use tree_hash::TreeHash; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use types::{EthSpec, Hash256}; pub const DEFAULT_PAYLOAD_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(10); diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 884aa9bf47..0016db9e0c 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -29,7 +29,7 @@ use tokio_stream::StreamExt; use tracing::{debug, error, info, warn}; use tree_hash::TreeHash; use types::ExecutionBlockHash; -use types::builder_bid::{ +use types::builder::{ BuilderBid, BuilderBidBellatrix, BuilderBidCapella, BuilderBidDeneb, BuilderBidElectra, BuilderBidFulu, SignedBuilderBid, }; diff --git a/beacon_node/http_api/src/block_rewards.rs b/beacon_node/http_api/src/block_rewards.rs index 29b23e89a7..891f024bf9 100644 --- a/beacon_node/http_api/src/block_rewards.rs +++ b/beacon_node/http_api/src/block_rewards.rs @@ -5,8 +5,8 @@ use state_processing::BlockReplayer; use std::num::NonZeroUsize; use std::sync::Arc; use tracing::{debug, warn}; -use types::beacon_block::BlindedBeaconBlock; -use types::non_zero_usize::new_non_zero_usize; +use types::block::BlindedBeaconBlock; +use types::new_non_zero_usize; use warp_utils::reject::{beacon_state_error, custom_bad_request, unhandled_error}; const STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(2); diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index a8c87523a5..939eca3b94 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -51,7 +51,7 @@ use types::{ChainSpec, EnrForkId, EthSpec}; mod subnet_predicate; use crate::discovery::enr::{NEXT_FORK_DIGEST_ENR_KEY, PEERDAS_CUSTODY_GROUP_COUNT_ENR_KEY}; pub use subnet_predicate::subnet_predicate; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; /// Local ENR storage filename. pub const ENR_FILENAME: &str = "enr.dat"; diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index dbcf39ac6e..0539877c72 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use strum::IntoStaticStr; use superstruct::superstruct; use types::data::BlobIdentifier; -use types::light_client_update::MAX_REQUEST_LIGHT_CLIENT_UPDATES; +use types::light_client::consts::MAX_REQUEST_LIGHT_CLIENT_UPDATES; use types::{ ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnsByRootIdentifier, Epoch, EthSpec, ForkContext, Hash256, LightClientBootstrap, LightClientFinalityUpdate, diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index eb70147c6e..ca25912934 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -40,7 +40,7 @@ use types::{ DataColumnSubnetId, EthSpec, Hash256, IndexedAttestation, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SingleAttestation, - Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, beacon_block::BlockImportSource, + Slot, SubnetId, SyncCommitteeMessage, SyncSubnetId, block::BlockImportSource, }; use beacon_processor::work_reprocessing_queue::QueuedColumnReconstruction; diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index c3a0cb2a86..38de6dffad 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -31,8 +31,8 @@ use std::sync::Arc; use std::time::Duration; use store::KzgCommitment; use tracing::{debug, debug_span, error, info, instrument, warn}; -use types::beacon_block_body::format_kzg_commitments; use types::data::FixedBlobSidecarList; +use types::kzg_ext::format_kzg_commitments; use types::{BlockImportSource, DataColumnSidecarList, Epoch, Hash256}; /// Id associated to a batch processing request, either a sync batch or a parent lookup. diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index 897a7e5ecc..045adfebe0 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -8,8 +8,8 @@ use state_processing::common::{ use std::collections::HashMap; use types::{ Attestation, BeaconState, ChainSpec, EthSpec, - beacon_state::BeaconStateBase, consts::altair::{PARTICIPATION_FLAG_WEIGHTS, PROPOSER_WEIGHT, WEIGHT_DENOMINATOR}, + state::BeaconStateBase, }; pub const PROPOSER_REWARD_DENOMINATOR: u64 = diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 0aa00e659b..29705283fa 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -8,7 +8,7 @@ use std::num::NonZeroUsize; use strum::{Display, EnumString, VariantNames}; use superstruct::superstruct; use types::EthSpec; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use zstd::{Decoder, Encoder}; #[cfg(all(feature = "redb", not(feature = "leveldb")))] diff --git a/beacon_node/store/src/hdiff.rs b/beacon_node/store/src/hdiff.rs index 323c87a914..3777c83b60 100644 --- a/beacon_node/store/src/hdiff.rs +++ b/beacon_node/store/src/hdiff.rs @@ -11,7 +11,7 @@ use std::ops::RangeInclusive; use std::str::FromStr; use std::sync::LazyLock; use superstruct::superstruct; -use types::historical_summary::HistoricalSummary; +use types::state::HistoricalSummary; use types::{BeaconState, ChainSpec, Epoch, EthSpec, Hash256, Slot, Validator}; static EMPTY_PUBKEY: LazyLock = LazyLock::new(PublicKeyBytes::empty); diff --git a/consensus/state_processing/src/epoch_cache.rs b/consensus/state_processing/src/epoch_cache.rs index ee03596d09..b890694a7e 100644 --- a/consensus/state_processing/src/epoch_cache.rs +++ b/consensus/state_processing/src/epoch_cache.rs @@ -5,7 +5,7 @@ use crate::metrics; use fixed_bytes::FixedBytesExtended; use safe_arith::SafeArith; use tracing::instrument; -use types::epoch_cache::{EpochCache, EpochCacheError, EpochCacheKey}; +use types::state::{EpochCache, EpochCacheError, EpochCacheKey}; use types::{ActivationQueue, BeaconState, ChainSpec, EthSpec, ForkName, Hash256}; /// Precursor to an `EpochCache`. diff --git a/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs index 9e8a36b6d5..34ead069f8 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/inactivity_updates.rs @@ -1,8 +1,7 @@ use crate::EpochProcessingError; use crate::per_epoch_processing::single_pass::{SinglePassConfig, process_epoch_single_pass}; -use types::beacon_state::BeaconState; -use types::chain_spec::ChainSpec; -use types::eth_spec::EthSpec; +use types::core::{ChainSpec, EthSpec}; +use types::state::BeaconState; /// Slow version of `process_inactivity_updates` that runs a subset of single-pass processing. /// diff --git a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs index 899d0dbb7d..f9b6043754 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs @@ -1,8 +1,8 @@ use crate::EpochProcessingError; use milhouse::List; use types::attestation::ParticipationFlags; -use types::beacon_state::BeaconState; -use types::eth_spec::EthSpec; +use types::core::EthSpec; +use types::state::BeaconState; pub fn process_participation_flag_updates( state: &mut BeaconState, diff --git a/consensus/state_processing/src/per_epoch_processing/altair/sync_committee_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/sync_committee_updates.rs index 3bb2ced982..12e80d608b 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/sync_committee_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/sync_committee_updates.rs @@ -1,9 +1,8 @@ use crate::EpochProcessingError; use safe_arith::SafeArith; use std::sync::Arc; -use types::beacon_state::BeaconState; -use types::chain_spec::ChainSpec; -use types::eth_spec::EthSpec; +use types::core::{ChainSpec, EthSpec}; +use types::state::BeaconState; pub fn process_sync_committee_updates( state: &mut BeaconState, diff --git a/consensus/state_processing/src/per_epoch_processing/base/participation_record_updates.rs b/consensus/state_processing/src/per_epoch_processing/base/participation_record_updates.rs index 52646e2269..74f71ad3bc 100644 --- a/consensus/state_processing/src/per_epoch_processing/base/participation_record_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/base/participation_record_updates.rs @@ -1,6 +1,6 @@ use crate::EpochProcessingError; -use types::beacon_state::BeaconState; -use types::eth_spec::EthSpec; +use types::core::EthSpec; +use types::state::BeaconState; pub fn process_participation_record_updates( state: &mut BeaconState, diff --git a/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs b/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs index 00adabdcfe..92ba3326f5 100644 --- a/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs +++ b/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs @@ -1,6 +1,6 @@ use crate::EpochProcessingError; use safe_arith::SafeArith; -use types::historical_summary::HistoricalSummary; +use types::state::HistoricalSummary; use types::{BeaconState, EthSpec}; pub fn process_historical_summaries_update( diff --git a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs index 8daad83a15..6d00cb77ff 100644 --- a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs @@ -1,8 +1,8 @@ use super::errors::EpochProcessingError; use crate::per_epoch_processing::single_pass::{SinglePassConfig, process_epoch_single_pass}; use safe_arith::SafeArith; -use types::beacon_state::BeaconState; -use types::chain_spec::ChainSpec; +use types::core::ChainSpec; +use types::state::BeaconState; use types::{BeaconStateError, EthSpec}; /// This implementation is now only used in phase0. Later hard forks use single-pass. diff --git a/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs b/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs index 9172d954bc..1bb814af05 100644 --- a/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs +++ b/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs @@ -2,8 +2,8 @@ use super::errors::EpochProcessingError; use safe_arith::SafeArith; use tree_hash::TreeHash; use typenum::Unsigned; -use types::beacon_state::BeaconState; -use types::eth_spec::EthSpec; +use types::core::EthSpec; +use types::state::BeaconState; pub fn process_historical_roots_update( state: &mut BeaconState, diff --git a/consensus/state_processing/src/per_epoch_processing/resets.rs b/consensus/state_processing/src/per_epoch_processing/resets.rs index e05fb30c33..c00cb7d635 100644 --- a/consensus/state_processing/src/per_epoch_processing/resets.rs +++ b/consensus/state_processing/src/per_epoch_processing/resets.rs @@ -2,8 +2,8 @@ use super::errors::EpochProcessingError; use milhouse::List; use safe_arith::SafeArith; use typenum::Unsigned; -use types::beacon_state::BeaconState; -use types::eth_spec::EthSpec; +use types::core::EthSpec; +use types::state::BeaconState; pub fn process_eth1_data_reset( state: &mut BeaconState, diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 0a02d100e6..f6d94331de 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -49,60 +49,6 @@ pub use sync_committee::*; pub use validator::*; pub use withdrawal::*; -// Temporary facade modules to maintain backwards compatibility for Lighthouse. -pub mod eth_spec { - pub use crate::core::EthSpec; -} - -pub mod chain_spec { - pub use crate::core::ChainSpec; -} - -pub mod beacon_block { - pub use crate::block::{BlindedBeaconBlock, BlockImportSource}; -} - -pub mod beacon_block_body { - pub use crate::kzg_ext::{KzgCommitments, format_kzg_commitments}; -} - -pub mod beacon_state { - pub use crate::state::{ - BeaconState, BeaconStateBase, CommitteeCache, compute_committee_index_in_epoch, - compute_committee_range_in_epoch, epoch_committee_count, - }; -} - pub mod graffiti { - pub use crate::core::GraffitiString; -} - -pub mod historical_summary { - pub use crate::state::HistoricalSummary; -} - -pub mod epoch_cache { - pub use crate::state::{EpochCache, EpochCacheError, EpochCacheKey}; -} - -pub mod non_zero_usize { - pub use crate::core::new_non_zero_usize; -} - -pub mod builder_bid { - pub use crate::builder::*; -} - -pub mod light_client_update { - pub use crate::light_client::consts::{ - CURRENT_SYNC_COMMITTEE_INDEX, CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA, FINALIZED_ROOT_INDEX, - FINALIZED_ROOT_INDEX_ELECTRA, MAX_REQUEST_LIGHT_CLIENT_UPDATES, NEXT_SYNC_COMMITTEE_INDEX, - NEXT_SYNC_COMMITTEE_INDEX_ELECTRA, - }; -} - -pub mod sync_committee_contribution { - pub use crate::sync_committee::{ - SyncCommitteeContributionError as Error, SyncContributionData, - }; + pub use crate::core::{GRAFFITI_BYTES_LEN, Graffiti, GraffitiString}; } diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 207324ea33..69817e5c9d 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -24,7 +24,7 @@ use std::str::FromStr; use std::string::ToString; use std::time::Duration; use tempfile::TempDir; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use types::{Address, Checkpoint, Epoch, Hash256, MainnetEthSpec}; const DEFAULT_EXECUTION_ENDPOINT: &str = "http://localhost:8551/"; diff --git a/slasher/src/config.rs b/slasher/src/config.rs index 144016efd2..50b4b7a858 100644 --- a/slasher/src/config.rs +++ b/slasher/src/config.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use std::num::NonZeroUsize; use std::path::PathBuf; use strum::{Display, EnumString, VariantNames}; -use types::non_zero_usize::new_non_zero_usize; +use types::new_non_zero_usize; use types::{Epoch, EthSpec, IndexedAttestation}; pub const DEFAULT_CHUNK_SIZE: usize = 16; diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index 52f5333df1..4aa9f980d4 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -6,7 +6,7 @@ use tree_hash::Hash256; use typenum::Unsigned; use types::{ BeaconBlockBody, BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, - BeaconBlockBodyFulu, BeaconBlockBodyGloas, BeaconState, FullPayload, light_client_update, + BeaconBlockBodyFulu, BeaconBlockBodyGloas, BeaconState, FullPayload, light_client, }; #[derive(Debug, Clone, Deserialize)] @@ -97,16 +97,16 @@ impl Case for BeaconStateMerkleProofValidity { state.update_tree_hash_cache().unwrap(); let proof = match self.merkle_proof.leaf_index { - light_client_update::CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA - | light_client_update::CURRENT_SYNC_COMMITTEE_INDEX => { + light_client::consts::CURRENT_SYNC_COMMITTEE_INDEX_ELECTRA + | light_client::consts::CURRENT_SYNC_COMMITTEE_INDEX => { state.compute_current_sync_committee_proof() } - light_client_update::NEXT_SYNC_COMMITTEE_INDEX_ELECTRA - | light_client_update::NEXT_SYNC_COMMITTEE_INDEX => { + light_client::consts::NEXT_SYNC_COMMITTEE_INDEX_ELECTRA + | light_client::consts::NEXT_SYNC_COMMITTEE_INDEX => { state.compute_next_sync_committee_proof() } - light_client_update::FINALIZED_ROOT_INDEX_ELECTRA - | light_client_update::FINALIZED_ROOT_INDEX => state.compute_finalized_root_proof(), + light_client::consts::FINALIZED_ROOT_INDEX_ELECTRA + | light_client::consts::FINALIZED_ROOT_INDEX => state.compute_finalized_root_proof(), _ => { return Err(Error::FailedToParseTest( "Could not retrieve merkle proof, invalid index".to_string(), diff --git a/testing/ef_tests/src/type_name.rs b/testing/ef_tests/src/type_name.rs index 2d13a6daa1..6dd1df1d8d 100644 --- a/testing/ef_tests/src/type_name.rs +++ b/testing/ef_tests/src/type_name.rs @@ -1,5 +1,5 @@ //! Mapping from types to canonical string identifiers used in testing. -use types::historical_summary::HistoricalSummary; +use types::state::HistoricalSummary; use types::*; pub trait TypeName { diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index 0cec69c97e..b2f65db6f3 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -239,7 +239,7 @@ macro_rules! ssz_static_test_no_run { #[cfg(feature = "fake_crypto")] mod ssz_static { use ef_tests::{Handler, SszStaticHandler, SszStaticTHCHandler, SszStaticWithSpecHandler}; - use types::historical_summary::HistoricalSummary; + use types::state::HistoricalSummary; use types::{ AttesterSlashingBase, AttesterSlashingElectra, ConsolidationRequest, DepositRequest, LightClientBootstrapAltair, PendingDeposit, PendingPartialWithdrawal, WithdrawalRequest, *,