Satisfy Clippy, remove non-tree-states code

This commit is contained in:
Michael Sproul
2022-03-28 11:42:55 +11:00
parent 705cba6443
commit c5212d0f98
45 changed files with 129 additions and 1153 deletions

View File

@@ -138,7 +138,7 @@ test-full: cargo-fmt test-release test-debug test-ef test-exec-engine
# Lints the code for bad style and potentially unsafe arithmetic using Clippy.
# Clippy lints are opt-in per-crate for now. By default, everything is allowed except for performance and correctness lints.
lint:
cargo clippy --workspace --tests --features lighthouse/tree-states -- \
cargo clippy --workspace --tests -- \
-D clippy::fn_to_numeric_cast_any \
-D warnings \
-A clippy::from-over-into \

View File

@@ -13,7 +13,6 @@ node_test_rig = { path = "../testing/node_test_rig" }
[features]
write_ssz_files = ["beacon_chain/write_ssz_files"] # Writes debugging .ssz files to /tmp during block processing.
tree-states = ["beacon_chain/tree-states"]
[dependencies]
eth2_config = { path = "../common/eth2_config" }

View File

@@ -10,7 +10,6 @@ default = ["participation_metrics"]
write_ssz_files = [] # Writes debugging .ssz files to /tmp during block processing.
participation_metrics = [] # Exposes validator participation metrics to Prometheus.
fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable
tree-states = ["store/milhouse"]
[dev-dependencies]
maplit = "1.0.2"

View File

@@ -84,7 +84,6 @@ use std::time::{Duration, Instant};
use store::iter::{BlockRootsIterator, ParentRootBlockIterator, StateRootsIterator};
use store::{Error as DBError, HotColdDB, KeyValueStore, KeyValueStoreOp, StoreItem, StoreOp};
use task_executor::ShutdownReason;
use types::beacon_state::CloneConfig;
use types::*;
pub type ForkChoiceError = fork_choice::Error<crate::ForkChoiceStoreError>;
@@ -545,12 +544,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let iter = self.store.forwards_block_roots_iterator_until(
start_slot,
end_slot,
|| {
(
head.beacon_state.clone_with_only_committee_caches(),
head.beacon_block_root,
)
},
|| (head.beacon_state.clone(), head.beacon_block_root),
&self.spec,
)?;
Ok(iter
@@ -713,12 +707,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let iter = self.store.forwards_state_roots_iterator_until(
start_slot,
end_slot,
|| {
(
head.beacon_state.clone_with_only_committee_caches(),
head.beacon_state_root(),
)
},
|| (head.beacon_state.clone(), head.beacon_state_root()),
&self.spec,
)?;
Ok(iter
@@ -993,7 +982,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
/// is the state as it was when the head block was received, which could be some slots prior to
/// now.
pub fn head(&self) -> Result<BeaconSnapshot<T::EthSpec>, Error> {
self.with_head(|head| Ok(head.clone_with(CloneConfig::committee_caches_only())))
self.with_head(|head| Ok(head.clone()))
}
/// Apply a function to the canonical head without cloning it.
@@ -1029,10 +1018,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// See `Self::head` for more information.
pub fn head_beacon_state(&self) -> Result<BeaconState<T::EthSpec>, Error> {
self.with_head(|s| {
Ok(s.beacon_state
.clone_with(CloneConfig::committee_caches_only()))
})
self.with_head(|s| Ok(s.beacon_state.clone()))
}
/// Return the sync committee at `slot + 1` from the canonical chain.
@@ -4209,11 +4195,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// to copy the head is liable to race-conditions.
let head_state_opt = self.with_head(|head| {
if head.beacon_block_root == head_block_root {
Ok(Some((
head.beacon_state
.clone_with(CloneConfig::committee_caches_only()),
head.beacon_state_root(),
)))
Ok(Some((head.beacon_state.clone(), head.beacon_state_root())))
} else {
Ok::<_, Error>(None)
}

View File

@@ -1,5 +1,5 @@
use serde_derive::Serialize;
use types::{beacon_state::CloneConfig, BeaconState, EthSpec, Hash256, SignedBeaconBlock};
use types::{BeaconState, EthSpec, Hash256, SignedBeaconBlock};
/// Represents some block and its associated state. Generally, this will be used for tracking the
/// head, justified head and finalized head.
@@ -57,12 +57,4 @@ impl<E: EthSpec> BeaconSnapshot<E> {
self.beacon_block_root = beacon_block_root;
self.beacon_state = beacon_state;
}
pub fn clone_with(&self, clone_config: CloneConfig) -> Self {
Self {
beacon_block: self.beacon_block.clone(),
beacon_block_root: self.beacon_block_root,
beacon_state: self.beacon_state.clone_with(clone_config),
}
}
}

View File

@@ -73,9 +73,9 @@ use std::io::Write;
use store::{Error as DBError, HotColdDB, KeyValueStore, StoreOp};
use tree_hash::TreeHash;
use types::{
BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec,
ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch,
SignedBeaconBlock, SignedBeaconBlockHeader, Slot,
BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExecutionBlockHash,
Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, SignedBeaconBlock,
SignedBeaconBlockHeader, Slot,
};
const POS_PANDA_BANNER: &str = r#"
@@ -1604,7 +1604,7 @@ fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec>(
parent_slot: state.slot(),
})
} else {
let mut state = state.clone_with(CloneConfig::committee_caches_only());
let mut state = state.clone();
let target_slot = block_epoch.start_slot(E::slots_per_epoch());
// Advance the state into the same epoch as the block. Use the "partial" method since state

View File

@@ -18,7 +18,7 @@ fn get_summary_v1<T: BeaconChainTypes>(
state_root: Hash256,
) -> Result<HotStateSummaryV1, Error> {
db.get_item(&state_root)?
.ok_or(HotColdDBError::MissingHotStateSummary(state_root).into())
.ok_or_else(|| HotColdDBError::MissingHotStateSummary(state_root).into())
}
fn get_state_by_replay<T: BeaconChainTypes>(
@@ -30,7 +30,7 @@ fn get_state_by_replay<T: BeaconChainTypes>(
slot,
latest_block_root,
epoch_boundary_state_root,
} = get_summary_v1::<T>(&db, state_root)?;
} = get_summary_v1::<T>(db, state_root)?;
// Load full state from the epoch boundary.
let (epoch_boundary_state, _) = db.load_hot_state_full(&epoch_boundary_state_root)?;

View File

@@ -109,14 +109,15 @@ mod test {
}
for v in state.validators() {
let creds = v.withdrawal_credentials.as_bytes();
let creds = v.withdrawal_credentials();
assert_eq!(
creds[0], spec.bls_withdrawal_prefix_byte,
creds.as_bytes()[0],
spec.bls_withdrawal_prefix_byte,
"first byte of withdrawal creds should be bls prefix"
);
assert_eq!(
&creds[1..],
&hash(&v.pubkey.as_ssz_bytes())[1..],
&creds.as_bytes()[1..],
&hash(&v.pubkey().as_ssz_bytes())[1..],
"rest of withdrawal creds should be pubkey hash"
)
}

View File

@@ -7,9 +7,7 @@ use beacon_chain::{
use eth2::types::{self as api_types};
use slot_clock::SlotClock;
use state_processing::state_advance::partial_state_advance;
use types::{
AttestationDuty, BeaconState, ChainSpec, CloneConfig, Epoch, EthSpec, Hash256, RelativeEpoch,
};
use types::{AttestationDuty, BeaconState, ChainSpec, Epoch, EthSpec, Hash256, RelativeEpoch};
/// The struct that is returned to the requesting HTTP client.
type ApiDuties = api_types::DutiesResponse<Vec<api_types::AttesterData>>;
@@ -82,11 +80,7 @@ fn compute_historic_attester_duties<T: BeaconChainTypes>(
let state_opt = chain
.with_head(|head| {
if head.beacon_state.current_epoch() <= request_epoch {
Ok(Some((
head.beacon_state_root(),
head.beacon_state
.clone_with(CloneConfig::committee_caches_only()),
)))
Ok(Some((head.beacon_state_root(), head.beacon_state.clone())))
} else {
Ok(None)
}

View File

@@ -10,7 +10,7 @@ use safe_arith::SafeArith;
use slog::{debug, Logger};
use slot_clock::SlotClock;
use std::cmp::Ordering;
use types::{CloneConfig, Epoch, EthSpec, Hash256, Slot};
use types::{Epoch, EthSpec, Hash256, Slot};
/// The struct that is returned to the requesting HTTP client.
type ApiDuties = api_types::DutiesResponse<Vec<api_types::ProposerData>>;
@@ -158,11 +158,7 @@ fn compute_historic_proposer_duties<T: BeaconChainTypes>(
let state_opt = chain
.with_head(|head| {
if head.beacon_state.current_epoch() <= epoch {
Ok(Some((
head.beacon_state_root(),
head.beacon_state
.clone_with(CloneConfig::committee_caches_only()),
)))
Ok(Some((head.beacon_state_root(), head.beacon_state.clone())))
} else {
Ok(None)
}

View File

@@ -30,6 +30,3 @@ tree_hash = "0.4.0"
take-until = "0.1.0"
zstd = "0.10.0"
strum = { version = "0.24", features = ["derive"] }
[features]
milhouse = ["state_processing/milhouse"]

View File

@@ -558,17 +558,6 @@ pub fn load_variable_list_from_db<F: VariableLengthField<E>, E: EthSpec, S: KeyV
Ok(result)
}
/// Index into a field of the state, avoiding out of bounds and division by 0.
#[cfg(not(feature = "milhouse"))]
fn safe_modulo_index<T: Copy>(values: &[T], index: u64) -> Result<T, ChunkError> {
if values.is_empty() {
Err(ChunkError::ZeroLengthVector)
} else {
Ok(values[index as usize % values.len()])
}
}
#[cfg(feature = "milhouse")]
fn safe_modulo_index_list<T: TreeHash + Copy, N: Unsigned>(
values: &VList<T, N>,
index: u64,
@@ -583,7 +572,6 @@ fn safe_modulo_index_list<T: TreeHash + Copy, N: Unsigned>(
}
}
#[cfg(feature = "milhouse")]
fn safe_modulo_index_vect<T: TreeHash + Copy, N: Unsigned>(
values: &FixedVector<T, N>,
index: u64,

View File

@@ -3,10 +3,7 @@ use crate::config::StoreConfigError;
use crate::hot_cold_store::HotColdDBError;
use ssz::DecodeError;
use state_processing::BlockReplayError;
use types::{BeaconStateError, Hash256, Slot};
#[cfg(feature = "milhouse")]
use types::milhouse;
use types::{milhouse, BeaconStateError, Hash256, Slot};
pub type Result<T> = std::result::Result<T, Error>;
@@ -46,7 +43,6 @@ pub enum Error {
MissingStateRoot(Slot),
MissingState(Hash256),
BlockReplayError(BlockReplayError),
#[cfg(feature = "milhouse")]
MilhouseError(milhouse::Error),
Compression(std::io::Error),
MissingPersistedBeaconChain,
@@ -110,7 +106,6 @@ impl From<StoreConfigError> for Error {
}
}
#[cfg(feature = "milhouse")]
impl From<milhouse::Error> for Error {
fn from(e: milhouse::Error) -> Self {
Self::MilhouseError(e)

View File

@@ -1037,13 +1037,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
.minimal_block_root_verification()
.state_root_iter(state_root_iter)
.apply_blocks(blocks, Some(target_slot))
.and_then(|block_replayer| {
.map(|block_replayer| {
// FIXME(sproul): tweak state miss condition
if block_replayer.state_root_miss() && false {
/*
if block_replayer.state_root_miss() {
Err(Error::MissingStateRoot(target_slot))
} else {
Ok(block_replayer.into_state())
}
*/
block_replayer.into_state()
})
}

View File

@@ -3,7 +3,7 @@ use ssz::{DecodeError, Encode};
use ssz_derive::Encode;
use std::convert::TryInto;
use std::sync::Arc;
use types::beacon_state::{CloneConfig, CommitteeCache, CACHED_EPOCHS};
use types::beacon_state::{CommitteeCache, CACHED_EPOCHS};
pub fn store_full_state<E: EthSpec>(
state_root: &Hash256,
@@ -56,7 +56,7 @@ impl<T: EthSpec> StorageContainer<T> {
/// Create a new instance for storing a `BeaconState`.
pub fn new(state: &BeaconState<T>) -> Self {
Self {
state: state.clone_with(CloneConfig::none()),
state: state.clone(),
committee_caches: state.committee_caches().to_vec(),
}
}

View File

@@ -191,10 +191,11 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
Box::new(
iter.take_while(move |(key, _)| key.matches_column(column))
.map(move |(bytes_key, value)| {
let key = bytes_key.remove_column(column).ok_or_else(|| {
HotColdDBError::IterationError {
let key =
bytes_key
.remove_column(column)
.ok_or(HotColdDBError::IterationError {
unexpected_key: bytes_key,
}
})?;
Ok((key, value))
}),

View File

@@ -44,6 +44,8 @@ use parking_lot::MutexGuard;
use strum::{EnumString, IntoStaticStr};
pub use types::*;
pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
/// Retrieve some bytes in `column` with `key`.
fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
@@ -78,10 +80,7 @@ pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
fn compact(&self) -> Result<(), Error>;
/// Iterate through all values in a particular column.
fn iter_column<'a>(
&'a self,
_column: DBColumn,
) -> Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a> {
fn iter_column(&self, _column: DBColumn) -> ColumnIter {
// Default impl for non LevelDB databases
Box::new(std::iter::empty())
}

View File

@@ -318,8 +318,6 @@ macro_rules! impl_try_into_beacon_state {
committee_caches: <_>::default(),
pubkey_cache: <_>::default(),
exit_cache: <_>::default(),
#[cfg(not(feature = "milhouse"))]
tree_hash_cache: <_>::default(),
// Variant-specific fields
$(

View File

@@ -203,22 +203,3 @@ impl BlockMap {
self.blocks.remove(block_root)
}
}
#[cfg(test)]
mod test {
use super::*;
use std::mem::size_of;
use types::{
beacon_state::PubkeyCache, BeaconBlockHeader, BeaconState, BeaconStateAltair,
BeaconStateMerge, MainnetEthSpec,
};
#[test]
fn state_size() {
println!("{}", size_of::<BeaconStateAltair<MainnetEthSpec>>());
println!("{}", size_of::<BeaconStateMerge<MainnetEthSpec>>());
println!("{}", size_of::<BeaconState<MainnetEthSpec>>());
println!("{}", size_of::<PubkeyCache>());
assert!(false);
}
}

View File

@@ -32,7 +32,6 @@ default = ["legacy-arith", "metrics"]
fake_crypto = ["bls/fake_crypto"]
legacy-arith = ["types/legacy-arith"]
metrics = ["lighthouse_metrics", "lazy_static"]
milhouse = ["types/milhouse"]
arbitrary-fuzz = [
"arbitrary",
"types/arbitrary-fuzz",

View File

@@ -217,15 +217,11 @@ where
pre_block_hook(&mut self.state, block)?;
}
let verify_block_root = self.verify_block_root.unwrap_or_else(|| {
// If no explicit policy is set, verify only the first 1 or 2 block roots if using
// accurate state roots. Inaccurate state roots require block root verification to
// be off.
if i <= 1 {
// If no explicit policy is set, verify only the first 1 or 2 block roots.
let verify_block_root = self.verify_block_root.unwrap_or(if i <= 1 {
VerifyBlockRoot::True
} else {
VerifyBlockRoot::False
}
});
let mut ctxt = ConsensusContext::new(block.slot());
per_block_processing(

View File

@@ -72,7 +72,6 @@ pub enum BlockProcessingError {
},
ExecutionInvalid,
ConsensusContext(ContextError),
#[cfg(feature = "milhouse")]
MilhouseError(milhouse::Error),
}
@@ -112,7 +111,6 @@ impl From<ContextError> for BlockProcessingError {
}
}
#[cfg(feature = "milhouse")]
impl From<milhouse::Error> for BlockProcessingError {
fn from(e: milhouse::Error) -> Self {
Self::MilhouseError(e)

View File

@@ -7,8 +7,8 @@ use crate::per_block_processing::errors::{
ProposerSlashingInvalid,
};
use crate::{
per_block_processing::process_operations, BlockSignatureStrategy, VerifyBlockRoot,
VerifySignatures,
per_block_processing::process_operations, BlockSignatureStrategy, ConsensusContext,
VerifyBlockRoot, VerifySignatures,
};
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use lazy_static::lazy_static;
@@ -63,12 +63,13 @@ fn valid_block_ok() {
let slot = state.slot();
let (block, mut state) = harness.make_block_return_pre_state(state, slot + Slot::new(1));
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&block,
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
@@ -87,12 +88,13 @@ fn invalid_block_header_state_slot() {
let (mut block, signature) = signed_block.deconstruct();
*block.slot_mut() = slot + Slot::new(1);
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, signature),
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
@@ -116,12 +118,13 @@ fn invalid_parent_block_root() {
let (mut block, signature) = signed_block.deconstruct();
*block.parent_root_mut() = Hash256::from([0xAA; 32]);
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, signature),
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
@@ -146,12 +149,13 @@ fn invalid_block_signature() {
let (signed_block, mut state) = harness.make_block_return_pre_state(state, slot + Slot::new(1));
let (block, _) = signed_block.deconstruct();
let mut ctxt = ConsensusContext::new(block.slot());
let result = per_block_processing(
&mut state,
&SignedBeaconBlock::from_block(block, Signature::empty()),
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);
@@ -176,12 +180,13 @@ fn invalid_randao_reveal_signature() {
*block.body_mut().randao_reveal_mut() = Signature::empty();
});
let mut ctxt = ConsensusContext::new(signed_block.slot());
let result = per_block_processing(
&mut state,
&signed_block,
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&spec,
);

View File

@@ -1,8 +1,5 @@
use crate::per_epoch_processing::altair::participation_cache::Error as ParticipationCacheError;
use types::{BeaconStateError, InconsistentFork};
#[cfg(feature = "milhouse")]
use types::milhouse;
use types::{milhouse, BeaconStateError, InconsistentFork};
#[derive(Debug, PartialEq)]
pub enum EpochProcessingError {
@@ -28,7 +25,6 @@ pub enum EpochProcessingError {
InvalidJustificationBit(ssz_types::Error),
InvalidFlagIndex(usize),
ParticipationCache(ParticipationCacheError),
#[cfg(feature = "milhouse")]
MilhouseError(milhouse::Error),
}
@@ -62,7 +58,6 @@ impl From<ParticipationCacheError> for EpochProcessingError {
}
}
#[cfg(feature = "milhouse")]
impl From<milhouse::Error> for EpochProcessingError {
fn from(e: milhouse::Error) -> Self {
Self::MilhouseError(e)

View File

@@ -104,8 +104,6 @@ pub fn upgrade_to_altair<E: EthSpec>(
committee_caches: mem::take(&mut pre.committee_caches),
pubkey_cache: mem::take(&mut pre.pubkey_cache),
exit_cache: mem::take(&mut pre.exit_cache),
#[cfg(not(feature = "milhouse"))]
tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
});
// Fill in previous epoch participation from the pre state's pending attestations.

View File

@@ -63,8 +63,6 @@ pub fn upgrade_to_bellatrix<E: EthSpec>(
committee_caches: mem::take(&mut pre.committee_caches),
pubkey_cache: mem::take(&mut pre.pubkey_cache),
exit_cache: mem::take(&mut pre.exit_cache),
#[cfg(not(feature = "milhouse"))]
tree_hash_cache: mem::take(&mut pre.tree_hash_cache),
});
*pre_state = post;

View File

@@ -18,7 +18,7 @@ impl tree_hash::TreeHash for HashVec {
tree_hash::TreeHashType::List
}
fn tree_hash_packed_encoding(&self) -> Vec<u8> {
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("List should never be packed.")
}

View File

@@ -46,7 +46,7 @@ itertools = "0.10.0"
superstruct = "0.4.0"
serde_json = "1.0.74"
smallvec = "1.8.0"
milhouse = { path = "../../../milhouse", optional = true }
milhouse = { path = "../../../milhouse" }
rpds = "0.11.0"
[dev-dependencies]

View File

@@ -4,7 +4,6 @@ use crate::test_utils::TestRandom;
use crate::*;
use compare_fields::CompareFields;
use compare_fields_derive::CompareFields;
use derivative::Derivative;
use eth2_hashing::hash;
use int_to_bytes::{int_to_bytes4, int_to_bytes8};
pub use pubkey_cache::PubkeyCache;
@@ -25,33 +24,25 @@ pub use self::committee_cache::{
compute_committee_index_in_epoch, compute_committee_range_in_epoch, epoch_committee_count,
CommitteeCache,
};
pub use clone_config::CloneConfig;
pub use diff::BeaconStateDiff;
pub use eth_spec::*;
pub use iter::BlockRootsIter;
#[cfg(feature = "milhouse")]
pub use milhouse::{interface::Interface, List as VList, List, Vector as FixedVector};
#[cfg(not(feature = "milhouse"))]
pub use {
ssz_types::FixedVector, ssz_types::VariableList as VList, tree_hash_cache::BeaconTreeHashCache,
};
#[macro_use]
mod committee_cache;
mod clone_config;
mod diff;
mod exit_cache;
mod iter;
mod pubkey_cache;
mod tests;
#[cfg(not(feature = "milhouse"))]
mod tree_hash_cache;
pub const CACHED_EPOCHS: usize = 3;
const MAX_RANDOM_BYTE: u64 = (1 << 8) - 1;
pub type Validators<T> = VList<Validator, <T as EthSpec>::ValidatorRegistryLimit>;
pub type Balances<T> = VList<u64, <T as EthSpec>::ValidatorRegistryLimit>;
#[derive(Debug, PartialEq, Clone)]
pub enum Error {
/// A state for a different hard-fork was required -- a severe logic error.
@@ -134,7 +125,6 @@ pub enum Error {
current_epoch: Epoch,
epoch: Epoch,
},
#[cfg(feature = "milhouse")]
MilhouseError(milhouse::Error),
CommitteeCacheDiffInvalidEpoch {
prev_current_epoch: Epoch,
@@ -199,9 +189,9 @@ impl From<BeaconStateHash> for Hash256 {
variants(Base, Altair, Merge),
variant_attributes(
derive(
Derivative,
Debug,
PartialEq,
Clone,
Serialize,
Deserialize,
Encode,
@@ -211,13 +201,12 @@ impl From<BeaconStateHash> for Hash256 {
CompareFields,
),
serde(bound = "T: EthSpec", deny_unknown_fields),
derivative(Clone),
cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))
),
cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"),
partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant")
)]
#[derive(Debug, PartialEq, Serialize, Deserialize, Encode, TreeHash)]
#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash)]
#[serde(untagged)]
#[serde(bound = "T: EthSpec")]
#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
@@ -321,39 +310,22 @@ where
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]
#[test_random(default)]
#[derivative(Clone(clone_with = "clone_default"))]
pub total_active_balance: Option<(Epoch, u64)>,
#[serde(skip_serializing, skip_deserializing)]
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]
#[test_random(default)]
#[derivative(Clone(clone_with = "clone_default"))]
pub committee_caches: [Arc<CommitteeCache>; CACHED_EPOCHS],
#[serde(skip_serializing, skip_deserializing)]
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]
#[test_random(default)]
#[derivative(Clone(clone_with = "clone_default"))]
pub pubkey_cache: PubkeyCache,
#[serde(skip_serializing, skip_deserializing)]
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]
#[test_random(default)]
#[derivative(Clone(clone_with = "clone_default"))]
pub exit_cache: ExitCache,
#[serde(skip_serializing, skip_deserializing)]
#[ssz(skip_serializing, skip_deserializing)]
#[tree_hash(skip_hashing)]
#[test_random(default)]
#[derivative(Clone(clone_with = "clone_default"))]
#[cfg(not(feature = "milhouse"))]
pub tree_hash_cache: BeaconTreeHashCache<T>,
}
impl<T: EthSpec> Clone for BeaconState<T> {
fn clone(&self) -> Self {
self.clone_with(CloneConfig::all())
}
}
impl<T: EthSpec> BeaconState<T> {
@@ -413,8 +385,6 @@ impl<T: EthSpec> BeaconState<T> {
],
pubkey_cache: PubkeyCache::default(),
exit_cache: ExitCache::default(),
#[cfg(not(feature = "milhouse"))]
tree_hash_cache: <_>::default(),
})
}
@@ -1142,12 +1112,7 @@ impl<T: EthSpec> BeaconState<T> {
}
/// Convenience accessor for validators and balances simultaneously.
pub fn validators_and_balances_mut(
&mut self,
) -> (
&mut VList<Validator, T::ValidatorRegistryLimit>,
&mut VList<u64, T::ValidatorRegistryLimit>,
) {
pub fn validators_and_balances_mut(&mut self) -> (&mut Validators<T>, &mut Balances<T>) {
match self {
BeaconState::Base(state) => (&mut state.validators, &mut state.balances),
BeaconState::Altair(state) => (&mut state.validators, &mut state.balances),
@@ -1434,7 +1399,6 @@ impl<T: EthSpec> BeaconState<T> {
self.drop_committee_cache(RelativeEpoch::Current)?;
self.drop_committee_cache(RelativeEpoch::Next)?;
self.drop_pubkey_cache();
self.drop_tree_hash_cache();
*self.exit_cache_mut() = ExitCache::default();
Ok(())
}
@@ -1645,101 +1609,21 @@ impl<T: EthSpec> BeaconState<T> {
Ok(())
}
/// Initialize but don't fill the tree hash cache, if it isn't already initialized.
pub fn initialize_tree_hash_cache(&mut self) {
#[cfg(not(feature = "milhouse"))]
if !self.tree_hash_cache().is_initialized() {
*self.tree_hash_cache_mut() = BeaconTreeHashCache::new(self)
}
}
/// Compute the tree hash root of the state using the tree hash cache.
///
/// Initialize the tree hash cache if it isn't already initialized.
pub fn update_tree_hash_cache(&mut self) -> Result<Hash256, Error> {
#[cfg(not(feature = "milhouse"))]
{
self.initialize_tree_hash_cache();
let cache = self.tree_hash_cache_mut().take();
if let Some(mut cache) = cache {
// Note: we return early if the tree hash fails, leaving `self.tree_hash_cache` as
// None. There's no need to keep a cache that fails.
let root = cache.recalculate_tree_hash_root(self)?;
self.tree_hash_cache_mut().restore(cache);
Ok(root)
} else {
Err(Error::TreeHashCacheNotInitialized)
}
}
#[cfg(feature = "milhouse")]
{
self.apply_pending_mutations()?;
Ok(self.tree_hash_root())
}
}
/// Compute the tree hash root of the validators using the tree hash cache.
///
/// Initialize the tree hash cache if it isn't already initialized.
pub fn update_validators_tree_hash_cache(&mut self) -> Result<Hash256, Error> {
#[cfg(not(feature = "milhouse"))]
{
self.initialize_tree_hash_cache();
let cache = self.tree_hash_cache_mut().take();
if let Some(mut cache) = cache {
// Note: we return early if the tree hash fails, leaving `self.tree_hash_cache` as
// None. There's no need to keep a cache that fails.
let root = cache.recalculate_validators_tree_hash_root(self.validators())?;
self.tree_hash_cache_mut().restore(cache);
Ok(root)
} else {
Err(Error::TreeHashCacheNotInitialized)
}
}
#[cfg(feature = "milhouse")]
{
self.validators_mut().apply_updates()?;
Ok(self.validators().tree_hash_root())
}
}
/// Completely drops the tree hash cache, replacing it with a new, empty cache.
pub fn drop_tree_hash_cache(&mut self) {
#[cfg(not(feature = "milhouse"))]
self.tree_hash_cache_mut().uninitialize();
}
/// Clone the state whilst preserving only the selected caches.
pub fn clone_with(&self, config: CloneConfig) -> Self {
let mut res = match self {
BeaconState::Base(inner) => BeaconState::Base(inner.clone()),
BeaconState::Altair(inner) => BeaconState::Altair(inner.clone()),
BeaconState::Merge(inner) => BeaconState::Merge(inner.clone()),
};
if config.committee_caches {
*res.committee_caches_mut() = self.committee_caches().clone();
*res.total_active_balance_mut() = *self.total_active_balance();
}
if config.pubkey_cache {
*res.pubkey_cache_mut() = self.pubkey_cache().clone();
}
if config.exit_cache {
*res.exit_cache_mut() = self.exit_cache().clone();
}
#[cfg(not(feature = "milhouse"))]
if config.tree_hash_cache {
*res.tree_hash_cache_mut() = self.tree_hash_cache().clone();
}
res
}
pub fn clone_with_only_committee_caches(&self) -> Self {
self.clone_with(CloneConfig::committee_caches_only())
}
pub fn is_eligible_validator(&self, val: &Validator) -> bool {
let previous_epoch = self.previous_epoch();
@@ -1811,18 +1695,12 @@ impl From<ArithError> for Error {
}
}
#[cfg(feature = "milhouse")]
impl From<milhouse::Error> for Error {
fn from(e: milhouse::Error) -> Self {
Self::MilhouseError(e)
}
}
/// Helper function for "cloning" a field by using its default value.
fn clone_default<T: Default>(_value: &T) -> T {
T::default()
}
impl<T: EthSpec> CompareFields for BeaconState<T> {
fn compare_fields(&self, other: &Self) -> Vec<compare_fields::Comparison> {
match (self, other) {

View File

@@ -1,43 +0,0 @@
/// Configuration struct for controlling which caches of a `BeaconState` should be cloned.
#[derive(Debug, Default, PartialEq, Eq, Clone, Copy)]
pub struct CloneConfig {
pub committee_caches: bool,
pub pubkey_cache: bool,
pub exit_cache: bool,
pub tree_hash_cache: bool,
}
impl CloneConfig {
pub fn all() -> Self {
Self {
committee_caches: true,
pubkey_cache: true,
exit_cache: true,
tree_hash_cache: true,
}
}
pub fn none() -> Self {
Self::default()
}
pub fn committee_caches_only() -> Self {
Self {
committee_caches: true,
..Self::none()
}
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn sanity() {
assert!(CloneConfig::all().pubkey_cache);
assert!(!CloneConfig::none().tree_hash_cache);
assert!(CloneConfig::committee_caches_only().committee_caches);
assert!(!CloneConfig::committee_caches_only().exit_cache);
}
}

View File

@@ -84,11 +84,10 @@ fn shuffles_for_the_right_epoch() {
let mut state = new_state::<MinimalEthSpec>(num_validators, slot);
let spec = &MinimalEthSpec::default_spec();
let distinct_hashes: Vec<Hash256> = (0..MinimalEthSpec::epochs_per_historical_vector())
.map(|i| Hash256::from_low_u64_be(i as u64))
.collect();
let distinct_hashes = (0..MinimalEthSpec::epochs_per_historical_vector())
.map(|i| Hash256::from_low_u64_be(i as u64));
*state.randao_mixes_mut() = FixedVector::from(distinct_hashes);
*state.randao_mixes_mut() = FixedVector::try_from_iter(distinct_hashes).unwrap();
let previous_seed = state
.get_seed(state.previous_epoch(), Domain::BeaconAttester, spec)

View File

@@ -175,8 +175,8 @@ impl Diff for CommitteeCachesDiff {
let (current_epoch, caches) = other;
// Sanity check the inputs to ensure we can compute a sensible diff.
check_committee_caches(&prev_caches, *prev_current_epoch)?;
check_committee_caches(&caches, *current_epoch)?;
check_committee_caches(prev_caches, *prev_current_epoch)?;
check_committee_caches(caches, *current_epoch)?;
let dist = compute_committee_cache_dist(*current_epoch, *prev_current_epoch)?;

View File

@@ -74,7 +74,7 @@ mod test {
let mut state: BeaconState<E> = BeaconState::new(0, <_>::default(), &spec);
for i in 0..state.block_roots().len() {
state.block_roots_mut()[i] = root_slot(i).1;
*state.block_roots_mut().get_mut(i).unwrap() = root_slot(i).1;
}
assert_eq!(
@@ -122,7 +122,7 @@ mod test {
let mut state: BeaconState<E> = BeaconState::new(0, <_>::default(), &spec);
for i in 0..state.block_roots().len() {
state.block_roots_mut()[i] = root_slot(i).1;
*state.block_roots_mut().get_mut(i).unwrap() = root_slot(i).1;
}
assert_eq!(

View File

@@ -14,6 +14,7 @@ pub struct PubkeyCache {
impl PubkeyCache {
/// Returns the number of validator indices added to the map so far.
#[allow(clippy::len_without_is_empty)]
pub fn len(&self) -> ValidatorIndex {
self.len
}

View File

@@ -1,18 +1,13 @@
#![cfg(test)]
use crate::test_utils::*;
use crate::test_utils::{SeedableRng, XorShiftRng};
use beacon_chain::test_utils::{
interop_genesis_state, test_spec, BeaconChainHarness, EphemeralHarnessType,
DEFAULT_ETH1_BLOCK_HASH,
};
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use beacon_chain::types::{
test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError,
ChainSpec, CloneConfig, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec,
ChainSpec, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec,
MinimalEthSpec, RelativeEpoch, Slot,
};
use safe_arith::SafeArith;
use ssz::{Decode, Encode};
use state_processing::per_slot_processing;
use std::ops::Mul;
use swap_or_not_shuffle::compute_shuffled_index;
use tree_hash::TreeHash;
@@ -100,7 +95,11 @@ fn test_beacon_proposer_index<T: EthSpec>() {
// Test with two validators per slot, first validator has zero balance.
let mut state = build_state::<T>((T::slots_per_epoch() as usize).mul(2));
let slot0_candidate0 = ith_candidate(&state, Slot::new(0), 0, &spec);
state.validators_mut()[slot0_candidate0].effective_balance = 0;
state
.validators_mut()
.get_mut(slot0_candidate0)
.unwrap()
.effective_balance = 0;
test(&state, Slot::new(0), 1);
for i in 1..T::slots_per_epoch() {
test(&state, Slot::from(i), 0);
@@ -158,83 +157,6 @@ fn cache_initialization() {
test_cache_initialization(&mut state, RelativeEpoch::Next, &spec);
}
fn test_clone_config<E: EthSpec>(base_state: &BeaconState<E>, clone_config: CloneConfig) {
let state = base_state.clone_with(clone_config);
if clone_config.committee_caches {
state
.committee_cache(RelativeEpoch::Previous)
.expect("committee cache exists");
state
.committee_cache(RelativeEpoch::Current)
.expect("committee cache exists");
state
.committee_cache(RelativeEpoch::Next)
.expect("committee cache exists");
state
.total_active_balance()
.expect("total active balance exists");
} else {
state
.committee_cache(RelativeEpoch::Previous)
.expect_err("shouldn't exist");
state
.committee_cache(RelativeEpoch::Current)
.expect_err("shouldn't exist");
state
.committee_cache(RelativeEpoch::Next)
.expect_err("shouldn't exist");
}
if clone_config.pubkey_cache {
assert_ne!(state.pubkey_cache().len(), 0);
} else {
assert_eq!(state.pubkey_cache().len(), 0);
}
if clone_config.exit_cache {
state
.exit_cache()
.check_initialized()
.expect("exit cache exists");
} else {
state
.exit_cache()
.check_initialized()
.expect_err("exit cache doesn't exist");
}
if clone_config.tree_hash_cache {
assert!(state.tree_hash_cache().is_initialized());
} else {
assert!(
!state.tree_hash_cache().is_initialized(),
"{:?}",
clone_config
);
}
}
#[test]
fn clone_config() {
let spec = MinimalEthSpec::default_spec();
let mut state = build_state::<MinimalEthSpec>(16);
state.build_all_caches(&spec).unwrap();
state
.update_tree_hash_cache()
.expect("should update tree hash cache");
let num_caches = 4;
let all_configs = (0..2u8.pow(num_caches)).map(|i| CloneConfig {
committee_caches: (i & 1) != 0,
pubkey_cache: ((i >> 1) & 1) != 0,
exit_cache: ((i >> 2) & 1) != 0,
tree_hash_cache: ((i >> 3) & 1) != 0,
});
for config in all_configs {
test_clone_config(&state, config);
}
}
/// Tests committee-specific components
#[cfg(test)]
mod committees {
@@ -325,10 +247,9 @@ mod committees {
let harness = get_harness::<T>(validator_count, slot);
let mut new_head_state = harness.get_current_state();
let distinct_hashes: Vec<Hash256> = (0..T::epochs_per_historical_vector())
.map(|i| Hash256::from_low_u64_be(i as u64))
.collect();
*new_head_state.randao_mixes_mut() = FixedVector::from(distinct_hashes);
let distinct_hashes =
(0..T::epochs_per_historical_vector()).map(|i| Hash256::from_low_u64_be(i as u64));
*new_head_state.randao_mixes_mut() = FixedVector::try_from_iter(distinct_hashes).unwrap();
new_head_state
.force_build_committee_cache(RelativeEpoch::Previous, spec)
@@ -547,66 +468,3 @@ fn tree_hash_cache_linear_history() {
let root = state.update_tree_hash_cache().unwrap();
assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]);
}
// Check how the cache behaves when there's a distance larger than `SLOTS_PER_HISTORICAL_ROOT`
// since its last update.
#[test]
fn tree_hash_cache_linear_history_long_skip() {
let validator_count = 128;
let keypairs = generate_deterministic_keypairs(validator_count);
let spec = &test_spec::<MinimalEthSpec>();
// This state has a cache that advances normally each slot.
let mut state: BeaconState<MinimalEthSpec> = interop_genesis_state(
&keypairs,
0,
Hash256::from_slice(DEFAULT_ETH1_BLOCK_HASH),
None,
spec,
)
.unwrap();
state.update_tree_hash_cache().unwrap();
// This state retains its original cache until it is updated after a long skip.
let mut original_cache_state = state.clone();
assert!(original_cache_state.tree_hash_cache().is_initialized());
// Advance the states to a slot beyond the historical state root limit, using the state root
// from the first state to avoid touching the original state's cache.
let start_slot = state.slot();
let target_slot = start_slot
.safe_add(MinimalEthSpec::slots_per_historical_root() as u64 + 1)
.unwrap();
let mut prev_state_root;
while state.slot() < target_slot {
prev_state_root = state.update_tree_hash_cache().unwrap();
per_slot_processing(&mut state, None, spec).unwrap();
per_slot_processing(&mut original_cache_state, Some(prev_state_root), spec).unwrap();
}
// The state with the original cache should still be initialized at the starting slot.
assert_eq!(
original_cache_state
.tree_hash_cache()
.initialized_slot()
.unwrap(),
start_slot
);
// Updating the tree hash cache should be successful despite the long skip.
assert_eq!(
original_cache_state.update_tree_hash_cache().unwrap(),
state.update_tree_hash_cache().unwrap()
);
assert_eq!(
original_cache_state
.tree_hash_cache()
.initialized_slot()
.unwrap(),
target_slot
);
}

View File

@@ -1,645 +0,0 @@
#![allow(clippy::integer_arithmetic)]
#![allow(clippy::disallowed_methods)]
#![allow(clippy::indexing_slicing)]
use super::Error;
use crate::{BeaconState, EthSpec, Hash256, ParticipationList, Slot, Unsigned, Validator};
use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache};
use rayon::prelude::*;
use ssz_derive::{Decode, Encode};
use ssz_types::VariableList;
use std::cmp::Ordering;
use std::iter::ExactSizeIterator;
use tree_hash::{mix_in_length, MerkleHasher, TreeHash};
/// The number of leaves (including padding) on the `BeaconState` Merkle tree.
///
/// ## Note
///
/// This constant is set with the assumption that there are `> 16` and `<= 32` fields on the
/// `BeaconState`. **Tree hashing will fail if this value is set incorrectly.**
const NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES: usize = 32;
/// The number of nodes in the Merkle tree of a validator record.
const NODES_PER_VALIDATOR: usize = 15;
/// The number of validator record tree hash caches stored in each arena.
///
/// This is primarily used for concurrency; if we have 16 validators and set `VALIDATORS_PER_ARENA
/// == 8` then it is possible to do a 2-core concurrent hash.
///
/// Do not set to 0.
const VALIDATORS_PER_ARENA: usize = 4_096;
#[derive(Debug, PartialEq, Clone, Encode, Decode)]
pub struct Eth1DataVotesTreeHashCache<T: EthSpec> {
arena: CacheArena,
tree_hash_cache: TreeHashCache,
voting_period: u64,
roots: VariableList<Hash256, T::SlotsPerEth1VotingPeriod>,
}
impl<T: EthSpec> Eth1DataVotesTreeHashCache<T> {
/// Instantiates a new cache.
///
/// Allocates the necessary memory to store all of the cached Merkle trees. Only the leaves are
/// hashed, leaving the internal nodes as all-zeros.
pub fn new(state: &BeaconState<T>) -> Self {
let mut arena = CacheArena::default();
let roots: VariableList<_, _> = state
.eth1_data_votes()
.iter()
.map(|eth1_data| eth1_data.tree_hash_root())
.collect::<Vec<_>>()
.into();
let tree_hash_cache = roots.new_tree_hash_cache(&mut arena);
Self {
arena,
tree_hash_cache,
voting_period: Self::voting_period(state.slot()),
roots,
}
}
fn voting_period(slot: Slot) -> u64 {
slot.as_u64() / T::SlotsPerEth1VotingPeriod::to_u64()
}
pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState<T>) -> Result<Hash256, Error> {
if state.eth1_data_votes().len() < self.roots.len()
|| Self::voting_period(state.slot()) != self.voting_period
{
*self = Self::new(state);
}
state
.eth1_data_votes()
.iter()
.skip(self.roots.len())
.try_for_each(|eth1_data| self.roots.push(eth1_data.tree_hash_root()))?;
self.roots
.recalculate_tree_hash_root(&mut self.arena, &mut self.tree_hash_cache)
.map_err(Into::into)
}
}
/// A cache that performs a caching tree hash of the entire `BeaconState` struct.
///
/// This type is a wrapper around the inner cache, which does all the work.
#[derive(Debug, Default, PartialEq, Clone)]
pub struct BeaconTreeHashCache<T: EthSpec> {
inner: Option<BeaconTreeHashCacheInner<T>>,
}
impl<T: EthSpec> BeaconTreeHashCache<T> {
pub fn new(state: &BeaconState<T>) -> Self {
Self {
inner: Some(BeaconTreeHashCacheInner::new(state)),
}
}
pub fn is_initialized(&self) -> bool {
self.inner.is_some()
}
/// Move the inner cache out so that the containing `BeaconState` can be borrowed.
pub fn take(&mut self) -> Option<BeaconTreeHashCacheInner<T>> {
self.inner.take()
}
/// Restore the inner cache after using `take`.
pub fn restore(&mut self, inner: BeaconTreeHashCacheInner<T>) {
self.inner = Some(inner);
}
/// Make the cache empty.
pub fn uninitialize(&mut self) {
self.inner = None;
}
/// Return the slot at which the cache was last updated.
///
/// This should probably only be used during testing.
pub fn initialized_slot(&self) -> Option<Slot> {
Some(self.inner.as_ref()?.previous_state?.1)
}
}
#[derive(Debug, PartialEq, Clone)]
pub struct BeaconTreeHashCacheInner<T: EthSpec> {
/// Tracks the previously generated state root to ensure the next state root provided descends
/// directly from this state.
previous_state: Option<(Hash256, Slot)>,
// Validators cache
validators: ValidatorsListTreeHashCache,
// Arenas
fixed_arena: CacheArena,
balances_arena: CacheArena,
slashings_arena: CacheArena,
// Caches
block_roots: TreeHashCache,
state_roots: TreeHashCache,
historical_roots: TreeHashCache,
balances: TreeHashCache,
randao_mixes: TreeHashCache,
slashings: TreeHashCache,
eth1_data_votes: Eth1DataVotesTreeHashCache<T>,
inactivity_scores: OptionalTreeHashCache,
// Participation caches
previous_epoch_participation: OptionalTreeHashCache,
current_epoch_participation: OptionalTreeHashCache,
}
impl<T: EthSpec> BeaconTreeHashCacheInner<T> {
/// Instantiates a new cache.
///
/// Allocates the necessary memory to store all of the cached Merkle trees. Only the leaves are
/// hashed, leaving the internal nodes as all-zeros.
pub fn new(state: &BeaconState<T>) -> Self {
let mut fixed_arena = CacheArena::default();
let block_roots = state.block_roots().new_tree_hash_cache(&mut fixed_arena);
let state_roots = state.state_roots().new_tree_hash_cache(&mut fixed_arena);
let historical_roots = state
.historical_roots()
.new_tree_hash_cache(&mut fixed_arena);
let randao_mixes = state.randao_mixes().new_tree_hash_cache(&mut fixed_arena);
let validators = ValidatorsListTreeHashCache::new::<T>(state.validators());
let mut balances_arena = CacheArena::default();
let balances = state.balances().new_tree_hash_cache(&mut balances_arena);
let mut slashings_arena = CacheArena::default();
let slashings = state.slashings().new_tree_hash_cache(&mut slashings_arena);
let inactivity_scores = OptionalTreeHashCache::new(state.inactivity_scores().ok());
let previous_epoch_participation = OptionalTreeHashCache::new(
state
.previous_epoch_participation()
.ok()
.map(ParticipationList::new)
.as_ref(),
);
let current_epoch_participation = OptionalTreeHashCache::new(
state
.current_epoch_participation()
.ok()
.map(ParticipationList::new)
.as_ref(),
);
Self {
previous_state: None,
validators,
fixed_arena,
balances_arena,
slashings_arena,
block_roots,
state_roots,
historical_roots,
balances,
randao_mixes,
slashings,
inactivity_scores,
eth1_data_votes: Eth1DataVotesTreeHashCache::new(state),
previous_epoch_participation,
current_epoch_participation,
}
}
/// Updates the cache and returns the tree hash root for the given `state`.
///
/// The provided `state` should be a descendant of the last `state` given to this function, or
/// the `Self::new` function. If the state is more than `SLOTS_PER_HISTORICAL_ROOT` slots
/// after `self.previous_state` then the whole cache will be re-initialized.
pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState<T>) -> Result<Hash256, Error> {
// If this cache has previously produced a root, ensure that it is in the state root
// history of this state.
//
// This ensures that the states applied have a linear history, this
// allows us to make assumptions about how the state changes over times and produce a more
// efficient algorithm.
if let Some((previous_root, previous_slot)) = self.previous_state {
// The previously-hashed state must not be newer than `state`.
if previous_slot > state.slot() {
return Err(Error::TreeHashCacheSkippedSlot {
cache: previous_slot,
state: state.slot(),
});
}
// If the state is newer, the previous root must be in the history of the given state.
// If the previous slot is out of range of the `state_roots` array (indicating a long
// gap between the cache's last use and the current state) then we re-initialize.
match state.get_state_root(previous_slot) {
Ok(state_previous_root) if *state_previous_root == previous_root => {}
Ok(_) => return Err(Error::NonLinearTreeHashCacheHistory),
Err(Error::SlotOutOfBounds) => {
*self = Self::new(state);
}
Err(e) => return Err(e),
}
}
let mut hasher = MerkleHasher::with_leaves(NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES);
hasher.write(state.genesis_time().tree_hash_root().as_bytes())?;
hasher.write(state.genesis_validators_root().tree_hash_root().as_bytes())?;
hasher.write(state.slot().tree_hash_root().as_bytes())?;
hasher.write(state.fork().tree_hash_root().as_bytes())?;
hasher.write(state.latest_block_header().tree_hash_root().as_bytes())?;
hasher.write(
state
.block_roots()
.recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.block_roots)?
.as_bytes(),
)?;
hasher.write(
state
.state_roots()
.recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.state_roots)?
.as_bytes(),
)?;
hasher.write(
state
.historical_roots()
.recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.historical_roots)?
.as_bytes(),
)?;
hasher.write(state.eth1_data().tree_hash_root().as_bytes())?;
hasher.write(
self.eth1_data_votes
.recalculate_tree_hash_root(state)?
.as_bytes(),
)?;
hasher.write(state.eth1_deposit_index().tree_hash_root().as_bytes())?;
hasher.write(
self.validators
.recalculate_tree_hash_root(state.validators())?
.as_bytes(),
)?;
hasher.write(
state
.balances()
.recalculate_tree_hash_root(&mut self.balances_arena, &mut self.balances)?
.as_bytes(),
)?;
hasher.write(
state
.randao_mixes()
.recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.randao_mixes)?
.as_bytes(),
)?;
hasher.write(
state
.slashings()
.recalculate_tree_hash_root(&mut self.slashings_arena, &mut self.slashings)?
.as_bytes(),
)?;
// Participation
if let BeaconState::Base(state) = state {
hasher.write(
state
.previous_epoch_attestations
.tree_hash_root()
.as_bytes(),
)?;
hasher.write(state.current_epoch_attestations.tree_hash_root().as_bytes())?;
} else {
hasher.write(
self.previous_epoch_participation
.recalculate_tree_hash_root(&ParticipationList::new(
state.previous_epoch_participation()?,
))?
.as_bytes(),
)?;
hasher.write(
self.current_epoch_participation
.recalculate_tree_hash_root(&ParticipationList::new(
state.current_epoch_participation()?,
))?
.as_bytes(),
)?;
}
hasher.write(state.justification_bits().tree_hash_root().as_bytes())?;
hasher.write(
state
.previous_justified_checkpoint()
.tree_hash_root()
.as_bytes(),
)?;
hasher.write(
state
.current_justified_checkpoint()
.tree_hash_root()
.as_bytes(),
)?;
hasher.write(state.finalized_checkpoint().tree_hash_root().as_bytes())?;
// Inactivity & light-client sync committees (Altair and later).
if let Ok(inactivity_scores) = state.inactivity_scores() {
hasher.write(
self.inactivity_scores
.recalculate_tree_hash_root(inactivity_scores)?
.as_bytes(),
)?;
}
if let Ok(current_sync_committee) = state.current_sync_committee() {
hasher.write(current_sync_committee.tree_hash_root().as_bytes())?;
}
if let Ok(next_sync_committee) = state.next_sync_committee() {
hasher.write(next_sync_committee.tree_hash_root().as_bytes())?;
}
// Execution payload (merge and later).
if let Ok(payload_header) = state.latest_execution_payload_header() {
hasher.write(payload_header.tree_hash_root().as_bytes())?;
}
let root = hasher.finish()?;
self.previous_state = Some((root, state.slot()));
Ok(root)
}
/// Updates the cache and provides the root of the given `validators`.
pub fn recalculate_validators_tree_hash_root(
&mut self,
validators: &[Validator],
) -> Result<Hash256, Error> {
self.validators.recalculate_tree_hash_root(validators)
}
}
/// A specialized cache for computing the tree hash root of `state.validators`.
#[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
struct ValidatorsListTreeHashCache {
list_arena: CacheArena,
list_cache: TreeHashCache,
values: ParallelValidatorTreeHash,
}
impl ValidatorsListTreeHashCache {
/// Instantiates a new cache.
///
/// Allocates the necessary memory to store all of the cached Merkle trees but does perform any
/// hashing.
fn new<E: EthSpec>(validators: &[Validator]) -> Self {
let mut list_arena = CacheArena::default();
Self {
list_cache: TreeHashCache::new(
&mut list_arena,
int_log(E::ValidatorRegistryLimit::to_usize()),
validators.len(),
),
list_arena,
values: ParallelValidatorTreeHash::new::<E>(validators),
}
}
/// Updates the cache and returns the tree hash root for the given `state`.
///
/// This function makes assumptions that the `validators` list will only change in accordance
/// with valid per-block/per-slot state transitions.
fn recalculate_tree_hash_root(&mut self, validators: &[Validator]) -> Result<Hash256, Error> {
let mut list_arena = std::mem::take(&mut self.list_arena);
let leaves = self.values.leaves(validators)?;
let num_leaves = leaves.iter().map(|arena| arena.len()).sum();
let leaves_iter = ForcedExactSizeIterator {
iter: leaves.into_iter().flatten().map(|h| h.to_fixed_bytes()),
len: num_leaves,
};
let list_root = self
.list_cache
.recalculate_merkle_root(&mut list_arena, leaves_iter)?;
self.list_arena = list_arena;
Ok(mix_in_length(&list_root, validators.len()))
}
}
/// Provides a wrapper around some `iter` if the number of items in the iterator is known to the
/// programmer but not the compiler. This allows use of `ExactSizeIterator` in some occasions.
///
/// Care should be taken to ensure `len` is accurate.
struct ForcedExactSizeIterator<I> {
iter: I,
len: usize,
}
impl<V, I: Iterator<Item = V>> Iterator for ForcedExactSizeIterator<I> {
type Item = V;
fn next(&mut self) -> Option<Self::Item> {
self.iter.next()
}
}
impl<V, I: Iterator<Item = V>> ExactSizeIterator for ForcedExactSizeIterator<I> {
fn len(&self) -> usize {
self.len
}
}
/// Provides a cache for each of the `Validator` objects in `state.validators` and computes the
/// roots of these using Rayon parallelization.
#[derive(Debug, PartialEq, Clone, Default, Encode, Decode)]
pub struct ParallelValidatorTreeHash {
/// Each arena and its associated sub-trees.
arenas: Vec<(CacheArena, Vec<TreeHashCache>)>,
}
impl ParallelValidatorTreeHash {
/// Instantiates a new cache.
///
/// Allocates the necessary memory to store all of the cached Merkle trees but does perform any
/// hashing.
fn new<E: EthSpec>(validators: &[Validator]) -> Self {
let num_arenas = std::cmp::max(
1,
(validators.len() + VALIDATORS_PER_ARENA - 1) / VALIDATORS_PER_ARENA,
);
let mut arenas = (1..=num_arenas)
.map(|i| {
let num_validators = if i == num_arenas {
validators.len() % VALIDATORS_PER_ARENA
} else {
VALIDATORS_PER_ARENA
};
NODES_PER_VALIDATOR * num_validators
})
.map(|capacity| (CacheArena::with_capacity(capacity), vec![]))
.collect::<Vec<_>>();
validators.iter().enumerate().for_each(|(i, v)| {
let (arena, caches) = &mut arenas[i / VALIDATORS_PER_ARENA];
caches.push(v.new_tree_hash_cache(arena))
});
Self { arenas }
}
/// Returns the number of validators stored in self.
fn len(&self) -> usize {
self.arenas.last().map_or(0, |last| {
// Subtraction cannot underflow because `.last()` ensures the `.len() > 0`.
(self.arenas.len() - 1) * VALIDATORS_PER_ARENA + last.1.len()
})
}
/// Updates the caches for each `Validator` in `validators` and returns a list that maps 1:1
/// with `validators` to the hash of each validator.
///
/// This function makes assumptions that the `validators` list will only change in accordance
/// with valid per-block/per-slot state transitions.
fn leaves(&mut self, validators: &[Validator]) -> Result<Vec<Vec<Hash256>>, Error> {
match self.len().cmp(&validators.len()) {
Ordering::Less => validators.iter().skip(self.len()).for_each(|v| {
if self
.arenas
.last()
.map_or(true, |last| last.1.len() >= VALIDATORS_PER_ARENA)
{
let mut arena = CacheArena::default();
let cache = v.new_tree_hash_cache(&mut arena);
self.arenas.push((arena, vec![cache]))
} else {
let (arena, caches) = &mut self
.arenas
.last_mut()
.expect("Cannot reach this block if arenas is empty.");
caches.push(v.new_tree_hash_cache(arena))
}
}),
Ordering::Greater => {
return Err(Error::ValidatorRegistryShrunk);
}
Ordering::Equal => (),
}
self.arenas
.par_iter_mut()
.enumerate()
.map(|(arena_index, (arena, caches))| {
caches
.iter_mut()
.enumerate()
.map(move |(cache_index, cache)| {
let val_index = (arena_index * VALIDATORS_PER_ARENA) + cache_index;
let validator = validators
.get(val_index)
.ok_or(Error::TreeHashCacheInconsistent)?;
validator
.recalculate_tree_hash_root(arena, cache)
.map_err(Error::CachedTreeHashError)
})
.collect()
})
.collect()
}
}
#[derive(Debug, PartialEq, Clone)]
pub struct OptionalTreeHashCache {
inner: Option<OptionalTreeHashCacheInner>,
}
#[derive(Debug, PartialEq, Clone)]
pub struct OptionalTreeHashCacheInner {
arena: CacheArena,
tree_hash_cache: TreeHashCache,
}
impl OptionalTreeHashCache {
/// Initialize a new cache if `item.is_some()`.
fn new<C: CachedTreeHash<TreeHashCache>>(item: Option<&C>) -> Self {
let inner = item.map(OptionalTreeHashCacheInner::new);
Self { inner }
}
/// Compute the tree hash root for the given `item`.
///
/// This function will initialize the inner cache if necessary (e.g. when crossing the fork).
fn recalculate_tree_hash_root<C: CachedTreeHash<TreeHashCache>>(
&mut self,
item: &C,
) -> Result<Hash256, Error> {
let cache = self
.inner
.get_or_insert_with(|| OptionalTreeHashCacheInner::new(item));
item.recalculate_tree_hash_root(&mut cache.arena, &mut cache.tree_hash_cache)
.map_err(Into::into)
}
}
impl OptionalTreeHashCacheInner {
fn new<C: CachedTreeHash<TreeHashCache>>(item: &C) -> Self {
let mut arena = CacheArena::default();
let tree_hash_cache = item.new_tree_hash_cache(&mut arena);
OptionalTreeHashCacheInner {
arena,
tree_hash_cache,
}
}
}
#[cfg(feature = "arbitrary-fuzz")]
impl<T: EthSpec> arbitrary::Arbitrary<'_> for BeaconTreeHashCache<T> {
fn arbitrary(_u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result<Self> {
Ok(Self::default())
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::{MainnetEthSpec, ParticipationFlags};
#[test]
fn validator_node_count() {
let mut arena = CacheArena::default();
let v = Validator::default();
let _cache = v.new_tree_hash_cache(&mut arena);
assert_eq!(arena.backing_len(), NODES_PER_VALIDATOR);
}
#[test]
fn participation_flags() {
type N = <MainnetEthSpec as EthSpec>::ValidatorRegistryLimit;
let len = 65;
let mut test_flag = ParticipationFlags::default();
test_flag.add_flag(0).unwrap();
let epoch_participation = VariableList::<_, N>::new(vec![test_flag; len]).unwrap();
let mut cache = OptionalTreeHashCache { inner: None };
let cache_root = cache
.recalculate_tree_hash_root(&ParticipationList::new(&epoch_participation))
.unwrap();
let recalc_root = cache
.recalculate_tree_hash_root(&ParticipationList::new(&epoch_participation))
.unwrap();
assert_eq!(cache_root, recalc_root, "recalculated root should match");
assert_eq!(
cache_root,
epoch_participation.tree_hash_root(),
"cached root should match uncached"
);
}
}

View File

@@ -1,15 +1,19 @@
use crate::test_utils::TestRandom;
use crate::*;
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use test_random_derive::TestRandom;
use tree_hash_derive::TreeHash;
/// Historical block and state roots.
///
/// Spec v0.12.1
#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom, TreeHash)]
pub struct HistoricalBatch<T: EthSpec> {
#[test_random(default)]
pub block_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
#[test_random(default)]
pub state_roots: FixedVector<Hash256, T::SlotsPerHistoricalRoot>,
}

View File

@@ -167,11 +167,6 @@ pub use bls::{
AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey,
Signature, SignatureBytes,
};
pub use milhouse::{self, Vector as FixedVector};
pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, VariableList};
pub use superstruct::superstruct;
#[cfg(feature = "milhouse")]
pub use milhouse::{self, Vector as FixedVector};
#[cfg(not(feature = "milhouse"))]
pub use ssz_types::FixedVector;

View File

@@ -36,6 +36,14 @@ impl Validator {
&self.immutable.pubkey
}
/// Replace the validator's pubkey (should only be used during testing).
pub fn replace_pubkey(&mut self, pubkey: PublicKeyBytes) {
self.immutable = Arc::new(ValidatorImmutable {
pubkey,
withdrawal_credentials: self.immutable.withdrawal_credentials,
});
}
pub fn withdrawal_credentials(&self) -> Hash256 {
self.immutable.withdrawal_credentials
}

View File

@@ -91,9 +91,10 @@ fn parse_client_config<E: EthSpec>(
cli_args: &ArgMatches,
_env: &Environment<E>,
) -> Result<ClientConfig, String> {
let mut client_config = ClientConfig::default();
client_config.data_dir = get_data_dir(cli_args);
let mut client_config = ClientConfig {
data_dir: get_data_dir(cli_args),
..Default::default()
};
if let Some(freezer_dir) = clap_utils::parse_optional(cli_args, "freezer-dir")? {
client_config.freezer_db_path = Some(freezer_dir);

View File

@@ -8,7 +8,6 @@ edition = "2021"
[features]
portable = ["bls/supranational-portable"]
fake_crypto = ['bls/fake_crypto']
tree-states = ["store/milhouse"]
[dependencies]
bls = { path = "../crypto/bls" }

View File

@@ -53,11 +53,14 @@ pub fn run<T: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(),
eprintln!("{}: {}", index, keypair.pk);
validators.get_mut(index).unwrap().pubkey = keypair.pk.into();
validators
.get_mut(index)
.unwrap()
.replace_pubkey(keypair.pk.into());
// Update the deposit tree.
let mut deposit_data = DepositData {
pubkey: validators.get(index).unwrap().pubkey,
pubkey: *validators.get(index).unwrap().pubkey(),
// Set this to a junk value since it's very time consuming to generate the withdrawal
// keys and it's not useful for the time being.
withdrawal_credentials: Hash256::zero(),
@@ -70,7 +73,6 @@ pub fn run<T: EthSpec>(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(),
.map_err(|e| format!("failed to create deposit tree: {:?}", e))?;
deposit_root = deposit_tree.root();
}
drop(validators);
// Update the genesis validators root since we changed the validators.
*state.genesis_validators_root_mut() = state.validators().tree_hash_root();

View File

@@ -7,7 +7,7 @@ autotests = false
rust-version = "1.58"
[features]
default = ["tree-states"]
default = ["malloc_utils/jemalloc"]
# Writes debugging .ssz files to /tmp during block processing.
write_ssz_files = ["beacon_node/write_ssz_files"]
# Compiles the BLS crypto code so that the binary is portable across machines.
@@ -20,8 +20,6 @@ milagro = ["bls/milagro"]
spec-minimal = []
# Support Gnosis spec and Gnosis Beacon Chain.
gnosis = []
# Use `milhouse` tree states.
tree-states = ["beacon_node/tree-states", "malloc_utils/jemalloc"]
[dependencies]
beacon_node = { "path" = "../beacon_node" }

View File

@@ -119,7 +119,6 @@ impl<E: EthSpec> Case for SszStaticTHC<BeaconState<E>> {
check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?;
let mut state = self.value.clone();
state.initialize_tree_hash_cache();
let cached_tree_hash_root = state.update_tree_hash_cache().unwrap();
check_tree_hash(&self.roots.root, cached_tree_hash_root.as_bytes())?;

View File

@@ -2,7 +2,7 @@ use super::*;
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use state_processing::{
per_block_processing, per_block_processing::errors::ExitInvalid, BlockProcessingError,
BlockSignatureStrategy, VerifyBlockRoot,
BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot,
};
use types::{BeaconBlock, BeaconState, Epoch, EthSpec, SignedBeaconBlock};
@@ -61,12 +61,13 @@ impl ExitTest {
block: &SignedBeaconBlock<E>,
state: &mut BeaconState<E>,
) -> Result<(), BlockProcessingError> {
let mut ctxt = ConsensusContext::new(block.slot());
per_block_processing(
state,
block,
None,
BlockSignatureStrategy::VerifyIndividual,
VerifyBlockRoot::True,
&mut ctxt,
&E::default_spec(),
)
}
@@ -122,7 +123,7 @@ vectors_and_tests!(
ExitTest {
block_modifier: Box::new(|_, block| {
// Duplicate the exit
let exit = block.body().voluntary_exits()[0].clone();
let exit = block.body().voluntary_exits().get(0).unwrap().clone();
block.body_mut().voluntary_exits_mut().push(exit).unwrap();
}),
expected: Err(BlockProcessingError::ExitInvalid {
@@ -141,7 +142,11 @@ vectors_and_tests!(
invalid_validator_unknown,
ExitTest {
block_modifier: Box::new(|_, block| {
block.body_mut().voluntary_exits_mut()[0]
block
.body_mut()
.voluntary_exits_mut()
.get_mut(0)
.unwrap()
.message
.validator_index = VALIDATOR_COUNT as u64;
}),
@@ -162,7 +167,7 @@ vectors_and_tests!(
invalid_exit_already_initiated,
ExitTest {
state_modifier: Box::new(|state| {
state.validators_mut()[0].exit_epoch = STATE_EPOCH + 1;
state.validators_mut().get_mut(0).unwrap().exit_epoch = STATE_EPOCH + 1;
}),
expected: Err(BlockProcessingError::ExitInvalid {
index: 0,
@@ -181,7 +186,8 @@ vectors_and_tests!(
invalid_not_active_before_activation_epoch,
ExitTest {
state_modifier: Box::new(|state| {
state.validators_mut()[0].activation_epoch = E::default_spec().far_future_epoch;
state.validators_mut().get_mut(0).unwrap().activation_epoch =
E::default_spec().far_future_epoch;
}),
expected: Err(BlockProcessingError::ExitInvalid {
index: 0,
@@ -200,7 +206,7 @@ vectors_and_tests!(
invalid_not_active_after_exit_epoch,
ExitTest {
state_modifier: Box::new(|state| {
state.validators_mut()[0].exit_epoch = STATE_EPOCH;
state.validators_mut().get_mut(0).unwrap().exit_epoch = STATE_EPOCH;
}),
expected: Err(BlockProcessingError::ExitInvalid {
index: 0,
@@ -300,7 +306,11 @@ vectors_and_tests!(
block_modifier: Box::new(|_, block| {
// Shift the validator index by 1 so that it's mismatched from the key that was
// used to sign.
block.body_mut().voluntary_exits_mut()[0]
block
.body_mut()
.voluntary_exits_mut()
.get_mut(0)
.unwrap()
.message
.validator_index = VALIDATOR_INDEX + 1;
}),