gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-08 01:05:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Repurpose the pubkey cache for validator de-dupe

Browse Source
This commit is contained in:
Michael Sproul
2022-09-28 15:08:30 +10:00
parent 9a1799f235
commit ea599a6d7f
12 changed files with 184 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
beacon_node/beacon_chain/src/beacon_chain.rs
View File

@@ -359,7 +359,7 @@ pub struct BeaconChain<T: BeaconChainTypes> {
/// Caches the beacon block proposer shuffling for a given epoch and shuffling key root.
pub beacon_proposer_cache: Mutex<BeaconProposerCache>,
/// Caches a map of `validator_index -> validator_pubkey`.
pub(crate) validator_pubkey_cache: TimeoutRwLock<ValidatorPubkeyCache<T>>,
pub(crate) validator_pubkey_cache: Arc<RwLock<ValidatorPubkeyCache<T>>>,
/// A cache used when producing attestations.
pub(crate) attester_cache: Arc<AttesterCache>,
/// A cache used when producing attestations whilst the head block is still being imported.
@@ -1170,10 +1170,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// May return an error if acquiring a read-lock on the `validator_pubkey_cache` times out.
pub fn validator_index(&self, pubkey: &PublicKeyBytes) -> Result<Option<usize>, Error> {
let pubkey_cache = self
.validator_pubkey_cache
.try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
let pubkey_cache = self.validator_pubkey_cache.read();
Ok(pubkey_cache.get_index(pubkey))
}
@@ -1186,10 +1183,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
validator_pubkeys: impl Iterator<Item = &'a PublicKeyBytes>,
) -> Result<Vec<u64>, Error> {
let pubkey_cache = self
.validator_pubkey_cache
.try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
let pubkey_cache = self.validator_pubkey_cache.read();
validator_pubkeys
.map(|pubkey| {
@@ -1214,10 +1208,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
///
/// May return an error if acquiring a read-lock on the `validator_pubkey_cache` times out.
pub fn validator_pubkey(&self, validator_index: usize) -> Result<Option<PublicKey>, Error> {
let pubkey_cache = self
.validator_pubkey_cache
.try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
let pubkey_cache = self.validator_pubkey_cache.read();
Ok(pubkey_cache.get(validator_index).cloned())
}
@@ -1227,11 +1218,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
validator_index: usize,
) -> Result<Option<PublicKeyBytes>, Error> {
let pubkey_cache = self
.validator_pubkey_cache
.try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
let pubkey_cache = self.validator_pubkey_cache.read();
Ok(pubkey_cache.get_pubkey_bytes(validator_index).copied())
}
@@ -1244,10 +1231,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
&self,
validator_indices: &[usize],
) -> Result<HashMap<usize, PublicKeyBytes>, Error> {
let pubkey_cache = self
.validator_pubkey_cache
.try_read_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?;
let pubkey_cache = self.validator_pubkey_cache.read();
let mut map = HashMap::with_capacity(validator_indices.len());
for &validator_index in validator_indices {
@@ -2612,9 +2596,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
// used by attestation processing which will only process an attestation if the block is
// known to fork choice. This ordering ensure that the pubkey cache is always up-to-date.
self.validator_pubkey_cache
.try_write_for(VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT)
.ok_or(Error::ValidatorPubkeyCacheLockTimeout)?
.import_new_pubkeys(&state)?;
.write()
.import_new_pubkeys(&state, &self.store)?;
// For the current and next epoch of this state, ensure we have the shuffling from this
// block in our cache.

8
beacon_node/beacon_chain/src/block_verification.rs
View File

@@ -545,7 +545,7 @@ pub fn signature_verify_chain_segment<T: BeaconChainTypes>(
)?;
let pubkey_cache = get_validator_pubkey_cache(chain)?;
let mut signature_verifier = get_signature_verifier(&state, &pubkey_cache, &chain.spec);
let mut signature_verifier = get_signature_verifier::<T>(&state, &pubkey_cache, &chain.spec);
for (block_root, block) in &chain_segment {
signature_verifier.include_all_signatures(block, Some(*block_root), true)?;
@@ -936,7 +936,8 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
let pubkey_cache = get_validator_pubkey_cache(chain)?;
let mut signature_verifier = get_signature_verifier(&state, &pubkey_cache, &chain.spec);
let mut signature_verifier =
get_signature_verifier::<T>(&state, &pubkey_cache, &chain.spec);
signature_verifier.include_all_signatures(&block, Some(block_root), true)?;
@@ -985,7 +986,8 @@ impl<T: BeaconChainTypes> SignatureVerifiedBlock<T> {
let pubkey_cache = get_validator_pubkey_cache(chain)?;
let mut signature_verifier = get_signature_verifier(&state, &pubkey_cache, &chain.spec);
let mut signature_verifier =
get_signature_verifier::<T>(&state, &pubkey_cache, &chain.spec);
signature_verifier.include_all_signatures_except_proposal(&block)?;

20
beacon_node/beacon_chain/src/builder.rs
View File

@@ -8,7 +8,6 @@ use crate::persisted_beacon_chain::PersistedBeaconChain;
use crate::shuffling_cache::ShufflingCache;
use crate::timeout_rw_lock::TimeoutRwLock;
use crate::validator_monitor::ValidatorMonitor;
use crate::validator_pubkey_cache::ValidatorPubkeyCache;
use crate::ChainConfig;
use crate::{
BeaconChain, BeaconChainTypes, BeaconForkChoiceStore, BeaconSnapshot, Eth1Chain,
@@ -81,7 +80,6 @@ pub struct BeaconChainBuilder<T: BeaconChainTypes> {
slot_clock: Option<T::SlotClock>,
shutdown_sender: Option<Sender<ShutdownReason>>,
head_tracker: Option<HeadTracker>,
validator_pubkey_cache: Option<ValidatorPubkeyCache<T>>,
spec: ChainSpec,
chain_config: ChainConfig,
log: Option<Logger>,
@@ -122,7 +120,6 @@ where
slot_clock: None,
shutdown_sender: None,
head_tracker: None,
validator_pubkey_cache: None,
spec: TEthSpec::default_spec(),
chain_config: ChainConfig::default(),
log: None,
@@ -280,16 +277,12 @@ where
.unwrap_or_else(OperationPool::new),
);
let pubkey_cache = ValidatorPubkeyCache::load_from_store(store)
.map_err(|e| format!("Unable to open persisted pubkey cache: {:?}", e))?;
self.genesis_block_root = Some(chain.genesis_block_root);
self.genesis_state_root = Some(genesis_block.state_root());
self.head_tracker = Some(
HeadTracker::from_ssz_container(&chain.ssz_head_tracker)
.map_err(|e| format!("Failed to decode head tracker for database: {:?}", e))?,
);
self.validator_pubkey_cache = Some(pubkey_cache);
self.fork_choice = Some(fork_choice);
Ok(self)
@@ -713,10 +706,13 @@ where
));
}
let validator_pubkey_cache = self.validator_pubkey_cache.map(Ok).unwrap_or_else(|| {
ValidatorPubkeyCache::new(&head_snapshot.beacon_state, store.clone())
.map_err(|e| format!("Unable to init validator pubkey cache: {:?}", e))
})?;
let validator_pubkey_cache = store.immutable_validators.clone();
// Update pubkey cache on first start in case we have started from genesis.
validator_pubkey_cache
.write()
.import_new_pubkeys(&head_snapshot.beacon_state, &store)
.map_err(|e| format!("error initializing pubkey cache: {e:?}"))?;
let migrator_config = self.store_migrator_config.unwrap_or_default();
let store_migrator = BackgroundMigrator::new(
@@ -816,7 +812,7 @@ where
beacon_proposer_cache: <_>::default(),
block_times_cache: <_>::default(),
pre_finalization_block_cache: <_>::default(),
validator_pubkey_cache: TimeoutRwLock::new(validator_pubkey_cache),
validator_pubkey_cache,
attester_cache: <_>::default(),
early_attester_cache: <_>::default(),
shutdown_sender: self

3
beacon_node/beacon_chain/src/errors.rs
View File

@@ -85,13 +85,10 @@ pub enum BeaconChainError {
ValidatorPubkeyCacheLockTimeout,
SnapshotCacheLockTimeout,
IncorrectStateForAttestation(RelativeEpochError),
InvalidValidatorPubkeyBytes(bls::Error),
ValidatorPubkeyCacheIncomplete(usize),
SignatureSetError(SignatureSetError),
BlockSignatureVerifierError(state_processing::block_signature_verifier::Error),
BlockReplayError(BlockReplayError),
DuplicateValidatorPublicKey,
ValidatorPubkeyCacheError(String),
ValidatorIndexUnknown(usize),
ValidatorPubkeyUnknown(PublicKeyBytes),
OpPoolError(OpPoolError),

12
beacon_node/beacon_chain/src/lib.rs
View File

@@ -40,7 +40,6 @@ pub mod sync_committee_verification;
pub mod test_utils;
mod timeout_rw_lock;
pub mod validator_monitor;
pub mod validator_pubkey_cache;
pub use self::beacon_chain::{
AttestationProcessingOutcome, BeaconChain, BeaconChainTypes, BeaconStore, ChainSegmentResult,
@@ -70,3 +69,14 @@ pub use state_processing::per_block_processing::errors::{
pub use store;
pub use timeout_rw_lock::TimeoutRwLock;
pub use types;
// FIXME(sproul): compatibility shim
pub mod validator_pubkey_cache {
use crate::BeaconChainTypes;
pub type ValidatorPubkeyCache<T> = store::ValidatorPubkeyCache<
<T as BeaconChainTypes>::EthSpec,
<T as BeaconChainTypes>::HotStore,
<T as BeaconChainTypes>::ColdStore,
>;
}

1
beacon_node/store/Cargo.toml
View File

@@ -30,3 +30,4 @@ tree_hash = "0.4.0"
take-until = "0.1.0"
zstd = "0.10.0"
strum = { version = "0.24.0", features = ["derive"] }
bls = { path = "../../crypto/bls" }

5
beacon_node/store/src/errors.rs
View File

@@ -59,7 +59,12 @@ pub enum Error {
AddPayloadLogicError,
ResyncRequiredForExecutionPayloadSeparation,
SlotClockUnavailableForMigration,
MissingImmutableValidator(usize),
V9MigrationFailure(Hash256),
ValidatorPubkeyCacheError(String),
DuplicateValidatorPublicKey,
InvalidValidatorPubkeyBytes(bls::Error),
ValidatorPubkeyCacheUninitialized,
}
pub trait HandleUnavailable<T> {

32
beacon_node/store/src/hot_cold_store.rs
View File

@@ -20,7 +20,7 @@ use crate::metrics;
use crate::state_cache::{PutStateOutcome, StateCache};
use crate::{
get_key_for_col, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp,
PartialBeaconState, StoreItem, StoreOp,
PartialBeaconState, StoreItem, StoreOp, ValidatorPubkeyCache,
};
use itertools::process_results;
use leveldb::iterator::LevelDBIterator;
@@ -36,13 +36,14 @@ use state_processing::{
block_replayer::PreSlotHook, BlockProcessingError, BlockReplayer, SlotProcessingError,
};
use std::cmp::min;
use std::convert::TryInto;
use std::io::Read;
use std::marker::PhantomData;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
use types::*;
use types::{beacon_state::BeaconStateDiff, EthSpec};
use zstd::Decoder;
pub const MAX_PARENT_STATES_TO_CACHE: u64 = 32;
@@ -70,6 +71,8 @@ pub struct HotColdDB<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
block_cache: Mutex<LruCache<Hash256, SignedBeaconBlock<E>>>,
/// Cache of beacon states.
state_cache: Mutex<StateCache<E>>,
/// Immutable validator cache.
pub immutable_validators: Arc<RwLock<ValidatorPubkeyCache<E, Hot, Cold>>>,
/// Chain spec.
pub(crate) spec: ChainSpec,
/// Logger.
@@ -141,6 +144,7 @@ impl<E: EthSpec> HotColdDB<E, MemoryStore<E>, MemoryStore<E>> {
hot_db: MemoryStore::open(),
block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
state_cache: Mutex::new(StateCache::new(config.state_cache_size)),
immutable_validators: Arc::new(RwLock::new(Default::default())),
config,
spec,
log,
@@ -176,6 +180,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
hot_db: LevelDB::open(hot_path)?,
block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
state_cache: Mutex::new(StateCache::new(config.state_cache_size)),
immutable_validators: Arc::new(RwLock::new(Default::default())),
config,
spec,
log,
@@ -217,6 +222,11 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
);
}
// Load validator pubkey cache.
// FIXME(sproul): probably breaks migrations, etc
let pubkey_cache = ValidatorPubkeyCache::load_from_store(&db)?;
*db.immutable_validators.write() = pubkey_cache;
// Ensure that the schema version of the on-disk database matches the software.
// If the version is mismatched, an automatic migration will be attempted.
let db = Arc::new(db);
@@ -1100,7 +1110,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
// 1. Convert to PartialBeaconState and store that in the DB.
let partial_state = PartialBeaconState::from_state_forgetful(state);
let op = partial_state.as_kv_store_op(*state_root);
let op = partial_state.as_kv_store_op(*state_root, &self.config)?;
ops.push(op);
// 2. Store updated vector entries.
@@ -1151,12 +1161,19 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// Load a restore point state by its `state_root`.
fn load_restore_point(&self, state_root: &Hash256) -> Result<BeaconState<E>, Error> {
let partial_state_bytes = self
let bytes = self
.cold_db
.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())?
.ok_or(HotColdDBError::MissingRestorePoint(*state_root))?;
let mut ssz_bytes = Vec::with_capacity(self.config.estimate_decompressed_size(bytes.len()));
let mut decoder = Decoder::new(&*bytes).map_err(Error::Compression)?;
decoder
.read_to_end(&mut ssz_bytes)
.map_err(Error::Compression)?;
let mut partial_state: PartialBeaconState<E> =
PartialBeaconState::from_ssz_bytes(&partial_state_bytes, &self.spec)?;
PartialBeaconState::from_ssz_bytes(&ssz_bytes, &self.spec)?;
// Fill in the fields of the partial state.
partial_state.load_block_roots(&self.cold_db, &self.spec)?;
@@ -1164,7 +1181,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
partial_state.load_historical_roots(&self.cold_db, &self.spec)?;
partial_state.load_randao_mixes(&self.cold_db, &self.spec)?;
partial_state.try_into()
let pubkey_cache = self.immutable_validators.read();
let immutable_validators = |i: usize| pubkey_cache.get_validator(i);
partial_state.try_into_full_state(immutable_validators)
}
/// Load a restore point state by its `restore_point_index`.

2
beacon_node/store/src/lib.rs
View File

@@ -28,6 +28,7 @@ mod partial_beacon_state;
pub mod reconstruct;
mod state_cache;
mod state_diff;
pub mod validator_pubkey_cache;
pub mod iter;
@@ -45,6 +46,7 @@ use parking_lot::MutexGuard;
use std::sync::Arc;
use strum::{EnumString, IntoStaticStr};
pub use types::*;
pub use validator_pubkey_cache::ValidatorPubkeyCache;
pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
pub type ColumnKeyIter<'a> = Box<dyn Iterator<Item = Result<Hash256, Error>> + 'a>;

56
beacon_node/store/src/partial_beacon_state.rs
View File

@@ -2,13 +2,15 @@ use crate::chunked_vector::{
load_variable_list_from_db, load_vector_from_db, BlockRoots, HistoricalRoots, RandaoMixes,
StateRoots,
};
use crate::{get_key_for_col, DBColumn, Error, KeyValueStore, KeyValueStoreOp};
use crate::{get_key_for_col, DBColumn, Error, KeyValueStore, KeyValueStoreOp, StoreConfig};
use itertools::process_results;
use ssz::{Decode, DecodeError, Encode};
use ssz_derive::{Decode, Encode};
use std::convert::TryInto;
use std::io::Write;
use std::sync::Arc;
use types::superstruct;
use types::*;
use zstd::Encoder;
/// Lightweight variant of the `BeaconState` that is stored in the database.
///
@@ -47,7 +49,7 @@ where
pub eth1_deposit_index: u64,
// Registry
pub validators: VList<Validator, T::ValidatorRegistryLimit>,
pub validators: Vec<ValidatorMutable>,
pub balances: VList<u64, T::ValidatorRegistryLimit>,
// Shuffling
@@ -114,7 +116,10 @@ macro_rules! impl_from_state_forgetful {
eth1_deposit_index: $s.eth1_deposit_index,
// Validator registry
validators: $s.validators.clone(),
validators: $s.validators.into_iter().map(|validator| {
validator.mutable.clone()
})
.collect(),
balances: $s.balances.clone(),
// Shuffling
@@ -204,9 +209,23 @@ impl<T: EthSpec> PartialBeaconState<T> {
}
/// Prepare the partial state for storage in the KV database.
pub fn as_kv_store_op(&self, state_root: Hash256) -> KeyValueStoreOp {
pub fn as_kv_store_op(
&self,
state_root: Hash256,
config: &StoreConfig,
) -> Result<KeyValueStoreOp, Error> {
let db_key = get_key_for_col(DBColumn::BeaconState.into(), state_root.as_bytes());
KeyValueStoreOp::PutKeyValue(db_key, self.as_ssz_bytes())
let ssz_bytes = self.as_ssz_bytes();
let mut compressed_value =
Vec::with_capacity(config.estimate_compressed_size(ssz_bytes.len()));
let mut encoder = Encoder::new(&mut compressed_value, config.compression_level)
.map_err(Error::Compression)?;
encoder.write_all(&ssz_bytes).map_err(Error::Compression)?;
encoder.finish().map_err(Error::Compression)?;
Ok(KeyValueStoreOp::PutKeyValue(db_key, compressed_value))
}
pub fn load_block_roots<S: KeyValueStore<T>>(
@@ -278,7 +297,7 @@ impl<T: EthSpec> PartialBeaconState<T> {
/// Implement the conversion from PartialBeaconState -> BeaconState.
macro_rules! impl_try_into_beacon_state {
($inner:ident, $variant_name:ident, $struct_name:ident, [$($extra_fields:ident),*]) => {
($inner:ident, $variant_name:ident, $struct_name:ident, $immutable_validators:ident, [$($extra_fields:ident),*]) => {
BeaconState::$variant_name($struct_name {
// Versioning
genesis_time: $inner.genesis_time,
@@ -298,7 +317,16 @@ macro_rules! impl_try_into_beacon_state {
eth1_deposit_index: $inner.eth1_deposit_index,
// Validator registry
validators: $inner.validators,
validators: process_results($inner.validators.into_iter().enumerate().map(|(i, mutable)| {
$immutable_validators(i)
.ok_or(Error::MissingImmutableValidator(i))
.map(move |immutable| {
Validator {
immutable,
mutable
}
})
}), |iter| VList::try_from_iter(iter))??,
balances: $inner.balances,
// Shuffling
@@ -331,21 +359,24 @@ fn unpack_field<T>(x: Option<T>) -> Result<T, Error> {
x.ok_or(Error::PartialBeaconStateError)
}
impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
type Error = Error;
fn try_into(self) -> Result<BeaconState<E>, Error> {
impl<E: EthSpec> PartialBeaconState<E> {
pub fn try_into_full_state<F>(self, immutable_validators: F) -> Result<BeaconState<E>, Error>
where
F: Fn(usize) -> Option<Arc<ValidatorImmutable>>,
{
let state = match self {
PartialBeaconState::Base(inner) => impl_try_into_beacon_state!(
inner,
Base,
BeaconStateBase,
immutable_validators,
[previous_epoch_attestations, current_epoch_attestations]
),
PartialBeaconState::Altair(inner) => impl_try_into_beacon_state!(
inner,
Altair,
BeaconStateAltair,
immutable_validators,
[
previous_epoch_participation,
current_epoch_participation,
@@ -358,6 +389,7 @@ impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
inner,
Merge,
BeaconStateMerge,
immutable_validators,
[
previous_epoch_participation,
current_epoch_participation,

122
beacon_node/beacon_chain/src/validator_pubkey_cache.rs → beacon_node/store/src/validator_pubkey_cache.rs
View File

@@ -1,10 +1,10 @@
use crate::errors::BeaconChainError;
use crate::{BeaconChainTypes, BeaconStore};
use crate::{DBColumn, Error, HotColdDB, ItemStore, StoreItem};
use ssz::{Decode, Encode};
use std::collections::HashMap;
use std::convert::TryInto;
use store::{DBColumn, Error as StoreError, StoreItem};
use types::{BeaconState, Hash256, PublicKey, PublicKeyBytes};
use std::marker::PhantomData;
use std::sync::Arc;
use types::{BeaconState, EthSpec, Hash256, PublicKey, PublicKeyBytes, ValidatorImmutable};
/// Provides a mapping of `validator_index -> validator_publickey`.
///
@@ -17,49 +17,63 @@ use types::{BeaconState, Hash256, PublicKey, PublicKeyBytes};
///
/// The cache has a `backing` that it uses to maintain a persistent, on-disk
/// copy of itself. This allows it to be restored between process invocations.
pub struct ValidatorPubkeyCache<T: BeaconChainTypes> {
#[derive(Debug)]
pub struct ValidatorPubkeyCache<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
pubkeys: Vec<PublicKey>,
indices: HashMap<PublicKeyBytes, usize>,
pubkey_bytes: Vec<PublicKeyBytes>,
store: BeaconStore<T>,
validators: Vec<Arc<ValidatorImmutable>>,
_phantom: PhantomData<(E, Hot, Cold)>,
}
impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
// Temp value.
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Default
for ValidatorPubkeyCache<E, Hot, Cold>
{
fn default() -> Self {
ValidatorPubkeyCache {
pubkeys: vec![],
indices: HashMap::new(),
validators: vec![],
_phantom: PhantomData,
}
}
}
impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> ValidatorPubkeyCache<E, Hot, Cold> {
/// Create a new public key cache using the keys in `state.validators`.
///
/// Also creates a new persistence file, returning an error if there is already a file at
/// `persistence_path`.
pub fn new(
state: &BeaconState<T::EthSpec>,
store: BeaconStore<T>,
) -> Result<Self, BeaconChainError> {
pub fn new(state: &BeaconState<E>, store: &HotColdDB<E, Hot, Cold>) -> Result<Self, Error> {
let mut cache = Self {
pubkeys: vec![],
indices: HashMap::new(),
pubkey_bytes: vec![],
store,
validators: vec![],
_phantom: PhantomData,
};
cache.import_new_pubkeys(state)?;
cache.import_new_pubkeys(state, store)?;
Ok(cache)
}
/// Load the pubkey cache from the given on-disk database.
pub fn load_from_store(store: BeaconStore<T>) -> Result<Self, BeaconChainError> {
pub fn load_from_store(store: &HotColdDB<E, Hot, Cold>) -> Result<Self, Error> {
let mut pubkeys = vec![];
let mut indices = HashMap::new();
let mut pubkey_bytes = vec![];
let mut validators = vec![];
for validator_index in 0.. {
if let Some(DatabasePubkey(pubkey)) =
store.get_item(&DatabasePubkey::key_for_index(validator_index))?
if let Some(DatabaseValidator(validator)) =
store.get_item(&DatabaseValidator::key_for_index(validator_index))?
{
pubkeys.push((&pubkey).try_into().map_err(|e| {
BeaconChainError::ValidatorPubkeyCacheError(format!("{:?}", e))
})?);
pubkey_bytes.push(pubkey);
indices.insert(pubkey, validator_index);
pubkeys.push(
(&validator.pubkey)
.try_into()
.map_err(|e| Error::ValidatorPubkeyCacheError(format!("{:?}", e)))?,
);
indices.insert(validator.pubkey, validator_index);
validators.push(validator);
} else {
break;
}
@@ -68,8 +82,8 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
Ok(ValidatorPubkeyCache {
pubkeys,
indices,
pubkey_bytes,
store,
validators,
_phantom: PhantomData,
})
}
@@ -78,15 +92,17 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
/// Does not delete any keys from `self` if they don't appear in `state`.
pub fn import_new_pubkeys(
&mut self,
state: &BeaconState<T::EthSpec>,
) -> Result<(), BeaconChainError> {
if state.validators().len() > self.pubkeys.len() {
state: &BeaconState<E>,
store: &HotColdDB<E, Hot, Cold>,
) -> Result<(), Error> {
if state.validators().len() > self.validators.len() {
self.import(
state
.validators()
.iter_from(self.pubkeys.len())
.unwrap() // FIXME(sproul)
.map(|v| *v.pubkey()),
.map(|v| v.immutable.clone()),
store,
)
} else {
Ok(())
@@ -94,19 +110,19 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
}
/// Adds zero or more validators to `self`.
fn import<I>(&mut self, validator_keys: I) -> Result<(), BeaconChainError>
fn import<I>(&mut self, validator_keys: I, store: &HotColdDB<E, Hot, Cold>) -> Result<(), Error>
where
I: Iterator<Item = PublicKeyBytes> + ExactSizeIterator,
I: Iterator<Item = Arc<ValidatorImmutable>> + ExactSizeIterator,
{
self.pubkey_bytes.reserve(validator_keys.len());
self.validators.reserve(validator_keys.len());
self.pubkeys.reserve(validator_keys.len());
self.indices.reserve(validator_keys.len());
for pubkey in validator_keys {
for validator in validator_keys {
let i = self.pubkeys.len();
if self.indices.contains_key(&pubkey) {
return Err(BeaconChainError::DuplicateValidatorPublicKey);
if self.indices.contains_key(&validator.pubkey) {
return Err(Error::DuplicateValidatorPublicKey);
}
// The item is written to disk _before_ it is written into
@@ -118,17 +134,18 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
// The motivation behind this ordering is that we do not want to have states that
// reference a pubkey that is not in our cache. However, it's fine to have pubkeys
// that are never referenced in a state.
self.store
.put_item(&DatabasePubkey::key_for_index(i), &DatabasePubkey(pubkey))?;
store.put_item(
&DatabaseValidator::key_for_index(i),
&DatabaseValidator(validator.clone()),
)?;
self.pubkeys.push(
(&pubkey)
(&validator.pubkey)
.try_into()
.map_err(BeaconChainError::InvalidValidatorPubkeyBytes)?,
.map_err(Error::InvalidValidatorPubkeyBytes)?,
);
self.pubkey_bytes.push(pubkey);
self.indices.insert(pubkey, i);
self.indices.insert(validator.pubkey, i);
self.validators.push(validator);
}
Ok(())
@@ -139,6 +156,11 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
self.pubkeys.get(i)
}
/// Get the immutable validator with index `i`.
pub fn get_validator(&self, i: usize) -> Option<Arc<ValidatorImmutable>> {
self.validators.get(i).cloned()
}
/// Get the `PublicKey` for a validator with `PublicKeyBytes`.
pub fn get_pubkey_from_pubkey_bytes(&self, pubkey: &PublicKeyBytes) -> Option<&PublicKey> {
self.get_index(pubkey).and_then(|index| self.get(index))
@@ -146,7 +168,7 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
/// Get the public key (in bytes form) for a validator with index `i`.
pub fn get_pubkey_bytes(&self, i: usize) -> Option<&PublicKeyBytes> {
self.pubkey_bytes.get(i)
self.validators.get(i).map(|validator| &validator.pubkey)
}
/// Get the index of a validator with `pubkey`.
@@ -168,23 +190,23 @@ impl<T: BeaconChainTypes> ValidatorPubkeyCache<T> {
/// Wrapper for a public key stored in the database.
///
/// Keyed by the validator index as `Hash256::from_low_u64_be(index)`.
struct DatabasePubkey(PublicKeyBytes);
struct DatabaseValidator(Arc<ValidatorImmutable>);
impl StoreItem for DatabasePubkey {
impl StoreItem for DatabaseValidator {
fn db_column() -> DBColumn {
DBColumn::PubkeyCache
}
fn as_store_bytes(&self) -> Result<Vec<u8>, StoreError> {
fn as_store_bytes(&self) -> Result<Vec<u8>, Error> {
Ok(self.0.as_ssz_bytes())
}
fn from_store_bytes(bytes: &[u8]) -> Result<Self, StoreError> {
Ok(Self(PublicKeyBytes::from_ssz_bytes(bytes)?))
fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
Ok(Self(Arc::new(ValidatorImmutable::from_ssz_bytes(bytes)?)))
}
}
impl DatabasePubkey {
impl DatabaseValidator {
fn key_for_index(index: usize) -> Hash256 {
Hash256::from_low_u64_be(index as u64)
}

3
consensus/types/src/validator.rs
View File

@@ -6,6 +6,7 @@ use ssz_derive::{Decode, Encode};
use std::sync::Arc;
use test_random_derive::TestRandom;
use tree_hash::TreeHash;
use tree_hash_derive::TreeHash;
const NUM_FIELDS: usize = 8;
@@ -20,7 +21,7 @@ pub struct Validator {
/// The mutable fields of a validator.
#[cfg_attr(feature = "arbitrary-fuzz", derive(arbitrary::Arbitrary))]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TestRandom)]
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode, TreeHash, TestRandom)]
pub struct ValidatorMutable {
#[serde(with = "eth2_serde_utils::quoted_u64")]
pub effective_balance: u64,