gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-19 13:58:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Altair consensus changes and refactors (#2279)

Browse Source 
## Proposed Changes

Implement the consensus changes necessary for the upcoming Altair hard fork.

## Additional Info

This is quite a heavy refactor, with pivotal types like the `BeaconState` and `BeaconBlock` changing from structs to enums. This ripples through the whole codebase with field accesses changing to methods, e.g. `state.slot` => `state.slot()`.


Co-authored-by: realbigsean <seananderson33@gmail.com>
This commit is contained in:
Michael Sproul
2021-07-09 06:15:32 +00:00
parent 89361573d4
commit b4689e20c6
271 changed files with 9652 additions and 8444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
beacon_node/store/src/chunked_vector.rs
View File

@@ -226,12 +226,12 @@ pub trait Field<E: EthSpec>: Copy {
/// Extract the genesis value for a fixed length field from an
///
/// Will only return a correct value if `slot_needs_genesis_value(state.slot, spec) == true`.
/// Will only return a correct value if `slot_needs_genesis_value(state.slot(), spec) == true`.
fn extract_genesis_value(
state: &BeaconState<E>,
spec: &ChainSpec,
) -> Result<Self::Value, Error> {
let (_, end_vindex) = Self::start_and_end_vindex(state.slot, spec);
let (_, end_vindex) = Self::start_and_end_vindex(state.slot(), spec);
match Self::update_pattern(spec) {
// Genesis value is guaranteed to exist at `end_vindex`, as it won't yet have been
// updated
@@ -295,7 +295,7 @@ field!(
T::SlotsPerHistoricalRoot,
DBColumn::BeaconBlockRoots,
|_| OncePerNSlots { n: 1 },
|state: &BeaconState<_>, index, _| safe_modulo_index(&state.block_roots, index)
|state: &BeaconState<_>, index, _| safe_modulo_index(state.block_roots(), index)
);
field!(
@@ -305,7 +305,7 @@ field!(
T::SlotsPerHistoricalRoot,
DBColumn::BeaconStateRoots,
|_| OncePerNSlots { n: 1 },
|state: &BeaconState<_>, index, _| safe_modulo_index(&state.state_roots, index)
|state: &BeaconState<_>, index, _| safe_modulo_index(state.state_roots(), index)
);
field!(
@@ -317,7 +317,7 @@ field!(
|_| OncePerNSlots {
n: T::SlotsPerHistoricalRoot::to_u64()
},
|state: &BeaconState<_>, index, _| safe_modulo_index(&state.historical_roots, index)
|state: &BeaconState<_>, index, _| safe_modulo_index(state.historical_roots(), index)
);
field!(
@@ -327,7 +327,7 @@ field!(
T::EpochsPerHistoricalVector,
DBColumn::BeaconRandaoMixes,
|_| OncePerEpoch { lag: 1 },
|state: &BeaconState<_>, index, _| safe_modulo_index(&state.randao_mixes, index)
|state: &BeaconState<_>, index, _| safe_modulo_index(state.randao_mixes(), index)
);
pub fn store_updated_vector<F: Field<E>, E: EthSpec, S: KeyValueStore<E>>(
@@ -338,12 +338,12 @@ pub fn store_updated_vector<F: Field<E>, E: EthSpec, S: KeyValueStore<E>>(
ops: &mut Vec<KeyValueStoreOp>,
) -> Result<(), Error> {
let chunk_size = F::chunk_size();
let (start_vindex, end_vindex) = F::start_and_end_vindex(state.slot, spec);
let (start_vindex, end_vindex) = F::start_and_end_vindex(state.slot(), spec);
let start_cindex = start_vindex / chunk_size;
let end_cindex = end_vindex / chunk_size;
// Store the genesis value if we have access to it, and it hasn't been stored already.
if F::slot_needs_genesis_value(state.slot, spec) {
if F::slot_needs_genesis_value(state.slot(), spec) {
let genesis_value = F::extract_genesis_value(state, spec)?;
F::check_and_store_genesis_value(store, genesis_value, ops)?;
}

4
beacon_node/store/src/forwards_iter.rs
View File

@@ -71,7 +71,7 @@ impl SimpleForwardsBlockRootsIterator {
) -> Result<Self> {
// Iterate backwards from the end state, stopping at the start slot.
let values = process_results(
std::iter::once(Ok((end_block_root, end_state.slot)))
std::iter::once(Ok((end_block_root, end_state.slot())))
.chain(BlockRootsIterator::owned(store, end_state)),
|iter| {
iter.take_while(|(_, slot)| *slot >= start_slot)
@@ -237,7 +237,7 @@ impl SimpleForwardsStateRootsIterator {
) -> Result<Self> {
// Iterate backwards from the end state, stopping at the start slot.
let values = process_results(
std::iter::once(Ok((end_state_root, end_state.slot)))
std::iter::once(Ok((end_state_root, end_state.slot())))
.chain(StateRootsIterator::owned(store, end_state)),
|iter| {
iter.take_while(|(_, slot)| *slot >= start_slot)

121
beacon_node/store/src/hot_cold_store.rs
View File

@@ -3,7 +3,10 @@ use crate::chunked_vector::{
};
use crate::config::{OnDiskStoreConfig, StoreConfig};
use crate::forwards_iter::{HybridForwardsBlockRootsIterator, HybridForwardsStateRootsIterator};
use crate::impls::beacon_state::{get_full_state, store_full_state};
use crate::impls::{
beacon_block_as_kv_store_op,
beacon_state::{get_full_state, store_full_state},
};
use crate::iter::{ParentRootBlockIterator, StateRootsIterator};
use crate::leveldb_store::BytesKey;
use crate::leveldb_store::LevelDB;
@@ -240,7 +243,8 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
block: SignedBeaconBlock<E>,
) -> Result<(), Error> {
// Store on disk.
self.hot_db.put(block_root, &block)?;
self.hot_db
.do_atomically(vec![beacon_block_as_kv_store_op(block_root, &block)])?;
// Update cache.
self.block_cache.lock().put(*block_root, block);
@@ -259,20 +263,34 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
}
// Fetch from database.
match self.hot_db.get::<SignedBeaconBlock<E>>(block_root)? {
Some(block) => {
match self
.hot_db
.get_bytes(DBColumn::BeaconBlock.into(), block_root.as_bytes())?
{
Some(block_bytes) => {
// Deserialize.
let block = SignedBeaconBlock::from_ssz_bytes(&block_bytes, &self.spec)?;
// Add to cache.
self.block_cache.lock().put(*block_root, block.clone());
Ok(Some(block))
}
None => Ok(None),
}
}
/// Determine whether a block exists in the database.
pub fn block_exists(&self, block_root: &Hash256) -> Result<bool, Error> {
self.hot_db
.key_exists(DBColumn::BeaconBlock.into(), block_root.as_bytes())
}
/// Delete a block from the store and the block cache.
pub fn delete_block(&self, block_root: &Hash256) -> Result<(), Error> {
self.block_cache.lock().pop(block_root);
self.hot_db.delete::<SignedBeaconBlock<E>>(block_root)
self.hot_db
.key_delete(DBColumn::BeaconBlock.into(), block_root.as_bytes())
}
pub fn put_state_summary(
@@ -286,7 +304,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// Store a state in the store.
pub fn put_state(&self, state_root: &Hash256, state: &BeaconState<E>) -> Result<(), Error> {
let mut ops: Vec<KeyValueStoreOp> = Vec::new();
if state.slot < self.get_split_slot() {
if state.slot() < self.get_split_slot() {
self.store_cold_state(state_root, &state, &mut ops)?;
self.cold_db.do_atomically(ops)
} else {
@@ -456,7 +474,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
for op in batch {
match op {
StoreOp::PutBlock(block_root, block) => {
key_value_batch.push(block.as_kv_store_op(*block_root));
key_value_batch.push(beacon_block_as_kv_store_op(block_root, block));
}
StoreOp::PutState(state_root, state) => {
@@ -538,11 +556,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
ops: &mut Vec<KeyValueStoreOp>,
) -> Result<(), Error> {
// On the epoch boundary, store the full state.
if state.slot % E::slots_per_epoch() == 0 {
if state.slot() % E::slots_per_epoch() == 0 {
trace!(
self.log,
"Storing full state on epoch boundary";
"slot" => state.slot.as_u64(),
"slot" => state.slot().as_u64(),
"state_root" => format!("{:?}", state_root)
);
store_full_state(state_root, &state, ops)?;
@@ -580,9 +598,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
epoch_boundary_state_root,
}) = self.load_hot_state_summary(state_root)?
{
let boundary_state = get_full_state(&self.hot_db, &epoch_boundary_state_root)?.ok_or(
HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root),
)?;
let boundary_state =
get_full_state(&self.hot_db, &epoch_boundary_state_root, &self.spec)?.ok_or(
HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root),
)?;
// Optimization to avoid even *thinking* about replaying blocks if we're already
// on an epoch boundary.
@@ -590,7 +609,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
boundary_state
} else {
let blocks =
self.load_blocks_to_replay(boundary_state.slot, slot, latest_block_root)?;
self.load_blocks_to_replay(boundary_state.slot(), slot, latest_block_root)?;
self.replay_blocks(boundary_state, blocks, slot, block_replay)?
};
@@ -610,11 +629,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
state: &BeaconState<E>,
ops: &mut Vec<KeyValueStoreOp>,
) -> Result<(), Error> {
if state.slot % self.config.slots_per_restore_point != 0 {
if state.slot() % self.config.slots_per_restore_point != 0 {
warn!(
self.log,
"Not storing non-restore point state in freezer";
"slot" => state.slot.as_u64(),
"slot" => state.slot().as_u64(),
"state_root" => format!("{:?}", state_root)
);
return Ok(());
@@ -623,7 +642,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
trace!(
self.log,
"Creating restore point";
"slot" => state.slot,
"slot" => state.slot(),
"state_root" => format!("{:?}", state_root)
);
@@ -640,7 +659,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
store_updated_vector(RandaoMixes, db, state, &self.spec, ops)?;
// 3. Store restore point.
let restore_point_index = state.slot.as_u64() / self.config.slots_per_restore_point;
let restore_point_index = state.slot().as_u64() / self.config.slots_per_restore_point;
self.store_restore_point_hash(restore_point_index, *state_root, ops);
Ok(())
@@ -670,10 +689,12 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// Load a restore point state by its `state_root`.
fn load_restore_point(&self, state_root: &Hash256) -> Result<BeaconState<E>, Error> {
let mut partial_state: PartialBeaconState<E> = self
let partial_state_bytes = self
.cold_db
.get(state_root)?
.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())?
.ok_or_else(|| HotColdDBError::MissingRestorePoint(*state_root))?;
let mut partial_state: PartialBeaconState<E> =
PartialBeaconState::from_ssz_bytes(&partial_state_bytes, &self.spec)?;
// Fill in the fields of the partial state.
partial_state.load_block_roots(&self.cold_db, &self.spec)?;
@@ -717,7 +738,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
// 2. Load the blocks from the high restore point back to the low restore point.
let blocks = self.load_blocks_to_replay(
low_restore_point.slot,
low_restore_point.slot(),
slot,
self.get_high_restore_point_block_root(&high_restore_point, slot)?,
)?;
@@ -759,14 +780,14 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
.filter(|result| {
result
.as_ref()
.map_or(true, |block| block.message.slot <= end_slot)
.map_or(true, |block| block.slot() <= end_slot)
})
// Include the block at the start slot (if any). Whilst it doesn't need to be applied
// to the state, it contains a potentially useful state root.
.take_while(|result| {
result
.as_ref()
.map_or(true, |block| block.message.slot >= start_slot)
.map_or(true, |block| block.slot() >= start_slot)
})
.collect::<Result<_, _>>()?;
blocks.reverse();
@@ -786,18 +807,36 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
) -> Result<BeaconState<E>, Error> {
if block_replay == BlockReplay::InconsistentStateRoots {
for i in 0..blocks.len() {
blocks[i].message.state_root = Hash256::zero();
let prev_block_root = if i > 0 {
blocks[i - 1].canonical_root()
} else {
// Not read.
Hash256::zero()
};
let (state_root, parent_root) = match &mut blocks[i] {
SignedBeaconBlock::Base(block) => (
&mut block.message.state_root,
&mut block.message.parent_root,
),
SignedBeaconBlock::Altair(block) => (
&mut block.message.state_root,
&mut block.message.parent_root,
),
};
*state_root = Hash256::zero();
if i > 0 {
blocks[i].message.parent_root = blocks[i - 1].canonical_root()
*parent_root = prev_block_root;
}
}
}
let state_root_from_prev_block = |i: usize, state: &BeaconState<E>| {
if i > 0 {
let prev_block = &blocks[i - 1].message;
if prev_block.slot == state.slot {
Some(prev_block.state_root)
let prev_block = blocks[i - 1].message();
if prev_block.slot() == state.slot() {
Some(prev_block.state_root())
} else {
None
}
@@ -807,11 +846,11 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
};
for (i, block) in blocks.iter().enumerate() {
if block.message.slot <= state.slot {
if block.slot() <= state.slot() {
continue;
}
while state.slot < block.message.slot {
while state.slot() < block.slot() {
let state_root = match block_replay {
BlockReplay::Accurate => state_root_from_prev_block(i, &state),
BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
@@ -830,7 +869,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
.map_err(HotColdDBError::BlockReplayBlockError)?;
}
while state.slot < target_slot {
while state.slot() < target_slot {
let state_root = match block_replay {
BlockReplay::Accurate => state_root_from_prev_block(blocks.len(), &state),
BlockReplay::InconsistentStateRoots => Some(Hash256::zero()),
@@ -1011,7 +1050,7 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
debug!(
store.log,
"Freezer migration started";
"slot" => frozen_head.slot
"slot" => frozen_head.slot()
);
// 0. Check that the migration is sensible.
@@ -1019,16 +1058,16 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
// boundary (in order for the hot state summary scheme to work).
let current_split_slot = store.split.read().slot;
if frozen_head.slot < current_split_slot {
if frozen_head.slot() < current_split_slot {
return Err(HotColdDBError::FreezeSlotError {
current_split_slot,
proposed_split_slot: frozen_head.slot,
proposed_split_slot: frozen_head.slot(),
}
.into());
}
if frozen_head.slot % E::slots_per_epoch() != 0 {
return Err(HotColdDBError::FreezeSlotUnaligned(frozen_head.slot).into());
if frozen_head.slot() % E::slots_per_epoch() != 0 {
return Err(HotColdDBError::FreezeSlotUnaligned(frozen_head.slot()).into());
}
let mut hot_db_ops: Vec<StoreOp<E>> = Vec::new();
@@ -1045,7 +1084,7 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
let mut cold_db_ops: Vec<KeyValueStoreOp> = Vec::new();
if slot % store.config.slots_per_restore_point == 0 {
let state: BeaconState<E> = get_full_state(&store.hot_db, &state_root)?
let state: BeaconState<E> = get_full_state(&store.hot_db, &state_root, &store.spec)?
.ok_or(HotColdDBError::MissingStateToFreeze(state_root))?;
store.store_cold_state(&state_root, &state, &mut cold_db_ops)?;
@@ -1102,7 +1141,7 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
// Before updating the in-memory split value, we flush it to disk first, so that should the
// OS process die at this point, we pick up from the right place after a restart.
let split = Split {
slot: frozen_head.slot,
slot: frozen_head.slot(),
state_root: frozen_head_root,
};
store.hot_db.put_sync(&SPLIT_KEY, &split)?;
@@ -1119,7 +1158,7 @@ pub fn migrate_database<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
debug!(
store.log,
"Freezer migration complete";
"slot" => frozen_head.slot
"slot" => frozen_head.slot()
);
Ok(())
@@ -1176,8 +1215,8 @@ impl HotStateSummary {
// Fill in the state root on the latest block header if necessary (this happens on all
// slots where there isn't a skip).
let latest_block_root = state.get_latest_block_root(*state_root);
let epoch_boundary_slot = state.slot / E::slots_per_epoch() * E::slots_per_epoch();
let epoch_boundary_state_root = if epoch_boundary_slot == state.slot {
let epoch_boundary_slot = state.slot() / E::slots_per_epoch() * E::slots_per_epoch();
let epoch_boundary_state_root = if epoch_boundary_slot == state.slot() {
*state_root
} else {
*state
@@ -1186,7 +1225,7 @@ impl HotStateSummary {
};
Ok(HotStateSummary {
slot: state.slot,
slot: state.slot(),
latest_block_root,
epoch_boundary_state_root,
})

40
beacon_node/store/src/impls.rs
View File

@@ -1,35 +1,15 @@
use crate::*;
use ssz::{Decode, Encode};
use ssz::Encode;
pub mod beacon_state;
pub mod partial_beacon_state;
impl<T: EthSpec> StoreItem for SignedBeaconBlock<T> {
fn db_column() -> DBColumn {
DBColumn::BeaconBlock
}
fn as_store_bytes(&self) -> Vec<u8> {
let timer = metrics::start_timer(&metrics::BEACON_BLOCK_WRITE_TIMES);
let bytes = self.as_ssz_bytes();
metrics::stop_timer(timer);
metrics::inc_counter(&metrics::BEACON_BLOCK_WRITE_COUNT);
metrics::inc_counter_by(&metrics::BEACON_BLOCK_WRITE_BYTES, bytes.len() as u64);
bytes
}
fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
let timer = metrics::start_timer(&metrics::BEACON_BLOCK_READ_TIMES);
let len = bytes.len();
let result = Self::from_ssz_bytes(bytes).map_err(Into::into);
metrics::stop_timer(timer);
metrics::inc_counter(&metrics::BEACON_BLOCK_READ_COUNT);
metrics::inc_counter_by(&metrics::BEACON_BLOCK_READ_BYTES, len as u64);
result
}
/// Prepare a signed beacon block for storage in the database.
#[must_use]
pub fn beacon_block_as_kv_store_op<T: EthSpec>(
key: &Hash256,
block: &SignedBeaconBlock<T>,
) -> KeyValueStoreOp {
// FIXME(altair): re-add block write/overhead metrics, or remove them
let db_key = get_key_for_col(DBColumn::BeaconBlock.into(), key.as_bytes());
KeyValueStoreOp::PutKeyValue(db_key, block.as_ssz_bytes())
}

32
beacon_node/store/src/impls/beacon_state.rs
View File

@@ -1,6 +1,6 @@
use crate::*;
use ssz::{Decode, DecodeError, Encode};
use ssz_derive::{Decode, Encode};
use ssz::{DecodeError, Encode};
use ssz_derive::Encode;
use std::convert::TryInto;
use types::beacon_state::{CloneConfig, CommitteeCache, CACHED_EPOCHS};
@@ -23,13 +23,14 @@ pub fn store_full_state<E: EthSpec>(
pub fn get_full_state<KV: KeyValueStore<E>, E: EthSpec>(
db: &KV,
state_root: &Hash256,
spec: &ChainSpec,
) -> Result<Option<BeaconState<E>>, Error> {
let total_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES);
match db.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())? {
Some(bytes) => {
let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_OVERHEAD_TIMES);
let container = StorageContainer::from_ssz_bytes(&bytes)?;
let container = StorageContainer::from_ssz_bytes(&bytes, spec)?;
metrics::stop_timer(overhead_timer);
metrics::stop_timer(total_timer);
@@ -44,7 +45,7 @@ pub fn get_full_state<KV: KeyValueStore<E>, E: EthSpec>(
/// A container for storing `BeaconState` components.
// TODO: would be more space efficient with the caches stored separately and referenced by hash
#[derive(Encode, Decode)]
#[derive(Encode)]
pub struct StorageContainer<T: EthSpec> {
state: BeaconState<T>,
committee_caches: Vec<CommitteeCache>,
@@ -55,9 +56,28 @@ impl<T: EthSpec> StorageContainer<T> {
pub fn new(state: &BeaconState<T>) -> Self {
Self {
state: state.clone_with(CloneConfig::none()),
committee_caches: state.committee_caches.to_vec(),
committee_caches: state.committee_caches().to_vec(),
}
}
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
// We need to use the slot-switching `from_ssz_bytes` of `BeaconState`, which doesn't
// compose with the other SSZ utils, so we duplicate some parts of `ssz_derive` here.
let mut builder = ssz::SszDecoderBuilder::new(bytes);
builder.register_anonymous_variable_length_item()?;
builder.register_type::<Vec<CommitteeCache>>()?;
let mut decoder = builder.build()?;
let state = decoder.decode_next_with(|bytes| BeaconState::from_ssz_bytes(bytes, spec))?;
let committee_caches = decoder.decode_next()?;
Ok(Self {
state,
committee_caches,
})
}
}
impl<T: EthSpec> TryInto<BeaconState<T>> for StorageContainer<T> {
@@ -73,7 +93,7 @@ impl<T: EthSpec> TryInto<BeaconState<T>> for StorageContainer<T> {
)));
};
state.committee_caches[i] = self.committee_caches.remove(i);
state.committee_caches_mut()[i] = self.committee_caches.remove(i);
}
Ok(state)

16
beacon_node/store/src/impls/partial_beacon_state.rs
View File

@@ -1,16 +0,0 @@
use crate::*;
use ssz::{Decode, Encode};
impl<T: EthSpec> StoreItem for PartialBeaconState<T> {
fn db_column() -> DBColumn {
DBColumn::BeaconState
}
fn as_store_bytes(&self) -> Vec<u8> {
self.as_ssz_bytes()
}
fn from_store_bytes(bytes: &[u8]) -> Result<Self, Error> {
Ok(Self::from_ssz_bytes(bytes)?)
}
}

56
beacon_node/store/src/iter.rs
View File

@@ -27,7 +27,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
store: Arc<HotColdDB<E, Hot, Cold>>,
) -> Option<BlockRootsIterator<'a, E, Hot, Cold>> {
let state = store
.get_state(&self.message.state_root, Some(self.message.slot))
.get_state(&self.message().state_root(), Some(self.slot()))
.ok()??;
Some(BlockRootsIterator::owned(store, state))
@@ -161,7 +161,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
pub fn new(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: &'a BeaconState<T>) -> Self {
Self {
store,
slot: beacon_state.slot,
slot: beacon_state.slot(),
beacon_state: Cow::Borrowed(beacon_state),
}
}
@@ -169,7 +169,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
pub fn owned(store: Arc<HotColdDB<T, Hot, Cold>>, beacon_state: BeaconState<T>) -> Self {
Self {
store,
slot: beacon_state.slot,
slot: beacon_state.slot(),
beacon_state: Cow::Owned(beacon_state),
}
}
@@ -188,7 +188,7 @@ impl<'a, T: EthSpec, Hot: ItemStore<T>, Cold: ItemStore<T>> RootsIterator<'a, T,
}
fn do_next(&mut self) -> Result<Option<(Hash256, Hash256, Slot)>, Error> {
if self.slot == 0 || self.slot > self.beacon_state.slot {
if self.slot == 0 || self.slot > self.beacon_state.slot() {
return Ok(None);
}
@@ -257,7 +257,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
.store
.get_block(&block_root)?
.ok_or(Error::BlockNotFound(block_root))?;
self.next_block_root = block.message.parent_root;
self.next_block_root = block.message().parent_root();
Ok(Some((block_root, block)))
}
}
@@ -323,7 +323,7 @@ fn next_historical_root_backtrack_state<E: EthSpec, Hot: ItemStore<E>, Cold: Ite
// a restore point slot (thus avoiding replaying blocks). In the case where we're
// not frozen, this just means we might not jump back by the maximum amount on
// our first jump (i.e. at most 1 extra state load).
let new_state_slot = slot_of_prev_restore_point::<E>(current_state.slot);
let new_state_slot = slot_of_prev_restore_point::<E>(current_state.slot());
let new_state_root = current_state.get_state_root(new_state_slot)?;
Ok(store
.get_state(new_state_root, Some(new_state_slot))?
@@ -339,46 +339,50 @@ fn slot_of_prev_restore_point<E: EthSpec>(current_slot: Slot) -> Slot {
#[cfg(test)]
mod test {
use super::*;
use crate::config::StoreConfig;
use crate::HotColdDB;
use crate::StoreConfig as Config;
use beacon_chain::store::StoreConfig;
use beacon_chain::test_utils::BeaconChainHarness;
use beacon_chain::types::{ChainSpec, Keypair, MainnetEthSpec};
use sloggers::{null::NullLoggerBuilder, Build};
use types::{test_utils::TestingBeaconStateBuilder, ChainSpec, Keypair, MainnetEthSpec};
fn get_state<T: EthSpec>() -> BeaconState<T> {
let builder = TestingBeaconStateBuilder::from_single_keypair(
0,
&Keypair::random(),
&T::default_spec(),
let harness = BeaconChainHarness::new_with_store_config(
T::default(),
None,
vec![Keypair::random()],
StoreConfig::default(),
);
let (state, _keypairs) = builder.build();
state
harness.advance_slot();
harness.get_current_state()
}
#[test]
fn block_root_iter() {
let log = NullLoggerBuilder.build().unwrap();
let store = Arc::new(
HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal(), log).unwrap(),
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap(),
);
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state();
let mut state_b: BeaconState<MainnetEthSpec> = get_state();
state_a.slot = Slot::from(slots_per_historical_root);
state_b.slot = Slot::from(slots_per_historical_root * 2);
*state_a.slot_mut() = Slot::from(slots_per_historical_root);
*state_b.slot_mut() = Slot::from(slots_per_historical_root * 2);
let mut hashes = (0..).map(Hash256::from_low_u64_be);
for root in &mut state_a.block_roots[..] {
*root = hashes.next().unwrap()
let roots_a = state_a.block_roots_mut();
for i in 0..roots_a.len() {
roots_a[i] = hashes.next().unwrap()
}
for root in &mut state_b.block_roots[..] {
*root = hashes.next().unwrap()
let roots_b = state_b.block_roots_mut();
for i in 0..roots_b.len() {
roots_b[i] = hashes.next().unwrap()
}
let state_a_root = hashes.next().unwrap();
state_b.state_roots[0] = state_a_root;
state_b.state_roots_mut()[0] = state_a_root;
store.put_state(&state_a_root, &state_a).unwrap();
let iter = BlockRootsIterator::new(store, &state_b);
@@ -405,15 +409,15 @@ mod test {
fn state_root_iter() {
let log = NullLoggerBuilder.build().unwrap();
let store = Arc::new(
HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal(), log).unwrap(),
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap(),
);
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state();
let mut state_b: BeaconState<MainnetEthSpec> = get_state();
state_a.slot = Slot::from(slots_per_historical_root);
state_b.slot = Slot::from(slots_per_historical_root * 2);
*state_a.slot_mut() = Slot::from(slots_per_historical_root);
*state_b.slot_mut() = Slot::from(slots_per_historical_root * 2);
let mut hashes = (0..).map(Hash256::from_low_u64_be);

273
beacon_node/store/src/partial_beacon_state.rs
View File

@@ -2,17 +2,21 @@ use crate::chunked_vector::{
load_variable_list_from_db, load_vector_from_db, BlockRoots, HistoricalRoots, RandaoMixes,
StateRoots,
};
use crate::{Error, KeyValueStore};
use crate::{get_key_for_col, DBColumn, Error, KeyValueStore, KeyValueStoreOp};
use ssz::{Decode, DecodeError, Encode};
use ssz_derive::{Decode, Encode};
use std::convert::TryInto;
use types::superstruct;
use types::*;
/// Lightweight variant of the `BeaconState` that is stored in the database.
///
/// Utilises lazy-loading from separate storage for its vector fields.
///
/// Spec v0.12.1
#[derive(Debug, PartialEq, Clone, Encode, Decode)]
#[superstruct(
variants(Base, Altair),
variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode))
)]
#[derive(Debug, PartialEq, Clone, Encode)]
pub struct PartialBeaconState<T>
where
T: EthSpec,
@@ -20,6 +24,7 @@ where
// Versioning
pub genesis_time: u64,
pub genesis_validators_root: Hash256,
#[superstruct(getter(copy))]
pub slot: Slot,
pub fork: Fork,
@@ -56,71 +61,152 @@ where
// Slashings
slashings: FixedVector<u64, T::EpochsPerSlashingsVector>,
// Attestations
// Attestations (genesis fork only)
#[superstruct(only(Base))]
pub previous_epoch_attestations: VariableList<PendingAttestation<T>, T::MaxPendingAttestations>,
#[superstruct(only(Base))]
pub current_epoch_attestations: VariableList<PendingAttestation<T>, T::MaxPendingAttestations>,
// Participation (Altair and later)
#[superstruct(only(Altair))]
pub previous_epoch_participation: VariableList<ParticipationFlags, T::ValidatorRegistryLimit>,
#[superstruct(only(Altair))]
pub current_epoch_participation: VariableList<ParticipationFlags, T::ValidatorRegistryLimit>,
// Finality
pub justification_bits: BitVector<T::JustificationBitsLength>,
pub previous_justified_checkpoint: Checkpoint,
pub current_justified_checkpoint: Checkpoint,
pub finalized_checkpoint: Checkpoint,
// Inactivity
#[superstruct(only(Altair))]
pub inactivity_scores: VariableList<u64, T::ValidatorRegistryLimit>,
// Light-client sync committees
#[superstruct(only(Altair))]
pub current_sync_committee: SyncCommittee<T>,
#[superstruct(only(Altair))]
pub next_sync_committee: SyncCommittee<T>,
}
impl<T: EthSpec> PartialBeaconState<T> {
/// Convert a `BeaconState` to a `PartialBeaconState`, while dropping the optional fields.
pub fn from_state_forgetful(s: &BeaconState<T>) -> Self {
// TODO: could use references/Cow for fields to avoid cloning
PartialBeaconState {
genesis_time: s.genesis_time,
genesis_validators_root: s.genesis_validators_root,
slot: s.slot,
fork: s.fork,
/// Implement the conversion function from BeaconState -> PartialBeaconState.
macro_rules! impl_from_state_forgetful {
($s:ident, $outer:ident, $variant_name:ident, $struct_name:ident, [$($extra_fields:ident),*]) => {
PartialBeaconState::$variant_name($struct_name {
// Versioning
genesis_time: $s.genesis_time,
genesis_validators_root: $s.genesis_validators_root,
slot: $s.slot,
fork: $s.fork,
// History
latest_block_header: s.latest_block_header.clone(),
latest_block_header: $s.latest_block_header.clone(),
block_roots: None,
state_roots: None,
historical_roots: None,
// Eth1
eth1_data: s.eth1_data.clone(),
eth1_data_votes: s.eth1_data_votes.clone(),
eth1_deposit_index: s.eth1_deposit_index,
eth1_data: $s.eth1_data.clone(),
eth1_data_votes: $s.eth1_data_votes.clone(),
eth1_deposit_index: $s.eth1_deposit_index,
// Validator registry
validators: s.validators.clone(),
balances: s.balances.clone(),
validators: $s.validators.clone(),
balances: $s.balances.clone(),
// Shuffling
latest_randao_value: *s
.get_randao_mix(s.current_epoch())
latest_randao_value: *$outer
.get_randao_mix($outer.current_epoch())
.expect("randao at current epoch is OK"),
randao_mixes: None,
// Slashings
slashings: s.get_all_slashings().to_vec().into(),
// Attestations
previous_epoch_attestations: s.previous_epoch_attestations.clone(),
current_epoch_attestations: s.current_epoch_attestations.clone(),
slashings: $s.slashings.clone(),
// Finality
justification_bits: s.justification_bits.clone(),
previous_justified_checkpoint: s.previous_justified_checkpoint,
current_justified_checkpoint: s.current_justified_checkpoint,
finalized_checkpoint: s.finalized_checkpoint,
justification_bits: $s.justification_bits.clone(),
previous_justified_checkpoint: $s.previous_justified_checkpoint,
current_justified_checkpoint: $s.current_justified_checkpoint,
finalized_checkpoint: $s.finalized_checkpoint,
// Variant-specific fields
$(
$extra_fields: $s.$extra_fields.clone()
),*
})
}
}
impl<T: EthSpec> PartialBeaconState<T> {
/// Convert a `BeaconState` to a `PartialBeaconState`, while dropping the optional fields.
pub fn from_state_forgetful(outer: &BeaconState<T>) -> Self {
match outer {
BeaconState::Base(s) => impl_from_state_forgetful!(
s,
outer,
Base,
PartialBeaconStateBase,
[previous_epoch_attestations, current_epoch_attestations]
),
BeaconState::Altair(s) => impl_from_state_forgetful!(
s,
outer,
Altair,
PartialBeaconStateAltair,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores
]
),
}
}
/// SSZ decode.
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
// Slot is after genesis_time (u64) and genesis_validators_root (Hash256).
let slot_offset = <u64 as Decode>::ssz_fixed_len() + <Hash256 as Decode>::ssz_fixed_len();
let slot_len = <Slot as Decode>::ssz_fixed_len();
let slot_bytes = bytes.get(slot_offset..slot_offset + slot_len).ok_or(
DecodeError::InvalidByteLength {
len: bytes.len(),
expected: slot_offset + slot_len,
},
)?;
let slot = Slot::from_ssz_bytes(slot_bytes)?;
let epoch = slot.epoch(T::slots_per_epoch());
if spec
.altair_fork_epoch
.map_or(true, |altair_epoch| epoch < altair_epoch)
{
PartialBeaconStateBase::from_ssz_bytes(bytes).map(Self::Base)
} else {
PartialBeaconStateAltair::from_ssz_bytes(bytes).map(Self::Altair)
}
}
/// Prepare the partial state for storage in the KV database.
#[must_use]
pub fn as_kv_store_op(&self, state_root: Hash256) -> KeyValueStoreOp {
let db_key = get_key_for_col(DBColumn::BeaconState.into(), state_root.as_bytes());
KeyValueStoreOp::PutKeyValue(db_key, self.as_ssz_bytes())
}
pub fn load_block_roots<S: KeyValueStore<T>>(
&mut self,
store: &S,
spec: &ChainSpec,
) -> Result<(), Error> {
if self.block_roots.is_none() {
self.block_roots = Some(load_vector_from_db::<BlockRoots, T, _>(
store, self.slot, spec,
if self.block_roots().is_none() {
*self.block_roots_mut() = Some(load_vector_from_db::<BlockRoots, T, _>(
store,
self.slot(),
spec,
)?);
}
Ok(())
@@ -131,9 +217,11 @@ impl<T: EthSpec> PartialBeaconState<T> {
store: &S,
spec: &ChainSpec,
) -> Result<(), Error> {
if self.state_roots.is_none() {
self.state_roots = Some(load_vector_from_db::<StateRoots, T, _>(
store, self.slot, spec,
if self.state_roots().is_none() {
*self.state_roots_mut() = Some(load_vector_from_db::<StateRoots, T, _>(
store,
self.slot(),
spec,
)?);
}
Ok(())
@@ -144,10 +232,10 @@ impl<T: EthSpec> PartialBeaconState<T> {
store: &S,
spec: &ChainSpec,
) -> Result<(), Error> {
if self.historical_roots.is_none() {
self.historical_roots = Some(load_variable_list_from_db::<HistoricalRoots, T, _>(
store, self.slot, spec,
)?);
if self.historical_roots().is_none() {
*self.historical_roots_mut() = Some(
load_variable_list_from_db::<HistoricalRoots, T, _>(store, self.slot(), spec)?,
);
}
Ok(())
}
@@ -157,72 +245,101 @@ impl<T: EthSpec> PartialBeaconState<T> {
store: &S,
spec: &ChainSpec,
) -> Result<(), Error> {
if self.randao_mixes.is_none() {
if self.randao_mixes().is_none() {
// Load the per-epoch values from the database
let mut randao_mixes =
load_vector_from_db::<RandaoMixes, T, _>(store, self.slot, spec)?;
load_vector_from_db::<RandaoMixes, T, _>(store, self.slot(), spec)?;
// Patch the value for the current slot into the index for the current epoch
let current_epoch = self.slot.epoch(T::slots_per_epoch());
let current_epoch = self.slot().epoch(T::slots_per_epoch());
let len = randao_mixes.len();
randao_mixes[current_epoch.as_usize() % len] = self.latest_randao_value;
randao_mixes[current_epoch.as_usize() % len] = *self.latest_randao_value();
self.randao_mixes = Some(randao_mixes)
*self.randao_mixes_mut() = Some(randao_mixes)
}
Ok(())
}
}
impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
type Error = Error;
fn try_into(self) -> Result<BeaconState<E>, Error> {
fn unpack<T>(x: Option<T>) -> Result<T, Error> {
x.ok_or(Error::PartialBeaconStateError)
}
Ok(BeaconState {
genesis_time: self.genesis_time,
genesis_validators_root: self.genesis_validators_root,
slot: self.slot,
fork: self.fork,
/// Implement the conversion from PartialBeaconState -> BeaconState.
macro_rules! impl_try_into_beacon_state {
($inner:ident, $variant_name:ident, $struct_name:ident, [$($extra_fields:ident),*]) => {
BeaconState::$variant_name($struct_name {
// Versioning
genesis_time: $inner.genesis_time,
genesis_validators_root: $inner.genesis_validators_root,
slot: $inner.slot,
fork: $inner.fork,
// History
latest_block_header: self.latest_block_header,
block_roots: unpack(self.block_roots)?,
state_roots: unpack(self.state_roots)?,
historical_roots: unpack(self.historical_roots)?,
latest_block_header: $inner.latest_block_header,
block_roots: unpack_field($inner.block_roots)?,
state_roots: unpack_field($inner.state_roots)?,
historical_roots: unpack_field($inner.historical_roots)?,
// Eth1
eth1_data: self.eth1_data,
eth1_data_votes: self.eth1_data_votes,
eth1_deposit_index: self.eth1_deposit_index,
eth1_data: $inner.eth1_data,
eth1_data_votes: $inner.eth1_data_votes,
eth1_deposit_index: $inner.eth1_deposit_index,
// Validator registry
validators: self.validators,
balances: self.balances,
validators: $inner.validators,
balances: $inner.balances,
// Shuffling
randao_mixes: unpack(self.randao_mixes)?,
randao_mixes: unpack_field($inner.randao_mixes)?,
// Slashings
slashings: self.slashings,
// Attestations
previous_epoch_attestations: self.previous_epoch_attestations,
current_epoch_attestations: self.current_epoch_attestations,
slashings: $inner.slashings,
// Finality
justification_bits: self.justification_bits,
previous_justified_checkpoint: self.previous_justified_checkpoint,
current_justified_checkpoint: self.current_justified_checkpoint,
finalized_checkpoint: self.finalized_checkpoint,
justification_bits: $inner.justification_bits,
previous_justified_checkpoint: $inner.previous_justified_checkpoint,
current_justified_checkpoint: $inner.current_justified_checkpoint,
finalized_checkpoint: $inner.finalized_checkpoint,
// Caching
committee_caches: <_>::default(),
pubkey_cache: <_>::default(),
exit_cache: <_>::default(),
tree_hash_cache: <_>::default(),
// Variant-specific fields
$(
$extra_fields: $inner.$extra_fields
),*
})
}
}
fn unpack_field<T>(x: Option<T>) -> Result<T, Error> {
x.ok_or(Error::PartialBeaconStateError)
}
impl<E: EthSpec> TryInto<BeaconState<E>> for PartialBeaconState<E> {
type Error = Error;
fn try_into(self) -> Result<BeaconState<E>, Error> {
let state = match self {
PartialBeaconState::Base(inner) => impl_try_into_beacon_state!(
inner,
Base,
BeaconStateBase,
[previous_epoch_attestations, current_epoch_attestations]
),
PartialBeaconState::Altair(inner) => impl_try_into_beacon_state!(
inner,
Altair,
BeaconStateAltair,
[
previous_epoch_participation,
current_epoch_participation,
current_sync_committee,
next_sync_committee,
inactivity_scores
]
),
};
Ok(state)
}
}