gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-19 21:04:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

In-memory tree states (#5533)

Browse Source 
* Consensus changes

* EF tests

* lcli

* common and watch

* account manager

* cargo

* fork choice

* promise cache

* beacon chain

* interop genesis

* http api

* lighthouse

* op pool

* beacon chain misc

* parallel state cache

* store

* fix issues in store

* IT COMPILES

* Remove some unnecessary module qualification

* Revert Arced pubkey optimization (#5536)

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Fix caching, rebasing and some tests

* Remove unused deps

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Small cleanups

* Revert shuffling cache/promise cache changes

* Fix state advance bugs

* Fix shuffling tests

* Remove some resolved FIXMEs

* Remove StateProcessingStrategy

* Optimise withdrawals calculation

* Don't reorg if state cache is missed

* Remove inconsistent state func

* Fix beta compiler

* Rebase early, rebase often

* Fix state caching behaviour

* Update to milhouse release

* Fix on-disk consensus context format

* Merge remote-tracking branch 'origin/unstable' into tree-states-memory

* Squashed commit of the following:

commit 3a16649023
Author: Michael Sproul <michael@sigmaprime.io>
Date:   Thu Apr 18 14:26:09 2024 +1000

    Fix on-disk consensus context format

* Keep indexed attestations, thanks Sean

* Merge branch 'on-disk-consensus-context' into tree-states-memory

* Merge branch 'unstable' into tree-states-memory

* Address half of Sean's review

* More simplifications from Sean's review

* Cache state after get_advanced_hot_state
This commit is contained in:
Michael Sproul
2024-04-24 11:22:36 +10:00
committed by GitHub
parent 4cad1fcbbe
commit 61962898e2
108 changed files with 2038 additions and 2762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
beacon_node/store/src/chunked_vector.rs
View File

@@ -1,6 +1,6 @@
//! Space-efficient storage for `BeaconState` vector fields.
//!
//! This module provides logic for splitting the `FixedVector` fields of a `BeaconState` into
//! This module provides logic for splitting the `Vector` fields of a `BeaconState` into
//! chunks, and storing those chunks in contiguous ranges in the on-disk database. The motiviation
//! for doing this is avoiding massive duplication in every on-disk state. For example, rather than
//! storing the whole `historical_roots` vector, which is updated once every couple of thousand
@@ -60,12 +60,13 @@ fn genesis_value_key() -> [u8; 8] {
/// type-level. We require their value-level witnesses to be `Copy` so that we can avoid the
/// turbofish when calling functions like `store_updated_vector`.
pub trait Field<E: EthSpec>: Copy {
/// The type of value stored in this field: the `T` from `FixedVector<T, N>`.
/// The type of value stored in this field: the `T` from `Vector<T, N>`.
///
/// The `Default` impl will be used to fill extra vector entries.
type Value: Decode + Encode + Default + Clone + PartialEq + std::fmt::Debug;
type Value: Default + std::fmt::Debug + milhouse::Value;
// Decode + Encode + Default + Clone + PartialEq + std::fmt::Debug
/// The length of this field: the `N` from `FixedVector<T, N>`.
/// The length of this field: the `N` from `Vector<T, N>`.
type Length: Unsigned;
/// The database column where the integer-indexed chunks for this field should be stored.
@@ -273,10 +274,10 @@ pub trait Field<E: EthSpec>: Copy {
}
}
/// Marker trait for fixed-length fields (`FixedVector<T, N>`).
/// Marker trait for fixed-length fields (`Vector<T, N>`).
pub trait FixedLengthField<E: EthSpec>: Field<E> {}
/// Marker trait for variable-length fields (`VariableList<T, N>`).
/// Marker trait for variable-length fields (`List<T, N>`).
pub trait VariableLengthField<E: EthSpec>: Field<E> {}
/// Macro to implement the `Field` trait on a new unit struct type.
@@ -331,7 +332,7 @@ field!(
activation_slot: Some(Slot::new(0)),
deactivation_slot: None
},
|state: &BeaconState<_>, index, _| safe_modulo_index(state.block_roots(), index)
|state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.block_roots(), index)
);
field!(
@@ -345,7 +346,7 @@ field!(
activation_slot: Some(Slot::new(0)),
deactivation_slot: None,
},
|state: &BeaconState<_>, index, _| safe_modulo_index(state.state_roots(), index)
|state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.state_roots(), index)
);
field!(
@@ -361,7 +362,7 @@ field!(
.capella_fork_epoch
.map(|fork_epoch| fork_epoch.start_slot(E::slots_per_epoch())),
},
|state: &BeaconState<_>, index, _| safe_modulo_index(state.historical_roots(), index)
|state: &BeaconState<_>, index, _| safe_modulo_list_index(state.historical_roots(), index)
);
field!(
@@ -371,7 +372,7 @@ field!(
E::EpochsPerHistoricalVector,
DBColumn::BeaconRandaoMixes,
|_| OncePerEpoch { lag: 1 },
|state: &BeaconState<_>, index, _| safe_modulo_index(state.randao_mixes(), index)
|state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.randao_mixes(), index)
);
field!(
@@ -387,7 +388,7 @@ field!(
.map(|fork_epoch| fork_epoch.start_slot(E::slots_per_epoch())),
deactivation_slot: None,
},
|state: &BeaconState<_>, index, _| safe_modulo_index(
|state: &BeaconState<_>, index, _| safe_modulo_list_index(
state
.historical_summaries()
.map_err(|_| ChunkError::InvalidFork)?,
@@ -565,7 +566,7 @@ pub fn load_vector_from_db<F: FixedLengthField<E>, E: EthSpec, S: KeyValueStore<
store: &S,
slot: Slot,
spec: &ChainSpec,
) -> Result<FixedVector<F::Value, F::Length>, Error> {
) -> Result<Vector<F::Value, F::Length>, Error> {
// Do a range query
let chunk_size = F::chunk_size();
let (start_vindex, end_vindex) = F::start_and_end_vindex(slot, spec);
@@ -589,7 +590,7 @@ pub fn load_vector_from_db<F: FixedLengthField<E>, E: EthSpec, S: KeyValueStore<
default,
)?;
Ok(result.into())
Ok(Vector::new(result).map_err(ChunkError::Milhouse)?)
}
/// The historical roots are stored in vector chunks, despite not actually being a vector.
@@ -597,7 +598,7 @@ pub fn load_variable_list_from_db<F: VariableLengthField<E>, E: EthSpec, S: KeyV
store: &S,
slot: Slot,
spec: &ChainSpec,
) -> Result<VariableList<F::Value, F::Length>, Error> {
) -> Result<List<F::Value, F::Length>, Error> {
let chunk_size = F::chunk_size();
let (start_vindex, end_vindex) = F::start_and_end_vindex(slot, spec);
let start_cindex = start_vindex / chunk_size;
@@ -617,15 +618,35 @@ pub fn load_variable_list_from_db<F: VariableLengthField<E>, E: EthSpec, S: KeyV
}
}
Ok(result.into())
Ok(List::new(result).map_err(ChunkError::Milhouse)?)
}
/// Index into a field of the state, avoiding out of bounds and division by 0.
fn safe_modulo_index<T: Copy>(values: &[T], index: u64) -> Result<T, ChunkError> {
/// Index into a `List` field of the state, avoiding out of bounds and division by 0.
fn safe_modulo_list_index<T: milhouse::Value + Copy, N: Unsigned>(
values: &List<T, N>,
index: u64,
) -> Result<T, ChunkError> {
if values.is_empty() {
Err(ChunkError::ZeroLengthList)
} else {
values
.get(index as usize % values.len())
.copied()
.ok_or(ChunkError::IndexOutOfBounds { index })
}
}
fn safe_modulo_vector_index<T: milhouse::Value + Copy, N: Unsigned>(
values: &Vector<T, N>,
index: u64,
) -> Result<T, ChunkError> {
if values.is_empty() {
Err(ChunkError::ZeroLengthVector)
} else {
Ok(values[index as usize % values.len()])
values
.get(index as usize % values.len())
.copied()
.ok_or(ChunkError::IndexOutOfBounds { index })
}
}
@@ -712,6 +733,10 @@ where
#[derive(Debug, PartialEq)]
pub enum ChunkError {
ZeroLengthVector,
ZeroLengthList,
IndexOutOfBounds {
index: u64,
},
InvalidSize {
chunk_index: usize,
expected: usize,
@@ -744,6 +769,13 @@ pub enum ChunkError {
length: usize,
},
InvalidFork,
Milhouse(milhouse::Error),
}
impl From<milhouse::Error> for ChunkError {
fn from(e: milhouse::Error) -> ChunkError {
Self::Milhouse(e)
}
}
#[cfg(test)]