gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-11 18:04:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement Overflow LRU Cache for Pending Blobs (#4203)

Browse Source 
* All Necessary Objects Implement Encode/Decode

* Major Components for LRUOverflowCache Implemented

* Finish Database Code

* Add Maintenance Methods

* Added Maintenance Service

* Persist Blobs on Shutdown / Reload on Startup

* Address Clippy Complaints

* Add (emum_behaviour = "tag") to ssz_derive

* Convert Encode/Decode Implementations to "tag"

* Started Adding Tests

* Added a ton of tests

* 1 character fix

* Feature Guard Minimal Spec Tests

* Update beacon_node/beacon_chain/src/data_availability_checker.rs

Co-authored-by: realbigsean <seananderson33@GMAIL.com>

* Address Sean's Comments

* Add iter_raw_keys method

* Remove TODOs

---------

Co-authored-by: realbigsean <seananderson33@GMAIL.com>
This commit is contained in:
ethDreamer
2023-05-12 09:08:24 -05:00
committed by GitHub
parent a22e4bf636
commit 46db30416d
23 changed files with 2364 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
consensus/types/src/beacon_block.rs
View File

@@ -201,13 +201,7 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_slot::<T>(self.slot());
let object_fork = match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -219,6 +213,19 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload<T>> BeaconBlockRef<'a, T, Payl
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconBlockRef::Base { .. } => ForkName::Base,
BeaconBlockRef::Altair { .. } => ForkName::Altair,
BeaconBlockRef::Merge { .. } => ForkName::Merge,
BeaconBlockRef::Capella { .. } => ForkName::Capella,
BeaconBlockRef::Deneb { .. } => ForkName::Deneb,
}
}
/// Convenience accessor for the `body` as a `BeaconBlockBodyRef`.
pub fn body(&self) -> BeaconBlockBodyRef<'a, T, Payload> {
map_beacon_block_ref_into_beacon_block_body_ref!(&'a _, *self, |block, cons| cons(

98
consensus/types/src/beacon_state.rs
View File

@@ -415,13 +415,7 @@ impl<T: EthSpec> BeaconState<T> {
/// dictated by `self.slot()`.
pub fn fork_name(&self, spec: &ChainSpec) -> Result<ForkName, InconsistentFork> {
let fork_at_slot = spec.fork_name_at_epoch(self.current_epoch());
let object_fork = match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
};
let object_fork = self.fork_name_unchecked();
if fork_at_slot == object_fork {
Ok(object_fork)
@@ -433,6 +427,19 @@ impl<T: EthSpec> BeaconState<T> {
}
}
/// Returns the name of the fork pertaining to `self`.
///
/// Does not check if `self` is consistent with the fork dictated by `self.slot()`.
pub fn fork_name_unchecked(&self) -> ForkName {
match self {
BeaconState::Base { .. } => ForkName::Base,
BeaconState::Altair { .. } => ForkName::Altair,
BeaconState::Merge { .. } => ForkName::Merge,
BeaconState::Capella { .. } => ForkName::Capella,
BeaconState::Deneb { .. } => ForkName::Deneb,
}
}
/// Specialised deserialisation method that uses the `ChainSpec` as context.
#[allow(clippy::integer_arithmetic)]
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
@@ -1870,3 +1877,80 @@ impl<T: EthSpec> ForkVersionDeserialize for BeaconState<T> {
))
}
}
/// This module can be used to encode and decode a `BeaconState` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used for *some* cases to store these objects in the
/// database and _NEVER_ for encoding / decoding states sent over the network!
pub mod ssz_tagged_beacon_state {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec>(state: &BeaconState<E>) -> usize {
state
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec>(state: &BeaconState<E>, buf: &mut Vec<u8>) {
let fork_name = state.fork_name_unchecked();
fork_name.ssz_append(buf);
state.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec>(state: &BeaconState<E>) -> Vec<u8> {
let mut buf = vec![];
ssz_append(state, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec>(bytes: &[u8]) -> Result<BeaconState<E>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(BeaconState::Base(BeaconStateBase::from_ssz_bytes(body)?)),
ForkName::Altair => Ok(BeaconState::Altair(BeaconStateAltair::from_ssz_bytes(
body,
)?)),
ForkName::Merge => Ok(BeaconState::Merge(BeaconStateMerge::from_ssz_bytes(body)?)),
ForkName::Capella => Ok(BeaconState::Capella(BeaconStateCapella::from_ssz_bytes(
body,
)?)),
ForkName::Deneb => Ok(BeaconState::Deneb(BeaconStateDeneb::from_ssz_bytes(body)?)),
}
}
}
}

4
consensus/types/src/fork_name.rs
View File

@@ -1,12 +1,14 @@
use crate::{ChainSpec, Epoch};
use serde_derive::{Deserialize, Serialize};
use ssz_derive::{Decode, Encode};
use std::convert::TryFrom;
use std::fmt::{self, Display, Formatter};
use std::str::FromStr;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[derive(Debug, Clone, Copy, Decode, Encode, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(try_from = "String")]
#[serde(into = "String")]
#[ssz(enum_behaviour = "tag")]
pub enum ForkName {
Base,
Altair,

6
consensus/types/src/lib.rs
View File

@@ -170,9 +170,9 @@ pub use crate::selection_proof::SelectionProof;
pub use crate::shuffling_id::AttestationShufflingId;
pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof;
pub use crate::signed_beacon_block::{
SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella,
SignedBeaconBlockDeneb, SignedBeaconBlockHash, SignedBeaconBlockMerge,
SignedBlindedBeaconBlock,
ssz_tagged_signed_beacon_block, SignedBeaconBlock, SignedBeaconBlockAltair,
SignedBeaconBlockBase, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockHash,
SignedBeaconBlockMerge, SignedBlindedBeaconBlock,
};
pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader;
pub use crate::signed_blob::*;

133
consensus/types/src/signed_beacon_block.rs
View File

@@ -92,6 +92,12 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> SignedBeaconBlock<E, Payload>
self.message().fork_name(spec)
}
/// Returns the name of the fork pertaining to `self`
/// Does not check that the fork is consistent with the slot.
pub fn fork_name_unchecked(&self) -> ForkName {
self.message().fork_name_unchecked()
}
/// SSZ decode with fork variant determined by slot.
pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result<Self, ssz::DecodeError> {
Self::from_ssz_bytes_with(bytes, |bytes| BeaconBlock::from_ssz_bytes(bytes, spec))
@@ -510,6 +516,99 @@ impl<E: EthSpec, Payload: AbstractExecPayload<E>> ForkVersionDeserialize
}
}
/// This module can be used to encode and decode a `SignedBeaconBlock` the same way it
/// would be done if we had tagged the superstruct enum with
/// `#[ssz(enum_behaviour = "union")]`
/// This should _only_ be used *some* cases when storing these objects in the database
/// and _NEVER_ for encoding / decoding blocks sent over the network!
pub mod ssz_tagged_signed_beacon_block {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> usize {
block
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max")
}
pub fn ssz_append<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
buf: &mut Vec<u8>,
) {
let fork_name = block.fork_name_unchecked();
fork_name.ssz_append(buf);
block.ssz_append(buf);
}
pub fn as_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
block: &SignedBeaconBlock<E, Payload>,
) -> Vec<u8> {
let mut buf = vec![];
ssz_append(block, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes<E: EthSpec, Payload: AbstractExecPayload<E>>(
bytes: &[u8],
) -> Result<SignedBeaconBlock<E, Payload>, DecodeError> {
let fork_byte = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
match ForkName::from_ssz_bytes(&[fork_byte])? {
ForkName::Base => Ok(SignedBeaconBlock::Base(
SignedBeaconBlockBase::from_ssz_bytes(body)?,
)),
ForkName::Altair => Ok(SignedBeaconBlock::Altair(
SignedBeaconBlockAltair::from_ssz_bytes(body)?,
)),
ForkName::Merge => Ok(SignedBeaconBlock::Merge(
SignedBeaconBlockMerge::from_ssz_bytes(body)?,
)),
ForkName::Capella => Ok(SignedBeaconBlock::Capella(
SignedBeaconBlockCapella::from_ssz_bytes(body)?,
)),
ForkName::Deneb => Ok(SignedBeaconBlock::Deneb(
SignedBeaconBlockDeneb::from_ssz_bytes(body)?,
)),
}
}
}
}
#[cfg(test)]
mod test {
use super::*;
@@ -551,4 +650,38 @@ mod test {
assert_eq!(reconstructed, block);
}
}
#[test]
fn test_ssz_tagged_signed_beacon_block() {
type E = MainnetEthSpec;
let spec = &E::default_spec();
let sig = Signature::empty();
let blocks = vec![
SignedBeaconBlock::<E>::from_block(
BeaconBlock::Base(BeaconBlockBase::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Altair(BeaconBlockAltair::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Merge(BeaconBlockMerge::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(
BeaconBlock::Capella(BeaconBlockCapella::empty(spec)),
sig.clone(),
),
SignedBeaconBlock::from_block(BeaconBlock::Deneb(BeaconBlockDeneb::empty(spec)), sig),
];
for block in blocks {
let encoded = ssz_tagged_signed_beacon_block::encode::as_ssz_bytes(&block);
let decoded = ssz_tagged_signed_beacon_block::decode::from_ssz_bytes::<E, _>(&encoded)
.expect("should decode");
assert_eq!(decoded, block);
}
}
}