gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-18 21:38:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'unstable' into deneb-free-blobs

Browse Source 
# Conflicts:
#	.github/workflows/docker.yml
#	.github/workflows/local-testnet.yml
#	.github/workflows/test-suite.yml
#	Cargo.lock
#	Cargo.toml
#	beacon_node/beacon_chain/src/beacon_chain.rs
#	beacon_node/beacon_chain/src/builder.rs
#	beacon_node/beacon_chain/src/test_utils.rs
#	beacon_node/execution_layer/src/engine_api/json_structures.rs
#	beacon_node/network/src/beacon_processor/mod.rs
#	beacon_node/network/src/beacon_processor/worker/gossip_methods.rs
#	beacon_node/network/src/sync/backfill_sync/mod.rs
#	beacon_node/store/src/config.rs
#	beacon_node/store/src/hot_cold_store.rs
#	common/eth2_network_config/Cargo.toml
#	consensus/ssz/src/decode/impls.rs
#	consensus/ssz_derive/src/lib.rs
#	consensus/ssz_derive/tests/tests.rs
#	consensus/ssz_types/src/serde_utils/mod.rs
#	consensus/tree_hash/src/impls.rs
#	consensus/tree_hash/src/lib.rs
#	consensus/types/Cargo.toml
#	consensus/types/src/beacon_state.rs
#	consensus/types/src/chain_spec.rs
#	consensus/types/src/eth_spec.rs
#	consensus/types/src/fork_name.rs
#	lcli/Cargo.toml
#	lcli/src/main.rs
#	lcli/src/new_testnet.rs
#	scripts/local_testnet/el_bootnode.sh
#	scripts/local_testnet/genesis.json
#	scripts/local_testnet/geth.sh
#	scripts/local_testnet/setup.sh
#	scripts/local_testnet/start_local_testnet.sh
#	scripts/local_testnet/vars.env
#	scripts/tests/doppelganger_protection.sh
#	scripts/tests/genesis.json
#	scripts/tests/vars.env
#	testing/ef_tests/Cargo.toml
#	validator_client/src/block_service.rs
This commit is contained in:
Jimmy Chen
2023-05-30 11:26:33 +10:00
parent 9b55d74c1c c547a11b0d
commit 70c4ae35ab
333 changed files with 5930 additions and 13386 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
beacon_node/store/src/config.rs
View File

@@ -7,6 +7,7 @@ use types::{EthSpec, MinimalEthSpec};
pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048;
pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192;
pub const DEFAULT_BLOCK_CACHE_SIZE: usize = 5;
pub const DEFAULT_HISTORIC_STATE_CACHE_SIZE: usize = 1;
pub const DEFAULT_BLOB_CACHE_SIZE: usize = 5;
pub const DEFAULT_EPOCHS_PER_BLOB_PRUNE: u64 = 1;
pub const DEFAULT_BLOB_PUNE_MARGIN_EPOCHS: u64 = 0;
@@ -20,6 +21,8 @@ pub struct StoreConfig {
pub slots_per_restore_point_set_explicitly: bool,
/// Maximum number of blocks to store in the in-memory block cache.
pub block_cache_size: usize,
/// Maximum number of states from freezer database to store in the in-memory state cache.
pub historic_state_cache_size: usize,
/// Maximum number of blobs to store in the in-memory blob cache.
pub blob_cache_size: usize,
/// Whether to compact the database on initialization.
@@ -55,6 +58,7 @@ impl Default for StoreConfig {
slots_per_restore_point: MinimalEthSpec::slots_per_historical_root() as u64,
slots_per_restore_point_set_explicitly: false,
block_cache_size: DEFAULT_BLOCK_CACHE_SIZE,
historic_state_cache_size: DEFAULT_HISTORIC_STATE_CACHE_SIZE,
blob_cache_size: DEFAULT_BLOB_CACHE_SIZE,
compact_on_init: false,
compact_on_prune: true,

75
beacon_node/store/src/hot_cold_store.rs
View File

@@ -30,7 +30,7 @@ use slog::{debug, error, info, trace, warn, Logger};
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use state_processing::{
BlockProcessingError, BlockReplayer, SlotProcessingError, StateRootStrategy,
BlockProcessingError, BlockReplayer, SlotProcessingError, StateProcessingStrategy,
};
use std::cmp::min;
use std::convert::TryInto;
@@ -70,6 +70,8 @@ pub struct HotColdDB<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
blob_cache: Mutex<LruCache<Hash256, BlobSidecarList<E>>>,
/// LRU cache of deserialized blocks. Updated whenever a block is loaded.
block_cache: Mutex<LruCache<Hash256, SignedBeaconBlock<E>>>,
/// LRU cache of replayed states.
state_cache: Mutex<LruCache<Slot, BeaconState<E>>>,
/// Chain spec.
pub(crate) spec: ChainSpec,
/// Logger.
@@ -143,6 +145,7 @@ impl<E: EthSpec> HotColdDB<E, MemoryStore<E>, MemoryStore<E>> {
blobs_db: Some(MemoryStore::open()),
hot_db: MemoryStore::open(),
block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)),
blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)),
config,
spec,
@@ -180,6 +183,7 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
blobs_db: None,
hot_db: LevelDB::open(hot_path)?,
block_cache: Mutex::new(LruCache::new(config.block_cache_size)),
state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)),
blob_cache: Mutex::new(LruCache::new(config.blob_cache_size)),
config,
spec,
@@ -632,10 +636,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
// chain. This way we avoid returning a state that doesn't match `state_root`.
self.load_cold_state(state_root)
} else {
self.load_hot_state(state_root, StateRootStrategy::Accurate)
self.load_hot_state(state_root, StateProcessingStrategy::Accurate)
}
} else {
match self.load_hot_state(state_root, StateRootStrategy::Accurate)? {
match self.load_hot_state(state_root, StateProcessingStrategy::Accurate)? {
Some(state) => Ok(Some(state)),
None => self.load_cold_state(state_root),
}
@@ -673,7 +677,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
}
.into())
} else {
self.load_hot_state(state_root, StateRootStrategy::Inconsistent)
self.load_hot_state(state_root, StateProcessingStrategy::Inconsistent)
}
}
@@ -763,10 +767,13 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
{
// NOTE: minor inefficiency here because we load an unnecessary hot state summary
//
// `StateRootStrategy` should be irrelevant here since we never replay blocks for an epoch
// `StateProcessingStrategy` should be irrelevant here since we never replay blocks for an epoch
// boundary state in the hot DB.
let state = self
.load_hot_state(&epoch_boundary_state_root, StateRootStrategy::Accurate)?
.load_hot_state(
&epoch_boundary_state_root,
StateProcessingStrategy::Accurate,
)?
.ok_or(HotColdDBError::MissingEpochBoundaryState(
epoch_boundary_state_root,
))?;
@@ -1026,7 +1033,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
pub fn load_hot_state(
&self,
state_root: &Hash256,
state_root_strategy: StateRootStrategy,
state_processing_strategy: StateProcessingStrategy,
) -> Result<Option<BeaconState<E>>, Error> {
metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT);
@@ -1059,7 +1066,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
blocks,
slot,
no_state_root_iter(),
state_root_strategy,
state_processing_strategy,
)?
};
@@ -1173,40 +1180,70 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// Load a frozen state that lies between restore points.
fn load_cold_intermediate_state(&self, slot: Slot) -> Result<BeaconState<E>, Error> {
if let Some(state) = self.state_cache.lock().get(&slot) {
return Ok(state.clone());
}
// 1. Load the restore points either side of the intermediate state.
let low_restore_point_idx = slot.as_u64() / self.config.slots_per_restore_point;
let high_restore_point_idx = low_restore_point_idx + 1;
// Use low restore point as the base state.
let mut low_slot: Slot =
Slot::new(low_restore_point_idx * self.config.slots_per_restore_point);
let mut low_state: Option<BeaconState<E>> = None;
// Try to get a more recent state from the cache to avoid massive blocks replay.
for (s, state) in self.state_cache.lock().iter() {
if s.as_u64() / self.config.slots_per_restore_point == low_restore_point_idx
&& *s < slot
&& low_slot < *s
{
low_slot = *s;
low_state = Some(state.clone());
}
}
// If low_state is still None, use load_restore_point_by_index to load the state.
let low_state = match low_state {
Some(state) => state,
None => self.load_restore_point_by_index(low_restore_point_idx)?,
};
// Acquire the read lock, so that the split can't change while this is happening.
let split = self.split.read_recursive();
let low_restore_point = self.load_restore_point_by_index(low_restore_point_idx)?;
let high_restore_point = self.get_restore_point(high_restore_point_idx, &split)?;
// 2. Load the blocks from the high restore point back to the low restore point.
// 2. Load the blocks from the high restore point back to the low point.
let blocks = self.load_blocks_to_replay(
low_restore_point.slot(),
low_slot,
slot,
self.get_high_restore_point_block_root(&high_restore_point, slot)?,
)?;
// 3. Replay the blocks on top of the low restore point.
// 3. Replay the blocks on top of the low point.
// Use a forwards state root iterator to avoid doing any tree hashing.
// The state root of the high restore point should never be used, so is safely set to 0.
let state_root_iter = self.forwards_state_roots_iterator_until(
low_restore_point.slot(),
low_slot,
slot,
|| Ok((high_restore_point, Hash256::zero())),
&self.spec,
)?;
self.replay_blocks(
low_restore_point,
let state = self.replay_blocks(
low_state,
blocks,
slot,
Some(state_root_iter),
StateRootStrategy::Accurate,
)
StateProcessingStrategy::Accurate,
)?;
// If state is not error, put it in the cache.
self.state_cache.lock().put(slot, state.clone());
Ok(state)
}
/// Get the restore point with the given index, or if it is out of bounds, the split state.
@@ -1292,10 +1329,10 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
blocks: Vec<SignedBeaconBlock<E, BlindedPayload<E>>>,
target_slot: Slot,
state_root_iter: Option<impl Iterator<Item = Result<(Hash256, Slot), Error>>>,
state_root_strategy: StateRootStrategy,
state_processing_strategy: StateProcessingStrategy,
) -> Result<BeaconState<E>, Error> {
let mut block_replayer = BlockReplayer::new(state, &self.spec)
.state_root_strategy(state_root_strategy)
.state_processing_strategy(state_processing_strategy)
.no_signature_verification()
.minimal_block_root_verification();

8
beacon_node/store/src/metadata.rs
View File

@@ -4,7 +4,7 @@ use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use types::{Checkpoint, Hash256, Slot};
pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(16);
pub const CURRENT_SCHEMA_VERSION: SchemaVersion = SchemaVersion(17);
// All the keys that get stored under the `BeaconMeta` column.
//
@@ -100,8 +100,10 @@ pub struct AnchorInfo {
impl AnchorInfo {
/// Returns true if the block backfill has completed.
pub fn block_backfill_complete(&self) -> bool {
self.oldest_block_slot == 0
/// This is a comparison between the oldest block slot and the target backfill slot (which is
/// likely to be the closest WSP).
pub fn block_backfill_complete(&self, target_slot: Slot) -> bool {
self.oldest_block_slot <= target_slot
}
}

3
beacon_node/store/src/reconstruct.rs
View File

@@ -5,7 +5,7 @@ use itertools::{process_results, Itertools};
use slog::info;
use state_processing::{
per_block_processing, per_slot_processing, BlockSignatureStrategy, ConsensusContext,
VerifyBlockRoot,
StateProcessingStrategy, VerifyBlockRoot,
};
use std::sync::Arc;
use types::{EthSpec, Hash256};
@@ -96,6 +96,7 @@ where
&mut state,
&block,
BlockSignatureStrategy::NoVerification,
StateProcessingStrategy::Accurate,
VerifyBlockRoot::True,
&mut ctxt,
&self.spec,