mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-10 12:11:59 +00:00
Hierarchical state diffs (#5978)
* Start extracting freezer changes for tree-states * Remove unused config args * Add comments * Remove unwraps * Subjective more clear implementation * Clean up hdiff * Update xdelta3 * Tree states archive metrics (#6040) * Add store cache size metrics * Add compress timer metrics * Add diff apply compute timer metrics * Add diff buffer cache hit metrics * Add hdiff buffer load times * Add blocks replayed metric * Move metrics to store * Future proof some metrics --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Port and clean up forwards iterator changes * Add and polish hierarchy-config flag * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Cleaner errors * Fix beacon_chain test compilation * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Patch a few more freezer block roots * Fix genesis block root bug * Fix test failing due to pending updates * Beacon chain tests passing * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix doc lint * Implement DB schema upgrade for hierarchical state diffs (#6193) * DB upgrade * Add flag * Delete RestorePointHash * Update docs * Update docs * Implement hierarchical state diffs config migration (#6245) * Implement hierarchical state diffs config migration * Review PR * Remove TODO * Set CURRENT_SCHEMA_VERSION correctly * Fix genesis state loading * Re-delete some PartialBeaconState stuff --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix test compilation * Update schema downgrade test * Fix tests * Fix null anchor migration * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix tree states upgrade migration (#6328) * Towards crash safety * Fix compilation * Move cold summaries and state roots to new columns * Rename StateRoots chunked field * Update prune states * Clean hdiff CLI flag and metrics * Fix "staged reconstruction" * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix alloy issues * Fix staged reconstruction logic * Prevent weird slot drift * Remove "allow" flag * Update CLI help * Remove FIXME about downgrade * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Remove some unnecessary error variants * Fix new test * Tree states archive - review comments and metrics (#6386) * Review PR comments and metrics * Comments * Add anchor metrics * drop prev comment * Update metadata.rs * Apply suggestions from code review --------- Co-authored-by: Michael Sproul <micsproul@gmail.com> * Update beacon_node/store/src/hot_cold_store.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Clarify comment and remove anchor_slot garbage * Simplify database anchor (#6397) * Simplify database anchor * Update beacon_node/store/src/reconstruct.rs * Add migration for anchor * Fix and simplify light_client store tests * Fix incompatible config test * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * More metrics * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * New historic state cache (#6475) * New historic state cache * Add more metrics * State cache hit rate metrics * Fix store metrics * More logs and metrics * Fix logger * Ensure cached states have built caches :O * Replay blocks in preference to diffing * Two separate caches * Distribute cache build time to next slot * Re-plumb historic-state-cache flag * Clean up metrics * Update book * Update beacon_node/store/src/hdiff.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update beacon_node/store/src/historic_state_cache.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update database docs * Update diagram * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Update lockbud to work with bindgen/etc * Correct pkg name for Debian * Remove vestigial epochs_per_state_diff * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Markdown lint * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Address Jimmy's review comments * Simplify ReplayFrom case * Fix and document genesis_state_root * Typo Co-authored-by: Jimmy Chen <jchen.tc@gmail.com> * Merge branch 'unstable' into tree-states-archive * Compute diff of validators list manually (#6556) * Split hdiff computation * Dedicated logic for historical roots and summaries * Benchmark against real states * Mutated source? * Version the hdiff * Add lighthouse DB config for hierarchy exponents * Tidy up hierarchy exponents flag * Apply suggestions from code review Co-authored-by: Michael Sproul <micsproul@gmail.com> * Address PR review * Remove hardcoded paths in benchmarks * Delete unused function in benches * lint --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Test hdiff binary format stability (#6585) * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Add deprecation warning for SPRP * Update xdelta to get rid of duplicate deps * Document test
This commit is contained in:
Michael Sproul
@@ -1060,21 +1060,21 @@ where
|
||||
self.db_path = Some(hot_path.into());
|
||||
self.freezer_db_path = Some(cold_path.into());
|
||||
|
||||
let inner_spec = spec.clone();
|
||||
let deposit_contract_deploy_block = context
|
||||
// Optionally grab the genesis state root.
|
||||
// This will only be required if a DB upgrade to V22 is needed.
|
||||
let genesis_state_root = context
|
||||
.eth2_network_config
|
||||
.as_ref()
|
||||
.map(|config| config.deposit_contract_deploy_block)
|
||||
.unwrap_or(0);
|
||||
.and_then(|config| config.genesis_state_root::<E>().transpose())
|
||||
.transpose()?;
|
||||
|
||||
let schema_upgrade = |db, from, to| {
|
||||
migrate_schema::<Witness<TSlotClock, TEth1Backend, _, _, _>>(
|
||||
db,
|
||||
deposit_contract_deploy_block,
|
||||
genesis_state_root,
|
||||
from,
|
||||
to,
|
||||
log,
|
||||
&inner_spec,
|
||||
)
|
||||
};
|
||||
|
||||
|
||||
@@ -45,10 +45,7 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
|
||||
let mut current_sync_state = network.sync_state();
|
||||
|
||||
// Store info if we are required to do a backfill sync.
|
||||
let original_anchor_slot = beacon_chain
|
||||
.store
|
||||
.get_anchor_info()
|
||||
.map(|ai| ai.oldest_block_slot);
|
||||
let original_oldest_block_slot = beacon_chain.store.get_anchor_info().oldest_block_slot;
|
||||
|
||||
let interval_future = async move {
|
||||
// Perform pre-genesis logging.
|
||||
@@ -141,22 +138,17 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
|
||||
match current_sync_state {
|
||||
SyncState::BackFillSyncing { .. } => {
|
||||
// Observe backfilling sync info.
|
||||
if let Some(oldest_slot) = original_anchor_slot {
|
||||
if let Some(current_anchor_slot) = beacon_chain
|
||||
.store
|
||||
.get_anchor_info()
|
||||
.map(|ai| ai.oldest_block_slot)
|
||||
{
|
||||
sync_distance = current_anchor_slot
|
||||
.saturating_sub(beacon_chain.genesis_backfill_slot);
|
||||
speedo
|
||||
// For backfill sync use a fake slot which is the distance we've progressed from the starting `oldest_block_slot`.
|
||||
.observe(
|
||||
oldest_slot.saturating_sub(current_anchor_slot),
|
||||
Instant::now(),
|
||||
);
|
||||
}
|
||||
}
|
||||
let current_oldest_block_slot =
|
||||
beacon_chain.store.get_anchor_info().oldest_block_slot;
|
||||
sync_distance = current_oldest_block_slot
|
||||
.saturating_sub(beacon_chain.genesis_backfill_slot);
|
||||
speedo
|
||||
// For backfill sync use a fake slot which is the distance we've progressed
|
||||
// from the starting `original_oldest_block_slot`.
|
||||
.observe(
|
||||
original_oldest_block_slot.saturating_sub(current_oldest_block_slot),
|
||||
Instant::now(),
|
||||
);
|
||||
}
|
||||
SyncState::SyncingFinalized { .. }
|
||||
| SyncState::SyncingHead { .. }
|
||||
@@ -213,14 +205,14 @@ pub fn spawn_notifier<T: BeaconChainTypes>(
|
||||
"Downloading historical blocks";
|
||||
"distance" => distance,
|
||||
"speed" => sync_speed_pretty(speed),
|
||||
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot).saturating_sub(beacon_chain.genesis_backfill_slot))),
|
||||
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_oldest_block_slot.saturating_sub(beacon_chain.genesis_backfill_slot))),
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
log,
|
||||
"Downloading historical blocks";
|
||||
"distance" => distance,
|
||||
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_anchor_slot.unwrap_or(current_slot).saturating_sub(beacon_chain.genesis_backfill_slot))),
|
||||
"est_time" => estimated_time_pretty(speedo.estimated_time_till_slot(original_oldest_block_slot.saturating_sub(beacon_chain.genesis_backfill_slot))),
|
||||
);
|
||||
}
|
||||
} else if !is_backfilling && last_backfill_log_slot.is_some() {
|
||||
|
||||
Reference in New Issue
Block a user