mirror of
https://github.com/sigp/lighthouse.git
synced 2026-04-26 01:03:40 +00:00
Hierarchical state diffs (#5978)
* Start extracting freezer changes for tree-states * Remove unused config args * Add comments * Remove unwraps * Subjective more clear implementation * Clean up hdiff * Update xdelta3 * Tree states archive metrics (#6040) * Add store cache size metrics * Add compress timer metrics * Add diff apply compute timer metrics * Add diff buffer cache hit metrics * Add hdiff buffer load times * Add blocks replayed metric * Move metrics to store * Future proof some metrics --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Port and clean up forwards iterator changes * Add and polish hierarchy-config flag * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Cleaner errors * Fix beacon_chain test compilation * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Patch a few more freezer block roots * Fix genesis block root bug * Fix test failing due to pending updates * Beacon chain tests passing * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix doc lint * Implement DB schema upgrade for hierarchical state diffs (#6193) * DB upgrade * Add flag * Delete RestorePointHash * Update docs * Update docs * Implement hierarchical state diffs config migration (#6245) * Implement hierarchical state diffs config migration * Review PR * Remove TODO * Set CURRENT_SCHEMA_VERSION correctly * Fix genesis state loading * Re-delete some PartialBeaconState stuff --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix test compilation * Update schema downgrade test * Fix tests * Fix null anchor migration * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix tree states upgrade migration (#6328) * Towards crash safety * Fix compilation * Move cold summaries and state roots to new columns * Rename StateRoots chunked field * Update prune states * Clean hdiff CLI flag and metrics * Fix "staged reconstruction" * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix alloy issues * Fix staged reconstruction logic * Prevent weird slot drift * Remove "allow" flag * Update CLI help * Remove FIXME about downgrade * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Remove some unnecessary error variants * Fix new test * Tree states archive - review comments and metrics (#6386) * Review PR comments and metrics * Comments * Add anchor metrics * drop prev comment * Update metadata.rs * Apply suggestions from code review --------- Co-authored-by: Michael Sproul <micsproul@gmail.com> * Update beacon_node/store/src/hot_cold_store.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Clarify comment and remove anchor_slot garbage * Simplify database anchor (#6397) * Simplify database anchor * Update beacon_node/store/src/reconstruct.rs * Add migration for anchor * Fix and simplify light_client store tests * Fix incompatible config test * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * More metrics * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * New historic state cache (#6475) * New historic state cache * Add more metrics * State cache hit rate metrics * Fix store metrics * More logs and metrics * Fix logger * Ensure cached states have built caches :O * Replay blocks in preference to diffing * Two separate caches * Distribute cache build time to next slot * Re-plumb historic-state-cache flag * Clean up metrics * Update book * Update beacon_node/store/src/hdiff.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update beacon_node/store/src/historic_state_cache.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update database docs * Update diagram * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Update lockbud to work with bindgen/etc * Correct pkg name for Debian * Remove vestigial epochs_per_state_diff * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Markdown lint * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Address Jimmy's review comments * Simplify ReplayFrom case * Fix and document genesis_state_root * Typo Co-authored-by: Jimmy Chen <jchen.tc@gmail.com> * Merge branch 'unstable' into tree-states-archive * Compute diff of validators list manually (#6556) * Split hdiff computation * Dedicated logic for historical roots and summaries * Benchmark against real states * Mutated source? * Version the hdiff * Add lighthouse DB config for hierarchy exponents * Tidy up hierarchy exponents flag * Apply suggestions from code review Co-authored-by: Michael Sproul <micsproul@gmail.com> * Address PR review * Remove hardcoded paths in benchmarks * Delete unused function in benches * lint --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Test hdiff binary format stability (#6585) * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Add deprecation warning for SPRP * Update xdelta to get rid of duplicate deps * Document test
This commit is contained in:
Michael Sproul
@@ -7,7 +7,6 @@
|
||||
//!
|
||||
//! Provides a simple API for storing/retrieving all types that sometimes needs type-hints. See
|
||||
//! tests for implementation examples.
|
||||
mod chunk_writer;
|
||||
pub mod chunked_iter;
|
||||
pub mod chunked_vector;
|
||||
pub mod config;
|
||||
@@ -15,25 +14,25 @@ pub mod consensus_context;
|
||||
pub mod errors;
|
||||
mod forwards_iter;
|
||||
mod garbage_collection;
|
||||
pub mod hdiff;
|
||||
pub mod historic_state_cache;
|
||||
pub mod hot_cold_store;
|
||||
mod impls;
|
||||
mod leveldb_store;
|
||||
mod memory_store;
|
||||
pub mod metadata;
|
||||
pub mod metrics;
|
||||
mod partial_beacon_state;
|
||||
pub mod partial_beacon_state;
|
||||
pub mod reconstruct;
|
||||
pub mod state_cache;
|
||||
|
||||
pub mod iter;
|
||||
|
||||
pub use self::chunk_writer::ChunkWriter;
|
||||
pub use self::config::StoreConfig;
|
||||
pub use self::consensus_context::OnDiskConsensusContext;
|
||||
pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split};
|
||||
pub use self::leveldb_store::LevelDB;
|
||||
pub use self::memory_store::MemoryStore;
|
||||
pub use self::partial_beacon_state::PartialBeaconState;
|
||||
pub use crate::metadata::BlobInfo;
|
||||
pub use errors::Error;
|
||||
pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer;
|
||||
@@ -251,6 +250,11 @@ pub enum DBColumn {
|
||||
/// For data related to the database itself.
|
||||
#[strum(serialize = "bma")]
|
||||
BeaconMeta,
|
||||
/// Data related to blocks.
|
||||
///
|
||||
/// - Key: `Hash256` block root.
|
||||
/// - Value in hot DB: SSZ-encoded blinded block.
|
||||
/// - Value in cold DB: 8-byte slot of block.
|
||||
#[strum(serialize = "blk")]
|
||||
BeaconBlock,
|
||||
#[strum(serialize = "blb")]
|
||||
@@ -260,9 +264,21 @@ pub enum DBColumn {
|
||||
/// For full `BeaconState`s in the hot database (finalized or fork-boundary states).
|
||||
#[strum(serialize = "ste")]
|
||||
BeaconState,
|
||||
/// For the mapping from state roots to their slots or summaries.
|
||||
/// For beacon state snapshots in the freezer DB.
|
||||
#[strum(serialize = "bsn")]
|
||||
BeaconStateSnapshot,
|
||||
/// For compact `BeaconStateDiff`s in the freezer DB.
|
||||
#[strum(serialize = "bsd")]
|
||||
BeaconStateDiff,
|
||||
/// Mapping from state root to `HotStateSummary` in the hot DB.
|
||||
///
|
||||
/// Previously this column also served a role in the freezer DB, mapping state roots to
|
||||
/// `ColdStateSummary`. However that role is now filled by `BeaconColdStateSummary`.
|
||||
#[strum(serialize = "bss")]
|
||||
BeaconStateSummary,
|
||||
/// Mapping from state root to `ColdStateSummary` in the cold DB.
|
||||
#[strum(serialize = "bcs")]
|
||||
BeaconColdStateSummary,
|
||||
/// For the list of temporary states stored during block import,
|
||||
/// and then made non-temporary by the deletion of their state root from this column.
|
||||
#[strum(serialize = "bst")]
|
||||
@@ -281,15 +297,37 @@ pub enum DBColumn {
|
||||
ForkChoice,
|
||||
#[strum(serialize = "pkc")]
|
||||
PubkeyCache,
|
||||
/// For the table mapping restore point numbers to state roots.
|
||||
/// For the legacy table mapping restore point numbers to state roots.
|
||||
///
|
||||
/// DEPRECATED. Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "brp")]
|
||||
BeaconRestorePoint,
|
||||
#[strum(serialize = "bbr")]
|
||||
BeaconBlockRoots,
|
||||
#[strum(serialize = "bsr")]
|
||||
/// Mapping from slot to beacon state root in the freezer DB.
|
||||
///
|
||||
/// This new column was created to replace the previous `bsr` column. The replacement was
|
||||
/// necessary to guarantee atomicity of the upgrade migration.
|
||||
#[strum(serialize = "bsx")]
|
||||
BeaconStateRoots,
|
||||
/// DEPRECATED. This is the previous column for beacon state roots stored by "chunk index".
|
||||
///
|
||||
/// Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "bsr")]
|
||||
BeaconStateRootsChunked,
|
||||
/// Mapping from slot to beacon block root in the freezer DB.
|
||||
///
|
||||
/// This new column was created to replace the previous `bbr` column. The replacement was
|
||||
/// necessary to guarantee atomicity of the upgrade migration.
|
||||
#[strum(serialize = "bbx")]
|
||||
BeaconBlockRoots,
|
||||
/// DEPRECATED. This is the previous column for beacon block roots stored by "chunk index".
|
||||
///
|
||||
/// Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "bbr")]
|
||||
BeaconBlockRootsChunked,
|
||||
/// DEPRECATED. Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "bhr")]
|
||||
BeaconHistoricalRoots,
|
||||
/// DEPRECATED. Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "brm")]
|
||||
BeaconRandaoMixes,
|
||||
#[strum(serialize = "dht")]
|
||||
@@ -297,6 +335,7 @@ pub enum DBColumn {
|
||||
/// For Optimistically Imported Merge Transition Blocks
|
||||
#[strum(serialize = "otb")]
|
||||
OptimisticTransitionBlock,
|
||||
/// DEPRECATED. Can be removed once schema v22 is buried by a hard fork.
|
||||
#[strum(serialize = "bhs")]
|
||||
BeaconHistoricalSummaries,
|
||||
#[strum(serialize = "olc")]
|
||||
@@ -338,6 +377,7 @@ impl DBColumn {
|
||||
| Self::BeaconState
|
||||
| Self::BeaconBlob
|
||||
| Self::BeaconStateSummary
|
||||
| Self::BeaconColdStateSummary
|
||||
| Self::BeaconStateTemporary
|
||||
| Self::ExecPayload
|
||||
| Self::BeaconChain
|
||||
@@ -349,10 +389,14 @@ impl DBColumn {
|
||||
| Self::DhtEnrs
|
||||
| Self::OptimisticTransitionBlock => 32,
|
||||
Self::BeaconBlockRoots
|
||||
| Self::BeaconBlockRootsChunked
|
||||
| Self::BeaconStateRoots
|
||||
| Self::BeaconStateRootsChunked
|
||||
| Self::BeaconHistoricalRoots
|
||||
| Self::BeaconHistoricalSummaries
|
||||
| Self::BeaconRandaoMixes
|
||||
| Self::BeaconStateSnapshot
|
||||
| Self::BeaconStateDiff
|
||||
| Self::SyncCommittee
|
||||
| Self::SyncCommitteeBranch
|
||||
| Self::LightClientUpdate => 8,
|
||||
|
||||
Reference in New Issue
Block a user