mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-10 04:01:51 +00:00
* Start extracting freezer changes for tree-states * Remove unused config args * Add comments * Remove unwraps * Subjective more clear implementation * Clean up hdiff * Update xdelta3 * Tree states archive metrics (#6040) * Add store cache size metrics * Add compress timer metrics * Add diff apply compute timer metrics * Add diff buffer cache hit metrics * Add hdiff buffer load times * Add blocks replayed metric * Move metrics to store * Future proof some metrics --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Port and clean up forwards iterator changes * Add and polish hierarchy-config flag * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Cleaner errors * Fix beacon_chain test compilation * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Patch a few more freezer block roots * Fix genesis block root bug * Fix test failing due to pending updates * Beacon chain tests passing * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix doc lint * Implement DB schema upgrade for hierarchical state diffs (#6193) * DB upgrade * Add flag * Delete RestorePointHash * Update docs * Update docs * Implement hierarchical state diffs config migration (#6245) * Implement hierarchical state diffs config migration * Review PR * Remove TODO * Set CURRENT_SCHEMA_VERSION correctly * Fix genesis state loading * Re-delete some PartialBeaconState stuff --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix test compilation * Update schema downgrade test * Fix tests * Fix null anchor migration * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix tree states upgrade migration (#6328) * Towards crash safety * Fix compilation * Move cold summaries and state roots to new columns * Rename StateRoots chunked field * Update prune states * Clean hdiff CLI flag and metrics * Fix "staged reconstruction" * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Fix alloy issues * Fix staged reconstruction logic * Prevent weird slot drift * Remove "allow" flag * Update CLI help * Remove FIXME about downgrade * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Remove some unnecessary error variants * Fix new test * Tree states archive - review comments and metrics (#6386) * Review PR comments and metrics * Comments * Add anchor metrics * drop prev comment * Update metadata.rs * Apply suggestions from code review --------- Co-authored-by: Michael Sproul <micsproul@gmail.com> * Update beacon_node/store/src/hot_cold_store.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Clarify comment and remove anchor_slot garbage * Simplify database anchor (#6397) * Simplify database anchor * Update beacon_node/store/src/reconstruct.rs * Add migration for anchor * Fix and simplify light_client store tests * Fix incompatible config test * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * More metrics * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * New historic state cache (#6475) * New historic state cache * Add more metrics * State cache hit rate metrics * Fix store metrics * More logs and metrics * Fix logger * Ensure cached states have built caches :O * Replay blocks in preference to diffing * Two separate caches * Distribute cache build time to next slot * Re-plumb historic-state-cache flag * Clean up metrics * Update book * Update beacon_node/store/src/hdiff.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update beacon_node/store/src/historic_state_cache.rs Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update database docs * Update diagram * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Update lockbud to work with bindgen/etc * Correct pkg name for Debian * Remove vestigial epochs_per_state_diff * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Markdown lint * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Address Jimmy's review comments * Simplify ReplayFrom case * Fix and document genesis_state_root * Typo Co-authored-by: Jimmy Chen <jchen.tc@gmail.com> * Merge branch 'unstable' into tree-states-archive * Compute diff of validators list manually (#6556) * Split hdiff computation * Dedicated logic for historical roots and summaries * Benchmark against real states * Mutated source? * Version the hdiff * Add lighthouse DB config for hierarchy exponents * Tidy up hierarchy exponents flag * Apply suggestions from code review Co-authored-by: Michael Sproul <micsproul@gmail.com> * Address PR review * Remove hardcoded paths in benchmarks * Delete unused function in benches * lint --------- Co-authored-by: Michael Sproul <michael@sigmaprime.io> * Test hdiff binary format stability (#6585) * Merge remote-tracking branch 'origin/unstable' into tree-states-archive * Add deprecation warning for SPRP * Update xdelta to get rid of duplicate deps * Document test
93 lines
3.0 KiB
Rust
93 lines
3.0 KiB
Rust
use crate::hdiff::{Error, HDiffBuffer};
|
|
use crate::metrics;
|
|
use lru::LruCache;
|
|
use std::num::NonZeroUsize;
|
|
use types::{BeaconState, ChainSpec, EthSpec, Slot};
|
|
|
|
/// Holds a combination of finalized states in two formats:
|
|
/// - `hdiff_buffers`: Format close to an SSZ serialized state for rapid application of diffs on top
|
|
/// of it
|
|
/// - `states`: Deserialized states for direct use or for rapid application of blocks (replay)
|
|
///
|
|
/// An example use: when requesting state data for consecutive slots, this cache allows the node to
|
|
/// apply diffs once on the first request, and latter just apply blocks one at a time.
|
|
#[derive(Debug)]
|
|
pub struct HistoricStateCache<E: EthSpec> {
|
|
hdiff_buffers: LruCache<Slot, HDiffBuffer>,
|
|
states: LruCache<Slot, BeaconState<E>>,
|
|
}
|
|
|
|
#[derive(Debug, Default)]
|
|
pub struct Metrics {
|
|
pub num_hdiff: usize,
|
|
pub num_state: usize,
|
|
pub hdiff_byte_size: usize,
|
|
}
|
|
|
|
impl<E: EthSpec> HistoricStateCache<E> {
|
|
pub fn new(hdiff_buffer_cache_size: NonZeroUsize, state_cache_size: NonZeroUsize) -> Self {
|
|
Self {
|
|
hdiff_buffers: LruCache::new(hdiff_buffer_cache_size),
|
|
states: LruCache::new(state_cache_size),
|
|
}
|
|
}
|
|
|
|
pub fn get_hdiff_buffer(&mut self, slot: Slot) -> Option<HDiffBuffer> {
|
|
if let Some(buffer_ref) = self.hdiff_buffers.get(&slot) {
|
|
let _timer = metrics::start_timer(&metrics::BEACON_HDIFF_BUFFER_CLONE_TIMES);
|
|
Some(buffer_ref.clone())
|
|
} else if let Some(state) = self.states.get(&slot) {
|
|
let buffer = HDiffBuffer::from_state(state.clone());
|
|
let _timer = metrics::start_timer(&metrics::BEACON_HDIFF_BUFFER_CLONE_TIMES);
|
|
let cloned = buffer.clone();
|
|
drop(_timer);
|
|
self.hdiff_buffers.put(slot, cloned);
|
|
Some(buffer)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
|
|
pub fn get_state(
|
|
&mut self,
|
|
slot: Slot,
|
|
spec: &ChainSpec,
|
|
) -> Result<Option<BeaconState<E>>, Error> {
|
|
if let Some(state) = self.states.get(&slot) {
|
|
Ok(Some(state.clone()))
|
|
} else if let Some(buffer) = self.hdiff_buffers.get(&slot) {
|
|
let state = buffer.as_state(spec)?;
|
|
self.states.put(slot, state.clone());
|
|
Ok(Some(state))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
}
|
|
|
|
pub fn put_state(&mut self, slot: Slot, state: BeaconState<E>) {
|
|
self.states.put(slot, state);
|
|
}
|
|
|
|
pub fn put_hdiff_buffer(&mut self, slot: Slot, buffer: HDiffBuffer) {
|
|
self.hdiff_buffers.put(slot, buffer);
|
|
}
|
|
|
|
pub fn put_both(&mut self, slot: Slot, state: BeaconState<E>, buffer: HDiffBuffer) {
|
|
self.put_state(slot, state);
|
|
self.put_hdiff_buffer(slot, buffer);
|
|
}
|
|
|
|
pub fn metrics(&self) -> Metrics {
|
|
let hdiff_byte_size = self
|
|
.hdiff_buffers
|
|
.iter()
|
|
.map(|(_, buffer)| buffer.size())
|
|
.sum::<usize>();
|
|
Metrics {
|
|
num_hdiff: self.hdiff_buffers.len(),
|
|
num_state: self.states.len(),
|
|
hdiff_byte_size,
|
|
}
|
|
}
|
|
}
|