mirror of
https://github.com/sigp/lighthouse.git
synced 2026-04-16 12:28:24 +00:00
Fix block backfill with genesis skip slots (#4820)
## Issue Addressed Closes #4817. ## Proposed Changes - Fill in the linear block roots array between 0 and the slot of the first block (e.g. slots 0 and 1 on Holesky). - Backport the `--freezer`, `--skip` and `--limit` options for `lighthouse db inspect` from tree-states. This allows us to easily view the database corruption of 4817 using `lighthouse db inspect --network holesky --freezer --column bbr --output values --limit 2`. - Backport the `iter_column_from` change and `MemoryStore` overhaul from tree-states. These are required to enable `lighthouse db inspect`. - Rework `freezer_upper_limit` to allow state lookups for slots below the `state_lower_limit`. Currently state lookups will fail until state reconstruction completes entirely. There is a new regression test for the main bug, but no test for the `freezer_upper_limit` fix because we don't currently support running state reconstruction partially (see #3026). This will be fixed once we merge `tree-states`! In lieu of an automated test, I've tested manually on a Holesky node while it was reconstructing. ## Additional Info Users who backfilled Holesky to slot 0 (e.g. using `--reconstruct-historic-states`) need to either: - Re-sync from genesis. - Re-sync using checkpoint sync and the changes from this PR. Due to the recency of the Holesky genesis, writing a custom pass to fix up broken databases (which would require its own thorough testing) was deemed unnecessary. This is the primary reason for this PR being marked `backwards-incompat`. This will create few conflicts with Deneb, which I've already resolved on `tree-states-deneb` and will be happy to backport to Deneb once this PR is merged to unstable.
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
use super::*;
|
||||
use crate::hot_cold_store::HotColdDBError;
|
||||
use crate::metrics;
|
||||
use db_key::Key;
|
||||
use leveldb::compaction::Compaction;
|
||||
use leveldb::database::batch::{Batch, Writebatch};
|
||||
use leveldb::database::kv::KV;
|
||||
@@ -176,10 +175,8 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Iterate through all keys and values in a particular column.
|
||||
fn iter_column(&self, column: DBColumn) -> ColumnIter {
|
||||
let start_key =
|
||||
BytesKey::from_vec(get_key_for_col(column.into(), Hash256::zero().as_bytes()));
|
||||
fn iter_column_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnIter<K> {
|
||||
let start_key = BytesKey::from_vec(get_key_for_col(column.into(), from));
|
||||
|
||||
let iter = self.db.iter(self.read_options());
|
||||
iter.seek(&start_key);
|
||||
@@ -187,13 +184,12 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
|
||||
Box::new(
|
||||
iter.take_while(move |(key, _)| key.matches_column(column))
|
||||
.map(move |(bytes_key, value)| {
|
||||
let key =
|
||||
bytes_key
|
||||
.remove_column(column)
|
||||
.ok_or(HotColdDBError::IterationError {
|
||||
unexpected_key: bytes_key,
|
||||
})?;
|
||||
Ok((key, value))
|
||||
let key = bytes_key.remove_column_variable(column).ok_or_else(|| {
|
||||
HotColdDBError::IterationError {
|
||||
unexpected_key: bytes_key.clone(),
|
||||
}
|
||||
})?;
|
||||
Ok((K::from_bytes(key)?, value))
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -254,12 +250,12 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
|
||||
impl<E: EthSpec> ItemStore<E> for LevelDB<E> {}
|
||||
|
||||
/// Used for keying leveldb.
|
||||
#[derive(Debug, PartialEq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct BytesKey {
|
||||
key: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Key for BytesKey {
|
||||
impl db_key::Key for BytesKey {
|
||||
fn from_u8(key: &[u8]) -> Self {
|
||||
Self { key: key.to_vec() }
|
||||
}
|
||||
@@ -275,12 +271,20 @@ impl BytesKey {
|
||||
self.key.starts_with(column.as_bytes())
|
||||
}
|
||||
|
||||
/// Remove the column from a key, returning its `Hash256` portion.
|
||||
/// Remove the column from a 32 byte key, yielding the `Hash256` key.
|
||||
pub fn remove_column(&self, column: DBColumn) -> Option<Hash256> {
|
||||
let key = self.remove_column_variable(column)?;
|
||||
(column.key_size() == 32).then(|| Hash256::from_slice(key))
|
||||
}
|
||||
|
||||
/// Remove the column from a key.
|
||||
///
|
||||
/// Will return `None` if the value doesn't match the column or has the wrong length.
|
||||
pub fn remove_column_variable(&self, column: DBColumn) -> Option<&[u8]> {
|
||||
if self.matches_column(column) {
|
||||
let subkey = &self.key[column.as_bytes().len()..];
|
||||
if subkey.len() == 32 {
|
||||
return Some(Hash256::from_slice(subkey));
|
||||
if subkey.len() == column.key_size() {
|
||||
return Some(subkey);
|
||||
}
|
||||
}
|
||||
None
|
||||
|
||||
Reference in New Issue
Block a user