gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-07-01 11:54:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/unstable' into import-blobs

Browse Source
This commit is contained in:
Michael Sproul
2025-04-23 16:02:13 +10:00
parent 95df7ea443 c13e069c9c
commit 1dd4f59610
821 changed files with 40869 additions and 52247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
database_manager/Cargo.toml
View File

@@ -10,9 +10,9 @@ clap = { workspace = true }
clap_utils = { workspace = true }
environment = { workspace = true }
hex = { workspace = true }
store = { workspace = true }
types = { workspace = true }
slog = { workspace = true }
strum = { workspace = true }
serde = { workspace = true }
ethereum_ssz = { workspace = true }
store = { workspace = true }
strum = { workspace = true }
tracing = { workspace = true }
types = { workspace = true }

21
database_manager/src/cli.rs
View File

@@ -3,6 +3,7 @@ use clap_utils::get_color_style;
use clap_utils::FLAG_HEADER;
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
use store::hdiff::HierarchyConfig;
use types::Slot;
use crate::InspectTarget;
@@ -22,13 +23,14 @@ use crate::InspectTarget;
pub struct DatabaseManager {
#[clap(
long,
value_name = "SLOT_COUNT",
help = "Specifies how often a freezer DB restore point should be stored. \
Cannot be changed after initialization. \
[default: 2048 (mainnet) or 64 (minimal)]",
global = true,
value_name = "N0,N1,N2,...",
help = "Specifies the frequency for storing full state snapshots and hierarchical \
diffs in the freezer DB.",
default_value_t = HierarchyConfig::default(),
display_order = 0
)]
pub slots_per_restore_point: Option<u64>,
pub hierarchy_exponents: HierarchyConfig,
#[clap(
long,
@@ -58,13 +60,12 @@ pub struct DatabaseManager {
#[clap(
long,
global = true,
help = "Prints help information",
action = clap::ArgAction::HelpLong,
value_name = "DATABASE",
help = "Set the database backend to be used by the beacon node.",
display_order = 0,
help_heading = FLAG_HEADER
default_value_t = store::config::DatabaseBackend::LevelDb
)]
help: Option<bool>,
pub backend: store::config::DatabaseBackend,
#[clap(subcommand)]
pub subcommand: DatabaseManagerSubcommand,

175
database_manager/src/lib.rs
View File

@@ -6,25 +6,27 @@ use beacon_chain::{
builder::Witness, eth1_chain::CachingEth1Backend, schema_change::migrate_schema,
slot_clock::SystemTimeSlotClock,
};
use beacon_node::{get_data_dir, get_slots_per_restore_point, ClientConfig};
use beacon_node::{get_data_dir, ClientConfig};
use clap::ArgMatches;
use clap::ValueEnum;
use cli::{Compact, Inspect};
use environment::{Environment, RuntimeContext};
use serde::{Deserialize, Serialize};
use slog::{debug, info, warn, Logger};
use ssz::Decode;
use std::fs;
use std::io::Write;
use std::path::{Path, PathBuf};
use store::metadata::STATE_UPPER_LIMIT_NO_RETAIN;
use store::KeyValueStore;
use store::{
database::interface::BeaconNodeBackend,
errors::Error,
hot_cold_store::HotColdDBError,
metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION},
DBColumn, HotColdDB, KeyValueStore, KeyValueStoreOp, LevelDB,
};
use strum::{EnumString, EnumVariantNames};
use tracing::{info, warn};
use types::{BeaconState, BlobSidecarList, EthSpec, Hash256, Slot};
fn parse_client_config<E: EthSpec>(
@@ -41,21 +43,15 @@ fn parse_client_config<E: EthSpec>(
client_config
.blobs_db_path
.clone_from(&database_manager_config.blobs_dir);
let (sprp, sprp_explicit) =
get_slots_per_restore_point::<E>(database_manager_config.slots_per_restore_point)?;
client_config.store.slots_per_restore_point = sprp;
client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit;
client_config.store.blob_prune_margin_epochs = database_manager_config.blob_prune_margin_epochs;
client_config.store.hierarchy_config = database_manager_config.hierarchy_exponents.clone();
client_config.store.backend = database_manager_config.backend;
Ok(client_config)
}
pub fn display_db_version<E: EthSpec>(
client_config: ClientConfig,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), Error> {
let spec = runtime_context.eth2_config.spec.clone();
let hot_path = client_config.get_db_path();
@@ -63,7 +59,7 @@ pub fn display_db_version<E: EthSpec>(
let blobs_path = client_config.get_blobs_db_path();
let mut version = CURRENT_SCHEMA_VERSION;
HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
@@ -73,16 +69,14 @@ pub fn display_db_version<E: EthSpec>(
},
client_config.store,
spec,
log.clone(),
)?;
info!(log, "Database version: {}", version.as_u64());
info!(version = version.as_u64(), "Database");
if version != CURRENT_SCHEMA_VERSION {
info!(
log,
"Latest schema version: {}",
CURRENT_SCHEMA_VERSION.as_u64(),
current_schema_version = CURRENT_SCHEMA_VERSION.as_u64(),
"Latest schema"
);
}
@@ -153,11 +147,14 @@ pub fn inspect_db<E: EthSpec>(
let mut num_keys = 0;
let sub_db = if inspect_config.freezer {
LevelDB::<E>::open(&cold_path).map_err(|e| format!("Unable to open freezer DB: {e:?}"))?
BeaconNodeBackend::<E>::open(&client_config.store, &cold_path)
.map_err(|e| format!("Unable to open freezer DB: {e:?}"))?
} else if inspect_config.blobs_db {
LevelDB::<E>::open(&blobs_path).map_err(|e| format!("Unable to open blobs DB: {e:?}"))?
BeaconNodeBackend::<E>::open(&client_config.store, &blobs_path)
.map_err(|e| format!("Unable to open blobs DB: {e:?}"))?
} else {
LevelDB::<E>::open(&hot_path).map_err(|e| format!("Unable to open hot DB: {e:?}"))?
BeaconNodeBackend::<E>::open(&client_config.store, &hot_path)
.map_err(|e| format!("Unable to open hot DB: {e:?}"))?
};
let skip = inspect_config.skip.unwrap_or(0);
@@ -263,7 +260,6 @@ fn parse_compact_config(compact_config: &Compact) -> Result<CompactConfig, Strin
pub fn compact_db<E: EthSpec>(
compact_config: CompactConfig,
client_config: ClientConfig,
log: Logger,
) -> Result<(), Error> {
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
@@ -271,17 +267,25 @@ pub fn compact_db<E: EthSpec>(
let column = compact_config.column;
let (sub_db, db_name) = if compact_config.freezer {
(LevelDB::<E>::open(&cold_path)?, "freezer_db")
(
BeaconNodeBackend::<E>::open(&client_config.store, &cold_path)?,
"freezer_db",
)
} else if compact_config.blobs_db {
(LevelDB::<E>::open(&blobs_path)?, "blobs_db")
(
BeaconNodeBackend::<E>::open(&client_config.store, &blobs_path)?,
"blobs_db",
)
} else {
(LevelDB::<E>::open(&hot_path)?, "hot_db")
(
BeaconNodeBackend::<E>::open(&client_config.store, &hot_path)?,
"hot_db",
)
};
info!(
log,
"Compacting database";
"db" => db_name,
"column" => ?column
db = db_name,
column = ?column,
"Compacting database"
);
sub_db.compact_column(column)?;
Ok(())
@@ -300,8 +304,8 @@ fn parse_migrate_config(migrate_config: &Migrate) -> Result<MigrateConfig, Strin
pub fn migrate_db<E: EthSpec>(
migrate_config: MigrateConfig,
client_config: ClientConfig,
mut genesis_state: BeaconState<E>,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), Error> {
let spec = runtime_context.eth2_config.spec.clone();
let hot_path = client_config.get_db_path();
@@ -310,7 +314,7 @@ pub fn migrate_db<E: EthSpec>(
let mut from = CURRENT_SCHEMA_VERSION;
let to = migrate_config.to;
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
@@ -320,44 +324,39 @@ pub fn migrate_db<E: EthSpec>(
},
client_config.store.clone(),
spec.clone(),
log.clone(),
)?;
info!(
log,
"Migrating database schema";
"from" => from.as_u64(),
"to" => to.as_u64(),
from = from.as_u64(),
to = to.as_u64(),
"Migrating database schema"
);
let genesis_state_root = genesis_state.canonical_root()?;
migrate_schema::<Witness<SystemTimeSlotClock, CachingEth1Backend<E>, _, _, _>>(
db,
client_config.eth1.deposit_contract_deploy_block,
Some(genesis_state_root),
from,
to,
log,
&spec,
)
}
pub fn prune_payloads<E: EthSpec>(
client_config: ClientConfig,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), Error> {
let spec = &runtime_context.eth2_config.spec;
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
log,
)?;
// If we're trigging a prune manually then ignore the check on the split's parent that bails
@@ -369,21 +368,19 @@ pub fn prune_payloads<E: EthSpec>(
pub fn prune_blobs<E: EthSpec>(
client_config: ClientConfig,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), Error> {
let spec = &runtime_context.eth2_config.spec;
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
log,
)?;
// If we're triggering a prune manually then ignore the check on `epochs_per_blob_prune` that
@@ -406,21 +403,19 @@ pub fn prune_states<E: EthSpec>(
prune_config: PruneStatesConfig,
mut genesis_state: BeaconState<E>,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), String> {
let spec = &runtime_context.eth2_config.spec;
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
log.clone(),
)
.map_err(|e| format!("Unable to open database: {e:?}"))?;
@@ -428,8 +423,7 @@ pub fn prune_states<E: EthSpec>(
// correct network, and that we don't end up storing the wrong genesis state.
let genesis_from_db = db
.load_cold_state_by_slot(Slot::new(0))
.map_err(|e| format!("Error reading genesis state: {e:?}"))?
.ok_or("Error: genesis state missing from database. Check schema version.")?;
.map_err(|e| format!("Error reading genesis state: {e:?}"))?;
if genesis_from_db.genesis_validators_root() != genesis_state.genesis_validators_root() {
return Err(format!(
@@ -440,27 +434,15 @@ pub fn prune_states<E: EthSpec>(
// Check that the user has confirmed they want to proceed.
if !prune_config.confirm {
match db.get_anchor_info() {
Some(anchor_info)
if anchor_info.state_lower_limit == 0
&& anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN =>
{
info!(log, "States have already been pruned");
return Ok(());
}
_ => {
info!(log, "Ready to prune states");
}
if db.get_anchor_info().full_state_pruning_enabled() {
info!("States have already been pruned");
return Ok(());
}
warn!(
log,
"Pruning states is irreversible";
);
warn!(
log,
"Re-run this command with --confirm to commit to state deletion"
);
info!(log, "Nothing has been pruned on this run");
info!("Ready to prune states");
warn!("Pruning states is irreversible");
warn!("Re-run this command with --confirm to commit to state deletion");
info!("Nothing has been pruned on this run");
return Err("Error: confirmation flag required".into());
}
@@ -471,7 +453,7 @@ pub fn prune_states<E: EthSpec>(
db.prune_historic_states(genesis_state_root, &genesis_state)
.map_err(|e| format!("Failed to prune due to error: {e:?}"))?;
info!(log, "Historic states pruned successfully");
info!("Historic states pruned successfully");
Ok(())
}
@@ -678,54 +660,55 @@ pub fn run<E: EthSpec>(
) -> Result<(), String> {
let client_config = parse_client_config(cli_args, db_manager_config, &env)?;
let context = env.core_context();
let log = context.log().clone();
let format_err = |e| format!("Fatal error: {:?}", e);
let get_genesis_state = || {
let executor = env.core_context().executor;
let network_config = context
.eth2_network_config
.clone()
.ok_or("Missing network config")?;
executor
.block_on_dangerous(
network_config.genesis_state::<E>(
client_config.genesis_state_url.as_deref(),
client_config.genesis_state_url_timeout,
),
"get_genesis_state",
)
.ok_or("Shutting down")?
.map_err(|e| format!("Error getting genesis state: {e}"))?
.ok_or("Genesis state missing".to_string())
};
match &db_manager_config.subcommand {
cli::DatabaseManagerSubcommand::Migrate(migrate_config) => {
let migrate_config = parse_migrate_config(migrate_config)?;
migrate_db(migrate_config, client_config, &context, log).map_err(format_err)
let genesis_state = get_genesis_state()?;
migrate_db(migrate_config, client_config, genesis_state, &context).map_err(format_err)
}
cli::DatabaseManagerSubcommand::Inspect(inspect_config) => {
let inspect_config = parse_inspect_config(inspect_config)?;
inspect_db::<E>(inspect_config, client_config)
}
cli::DatabaseManagerSubcommand::Version(_) => {
display_db_version(client_config, &context, log).map_err(format_err)
display_db_version(client_config, &context).map_err(format_err)
}
cli::DatabaseManagerSubcommand::PrunePayloads(_) => {
prune_payloads(client_config, &context, log).map_err(format_err)
prune_payloads(client_config, &context).map_err(format_err)
}
cli::DatabaseManagerSubcommand::PruneBlobs(_) => {
prune_blobs(client_config, &context, log).map_err(format_err)
prune_blobs(client_config, &context).map_err(format_err)
}
cli::DatabaseManagerSubcommand::PruneStates(prune_states_config) => {
let executor = env.core_context().executor;
let network_config = context
.eth2_network_config
.clone()
.ok_or("Missing network config")?;
let genesis_state = executor
.block_on_dangerous(
network_config.genesis_state::<E>(
client_config.genesis_state_url.as_deref(),
client_config.genesis_state_url_timeout,
&log,
),
"get_genesis_state",
)
.ok_or("Shutting down")?
.map_err(|e| format!("Error getting genesis state: {e}"))?
.ok_or("Genesis state missing")?;
let prune_config = parse_prune_states_config(prune_states_config)?;
prune_states(client_config, prune_config, genesis_state, &context, log)
let genesis_state = get_genesis_state()?;
prune_states(client_config, prune_config, genesis_state, &context)
}
cli::DatabaseManagerSubcommand::Compact(compact_config) => {
let compact_config = parse_compact_config(compact_config)?;
compact_db::<E>(compact_config, client_config, log).map_err(format_err)
compact_db::<E>(compact_config, client_config).map_err(format_err)
}
cli::DatabaseManagerSubcommand::SetOldestBlobSlot(blob_slot_config) => {
set_oldest_blob_slot(blob_slot_config.slot, client_config, &context, log)