gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-20 05:14:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/unstable' into tree-states

Browse Source
This commit is contained in:
Michael Sproul
2023-11-12 22:19:07 +03:00
parent d36ebba1ea 051c3e842f
commit 9cdc4b989a
161 changed files with 3635 additions and 2227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
database_manager/src/lib.rs
View File

@@ -5,17 +5,18 @@ use beacon_chain::{
use beacon_node::{get_data_dir, ClientConfig};
use clap::{App, Arg, ArgMatches};
use environment::{Environment, RuntimeContext};
use slog::{info, Logger};
use slog::{info, warn, Logger};
use std::fs;
use std::io::Write;
use std::path::PathBuf;
use store::metadata::STATE_UPPER_LIMIT_NO_RETAIN;
use store::{
errors::Error,
metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION},
DBColumn, HotColdDB, KeyValueStore, LevelDB,
};
use strum::{EnumString, EnumVariantNames, VariantNames};
use types::{EthSpec, VList};
use types::{BeaconState, EthSpec, Slot, VList};
pub const CMD: &str = "database_manager";
@@ -88,7 +89,8 @@ pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> {
}
pub fn prune_payloads_app<'a, 'b>() -> App<'a, 'b> {
App::new("prune_payloads")
App::new("prune-payloads")
.alias("prune_payloads")
.setting(clap::AppSettings::ColoredHelp)
.about("Prune finalized execution payloads")
}
@@ -114,11 +116,28 @@ pub fn diff_app<'a, 'b>() -> App<'a, 'b> {
}
pub fn prune_blobs_app<'a, 'b>() -> App<'a, 'b> {
App::new("prune_blobs")
App::new("prune-blobs")
.alias("prune_blobs")
.setting(clap::AppSettings::ColoredHelp)
.about("Prune blobs older than data availability boundary")
}
pub fn prune_states_app<'a, 'b>() -> App<'a, 'b> {
App::new("prune-states")
.alias("prune_states")
.arg(
Arg::with_name("confirm")
.long("confirm")
.help(
"Commit to pruning states irreversably. Without this flag the command will \
just check that the database is capable of being pruned.",
)
.takes_value(false),
)
.setting(clap::AppSettings::ColoredHelp)
.about("Prune all beacon states from the freezer database")
}
pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
App::new(CMD)
.visible_aliases(&["db"])
@@ -166,6 +185,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
.subcommand(prune_payloads_app())
.subcommand(prune_blobs_app())
.subcommand(diff_app())
.subcommand(prune_states_app())
}
fn parse_client_config<E: EthSpec>(
@@ -207,7 +227,7 @@ pub fn display_db_version<E: EthSpec>(
HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
blobs_path,
&blobs_path,
|_, from, _| {
version = from;
Ok(())
@@ -285,7 +305,7 @@ pub fn inspect_db<E: EthSpec>(
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
blobs_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec,
@@ -407,7 +427,7 @@ pub fn migrate_db<E: EthSpec>(
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
blobs_path,
&blobs_path,
|_, db_initial_version, _| {
from = db_initial_version;
Ok(())
@@ -498,7 +518,7 @@ pub fn prune_payloads<E: EthSpec>(
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
blobs_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
@@ -524,7 +544,7 @@ pub fn prune_blobs<E: EthSpec>(
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
blobs_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
@@ -536,6 +556,86 @@ pub fn prune_blobs<E: EthSpec>(
db.try_prune_most_blobs(true)
}
pub struct PruneStatesConfig {
confirm: bool,
}
fn parse_prune_states_config(cli_args: &ArgMatches) -> Result<PruneStatesConfig, String> {
let confirm = cli_args.is_present("confirm");
Ok(PruneStatesConfig { confirm })
}
pub fn prune_states<E: EthSpec>(
client_config: ClientConfig,
prune_config: PruneStatesConfig,
mut genesis_state: BeaconState<E>,
runtime_context: &RuntimeContext<E>,
log: Logger,
) -> Result<(), String> {
let spec = &runtime_context.eth2_config.spec;
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
&hot_path,
&cold_path,
&blobs_path,
|_, _, _| Ok(()),
client_config.store,
spec.clone(),
log.clone(),
)
.map_err(|e| format!("Unable to open database: {e:?}"))?;
// Load the genesis state from the database to ensure we're deleting states for the
// correct network, and that we don't end up storing the wrong genesis state.
let genesis_from_db = db
.load_cold_state_by_slot(Slot::new(0))
.map_err(|e| format!("Error reading genesis state: {e:?}"))?
.ok_or("Error: genesis state missing from database. Check schema version.")?;
if genesis_from_db.genesis_validators_root() != genesis_state.genesis_validators_root() {
return Err(format!(
"Error: Wrong network. Genesis state in DB does not match {} genesis.",
spec.config_name.as_deref().unwrap_or("<unknown network>")
));
}
// Check that the user has confirmed they want to proceed.
if !prune_config.confirm {
match db.get_anchor_info() {
Some(anchor_info) if anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN => {
info!(log, "States have already been pruned");
return Ok(());
}
_ => {
info!(log, "Ready to prune states");
}
}
warn!(
log,
"Pruning states is irreversible";
);
warn!(
log,
"Re-run this command with --confirm to commit to state deletion"
);
info!(log, "Nothing has been pruned on this run");
return Err("Error: confirmation flag required".into());
}
// Delete all historic state data and *re-store* the genesis state.
let genesis_state_root = genesis_state
.update_tree_hash_cache()
.map_err(|e| format!("Error computing genesis state root: {e:?}"))?;
db.prune_historic_states(genesis_state_root, &genesis_state)
.map_err(|e| format!("Failed to prune due to error: {e:?}"))?;
info!(log, "Historic states pruned successfully");
Ok(())
}
/// Run the database manager, returning an error string if the operation did not succeed.
pub fn run<T: EthSpec>(cli_args: &ArgMatches<'_>, env: Environment<T>) -> Result<(), String> {
let client_config = parse_client_config(cli_args, &env)?;
@@ -555,14 +655,38 @@ pub fn run<T: EthSpec>(cli_args: &ArgMatches<'_>, env: Environment<T>) -> Result
let inspect_config = parse_inspect_config(cli_args)?;
inspect_db(inspect_config, client_config, &context, log)
}
("prune_payloads", Some(_)) => {
("prune-payloads", Some(_)) => {
prune_payloads(client_config, &context, log).map_err(format_err)
}
("diff", Some(cli_args)) => {
let diff_config = parse_diff_config(cli_args)?;
diff::<T>(&diff_config, log).map_err(format_err)
}
("prune_blobs", Some(_)) => prune_blobs(client_config, &context, log).map_err(format_err),
("prune-blobs", Some(_)) => prune_blobs(client_config, &context, log).map_err(format_err),
("prune-states", Some(cli_args)) => {
let executor = env.core_context().executor;
let network_config = context
.eth2_network_config
.clone()
.ok_or("Missing network config")?;
let genesis_state = executor
.block_on_dangerous(
network_config.genesis_state::<T>(
client_config.genesis_state_url.as_deref(),
client_config.genesis_state_url_timeout,
&log,
),
"get_genesis_state",
)
.ok_or("Shutting down")?
.map_err(|e| format!("Error getting genesis state: {e}"))?
.ok_or("Genesis state missing")?;
let prune_config = parse_prune_states_config(cli_args)?;
prune_states(client_config, prune_config, genesis_state, &context, log)
}
_ => Err("Unknown subcommand, for help `lighthouse database_manager --help`".into()),
}
}