gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-16 19:32:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix tree-states sub-epoch diffs (#5097)

Browse Source
This commit is contained in:
Michael Sproul
2024-01-30 15:56:48 +11:00
committed by GitHub
parent 11461d8f51
commit 7862c71bb5
12 changed files with 255 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
database_manager/src/lib.rs
View File

@@ -77,7 +77,15 @@ pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> {
Arg::with_name("freezer")
.long("freezer")
.help("Inspect the freezer DB rather than the hot DB")
.takes_value(false),
.takes_value(false)
.conflicts_with("blobs-db"),
)
.arg(
Arg::with_name("blobs-db")
.long("blobs-db")
.help("Inspect the blobs DB rather than the hot DB")
.takes_value(false)
.conflicts_with("freezer"),
)
.arg(
Arg::with_name("output-dir")
@@ -88,6 +96,34 @@ pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> {
)
}
pub fn compact_cli_app<'a, 'b>() -> App<'a, 'b> {
App::new("compact")
.setting(clap::AppSettings::ColoredHelp)
.about("Compact database manually")
.arg(
Arg::with_name("column")
.long("column")
.value_name("TAG")
.help("3-byte column ID (see `DBColumn`)")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("freezer")
.long("freezer")
.help("Inspect the freezer DB rather than the hot DB")
.takes_value(false)
.conflicts_with("blobs-db"),
)
.arg(
Arg::with_name("blobs-db")
.long("blobs-db")
.help("Inspect the blobs DB rather than the hot DB")
.takes_value(false)
.conflicts_with("freezer"),
)
}
pub fn prune_payloads_app<'a, 'b>() -> App<'a, 'b> {
App::new("prune-payloads")
.alias("prune_payloads")
@@ -143,17 +179,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
.visible_aliases(&["db"])
.setting(clap::AppSettings::ColoredHelp)
.about("Manage a beacon node database")
.arg(
Arg::with_name("slots-per-restore-point")
.long("slots-per-restore-point")
.value_name("SLOT_COUNT")
.help(
"Specifies how often a freezer DB restore point should be stored. \
Cannot be changed after initialization. \
[default: 2048 (mainnet) or 64 (minimal)]",
)
.takes_value(true),
)
.arg(
Arg::with_name("freezer-dir")
.long("freezer-dir")
@@ -179,9 +204,25 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
.help("Data directory for the blobs database.")
.takes_value(true),
)
.arg(
Arg::with_name("hierarchy-exponents")
.long("hierarchy-exponents")
.value_name("EXPONENTS")
.help("Specifies the frequency for storing full state snapshots and hierarchical \
diffs in the freezer DB. Accepts a comma-separated list of ascending \
exponents. Each exponent defines an interval for storing diffs to the layer \
above. The last exponent defines the interval for full snapshots. \
For example, a config of '4,8,12' would store a full snapshot every \
4096 (2^12) slots, first-level diffs every 256 (2^8) slots, and second-level \
diffs every 16 (2^4) slots. \
Cannot be changed after initialization. \
[default: 5,9,11,13,16,18,21]")
.takes_value(true)
)
.subcommand(migrate_cli_app())
.subcommand(version_cli_app())
.subcommand(inspect_cli_app())
.subcommand(compact_cli_app())
.subcommand(prune_payloads_app())
.subcommand(prune_blobs_app())
.subcommand(diff_app())
@@ -210,6 +251,10 @@ fn parse_client_config<E: EthSpec>(
client_config.store.blob_prune_margin_epochs = blob_prune_margin_epochs;
}
if let Some(hierarchy_config) = clap_utils::parse_optional(cli_args, "hierarchy-exponents")? {
client_config.store.hierarchy_config = hierarchy_config;
}
Ok(client_config)
}
@@ -268,6 +313,7 @@ pub struct InspectConfig {
skip: Option<usize>,
limit: Option<usize>,
freezer: bool,
blobs_db: bool,
/// Configures where the inspect output should be stored.
output_dir: PathBuf,
}
@@ -278,6 +324,7 @@ fn parse_inspect_config(cli_args: &ArgMatches) -> Result<InspectConfig, String>
let skip = clap_utils::parse_optional(cli_args, "skip")?;
let limit = clap_utils::parse_optional(cli_args, "limit")?;
let freezer = cli_args.is_present("freezer");
let blobs_db = cli_args.is_present("blobs-db");
let output_dir: PathBuf =
clap_utils::parse_optional(cli_args, "output-dir")?.unwrap_or_else(PathBuf::new);
@@ -287,6 +334,7 @@ fn parse_inspect_config(cli_args: &ArgMatches) -> Result<InspectConfig, String>
skip,
limit,
freezer,
blobs_db,
output_dir,
})
}
@@ -297,12 +345,15 @@ pub fn inspect_db<E: EthSpec>(
) -> Result<(), String> {
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let mut total = 0;
let mut num_keys = 0;
let sub_db = if inspect_config.freezer {
LevelDB::<E>::open(&cold_path).map_err(|e| format!("Unable to open freezer DB: {e:?}"))?
} else if inspect_config.blobs_db {
LevelDB::<E>::open(&blobs_path).map_err(|e| format!("Unable to open blobs DB: {e:?}"))?
} else {
LevelDB::<E>::open(&hot_path).map_err(|e| format!("Unable to open hot DB: {e:?}"))?
};
@@ -386,6 +437,50 @@ pub fn inspect_db<E: EthSpec>(
Ok(())
}
pub struct CompactConfig {
column: DBColumn,
freezer: bool,
blobs_db: bool,
}
fn parse_compact_config(cli_args: &ArgMatches) -> Result<CompactConfig, String> {
let column = clap_utils::parse_required(cli_args, "column")?;
let freezer = cli_args.is_present("freezer");
let blobs_db = cli_args.is_present("blobs-db");
Ok(CompactConfig {
column,
freezer,
blobs_db,
})
}
pub fn compact_db<E: EthSpec>(
compact_config: CompactConfig,
client_config: ClientConfig,
log: Logger,
) -> Result<(), Error> {
let hot_path = client_config.get_db_path();
let cold_path = client_config.get_freezer_db_path();
let blobs_path = client_config.get_blobs_db_path();
let column = compact_config.column;
let (sub_db, db_name) = if compact_config.freezer {
(LevelDB::<E>::open(&cold_path)?, "freezer_db")
} else if compact_config.blobs_db {
(LevelDB::<E>::open(&blobs_path)?, "blobs_db")
} else {
(LevelDB::<E>::open(&hot_path)?, "hot_db")
};
info!(
log,
"Compacting database";
"db" => db_name,
"column" => ?column
);
sub_db.compact_column(column)?;
Ok(())
}
pub struct MigrateConfig {
to: SchemaVersion,
}
@@ -590,7 +685,10 @@ pub fn prune_states<E: EthSpec>(
// Check that the user has confirmed they want to proceed.
if !prune_config.confirm {
match db.get_anchor_info() {
Some(anchor_info) if anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN => {
Some(anchor_info)
if anchor_info.state_lower_limit == 0
&& anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN =>
{
info!(log, "States have already been pruned");
return Ok(());
}
@@ -640,6 +738,10 @@ pub fn run<T: EthSpec>(cli_args: &ArgMatches<'_>, env: Environment<T>) -> Result
let inspect_config = parse_inspect_config(cli_args)?;
inspect_db::<T>(inspect_config, client_config)
}
("compact", Some(cli_args)) => {
let compact_config = parse_compact_config(cli_args)?;
compact_db::<T>(compact_config, client_config, log).map_err(format_err)
}
("prune-payloads", Some(_)) => {
prune_payloads(client_config, &context, log).map_err(format_err)
}