mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-03 00:31:50 +00:00
Add lighthouse db command (#3129)
## Proposed Changes Add a `lighthouse db` command with three initial subcommands: - `lighthouse db version`: print the database schema version. - `lighthouse db migrate --to N`: manually upgrade (or downgrade!) the database to a different version. - `lighthouse db inspect --column C`: log the key and size in bytes of every value in a given `DBColumn`. This PR lays the groundwork for other changes, namely: - Mark's fast-deposit sync (https://github.com/sigp/lighthouse/pull/2915), for which I think we should implement a database downgrade (from v9 to v8). - My `tree-states` work, which already implements a downgrade (v10 to v8). - Standalone purge commands like `lighthouse db purge-dht` per https://github.com/sigp/lighthouse/issues/2824. ## Additional Info I updated the `strum` crate to 0.24.0, which necessitated some changes in the network code to remove calls to deprecated methods. Thanks to @winksaville for the motivation, and implementation work that I used as a source of inspiration (https://github.com/sigp/lighthouse/pull/2685).
This commit is contained in:
Michael Sproul
18
database_manager/Cargo.toml
Normal file
18
database_manager/Cargo.toml
Normal file
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "database_manager"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
beacon_chain = { path = "../beacon_node/beacon_chain" }
|
||||
beacon_node = { path = "../beacon_node" }
|
||||
clap = "2.33.3"
|
||||
clap_utils = { path = "../common/clap_utils" }
|
||||
environment = { path = "../lighthouse/environment" }
|
||||
logging = { path = "../common/logging" }
|
||||
sloggers = "2.0.2"
|
||||
store = { path = "../beacon_node/store" }
|
||||
tempfile = "3.1.0"
|
||||
types = { path = "../consensus/types" }
|
||||
slog = "2.5.2"
|
||||
strum = { version = "0.24.0", features = ["derive"] }
|
||||
278
database_manager/src/lib.rs
Normal file
278
database_manager/src/lib.rs
Normal file
@@ -0,0 +1,278 @@
|
||||
use beacon_chain::{
|
||||
builder::Witness, eth1_chain::CachingEth1Backend, schema_change::migrate_schema,
|
||||
slot_clock::SystemTimeSlotClock,
|
||||
};
|
||||
use beacon_node::{get_data_dir, get_slots_per_restore_point, ClientConfig};
|
||||
use clap::{App, Arg, ArgMatches};
|
||||
use environment::{Environment, RuntimeContext};
|
||||
use slog::{info, Logger};
|
||||
use store::{
|
||||
errors::Error,
|
||||
metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION},
|
||||
DBColumn, HotColdDB, KeyValueStore, LevelDB,
|
||||
};
|
||||
use strum::{EnumString, EnumVariantNames, VariantNames};
|
||||
use types::EthSpec;
|
||||
|
||||
pub const CMD: &str = "database_manager";
|
||||
|
||||
pub fn version_cli_app<'a, 'b>() -> App<'a, 'b> {
|
||||
App::new("version")
|
||||
.visible_aliases(&["v"])
|
||||
.setting(clap::AppSettings::ColoredHelp)
|
||||
.about("Display database schema version")
|
||||
}
|
||||
|
||||
pub fn migrate_cli_app<'a, 'b>() -> App<'a, 'b> {
|
||||
App::new("migrate")
|
||||
.setting(clap::AppSettings::ColoredHelp)
|
||||
.about("Migrate the database to a specific schema version")
|
||||
.arg(
|
||||
Arg::with_name("to")
|
||||
.long("to")
|
||||
.value_name("VERSION")
|
||||
.help("Schema version to migrate to")
|
||||
.takes_value(true)
|
||||
.required(true),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> {
|
||||
App::new("inspect")
|
||||
.setting(clap::AppSettings::ColoredHelp)
|
||||
.about("Inspect raw database values")
|
||||
.arg(
|
||||
Arg::with_name("column")
|
||||
.long("column")
|
||||
.value_name("TAG")
|
||||
.help("3-byte column ID (see `DBColumn`)")
|
||||
.takes_value(true)
|
||||
.required(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("output")
|
||||
.long("output")
|
||||
.value_name("TARGET")
|
||||
.help("Select the type of output to show")
|
||||
.default_value("sizes")
|
||||
.possible_values(InspectTarget::VARIANTS),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
|
||||
App::new(CMD)
|
||||
.visible_aliases(&["db"])
|
||||
.setting(clap::AppSettings::ColoredHelp)
|
||||
.about("Manage a beacon node database")
|
||||
.arg(
|
||||
Arg::with_name("slots-per-restore-point")
|
||||
.long("slots-per-restore-point")
|
||||
.value_name("SLOT_COUNT")
|
||||
.help(
|
||||
"Specifies how often a freezer DB restore point should be stored. \
|
||||
Cannot be changed after initialization. \
|
||||
[default: 2048 (mainnet) or 64 (minimal)]",
|
||||
)
|
||||
.takes_value(true),
|
||||
)
|
||||
.arg(
|
||||
Arg::with_name("freezer-dir")
|
||||
.long("freezer-dir")
|
||||
.value_name("DIR")
|
||||
.help("Data directory for the freezer database.")
|
||||
.takes_value(true),
|
||||
)
|
||||
.subcommand(migrate_cli_app())
|
||||
.subcommand(version_cli_app())
|
||||
.subcommand(inspect_cli_app())
|
||||
}
|
||||
|
||||
fn parse_client_config<E: EthSpec>(
|
||||
cli_args: &ArgMatches,
|
||||
_env: &Environment<E>,
|
||||
) -> Result<ClientConfig, String> {
|
||||
let mut client_config = ClientConfig {
|
||||
data_dir: get_data_dir(cli_args),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Some(freezer_dir) = clap_utils::parse_optional(cli_args, "freezer-dir")? {
|
||||
client_config.freezer_db_path = Some(freezer_dir);
|
||||
}
|
||||
|
||||
client_config.store.slots_per_restore_point = get_slots_per_restore_point::<E>(cli_args)?;
|
||||
|
||||
Ok(client_config)
|
||||
}
|
||||
|
||||
pub fn display_db_version<E: EthSpec>(
|
||||
client_config: ClientConfig,
|
||||
runtime_context: &RuntimeContext<E>,
|
||||
log: Logger,
|
||||
) -> Result<(), Error> {
|
||||
let spec = runtime_context.eth2_config.spec.clone();
|
||||
let hot_path = client_config.get_db_path();
|
||||
let cold_path = client_config.get_freezer_db_path();
|
||||
|
||||
let mut version = CURRENT_SCHEMA_VERSION;
|
||||
HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
|
||||
&hot_path,
|
||||
&cold_path,
|
||||
|_, from, _| {
|
||||
version = from;
|
||||
Ok(())
|
||||
},
|
||||
client_config.store,
|
||||
spec,
|
||||
log.clone(),
|
||||
)?;
|
||||
|
||||
info!(log, "Database version: {}", version.as_u64());
|
||||
|
||||
if version != CURRENT_SCHEMA_VERSION {
|
||||
info!(
|
||||
log,
|
||||
"Latest schema version: {}",
|
||||
CURRENT_SCHEMA_VERSION.as_u64(),
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, EnumString, EnumVariantNames)]
|
||||
pub enum InspectTarget {
|
||||
#[strum(serialize = "sizes")]
|
||||
ValueSizes,
|
||||
#[strum(serialize = "total")]
|
||||
ValueTotal,
|
||||
}
|
||||
|
||||
pub struct InspectConfig {
|
||||
column: DBColumn,
|
||||
target: InspectTarget,
|
||||
}
|
||||
|
||||
fn parse_inspect_config(cli_args: &ArgMatches) -> Result<InspectConfig, String> {
|
||||
let column = clap_utils::parse_required(cli_args, "column")?;
|
||||
let target = clap_utils::parse_required(cli_args, "output")?;
|
||||
|
||||
Ok(InspectConfig { column, target })
|
||||
}
|
||||
|
||||
pub fn inspect_db<E: EthSpec>(
|
||||
inspect_config: InspectConfig,
|
||||
client_config: ClientConfig,
|
||||
runtime_context: &RuntimeContext<E>,
|
||||
log: Logger,
|
||||
) -> Result<(), Error> {
|
||||
let spec = runtime_context.eth2_config.spec.clone();
|
||||
let hot_path = client_config.get_db_path();
|
||||
let cold_path = client_config.get_freezer_db_path();
|
||||
|
||||
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
|
||||
&hot_path,
|
||||
&cold_path,
|
||||
|_, _, _| Ok(()),
|
||||
client_config.store,
|
||||
spec,
|
||||
log,
|
||||
)?;
|
||||
|
||||
let mut total = 0;
|
||||
|
||||
for res in db.hot_db.iter_column(inspect_config.column) {
|
||||
let (key, value) = res?;
|
||||
|
||||
match inspect_config.target {
|
||||
InspectTarget::ValueSizes => {
|
||||
println!("{:?}: {} bytes", key, value.len());
|
||||
total += value.len();
|
||||
}
|
||||
InspectTarget::ValueTotal => {
|
||||
total += value.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match inspect_config.target {
|
||||
InspectTarget::ValueSizes | InspectTarget::ValueTotal => {
|
||||
println!("Total: {} bytes", total);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub struct MigrateConfig {
|
||||
to: SchemaVersion,
|
||||
}
|
||||
|
||||
fn parse_migrate_config(cli_args: &ArgMatches) -> Result<MigrateConfig, String> {
|
||||
let to = SchemaVersion(clap_utils::parse_required(cli_args, "to")?);
|
||||
|
||||
Ok(MigrateConfig { to })
|
||||
}
|
||||
|
||||
pub fn migrate_db<E: EthSpec>(
|
||||
migrate_config: MigrateConfig,
|
||||
client_config: ClientConfig,
|
||||
runtime_context: &RuntimeContext<E>,
|
||||
log: Logger,
|
||||
) -> Result<(), Error> {
|
||||
let spec = runtime_context.eth2_config.spec.clone();
|
||||
let hot_path = client_config.get_db_path();
|
||||
let cold_path = client_config.get_freezer_db_path();
|
||||
|
||||
let mut from = CURRENT_SCHEMA_VERSION;
|
||||
let to = migrate_config.to;
|
||||
let db = HotColdDB::<E, LevelDB<E>, LevelDB<E>>::open(
|
||||
&hot_path,
|
||||
&cold_path,
|
||||
|_, db_initial_version, _| {
|
||||
from = db_initial_version;
|
||||
Ok(())
|
||||
},
|
||||
client_config.store.clone(),
|
||||
spec,
|
||||
log.clone(),
|
||||
)?;
|
||||
|
||||
info!(
|
||||
log,
|
||||
"Migrating database schema";
|
||||
"from" => from.as_u64(),
|
||||
"to" => to.as_u64(),
|
||||
);
|
||||
|
||||
migrate_schema::<Witness<SystemTimeSlotClock, CachingEth1Backend<E>, _, _, _>>(
|
||||
db,
|
||||
&client_config.get_data_dir(),
|
||||
from,
|
||||
to,
|
||||
log,
|
||||
)
|
||||
}
|
||||
|
||||
/// Run the database manager, returning an error string if the operation did not succeed.
|
||||
pub fn run<T: EthSpec>(cli_args: &ArgMatches<'_>, mut env: Environment<T>) -> Result<(), String> {
|
||||
let client_config = parse_client_config(cli_args, &env)?;
|
||||
let context = env.core_context();
|
||||
let log = context.log().clone();
|
||||
|
||||
match cli_args.subcommand() {
|
||||
("version", Some(_)) => display_db_version(client_config, &context, log),
|
||||
("migrate", Some(cli_args)) => {
|
||||
let migrate_config = parse_migrate_config(cli_args)?;
|
||||
migrate_db(migrate_config, client_config, &context, log)
|
||||
}
|
||||
("inspect", Some(cli_args)) => {
|
||||
let inspect_config = parse_inspect_config(cli_args)?;
|
||||
inspect_db(inspect_config, client_config, &context, log)
|
||||
}
|
||||
_ => {
|
||||
return Err("Unknown subcommand, for help `lighthouse database_manager --help`".into())
|
||||
}
|
||||
}
|
||||
.map_err(|e| format!("Fatal error: {:?}", e))
|
||||
}
|
||||
Reference in New Issue
Block a user