Implement ERA consumer and producer in lcli

lcli/src/consume_era_files.rs

@@ -0,0 +1,128 @@
+use beacon_chain::era::consumer::EraFileDir;
+use clap::ArgMatches;
+use clap_utils::parse_required;
+use environment::Environment;
+use eth2_network_config::Eth2NetworkConfig;
+use std::path::PathBuf;
+use std::time::Duration;
+use store::database::interface::BeaconNodeBackend;
+use store::{HotColdDB, StoreConfig};
+use tracing::info;
+use types::EthSpec;
+
+fn is_dir_non_empty(path: &PathBuf) -> bool {
+    path.exists()
+        && std::fs::read_dir(path)
+            .map(|mut entries| entries.next().is_some())
+            .unwrap_or(false)
+}
+
+pub fn run<E: EthSpec>(
+    env: Environment<E>,
+    network_config: Eth2NetworkConfig,
+    matches: &ArgMatches,
+) -> Result<(), String> {
+    let datadir: PathBuf = parse_required(matches, "datadir")?;
+    let era_dir: PathBuf = parse_required(matches, "era-dir")?;
+
+    let hot_path = datadir.join("chain_db");
+    let cold_path = datadir.join("freezer_db");
+    let blobs_path = datadir.join("blobs_db");
+
+    // Fail fast if database directories already contain data
+    if is_dir_non_empty(&hot_path) || is_dir_non_empty(&cold_path) {
+        return Err(format!(
+            "Database directories are not empty: {} / {}. \
+             This command expects a fresh datadir.",
+            hot_path.display(),
+            cold_path.display(),
+        ));
+    }
+
+    let spec = env.eth2_config.spec.clone();
+
+    info!(
+        hot_path = %hot_path.display(),
+        cold_path = %cold_path.display(),
+        era_dir = %era_dir.display(),
+        "Opening database"
+    );
+
+    std::fs::create_dir_all(&hot_path).map_err(|e| format!("Failed to create hot db dir: {e}"))?;
+    std::fs::create_dir_all(&cold_path)
+        .map_err(|e| format!("Failed to create cold db dir: {e}"))?;
+    std::fs::create_dir_all(&blobs_path)
+        .map_err(|e| format!("Failed to create blobs db dir: {e}"))?;
+
+    let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
+        &hot_path,
+        &cold_path,
+        &blobs_path,
+        |_, _, _| Ok(()),
+        StoreConfig::default(),
+        spec.clone(),
+    )
+    .map_err(|e| format!("Failed to open database: {e:?}"))?;
+
+    // Load genesis state from the network config
+    let mut genesis_state = env
+        .runtime()
+        .block_on(network_config.genesis_state::<E>(None, Duration::from_secs(120)))
+        .map_err(|e| format!("Failed to load genesis state: {e}"))?
+        .ok_or("No genesis state available for this network")?;
+
+    // Open ERA files directory and validate against genesis
+    let era_file_dir = EraFileDir::new::<E>(&era_dir, &spec)
+        .map_err(|e| format!("Failed to open ERA dir: {e}"))?;
+
+    // Verify ERA files match the network's genesis
+    if era_file_dir.genesis_validators_root() != genesis_state.genesis_validators_root() {
+        return Err(format!(
+            "ERA files genesis_validators_root ({:?}) does not match network genesis ({:?}). \
+             Are the ERA files from the correct network?",
+            era_file_dir.genesis_validators_root(),
+            genesis_state.genesis_validators_root(),
+        ));
+    }
+
+    info!(
+        genesis_validators_root = %genesis_state.genesis_validators_root(),
+        "Storing genesis state"
+    );
+
+    let genesis_root = genesis_state
+        .canonical_root()
+        .map_err(|e| format!("Failed to hash genesis state: {e:?}"))?;
+    db.put_cold_state(&genesis_root, &genesis_state)
+        .map_err(|e| format!("Failed to store genesis state: {e:?}"))?;
+
+    let max_era = era_file_dir.max_era();
+    info!(max_era, "Importing ERA files");
+
+    let start = std::time::Instant::now();
+    for era_number in 1..=max_era {
+        era_file_dir
+            .import_era_file(&db, era_number, &spec, None)
+            .map_err(|e| format!("Failed to import ERA {era_number}: {e}"))?;
+
+        if era_number % 100 == 0 || era_number == max_era {
+            let elapsed = start.elapsed();
+            let rate = era_number as f64 / elapsed.as_secs_f64();
+            info!(
+                era_number,
+                max_era,
+                ?elapsed,
+                rate = format!("{rate:.1} era/s"),
+                "Progress"
+            );
+        }
+    }
+
+    info!(
+        max_era,
+        elapsed = ?start.elapsed(),
+        "ERA file import complete. Database is ready."
+    );
+
+    Ok(())
+}

lcli/src/main.rs

@@ -1,11 +1,13 @@
 mod block_root;
 mod check_deposit_data;
+mod consume_era_files;
 mod generate_bootnode_enr;
 mod http_sync;
 mod indexed_attestations;
 mod mnemonic_validators;
 mod mock_el;
 mod parse_ssz;
+mod produce_era_files;
 mod skip_slots;
 mod state_root;
 mod transition_blocks;
@@ -571,6 +573,50 @@ fn main() {
                         .display_order(0)
                 )
         )
+        .subcommand(
+            Command::new("consume-era-files")
+                .about("Import ERA files into an empty database, producing a ready-to-use beacon node DB.")
+                .arg(
+                    Arg::new("datadir")
+                        .long("datadir")
+                        .value_name("PATH")
+                        .action(ArgAction::Set)
+                        .required(true)
+                        .help("Path to the beacon node data directory (will create chain_db, freezer_db, blobs_db inside).")
+                        .display_order(0)
+                )
+                .arg(
+                    Arg::new("era-dir")
+                        .long("era-dir")
+                        .value_name("PATH")
+                        .action(ArgAction::Set)
+                        .required(true)
+                        .help("Directory containing ERA files to import.")
+                        .display_order(0)
+                )
+        )
+        .subcommand(
+            Command::new("produce-era-files")
+                .about("Produce ERA files from a fully reconstructed beacon node database.")
+                .arg(
+                    Arg::new("datadir")
+                        .long("datadir")
+                        .value_name("PATH")
+                        .action(ArgAction::Set)
+                        .required(true)
+                        .help("Path to the beacon node data directory (containing chain_db, freezer_db, blobs_db).")
+                        .display_order(0)
+                )
+                .arg(
+                    Arg::new("output-dir")
+                        .long("output-dir")
+                        .value_name("PATH")
+                        .action(ArgAction::Set)
+                        .required(true)
+                        .help("Directory to write ERA files to. Created if it does not exist.")
+                        .display_order(0)
+                )
+        )
         .subcommand(
             Command::new("http-sync")
                 .about("Manual sync")
@@ -765,6 +811,13 @@ fn run<E: EthSpec>(env_builder: EnvironmentBuilder<E>, matches: &ArgMatches) ->
         }
         Some(("mock-el", matches)) => mock_el::run::<E>(env, matches)
             .map_err(|e| format!("Failed to run mock-el command: {}", e)),
+        Some(("consume-era-files", matches)) => {
+            let network_config = get_network_config()?;
+            consume_era_files::run::<E>(env, network_config, matches)
+                .map_err(|e| format!("Failed to consume ERA files: {}", e))
+        }
+        Some(("produce-era-files", matches)) => produce_era_files::run::<E>(env, matches)
+            .map_err(|e| format!("Failed to produce ERA files: {}", e)),
         Some(("http-sync", matches)) => {
             let network_config = get_network_config()?;
             http_sync::run::<E>(env, network_config, matches)

lcli/src/produce_era_files.rs

@@ -0,0 +1,90 @@
+use beacon_chain::era::producer;
+use clap::ArgMatches;
+use clap_utils::parse_required;
+use environment::Environment;
+use std::path::PathBuf;
+use store::database::interface::BeaconNodeBackend;
+use store::{HotColdDB, StoreConfig};
+use tracing::info;
+use types::EthSpec;
+
+pub fn run<E: EthSpec>(env: Environment<E>, matches: &ArgMatches) -> Result<(), String> {
+    let datadir: PathBuf = parse_required(matches, "datadir")?;
+    let output_dir: PathBuf = parse_required(matches, "output-dir")?;
+
+    let hot_path = datadir.join("chain_db");
+    let cold_path = datadir.join("freezer_db");
+    let blobs_path = datadir.join("blobs_db");
+
+    let spec = env.eth2_config.spec.clone();
+
+    info!(
+        hot_path = %hot_path.display(),
+        cold_path = %cold_path.display(),
+        output_dir = %output_dir.display(),
+        "Opening database"
+    );
+
+    let db = HotColdDB::<E, BeaconNodeBackend<E>, BeaconNodeBackend<E>>::open(
+        &hot_path,
+        &cold_path,
+        &blobs_path,
+        |_, _, _| Ok(()),
+        StoreConfig::default(),
+        spec,
+    )
+    .map_err(|e| format!("Failed to open database: {e:?}"))?;
+
+    let anchor = db.get_anchor_info();
+    let split = db.get_split_info();
+
+    info!(
+        anchor_slot = %anchor.anchor_slot,
+        state_lower_limit = %anchor.state_lower_limit,
+        state_upper_limit = %anchor.state_upper_limit,
+        oldest_block_slot = %anchor.oldest_block_slot,
+        split_slot = %split.slot,
+        "Database info"
+    );
+
+    // Verify reconstruction is complete: state_lower_limit should equal state_upper_limit
+    if !anchor.all_historic_states_stored() {
+        return Err(format!(
+            "State reconstruction is not complete. \
+             state_lower_limit={}, state_upper_limit={}. \
+             Run with --reconstruct-historic-states first.",
+            anchor.state_lower_limit, anchor.state_upper_limit,
+        ));
+    }
+
+    // Verify block backfill is complete
+    if anchor.oldest_block_slot > 0 {
+        return Err(format!(
+            "Block backfill is not complete. oldest_block_slot={}. \
+             Complete backfill sync first.",
+            anchor.oldest_block_slot,
+        ));
+    }
+
+    let slots_per_historical_root = E::slots_per_historical_root() as u64;
+    // An ERA can only be created if its end slot <= split slot (finalized boundary)
+    let max_era = split.slot.as_u64() / slots_per_historical_root;
+
+    info!(max_era, "Producing ERA files from 0 to max_era");
+
+    std::fs::create_dir_all(&output_dir)
+        .map_err(|e| format!("Failed to create output directory: {e}"))?;
+
+    for era_number in 0..=max_era {
+        producer::create_era_file(&db, era_number, &output_dir)
+            .map_err(|e| format!("Failed to produce ERA file {era_number}: {e}"))?;
+
+        if (era_number + 1) % 100 == 0 || era_number == max_era {
+            info!(era_number, max_era, "Progress");
+        }
+    }
+
+    info!(max_era, "ERA file production complete");
+
+    Ok(())
+}