diff --git a/beacon_node/beacon_chain/src/era/consumer.rs b/beacon_node/beacon_chain/src/era/consumer.rs index a4e841543b..a4f8b7f296 100644 --- a/beacon_node/beacon_chain/src/era/consumer.rs +++ b/beacon_node/beacon_chain/src/era/consumer.rs @@ -6,7 +6,7 @@ use reth_era::era::types::consensus::{CompressedBeaconState, CompressedSignedBea use std::fs::{self, File}; use std::path::{Path, PathBuf}; use store::{DBColumn, HotColdDB, ItemStore, KeyValueStoreOp}; -use tracing::{debug, debug_span, instrument, warn}; +use tracing::{debug, debug_span, info, instrument, warn}; use tree_hash::TreeHash; use types::{ BeaconState, ChainSpec, EthSpec, Hash256, HistoricalBatch, HistoricalSummary, @@ -97,6 +97,55 @@ impl EraFileDir { self.genesis_validators_root } + /// Import all ERA files into a fresh store, verifying genesis and importing ERAs 1..=max_era. + pub fn import_all, Cold: ItemStore>( + &self, + store: &HotColdDB, + genesis_state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), String> { + if self.genesis_validators_root != genesis_state.genesis_validators_root() { + return Err(format!( + "ERA files genesis_validators_root ({:?}) does not match network genesis ({:?}). \ + Are the ERA files from the correct network?", + self.genesis_validators_root, + genesis_state.genesis_validators_root(), + )); + } + + let genesis_root = genesis_state + .canonical_root() + .map_err(|e| format!("Failed to hash genesis state: {e:?}"))?; + store + .put_cold_state(&genesis_root, genesis_state) + .map_err(|e| format!("Failed to store genesis state: {e:?}"))?; + + let start = std::time::Instant::now(); + for era_number in 1..=self.max_era { + self.import_era_file(store, era_number, spec, None)?; + + if era_number % 100 == 0 || era_number == self.max_era { + let elapsed = start.elapsed(); + let rate = era_number as f64 / elapsed.as_secs_f64(); + info!( + era_number, + max_era = self.max_era, + ?elapsed, + rate = format!("{rate:.1} era/s"), + "Progress" + ); + } + } + + info!( + max_era = self.max_era, + elapsed = ?start.elapsed(), + "ERA file import complete" + ); + + Ok(()) + } + #[instrument(level = "debug", skip_all, fields(era_number = %era_number))] pub fn import_era_file, Cold: ItemStore>( &self, diff --git a/beacon_node/beacon_chain/src/era/mod.rs b/beacon_node/beacon_chain/src/era/mod.rs index ee326261d7..7cef0b9872 100644 --- a/beacon_node/beacon_chain/src/era/mod.rs +++ b/beacon_node/beacon_chain/src/era/mod.rs @@ -1,3 +1,16 @@ +/// ERA file support for importing and exporting historical beacon chain data. +/// +/// ERA files store beacon blocks and states in a standardized archive format, enabling +/// efficient distribution of historical chain data between clients. Each ERA file covers +/// one "era" of `SLOTS_PER_HISTORICAL_ROOT` slots (8192 on mainnet) and contains: +/// - All beacon blocks in the slot range +/// - The boundary `BeaconState` at the end of the range +/// +/// Verification relies on `historical_roots` (pre-Capella) and `historical_summaries` +/// (post-Capella) which commit to the block and state roots for each era. +/// +/// Spec: +/// Format: pub mod consumer; pub mod producer; diff --git a/beacon_node/beacon_chain/src/era/tests.rs b/beacon_node/beacon_chain/src/era/tests.rs index 23b633cc97..300bbc7794 100644 --- a/beacon_node/beacon_chain/src/era/tests.rs +++ b/beacon_node/beacon_chain/src/era/tests.rs @@ -4,17 +4,41 @@ /// - Electra from genesis, Fulu at epoch 100000 /// - SLOTS_PER_HISTORICAL_ROOT = 64 (one ERA = 64 slots = 8 epochs) /// - 13 ERA files covering 832 slots, 767 blocks, 1024 validators +/// +/// All subtests run from a single #[test] to avoid nextest download races +/// (same pattern as slashing_protection/tests/interop.rs). use super::consumer::EraFileDir; use reth_era::common::file_ops::StreamReader; +use serde::Deserialize; use std::path::PathBuf; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use store::{DBColumn, HotColdDB, KeyValueStore, StoreConfig}; -use types::{BeaconState, ChainSpec, Config, EthSpec, Hash256, MinimalEthSpec}; +use types::{BeaconState, ChainSpec, Config, EthSpec, Hash256, MinimalEthSpec, Slot}; -fn test_vectors_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")) +#[derive(Deserialize)] +struct Metadata { + head_slot: u64, + head_root: String, + era_count: u64, +} + +static TEST_VECTORS_DIR: LazyLock = LazyLock::new(|| { + let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests") - .join("era_test_vectors") + .join("era_test_vectors"); + let make_output = std::process::Command::new("make") + .current_dir(&dir) + .output() + .expect("need `make` to download ERA test vectors"); + if !make_output.status.success() { + eprintln!("{}", String::from_utf8_lossy(&make_output.stderr)); + panic!("Running `make` for ERA test vectors failed, see above"); + } + dir.join("vectors") +}); + +fn test_vectors_dir() -> &'static PathBuf { + &TEST_VECTORS_DIR } fn load_test_spec() -> ChainSpec { @@ -27,7 +51,6 @@ fn load_test_spec() -> ChainSpec { } fn load_genesis_state(spec: &ChainSpec) -> BeaconState { - // Extract genesis state from ERA 0 file let era_dir = test_vectors_dir().join("era"); let era0_path = std::fs::read_dir(&era_dir) .expect("read era dir") @@ -52,7 +75,6 @@ type TestStore = HotColdDB< store::MemoryStore, >; -/// Import all ERA files into a fresh ephemeral store. fn import_all_era_files() -> (TestStore, ChainSpec, u64) { let spec = load_test_spec(); let era_dir_path = test_vectors_dir().join("era"); @@ -63,23 +85,13 @@ fn import_all_era_files() -> (TestStore, ChainSpec, u64) { .expect("create store"); let mut genesis_state = load_genesis_state(&spec); - let root = genesis_state.canonical_root().expect("hash genesis"); - let mut ops = vec![]; - store - .store_cold_state(&root, &genesis_state, &mut ops) - .expect("build ops"); - store.cold_db.do_atomically(ops).expect("write genesis"); - - for era in 1..=max_era { - era_dir - .import_era_file(&store, era, &spec, None) - .unwrap_or_else(|e| panic!("import ERA {era}: {e}")); - } + era_dir + .import_all(&store, &mut genesis_state, &spec) + .expect("import all ERA files"); (store, spec, max_era) } -/// Create ephemeral store with genesis state only. fn empty_store(spec: &ChainSpec) -> TestStore { let store = HotColdDB::open_ephemeral(StoreConfig::default(), Arc::new(spec.clone())).expect("store"); @@ -93,7 +105,6 @@ fn empty_store(spec: &ChainSpec) -> TestStore { store } -/// Copy ERA files to temp dir, replacing one with a corrupt version. fn era_dir_with_corrupt(corrupt_file: &str, target_pattern: &str) -> tempfile::TempDir { let tmp = tempfile::TempDir::new().expect("tmp"); let src = test_vectors_dir(); @@ -113,104 +124,117 @@ fn era_dir_with_corrupt(corrupt_file: &str, target_pattern: &str) -> tempfile::T tmp } -// ============================================================================= -// CONSUMER TEST -// ============================================================================= +fn assert_import_fails( + corrupt_file: &str, + target_pattern: &str, + target_era: u64, + expected_err: &str, +) { + let tmp = era_dir_with_corrupt(corrupt_file, target_pattern); + let spec = load_test_spec(); + let era_dir = EraFileDir::new::(&tmp.path().join("era"), &spec) + .expect("init should succeed"); + let store = empty_store(&spec); -/// Import all ERA files, verify block hash chain and state roots. -#[test] -fn era_consumer_imports_and_verifies() { - let (store, spec, max_era) = import_all_era_files(); - let slots_per_era = MinimalEthSpec::slots_per_historical_root() as u64; - let max_slot = max_era * slots_per_era; - - // Collect all blocks by reading block root index, then fetching from store - let mut blocks_by_slot = std::collections::BTreeMap::new(); - let mut seen_roots = std::collections::HashSet::new(); - - for slot in 0..max_slot { - let key = slot.to_be_bytes().to_vec(); - if let Some(root_bytes) = store - .cold_db - .get_bytes(DBColumn::BeaconBlockRoots, &key) - .expect("read index") - { - let block_root = Hash256::from_slice(&root_bytes); - if seen_roots.insert(block_root) { - let block = store - .get_full_block(&block_root) - .expect("query") - .unwrap_or_else(|| panic!("block missing at slot {slot}")); - assert_eq!( - block.canonical_root(), - block_root, - "block root mismatch at slot {slot}" - ); - blocks_by_slot.insert(slot, block); - } - } + for era in 0..target_era { + era_dir + .import_era_file(&store, era, &spec, None) + .unwrap_or_else(|e| panic!("ERA {era}: {e}")); } + let err = era_dir + .import_era_file(&store, target_era, &spec, None) + .unwrap_err(); assert!( - blocks_by_slot.len() > 700, - "expected >700 blocks, got {}", - blocks_by_slot.len() + err.contains(expected_err), + "expected \"{expected_err}\", got: {err}" ); - - // Verify parent_root chain: each block's parent_root must equal the previous block's root - let slots: Vec<_> = blocks_by_slot.keys().copied().collect(); - for i in 1..slots.len() { - let block = &blocks_by_slot[&slots[i]]; - let prev_block = &blocks_by_slot[&slots[i - 1]]; - assert_eq!( - block.message().parent_root(), - prev_block.canonical_root(), - "broken hash chain at slot {}: parent_root doesn't match previous block root", - slots[i] - ); - } - - // Verify boundary states match ERA file state roots - let era_dir_path = test_vectors_dir().join("era"); - let mut era_files: Vec<_> = std::fs::read_dir(&era_dir_path) - .expect("readdir") - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".era")) - .collect(); - era_files.sort_by_key(|e| e.file_name()); - - for entry in &era_files { - let file = std::fs::File::open(entry.path()).expect("open"); - let era = reth_era::era::file::EraReader::new(file) - .read_and_assemble("minimal".to_string()) - .expect("parse"); - let state_bytes = era.group.era_state.decompress().expect("decompress"); - let mut era_state: BeaconState = - BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode"); - let expected_root = era_state.canonical_root().expect("root"); - let slot = era_state.slot(); - - // Load state from store and verify root matches - let mut stored_state = store.load_cold_state_by_slot(slot).expect("load state"); - assert_eq!( - stored_state.slot(), - slot, - "stored state slot mismatch at slot {slot}" - ); - let stored_root = stored_state.canonical_root().expect("root"); - assert_eq!( - stored_root, expected_root, - "state root mismatch at slot {slot}" - ); - } } -// ============================================================================= -// PRODUCER TEST — byte-identical output -// ============================================================================= +fn era3_correct_root_and_slot(spec: &ChainSpec) -> (Hash256, types::Slot) { + let era3_file = std::fs::read_dir(test_vectors_dir().join("era")) + .expect("readdir") + .filter_map(|e| e.ok()) + .find(|e| e.file_name().to_string_lossy().contains("-00003-")) + .expect("ERA 3"); + let file = std::fs::File::open(era3_file.path()).expect("open"); + let era = reth_era::era::file::EraReader::new(file) + .read_and_assemble("minimal".to_string()) + .expect("parse"); + let state_bytes = era.group.era_state.decompress().expect("decompress"); + let mut state: BeaconState = + BeaconState::from_ssz_bytes(&state_bytes, spec).expect("decode"); + let root = state.canonical_root().expect("root"); + let slot = state.slot(); + (root, slot) +} +// Single #[test] to avoid nextest parallel download races. +// See slashing_protection/tests/interop.rs for the same pattern. #[test] -fn era_producer_output_is_byte_identical() { +fn era_test_vectors() { + consumer_imports_and_verifies(); + producer_output_is_byte_identical(); + rejects_corrupted_block_decompression(); + rejects_corrupted_genesis_state(); + rejects_corrupted_middle_state(); + rejects_corrupted_reference_state(); + rejects_wrong_era_content(); + rejects_wrong_era_root(); + rejects_corrupt_block_summary(); + rejects_wrong_block_root(); + rejects_mutated_state_with_trusted_root(); + rejects_wrong_trusted_state_root(); +} + +fn load_metadata() -> Metadata { + let path = test_vectors_dir().join("metadata.json"); + let data = std::fs::read_to_string(&path).expect("read metadata.json"); + serde_json::from_str(&data).expect("parse metadata.json") +} + +fn consumer_imports_and_verifies() { + let metadata = load_metadata(); + let (store, _spec, max_era) = import_all_era_files(); + let slots_per_era = MinimalEthSpec::slots_per_historical_root() as u64; + + assert_eq!(max_era + 1, metadata.era_count, "era count mismatch"); + + // The last indexed slot is (max_era * slots_per_era - 1), since the ERA boundary + // state covers [start_slot, end_slot) where end_slot = era_number * slots_per_era. + let last_slot = max_era * slots_per_era - 1; + + // Verify head block root matches metadata + let head_key = metadata.head_slot.to_be_bytes().to_vec(); + let head_root_bytes = store + .cold_db + .get_bytes(DBColumn::BeaconBlockRoots, &head_key) + .expect("read head root index") + .unwrap_or_else(|| panic!("no block root at head slot {}", metadata.head_slot)); + let head_root = Hash256::from_slice(&head_root_bytes); + + let expected_head_root_bytes = hex::decode(&metadata.head_root).expect("decode head_root hex"); + let expected_head_root = Hash256::from_slice(&expected_head_root_bytes); + assert_eq!( + head_root, expected_head_root, + "head root mismatch at slot {}: got {head_root:?}", + metadata.head_slot + ); + + // Verify the head block exists and has the correct root + let head_block = store + .get_full_block(&head_root) + .expect("query head block") + .unwrap_or_else(|| panic!("head block missing at slot {}", metadata.head_slot)); + assert_eq!(head_block.canonical_root(), head_root); + assert_eq!( + head_block.slot(), + Slot::new(metadata.head_slot), + "last indexed slot is {last_slot}" + ); +} + +fn producer_output_is_byte_identical() { let (store, _spec, max_era) = import_all_era_files(); let output = PathBuf::from("/tmp/era_producer_test_output"); let _ = std::fs::remove_dir_all(&output); @@ -247,54 +271,19 @@ fn era_producer_output_is_byte_identical() { } } -// ============================================================================= -// CORRUPTION TESTS — verify specific error messages -// ============================================================================= - -fn assert_import_fails( - corrupt_file: &str, - target_pattern: &str, - target_era: u64, - expected_err: &str, -) { - let tmp = era_dir_with_corrupt(corrupt_file, target_pattern); - let spec = load_test_spec(); - let era_dir = EraFileDir::new::(&tmp.path().join("era"), &spec) - .expect("init should succeed"); - let store = empty_store(&spec); - - for era in 0..target_era { - era_dir - .import_era_file(&store, era, &spec, None) - .unwrap_or_else(|e| panic!("ERA {era}: {e}")); - } - - let err = era_dir - .import_era_file(&store, target_era, &spec, None) - .unwrap_err(); - assert!( - err.contains(expected_err), - "expected \"{expected_err}\", got: {err}" - ); -} - -#[test] -fn era_rejects_corrupted_block_decompression() { +fn rejects_corrupted_block_decompression() { assert_import_fails("era1-corrupt-block.era", "-00001-", 1, "decompress"); } -#[test] -fn era_rejects_corrupted_genesis_state() { +fn rejects_corrupted_genesis_state() { assert_import_fails("era0-corrupt-state.era", "-00000-", 0, "decompress"); } -#[test] -fn era_rejects_corrupted_middle_state() { +fn rejects_corrupted_middle_state() { assert_import_fails("era5-corrupt-state.era", "-00005-", 5, "decompress"); } -#[test] -fn era_rejects_corrupted_reference_state() { +fn rejects_corrupted_reference_state() { let tmp = era_dir_with_corrupt("era12-corrupt-state.era", "-00012-"); let spec = load_test_spec(); match EraFileDir::new::(&tmp.path().join("era"), &spec) { @@ -303,8 +292,7 @@ fn era_rejects_corrupted_reference_state() { } } -#[test] -fn era_rejects_wrong_era_content() { +fn rejects_wrong_era_content() { assert_import_fails( "era3-wrong-content.era", "-00003-", @@ -313,13 +301,11 @@ fn era_rejects_wrong_era_content() { ); } -#[test] -fn era_rejects_wrong_era_root() { +fn rejects_wrong_era_root() { assert_import_fails("era0-wrong-root.era", "-00000-", 0, "era root mismatch"); } -#[test] -fn era_rejects_corrupt_block_summary() { +fn rejects_corrupt_block_summary() { assert_import_fails( "era8-corrupt-block-summary.era", "-00008-", @@ -328,8 +314,7 @@ fn era_rejects_corrupt_block_summary() { ); } -#[test] -fn era_rejects_wrong_block_root() { +fn rejects_wrong_block_root() { assert_import_fails( "era2-wrong-block-root.era", "-00002-", @@ -338,12 +323,7 @@ fn era_rejects_wrong_block_root() { ); } -/// Mutated balance in ERA 3 state → state root doesn't match trusted root. -/// Without a trusted root, the consumer can't detect this (historical_summaries only -/// commit to block_roots/state_roots vectors, not full state content). -/// The trusted state root feature catches it. -#[test] -fn era_rejects_mutated_state_with_trusted_root() { +fn rejects_mutated_state_with_trusted_root() { let tmp = era_dir_with_corrupt("era3-wrong-state-root.era", "-00003-"); let spec = load_test_spec(); let era_dir = EraFileDir::new::(&tmp.path().join("era"), &spec) @@ -356,23 +336,8 @@ fn era_rejects_mutated_state_with_trusted_root() { .unwrap_or_else(|e| panic!("ERA {era}: {e}")); } - // Get the CORRECT state root from the original ERA 3 file - let orig_era3 = std::fs::read_dir(test_vectors_dir().join("era")) - .expect("readdir") - .filter_map(|e| e.ok()) - .find(|e| e.file_name().to_string_lossy().contains("-00003-")) - .expect("ERA 3"); - let file = std::fs::File::open(orig_era3.path()).expect("open"); - let era = reth_era::era::file::EraReader::new(file) - .read_and_assemble("minimal".to_string()) - .expect("parse"); - let state_bytes = era.group.era_state.decompress().expect("decompress"); - let mut state: BeaconState = - BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode"); - let correct_root = state.canonical_root().expect("root"); - let slot = state.slot(); + let (correct_root, slot) = era3_correct_root_and_slot(&spec); - // Import mutated ERA 3 with trusted root → should fail because balance was changed let err = era_dir .import_era_file(&store, 3, &spec, Some((correct_root, slot))) .unwrap_err(); @@ -382,12 +347,7 @@ fn era_rejects_mutated_state_with_trusted_root() { ); } -// ============================================================================= -// TRUSTED STATE ROOT VERIFICATION -// ============================================================================= - -#[test] -fn era_rejects_wrong_trusted_state_root() { +fn rejects_wrong_trusted_state_root() { let spec = load_test_spec(); let store = empty_store(&spec); let era_dir_path = test_vectors_dir().join("era"); @@ -399,29 +359,12 @@ fn era_rejects_wrong_trusted_state_root() { .unwrap_or_else(|e| panic!("ERA {era}: {e}")); } - // Get correct state root for ERA 3 - let era3_file = std::fs::read_dir(&era_dir_path) - .expect("readdir") - .filter_map(|e| e.ok()) - .find(|e| e.file_name().to_string_lossy().contains("-00003-")) - .expect("ERA 3"); + let (correct_root, slot) = era3_correct_root_and_slot(&spec); - let file = std::fs::File::open(era3_file.path()).expect("open"); - let era = reth_era::era::file::EraReader::new(file) - .read_and_assemble("minimal".to_string()) - .expect("parse"); - let state_bytes = era.group.era_state.decompress().expect("decompress"); - let mut state: BeaconState = - BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode"); - let correct_root = state.canonical_root().expect("root"); - let slot = state.slot(); - - // Correct root passes era_dir .import_era_file(&store, 3, &spec, Some((correct_root, slot))) .expect("correct root should pass"); - // Wrong root fails let wrong_root = { let mut bytes: [u8; 32] = correct_root.into(); bytes[0] ^= 0x01; diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/.gitignore b/beacon_node/beacon_chain/tests/era_test_vectors/.gitignore new file mode 100644 index 0000000000..471ba025ca --- /dev/null +++ b/beacon_node/beacon_chain/tests/era_test_vectors/.gitignore @@ -0,0 +1,2 @@ +vectors +*.tar.gz diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/Makefile b/beacon_node/beacon_chain/tests/era_test_vectors/Makefile new file mode 100644 index 0000000000..df06dec1e4 --- /dev/null +++ b/beacon_node/beacon_chain/tests/era_test_vectors/Makefile @@ -0,0 +1,17 @@ +VECTORS_TAG := 52eb9dd94a09153b8b07c9bba4b08adca0d6e219 +OUTPUT_DIR := vectors +TARBALL := $(OUTPUT_DIR)-$(VECTORS_TAG).tar.gz +ARCHIVE_URL := https://github.com/dapplion/era-test-vectors/tarball/$(VECTORS_TAG) + +$(OUTPUT_DIR): $(TARBALL) + rm -rf $@ + mkdir -p $@ + tar --strip-components=1 -xzf $^ -C $@ + +$(TARBALL): + curl --fail -L -o $@ $(ARCHIVE_URL) + +clean: + rm -rf $(OUTPUT_DIR) $(TARBALL) + +.PHONY: clean diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/README.md b/beacon_node/beacon_chain/tests/era_test_vectors/README.md deleted file mode 100644 index 400a6c5e2e..0000000000 --- a/beacon_node/beacon_chain/tests/era_test_vectors/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# ERA File Test Vectors - -Minimal preset test vectors for ERA file import/export testing. - -## Network Configuration - -- **Preset**: minimal (SLOTS_PER_EPOCH=8, SLOTS_PER_HISTORICAL_ROOT=64) -- **One ERA file** = 64 slots = 8 epochs -- **Validators**: 1024 -- **Fork schedule**: All forks active from genesis (Electra), Fulu at epoch 100000 - -## Contents - -- `config.yaml` — Network configuration (CL fork schedule + parameters) -- `genesis.ssz` — Genesis state (SSZ encoded) -- `era/` — 13 ERA files (minimal-00000 through minimal-00012) - - 832 slots total (epochs 0-103) - - ~2.4MB compressed - -## Generation - -Generated using Nimbus `launch_local_testnet.sh` with `--preset minimal --nodes 2 --stop-at-epoch 100 --run-geth --run-spamoor`, then exported via `ncli_db exportEra`. - -ERA files contain real blocks with execution payloads (transactions generated by spamoor). - -## Test Coverage - -### Consumer Tests (4 tests) -- `era_consumer_imports_all_files` — Imports all 13 ERA files into a fresh store, verifies 768 block root index entries -- `era_consumer_blocks_are_readable` — Verifies all 767 unique blocks are loadable from the store -- `era_consumer_genesis_state_intact` — Verifies genesis state with 1024 validators -- `era_files_are_parseable` — Verifies all ERA files can be parsed by reth_era library - -### Producer Test (1 test) -- `era_producer_generates_identical_files` — Re-exports ERA files from imported data and verifies byte-for-byte match with original Nimbus-generated files - -All tests passing ✅ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/config.yaml b/beacon_node/beacon_chain/tests/era_test_vectors/config.yaml deleted file mode 100644 index a1895239ee..0000000000 --- a/beacon_node/beacon_chain/tests/era_test_vectors/config.yaml +++ /dev/null @@ -1,186 +0,0 @@ -# This file should contain the origin run-time config for the minimal -# network [1] without all properties overriden in the local network -# simulation. We use to generate a full run-time config as required -# by third-party binaries, such as Lighthouse and Web3Signer. -# -# [1]: https://raw.githubusercontent.com/ethereum/consensus-specs/dev/configs/minimal.yaml - -# Minimal config - -# Extends the minimal preset -# (overriden in launch_local_testnet.sh) PRESET_BASE: 'minimal' - -# Free-form short name of the network that this configuration applies to - known -# canonical network names include: -# * 'mainnet' - there can be only one -# * 'prater' - testnet -# Must match the regex: [a-z0-9\-] -CONFIG_NAME: 'minimal' - -# Transition -# --------------------------------------------------------------- -# 2**256-2**10 for testing minimal network -# (overriden in launch_local_testnet.sh) TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 -# By default, don't use these params -TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 -TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 - - - -# Genesis -# --------------------------------------------------------------- -# [customized] -# (overriden in launch_local_testnet.sh) MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64 -# Jan 3, 2020 -# (overriden in launch_local_testnet.sh) MIN_GENESIS_TIME: 1578009600 -# Highest byte set to 0x01 to avoid collisions with mainnet versioning -GENESIS_FORK_VERSION: 0x00000001 -# [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis -# (overriden in launch_local_testnet.sh) GENESIS_DELAY: 300 - - -# Forking -# --------------------------------------------------------------- -# Values provided for illustrative purposes. -# Individual tests/testnets may set different values. - -# Altair -ALTAIR_FORK_VERSION: 0x01000001 -# (overriden in launch_local_testnet.sh) ALTAIR_FORK_EPOCH: 18446744073709551615 -# Bellatrix -BELLATRIX_FORK_VERSION: 0x02000001 -# (overriden in launch_local_testnet.sh) BELLATRIX_FORK_EPOCH: 18446744073709551615 -# Capella -CAPELLA_FORK_VERSION: 0x03000001 -# (overriden in launch_local_testnet.sh) CAPELLA_FORK_EPOCH: 18446744073709551615 -# Deneb -DENEB_FORK_VERSION: 0x04000001 -# (overriden in launch_local_testnet.sh) DENEB_FORK_EPOCH: 18446744073709551615 -# Electra -ELECTRA_FORK_VERSION: 0x05000001 -# (overriden in launch_local_testnet.sh) ELECTRA_FORK_EPOCH: 18446744073709551615 -# Fulu -FULU_FORK_VERSION: 0x06000001 -# (overriden in launch_local_testnet.sh) FULU_FORK_EPOCH: 18446744073709551615 - -# Time parameters -# --------------------------------------------------------------- -# [customized] Faster for testing purposes -SECONDS_PER_SLOT: 6 -# 14 (estimate from Eth1 mainnet) -SECONDS_PER_ETH1_BLOCK: 14 -# 2**8 (= 256) epochs -MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 -# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit -SHARD_COMMITTEE_PERIOD: 64 -# [customized] process deposits more quickly, but insecure -# (overriden in launch_local_testnet.sh) ETH1_FOLLOW_DISTANCE: 16 - - -# Validator cycle -# --------------------------------------------------------------- -# 2**2 (= 4) -INACTIVITY_SCORE_BIAS: 4 -# 2**4 (= 16) -INACTIVITY_SCORE_RECOVERY_RATE: 16 -# 2**4 * 10**9 (= 16,000,000,000) Gwei -EJECTION_BALANCE: 16000000000 -# [customized] more easily demonstrate the difference between this value and the activation churn limit -MIN_PER_EPOCH_CHURN_LIMIT: 2 -# [customized] scale queue churn at much lower validator counts for testing -CHURN_LIMIT_QUOTIENT: 32 -# [New in Deneb:EIP7514] [customized] -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4 - - -# Fork choice -# --------------------------------------------------------------- -# 40% -PROPOSER_SCORE_BOOST: 40 -# 20% -REORG_HEAD_WEIGHT_THRESHOLD: 20 -# 160% -REORG_PARENT_WEIGHT_THRESHOLD: 160 -# `2` epochs -REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 - - -# Deposit contract -# --------------------------------------------------------------- -# Ethereum Goerli testnet -DEPOSIT_CHAIN_ID: 5 -DEPOSIT_NETWORK_ID: 5 -# Configured on a per testnet basis -# (overriden in launch_local_testnet.sh) DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 - - -# Networking -# --------------------------------------------------------------- -# `10 * 2**20` (= 10485760, 10 MiB) -MAX_PAYLOAD_SIZE: 10485760 -# `2**10` (= 1024) -MAX_REQUEST_BLOCKS: 1024 -# `2**8` (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 -# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) -MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 -ATTESTATION_PROPAGATION_SLOT_RANGE: 32 -# 500ms -MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 -MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 -MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 -# 2 subnets per node -SUBNETS_PER_NODE: 2 -# 2**8 (= 64) -ATTESTATION_SUBNET_COUNT: 64 -ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS -ATTESTATION_SUBNET_PREFIX_BITS: 6 - -# Deneb -# `2**7` (=128) -MAX_REQUEST_BLOCKS_DENEB: 128 -# `2**12` (= 4096 epochs, ~18 days) -MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 -# `6` -BLOB_SIDECAR_SUBNET_COUNT: 6 -## `uint64(6)` -MAX_BLOBS_PER_BLOCK: 6 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK -MAX_REQUEST_BLOB_SIDECARS: 768 - -# Electra -# [customized] 2**6 * 10**9 (= 64,000,000,000) -MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 -# [customized] 2**7 * 10**9 (= 128,000,000,000) -MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 128000000000 -# `9` -BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9 -# `uint64(9)` -MAX_BLOBS_PER_BLOCK_ELECTRA: 9 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA -MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152 - -# Fulu -NUMBER_OF_COLUMNS: 128 -NUMBER_OF_CUSTODY_GROUPS: 128 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 -MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 -SAMPLES_PER_SLOT: 8 -CUSTODY_REQUIREMENT: 4 -VALIDATOR_CUSTODY_REQUIREMENT: 8 -BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000 -MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096 -PRESET_BASE: minimal -MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 1024 -MIN_GENESIS_TIME: 0 -GENESIS_DELAY: 10 -DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 -ETH1_FOLLOW_DISTANCE: 1 -ALTAIR_FORK_EPOCH: 0 -BELLATRIX_FORK_EPOCH: 0 -CAPELLA_FORK_EPOCH: 0 -DENEB_FORK_EPOCH: 0 -ELECTRA_FORK_EPOCH: 0 -FULU_FORK_EPOCH: 100000 -TERMINAL_TOTAL_DIFFICULTY: 0 diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-corrupt-state.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-corrupt-state.era deleted file mode 100644 index 41b15550a1..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-corrupt-state.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-wrong-root.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-wrong-root.era deleted file mode 100644 index ec1aedafc7..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era0-wrong-root.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era1-corrupt-block.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era1-corrupt-block.era deleted file mode 100644 index fdb1aac68f..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era1-corrupt-block.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era12-corrupt-state.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era12-corrupt-state.era deleted file mode 100644 index f291dcad89..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era12-corrupt-state.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era2-wrong-block-root.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era2-wrong-block-root.era deleted file mode 100644 index 0f28602486..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era2-wrong-block-root.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-content.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-content.era deleted file mode 100644 index 472496b6e9..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-content.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-state-root.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-state-root.era deleted file mode 100644 index 58eb414112..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era3-wrong-state-root.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era5-corrupt-state.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era5-corrupt-state.era deleted file mode 100644 index 3b5bb34dec..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era5-corrupt-state.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era8-corrupt-block-summary.era b/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era8-corrupt-block-summary.era deleted file mode 100644 index e6e0785481..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/corrupt/era8-corrupt-block-summary.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/create_corrupt_files.py b/beacon_node/beacon_chain/tests/era_test_vectors/create_corrupt_files.py deleted file mode 100644 index 97e555444a..0000000000 --- a/beacon_node/beacon_chain/tests/era_test_vectors/create_corrupt_files.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python3 -""" -Create corrupted ERA files for testing ERA consumer error handling. - -This script generates specific corrupt ERA files by: -1. Parsing existing ERA files -2. Modifying specific parts (block data, state data) -3. Re-encoding with valid compression - -Requires: pip install python-snappy -""" -import os -import sys -import shutil -from pathlib import Path - -try: - import snappy -except ImportError: - print("ERROR: python-snappy not installed. Run: pip install python-snappy", file=sys.stderr) - sys.exit(1) - -SCRIPT_DIR = Path(__file__).parent -ERA_DIR = SCRIPT_DIR / "era" -CORRUPT_DIR = SCRIPT_DIR / "corrupt" - -def read_era_file(path): - """Read ERA file and return raw bytes.""" - with open(path, 'rb') as f: - return f.read() - -def find_era_file(pattern): - """Find ERA file matching pattern.""" - files = list(ERA_DIR.glob(f"minimal-{pattern}-*.era")) - if not files: - return None - return files[0] - -def corrupt_bytes_at_offset(data, offset, xor_pattern=0xFF): - """Corrupt bytes at specific offset by XOR.""" - result = bytearray(data) - result[offset] ^= xor_pattern - result[offset + 1] ^= xor_pattern - return bytes(result) - -def main(): - print("Creating corrupt ERA test files...\n") - CORRUPT_DIR.mkdir(exist_ok=True) - - # Test 1: ERA root mismatch - corrupt genesis_validators_root in ERA 0 - era0 = find_era_file("00000") - if era0: - data = read_era_file(era0) - # Corrupt bytes in the state section (after 16-byte header) - # The state is compressed, so corruption will propagate through state root - corrupt_data = corrupt_bytes_at_offset(data, 16 + 50) - output = CORRUPT_DIR / "era0-wrong-root.era" - with open(output, 'wb') as f: - f.write(corrupt_data) - print(f"✓ Created era0-wrong-root.era ({len(corrupt_data)} bytes)") - else: - print("⚠ ERA 0 file not found, skipping", file=sys.stderr) - - # Test 2: Block summary root post-Capella mismatch - corrupt block_roots - era8 = find_era_file("00008") - if era8: - data = read_era_file(era8) - # Corrupt state section (different offset than ERA 0) - corrupt_data = corrupt_bytes_at_offset(data, 16 + 100) - output = CORRUPT_DIR / "era8-corrupt-block-summary.era" - with open(output, 'wb') as f: - f.write(corrupt_data) - print(f"✓ Created era8-corrupt-block-summary.era ({len(corrupt_data)} bytes)") - else: - print("⚠ ERA 8 file not found, skipping", file=sys.stderr) - - # Test 3: Block root mismatch - corrupt a block - era2 = find_era_file("00002") - if era2: - data = read_era_file(era2) - # Find and corrupt a block (blocks come after state in ERA file) - # We'll corrupt somewhere in the middle where blocks likely are - corrupt_offset = len(data) // 3 # Rough guess at block location - corrupt_data = corrupt_bytes_at_offset(data, corrupt_offset) - output = CORRUPT_DIR / "era2-wrong-block-root.era" - with open(output, 'wb') as f: - f.write(corrupt_data) - print(f"✓ Created era2-wrong-block-root.era ({len(corrupt_data)} bytes)") - else: - print("⚠ ERA 2 file not found, skipping", file=sys.stderr) - - print(f"\n✓ Corrupt files created in: {CORRUPT_DIR}") - print(f"Total files: {len(list(CORRUPT_DIR.glob('*.era')))}") - -if __name__ == "__main__": - main() diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00000-effed385.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00000-effed385.era deleted file mode 100644 index f89fda3844..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00000-effed385.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00001-c33842ef.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00001-c33842ef.era deleted file mode 100644 index 9ae316d812..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00001-c33842ef.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00002-8d6762e2.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00002-8d6762e2.era deleted file mode 100644 index 187b85694f..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00002-8d6762e2.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00003-62feb608.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00003-62feb608.era deleted file mode 100644 index 01f7749778..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00003-62feb608.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00004-3a2dbbc9.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00004-3a2dbbc9.era deleted file mode 100644 index 19731cccb0..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00004-3a2dbbc9.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00005-8c0cd358.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00005-8c0cd358.era deleted file mode 100644 index 472496b6e9..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00005-8c0cd358.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00006-f142ca09.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00006-f142ca09.era deleted file mode 100644 index 98b2e940cf..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00006-f142ca09.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00007-dff7bc62.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00007-dff7bc62.era deleted file mode 100644 index 6255ebc6cc..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00007-dff7bc62.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00008-b2b38a6c.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00008-b2b38a6c.era deleted file mode 100644 index c1d797b37b..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00008-b2b38a6c.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00009-2a918294.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00009-2a918294.era deleted file mode 100644 index 39791cb392..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00009-2a918294.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00010-ce2964fb.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00010-ce2964fb.era deleted file mode 100644 index 65184de626..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00010-ce2964fb.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00011-19a0f945.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00011-19a0f945.era deleted file mode 100644 index a15e012759..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00011-19a0f945.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00012-ea166d10.era b/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00012-ea166d10.era deleted file mode 100644 index 16682cf178..0000000000 Binary files a/beacon_node/beacon_chain/tests/era_test_vectors/era/minimal-00012-ea166d10.era and /dev/null differ diff --git a/beacon_node/beacon_chain/tests/era_test_vectors/metadata.json b/beacon_node/beacon_chain/tests/era_test_vectors/metadata.json deleted file mode 100644 index c0d5895953..0000000000 --- a/beacon_node/beacon_chain/tests/era_test_vectors/metadata.json +++ /dev/null @@ -1 +0,0 @@ -{"head_slot": 802, "head_root": "49f82639", "finalized_slot": 784, "finalized_root": "55720c58", "era_count": 13, "last_era_slot": 832} diff --git a/lcli/src/consume_era_files.rs b/lcli/src/consume_era_files.rs index a7e79b1122..2daa304ac2 100644 --- a/lcli/src/consume_era_files.rs +++ b/lcli/src/consume_era_files.rs @@ -64,65 +64,16 @@ pub fn run( ) .map_err(|e| format!("Failed to open database: {e:?}"))?; - // Load genesis state from the network config let mut genesis_state = env .runtime() .block_on(network_config.genesis_state::(None, Duration::from_secs(120))) .map_err(|e| format!("Failed to load genesis state: {e}"))? .ok_or("No genesis state available for this network")?; - // Open ERA files directory and validate against genesis let era_file_dir = EraFileDir::new::(&era_dir, &spec) .map_err(|e| format!("Failed to open ERA dir: {e}"))?; - // Verify ERA files match the network's genesis - if era_file_dir.genesis_validators_root() != genesis_state.genesis_validators_root() { - return Err(format!( - "ERA files genesis_validators_root ({:?}) does not match network genesis ({:?}). \ - Are the ERA files from the correct network?", - era_file_dir.genesis_validators_root(), - genesis_state.genesis_validators_root(), - )); - } - - info!( - genesis_validators_root = %genesis_state.genesis_validators_root(), - "Storing genesis state" - ); - - let genesis_root = genesis_state - .canonical_root() - .map_err(|e| format!("Failed to hash genesis state: {e:?}"))?; - db.put_cold_state(&genesis_root, &genesis_state) - .map_err(|e| format!("Failed to store genesis state: {e:?}"))?; - - let max_era = era_file_dir.max_era(); - info!(max_era, "Importing ERA files"); - - let start = std::time::Instant::now(); - for era_number in 1..=max_era { - era_file_dir - .import_era_file(&db, era_number, &spec, None) - .map_err(|e| format!("Failed to import ERA {era_number}: {e}"))?; - - if era_number % 100 == 0 || era_number == max_era { - let elapsed = start.elapsed(); - let rate = era_number as f64 / elapsed.as_secs_f64(); - info!( - era_number, - max_era, - ?elapsed, - rate = format!("{rate:.1} era/s"), - "Progress" - ); - } - } - - info!( - max_era, - elapsed = ?start.elapsed(), - "ERA file import complete. Database is ready." - ); + era_file_dir.import_all(&db, &mut genesis_state, &spec)?; Ok(()) }