Move ERA test vectors to external repo, download at test time

Test vectors are now hosted at dapplion/era-test-vectors and downloaded
via Makefile (same pattern as slashing_protection interchange tests).

beacon_node/beacon_chain/src/era/consumer.rs

@@ -6,7 +6,7 @@ use reth_era::era::types::consensus::{CompressedBeaconState, CompressedSignedBea
 use std::fs::{self, File};
 use std::path::{Path, PathBuf};
 use store::{DBColumn, HotColdDB, ItemStore, KeyValueStoreOp};
-use tracing::{debug, debug_span, instrument, warn};
+use tracing::{debug, debug_span, info, instrument, warn};
 use tree_hash::TreeHash;
 use types::{
     BeaconState, ChainSpec, EthSpec, Hash256, HistoricalBatch, HistoricalSummary,
@@ -97,6 +97,55 @@ impl EraFileDir {
         self.genesis_validators_root
     }
 
+    /// Import all ERA files into a fresh store, verifying genesis and importing ERAs 1..=max_era.
+    pub fn import_all<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
+        &self,
+        store: &HotColdDB<E, Hot, Cold>,
+        genesis_state: &mut BeaconState<E>,
+        spec: &ChainSpec,
+    ) -> Result<(), String> {
+        if self.genesis_validators_root != genesis_state.genesis_validators_root() {
+            return Err(format!(
+                "ERA files genesis_validators_root ({:?}) does not match network genesis ({:?}). \
+                 Are the ERA files from the correct network?",
+                self.genesis_validators_root,
+                genesis_state.genesis_validators_root(),
+            ));
+        }
+
+        let genesis_root = genesis_state
+            .canonical_root()
+            .map_err(|e| format!("Failed to hash genesis state: {e:?}"))?;
+        store
+            .put_cold_state(&genesis_root, genesis_state)
+            .map_err(|e| format!("Failed to store genesis state: {e:?}"))?;
+
+        let start = std::time::Instant::now();
+        for era_number in 1..=self.max_era {
+            self.import_era_file(store, era_number, spec, None)?;
+
+            if era_number % 100 == 0 || era_number == self.max_era {
+                let elapsed = start.elapsed();
+                let rate = era_number as f64 / elapsed.as_secs_f64();
+                info!(
+                    era_number,
+                    max_era = self.max_era,
+                    ?elapsed,
+                    rate = format!("{rate:.1} era/s"),
+                    "Progress"
+                );
+            }
+        }
+
+        info!(
+            max_era = self.max_era,
+            elapsed = ?start.elapsed(),
+            "ERA file import complete"
+        );
+
+        Ok(())
+    }
+
     #[instrument(level = "debug", skip_all, fields(era_number = %era_number))]
     pub fn import_era_file<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>(
         &self,

beacon_node/beacon_chain/src/era/mod.rs

@@ -1,3 +1,16 @@
+/// ERA file support for importing and exporting historical beacon chain data.
+///
+/// ERA files store beacon blocks and states in a standardized archive format, enabling
+/// efficient distribution of historical chain data between clients. Each ERA file covers
+/// one "era" of `SLOTS_PER_HISTORICAL_ROOT` slots (8192 on mainnet) and contains:
+/// - All beacon blocks in the slot range
+/// - The boundary `BeaconState` at the end of the range
+///
+/// Verification relies on `historical_roots` (pre-Capella) and `historical_summaries`
+/// (post-Capella) which commit to the block and state roots for each era.
+///
+/// Spec: <https://github.com/ethereum/consensus-specs/blob/dev/specs/phase0/beacon-chain.md#historical-roots-updates>
+/// Format: <https://github.com/status-im/nimbus-eth2/blob/stable/docs/the_auditors_handbook/src/02.4_the_era_file_format.md>
 pub mod consumer;
 pub mod producer;
 

beacon_node/beacon_chain/src/era/tests.rs

@@ -4,17 +4,41 @@
 /// - Electra from genesis, Fulu at epoch 100000
 /// - SLOTS_PER_HISTORICAL_ROOT = 64 (one ERA = 64 slots = 8 epochs)
 /// - 13 ERA files covering 832 slots, 767 blocks, 1024 validators
+///
+/// All subtests run from a single #[test] to avoid nextest download races
+/// (same pattern as slashing_protection/tests/interop.rs).
 use super::consumer::EraFileDir;
 use reth_era::common::file_ops::StreamReader;
+use serde::Deserialize;
 use std::path::PathBuf;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 use store::{DBColumn, HotColdDB, KeyValueStore, StoreConfig};
-use types::{BeaconState, ChainSpec, Config, EthSpec, Hash256, MinimalEthSpec};
+use types::{BeaconState, ChainSpec, Config, EthSpec, Hash256, MinimalEthSpec, Slot};
 
-fn test_vectors_dir() -> PathBuf {
-    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+#[derive(Deserialize)]
+struct Metadata {
+    head_slot: u64,
+    head_root: String,
+    era_count: u64,
+}
+
+static TEST_VECTORS_DIR: LazyLock<PathBuf> = LazyLock::new(|| {
+    let dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
         .join("tests")
-        .join("era_test_vectors")
+        .join("era_test_vectors");
+    let make_output = std::process::Command::new("make")
+        .current_dir(&dir)
+        .output()
+        .expect("need `make` to download ERA test vectors");
+    if !make_output.status.success() {
+        eprintln!("{}", String::from_utf8_lossy(&make_output.stderr));
+        panic!("Running `make` for ERA test vectors failed, see above");
+    }
+    dir.join("vectors")
+});
+
+fn test_vectors_dir() -> &'static PathBuf {
+    &TEST_VECTORS_DIR
 }
 
 fn load_test_spec() -> ChainSpec {
@@ -27,7 +51,6 @@ fn load_test_spec() -> ChainSpec {
 }
 
 fn load_genesis_state(spec: &ChainSpec) -> BeaconState<MinimalEthSpec> {
-    // Extract genesis state from ERA 0 file
     let era_dir = test_vectors_dir().join("era");
     let era0_path = std::fs::read_dir(&era_dir)
         .expect("read era dir")
@@ -52,7 +75,6 @@ type TestStore = HotColdDB<
     store::MemoryStore<MinimalEthSpec>,
 >;
 
-/// Import all ERA files into a fresh ephemeral store.
 fn import_all_era_files() -> (TestStore, ChainSpec, u64) {
     let spec = load_test_spec();
     let era_dir_path = test_vectors_dir().join("era");
@@ -63,23 +85,13 @@ fn import_all_era_files() -> (TestStore, ChainSpec, u64) {
         .expect("create store");
 
     let mut genesis_state = load_genesis_state(&spec);
-    let root = genesis_state.canonical_root().expect("hash genesis");
-    let mut ops = vec![];
-    store
-        .store_cold_state(&root, &genesis_state, &mut ops)
-        .expect("build ops");
-    store.cold_db.do_atomically(ops).expect("write genesis");
-
-    for era in 1..=max_era {
-        era_dir
-            .import_era_file(&store, era, &spec, None)
-            .unwrap_or_else(|e| panic!("import ERA {era}: {e}"));
-    }
+    era_dir
+        .import_all(&store, &mut genesis_state, &spec)
+        .expect("import all ERA files");
 
     (store, spec, max_era)
 }
 
-/// Create ephemeral store with genesis state only.
 fn empty_store(spec: &ChainSpec) -> TestStore {
     let store =
         HotColdDB::open_ephemeral(StoreConfig::default(), Arc::new(spec.clone())).expect("store");
@@ -93,7 +105,6 @@ fn empty_store(spec: &ChainSpec) -> TestStore {
     store
 }
 
-/// Copy ERA files to temp dir, replacing one with a corrupt version.
 fn era_dir_with_corrupt(corrupt_file: &str, target_pattern: &str) -> tempfile::TempDir {
     let tmp = tempfile::TempDir::new().expect("tmp");
     let src = test_vectors_dir();
@@ -113,104 +124,117 @@ fn era_dir_with_corrupt(corrupt_file: &str, target_pattern: &str) -> tempfile::T
     tmp
 }
 
-// =============================================================================
-// CONSUMER TEST
-// =============================================================================
+fn assert_import_fails(
+    corrupt_file: &str,
+    target_pattern: &str,
+    target_era: u64,
+    expected_err: &str,
+) {
+    let tmp = era_dir_with_corrupt(corrupt_file, target_pattern);
+    let spec = load_test_spec();
+    let era_dir = EraFileDir::new::<MinimalEthSpec>(&tmp.path().join("era"), &spec)
+        .expect("init should succeed");
+    let store = empty_store(&spec);
 
-/// Import all ERA files, verify block hash chain and state roots.
-#[test]
-fn era_consumer_imports_and_verifies() {
-    let (store, spec, max_era) = import_all_era_files();
-    let slots_per_era = MinimalEthSpec::slots_per_historical_root() as u64;
-    let max_slot = max_era * slots_per_era;
-
-    // Collect all blocks by reading block root index, then fetching from store
-    let mut blocks_by_slot = std::collections::BTreeMap::new();
-    let mut seen_roots = std::collections::HashSet::new();
-
-    for slot in 0..max_slot {
-        let key = slot.to_be_bytes().to_vec();
-        if let Some(root_bytes) = store
-            .cold_db
-            .get_bytes(DBColumn::BeaconBlockRoots, &key)
-            .expect("read index")
-        {
-            let block_root = Hash256::from_slice(&root_bytes);
-            if seen_roots.insert(block_root) {
-                let block = store
-                    .get_full_block(&block_root)
-                    .expect("query")
-                    .unwrap_or_else(|| panic!("block missing at slot {slot}"));
-                assert_eq!(
-                    block.canonical_root(),
-                    block_root,
-                    "block root mismatch at slot {slot}"
-                );
-                blocks_by_slot.insert(slot, block);
-            }
-        }
+    for era in 0..target_era {
+        era_dir
+            .import_era_file(&store, era, &spec, None)
+            .unwrap_or_else(|e| panic!("ERA {era}: {e}"));
     }
 
+    let err = era_dir
+        .import_era_file(&store, target_era, &spec, None)
+        .unwrap_err();
     assert!(
-        blocks_by_slot.len() > 700,
-        "expected >700 blocks, got {}",
-        blocks_by_slot.len()
+        err.contains(expected_err),
+        "expected \"{expected_err}\", got: {err}"
     );
-
-    // Verify parent_root chain: each block's parent_root must equal the previous block's root
-    let slots: Vec<_> = blocks_by_slot.keys().copied().collect();
-    for i in 1..slots.len() {
-        let block = &blocks_by_slot[&slots[i]];
-        let prev_block = &blocks_by_slot[&slots[i - 1]];
-        assert_eq!(
-            block.message().parent_root(),
-            prev_block.canonical_root(),
-            "broken hash chain at slot {}: parent_root doesn't match previous block root",
-            slots[i]
-        );
-    }
-
-    // Verify boundary states match ERA file state roots
-    let era_dir_path = test_vectors_dir().join("era");
-    let mut era_files: Vec<_> = std::fs::read_dir(&era_dir_path)
-        .expect("readdir")
-        .filter_map(|e| e.ok())
-        .filter(|e| e.file_name().to_string_lossy().ends_with(".era"))
-        .collect();
-    era_files.sort_by_key(|e| e.file_name());
-
-    for entry in &era_files {
-        let file = std::fs::File::open(entry.path()).expect("open");
-        let era = reth_era::era::file::EraReader::new(file)
-            .read_and_assemble("minimal".to_string())
-            .expect("parse");
-        let state_bytes = era.group.era_state.decompress().expect("decompress");
-        let mut era_state: BeaconState<MinimalEthSpec> =
-            BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode");
-        let expected_root = era_state.canonical_root().expect("root");
-        let slot = era_state.slot();
-
-        // Load state from store and verify root matches
-        let mut stored_state = store.load_cold_state_by_slot(slot).expect("load state");
-        assert_eq!(
-            stored_state.slot(),
-            slot,
-            "stored state slot mismatch at slot {slot}"
-        );
-        let stored_root = stored_state.canonical_root().expect("root");
-        assert_eq!(
-            stored_root, expected_root,
-            "state root mismatch at slot {slot}"
-        );
-    }
 }
 
-// =============================================================================
-// PRODUCER TEST — byte-identical output
-// =============================================================================
+fn era3_correct_root_and_slot(spec: &ChainSpec) -> (Hash256, types::Slot) {
+    let era3_file = std::fs::read_dir(test_vectors_dir().join("era"))
+        .expect("readdir")
+        .filter_map(|e| e.ok())
+        .find(|e| e.file_name().to_string_lossy().contains("-00003-"))
+        .expect("ERA 3");
+    let file = std::fs::File::open(era3_file.path()).expect("open");
+    let era = reth_era::era::file::EraReader::new(file)
+        .read_and_assemble("minimal".to_string())
+        .expect("parse");
+    let state_bytes = era.group.era_state.decompress().expect("decompress");
+    let mut state: BeaconState<MinimalEthSpec> =
+        BeaconState::from_ssz_bytes(&state_bytes, spec).expect("decode");
+    let root = state.canonical_root().expect("root");
+    let slot = state.slot();
+    (root, slot)
+}
 
+// Single #[test] to avoid nextest parallel download races.
+// See slashing_protection/tests/interop.rs for the same pattern.
 #[test]
-fn era_producer_output_is_byte_identical() {
+fn era_test_vectors() {
+    consumer_imports_and_verifies();
+    producer_output_is_byte_identical();
+    rejects_corrupted_block_decompression();
+    rejects_corrupted_genesis_state();
+    rejects_corrupted_middle_state();
+    rejects_corrupted_reference_state();
+    rejects_wrong_era_content();
+    rejects_wrong_era_root();
+    rejects_corrupt_block_summary();
+    rejects_wrong_block_root();
+    rejects_mutated_state_with_trusted_root();
+    rejects_wrong_trusted_state_root();
+}
+
+fn load_metadata() -> Metadata {
+    let path = test_vectors_dir().join("metadata.json");
+    let data = std::fs::read_to_string(&path).expect("read metadata.json");
+    serde_json::from_str(&data).expect("parse metadata.json")
+}
+
+fn consumer_imports_and_verifies() {
+    let metadata = load_metadata();
+    let (store, _spec, max_era) = import_all_era_files();
+    let slots_per_era = MinimalEthSpec::slots_per_historical_root() as u64;
+
+    assert_eq!(max_era + 1, metadata.era_count, "era count mismatch");
+
+    // The last indexed slot is (max_era * slots_per_era - 1), since the ERA boundary
+    // state covers [start_slot, end_slot) where end_slot = era_number * slots_per_era.
+    let last_slot = max_era * slots_per_era - 1;
+
+    // Verify head block root matches metadata
+    let head_key = metadata.head_slot.to_be_bytes().to_vec();
+    let head_root_bytes = store
+        .cold_db
+        .get_bytes(DBColumn::BeaconBlockRoots, &head_key)
+        .expect("read head root index")
+        .unwrap_or_else(|| panic!("no block root at head slot {}", metadata.head_slot));
+    let head_root = Hash256::from_slice(&head_root_bytes);
+
+    let expected_head_root_bytes = hex::decode(&metadata.head_root).expect("decode head_root hex");
+    let expected_head_root = Hash256::from_slice(&expected_head_root_bytes);
+    assert_eq!(
+        head_root, expected_head_root,
+        "head root mismatch at slot {}: got {head_root:?}",
+        metadata.head_slot
+    );
+
+    // Verify the head block exists and has the correct root
+    let head_block = store
+        .get_full_block(&head_root)
+        .expect("query head block")
+        .unwrap_or_else(|| panic!("head block missing at slot {}", metadata.head_slot));
+    assert_eq!(head_block.canonical_root(), head_root);
+    assert_eq!(
+        head_block.slot(),
+        Slot::new(metadata.head_slot),
+        "last indexed slot is {last_slot}"
+    );
+}
+
+fn producer_output_is_byte_identical() {
     let (store, _spec, max_era) = import_all_era_files();
     let output = PathBuf::from("/tmp/era_producer_test_output");
     let _ = std::fs::remove_dir_all(&output);
@@ -247,54 +271,19 @@ fn era_producer_output_is_byte_identical() {
     }
 }
 
-// =============================================================================
-// CORRUPTION TESTS — verify specific error messages
-// =============================================================================
-
-fn assert_import_fails(
-    corrupt_file: &str,
-    target_pattern: &str,
-    target_era: u64,
-    expected_err: &str,
-) {
-    let tmp = era_dir_with_corrupt(corrupt_file, target_pattern);
-    let spec = load_test_spec();
-    let era_dir = EraFileDir::new::<MinimalEthSpec>(&tmp.path().join("era"), &spec)
-        .expect("init should succeed");
-    let store = empty_store(&spec);
-
-    for era in 0..target_era {
-        era_dir
-            .import_era_file(&store, era, &spec, None)
-            .unwrap_or_else(|e| panic!("ERA {era}: {e}"));
-    }
-
-    let err = era_dir
-        .import_era_file(&store, target_era, &spec, None)
-        .unwrap_err();
-    assert!(
-        err.contains(expected_err),
-        "expected \"{expected_err}\", got: {err}"
-    );
-}
-
-#[test]
-fn era_rejects_corrupted_block_decompression() {
+fn rejects_corrupted_block_decompression() {
     assert_import_fails("era1-corrupt-block.era", "-00001-", 1, "decompress");
 }
 
-#[test]
-fn era_rejects_corrupted_genesis_state() {
+fn rejects_corrupted_genesis_state() {
     assert_import_fails("era0-corrupt-state.era", "-00000-", 0, "decompress");
 }
 
-#[test]
-fn era_rejects_corrupted_middle_state() {
+fn rejects_corrupted_middle_state() {
     assert_import_fails("era5-corrupt-state.era", "-00005-", 5, "decompress");
 }
 
-#[test]
-fn era_rejects_corrupted_reference_state() {
+fn rejects_corrupted_reference_state() {
     let tmp = era_dir_with_corrupt("era12-corrupt-state.era", "-00012-");
     let spec = load_test_spec();
     match EraFileDir::new::<MinimalEthSpec>(&tmp.path().join("era"), &spec) {
@@ -303,8 +292,7 @@ fn era_rejects_corrupted_reference_state() {
     }
 }
 
-#[test]
-fn era_rejects_wrong_era_content() {
+fn rejects_wrong_era_content() {
     assert_import_fails(
         "era3-wrong-content.era",
         "-00003-",
@@ -313,13 +301,11 @@ fn era_rejects_wrong_era_content() {
     );
 }
 
-#[test]
-fn era_rejects_wrong_era_root() {
+fn rejects_wrong_era_root() {
     assert_import_fails("era0-wrong-root.era", "-00000-", 0, "era root mismatch");
 }
 
-#[test]
-fn era_rejects_corrupt_block_summary() {
+fn rejects_corrupt_block_summary() {
     assert_import_fails(
         "era8-corrupt-block-summary.era",
         "-00008-",
@@ -328,8 +314,7 @@ fn era_rejects_corrupt_block_summary() {
     );
 }
 
-#[test]
-fn era_rejects_wrong_block_root() {
+fn rejects_wrong_block_root() {
     assert_import_fails(
         "era2-wrong-block-root.era",
         "-00002-",
@@ -338,12 +323,7 @@ fn era_rejects_wrong_block_root() {
     );
 }
 
-/// Mutated balance in ERA 3 state → state root doesn't match trusted root.
-/// Without a trusted root, the consumer can't detect this (historical_summaries only
-/// commit to block_roots/state_roots vectors, not full state content).
-/// The trusted state root feature catches it.
-#[test]
-fn era_rejects_mutated_state_with_trusted_root() {
+fn rejects_mutated_state_with_trusted_root() {
     let tmp = era_dir_with_corrupt("era3-wrong-state-root.era", "-00003-");
     let spec = load_test_spec();
     let era_dir = EraFileDir::new::<MinimalEthSpec>(&tmp.path().join("era"), &spec)
@@ -356,23 +336,8 @@ fn era_rejects_mutated_state_with_trusted_root() {
             .unwrap_or_else(|e| panic!("ERA {era}: {e}"));
     }
 
-    // Get the CORRECT state root from the original ERA 3 file
-    let orig_era3 = std::fs::read_dir(test_vectors_dir().join("era"))
-        .expect("readdir")
-        .filter_map(|e| e.ok())
-        .find(|e| e.file_name().to_string_lossy().contains("-00003-"))
-        .expect("ERA 3");
-    let file = std::fs::File::open(orig_era3.path()).expect("open");
-    let era = reth_era::era::file::EraReader::new(file)
-        .read_and_assemble("minimal".to_string())
-        .expect("parse");
-    let state_bytes = era.group.era_state.decompress().expect("decompress");
-    let mut state: BeaconState<MinimalEthSpec> =
-        BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode");
-    let correct_root = state.canonical_root().expect("root");
-    let slot = state.slot();
+    let (correct_root, slot) = era3_correct_root_and_slot(&spec);
 
-    // Import mutated ERA 3 with trusted root → should fail because balance was changed
     let err = era_dir
         .import_era_file(&store, 3, &spec, Some((correct_root, slot)))
         .unwrap_err();
@@ -382,12 +347,7 @@ fn era_rejects_mutated_state_with_trusted_root() {
     );
 }
 
-// =============================================================================
-// TRUSTED STATE ROOT VERIFICATION
-// =============================================================================
-
-#[test]
-fn era_rejects_wrong_trusted_state_root() {
+fn rejects_wrong_trusted_state_root() {
     let spec = load_test_spec();
     let store = empty_store(&spec);
     let era_dir_path = test_vectors_dir().join("era");
@@ -399,29 +359,12 @@ fn era_rejects_wrong_trusted_state_root() {
             .unwrap_or_else(|e| panic!("ERA {era}: {e}"));
     }
 
-    // Get correct state root for ERA 3
-    let era3_file = std::fs::read_dir(&era_dir_path)
-        .expect("readdir")
-        .filter_map(|e| e.ok())
-        .find(|e| e.file_name().to_string_lossy().contains("-00003-"))
-        .expect("ERA 3");
+    let (correct_root, slot) = era3_correct_root_and_slot(&spec);
 
-    let file = std::fs::File::open(era3_file.path()).expect("open");
-    let era = reth_era::era::file::EraReader::new(file)
-        .read_and_assemble("minimal".to_string())
-        .expect("parse");
-    let state_bytes = era.group.era_state.decompress().expect("decompress");
-    let mut state: BeaconState<MinimalEthSpec> =
-        BeaconState::from_ssz_bytes(&state_bytes, &spec).expect("decode");
-    let correct_root = state.canonical_root().expect("root");
-    let slot = state.slot();
-
-    // Correct root passes
     era_dir
         .import_era_file(&store, 3, &spec, Some((correct_root, slot)))
         .expect("correct root should pass");
 
-    // Wrong root fails
     let wrong_root = {
         let mut bytes: [u8; 32] = correct_root.into();
         bytes[0] ^= 0x01;

beacon_node/beacon_chain/tests/era_test_vectors/.gitignore

@@ -0,0 +1,2 @@
+vectors
+*.tar.gz

beacon_node/beacon_chain/tests/era_test_vectors/Makefile

@@ -0,0 +1,17 @@
+VECTORS_TAG := 52eb9dd94a09153b8b07c9bba4b08adca0d6e219
+OUTPUT_DIR := vectors
+TARBALL := $(OUTPUT_DIR)-$(VECTORS_TAG).tar.gz
+ARCHIVE_URL := https://github.com/dapplion/era-test-vectors/tarball/$(VECTORS_TAG)
+
+$(OUTPUT_DIR): $(TARBALL)
+	rm -rf $@
+	mkdir -p $@
+	tar --strip-components=1 -xzf $^ -C $@
+
+$(TARBALL):
+	curl --fail -L -o $@ $(ARCHIVE_URL)
+
+clean:
+	rm -rf $(OUTPUT_DIR) $(TARBALL)
+
+.PHONY: clean

beacon_node/beacon_chain/tests/era_test_vectors/README.md

@@ -1,37 +0,0 @@
-# ERA File Test Vectors
-
-Minimal preset test vectors for ERA file import/export testing.
-
-## Network Configuration
-
-- **Preset**: minimal (SLOTS_PER_EPOCH=8, SLOTS_PER_HISTORICAL_ROOT=64)
-- **One ERA file** = 64 slots = 8 epochs
-- **Validators**: 1024
-- **Fork schedule**: All forks active from genesis (Electra), Fulu at epoch 100000
-
-## Contents
-
-- `config.yaml` — Network configuration (CL fork schedule + parameters)
-- `genesis.ssz` — Genesis state (SSZ encoded)
-- `era/` — 13 ERA files (minimal-00000 through minimal-00012)
-  - 832 slots total (epochs 0-103)
-  - ~2.4MB compressed
-
-## Generation
-
-Generated using Nimbus `launch_local_testnet.sh` with `--preset minimal --nodes 2 --stop-at-epoch 100 --run-geth --run-spamoor`, then exported via `ncli_db exportEra`.
-
-ERA files contain real blocks with execution payloads (transactions generated by spamoor).
-
-## Test Coverage
-
-### Consumer Tests (4 tests)
-- `era_consumer_imports_all_files` — Imports all 13 ERA files into a fresh store, verifies 768 block root index entries
-- `era_consumer_blocks_are_readable` — Verifies all 767 unique blocks are loadable from the store
-- `era_consumer_genesis_state_intact` — Verifies genesis state with 1024 validators
-- `era_files_are_parseable` — Verifies all ERA files can be parsed by reth_era library
-
-### Producer Test (1 test)
-- `era_producer_generates_identical_files` — Re-exports ERA files from imported data and verifies byte-for-byte match with original Nimbus-generated files
-
-All tests passing ✅

beacon_node/beacon_chain/tests/era_test_vectors/config.yaml

@@ -1,186 +0,0 @@
-# This file should contain the origin run-time config for the minimal
-# network [1] without all properties overriden in the local network
-# simulation. We use to generate a full run-time config as required
-# by third-party binaries, such as Lighthouse and Web3Signer.
-#
-# [1]: https://raw.githubusercontent.com/ethereum/consensus-specs/dev/configs/minimal.yaml
-
-# Minimal config
-
-# Extends the minimal preset
-# (overriden in launch_local_testnet.sh) PRESET_BASE: 'minimal'
-
-# Free-form short name of the network that this configuration applies to - known
-# canonical network names include:
-# * 'mainnet' - there can be only one
-# * 'prater' - testnet
-# Must match the regex: [a-z0-9\-]
-CONFIG_NAME: 'minimal'
-
-# Transition
-# ---------------------------------------------------------------
-# 2**256-2**10 for testing minimal network
-# (overriden in launch_local_testnet.sh) TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912
-# By default, don't use these params
-TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000
-TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615
-
-
-
-# Genesis
-# ---------------------------------------------------------------
-# [customized]
-# (overriden in launch_local_testnet.sh) MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 64
-# Jan 3, 2020
-# (overriden in launch_local_testnet.sh) MIN_GENESIS_TIME: 1578009600
-# Highest byte set to 0x01 to avoid collisions with mainnet versioning
-GENESIS_FORK_VERSION: 0x00000001
-# [customized] Faster to spin up testnets, but does not give validator reasonable warning time for genesis
-# (overriden in launch_local_testnet.sh) GENESIS_DELAY: 300
-
-
-# Forking
-# ---------------------------------------------------------------
-# Values provided for illustrative purposes.
-# Individual tests/testnets may set different values.
-
-# Altair
-ALTAIR_FORK_VERSION: 0x01000001
-# (overriden in launch_local_testnet.sh) ALTAIR_FORK_EPOCH: 18446744073709551615
-# Bellatrix
-BELLATRIX_FORK_VERSION: 0x02000001
-# (overriden in launch_local_testnet.sh) BELLATRIX_FORK_EPOCH: 18446744073709551615
-# Capella
-CAPELLA_FORK_VERSION: 0x03000001
-# (overriden in launch_local_testnet.sh) CAPELLA_FORK_EPOCH: 18446744073709551615
-# Deneb
-DENEB_FORK_VERSION: 0x04000001
-# (overriden in launch_local_testnet.sh) DENEB_FORK_EPOCH: 18446744073709551615
-# Electra
-ELECTRA_FORK_VERSION: 0x05000001
-# (overriden in launch_local_testnet.sh) ELECTRA_FORK_EPOCH: 18446744073709551615
-# Fulu
-FULU_FORK_VERSION: 0x06000001
-# (overriden in launch_local_testnet.sh) FULU_FORK_EPOCH: 18446744073709551615
-
-# Time parameters
-# ---------------------------------------------------------------
-# [customized] Faster for testing purposes
-SECONDS_PER_SLOT: 6
-# 14 (estimate from Eth1 mainnet)
-SECONDS_PER_ETH1_BLOCK: 14
-# 2**8 (= 256) epochs
-MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
-# [customized] higher frequency of committee turnover and faster time to acceptable voluntary exit
-SHARD_COMMITTEE_PERIOD: 64
-# [customized] process deposits more quickly, but insecure
-# (overriden in launch_local_testnet.sh) ETH1_FOLLOW_DISTANCE: 16
-
-
-# Validator cycle
-# ---------------------------------------------------------------
-# 2**2 (= 4)
-INACTIVITY_SCORE_BIAS: 4
-# 2**4 (= 16)
-INACTIVITY_SCORE_RECOVERY_RATE: 16
-# 2**4 * 10**9 (= 16,000,000,000) Gwei
-EJECTION_BALANCE: 16000000000
-# [customized] more easily demonstrate the difference between this value and the activation churn limit
-MIN_PER_EPOCH_CHURN_LIMIT: 2
-# [customized] scale queue churn at much lower validator counts for testing
-CHURN_LIMIT_QUOTIENT: 32
-# [New in Deneb:EIP7514] [customized]
-MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 4
-
-
-# Fork choice
-# ---------------------------------------------------------------
-# 40%
-PROPOSER_SCORE_BOOST: 40
-# 20%
-REORG_HEAD_WEIGHT_THRESHOLD: 20
-# 160%
-REORG_PARENT_WEIGHT_THRESHOLD: 160
-# `2` epochs
-REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
-
-
-# Deposit contract
-# ---------------------------------------------------------------
-# Ethereum Goerli testnet
-DEPOSIT_CHAIN_ID: 5
-DEPOSIT_NETWORK_ID: 5
-# Configured on a per testnet basis
-# (overriden in launch_local_testnet.sh) DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
-
-
-# Networking
-# ---------------------------------------------------------------
-# `10 * 2**20` (= 10485760, 10 MiB)
-MAX_PAYLOAD_SIZE: 10485760
-# `2**10` (= 1024)
-MAX_REQUEST_BLOCKS: 1024
-# `2**8` (= 256)
-EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
-# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
-MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
-ATTESTATION_PROPAGATION_SLOT_RANGE: 32
-# 500ms
-MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500
-MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000
-MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000
-# 2 subnets per node
-SUBNETS_PER_NODE: 2
-# 2**8 (= 64)
-ATTESTATION_SUBNET_COUNT: 64
-ATTESTATION_SUBNET_EXTRA_BITS: 0
-# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
-ATTESTATION_SUBNET_PREFIX_BITS: 6
-
-# Deneb
-# `2**7` (=128)
-MAX_REQUEST_BLOCKS_DENEB: 128
-# `2**12` (= 4096 epochs, ~18 days)
-MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
-# `6`
-BLOB_SIDECAR_SUBNET_COUNT: 6
-## `uint64(6)`
-MAX_BLOBS_PER_BLOCK: 6
-# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
-MAX_REQUEST_BLOB_SIDECARS: 768
-
-# Electra
-# [customized] 2**6 * 10**9 (= 64,000,000,000)
-MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000
-# [customized] 2**7 * 10**9 (= 128,000,000,000)
-MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT: 128000000000
-# `9`
-BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: 9
-# `uint64(9)`
-MAX_BLOBS_PER_BLOCK_ELECTRA: 9
-# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK_ELECTRA
-MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152
-
-# Fulu
-NUMBER_OF_COLUMNS: 128
-NUMBER_OF_CUSTODY_GROUPS: 128
-DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
-MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
-SAMPLES_PER_SLOT: 8
-CUSTODY_REQUIREMENT: 4
-VALIDATOR_CUSTODY_REQUIREMENT: 8
-BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
-MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
-PRESET_BASE: minimal
-MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 1024
-MIN_GENESIS_TIME: 0
-GENESIS_DELAY: 10
-DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242
-ETH1_FOLLOW_DISTANCE: 1
-ALTAIR_FORK_EPOCH: 0
-BELLATRIX_FORK_EPOCH: 0
-CAPELLA_FORK_EPOCH: 0
-DENEB_FORK_EPOCH: 0
-ELECTRA_FORK_EPOCH: 0
-FULU_FORK_EPOCH: 100000
-TERMINAL_TOTAL_DIFFICULTY: 0

beacon_node/beacon_chain/tests/era_test_vectors/create_corrupt_files.py

@@ -1,96 +0,0 @@
-#!/usr/bin/env python3
-"""
-Create corrupted ERA files for testing ERA consumer error handling.
-
-This script generates specific corrupt ERA files by:
-1. Parsing existing ERA files
-2. Modifying specific parts (block data, state data)
-3. Re-encoding with valid compression
-
-Requires: pip install python-snappy
-"""
-import os
-import sys
-import shutil
-from pathlib import Path
-
-try:
-    import snappy
-except ImportError:
-    print("ERROR: python-snappy not installed. Run: pip install python-snappy", file=sys.stderr)
-    sys.exit(1)
-
-SCRIPT_DIR = Path(__file__).parent
-ERA_DIR = SCRIPT_DIR / "era"
-CORRUPT_DIR = SCRIPT_DIR / "corrupt"
-
-def read_era_file(path):
-    """Read ERA file and return raw bytes."""
-    with open(path, 'rb') as f:
-        return f.read()
-
-def find_era_file(pattern):
-    """Find ERA file matching pattern."""
-    files = list(ERA_DIR.glob(f"minimal-{pattern}-*.era"))
-    if not files:
-        return None
-    return files[0]
-
-def corrupt_bytes_at_offset(data, offset, xor_pattern=0xFF):
-    """Corrupt bytes at specific offset by XOR."""
-    result = bytearray(data)
-    result[offset] ^= xor_pattern
-    result[offset + 1] ^= xor_pattern
-    return bytes(result)
-
-def main():
-    print("Creating corrupt ERA test files...\n")
-    CORRUPT_DIR.mkdir(exist_ok=True)
-
-    # Test 1: ERA root mismatch - corrupt genesis_validators_root in ERA 0
-    era0 = find_era_file("00000")
-    if era0:
-        data = read_era_file(era0)
-        # Corrupt bytes in the state section (after 16-byte header)
-        # The state is compressed, so corruption will propagate through state root
-        corrupt_data = corrupt_bytes_at_offset(data, 16 + 50)
-        output = CORRUPT_DIR / "era0-wrong-root.era"
-        with open(output, 'wb') as f:
-            f.write(corrupt_data)
-        print(f"✓ Created era0-wrong-root.era ({len(corrupt_data)} bytes)")
-    else:
-        print("⚠ ERA 0 file not found, skipping", file=sys.stderr)
-
-    # Test 2: Block summary root post-Capella mismatch - corrupt block_roots
-    era8 = find_era_file("00008")
-    if era8:
-        data = read_era_file(era8)
-        # Corrupt state section (different offset than ERA 0)
-        corrupt_data = corrupt_bytes_at_offset(data, 16 + 100)
-        output = CORRUPT_DIR / "era8-corrupt-block-summary.era"
-        with open(output, 'wb') as f:
-            f.write(corrupt_data)
-        print(f"✓ Created era8-corrupt-block-summary.era ({len(corrupt_data)} bytes)")
-    else:
-        print("⚠ ERA 8 file not found, skipping", file=sys.stderr)
-
-    # Test 3: Block root mismatch - corrupt a block
-    era2 = find_era_file("00002")
-    if era2:
-        data = read_era_file(era2)
-        # Find and corrupt a block (blocks come after state in ERA file)
-        # We'll corrupt somewhere in the middle where blocks likely are
-        corrupt_offset = len(data) // 3  # Rough guess at block location
-        corrupt_data = corrupt_bytes_at_offset(data, corrupt_offset)
-        output = CORRUPT_DIR / "era2-wrong-block-root.era"
-        with open(output, 'wb') as f:
-            f.write(corrupt_data)
-        print(f"✓ Created era2-wrong-block-root.era ({len(corrupt_data)} bytes)")
-    else:
-        print("⚠ ERA 2 file not found, skipping", file=sys.stderr)
-
-    print(f"\n✓ Corrupt files created in: {CORRUPT_DIR}")
-    print(f"Total files: {len(list(CORRUPT_DIR.glob('*.era')))}")
-
-if __name__ == "__main__":
-    main()

beacon_node/beacon_chain/tests/era_test_vectors/metadata.json

@@ -1 +0,0 @@
-{"head_slot": 802, "head_root": "49f82639", "finalized_slot": 784, "finalized_root": "55720c58", "era_count": 13, "last_era_slot": 832}

lcli/src/consume_era_files.rs

@@ -64,65 +64,16 @@ pub fn run<E: EthSpec>(
     )
     .map_err(|e| format!("Failed to open database: {e:?}"))?;
 
-    // Load genesis state from the network config
     let mut genesis_state = env
         .runtime()
         .block_on(network_config.genesis_state::<E>(None, Duration::from_secs(120)))
         .map_err(|e| format!("Failed to load genesis state: {e}"))?
         .ok_or("No genesis state available for this network")?;
 
-    // Open ERA files directory and validate against genesis
     let era_file_dir = EraFileDir::new::<E>(&era_dir, &spec)
         .map_err(|e| format!("Failed to open ERA dir: {e}"))?;
 
-    // Verify ERA files match the network's genesis
-    if era_file_dir.genesis_validators_root() != genesis_state.genesis_validators_root() {
-        return Err(format!(
-            "ERA files genesis_validators_root ({:?}) does not match network genesis ({:?}). \
-             Are the ERA files from the correct network?",
-            era_file_dir.genesis_validators_root(),
-            genesis_state.genesis_validators_root(),
-        ));
-    }
-
-    info!(
-        genesis_validators_root = %genesis_state.genesis_validators_root(),
-        "Storing genesis state"
-    );
-
-    let genesis_root = genesis_state
-        .canonical_root()
-        .map_err(|e| format!("Failed to hash genesis state: {e:?}"))?;
-    db.put_cold_state(&genesis_root, &genesis_state)
-        .map_err(|e| format!("Failed to store genesis state: {e:?}"))?;
-
-    let max_era = era_file_dir.max_era();
-    info!(max_era, "Importing ERA files");
-
-    let start = std::time::Instant::now();
-    for era_number in 1..=max_era {
-        era_file_dir
-            .import_era_file(&db, era_number, &spec, None)
-            .map_err(|e| format!("Failed to import ERA {era_number}: {e}"))?;
-
-        if era_number % 100 == 0 || era_number == max_era {
-            let elapsed = start.elapsed();
-            let rate = era_number as f64 / elapsed.as_secs_f64();
-            info!(
-                era_number,
-                max_era,
-                ?elapsed,
-                rate = format!("{rate:.1} era/s"),
-                "Progress"
-            );
-        }
-    }
-
-    info!(
-        max_era,
-        elapsed = ?start.elapsed(),
-        "ERA file import complete. Database is ready."
-    );
+    era_file_dir.import_all(&db, &mut genesis_state, &spec)?;
 
     Ok(())
 }