diff --git a/Cargo.toml b/Cargo.toml index 27c473bb9a..ef17b431ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,5 @@ [workspace] members = [ - "eth2/fork_choice", "eth2/lmd_ghost", "eth2/operation_pool", "eth2/state_processing", diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 2b6f44e949..f97302a7c0 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -10,7 +10,6 @@ network = { path = "../network" } store = { path = "../store" } http_server = { path = "../http_server" } rpc = { path = "../rpc" } -fork_choice = { path = "../../eth2/fork_choice" } prometheus = "^0.6" types = { path = "../../eth2/types" } tree_hash = { path = "../../eth2/utils/tree_hash" } diff --git a/beacon_node/http_server/Cargo.toml b/beacon_node/http_server/Cargo.toml index 098c3e1c9d..45e0349f55 100644 --- a/beacon_node/http_server/Cargo.toml +++ b/beacon_node/http_server/Cargo.toml @@ -16,7 +16,6 @@ types = { path = "../../eth2/types" } ssz = { path = "../../eth2/utils/ssz" } slot_clock = { path = "../../eth2/utils/slot_clock" } protos = { path = "../../protos" } -fork_choice = { path = "../../eth2/fork_choice" } grpcio = { version = "0.4", default-features = false, features = ["protobuf-codec"] } persistent = "^0.4" protobuf = "2.0.2" diff --git a/beacon_node/network/src/sync/simple_sync.rs b/beacon_node/network/src/sync/simple_sync.rs index 528edca98e..0a082afcf7 100644 --- a/beacon_node/network/src/sync/simple_sync.rs +++ b/beacon_node/network/src/sync/simple_sync.rs @@ -517,7 +517,7 @@ impl SimpleSync { self.process_block(peer_id.clone(), block.clone(), network, &"gossip") { match outcome { - BlockProcessingOutcome::Processed => SHOULD_FORWARD_GOSSIP_BLOCK, + BlockProcessingOutcome::Processed { .. } => SHOULD_FORWARD_GOSSIP_BLOCK, BlockProcessingOutcome::ParentUnknown { .. } => { self.import_queue .enqueue_full_blocks(vec![block], peer_id.clone()); @@ -582,7 +582,7 @@ impl SimpleSync { _ => true, }; - if processing_result == Some(BlockProcessingOutcome::Processed) { + if processing_result == Some(BlockProcessingOutcome::Processed { block_root }) { successful += 1; } @@ -695,11 +695,12 @@ impl SimpleSync { if let Ok(outcome) = processing_result { match outcome { - BlockProcessingOutcome::Processed => { + BlockProcessingOutcome::Processed { block_root } => { info!( self.log, "Imported block from network"; "source" => source, "slot" => block.slot, + "block_root" => format!("{}", block_root), "peer" => format!("{:?}", peer_id), ); } diff --git a/beacon_node/rpc/src/beacon_block.rs b/beacon_node/rpc/src/beacon_block.rs index d36cb1f313..533fd285a3 100644 --- a/beacon_node/rpc/src/beacon_block.rs +++ b/beacon_node/rpc/src/beacon_block.rs @@ -95,12 +95,13 @@ impl BeaconBlockService for BeaconBlockServiceInstance { Ok(block) => { match self.chain.process_block(block.clone()) { Ok(outcome) => { - if outcome == BlockProcessingOutcome::Processed { + if let BlockProcessingOutcome::Processed { block_root } = outcome { // Block was successfully processed. info!( self.log, "Valid block from RPC"; "block_slot" => block.slot, + "block_root" => format!("{}", block_root), ); // TODO: Obtain topics from the network service properly. diff --git a/eth2/fork_choice/Cargo.toml b/eth2/fork_choice/Cargo.toml deleted file mode 100644 index e37e415e49..0000000000 --- a/eth2/fork_choice/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "fork_choice" -version = "0.1.0" -authors = ["Age Manning "] -edition = "2018" - -[[bench]] -name = "benches" -harness = false - -[dependencies] -store = { path = "../../beacon_node/store" } -ssz = { path = "../utils/ssz" } -types = { path = "../types" } -log = "0.4.6" -bit-vec = "0.5.0" - -[dev-dependencies] -criterion = "0.2" -hex = "0.3.2" -yaml-rust = "0.4.2" -bls = { path = "../utils/bls" } -slot_clock = { path = "../utils/slot_clock" } -beacon_chain = { path = "../../beacon_node/beacon_chain" } -env_logger = "0.6.0" diff --git a/eth2/fork_choice/benches/benches.rs b/eth2/fork_choice/benches/benches.rs deleted file mode 100644 index f311e1ccbb..0000000000 --- a/eth2/fork_choice/benches/benches.rs +++ /dev/null @@ -1,75 +0,0 @@ -use criterion::Criterion; -use criterion::{criterion_group, criterion_main, Benchmark}; -use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; -use std::sync::Arc; -use store::MemoryStore; -use types::{ChainSpec, EthSpec, MainnetEthSpec}; - -pub type TestedForkChoice = OptimizedLMDGhost; -pub type TestedEthSpec = MainnetEthSpec; - -/// Helper function to setup a builder and spec. -fn setup( - validator_count: usize, - chain_length: usize, -) -> ( - TestingForkChoiceBuilder, - ChainSpec, -) { - let store = MemoryStore::open(); - let builder: TestingForkChoiceBuilder = - TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - let spec = TestedEthSpec::default_spec(); - - (builder, spec) -} - -/// Benches adding blocks to fork_choice. -fn add_block(c: &mut Criterion) { - let validator_count = 16; - let chain_length = 100; - - let (builder, spec) = setup(validator_count, chain_length); - - c.bench( - &format!("{}_blocks", chain_length), - Benchmark::new("add_blocks", move |b| { - b.iter(|| { - let mut fc = builder.build::>(); - for (root, block) in builder.chain.iter().skip(1) { - fc.add_block(block, root, &spec).unwrap(); - } - }) - }) - .sample_size(10), - ); -} - -/// Benches fork choice head finding. -fn find_head(c: &mut Criterion) { - let validator_count = 16; - let chain_length = 64 * 2; - - let (builder, spec) = setup(validator_count, chain_length); - - let mut fc = builder.build::>(); - for (root, block) in builder.chain.iter().skip(1) { - fc.add_block(block, root, &spec).unwrap(); - } - - let head_root = builder.chain.last().unwrap().0; - for i in 0..validator_count { - fc.add_attestation(i as u64, &head_root, &spec).unwrap(); - } - - c.bench( - &format!("{}_blocks", chain_length), - Benchmark::new("find_head", move |b| { - b.iter(|| fc.find_head(&builder.genesis_root(), &spec).unwrap()) - }) - .sample_size(10), - ); -} - -criterion_group!(benches, add_block, find_head); -criterion_main!(benches); diff --git a/eth2/fork_choice/examples/example.rs b/eth2/fork_choice/examples/example.rs deleted file mode 100644 index a912c3753c..0000000000 --- a/eth2/fork_choice/examples/example.rs +++ /dev/null @@ -1,40 +0,0 @@ -use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; -use std::sync::Arc; -use store::{MemoryStore, Store}; -use types::{BeaconBlock, ChainSpec, EthSpec, Hash256, MainnetEthSpec}; - -fn main() { - let validator_count = 16; - let chain_length = 100; - let repetitions = 50; - - let store = MemoryStore::open(); - let builder: TestingForkChoiceBuilder = - TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - - let fork_choosers: Vec> = (0..repetitions) - .into_iter() - .map(|_| builder.build()) - .collect(); - - let spec = &MainnetEthSpec::default_spec(); - - println!("Running {} times...", repetitions); - for fc in fork_choosers { - do_thing(fc, &builder.chain, builder.genesis_root(), spec); - } -} - -#[inline(never)] -fn do_thing, S: Store>( - mut fc: F, - chain: &[(Hash256, BeaconBlock)], - genesis_root: Hash256, - spec: &ChainSpec, -) { - for (root, block) in chain.iter().skip(1) { - fc.add_block(block, root, spec).unwrap(); - } - - let _head = fc.find_head(&genesis_root, spec).unwrap(); -} diff --git a/eth2/fork_choice/fork_choice/Cargo.toml b/eth2/fork_choice/fork_choice/Cargo.toml deleted file mode 100644 index e37e415e49..0000000000 --- a/eth2/fork_choice/fork_choice/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "fork_choice" -version = "0.1.0" -authors = ["Age Manning "] -edition = "2018" - -[[bench]] -name = "benches" -harness = false - -[dependencies] -store = { path = "../../beacon_node/store" } -ssz = { path = "../utils/ssz" } -types = { path = "../types" } -log = "0.4.6" -bit-vec = "0.5.0" - -[dev-dependencies] -criterion = "0.2" -hex = "0.3.2" -yaml-rust = "0.4.2" -bls = { path = "../utils/bls" } -slot_clock = { path = "../utils/slot_clock" } -beacon_chain = { path = "../../beacon_node/beacon_chain" } -env_logger = "0.6.0" diff --git a/eth2/fork_choice/fork_choice/benches/benches.rs b/eth2/fork_choice/fork_choice/benches/benches.rs deleted file mode 100644 index f311e1ccbb..0000000000 --- a/eth2/fork_choice/fork_choice/benches/benches.rs +++ /dev/null @@ -1,75 +0,0 @@ -use criterion::Criterion; -use criterion::{criterion_group, criterion_main, Benchmark}; -use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; -use std::sync::Arc; -use store::MemoryStore; -use types::{ChainSpec, EthSpec, MainnetEthSpec}; - -pub type TestedForkChoice = OptimizedLMDGhost; -pub type TestedEthSpec = MainnetEthSpec; - -/// Helper function to setup a builder and spec. -fn setup( - validator_count: usize, - chain_length: usize, -) -> ( - TestingForkChoiceBuilder, - ChainSpec, -) { - let store = MemoryStore::open(); - let builder: TestingForkChoiceBuilder = - TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - let spec = TestedEthSpec::default_spec(); - - (builder, spec) -} - -/// Benches adding blocks to fork_choice. -fn add_block(c: &mut Criterion) { - let validator_count = 16; - let chain_length = 100; - - let (builder, spec) = setup(validator_count, chain_length); - - c.bench( - &format!("{}_blocks", chain_length), - Benchmark::new("add_blocks", move |b| { - b.iter(|| { - let mut fc = builder.build::>(); - for (root, block) in builder.chain.iter().skip(1) { - fc.add_block(block, root, &spec).unwrap(); - } - }) - }) - .sample_size(10), - ); -} - -/// Benches fork choice head finding. -fn find_head(c: &mut Criterion) { - let validator_count = 16; - let chain_length = 64 * 2; - - let (builder, spec) = setup(validator_count, chain_length); - - let mut fc = builder.build::>(); - for (root, block) in builder.chain.iter().skip(1) { - fc.add_block(block, root, &spec).unwrap(); - } - - let head_root = builder.chain.last().unwrap().0; - for i in 0..validator_count { - fc.add_attestation(i as u64, &head_root, &spec).unwrap(); - } - - c.bench( - &format!("{}_blocks", chain_length), - Benchmark::new("find_head", move |b| { - b.iter(|| fc.find_head(&builder.genesis_root(), &spec).unwrap()) - }) - .sample_size(10), - ); -} - -criterion_group!(benches, add_block, find_head); -criterion_main!(benches); diff --git a/eth2/fork_choice/fork_choice/examples/example.rs b/eth2/fork_choice/fork_choice/examples/example.rs deleted file mode 100644 index a912c3753c..0000000000 --- a/eth2/fork_choice/fork_choice/examples/example.rs +++ /dev/null @@ -1,40 +0,0 @@ -use fork_choice::{test_utils::TestingForkChoiceBuilder, ForkChoice, OptimizedLMDGhost}; -use std::sync::Arc; -use store::{MemoryStore, Store}; -use types::{BeaconBlock, ChainSpec, EthSpec, Hash256, MainnetEthSpec}; - -fn main() { - let validator_count = 16; - let chain_length = 100; - let repetitions = 50; - - let store = MemoryStore::open(); - let builder: TestingForkChoiceBuilder = - TestingForkChoiceBuilder::new(validator_count, chain_length, Arc::new(store)); - - let fork_choosers: Vec> = (0..repetitions) - .into_iter() - .map(|_| builder.build()) - .collect(); - - let spec = &MainnetEthSpec::default_spec(); - - println!("Running {} times...", repetitions); - for fc in fork_choosers { - do_thing(fc, &builder.chain, builder.genesis_root(), spec); - } -} - -#[inline(never)] -fn do_thing, S: Store>( - mut fc: F, - chain: &[(Hash256, BeaconBlock)], - genesis_root: Hash256, - spec: &ChainSpec, -) { - for (root, block) in chain.iter().skip(1) { - fc.add_block(block, root, spec).unwrap(); - } - - let _head = fc.find_head(&genesis_root, spec).unwrap(); -} diff --git a/eth2/fork_choice/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml deleted file mode 100644 index 61b0b05c40..0000000000 --- a/eth2/fork_choice/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ /dev/null @@ -1,144 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests for bitwise lmd ghost. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - weights: - - b0: 0 - - b1: 0 - - b2: 5 - - b3: 10 - heads: - - id: 'b3' -# bitwise LMD ghost example. bitwise GHOST gives b2 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 4 - - b3: 3 - heads: - - id: 'b2' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b1' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b6' - weights: - - b0: 0 - - b1: 3 - - b2: 2 - - b3: 1 - - b4: 1 - - b5: 1 - - b6: 2 - - b7: 2 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b2' - - id: 'b8' - parent: 'b3' - - id: 'b9' - parent: 'b3' - weights: - - b1: 2 - - b2: 1 - - b3: 1 - - b4: 7 - - b5: 5 - - b6: 2 - - b7: 4 - - b8: 4 - - b9: 2 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b2' - - id: 'b8' - parent: 'b3' - - id: 'b9' - parent: 'b3' - weights: - - b1: 1 - - b2: 1 - - b3: 1 - - b4: 7 - - b5: 5 - - b6: 2 - - b7: 4 - - b8: 4 - - b9: 2 - heads: - - id: 'b7' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - weights: - - b1: 0 - - b2: 0 - heads: - - id: 'b1' - diff --git a/eth2/fork_choice/fork_choice/tests/lmd_ghost_test_vectors.yaml b/eth2/fork_choice/fork_choice/tests/lmd_ghost_test_vectors.yaml deleted file mode 100644 index e7847de11a..0000000000 --- a/eth2/fork_choice/fork_choice/tests/lmd_ghost_test_vectors.yaml +++ /dev/null @@ -1,65 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests for lmd ghost. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - weights: - - b0: 0 - - b1: 0 - - b2: 5 - - b3: 10 - heads: - - id: 'b3' -# bitwise LMD ghost example. GHOST gives b1 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 4 - - b3: 3 - heads: - - id: 'b1' -# equal weights children. Should choose lower hash b2 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 6 - - b3: 6 - heads: - - id: 'b2' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - weights: - - b1: 0 - - b2: 0 - heads: - - id: 'b1' diff --git a/eth2/fork_choice/fork_choice/tests/longest_chain_test_vectors.yaml b/eth2/fork_choice/fork_choice/tests/longest_chain_test_vectors.yaml deleted file mode 100644 index e1cd61f06a..0000000000 --- a/eth2/fork_choice/fork_choice/tests/longest_chain_test_vectors.yaml +++ /dev/null @@ -1,51 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests to verify the longest chain fork-choice rule. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - - id: 'b4' - parent: 'b3' - weights: - - b0: 0 - - b1: 0 - - b2: 10 - - b3: 1 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b2' - - id: 'b4' - parent: 'b3' - - id: 'b5' - parent: 'b0' - - id: 'b6' - parent: 'b5' - - id: 'b7' - parent: 'b6' - - id: 'b8' - parent: 'b7' - - id: 'b9' - parent: 'b8' - weights: - - b0: 5 - - b1: 20 - - b2: 10 - - b3: 10 - heads: - - id: 'b9' diff --git a/eth2/fork_choice/fork_choice/tests/tests.rs b/eth2/fork_choice/fork_choice/tests/tests.rs deleted file mode 100644 index 39e70a7ddb..0000000000 --- a/eth2/fork_choice/fork_choice/tests/tests.rs +++ /dev/null @@ -1,231 +0,0 @@ -#![cfg(not(debug_assertions))] -/// Tests the available fork-choice algorithms -pub use beacon_chain::BeaconChain; -use bls::Signature; -use store::MemoryStore; -use store::Store; -// use env_logger::{Builder, Env}; -use fork_choice::{BitwiseLMDGhost, ForkChoice, LongestChain, OptimizedLMDGhost, SlowLMDGhost}; -use std::collections::HashMap; -use std::sync::Arc; -use std::{fs::File, io::prelude::*, path::PathBuf}; -use types::test_utils::TestingBeaconStateBuilder; -use types::{ - BeaconBlock, BeaconBlockBody, Eth1Data, EthSpec, Hash256, Keypair, MainnetEthSpec, Slot, -}; -use yaml_rust::yaml; - -// Note: We Assume the block Id's are hex-encoded. - -#[test] -fn test_optimized_lmd_ghost() { - // set up logging - // Builder::from_env(Env::default().default_filter_or("trace")).init(); - - test_yaml_vectors::>( - "tests/lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_bitwise_lmd_ghost() { - // set up logging - //Builder::from_env(Env::default().default_filter_or("trace")).init(); - - test_yaml_vectors::>( - "tests/bitwise_lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_slow_lmd_ghost() { - test_yaml_vectors::>( - "tests/lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_longest_chain() { - test_yaml_vectors::>("tests/longest_chain_test_vectors.yaml", 100); -} - -// run a generic test over given YAML test vectors -fn test_yaml_vectors>( - yaml_file_path: &str, - emulated_validators: usize, // the number of validators used to give weights. -) { - // load test cases from yaml - let test_cases = load_test_cases_from_yaml(yaml_file_path); - - // default vars - let spec = MainnetEthSpec::default_spec(); - let zero_hash = Hash256::zero(); - let eth1_data = Eth1Data { - deposit_count: 0, - deposit_root: zero_hash.clone(), - block_hash: zero_hash.clone(), - }; - let randao_reveal = Signature::empty_signature(); - let signature = Signature::empty_signature(); - let body = BeaconBlockBody { - eth1_data, - randao_reveal, - graffiti: [0; 32], - proposer_slashings: vec![], - attester_slashings: vec![], - attestations: vec![], - deposits: vec![], - voluntary_exits: vec![], - transfers: vec![], - }; - - // process the tests - for test_case in test_cases { - // setup a fresh test - let (mut fork_choice, store, state_root) = setup_inital_state::(emulated_validators); - - // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) - //let mut block_id_map: HashMap = HashMap::new(); - // keep a list of hash to slot - let mut block_slot: HashMap = HashMap::new(); - // assume the block tree is given to us in order. - let mut genesis_hash = None; - for block in test_case["blocks"].clone().into_vec().unwrap() { - let block_id = block["id"].as_str().unwrap().to_string(); - let parent_id = block["parent"].as_str().unwrap().to_string(); - - // default params for genesis - let block_hash = id_to_hash(&block_id); - let mut slot = spec.genesis_slot; - let previous_block_root = id_to_hash(&parent_id); - - // set the slot and parent based off the YAML. Start with genesis; - // if not the genesis, update slot - if parent_id != block_id { - // find parent slot - slot = *(block_slot - .get(&previous_block_root) - .expect("Parent should have a slot number")) - + 1; - } else { - genesis_hash = Some(block_hash); - } - - // update slot mapping - block_slot.insert(block_hash, slot); - - // build the BeaconBlock - let beacon_block = BeaconBlock { - slot, - previous_block_root, - state_root: state_root.clone(), - signature: signature.clone(), - body: body.clone(), - }; - - // Store the block. - store.put(&block_hash, &beacon_block).unwrap(); - - // run add block for fork choice if not genesis - if parent_id != block_id { - fork_choice - .add_block(&beacon_block, &block_hash, &spec) - .unwrap(); - } - } - - // add the weights (attestations) - let mut current_validator = 0; - for id_map in test_case["weights"].clone().into_vec().unwrap() { - // get the block id and weights - for (map_id, map_weight) in id_map.as_hash().unwrap().iter() { - let id = map_id.as_str().unwrap(); - let block_root = id_to_hash(&id.to_string()); - let weight = map_weight.as_i64().unwrap(); - // we assume a validator has a value 1 and add an attestation for to achieve the - // correct weight - for _ in 0..weight { - assert!( - current_validator <= emulated_validators, - "Not enough validators to emulate weights" - ); - fork_choice - .add_attestation(current_validator as u64, &block_root, &spec) - .unwrap(); - current_validator += 1; - } - } - } - - // everything is set up, run the fork choice, using genesis as the head - let head = fork_choice - .find_head(&genesis_hash.unwrap(), &spec) - .unwrap(); - - // compare the result to the expected test - let success = test_case["heads"] - .clone() - .into_vec() - .unwrap() - .iter() - .find(|heads| id_to_hash(&heads["id"].as_str().unwrap().to_string()) == head) - .is_some(); - - println!("Head found: {}", head); - assert!(success, "Did not find one of the possible heads"); - } -} - -// loads the test_cases from the supplied yaml file -fn load_test_cases_from_yaml(file_path: &str) -> Vec { - // load the yaml - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push(file_path); - File::open(file_path_buf).unwrap() - }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - doc["test_cases"].as_vec().unwrap().clone() -} - -fn setup_inital_state( - // fork_choice_algo: &ForkChoiceAlgorithm, - num_validators: usize -) -> (T, Arc, Hash256) -where - T: ForkChoice, -{ - let store = Arc::new(MemoryStore::open()); - - let fork_choice = ForkChoice::new(store.clone()); - let spec = MainnetEthSpec::default_spec(); - - let mut state_builder: TestingBeaconStateBuilder = - TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec); - state_builder.build_caches(&spec).unwrap(); - let (state, _keypairs) = state_builder.build(); - - let state_root = state.canonical_root(); - store.put(&state_root, &state).unwrap(); - - // return initialised vars - (fork_choice, store, state_root) -} - -// convert a block_id into a Hash256 -- assume input is hex encoded; -fn id_to_hash(id: &String) -> Hash256 { - let bytes = hex::decode(id).expect("Block ID should be hex"); - - let len = std::cmp::min(bytes.len(), 32); - let mut fixed_bytes = [0u8; 32]; - for (index, byte) in bytes.iter().take(32).enumerate() { - fixed_bytes[32 - len + index] = *byte; - } - Hash256::from(fixed_bytes) -} diff --git a/eth2/fork_choice/src/bitwise_lmd_ghost.rs b/eth2/fork_choice/src/bitwise_lmd_ghost.rs deleted file mode 100644 index 3ed57bf4dd..0000000000 --- a/eth2/fork_choice/src/bitwise_lmd_ghost.rs +++ /dev/null @@ -1,476 +0,0 @@ -//! The optimised bitwise LMD-GHOST fork choice rule. -use crate::{ForkChoice, ForkChoiceError}; -use bit_vec::BitVec; -use log::{debug, trace}; -use std::collections::HashMap; -use std::marker::PhantomData; -use std::sync::Arc; -use store::Store; -use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight}; - -//TODO: Pruning - Children -//TODO: Handle Syncing - -// NOTE: This uses u32 to represent difference between block heights. Thus this is only -// applicable for block height differences in the range of a u32. -// This can potentially be parallelized in some parts. - -/// Compute the base-2 logarithm of an integer, floored (rounded down) -#[inline] -fn log2_int(x: u64) -> u32 { - if x == 0 { - return 0; - } - 63 - x.leading_zeros() -} - -fn power_of_2_below(x: u64) -> u64 { - 2u64.pow(log2_int(x)) -} - -/// Stores the necessary data structures to run the optimised bitwise lmd ghost algorithm. -pub struct BitwiseLMDGhost { - /// A cache of known ancestors at given heights for a specific block. - //TODO: Consider FnvHashMap - cache: HashMap, Hash256>, - /// Log lookup table for blocks to their ancestors. - //TODO: Verify we only want/need a size 16 log lookup - ancestors: Vec>, - /// Stores the children for any given parent. - children: HashMap>, - /// The latest attestation targets as a map of validator index to block hash. - //TODO: Could this be a fixed size vec - latest_attestation_targets: HashMap, - /// Block and state storage. - store: Arc, - max_known_height: SlotHeight, - _phantom: PhantomData, -} - -impl BitwiseLMDGhost { - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to - /// weighted votes. - pub fn get_latest_votes( - &self, - state_root: &Hash256, - block_slot: Slot, - spec: &ChainSpec, - ) -> Result, ForkChoiceError> { - // get latest votes - // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // - // FORK_CHOICE_BALANCE_INCREMENT - // build a hashmap of block_hash to weighted votes - let mut latest_votes: HashMap = HashMap::new(); - // gets the current weighted votes - let current_state: BeaconState = self - .store - .get(&state_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; - - let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); - - for index in active_validator_indices { - let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) - / spec.effective_balance_increment; - if balance > 0 { - if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { - *latest_votes.entry(*target).or_insert_with(|| 0) += balance; - } - } - } - trace!("Latest votes: {:?}", latest_votes); - Ok(latest_votes) - } - - /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. - fn get_ancestor( - &mut self, - block_hash: Hash256, - target_height: SlotHeight, - spec: &ChainSpec, - ) -> Option { - // return None if we can't get the block from the db. - let block_height = { - let block_slot = self - .store - .get::(&block_hash) - .ok()? - .expect("Should have returned already if None") - .slot; - - block_slot.height(spec.genesis_slot) - }; - - // verify we haven't exceeded the block height - if target_height >= block_height { - if target_height > block_height { - return None; - } else { - return Some(block_hash); - } - } - // check if the result is stored in our cache - let cache_key = CacheKey::new(&block_hash, target_height.as_u64()); - if let Some(ancestor) = self.cache.get(&cache_key) { - return Some(*ancestor); - } - - // not in the cache recursively search for ancestors using a log-lookup - if let Some(ancestor) = { - let ancestor_lookup = *self.ancestors - [log2_int((block_height - target_height - 1u64).as_u64()) as usize] - .get(&block_hash) - //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(ancestor_lookup, target_height, &spec) - } { - // add the result to the cache - self.cache.insert(cache_key, ancestor); - return Some(ancestor); - } - - None - } - - // looks for an obvious block winner given the latest votes for a specific height - fn get_clear_winner( - &mut self, - latest_votes: &HashMap, - block_height: SlotHeight, - spec: &ChainSpec, - ) -> Option { - // map of vote counts for every hash at this height - let mut current_votes: HashMap = HashMap::new(); - let mut total_vote_count = 0; - - trace!("Clear winner at block height: {}", block_height); - // loop through the latest votes and count all votes - // these have already been weighted by balance - for (hash, votes) in latest_votes.iter() { - if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = *current_votes.get(&ancestor).unwrap_or_else(|| &0); - current_votes.insert(ancestor, current_vote_value + *votes); - total_vote_count += votes; - } - } - // Check if there is a clear block winner at this height. If so return it. - for (hash, votes) in current_votes.iter() { - if *votes > total_vote_count / 2 { - // we have a clear winner, return it - return Some(*hash); - } - } - // didn't find a clear winner - None - } - - // Finds the best child, splitting children into a binary tree, based on their hashes (Bitwise - // LMD Ghost) - fn choose_best_child(&self, votes: &HashMap) -> Option { - if votes.is_empty() { - return None; - } - let mut bitmask: BitVec = BitVec::new(); - // loop through all bits - for bit in 0..=256 { - let mut zero_votes = 0; - let mut one_votes = 0; - let mut single_candidate = (None, false); - - trace!("Child vote length: {}", votes.len()); - for (candidate, votes) in votes.iter() { - let candidate_bit: BitVec = BitVec::from_bytes(candidate.as_bytes()); - - // if the bitmasks don't match, exclude candidate - if !bitmask.iter().eq(candidate_bit.iter().take(bit)) { - trace!( - "Child: {} was removed in bit: {} with the bitmask: {:?}", - candidate, - bit, - bitmask - ); - continue; - } - if candidate_bit.get(bit) == Some(false) { - zero_votes += votes; - } else { - one_votes += votes; - } - - if single_candidate.0.is_none() { - single_candidate.0 = Some(candidate); - single_candidate.1 = true; - } else { - single_candidate.1 = false; - } - } - bitmask.push(one_votes > zero_votes); - if single_candidate.1 { - return Some(*single_candidate.0.expect("Cannot reach this")); - } - } - // should never reach here - None - } -} - -impl ForkChoice for BitwiseLMDGhost { - fn new(store: Arc) -> Self { - BitwiseLMDGhost { - cache: HashMap::new(), - ancestors: vec![HashMap::new(); 16], - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - max_known_height: SlotHeight::new(0), - store, - _phantom: PhantomData, - } - } - - fn add_block( - &mut self, - block: &BeaconBlock, - block_hash: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // get the height of the parent - let parent_height = self - .store - .get::(&block.previous_block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? - .slot - .height(spec.genesis_slot); - - let parent_hash = &block.previous_block_root; - - // add the new block to the children of parent - (*self - .children - .entry(block.previous_block_root) - .or_insert_with(|| vec![])) - .push(block_hash.clone()); - - // build the ancestor data structure - for index in 0..16 { - if parent_height % (1 << index) == 0 { - self.ancestors[index].insert(*block_hash, *parent_hash); - } else { - // TODO: This is unsafe. Will panic if parent_hash doesn't exist. Using it for debugging - let parent_ancestor = self.ancestors[index][parent_hash]; - self.ancestors[index].insert(*block_hash, parent_ancestor); - } - } - // update the max height - self.max_known_height = std::cmp::max(self.max_known_height, parent_height + 1); - Ok(()) - } - - fn add_attestation( - &mut self, - validator_index: u64, - target_block_root: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // simply add the attestation to the latest_attestation_target if the block_height is - // larger - trace!( - "Adding attestation of validator: {:?} for block: {}", - validator_index, - target_block_root - ); - let attestation_target = self - .latest_attestation_targets - .entry(validator_index) - .or_insert_with(|| *target_block_root); - // if we already have a value - if attestation_target != target_block_root { - trace!("Old attestation found: {:?}", attestation_target); - // get the height of the target block - let block_height = self - .store - .get::(&target_block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? - .slot - .height(spec.genesis_slot); - - // get the height of the past target block - let past_block_height = self - .store - .get::(&attestation_target)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? - .slot - .height(spec.genesis_slot); - // update the attestation only if the new target is higher - if past_block_height < block_height { - trace!("Updating old attestation"); - *attestation_target = *target_block_root; - } - } - Ok(()) - } - - /// Perform lmd_ghost on the current chain to find the head. - fn find_head( - &mut self, - justified_block_start: &Hash256, - spec: &ChainSpec, - ) -> Result { - debug!( - "Starting optimised fork choice at block: {}", - justified_block_start - ); - let block = self - .store - .get::(&justified_block_start)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; - - let block_slot = block.slot; - let state_root = block.state_root; - let mut block_height = block_slot.height(spec.genesis_slot); - - let mut current_head = *justified_block_start; - - let mut latest_votes = self.get_latest_votes(&state_root, block_slot, spec)?; - - // remove any votes that don't relate to our current head. - latest_votes - .retain(|hash, _| self.get_ancestor(*hash, block_height, spec) == Some(current_head)); - - // begin searching for the head - loop { - debug!( - "Iteration for block: {} with vote length: {}", - current_head, - latest_votes.len() - ); - // if there are no children, we are done, return the current_head - let children = match self.children.get(¤t_head) { - Some(children) => children.clone(), - None => { - debug!("Head found: {}", current_head); - return Ok(current_head); - } - }; - - // logarithmic lookup blocks to see if there are obvious winners, if so, - // progress to the next iteration. - let mut step = - power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u64()) / 2; - while step > 0 { - trace!("Current Step: {}", step); - if let Some(clear_winner) = self.get_clear_winner( - &latest_votes, - block_height - (block_height % step) + step, - spec, - ) { - current_head = clear_winner; - break; - } - step /= 2; - } - if step > 0 { - trace!("Found clear winner: {}", current_head); - } - // if our skip lookup failed and we only have one child, progress to that child - else if children.len() == 1 { - current_head = children[0]; - trace!( - "Lookup failed, only one child, proceeding to child: {}", - current_head - ); - } - // we need to find the best child path to progress down. - else { - trace!("Searching for best child"); - let mut child_votes = HashMap::new(); - for (voted_hash, vote) in latest_votes.iter() { - // if the latest votes correspond to a child - if let Some(child) = self.get_ancestor(*voted_hash, block_height + 1, spec) { - // add up the votes for each child - *child_votes.entry(child).or_insert_with(|| 0) += vote; - } - } - // check if we have votes of children, if not select the smallest hash child - if child_votes.is_empty() { - current_head = *children - .iter() - .min_by(|child1, child2| child1.cmp(child2)) - .expect("Must be children here"); - trace!( - "Children have no votes - smallest hash chosen: {}", - current_head - ); - } else { - // given the votes on the children, find the best child - current_head = self - .choose_best_child(&child_votes) - .ok_or(ForkChoiceError::CannotFindBestChild)?; - trace!("Best child found: {}", current_head); - } - } - - // didn't find head yet, proceed to next iteration - // update block height - block_height = self - .store - .get::(¤t_head)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? - .slot - .height(spec.genesis_slot); - // prune the latest votes for votes that are not part of current chosen chain - // more specifically, only keep votes that have head as an ancestor - for hash in latest_votes.keys() { - trace!( - "Ancestor for vote: {} at height: {} is: {:?}", - hash, - block_height, - self.get_ancestor(*hash, block_height, spec) - ); - } - latest_votes.retain(|hash, _| { - self.get_ancestor(*hash, block_height, spec) == Some(current_head) - }); - } - } -} - -/// Type for storing blocks in a memory cache. Key is comprised of block-hash plus the height. -#[derive(PartialEq, Eq, Hash)] -pub struct CacheKey { - block_hash: Hash256, - block_height: T, -} - -impl CacheKey { - pub fn new(block_hash: &Hash256, block_height: T) -> Self { - CacheKey { - block_hash: *block_hash, - block_height, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - pub fn test_power_of_2_below() { - assert_eq!(power_of_2_below(4), 4); - assert_eq!(power_of_2_below(5), 4); - assert_eq!(power_of_2_below(7), 4); - assert_eq!(power_of_2_below(24), 16); - assert_eq!(power_of_2_below(32), 32); - assert_eq!(power_of_2_below(33), 32); - assert_eq!(power_of_2_below(63), 32); - } - - #[test] - pub fn test_power_of_2_below_large() { - let pow: u64 = 1 << 24; - for x in (pow - 20)..(pow + 20) { - assert!(power_of_2_below(x) <= x, "{}", x); - } - } -} diff --git a/eth2/fork_choice/src/lib.rs b/eth2/fork_choice/src/lib.rs deleted file mode 100644 index f4a1fa5cb6..0000000000 --- a/eth2/fork_choice/src/lib.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! This crate stores the various implementations of fork-choice rules that can be used for the -//! beacon blockchain. -//! -//! There are three implementations. One is the naive longest chain rule (primarily for testing -//! purposes). The other two are proposed implementations of the LMD-GHOST fork-choice rule with various forms of optimisation. -//! -//! The current implementations are: -//! - [`longest-chain`]: Simplistic longest-chain fork choice - primarily for testing, **not for -//! production**. -//! - [`slow_lmd_ghost`]: This is a simple and very inefficient implementation given in the ethereum 2.0 -//! specifications (https://github.com/ethereum/eth2.0-specs/blob/v0.1/specs/core/0_beacon-chain.md#get_block_root). -//! - [`bitwise_lmd_ghost`]: This is an optimised version of bitwise LMD-GHOST as proposed -//! by Vitalik. The reference implementation can be found at: https://github.com/ethereum/research/blob/master/ghost/ghost.py -//! -//! [`longest-chain`]: struct.LongestChain.html -//! [`slow_lmd_ghost`]: struct.SlowLmdGhost.html -//! [`bitwise_lmd_ghost`]: struct.OptimisedLmdGhost.html - -pub mod bitwise_lmd_ghost; -pub mod longest_chain; -pub mod optimized_lmd_ghost; -pub mod slow_lmd_ghost; -pub mod test_utils; - -use std::sync::Arc; -use store::Error as DBError; -use types::{BeaconBlock, ChainSpec, Hash256}; - -pub use bitwise_lmd_ghost::BitwiseLMDGhost; -pub use longest_chain::LongestChain; -pub use optimized_lmd_ghost::OptimizedLMDGhost; -pub use slow_lmd_ghost::SlowLMDGhost; - -/// Defines the interface for Fork Choices. Each Fork choice will define their own data structures -/// which can be built in block processing through the `add_block` and `add_attestation` functions. -/// The main fork choice algorithm is specified in `find_head -pub trait ForkChoice: Send + Sync { - /// Create a new `ForkChoice` which reads from `store`. - fn new(store: Arc) -> Self; - - /// Called when a block has been added. Allows generic block-level data structures to be - /// built for a given fork-choice. - fn add_block( - &mut self, - block: &BeaconBlock, - block_hash: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError>; - /// Called when an attestation has been added. Allows generic attestation-level data structures to be built for a given fork choice. - // This can be generalised to a full attestation if required later. - fn add_attestation( - &mut self, - validator_index: u64, - target_block_hash: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError>; - /// The fork-choice algorithm to find the current canonical head of the chain. - // TODO: Remove the justified_start_block parameter and make it internal - fn find_head( - &mut self, - justified_start_block: &Hash256, - spec: &ChainSpec, - ) -> Result; -} - -/// Possible fork choice errors that can occur. -#[derive(Debug, PartialEq)] -pub enum ForkChoiceError { - MissingBeaconBlock(Hash256), - MissingBeaconState(Hash256), - IncorrectBeaconState(Hash256), - CannotFindBestChild, - ChildrenNotFound, - StorageError(String), - HeadNotFound, -} - -impl From for ForkChoiceError { - fn from(e: DBError) -> ForkChoiceError { - ForkChoiceError::StorageError(format!("{:?}", e)) - } -} - -/// Fork choice options that are currently implemented. -#[derive(Debug, Clone)] -pub enum ForkChoiceAlgorithm { - /// Chooses the longest chain becomes the head. Not for production. - LongestChain, - /// A simple and highly inefficient implementation of LMD ghost. - SlowLMDGhost, - /// An optimised version of bitwise LMD-GHOST by Vitalik. - BitwiseLMDGhost, - /// An optimised implementation of LMD ghost. - OptimizedLMDGhost, -} diff --git a/eth2/fork_choice/src/longest_chain.rs b/eth2/fork_choice/src/longest_chain.rs deleted file mode 100644 index 08e47cf393..0000000000 --- a/eth2/fork_choice/src/longest_chain.rs +++ /dev/null @@ -1,100 +0,0 @@ -use crate::{ForkChoice, ForkChoiceError}; -use std::sync::Arc; -use store::Store; -use types::{BeaconBlock, ChainSpec, Hash256, Slot}; - -pub struct LongestChain { - /// List of head block hashes - head_block_hashes: Vec, - /// Block storage. - store: Arc, -} - -impl ForkChoice for LongestChain { - fn new(store: Arc) -> Self { - LongestChain { - head_block_hashes: Vec::new(), - store, - } - } - - fn add_block( - &mut self, - block: &BeaconBlock, - block_hash: &Hash256, - _: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // add the block hash to head_block_hashes removing the parent if it exists - self.head_block_hashes - .retain(|hash| *hash != block.previous_block_root); - self.head_block_hashes.push(*block_hash); - Ok(()) - } - - fn add_attestation( - &mut self, - _: u64, - _: &Hash256, - _: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // do nothing - Ok(()) - } - - fn find_head(&mut self, _: &Hash256, _: &ChainSpec) -> Result { - let mut head_blocks: Vec<(usize, BeaconBlock)> = vec![]; - /* - * Load all the head_block hashes from the DB as SszBeaconBlocks. - */ - for (index, block_hash) in self.head_block_hashes.iter().enumerate() { - let block: BeaconBlock = self - .store - .get(&block_hash)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_hash))?; - head_blocks.push((index, block)); - } - - /* - * Loop through all the head blocks and find the highest slot. - */ - let highest_slot = head_blocks - .iter() - .fold(Slot::from(0u64), |highest, (_, block)| { - std::cmp::max(block.slot, highest) - }); - - // if we find no blocks, return Error - if highest_slot == 0 { - return Err(ForkChoiceError::HeadNotFound); - } - - /* - * Loop through all the highest blocks and sort them by highest hash. - * - * Ultimately, the index of the head_block hash with the highest slot and highest block - * hash will be the winner. - */ - - let head_index: Option = - head_blocks - .iter() - .fold(None, |smallest_index, (index, block)| { - if block.slot == highest_slot { - if smallest_index.is_none() { - return Some(*index); - } - return Some(std::cmp::min( - *index, - smallest_index.expect("Cannot be None"), - )); - } - smallest_index - }); - - if head_index.is_none() { - return Err(ForkChoiceError::HeadNotFound); - } - - Ok(self.head_block_hashes[head_index.unwrap()]) - } -} diff --git a/eth2/fork_choice/src/optimized_lmd_ghost.rs b/eth2/fork_choice/src/optimized_lmd_ghost.rs deleted file mode 100644 index 7a48c461e4..0000000000 --- a/eth2/fork_choice/src/optimized_lmd_ghost.rs +++ /dev/null @@ -1,447 +0,0 @@ -//! The optimised bitwise LMD-GHOST fork choice rule. -use crate::{ForkChoice, ForkChoiceError}; -use log::{debug, trace}; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::marker::PhantomData; -use std::sync::Arc; -use store::Store; -use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot, SlotHeight}; - -//TODO: Pruning - Children -//TODO: Handle Syncing - -// NOTE: This uses u32 to represent difference between block heights. Thus this is only -// applicable for block height differences in the range of a u32. -// This can potentially be parallelized in some parts. - -/// Compute the base-2 logarithm of an integer, floored (rounded down) -#[inline] -fn log2_int(x: u64) -> u32 { - if x == 0 { - return 0; - } - 63 - x.leading_zeros() -} - -fn power_of_2_below(x: u64) -> u64 { - 2u64.pow(log2_int(x)) -} - -/// Stores the necessary data structures to run the optimised lmd ghost algorithm. -pub struct OptimizedLMDGhost { - /// A cache of known ancestors at given heights for a specific block. - //TODO: Consider FnvHashMap - cache: HashMap, Hash256>, - /// Log lookup table for blocks to their ancestors. - //TODO: Verify we only want/need a size 16 log lookup - ancestors: Vec>, - /// Stores the children for any given parent. - children: HashMap>, - /// The latest attestation targets as a map of validator index to block hash. - //TODO: Could this be a fixed size vec - latest_attestation_targets: HashMap, - /// Block and state storage. - store: Arc, - max_known_height: SlotHeight, - _phantom: PhantomData, -} - -impl OptimizedLMDGhost { - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to - /// weighted votes. - pub fn get_latest_votes( - &self, - state_root: &Hash256, - block_slot: Slot, - spec: &ChainSpec, - ) -> Result, ForkChoiceError> { - // get latest votes - // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // - // FORK_CHOICE_BALANCE_INCREMENT - // build a hashmap of block_hash to weighted votes - let mut latest_votes: HashMap = HashMap::new(); - // gets the current weighted votes - let current_state: BeaconState = self - .store - .get(&state_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; - - let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); - - for index in active_validator_indices { - let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) - / spec.effective_balance_increment; - if balance > 0 { - if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { - *latest_votes.entry(*target).or_insert_with(|| 0) += balance; - } - } - } - trace!("Latest votes: {:?}", latest_votes); - Ok(latest_votes) - } - - /// Gets the ancestor at a given height `at_height` of a block specified by `block_hash`. - fn get_ancestor( - &mut self, - block_hash: Hash256, - target_height: SlotHeight, - spec: &ChainSpec, - ) -> Option { - // return None if we can't get the block from the db. - let block_height = { - let block_slot = self - .store - .get::(&block_hash) - .ok()? - .expect("Should have returned already if None") - .slot; - - block_slot.height(spec.genesis_slot) - }; - - // verify we haven't exceeded the block height - if target_height >= block_height { - if target_height > block_height { - return None; - } else { - return Some(block_hash); - } - } - // check if the result is stored in our cache - let cache_key = CacheKey::new(&block_hash, target_height.as_u64()); - if let Some(ancestor) = self.cache.get(&cache_key) { - return Some(*ancestor); - } - - // not in the cache recursively search for ancestors using a log-lookup - if let Some(ancestor) = { - let ancestor_lookup = *self.ancestors - [log2_int((block_height - target_height - 1u64).as_u64()) as usize] - .get(&block_hash) - //TODO: Panic if we can't lookup and fork choice fails - .expect("All blocks should be added to the ancestor log lookup table"); - self.get_ancestor(ancestor_lookup, target_height, &spec) - } { - // add the result to the cache - self.cache.insert(cache_key, ancestor); - return Some(ancestor); - } - - None - } - - // looks for an obvious block winner given the latest votes for a specific height - fn get_clear_winner( - &mut self, - latest_votes: &HashMap, - block_height: SlotHeight, - spec: &ChainSpec, - ) -> Option { - // map of vote counts for every hash at this height - let mut current_votes: HashMap = HashMap::new(); - let mut total_vote_count = 0; - - trace!("Clear winner at block height: {}", block_height); - // loop through the latest votes and count all votes - // these have already been weighted by balance - for (hash, votes) in latest_votes.iter() { - if let Some(ancestor) = self.get_ancestor(*hash, block_height, spec) { - let current_vote_value = *current_votes.get(&ancestor).unwrap_or_else(|| &0); - current_votes.insert(ancestor, current_vote_value + *votes); - total_vote_count += votes; - } - } - // Check if there is a clear block winner at this height. If so return it. - for (hash, votes) in current_votes.iter() { - if *votes > total_vote_count / 2 { - // we have a clear winner, return it - return Some(*hash); - } - } - // didn't find a clear winner - None - } - - // Finds the best child (one with highest votes) - fn choose_best_child(&self, votes: &HashMap) -> Option { - if votes.is_empty() { - return None; - } - - // Iterate through hashmap to get child with maximum votes - let best_child = votes.iter().max_by(|(child1, v1), (child2, v2)| { - let mut result = v1.cmp(v2); - // If votes are equal, choose smaller hash to break ties deterministically - if result == Ordering::Equal { - // Reverse so that max_by chooses smaller hash - result = child1.cmp(child2).reverse(); - } - result - }); - - Some(*best_child.unwrap().0) - } -} - -impl ForkChoice for OptimizedLMDGhost { - fn new(store: Arc) -> Self { - OptimizedLMDGhost { - cache: HashMap::new(), - ancestors: vec![HashMap::new(); 16], - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - max_known_height: SlotHeight::new(0), - store, - _phantom: PhantomData, - } - } - - fn add_block( - &mut self, - block: &BeaconBlock, - block_hash: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // get the height of the parent - let parent_height = self - .store - .get::(&block.previous_block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(block.previous_block_root))? - .slot - .height(spec.genesis_slot); - - let parent_hash = &block.previous_block_root; - - // add the new block to the children of parent - (*self - .children - .entry(block.previous_block_root) - .or_insert_with(|| vec![])) - .push(block_hash.clone()); - - // build the ancestor data structure - for index in 0..16 { - if parent_height % (1 << index) == 0 { - self.ancestors[index].insert(*block_hash, *parent_hash); - } else { - // TODO: This is unsafe. Will panic if parent_hash doesn't exist. Using it for debugging - let parent_ancestor = self.ancestors[index][parent_hash]; - self.ancestors[index].insert(*block_hash, parent_ancestor); - } - } - // update the max height - self.max_known_height = std::cmp::max(self.max_known_height, parent_height + 1); - Ok(()) - } - - fn add_attestation( - &mut self, - validator_index: u64, - target_block_root: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // simply add the attestation to the latest_attestation_target if the block_height is - // larger - trace!( - "Adding attestation of validator: {:?} for block: {}", - validator_index, - target_block_root - ); - let attestation_target = self - .latest_attestation_targets - .entry(validator_index) - .or_insert_with(|| *target_block_root); - // if we already have a value - if attestation_target != target_block_root { - trace!("Old attestation found: {:?}", attestation_target); - // get the height of the target block - let block_height = self - .store - .get::(&target_block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? - .slot - .height(spec.genesis_slot); - - // get the height of the past target block - let past_block_height = self - .store - .get::(&attestation_target)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? - .slot - .height(spec.genesis_slot); - // update the attestation only if the new target is higher - if past_block_height < block_height { - trace!("Updating old attestation"); - *attestation_target = *target_block_root; - } - } - Ok(()) - } - - /// Perform lmd_ghost on the current chain to find the head. - fn find_head( - &mut self, - justified_block_start: &Hash256, - spec: &ChainSpec, - ) -> Result { - debug!( - "Starting optimised fork choice at block: {}", - justified_block_start - ); - let block = self - .store - .get::(&justified_block_start)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; - - let block_slot = block.slot; - let state_root = block.state_root; - let mut block_height = block_slot.height(spec.genesis_slot); - - let mut current_head = *justified_block_start; - - let mut latest_votes = self.get_latest_votes(&state_root, block_slot, spec)?; - - // remove any votes that don't relate to our current head. - latest_votes - .retain(|hash, _| self.get_ancestor(*hash, block_height, spec) == Some(current_head)); - - // begin searching for the head - loop { - debug!( - "Iteration for block: {} with vote length: {}", - current_head, - latest_votes.len() - ); - // if there are no children, we are done, return the current_head - let children = match self.children.get(¤t_head) { - Some(children) => children.clone(), - None => { - debug!("Head found: {}", current_head); - return Ok(current_head); - } - }; - - // logarithmic lookup blocks to see if there are obvious winners, if so, - // progress to the next iteration. - let mut step = - power_of_2_below(self.max_known_height.saturating_sub(block_height).as_u64()) / 2; - while step > 0 { - trace!("Current Step: {}", step); - if let Some(clear_winner) = self.get_clear_winner( - &latest_votes, - block_height - (block_height % step) + step, - spec, - ) { - current_head = clear_winner; - break; - } - step /= 2; - } - if step > 0 { - trace!("Found clear winner: {}", current_head); - } - // if our skip lookup failed and we only have one child, progress to that child - else if children.len() == 1 { - current_head = children[0]; - trace!( - "Lookup failed, only one child, proceeding to child: {}", - current_head - ); - } - // we need to find the best child path to progress down. - else { - trace!("Searching for best child"); - let mut child_votes = HashMap::new(); - for (voted_hash, vote) in latest_votes.iter() { - // if the latest votes correspond to a child - if let Some(child) = self.get_ancestor(*voted_hash, block_height + 1, spec) { - // add up the votes for each child - *child_votes.entry(child).or_insert_with(|| 0) += vote; - } - } - // check if we have votes of children, if not select the smallest hash child - if child_votes.is_empty() { - current_head = *children - .iter() - .min_by(|child1, child2| child1.cmp(child2)) - .expect("Must be children here"); - trace!( - "Children have no votes - smallest hash chosen: {}", - current_head - ); - } else { - // given the votes on the children, find the best child - current_head = self - .choose_best_child(&child_votes) - .ok_or(ForkChoiceError::CannotFindBestChild)?; - trace!("Best child found: {}", current_head); - } - } - - // didn't find head yet, proceed to next iteration - // update block height - block_height = self - .store - .get::(¤t_head)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(current_head))? - .slot - .height(spec.genesis_slot); - // prune the latest votes for votes that are not part of current chosen chain - // more specifically, only keep votes that have head as an ancestor - for hash in latest_votes.keys() { - trace!( - "Ancestor for vote: {} at height: {} is: {:?}", - hash, - block_height, - self.get_ancestor(*hash, block_height, spec) - ); - } - latest_votes.retain(|hash, _| { - self.get_ancestor(*hash, block_height, spec) == Some(current_head) - }); - } - } -} - -/// Type for storing blocks in a memory cache. Key is comprised of block-hash plus the height. -#[derive(PartialEq, Eq, Hash)] -pub struct CacheKey { - block_hash: Hash256, - block_height: T, -} - -impl CacheKey { - pub fn new(block_hash: &Hash256, block_height: T) -> Self { - CacheKey { - block_hash: *block_hash, - block_height, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - pub fn test_power_of_2_below() { - assert_eq!(power_of_2_below(4), 4); - assert_eq!(power_of_2_below(5), 4); - assert_eq!(power_of_2_below(7), 4); - assert_eq!(power_of_2_below(24), 16); - assert_eq!(power_of_2_below(32), 32); - assert_eq!(power_of_2_below(33), 32); - assert_eq!(power_of_2_below(63), 32); - } - - #[test] - pub fn test_power_of_2_below_large() { - let pow: u64 = 1 << 24; - for x in (pow - 20)..(pow + 20) { - assert!(power_of_2_below(x) <= x, "{}", x); - } - } -} diff --git a/eth2/fork_choice/src/slow_lmd_ghost.rs b/eth2/fork_choice/src/slow_lmd_ghost.rs deleted file mode 100644 index 9b7a204002..0000000000 --- a/eth2/fork_choice/src/slow_lmd_ghost.rs +++ /dev/null @@ -1,212 +0,0 @@ -use crate::{ForkChoice, ForkChoiceError}; -use log::{debug, trace}; -use std::collections::HashMap; -use std::marker::PhantomData; -use std::sync::Arc; -use store::Store; -use types::{BeaconBlock, BeaconState, ChainSpec, EthSpec, Hash256, Slot}; - -//TODO: Pruning and syncing - -pub struct SlowLMDGhost { - /// The latest attestation targets as a map of validator index to block hash. - //TODO: Could this be a fixed size vec - latest_attestation_targets: HashMap, - /// Stores the children for any given parent. - children: HashMap>, - /// Block and state storage. - store: Arc, - _phantom: PhantomData, -} - -impl SlowLMDGhost { - /// Finds the latest votes weighted by validator balance. Returns a hashmap of block_hash to - /// weighted votes. - pub fn get_latest_votes( - &self, - state_root: &Hash256, - block_slot: Slot, - spec: &ChainSpec, - ) -> Result, ForkChoiceError> { - // get latest votes - // Note: Votes are weighted by min(balance, MAX_DEPOSIT_AMOUNT) // - // FORK_CHOICE_BALANCE_INCREMENT - // build a hashmap of block_hash to weighted votes - let mut latest_votes: HashMap = HashMap::new(); - // gets the current weighted votes - let current_state: BeaconState = self - .store - .get(state_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconState(*state_root))?; - - let active_validator_indices = - current_state.get_active_validator_indices(block_slot.epoch(E::slots_per_epoch())); - - for index in active_validator_indices { - let balance = std::cmp::min(current_state.balances[index], spec.max_effective_balance) - / spec.effective_balance_increment; - if balance > 0 { - if let Some(target) = self.latest_attestation_targets.get(&(index as u64)) { - *latest_votes.entry(*target).or_insert_with(|| 0) += balance; - } - } - } - trace!("Latest votes: {:?}", latest_votes); - Ok(latest_votes) - } - - /// Get the total number of votes for some given block root. - /// - /// The vote count is incremented each time an attestation target votes for a block root. - fn get_vote_count( - &self, - latest_votes: &HashMap, - block_root: &Hash256, - ) -> Result { - let mut count = 0; - let block_slot = self - .store - .get::(&block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))? - .slot; - - for (vote_hash, votes) in latest_votes.iter() { - let (root_at_slot, _) = self - .store - .get_block_at_preceeding_slot(*vote_hash, block_slot)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*block_root))?; - if root_at_slot == *block_root { - count += votes; - } - } - Ok(count) - } -} - -impl ForkChoice for SlowLMDGhost { - fn new(store: Arc) -> Self { - SlowLMDGhost { - latest_attestation_targets: HashMap::new(), - children: HashMap::new(), - store, - _phantom: PhantomData, - } - } - - /// Process when a block is added - fn add_block( - &mut self, - block: &BeaconBlock, - block_hash: &Hash256, - _: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // build the children hashmap - // add the new block to the children of parent - (*self - .children - .entry(block.previous_block_root) - .or_insert_with(|| vec![])) - .push(block_hash.clone()); - - // complete - Ok(()) - } - - fn add_attestation( - &mut self, - validator_index: u64, - target_block_root: &Hash256, - spec: &ChainSpec, - ) -> Result<(), ForkChoiceError> { - // simply add the attestation to the latest_attestation_target if the block_height is - // larger - trace!( - "Adding attestation of validator: {:?} for block: {}", - validator_index, - target_block_root - ); - let attestation_target = self - .latest_attestation_targets - .entry(validator_index) - .or_insert_with(|| *target_block_root); - // if we already have a value - if attestation_target != target_block_root { - trace!("Old attestation found: {:?}", attestation_target); - // get the height of the target block - let block_height = self - .store - .get::(&target_block_root)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*target_block_root))? - .slot - .height(spec.genesis_slot); - - // get the height of the past target block - let past_block_height = self - .store - .get::(&attestation_target)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*attestation_target))? - .slot - .height(spec.genesis_slot); - // update the attestation only if the new target is higher - if past_block_height < block_height { - trace!("Updating old attestation"); - *attestation_target = *target_block_root; - } - } - Ok(()) - } - - /// A very inefficient implementation of LMD ghost. - fn find_head( - &mut self, - justified_block_start: &Hash256, - spec: &ChainSpec, - ) -> Result { - debug!("Running LMD Ghost Fork-choice rule"); - let start = self - .store - .get::(&justified_block_start)? - .ok_or_else(|| ForkChoiceError::MissingBeaconBlock(*justified_block_start))?; - - let start_state_root = start.state_root; - - let latest_votes = self.get_latest_votes(&start_state_root, start.slot, spec)?; - - let mut head_hash = *justified_block_start; - - loop { - debug!("Iteration for block: {}", head_hash); - - let children = match self.children.get(&head_hash) { - Some(children) => children, - // we have found the head, exit - None => break, - }; - - // if we only have one child, use it - if children.len() == 1 { - trace!("Single child found."); - head_hash = children[0]; - continue; - } - trace!("Children found: {:?}", children); - - let mut head_vote_count = 0; - head_hash = children[0]; - for child_hash in children { - let vote_count = self.get_vote_count(&latest_votes, &child_hash)?; - trace!("Vote count for child: {} is: {}", child_hash, vote_count); - - if vote_count > head_vote_count { - head_hash = *child_hash; - head_vote_count = vote_count; - } - // resolve ties - choose smaller hash - else if vote_count == head_vote_count && *child_hash < head_hash { - head_hash = *child_hash; - } - } - } - Ok(head_hash) - } -} diff --git a/eth2/fork_choice/src/test_utils.rs b/eth2/fork_choice/src/test_utils.rs deleted file mode 100644 index 8ef20108aa..0000000000 --- a/eth2/fork_choice/src/test_utils.rs +++ /dev/null @@ -1,91 +0,0 @@ -use crate::ForkChoice; -use std::marker::PhantomData; -use std::sync::Arc; -use store::Store; -use types::{ - test_utils::{SeedableRng, TestRandom, TestingBeaconStateBuilder, XorShiftRng}, - BeaconBlock, BeaconState, EthSpec, Hash256, Keypair, MainnetEthSpec, -}; - -/// Creates a chain of blocks and produces `ForkChoice` instances with pre-filled stores. -pub struct TestingForkChoiceBuilder { - store: Arc, - pub chain: Vec<(Hash256, BeaconBlock)>, - _phantom: PhantomData, -} - -impl TestingForkChoiceBuilder { - pub fn new(validator_count: usize, chain_length: usize, store: Arc) -> Self { - let chain = - get_chain_of_blocks::(chain_length, validator_count, store.clone()); - - Self { - store, - chain, - _phantom: PhantomData, - } - } - - pub fn genesis_root(&self) -> Hash256 { - self.chain[0].0 - } - - /// Return a new `ForkChoice` instance with a chain stored in it's `Store`. - pub fn build>(&self) -> F { - F::new(self.store.clone()) - } -} - -fn get_state(validator_count: usize) -> BeaconState { - let spec = T::default_spec(); - - let builder: TestingBeaconStateBuilder = - TestingBeaconStateBuilder::from_single_keypair(validator_count, &Keypair::random(), &spec); - let (state, _keypairs) = builder.build(); - state -} - -/// Generates a chain of blocks of length `len`. -/// -/// Creates a `BeaconState` for the block and stores it in `Store`, along with the block. -/// -/// Returns the chain of blocks. -fn get_chain_of_blocks( - len: usize, - validator_count: usize, - store: Arc, -) -> Vec<(Hash256, BeaconBlock)> { - let spec = T::default_spec(); - let mut blocks_and_roots: Vec<(Hash256, BeaconBlock)> = vec![]; - let mut unique_hashes = (0..).map(Hash256::from); - let mut random_block = BeaconBlock::random_for_test(&mut XorShiftRng::from_seed([42; 16])); - random_block.previous_block_root = Hash256::zero(); - let beacon_state = get_state::(validator_count); - - for i in 0..len { - let slot = spec.genesis_slot + i as u64; - - // Generate and store the state. - let mut state = beacon_state.clone(); - state.slot = slot; - let state_root = unique_hashes.next().unwrap(); - store.put(&state_root, &state).unwrap(); - - // Generate the block. - let mut block = random_block.clone(); - block.slot = slot; - block.state_root = state_root; - - // Chain all the blocks to their parents. - if i > 0 { - block.previous_block_root = blocks_and_roots[i - 1].0; - } - - // Store the block. - let block_root = unique_hashes.next().unwrap(); - store.put(&block_root, &block).unwrap(); - blocks_and_roots.push((block_root, block)); - } - - blocks_and_roots -} diff --git a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml deleted file mode 100644 index 61b0b05c40..0000000000 --- a/eth2/fork_choice/tests/bitwise_lmd_ghost_test_vectors.yaml +++ /dev/null @@ -1,144 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests for bitwise lmd ghost. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - weights: - - b0: 0 - - b1: 0 - - b2: 5 - - b3: 10 - heads: - - id: 'b3' -# bitwise LMD ghost example. bitwise GHOST gives b2 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 4 - - b3: 3 - heads: - - id: 'b2' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b1' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b6' - weights: - - b0: 0 - - b1: 3 - - b2: 2 - - b3: 1 - - b4: 1 - - b5: 1 - - b6: 2 - - b7: 2 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b2' - - id: 'b8' - parent: 'b3' - - id: 'b9' - parent: 'b3' - weights: - - b1: 2 - - b2: 1 - - b3: 1 - - b4: 7 - - b5: 5 - - b6: 2 - - b7: 4 - - b8: 4 - - b9: 2 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - - id: 'b4' - parent: 'b1' - - id: 'b5' - parent: 'b1' - - id: 'b6' - parent: 'b2' - - id: 'b7' - parent: 'b2' - - id: 'b8' - parent: 'b3' - - id: 'b9' - parent: 'b3' - weights: - - b1: 1 - - b2: 1 - - b3: 1 - - b4: 7 - - b5: 5 - - b6: 2 - - b7: 4 - - b8: 4 - - b9: 2 - heads: - - id: 'b7' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - weights: - - b1: 0 - - b2: 0 - heads: - - id: 'b1' - diff --git a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml b/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml deleted file mode 100644 index e7847de11a..0000000000 --- a/eth2/fork_choice/tests/lmd_ghost_test_vectors.yaml +++ /dev/null @@ -1,65 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests for lmd ghost. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - weights: - - b0: 0 - - b1: 0 - - b2: 5 - - b3: 10 - heads: - - id: 'b3' -# bitwise LMD ghost example. GHOST gives b1 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 4 - - b3: 3 - heads: - - id: 'b1' -# equal weights children. Should choose lower hash b2 -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - - id: 'b3' - parent: 'b0' - weights: - - b1: 5 - - b2: 6 - - b3: 6 - heads: - - id: 'b2' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b0' - weights: - - b1: 0 - - b2: 0 - heads: - - id: 'b1' diff --git a/eth2/fork_choice/tests/longest_chain_test_vectors.yaml b/eth2/fork_choice/tests/longest_chain_test_vectors.yaml deleted file mode 100644 index e1cd61f06a..0000000000 --- a/eth2/fork_choice/tests/longest_chain_test_vectors.yaml +++ /dev/null @@ -1,51 +0,0 @@ -title: Fork-choice Tests -summary: A collection of abstract fork-choice tests to verify the longest chain fork-choice rule. -test_suite: Fork-Choice - -test_cases: -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b1' - - id: 'b4' - parent: 'b3' - weights: - - b0: 0 - - b1: 0 - - b2: 10 - - b3: 1 - heads: - - id: 'b4' -- blocks: - - id: 'b0' - parent: 'b0' - - id: 'b1' - parent: 'b0' - - id: 'b2' - parent: 'b1' - - id: 'b3' - parent: 'b2' - - id: 'b4' - parent: 'b3' - - id: 'b5' - parent: 'b0' - - id: 'b6' - parent: 'b5' - - id: 'b7' - parent: 'b6' - - id: 'b8' - parent: 'b7' - - id: 'b9' - parent: 'b8' - weights: - - b0: 5 - - b1: 20 - - b2: 10 - - b3: 10 - heads: - - id: 'b9' diff --git a/eth2/fork_choice/tests/tests.rs b/eth2/fork_choice/tests/tests.rs deleted file mode 100644 index 4fa266d306..0000000000 --- a/eth2/fork_choice/tests/tests.rs +++ /dev/null @@ -1,231 +0,0 @@ -#![cfg(not(debug_assertions))] -/// Tests the available fork-choice algorithms -pub use beacon_chain::BeaconChain; -use bls::Signature; -use store::MemoryStore; -use store::Store; -// use env_logger::{Builder, Env}; -use fork_choice::{BitwiseLMDGhost, ForkChoice, LongestChain, OptimizedLMDGhost, SlowLMDGhost}; -use std::collections::HashMap; -use std::sync::Arc; -use std::{fs::File, io::prelude::*, path::PathBuf}; -use types::test_utils::TestingBeaconStateBuilder; -use types::{ - BeaconBlock, BeaconBlockBody, Eth1Data, EthSpec, Hash256, Keypair, MainnetEthSpec, Slot, -}; -use yaml_rust::yaml; - -// Note: We Assume the block Id's are hex-encoded. - -#[test] -fn test_optimized_lmd_ghost() { - // set up logging - // Builder::from_env(Env::default().default_filter_or("trace")).init(); - - test_yaml_vectors::>( - "tests/lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_bitwise_lmd_ghost() { - // set up logging - //Builder::from_env(Env::default().default_filter_or("trace")).init(); - - test_yaml_vectors::>( - "tests/bitwise_lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_slow_lmd_ghost() { - test_yaml_vectors::>( - "tests/lmd_ghost_test_vectors.yaml", - 100, - ); -} - -#[test] -fn test_longest_chain() { - test_yaml_vectors::>("tests/longest_chain_test_vectors.yaml", 100); -} - -// run a generic test over given YAML test vectors -fn test_yaml_vectors>( - yaml_file_path: &str, - emulated_validators: usize, // the number of validators used to give weights. -) { - // load test cases from yaml - let test_cases = load_test_cases_from_yaml(yaml_file_path); - - // default vars - let spec = MainnetEthSpec::default_spec(); - let zero_hash = Hash256::zero(); - let eth1_data = Eth1Data { - deposit_count: 0, - deposit_root: zero_hash.clone(), - block_hash: zero_hash.clone(), - }; - let randao_reveal = Signature::empty_signature(); - let signature = Signature::empty_signature(); - let body = BeaconBlockBody { - eth1_data, - randao_reveal, - graffiti: [0; 32], - proposer_slashings: vec![], - attester_slashings: vec![], - attestations: vec![], - deposits: vec![], - voluntary_exits: vec![], - transfers: vec![], - }; - - // process the tests - for test_case in test_cases { - // setup a fresh test - let (mut fork_choice, store, state_root) = setup_inital_state::(emulated_validators); - - // keep a hashmap of block_id's to block_hashes (random hashes to abstract block_id) - //let mut block_id_map: HashMap = HashMap::new(); - // keep a list of hash to slot - let mut block_slot: HashMap = HashMap::new(); - // assume the block tree is given to us in order. - let mut genesis_hash = None; - for block in test_case["blocks"].clone().into_vec().unwrap() { - let block_id = block["id"].as_str().unwrap().to_string(); - let parent_id = block["parent"].as_str().unwrap().to_string(); - - // default params for genesis - let block_hash = id_to_hash(&block_id); - let mut slot = spec.genesis_slot; - let previous_block_root = id_to_hash(&parent_id); - - // set the slot and parent based off the YAML. Start with genesis; - // if not the genesis, update slot - if parent_id != block_id { - // find parent slot - slot = *(block_slot - .get(&previous_block_root) - .expect("Parent should have a slot number")) - + 1; - } else { - genesis_hash = Some(block_hash); - } - - // update slot mapping - block_slot.insert(block_hash, slot); - - // build the BeaconBlock - let beacon_block = BeaconBlock { - slot, - previous_block_root, - state_root: state_root.clone(), - signature: signature.clone(), - body: body.clone(), - }; - - // Store the block. - store.put(&block_hash, &beacon_block).unwrap(); - - // run add block for fork choice if not genesis - if parent_id != block_id { - fork_choice - .add_block(&beacon_block, &block_hash, &spec) - .unwrap(); - } - } - - // add the weights (attestations) - let mut current_validator = 0; - for id_map in test_case["weights"].clone().into_vec().unwrap() { - // get the block id and weights - for (map_id, map_weight) in id_map.as_hash().unwrap().iter() { - let id = map_id.as_str().unwrap(); - let block_root = id_to_hash(&id.to_string()); - let weight = map_weight.as_i64().unwrap(); - // we assume a validator has a value 1 and add an attestation for to achieve the - // correct weight - for _ in 0..weight { - assert!( - current_validator <= emulated_validators, - "Not enough validators to emulate weights" - ); - fork_choice - .add_attestation(current_validator as u64, &block_root, &spec) - .unwrap(); - current_validator += 1; - } - } - } - - // everything is set up, run the fork choice, using genesis as the head - let head = fork_choice - .find_head(&genesis_hash.unwrap(), &spec) - .unwrap(); - - // compare the result to the expected test - let success = test_case["heads"] - .clone() - .into_vec() - .unwrap() - .iter() - .find(|heads| id_to_hash(&heads["id"].as_str().unwrap().to_string()) == head) - .is_some(); - - println!("Head found: {}", head); - assert!(success, "Did not find one of the possible heads"); - } -} - -// loads the test_cases from the supplied yaml file -fn load_test_cases_from_yaml(file_path: &str) -> Vec { - // load the yaml - let mut file = { - let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file_path_buf.push(file_path); - File::open(file_path_buf).unwrap() - }; - let mut yaml_str = String::new(); - file.read_to_string(&mut yaml_str).unwrap(); - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - doc["test_cases"].as_vec().unwrap().clone() -} - -fn setup_inital_state( - // fork_choice_algo: &ForkChoiceAlgorithm, - num_validators: usize, -) -> (T, Arc, Hash256) -where - T: ForkChoice, -{ - let store = Arc::new(MemoryStore::open()); - - let fork_choice = ForkChoice::new(store.clone()); - let spec = MainnetEthSpec::default_spec(); - - let mut state_builder: TestingBeaconStateBuilder = - TestingBeaconStateBuilder::from_single_keypair(num_validators, &Keypair::random(), &spec); - state_builder.build_caches(&spec).unwrap(); - let (state, _keypairs) = state_builder.build(); - - let state_root = state.canonical_root(); - store.put(&state_root, &state).unwrap(); - - // return initialised vars - (fork_choice, store, state_root) -} - -// convert a block_id into a Hash256 -- assume input is hex encoded; -fn id_to_hash(id: &String) -> Hash256 { - let bytes = hex::decode(id).expect("Block ID should be hex"); - - let len = std::cmp::min(bytes.len(), 32); - let mut fixed_bytes = [0u8; 32]; - for (index, byte) in bytes.iter().take(32).enumerate() { - fixed_bytes[32 - len + index] = *byte; - } - Hash256::from(fixed_bytes) -}