From 41ef019d57b92e0455e4d0d60db646af20800c5f Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sat, 29 Dec 2018 14:39:56 +1100 Subject: [PATCH 01/15] Add unfinished chain code --- beacon_chain/abstract_chain/Cargo.toml | 10 + .../abstract_chain/src/block_reader.rs | 16 ++ beacon_chain/abstract_chain/src/lib.rs | 185 ++++++++++++++++++ .../db/src/stores/beacon_state_store.rs | 33 ++++ 4 files changed, 244 insertions(+) create mode 100644 beacon_chain/abstract_chain/Cargo.toml create mode 100644 beacon_chain/abstract_chain/src/block_reader.rs create mode 100644 beacon_chain/abstract_chain/src/lib.rs create mode 100644 lighthouse/db/src/stores/beacon_state_store.rs diff --git a/beacon_chain/abstract_chain/Cargo.toml b/beacon_chain/abstract_chain/Cargo.toml new file mode 100644 index 0000000000..6e11036cba --- /dev/null +++ b/beacon_chain/abstract_chain/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "abstract_chain" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +hashing = { path = "../utils/hashing" } +ssz = { path = "../utils/ssz" } +types = { path = "../types" } diff --git a/beacon_chain/abstract_chain/src/block_reader.rs b/beacon_chain/abstract_chain/src/block_reader.rs new file mode 100644 index 0000000000..9d66dc5b69 --- /dev/null +++ b/beacon_chain/abstract_chain/src/block_reader.rs @@ -0,0 +1,16 @@ +use types::{BeaconBlock, Hash256}; + +pub trait BlockReader { + fn slot(&self) -> u64; + fn parent_root(&self) -> Hash256; +} + +impl BlockReader for BeaconBlock { + fn slot(&self) -> u64 { + self.slot + } + + fn parent_root(&self) -> Hash256 { + self.parent_root + } +} diff --git a/beacon_chain/abstract_chain/src/lib.rs b/beacon_chain/abstract_chain/src/lib.rs new file mode 100644 index 0000000000..e348dfdba6 --- /dev/null +++ b/beacon_chain/abstract_chain/src/lib.rs @@ -0,0 +1,185 @@ +/* + * + */ + + +use hashing::canonical_hash; +use ssz::{Encodable, Decodable, DecodeError, SszStream, ssz_encode}; +use std::collections::{HashMap, HashSet}; +use types::Hash256; + +mod block_reader; + +use crate::block_reader::BlockReader; + +pub trait AbstractChain { + type Block; + type Hash; + + fn genesis(&mut self); + + fn receive_block(&mut self, block: &BlockReader) -> bool; + + fn block_by_root(&self, root: &Self::Hash) -> Option<&Self::Block>; +} + +pub struct TestBlock { + slot: u64, + parent_root: Hash256, + state_root: Hash256, + weight: u8, +} + +impl Encodable for TestBlock { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.slot); + s.append(&self.parent_root); + } +} + +impl Decodable for TestBlock { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (slot, i) = <_>::ssz_decode(bytes, i)?; + let (parent_root, i) = <_>::ssz_decode(bytes, i)?; + let (state_root, i) = <_>::ssz_decode(bytes, i)?; + let (weight, i) = <_>::ssz_decode(bytes, i)?; + + Ok(( + Self { + slot, + parent_root, + state_root, + weight, + }, + i, + )) + } +} + +impl TestBlock { + fn canonical_hash(&self) -> Hash256 { + Hash256::from(&canonical_hash(&ssz_encode(self))[..]) + } +} + +impl BlockReader for TestBlock { + fn slot(&self) -> u64 { + self.slot + } + + fn parent_root(&self) -> Hash256 { + self.parent_root + } +} + +#[derive(Clone)] +pub struct TestState { + total_weight: u64, + skipped_slots: u64, +} + +impl Encodable for TestState { + fn ssz_append(&self, s: &mut SszStream) { + s.append(&self.total_weight); + s.append(&self.skipped_slots); + } +} + +impl TestState { + fn canonical_hash(&self) -> Hash256 { + Hash256::from(&canonical_hash(&ssz_encode(self))[..]) + } +} + +impl Decodable for TestState { + fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { + let (total_weight, i) = <_>::ssz_decode(bytes, i)?; + let (skipped_slots, i) = <_>::ssz_decode(bytes, i)?; + + Ok(( + Self { + total_weight, + skipped_slots, + }, + i, + )) + } +} + +pub struct TestChain { + slot: u64, + block_store: HashMap, + state_store: HashMap, + leaf_nodes: HashSet, + canonical_leaf_node: Hash256, +} + +impl AbstractChain for TestChain { + type Block = TestBlock; + type Hash = Hash256; + + fn genesis(&mut self) { + let genesis_state = TestState { + total_weight: 255, + skipped_slots: 0, + }; + let state_root = genesis_state.canonical_hash(); + let genesis_block = TestBlock { + slot: 0, + parent_root: Hash256::zero(), + state_root: state_root.clone(), + weight: 255, + }; + let block_root = genesis_block.canonical_hash(); + self.block_store.insert(block_root, genesis_block); + self.state_store.insert(state_root, genesis_state); + self.leaf_nodes.insert(block_root); + self.canonical_leaf_node = block_root; + self.slot = 0; + } + + fn block_by_root(&self, root: &Hash256) -> Option<&Self::Block> { + self.block_store.get(root) + } + + fn receive_block(&mut self, block: &BlockReader) -> bool { + let block_hash = block.canonical_hash(); + let parent_block = match self.block_by_root(&block.parent_root()) { + Some(block) => block, + None => return false + }; + let slot_distance = block.slot().saturating_sub(parent_block.slot()); + if slot_distance > 0 { + let mut state = self.state_store.get(&block.parent_root()).unwrap().clone(); + for _ in parent_block.slot()..(block.slot() - 1) { + state.skipped_slots += 1; + } + state.total_weight += u64::from(block.weight); + let state_root = state.canonical_hash(); + if state_root != block.state_root { + return false; + } + self.leaf_nodes.remove(&block.parent_root()); + self.leaf_nodes.insert(block_hash); + + let canonical_state = { + let canonical_block = self.block_store.get(&self.canonical_leaf_node).unwrap(); + self.state_store.get(&canonical_block.state_root).unwrap() + }; + // New block is canonical, chain weight is greatest. + if state.total_weight > canonical_state.total_weight { + self.canonical_leaf_node = block_hash; + } + // New block chain weight equals existing canonical block chain weight. + else if state.total_weight == canonical_state.total_weight { + // New block is canonical as hash is highest. + if block_hash > self.canonical_leaf_node { + self.canonical_leaf_node = block_hash; + } + } + true + } else { + false + } + } +} diff --git a/lighthouse/db/src/stores/beacon_state_store.rs b/lighthouse/db/src/stores/beacon_state_store.rs new file mode 100644 index 0000000000..4d3bbb36c9 --- /dev/null +++ b/lighthouse/db/src/stores/beacon_state_store.rs @@ -0,0 +1,33 @@ +use super::STATES_DB_COLUMN as DB_COLUMN; +use super::{ClientDB, DBError}; +use std::sync::Arc; +use types::Hash256; + +pub struct BeaconStateStore +where + T: ClientDB, +{ + db: Arc, +} + +impl BeaconStateStore { + pub fn new(db: Arc) -> Self { + Self { db } + } + + pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { + self.db.put(DB_COLUMN, hash, ssz) + } + + pub fn get(&self, hash: &[u8]) -> Result>, DBError> { + self.db.get(DB_COLUMN, hash) + } + + pub fn exists(&self, hash: &[u8]) -> Result { + self.db.exists(DB_COLUMN, hash) + } + + pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { + self.db.delete(DB_COLUMN, hash) + } +} From dc8fbf813fd44726c0fe287b9f0602bccb4a9197 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Dec 2018 12:56:07 +1100 Subject: [PATCH 02/15] Update db function signatures --- .../db/src/stores/beacon_block_store.rs | 53 ++++++++++++------- .../db/src/stores/beacon_state_store.rs | 16 +++++- lighthouse/db/src/stores/mod.rs | 5 +- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/lighthouse/db/src/stores/beacon_block_store.rs b/lighthouse/db/src/stores/beacon_block_store.rs index 5f777ea0ae..5faa208dff 100644 --- a/lighthouse/db/src/stores/beacon_block_store.rs +++ b/lighthouse/db/src/stores/beacon_block_store.rs @@ -2,7 +2,7 @@ use super::BLOCKS_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; use ssz::{Decodable, DecodeError}; use std::sync::Arc; -use types::Hash256; +use types::{readers::BeaconBlockReader, BeaconBlock, Hash256}; type BeaconBlockHash = Vec; type BeaconBlockSsz = Vec; @@ -26,22 +26,35 @@ impl BeaconBlockStore { Self { db } } - pub fn put_serialized_block(&self, hash: &[u8], ssz: &[u8]) -> Result<(), DBError> { + pub fn put(&self, hash: &[u8], ssz: &[u8]) -> Result<(), DBError> { self.db.put(DB_COLUMN, hash, ssz) } - pub fn get_serialized_block(&self, hash: &[u8]) -> Result>, DBError> { + pub fn get(&self, hash: &[u8]) -> Result>, DBError> { self.db.get(DB_COLUMN, hash) } - pub fn block_exists(&self, hash: &[u8]) -> Result { + pub fn exists(&self, hash: &[u8]) -> Result { self.db.exists(DB_COLUMN, hash) } - pub fn delete_block(&self, hash: &[u8]) -> Result<(), DBError> { + pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { self.db.delete(DB_COLUMN, hash) } + /// Retuns a fully de-serialized `BeaconBlock` (or `None` if hash not known). + pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad Block SSZ.".to_string(), + })?; + Ok(Some(block)) + } + } + } + /// Retrieve the block at a slot given a "head_hash" and a slot. /// /// A "head_hash" must be a block hash with a slot number greater than or equal to the desired @@ -56,7 +69,7 @@ impl BeaconBlockStore { head_hash: &[u8], slot: u64, ) -> Result, BeaconBlockAtSlotError> { - match self.get_serialized_block(head_hash)? { + match self.get(head_hash)? { None => Err(BeaconBlockAtSlotError::UnknownBeaconBlock), Some(ssz) => { let (retrieved_slot, parent_hash) = slot_and_parent_from_block_ssz(&ssz, 0) @@ -102,19 +115,19 @@ mod tests { use types::Hash256; #[test] - fn test_put_serialized_block() { + fn test_put() { let db = Arc::new(MemoryDB::open()); let store = BeaconBlockStore::new(db.clone()); let ssz = "some bytes".as_bytes(); let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - store.put_serialized_block(hash, ssz).unwrap(); + store.put(hash, ssz).unwrap(); assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); } #[test] - fn test_get_serialized_block() { + fn test_get() { let db = Arc::new(MemoryDB::open()); let store = BeaconBlockStore::new(db.clone()); @@ -122,11 +135,11 @@ mod tests { let hash = &Hash256::from("some hash".as_bytes()).to_vec(); db.put(DB_COLUMN, hash, ssz).unwrap(); - assert_eq!(store.get_serialized_block(hash).unwrap().unwrap(), ssz); + assert_eq!(store.get(hash).unwrap().unwrap(), ssz); } #[test] - fn test_get_unknown_serialized_block() { + fn test_get_unknown() { let db = Arc::new(MemoryDB::open()); let store = BeaconBlockStore::new(db.clone()); @@ -135,11 +148,11 @@ mod tests { let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); db.put(DB_COLUMN, other_hash, ssz).unwrap(); - assert_eq!(store.get_serialized_block(hash).unwrap(), None); + assert_eq!(store.get(hash).unwrap(), None); } #[test] - fn test_block_exists() { + fn test_exists() { let db = Arc::new(MemoryDB::open()); let store = BeaconBlockStore::new(db.clone()); @@ -147,7 +160,7 @@ mod tests { let hash = &Hash256::from("some hash".as_bytes()).to_vec(); db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(store.block_exists(hash).unwrap()); + assert!(store.exists(hash).unwrap()); } #[test] @@ -160,11 +173,11 @@ mod tests { let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(!store.block_exists(other_hash).unwrap()); + assert!(!store.exists(other_hash).unwrap()); } #[test] - fn test_delete_block() { + fn test_delete() { let db = Arc::new(MemoryDB::open()); let store = BeaconBlockStore::new(db.clone()); @@ -174,7 +187,7 @@ mod tests { db.put(DB_COLUMN, hash, ssz).unwrap(); assert!(db.exists(DB_COLUMN, hash).unwrap()); - store.delete_block(hash).unwrap(); + store.delete(hash).unwrap(); assert!(!db.exists(DB_COLUMN, hash).unwrap()); } @@ -228,7 +241,7 @@ mod tests { for w in 0..wc { let key = (t * w) as u8; let val = 42; - bs.put_serialized_block(&vec![key], &vec![val]).unwrap(); + bs.put(&vec![key], &vec![val]).unwrap(); } }); handles.push(handle); @@ -241,8 +254,8 @@ mod tests { for t in 0..thread_count { for w in 0..write_count { let key = (t * w) as u8; - assert!(bs.block_exists(&vec![key]).unwrap()); - let val = bs.get_serialized_block(&vec![key]).unwrap().unwrap(); + assert!(bs.exists(&vec![key]).unwrap()); + let val = bs.get(&vec![key]).unwrap().unwrap(); assert_eq!(vec![42], val); } } diff --git a/lighthouse/db/src/stores/beacon_state_store.rs b/lighthouse/db/src/stores/beacon_state_store.rs index 4d3bbb36c9..687fc0f222 100644 --- a/lighthouse/db/src/stores/beacon_state_store.rs +++ b/lighthouse/db/src/stores/beacon_state_store.rs @@ -1,7 +1,8 @@ use super::STATES_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; +use ssz::Decodable; use std::sync::Arc; -use types::Hash256; +use types::{readers::BeaconStateReader, BeaconState, Hash256}; pub struct BeaconStateStore where @@ -30,4 +31,17 @@ impl BeaconStateStore { pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { self.db.delete(DB_COLUMN, hash) } + + /// Retuns a fully de-serialized `BeaconState` (or `None` if hash not known). + pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + match self.get(&hash)? { + None => Ok(None), + Some(ssz) => { + let (state, _) = BeaconState::ssz_decode(&ssz, 0).map_err(|_| DBError { + message: "Bad State SSZ.".to_string(), + })?; + Ok(Some(state)) + } + } + } } diff --git a/lighthouse/db/src/stores/mod.rs b/lighthouse/db/src/stores/mod.rs index f3adec2576..7fc51abe70 100644 --- a/lighthouse/db/src/stores/mod.rs +++ b/lighthouse/db/src/stores/mod.rs @@ -1,17 +1,20 @@ use super::{ClientDB, DBError}; mod beacon_block_store; +mod beacon_state_store; mod pow_chain_store; mod validator_store; pub use self::beacon_block_store::{BeaconBlockAtSlotError, BeaconBlockStore}; +pub use self::beacon_state_store::BeaconStateStore; pub use self::pow_chain_store::PoWChainStore; pub use self::validator_store::{ValidatorStore, ValidatorStoreError}; use super::bls; pub const BLOCKS_DB_COLUMN: &str = "blocks"; +pub const STATES_DB_COLUMN: &str = "states"; pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; pub const VALIDATOR_DB_COLUMN: &str = "validator"; -pub const COLUMNS: [&str; 3] = [BLOCKS_DB_COLUMN, POW_CHAIN_DB_COLUMN, VALIDATOR_DB_COLUMN]; +pub const COLUMNS: [&str; 4] = [BLOCKS_DB_COLUMN, STATES_DB_COLUMN, POW_CHAIN_DB_COLUMN, VALIDATOR_DB_COLUMN]; From 2b63ece2445faadb6ef774e29fa714260bf6ad07 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Dec 2018 12:59:24 +1100 Subject: [PATCH 03/15] Move beacon_chain into lighthouse dir --- beacon_chain/chain/Cargo.toml | 16 --- beacon_chain/chain/src/block_processing.rs | 29 ----- beacon_chain/chain/src/lib.rs | 110 ------------------ lighthouse/beacon_chain/Cargo.toml | 17 +++ .../beacon_chain/src/block_processing.rs | 72 ++++++++++++ .../beacon_chain/src/block_production.rs | 64 ++++++++++ lighthouse/beacon_chain/src/lib.rs | 83 +++++++++++++ .../beacon_chain}/src/maps.rs | 0 .../beacon_chain}/src/stores.rs | 0 .../beacon_chain}/src/transition.rs | 2 +- lighthouse/beacon_chain/tests/chain_test.rs | 49 ++++++++ 11 files changed, 286 insertions(+), 156 deletions(-) delete mode 100644 beacon_chain/chain/Cargo.toml delete mode 100644 beacon_chain/chain/src/block_processing.rs delete mode 100644 beacon_chain/chain/src/lib.rs create mode 100644 lighthouse/beacon_chain/Cargo.toml create mode 100644 lighthouse/beacon_chain/src/block_processing.rs create mode 100644 lighthouse/beacon_chain/src/block_production.rs create mode 100644 lighthouse/beacon_chain/src/lib.rs rename {beacon_chain/chain => lighthouse/beacon_chain}/src/maps.rs (100%) rename {beacon_chain/chain => lighthouse/beacon_chain}/src/stores.rs (100%) rename {beacon_chain/chain => lighthouse/beacon_chain}/src/transition.rs (97%) create mode 100644 lighthouse/beacon_chain/tests/chain_test.rs diff --git a/beacon_chain/chain/Cargo.toml b/beacon_chain/chain/Cargo.toml deleted file mode 100644 index 3302d15d47..0000000000 --- a/beacon_chain/chain/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "chain" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -bls = { path = "../utils/bls" } -db = { path = "../../lighthouse/db" } -genesis = { path = "../genesis" } -naive_fork_choice = { path = "../naive_fork_choice" } -spec = { path = "../spec" } -ssz = { path = "../utils/ssz" } -types = { path = "../types" } -validator_induction = { path = "../validator_induction" } -validator_shuffling = { path = "../validator_shuffling" } diff --git a/beacon_chain/chain/src/block_processing.rs b/beacon_chain/chain/src/block_processing.rs deleted file mode 100644 index b576bdbcec..0000000000 --- a/beacon_chain/chain/src/block_processing.rs +++ /dev/null @@ -1,29 +0,0 @@ -use super::BeaconChain; -use db::ClientDB; -use types::Hash256; - -pub enum BlockProcessingOutcome { - BlockAlreadyKnown, - NewCanonicalBlock, - NewReorgBlock, - NewForkBlock, -} - -pub enum Error { - NotImplemented, -} - -impl BeaconChain -where - T: ClientDB + Sized, -{ - pub fn process_block( - &mut self, - _ssz: &[u8], - _present_slot: u64, - ) -> Result<(BlockProcessingOutcome, Hash256), Error> { - // TODO: block processing has been removed. - // https://github.com/sigp/lighthouse/issues/98 - Err(Error::NotImplemented) - } -} diff --git a/beacon_chain/chain/src/lib.rs b/beacon_chain/chain/src/lib.rs deleted file mode 100644 index b1499ac1ac..0000000000 --- a/beacon_chain/chain/src/lib.rs +++ /dev/null @@ -1,110 +0,0 @@ -extern crate db; -extern crate naive_fork_choice; -extern crate genesis; -extern crate spec; -extern crate ssz; -extern crate types; -extern crate validator_induction; -extern crate validator_shuffling; - -mod block_processing; -mod maps; -mod stores; - -use db::ClientDB; -use crate::maps::{generate_attester_and_proposer_maps, AttesterAndProposerMapError}; -use crate::stores::BeaconChainStore; -use genesis::{genesis_beacon_state, GenesisError}; -use spec::ChainSpec; -use std::collections::HashMap; -use std::sync::Arc; -use types::{AttesterMap, BeaconState, Hash256, ProposerMap}; - -#[derive(Debug, PartialEq)] -pub enum BeaconChainError { - InvalidGenesis, - InsufficientValidators, - UnableToGenerateMaps(AttesterAndProposerMapError), - GenesisError(GenesisError), - DBError(String), -} - -pub struct BeaconChain { - /// The last slot which has been finalized, this is common to all forks. - pub last_finalized_slot: u64, - /// A vec of all block heads (tips of chains). - pub head_block_hashes: Vec, - /// The index of the canonical block in `head_block_hashes`. - pub canonical_head_block_hash: usize, - /// An in-memory map of root hash to beacon state. - pub beacon_states: HashMap, - /// A map of crystallized state to a proposer and attester map. - pub attester_proposer_maps: HashMap, Arc)>, - /// A collection of database stores used by the chain. - pub store: BeaconChainStore, - /// The chain configuration. - pub spec: ChainSpec, -} - -impl BeaconChain -where - T: ClientDB + Sized, -{ - pub fn new(store: BeaconChainStore, spec: ChainSpec) -> Result { - if spec.initial_validators.is_empty() { - return Err(BeaconChainError::InsufficientValidators); - } - - /* - * Generate and process the genesis state. - */ - let genesis_state = genesis_beacon_state(&spec)?; - let mut beacon_states = HashMap::new(); - beacon_states.insert(genesis_state.canonical_root(), genesis_state.clone()); - - // TODO: implement genesis block - // https://github.com/sigp/lighthouse/issues/105 - let canonical_latest_block_hash = Hash256::zero(); - - let head_block_hashes = vec![canonical_latest_block_hash]; - let canonical_head_block_hash = 0; - - let mut attester_proposer_maps = HashMap::new(); - - let (attester_map, proposer_map) = generate_attester_and_proposer_maps( - &genesis_state.shard_committees_at_slots, - 0, - )?; - - attester_proposer_maps.insert( - canonical_latest_block_hash, - (Arc::new(attester_map), Arc::new(proposer_map)), - ); - - Ok(Self { - last_finalized_slot: 0, - head_block_hashes, - canonical_head_block_hash, - beacon_states, - attester_proposer_maps, - store, - spec, - }) - } - - pub fn canonical_block_hash(&self) -> Hash256 { - self.head_block_hashes[self.canonical_head_block_hash] - } -} - -impl From for BeaconChainError { - fn from(e: AttesterAndProposerMapError) -> BeaconChainError { - BeaconChainError::UnableToGenerateMaps(e) - } -} - -impl From for BeaconChainError { - fn from(e: GenesisError) -> BeaconChainError { - BeaconChainError::GenesisError(e) - } -} diff --git a/lighthouse/beacon_chain/Cargo.toml b/lighthouse/beacon_chain/Cargo.toml new file mode 100644 index 0000000000..5e27949773 --- /dev/null +++ b/lighthouse/beacon_chain/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "chain" +version = "0.1.0" +authors = ["Paul Hauner "] +edition = "2018" + +[dependencies] +bls = { path = "../../beacon_chain/utils/bls" } +db = { path = "../db" } +genesis = { path = "../../beacon_chain/genesis" } +naive_fork_choice = { path = "../../beacon_chain/naive_fork_choice" } +slot_clock = { path = "../../beacon_chain/utils/slot_clock" } +spec = { path = "../../beacon_chain/spec" } +ssz = { path = "../../beacon_chain/utils/ssz" } +types = { path = "../../beacon_chain/types" } +validator_induction = { path = "../../beacon_chain/validator_induction" } +validator_shuffling = { path = "../../beacon_chain/validator_shuffling" } diff --git a/lighthouse/beacon_chain/src/block_processing.rs b/lighthouse/beacon_chain/src/block_processing.rs new file mode 100644 index 0000000000..b24a6f1f81 --- /dev/null +++ b/lighthouse/beacon_chain/src/block_processing.rs @@ -0,0 +1,72 @@ +use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use slot_clock::TestingSlotClockError; +use ssz::{ssz_encode, Encodable}; +use types::{readers::BeaconBlockReader, Hash256}; + +#[derive(Debug, PartialEq)] +pub enum Outcome { + FutureSlot, + Processed, + + NewCanonicalBlock, + NewReorgBlock, + NewForkBlock, +} + +#[derive(Debug, PartialEq)] +pub enum Error { + DBError(String), + NotImplemented, + PresentSlotIsNone, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, + Error: From<::Error>, +{ + pub fn process_block(&mut self, block: &V) -> Result<(Outcome, Hash256), Error> + where + V: BeaconBlockReader + Encodable + Sized, + { + let block_root = block.canonical_root(); + + let present_slot = self + .slot_clock + .present_slot()? + .ok_or(Error::PresentSlotIsNone)?; + + // Block from future slots (i.e., greater than the present slot) should not be processed. + if block.slot() > present_slot { + return Ok((Outcome::FutureSlot, block_root)); + } + + // TODO: block processing has been removed. + // https://github.com/sigp/lighthouse/issues/98 + + // Update leaf blocks. + self.block_store.put(&block_root, &ssz_encode(block)[..])?; + if self.leaf_blocks.contains(&block.parent_root()) { + self.leaf_blocks.remove(&block.parent_root()); + } + if self.canonical_leaf_block == block.parent_root() { + self.canonical_leaf_block = block_root; + } + self.leaf_blocks.insert(block_root); + + Ok((Outcome::Processed, block_root)) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(_: TestingSlotClockError) -> Error { + unreachable!(); // Testing clock never throws an error. + } +} diff --git a/lighthouse/beacon_chain/src/block_production.rs b/lighthouse/beacon_chain/src/block_production.rs new file mode 100644 index 0000000000..9754b4a54c --- /dev/null +++ b/lighthouse/beacon_chain/src/block_production.rs @@ -0,0 +1,64 @@ +use super::{BeaconChain, ClientDB, DBError, SlotClock}; +use slot_clock::TestingSlotClockError; +use types::{ + readers::{BeaconBlockReader, BeaconStateReader}, + BeaconBlock, BeaconState, Hash256, +}; + +#[derive(Debug, PartialEq)] +pub enum Error { + DBError(String), + PresentSlotIsNone, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, + Error: From<::Error>, +{ + pub fn produce_block(&mut self) -> Result<(BeaconBlock, BeaconState), Error> { + let present_slot = self + .slot_clock + .present_slot()? + .ok_or(Error::PresentSlotIsNone)?; + let parent_root = self.canonical_leaf_block; + let parent_block = self + .block_store + .get_deserialized(&parent_root)? + .ok_or(Error::DBError("Block not found.".to_string()))?; + let parent_state = self + .state_store + .get_deserialized(&parent_block.state_root())? + .ok_or(Error::DBError("State not found.".to_string()))?; + + let mut block = BeaconBlock { + slot: present_slot, + parent_root, + state_root: Hash256::zero(), // Updated after the state is calculated. + ..parent_block.to_beacon_block() + }; + + let state = BeaconState { + slot: present_slot, + ..parent_state.to_beacon_state() + }; + let state_root = state.canonical_root(); + + block.state_root = state_root; + + Ok((block, state)) + } +} + +impl From for Error { + fn from(e: DBError) -> Error { + Error::DBError(e.message) + } +} + +impl From for Error { + fn from(_: TestingSlotClockError) -> Error { + unreachable!(); // Testing clock never throws an error. + } +} diff --git a/lighthouse/beacon_chain/src/lib.rs b/lighthouse/beacon_chain/src/lib.rs new file mode 100644 index 0000000000..632aa2c852 --- /dev/null +++ b/lighthouse/beacon_chain/src/lib.rs @@ -0,0 +1,83 @@ +mod block_processing; +mod block_production; +mod maps; +mod stores; + +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + ClientDB, DBError, +}; +use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; +use slot_clock::{SlotClock, TestingSlotClockError}; +use spec::ChainSpec; +use ssz::ssz_encode; +use std::collections::HashSet; +use std::sync::Arc; +use types::Hash256; + +pub use crate::block_processing::Outcome as BlockProcessingOutcome; + +#[derive(Debug, PartialEq)] +pub enum BeaconChainError { + InsufficientValidators, + GenesisError(GenesisError), + DBError(String), +} + +pub struct BeaconChain { + pub block_store: Arc>, + pub state_store: Arc>, + pub slot_clock: U, + pub leaf_blocks: HashSet, + pub canonical_leaf_block: Hash256, + pub spec: ChainSpec, +} + +impl BeaconChain +where + T: ClientDB, + U: SlotClock, +{ + pub fn genesis( + state_store: Arc>, + block_store: Arc>, + slot_clock: U, + spec: ChainSpec, + ) -> Result { + if spec.initial_validators.is_empty() { + return Err(BeaconChainError::InsufficientValidators); + } + + let genesis_state = genesis_beacon_state(&spec)?; + let state_root = genesis_state.canonical_root(); + state_store.put(&state_root, &ssz_encode(&genesis_state)[..])?; + + let genesis_block = genesis_beacon_block(state_root, &spec); + let block_root = genesis_block.canonical_root(); + block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; + + let mut leaf_blocks = HashSet::new(); + leaf_blocks.insert(block_root.clone()); + + Ok(Self { + block_store, + state_store, + slot_clock, + leaf_blocks, + canonical_leaf_block: block_root, + spec, + }) + } +} + +impl From for BeaconChainError { + fn from(e: DBError) -> BeaconChainError { + BeaconChainError::DBError(e.message) + } +} + +impl From for BeaconChainError { + fn from(e: GenesisError) -> BeaconChainError { + BeaconChainError::GenesisError(e) + } +} diff --git a/beacon_chain/chain/src/maps.rs b/lighthouse/beacon_chain/src/maps.rs similarity index 100% rename from beacon_chain/chain/src/maps.rs rename to lighthouse/beacon_chain/src/maps.rs diff --git a/beacon_chain/chain/src/stores.rs b/lighthouse/beacon_chain/src/stores.rs similarity index 100% rename from beacon_chain/chain/src/stores.rs rename to lighthouse/beacon_chain/src/stores.rs diff --git a/beacon_chain/chain/src/transition.rs b/lighthouse/beacon_chain/src/transition.rs similarity index 97% rename from beacon_chain/chain/src/transition.rs rename to lighthouse/beacon_chain/src/transition.rs index df8803b27f..df434cc0c0 100644 --- a/beacon_chain/chain/src/transition.rs +++ b/lighthouse/beacon_chain/src/transition.rs @@ -3,7 +3,7 @@ use db::ClientDB; use state_transition::{extend_active_state, StateTransitionError}; use types::{ActiveState, BeaconBlock, CrystallizedState, Hash256}; -impl BeaconChain +impl BeaconChain where T: ClientDB + Sized, { diff --git a/lighthouse/beacon_chain/tests/chain_test.rs b/lighthouse/beacon_chain/tests/chain_test.rs new file mode 100644 index 0000000000..aec6c08e66 --- /dev/null +++ b/lighthouse/beacon_chain/tests/chain_test.rs @@ -0,0 +1,49 @@ +use chain::{BlockProcessingOutcome, BeaconChain}; +use db::{ + stores::{BeaconBlockStore, BeaconStateStore}, + MemoryDB, +}; +use slot_clock::TestingSlotClock; +use spec::ChainSpec; +use std::sync::Arc; + +fn in_memory_test_stores() -> ( + Arc, + Arc>, + Arc>, +) { + let db = Arc::new(MemoryDB::open()); + let block_store = Arc::new(BeaconBlockStore::new(db.clone())); + let state_store = Arc::new(BeaconStateStore::new(db.clone())); + (db, block_store, state_store) +} + +fn in_memory_test_chain( + spec: ChainSpec, +) -> (Arc, BeaconChain) { + let (db, block_store, state_store) = in_memory_test_stores(); + let slot_clock = TestingSlotClock::new(0); + + let chain = BeaconChain::genesis(state_store, block_store, slot_clock, spec); + (db, chain.unwrap()) +} + +#[test] +fn it_constructs() { + let (_db, _chain) = in_memory_test_chain(ChainSpec::foundation()); +} + +#[test] +fn it_produces() { + let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); + let (_block, _state) = chain.produce_block().unwrap(); +} + +#[test] +fn it_processes_a_block_it_produces() { + let (_db, mut chain) = in_memory_test_chain(ChainSpec::foundation()); + let (block, _state) = chain.produce_block().unwrap(); + let (outcome, new_block_hash) = chain.process_block(&block).unwrap(); + assert_eq!(outcome, BlockProcessingOutcome::Processed); + assert_eq!(chain.canonical_leaf_block, new_block_hash); +} From 0ba9b86e874ee2be2e2491c99767b2fe2f807909 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Dec 2018 13:00:40 +1100 Subject: [PATCH 04/15] Remove old `abstract_chain` crate --- beacon_chain/abstract_chain/Cargo.toml | 10 - .../abstract_chain/src/block_reader.rs | 16 -- beacon_chain/abstract_chain/src/lib.rs | 185 ------------------ 3 files changed, 211 deletions(-) delete mode 100644 beacon_chain/abstract_chain/Cargo.toml delete mode 100644 beacon_chain/abstract_chain/src/block_reader.rs delete mode 100644 beacon_chain/abstract_chain/src/lib.rs diff --git a/beacon_chain/abstract_chain/Cargo.toml b/beacon_chain/abstract_chain/Cargo.toml deleted file mode 100644 index 6e11036cba..0000000000 --- a/beacon_chain/abstract_chain/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "abstract_chain" -version = "0.1.0" -authors = ["Paul Hauner "] -edition = "2018" - -[dependencies] -hashing = { path = "../utils/hashing" } -ssz = { path = "../utils/ssz" } -types = { path = "../types" } diff --git a/beacon_chain/abstract_chain/src/block_reader.rs b/beacon_chain/abstract_chain/src/block_reader.rs deleted file mode 100644 index 9d66dc5b69..0000000000 --- a/beacon_chain/abstract_chain/src/block_reader.rs +++ /dev/null @@ -1,16 +0,0 @@ -use types::{BeaconBlock, Hash256}; - -pub trait BlockReader { - fn slot(&self) -> u64; - fn parent_root(&self) -> Hash256; -} - -impl BlockReader for BeaconBlock { - fn slot(&self) -> u64 { - self.slot - } - - fn parent_root(&self) -> Hash256 { - self.parent_root - } -} diff --git a/beacon_chain/abstract_chain/src/lib.rs b/beacon_chain/abstract_chain/src/lib.rs deleted file mode 100644 index e348dfdba6..0000000000 --- a/beacon_chain/abstract_chain/src/lib.rs +++ /dev/null @@ -1,185 +0,0 @@ -/* - * - */ - - -use hashing::canonical_hash; -use ssz::{Encodable, Decodable, DecodeError, SszStream, ssz_encode}; -use std::collections::{HashMap, HashSet}; -use types::Hash256; - -mod block_reader; - -use crate::block_reader::BlockReader; - -pub trait AbstractChain { - type Block; - type Hash; - - fn genesis(&mut self); - - fn receive_block(&mut self, block: &BlockReader) -> bool; - - fn block_by_root(&self, root: &Self::Hash) -> Option<&Self::Block>; -} - -pub struct TestBlock { - slot: u64, - parent_root: Hash256, - state_root: Hash256, - weight: u8, -} - -impl Encodable for TestBlock { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.slot); - s.append(&self.parent_root); - } -} - -impl Decodable for TestBlock { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (slot, i) = <_>::ssz_decode(bytes, i)?; - let (parent_root, i) = <_>::ssz_decode(bytes, i)?; - let (state_root, i) = <_>::ssz_decode(bytes, i)?; - let (weight, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - slot, - parent_root, - state_root, - weight, - }, - i, - )) - } -} - -impl TestBlock { - fn canonical_hash(&self) -> Hash256 { - Hash256::from(&canonical_hash(&ssz_encode(self))[..]) - } -} - -impl BlockReader for TestBlock { - fn slot(&self) -> u64 { - self.slot - } - - fn parent_root(&self) -> Hash256 { - self.parent_root - } -} - -#[derive(Clone)] -pub struct TestState { - total_weight: u64, - skipped_slots: u64, -} - -impl Encodable for TestState { - fn ssz_append(&self, s: &mut SszStream) { - s.append(&self.total_weight); - s.append(&self.skipped_slots); - } -} - -impl TestState { - fn canonical_hash(&self) -> Hash256 { - Hash256::from(&canonical_hash(&ssz_encode(self))[..]) - } -} - -impl Decodable for TestState { - fn ssz_decode(bytes: &[u8], i: usize) -> Result<(Self, usize), DecodeError> { - let (total_weight, i) = <_>::ssz_decode(bytes, i)?; - let (skipped_slots, i) = <_>::ssz_decode(bytes, i)?; - - Ok(( - Self { - total_weight, - skipped_slots, - }, - i, - )) - } -} - -pub struct TestChain { - slot: u64, - block_store: HashMap, - state_store: HashMap, - leaf_nodes: HashSet, - canonical_leaf_node: Hash256, -} - -impl AbstractChain for TestChain { - type Block = TestBlock; - type Hash = Hash256; - - fn genesis(&mut self) { - let genesis_state = TestState { - total_weight: 255, - skipped_slots: 0, - }; - let state_root = genesis_state.canonical_hash(); - let genesis_block = TestBlock { - slot: 0, - parent_root: Hash256::zero(), - state_root: state_root.clone(), - weight: 255, - }; - let block_root = genesis_block.canonical_hash(); - self.block_store.insert(block_root, genesis_block); - self.state_store.insert(state_root, genesis_state); - self.leaf_nodes.insert(block_root); - self.canonical_leaf_node = block_root; - self.slot = 0; - } - - fn block_by_root(&self, root: &Hash256) -> Option<&Self::Block> { - self.block_store.get(root) - } - - fn receive_block(&mut self, block: &BlockReader) -> bool { - let block_hash = block.canonical_hash(); - let parent_block = match self.block_by_root(&block.parent_root()) { - Some(block) => block, - None => return false - }; - let slot_distance = block.slot().saturating_sub(parent_block.slot()); - if slot_distance > 0 { - let mut state = self.state_store.get(&block.parent_root()).unwrap().clone(); - for _ in parent_block.slot()..(block.slot() - 1) { - state.skipped_slots += 1; - } - state.total_weight += u64::from(block.weight); - let state_root = state.canonical_hash(); - if state_root != block.state_root { - return false; - } - self.leaf_nodes.remove(&block.parent_root()); - self.leaf_nodes.insert(block_hash); - - let canonical_state = { - let canonical_block = self.block_store.get(&self.canonical_leaf_node).unwrap(); - self.state_store.get(&canonical_block.state_root).unwrap() - }; - // New block is canonical, chain weight is greatest. - if state.total_weight > canonical_state.total_weight { - self.canonical_leaf_node = block_hash; - } - // New block chain weight equals existing canonical block chain weight. - else if state.total_weight == canonical_state.total_weight { - // New block is canonical as hash is highest. - if block_hash > self.canonical_leaf_node { - self.canonical_leaf_node = block_hash; - } - } - true - } else { - false - } - } -} From 31c78b77183422d4906482ab27343a3abe156da4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Dec 2018 13:02:44 +1100 Subject: [PATCH 05/15] Rename slot-clock -> slock_clock --- beacon_chain/utils/slot-clock/src/lib.rs | 74 ---------- .../{slot-clock => slot_clock}/Cargo.toml | 2 +- beacon_chain/utils/slot_clock/src/lib.rs | 11 ++ .../slot_clock/src/system_time_slot_clock.rs | 135 ++++++++++++++++++ .../slot_clock/src/testing_slot_clock.rs | 43 ++++++ 5 files changed, 190 insertions(+), 75 deletions(-) delete mode 100644 beacon_chain/utils/slot-clock/src/lib.rs rename beacon_chain/utils/{slot-clock => slot_clock}/Cargo.toml (84%) create mode 100644 beacon_chain/utils/slot_clock/src/lib.rs create mode 100644 beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs create mode 100644 beacon_chain/utils/slot_clock/src/testing_slot_clock.rs diff --git a/beacon_chain/utils/slot-clock/src/lib.rs b/beacon_chain/utils/slot-clock/src/lib.rs deleted file mode 100644 index ffd1b5a9ed..0000000000 --- a/beacon_chain/utils/slot-clock/src/lib.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::time::{Duration, SystemTime, SystemTimeError}; - -pub fn slot_now( - genesis_seconds: u64, - slot_duration_seconds: u64, -) -> Result, SystemTimeError> { - let sys_time = SystemTime::now(); - let duration_since_epoch = sys_time.duration_since(SystemTime::UNIX_EPOCH)?; - let duration_since_genesis = - duration_since_epoch.checked_sub(Duration::from_secs(genesis_seconds)); - match duration_since_genesis { - None => Ok(None), - Some(d) => Ok(slot_from_duration(slot_duration_seconds, d)), - } -} - -fn slot_from_duration(slot_duration_seconds: u64, duration: Duration) -> Option { - duration.as_secs().checked_div(slot_duration_seconds) -} - -#[cfg(test)] -mod tests { - use super::*; - - /* - * Note: these tests are using actual system times and could fail if they are executed on a - * very slow machine. - */ - #[test] - fn test_slot_now() { - let s_time = 100; - - let now = SystemTime::now(); - let since_epoch = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); - - let genesis = since_epoch.as_secs() - s_time * 89; - assert_eq!(slot_now(genesis, s_time).unwrap(), Some(89)); - - let genesis = since_epoch.as_secs(); - assert_eq!(slot_now(genesis, s_time).unwrap(), Some(0)); - - let genesis = since_epoch.as_secs() - s_time * 42 - 5; - assert_eq!(slot_now(genesis, s_time).unwrap(), Some(42)); - } - - #[test] - fn test_slot_from_duration() { - let s_time = 100; - - assert_eq!(slot_from_duration(s_time, Duration::from_secs(0)), Some(0)); - assert_eq!(slot_from_duration(s_time, Duration::from_secs(10)), Some(0)); - assert_eq!( - slot_from_duration(s_time, Duration::from_secs(100)), - Some(1) - ); - assert_eq!( - slot_from_duration(s_time, Duration::from_secs(101)), - Some(1) - ); - assert_eq!( - slot_from_duration(s_time, Duration::from_secs(1000)), - Some(10) - ); - } - - #[test] - fn test_slot_from_duration_slot_time_zero() { - let s_time = 0; - - assert_eq!(slot_from_duration(s_time, Duration::from_secs(0)), None); - assert_eq!(slot_from_duration(s_time, Duration::from_secs(10)), None); - assert_eq!(slot_from_duration(s_time, Duration::from_secs(1000)), None); - } -} diff --git a/beacon_chain/utils/slot-clock/Cargo.toml b/beacon_chain/utils/slot_clock/Cargo.toml similarity index 84% rename from beacon_chain/utils/slot-clock/Cargo.toml rename to beacon_chain/utils/slot_clock/Cargo.toml index ccb2e4ed49..166f397fd1 100644 --- a/beacon_chain/utils/slot-clock/Cargo.toml +++ b/beacon_chain/utils/slot_clock/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "slot-clock" +name = "slot_clock" version = "0.1.0" authors = ["Paul Hauner "] edition = "2018" diff --git a/beacon_chain/utils/slot_clock/src/lib.rs b/beacon_chain/utils/slot_clock/src/lib.rs new file mode 100644 index 0000000000..7bdb775afe --- /dev/null +++ b/beacon_chain/utils/slot_clock/src/lib.rs @@ -0,0 +1,11 @@ +mod system_time_slot_clock; +mod testing_slot_clock; + +pub use crate::system_time_slot_clock::{Error as SystemTimeSlotClockError, SystemTimeSlotClock}; +pub use crate::testing_slot_clock::{Error as TestingSlotClockError, TestingSlotClock}; + +pub trait SlotClock { + type Error; + + fn present_slot(&self) -> Result, Self::Error>; +} diff --git a/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs b/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs new file mode 100644 index 0000000000..be7265ae8f --- /dev/null +++ b/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs @@ -0,0 +1,135 @@ +use super::SlotClock; +use std::time::{Duration, SystemTime}; + +pub use std::time::SystemTimeError; + +#[derive(Debug)] +pub enum Error { + SlotDurationIsZero, + SystemTimeError(SystemTimeError), +} + +/// Determines the present slot based upon the present system time. +pub struct SystemTimeSlotClock { + genesis_seconds: u64, + slot_duration_seconds: u64, +} + +impl SystemTimeSlotClock { + /// Create a new `SystemTimeSlotClock`. + /// + /// Returns an Error if `slot_duration_seconds == 0`. + pub fn new( + genesis_seconds: u64, + slot_duration_seconds: u64, + ) -> Result { + if slot_duration_seconds == 0 { + Err(Error::SlotDurationIsZero) + } else { + Ok(SystemTimeSlotClock { + genesis_seconds, + slot_duration_seconds, + }) + } + } +} + +impl SlotClock for SystemTimeSlotClock { + type Error = Error; + + fn present_slot(&self) -> Result, Error> { + let syslot_time = SystemTime::now(); + let duration_since_epoch = syslot_time.duration_since(SystemTime::UNIX_EPOCH)?; + let duration_since_genesis = + duration_since_epoch.checked_sub(Duration::from_secs(self.genesis_seconds)); + match duration_since_genesis { + None => Ok(None), + Some(d) => Ok(slot_from_duration(self.slot_duration_seconds, d)), + } + } +} + +impl From for Error { + fn from(e: SystemTimeError) -> Error { + Error::SystemTimeError(e) + } +} + +fn slot_from_duration(slot_duration_seconds: u64, duration: Duration) -> Option { + duration.as_secs().checked_div(slot_duration_seconds) +} + +#[cfg(test)] +mod tests { + use super::*; + + /* + * Note: these tests are using actual system times and could fail if they are executed on a + * very slow machine. + */ + #[test] + fn test_slot_now() { + let slot_time = 100; + + let now = SystemTime::now(); + let since_epoch = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + + let genesis = since_epoch.as_secs() - slot_time * 89; + + let clock = SystemTimeSlotClock { + genesis_seconds: genesis, + slot_duration_seconds: slot_time, + }; + assert_eq!(clock.present_slot().unwrap(), Some(89)); + + let clock = SystemTimeSlotClock { + genesis_seconds: since_epoch.as_secs(), + slot_duration_seconds: slot_time, + }; + assert_eq!(clock.present_slot().unwrap(), Some(0)); + + let clock = SystemTimeSlotClock { + genesis_seconds: since_epoch.as_secs() - slot_time * 42 - 5, + slot_duration_seconds: slot_time, + }; + assert_eq!(clock.present_slot().unwrap(), Some(42)); + } + + #[test] + fn test_slot_from_duration() { + let slot_time = 100; + + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(0)), + Some(0) + ); + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(10)), + Some(0) + ); + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(100)), + Some(1) + ); + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(101)), + Some(1) + ); + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(1000)), + Some(10) + ); + } + + #[test] + fn test_slot_from_duration_slot_time_zero() { + let slot_time = 0; + + assert_eq!(slot_from_duration(slot_time, Duration::from_secs(0)), None); + assert_eq!(slot_from_duration(slot_time, Duration::from_secs(10)), None); + assert_eq!( + slot_from_duration(slot_time, Duration::from_secs(1000)), + None + ); + } +} diff --git a/beacon_chain/utils/slot_clock/src/testing_slot_clock.rs b/beacon_chain/utils/slot_clock/src/testing_slot_clock.rs new file mode 100644 index 0000000000..de1d7ddb3e --- /dev/null +++ b/beacon_chain/utils/slot_clock/src/testing_slot_clock.rs @@ -0,0 +1,43 @@ +use super::SlotClock; + +#[derive(Debug, PartialEq)] +pub enum Error {} + +/// Determines the present slot based upon the present system time. +pub struct TestingSlotClock { + slot: u64, +} + +impl TestingSlotClock { + /// Create a new `TestingSlotClock`. + /// + /// Returns an Error if `slot_duration_seconds == 0`. + pub fn new(slot: u64) -> TestingSlotClock { + TestingSlotClock { slot } + } + + pub fn set_slot(&mut self, slot: u64) { + self.slot = slot; + } +} + +impl SlotClock for TestingSlotClock { + type Error = Error; + + fn present_slot(&self) -> Result, Error> { + Ok(Some(self.slot)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_slot_now() { + let mut clock = TestingSlotClock::new(10); + assert_eq!(clock.present_slot(), Ok(Some(10))); + clock.set_slot(123); + assert_eq!(clock.present_slot(), Ok(Some(123))); + } +} From 1081529cc7ac59b5cbebdb8fab3ba0651dd77f20 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Sun, 30 Dec 2018 13:03:20 +1100 Subject: [PATCH 06/15] Add basic `BeaconChain` struct --- Cargo.toml | 4 +-- beacon_chain/genesis/src/beacon_block.rs | 9 +++--- beacon_chain/naive_fork_choice/src/lib.rs | 2 +- beacon_chain/types/Cargo.toml | 1 + beacon_chain/types/src/beacon_block.rs | 11 ++++++- beacon_chain/types/src/beacon_state.rs | 5 +-- beacon_chain/types/src/lib.rs | 3 ++ .../types/src/readers/block_reader.rs | 31 +++++++++++++++++++ beacon_chain/types/src/readers/mod.rs | 5 +++ .../types/src/readers/state_reader.rs | 21 +++++++++++++ 10 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 beacon_chain/types/src/readers/block_reader.rs create mode 100644 beacon_chain/types/src/readers/mod.rs create mode 100644 beacon_chain/types/src/readers/state_reader.rs diff --git a/Cargo.toml b/Cargo.toml index 74bff6b731..be3bdf3816 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,6 @@ name = "lighthouse" [workspace] members = [ "beacon_chain/attestation_validation", - "beacon_chain/chain", "beacon_chain/genesis", "beacon_chain/naive_fork_choice", "beacon_chain/spec", @@ -43,11 +42,12 @@ members = [ "beacon_chain/utils/boolean-bitfield", "beacon_chain/utils/hashing", "beacon_chain/utils/honey-badger-split", - "beacon_chain/utils/slot-clock", + "beacon_chain/utils/slot_clock", "beacon_chain/utils/ssz", "beacon_chain/utils/vec_shuffle", "beacon_chain/validator_change", "beacon_chain/validator_induction", "beacon_chain/validator_shuffling", + "lighthouse/beacon_chain", "lighthouse/db", ] diff --git a/beacon_chain/genesis/src/beacon_block.rs b/beacon_chain/genesis/src/beacon_block.rs index 120903564a..65adc1cf76 100644 --- a/beacon_chain/genesis/src/beacon_block.rs +++ b/beacon_chain/genesis/src/beacon_block.rs @@ -1,14 +1,14 @@ use bls::{Signature, BLS_AGG_SIG_BYTE_SIZE}; use spec::ChainSpec; use ssz::{encode::encode_length, Decodable, LENGTH_BYTES}; -use types::{BeaconBlock, BeaconBlockBody}; +use types::{BeaconBlock, BeaconBlockBody, Hash256}; /// Generate a genesis BeaconBlock. -pub fn genesis_beacon_block(spec: &ChainSpec) -> BeaconBlock { +pub fn genesis_beacon_block(state_root: Hash256, spec: &ChainSpec) -> BeaconBlock { BeaconBlock { slot: spec.initial_slot_number, parent_root: spec.zero_hash, - state_root: spec.zero_hash, + state_root: state_root, randao_reveal: spec.zero_hash, candidate_pow_receipt_root: spec.zero_hash, signature: genesis_signature(), @@ -42,8 +42,9 @@ mod tests { #[test] fn test_genesis() { let spec = ChainSpec::foundation(); + let state_root = Hash256::from("cats".as_bytes()); // This only checks that the function runs without panic. - genesis_beacon_block(&spec); + genesis_beacon_block(state_root, &spec); } } diff --git a/beacon_chain/naive_fork_choice/src/lib.rs b/beacon_chain/naive_fork_choice/src/lib.rs index 5270fb1f95..385c9c50d0 100644 --- a/beacon_chain/naive_fork_choice/src/lib.rs +++ b/beacon_chain/naive_fork_choice/src/lib.rs @@ -28,7 +28,7 @@ where */ for (index, block_hash) in head_block_hashes.iter().enumerate() { let ssz = block_store - .get_serialized_block(&block_hash.to_vec()[..])? + .get(&block_hash.to_vec()[..])? .ok_or(ForkChoiceError::MissingBlock)?; let (block, _) = BeaconBlock::ssz_decode(&ssz, 0)?; head_blocks.push((index, block)); diff --git a/beacon_chain/types/Cargo.toml b/beacon_chain/types/Cargo.toml index f497022807..af53fa5978 100644 --- a/beacon_chain/types/Cargo.toml +++ b/beacon_chain/types/Cargo.toml @@ -8,5 +8,6 @@ edition = "2018" bls = { path = "../utils/bls" } boolean-bitfield = { path = "../utils/boolean-bitfield" } ethereum-types = "0.4.0" +hashing = { path = "../utils/hashing" } rand = "0.5.5" ssz = { path = "../utils/ssz" } diff --git a/beacon_chain/types/src/beacon_block.rs b/beacon_chain/types/src/beacon_block.rs index a0c4262330..826667d5ef 100644 --- a/beacon_chain/types/src/beacon_block.rs +++ b/beacon_chain/types/src/beacon_block.rs @@ -1,7 +1,8 @@ -use super::ssz::{Decodable, DecodeError, Encodable, SszStream}; +use super::ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; use super::{BeaconBlockBody, Hash256}; use crate::test_utils::TestRandom; use bls::Signature; +use hashing::canonical_hash; use rand::RngCore; #[derive(Debug, PartialEq, Clone)] @@ -15,6 +16,14 @@ pub struct BeaconBlock { pub body: BeaconBlockBody, } +impl BeaconBlock { + pub fn canonical_root(&self) -> Hash256 { + // TODO: implement tree hashing. + // https://github.com/sigp/lighthouse/issues/70 + Hash256::from(&canonical_hash(&ssz_encode(self))[..]) + } +} + impl Encodable for BeaconBlock { fn ssz_append(&self, s: &mut SszStream) { s.append(&self.slot); diff --git a/beacon_chain/types/src/beacon_state.rs b/beacon_chain/types/src/beacon_state.rs index 7c61d1d207..f2d8f6aeb5 100644 --- a/beacon_chain/types/src/beacon_state.rs +++ b/beacon_chain/types/src/beacon_state.rs @@ -7,8 +7,9 @@ use super::shard_reassignment_record::ShardReassignmentRecord; use super::validator_record::ValidatorRecord; use super::Hash256; use crate::test_utils::TestRandom; +use hashing::canonical_hash; use rand::RngCore; -use ssz::{Decodable, DecodeError, Encodable, SszStream}; +use ssz::{ssz_encode, Decodable, DecodeError, Encodable, SszStream}; #[derive(Debug, PartialEq, Clone)] pub struct BeaconState { @@ -51,7 +52,7 @@ impl BeaconState { pub fn canonical_root(&self) -> Hash256 { // TODO: implement tree hashing. // https://github.com/sigp/lighthouse/issues/70 - Hash256::zero() + Hash256::from(&canonical_hash(&ssz_encode(self))[..]) } } diff --git a/beacon_chain/types/src/lib.rs b/beacon_chain/types/src/lib.rs index 08b871c1cb..7863944d70 100644 --- a/beacon_chain/types/src/lib.rs +++ b/beacon_chain/types/src/lib.rs @@ -31,6 +31,9 @@ pub mod slashable_vote_data; pub mod validator_record; pub mod validator_registration; +pub mod readers; + + use self::ethereum_types::{H160, H256, U256}; use std::collections::HashMap; diff --git a/beacon_chain/types/src/readers/block_reader.rs b/beacon_chain/types/src/readers/block_reader.rs new file mode 100644 index 0000000000..e1cba224d2 --- /dev/null +++ b/beacon_chain/types/src/readers/block_reader.rs @@ -0,0 +1,31 @@ +use crate::{BeaconBlock, Hash256}; + +pub trait BeaconBlockReader { + fn slot(&self) -> u64; + fn parent_root(&self) -> Hash256; + fn state_root(&self) -> Hash256; + fn canonical_root(&self) -> Hash256; + fn to_beacon_block(self) -> BeaconBlock; +} + +impl BeaconBlockReader for BeaconBlock { + fn slot(&self) -> u64 { + self.slot + } + + fn parent_root(&self) -> Hash256 { + self.parent_root + } + + fn state_root(&self) -> Hash256 { + self.state_root + } + + fn canonical_root(&self) -> Hash256 { + self.canonical_root() + } + + fn to_beacon_block(self) -> BeaconBlock { + self + } +} diff --git a/beacon_chain/types/src/readers/mod.rs b/beacon_chain/types/src/readers/mod.rs new file mode 100644 index 0000000000..4ccb14a8cd --- /dev/null +++ b/beacon_chain/types/src/readers/mod.rs @@ -0,0 +1,5 @@ +mod block_reader; +mod state_reader; + +pub use self::block_reader::BeaconBlockReader; +pub use self::state_reader::BeaconStateReader; diff --git a/beacon_chain/types/src/readers/state_reader.rs b/beacon_chain/types/src/readers/state_reader.rs new file mode 100644 index 0000000000..a43ef9930e --- /dev/null +++ b/beacon_chain/types/src/readers/state_reader.rs @@ -0,0 +1,21 @@ +use crate::{BeaconState, Hash256}; + +pub trait BeaconStateReader { + fn slot(&self) -> u64; + fn canonical_root(&self) -> Hash256; + fn to_beacon_state(self) -> BeaconState; +} + +impl BeaconStateReader for BeaconState { + fn slot(&self) -> u64 { + self.slot + } + + fn canonical_root(&self) -> Hash256 { + self.canonical_root() + } + + fn to_beacon_state(self) -> BeaconState { + self + } +} From 8aa9f83c48a28681dc5e6db861b2a98dcb20c405 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 4 Jan 2019 17:55:15 +1100 Subject: [PATCH 07/15] Implement new API for db stores --- .../db/src/stores/beacon_block_store.rs | 176 ++++-------------- .../db/src/stores/beacon_state_store.rs | 57 ++++-- lighthouse/db/src/stores/macros.rs | 102 ++++++++++ lighthouse/db/src/stores/mod.rs | 9 +- lighthouse/db/src/stores/pow_chain_store.rs | 4 +- 5 files changed, 187 insertions(+), 161 deletions(-) create mode 100644 lighthouse/db/src/stores/macros.rs diff --git a/lighthouse/db/src/stores/beacon_block_store.rs b/lighthouse/db/src/stores/beacon_block_store.rs index 5faa208dff..ff7247b1c4 100644 --- a/lighthouse/db/src/stores/beacon_block_store.rs +++ b/lighthouse/db/src/stores/beacon_block_store.rs @@ -1,12 +1,9 @@ use super::BLOCKS_DB_COLUMN as DB_COLUMN; use super::{ClientDB, DBError}; -use ssz::{Decodable, DecodeError}; +use ssz::Decodable; use std::sync::Arc; use types::{readers::BeaconBlockReader, BeaconBlock, Hash256}; -type BeaconBlockHash = Vec; -type BeaconBlockSsz = Vec; - #[derive(Clone, Debug, PartialEq)] pub enum BeaconBlockAtSlotError { UnknownBeaconBlock, @@ -21,34 +18,25 @@ where db: Arc, } +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconBlockStore, DB_COLUMN); + impl BeaconBlockStore { pub fn new(db: Arc) -> Self { Self { db } } - pub fn put(&self, hash: &[u8], ssz: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, ssz) - } - - pub fn get(&self, hash: &[u8]) -> Result>, DBError> { - self.db.get(DB_COLUMN, hash) - } - - pub fn exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } - - pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.delete(DB_COLUMN, hash) - } - - /// Retuns a fully de-serialized `BeaconBlock` (or `None` if hash not known). - pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + /// Retuns an object implementing `BeaconBlockReader`, or `None` (if hash not known). + /// + /// Note: Presently, this function fully deserializes a `BeaconBlock` and returns that. In the + /// future, it would be ideal to return an object capable of reading directly from serialized + /// SSZ bytes. + pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { match self.get(&hash)? { None => Ok(None), Some(ssz) => { let (block, _) = BeaconBlock::ssz_decode(&ssz, 0).map_err(|_| DBError { - message: "Bad Block SSZ.".to_string(), + message: "Bad BeaconBlock SSZ.".to_string(), })?; Ok(Some(block)) } @@ -66,35 +54,25 @@ impl BeaconBlockStore { /// If a block is found, a tuple of (block_hash, serialized_block) is returned. pub fn block_at_slot( &self, - head_hash: &[u8], + head_hash: &Hash256, slot: u64, - ) -> Result, BeaconBlockAtSlotError> { - match self.get(head_hash)? { + ) -> Result, BeaconBlockAtSlotError> { + match self.get_reader(head_hash)? { None => Err(BeaconBlockAtSlotError::UnknownBeaconBlock), - Some(ssz) => { - let (retrieved_slot, parent_hash) = slot_and_parent_from_block_ssz(&ssz, 0) - .map_err(|_| BeaconBlockAtSlotError::InvalidBeaconBlock)?; - match retrieved_slot { - s if s == slot => Ok(Some((head_hash.to_vec(), ssz.to_vec()))), - s if s < slot => Ok(None), - _ => self.block_at_slot(&parent_hash, slot), + Some(block_reader) => match block_reader.slot() { + s if s == slot => { + let block = block_reader + .into_beacon_block() + .ok_or(BeaconBlockAtSlotError::InvalidBeaconBlock)?; + Ok(Some((*head_hash, block))) } - } + s if s < slot => Ok(None), + _ => self.block_at_slot(&block_reader.parent_root(), slot), + }, } } } -/// Read `block.slot` and `block.parent_root` from a SSZ-encoded block bytes. -/// -/// Assumes the block starts at byte `i`. -fn slot_and_parent_from_block_ssz(ssz: &[u8], i: usize) -> Result<(u64, Hash256), DecodeError> { - // Assuming the slot is the first field on a block. - let (slot, i) = u64::ssz_decode(&ssz, i)?; - // Assuming the parent has is the second field on a block. - let (parent_root, _) = Hash256::ssz_decode(&ssz, i)?; - Ok((slot, parent_root)) -} - impl From for BeaconBlockAtSlotError { fn from(e: DBError) -> Self { BeaconBlockAtSlotError::DBError(e.message) @@ -114,82 +92,7 @@ mod tests { use types::BeaconBlock; use types::Hash256; - #[test] - fn test_put() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - store.put(hash, ssz).unwrap(); - assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert_eq!(store.get(hash).unwrap().unwrap(), ssz); - } - - #[test] - fn test_get_unknown() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, other_hash, ssz).unwrap(); - assert_eq!(store.get(hash).unwrap(), None); - } - - #[test] - fn test_exists() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(store.exists(hash).unwrap()); - } - - #[test] - fn test_block_does_not_exist() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(!store.exists(other_hash).unwrap()); - } - - #[test] - fn test_delete() { - let db = Arc::new(MemoryDB::open()); - let store = BeaconBlockStore::new(db.clone()); - - let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - - db.put(DB_COLUMN, hash, ssz).unwrap(); - assert!(db.exists(DB_COLUMN, hash).unwrap()); - - store.delete(hash).unwrap(); - assert!(!db.exists(DB_COLUMN, hash).unwrap()); - } + test_crud_for_store!(BeaconBlockStore, DB_COLUMN); #[test] fn test_invalid_block_at_slot() { @@ -197,12 +100,14 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "definitly not a valid block".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); + let hash = &Hash256::from("some hash".as_bytes()); db.put(DB_COLUMN, hash, ssz).unwrap(); assert_eq!( store.block_at_slot(hash, 42), - Err(BeaconBlockAtSlotError::InvalidBeaconBlock) + Err(BeaconBlockAtSlotError::DBError( + "Bad BeaconBlock SSZ.".into() + )) ); } @@ -212,8 +117,8 @@ mod tests { let store = BeaconBlockStore::new(db.clone()); let ssz = "some bytes".as_bytes(); - let hash = &Hash256::from("some hash".as_bytes()).to_vec(); - let other_hash = &Hash256::from("another hash".as_bytes()).to_vec(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); db.put(DB_COLUMN, hash, ssz).unwrap(); assert_eq!( @@ -241,7 +146,7 @@ mod tests { for w in 0..wc { let key = (t * w) as u8; let val = 42; - bs.put(&vec![key], &vec![val]).unwrap(); + bs.put(&[key][..].into(), &vec![val]).unwrap(); } }); handles.push(handle); @@ -254,8 +159,8 @@ mod tests { for t in 0..thread_count { for w in 0..write_count { let key = (t * w) as u8; - assert!(bs.exists(&vec![key]).unwrap()); - let val = bs.get(&vec![key]).unwrap().unwrap(); + assert!(bs.exists(&[key][..].into()).unwrap()); + let val = bs.get(&[key][..].into()).unwrap().unwrap(); assert_eq!(vec![42], val); } } @@ -294,13 +199,7 @@ mod tests { block.slot = slots[i]; let ssz = ssz_encode(&block); - db.put(DB_COLUMN, &hashes[i].to_vec(), &ssz).unwrap(); - - // Ensure the slot and parent_root decoding fn works correctly. - let (decoded_slot, decoded_parent_root) = - slot_and_parent_from_block_ssz(&ssz, 0).unwrap(); - assert_eq!(decoded_slot, block.slot); - assert_eq!(decoded_parent_root, block.parent_root); + db.put(DB_COLUMN, &hashes[i], &ssz).unwrap(); blocks.push(block); } @@ -308,14 +207,11 @@ mod tests { // Test that certain slots can be reached from certain hashes. let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, matched_block_ssz) = bs + let (matched_block_hash, _) = bs .block_at_slot(&hashes[hashes_index], slots[slot_index]) .unwrap() .unwrap(); - let (retrieved_slot, _) = - slot_and_parent_from_block_ssz(&matched_block_ssz, 0).unwrap(); - assert_eq!(retrieved_slot, slots[slot_index]); - assert_eq!(matched_block_hash, hashes[slot_index].to_vec()); + assert_eq!(matched_block_hash, hashes[slot_index]); } let ssz = bs.block_at_slot(&hashes[4], 2).unwrap(); diff --git a/lighthouse/db/src/stores/beacon_state_store.rs b/lighthouse/db/src/stores/beacon_state_store.rs index 687fc0f222..122b225094 100644 --- a/lighthouse/db/src/stores/beacon_state_store.rs +++ b/lighthouse/db/src/stores/beacon_state_store.rs @@ -11,29 +11,20 @@ where db: Arc, } +// Implements `put`, `get`, `exists` and `delete` for the store. +impl_crud_for_store!(BeaconStateStore, DB_COLUMN); + impl BeaconStateStore { pub fn new(db: Arc) -> Self { Self { db } } - pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { - self.db.put(DB_COLUMN, hash, ssz) - } - - pub fn get(&self, hash: &[u8]) -> Result>, DBError> { - self.db.get(DB_COLUMN, hash) - } - - pub fn exists(&self, hash: &[u8]) -> Result { - self.db.exists(DB_COLUMN, hash) - } - - pub fn delete(&self, hash: &[u8]) -> Result<(), DBError> { - self.db.delete(DB_COLUMN, hash) - } - - /// Retuns a fully de-serialized `BeaconState` (or `None` if hash not known). - pub fn get_deserialized(&self, hash: &[u8]) -> Result, DBError> { + /// Retuns an object implementing `BeaconStateReader`, or `None` (if hash not known). + /// + /// Note: Presently, this function fully deserializes a `BeaconState` and returns that. In the + /// future, it would be ideal to return an object capable of reading directly from serialized + /// SSZ bytes. + pub fn get_reader(&self, hash: &Hash256) -> Result, DBError> { match self.get(&hash)? { None => Ok(None), Some(ssz) => { @@ -45,3 +36,33 @@ impl BeaconStateStore { } } } + +#[cfg(test)] +mod tests { + use super::super::super::MemoryDB; + use super::*; + + use std::sync::Arc; + use ssz::ssz_encode; + use types::Hash256; + use types::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + + test_crud_for_store!(BeaconStateStore, DB_COLUMN); + + #[test] + fn test_reader() { + let db = Arc::new(MemoryDB::open()); + let store = BeaconStateStore::new(db.clone()); + + let mut rng = XorShiftRng::from_seed([42; 16]); + let state = BeaconState::random_for_test(&mut rng); + let state_root = state.canonical_root(); + + store.put(&state_root, &ssz_encode(&state)).unwrap(); + + let reader = store.get_reader(&state_root).unwrap().unwrap(); + let decoded = reader.into_beacon_state().unwrap(); + + assert_eq!(state, decoded); + } +} diff --git a/lighthouse/db/src/stores/macros.rs b/lighthouse/db/src/stores/macros.rs new file mode 100644 index 0000000000..2086dcbacd --- /dev/null +++ b/lighthouse/db/src/stores/macros.rs @@ -0,0 +1,102 @@ +macro_rules! impl_crud_for_store { + ($store: ident, $db_column: expr) => { + impl $store { + pub fn put(&self, hash: &Hash256, ssz: &[u8]) -> Result<(), DBError> { + self.db.put($db_column, hash, ssz) + } + + pub fn get(&self, hash: &Hash256) -> Result>, DBError> { + self.db.get($db_column, hash) + } + + pub fn exists(&self, hash: &Hash256) -> Result { + self.db.exists($db_column, hash) + } + + pub fn delete(&self, hash: &Hash256) -> Result<(), DBError> { + self.db.delete($db_column, hash) + } + } + }; +} + +macro_rules! test_crud_for_store { + ($store: ident, $db_column: expr) => { + #[test] + fn test_put() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + store.put(hash, ssz).unwrap(); + assert_eq!(db.get(DB_COLUMN, hash).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert_eq!(store.get(hash).unwrap().unwrap(), ssz); + } + + #[test] + fn test_get_unknown() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); + + db.put(DB_COLUMN, other_hash, ssz).unwrap(); + assert_eq!(store.get(hash).unwrap(), None); + } + + #[test] + fn test_exists() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(store.exists(hash).unwrap()); + } + + #[test] + fn test_block_does_not_exist() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + let other_hash = &Hash256::from("another hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(!store.exists(other_hash).unwrap()); + } + + #[test] + fn test_delete() { + let db = Arc::new(MemoryDB::open()); + let store = $store::new(db.clone()); + + let ssz = "some bytes".as_bytes(); + let hash = &Hash256::from("some hash".as_bytes()); + + db.put(DB_COLUMN, hash, ssz).unwrap(); + assert!(db.exists(DB_COLUMN, hash).unwrap()); + + store.delete(hash).unwrap(); + assert!(!db.exists(DB_COLUMN, hash).unwrap()); + } + }; +} diff --git a/lighthouse/db/src/stores/mod.rs b/lighthouse/db/src/stores/mod.rs index 7fc51abe70..c78d10dbf9 100644 --- a/lighthouse/db/src/stores/mod.rs +++ b/lighthouse/db/src/stores/mod.rs @@ -1,5 +1,7 @@ use super::{ClientDB, DBError}; +#[macro_use] +mod macros; mod beacon_block_store; mod beacon_state_store; mod pow_chain_store; @@ -17,4 +19,9 @@ pub const STATES_DB_COLUMN: &str = "states"; pub const POW_CHAIN_DB_COLUMN: &str = "powchain"; pub const VALIDATOR_DB_COLUMN: &str = "validator"; -pub const COLUMNS: [&str; 4] = [BLOCKS_DB_COLUMN, STATES_DB_COLUMN, POW_CHAIN_DB_COLUMN, VALIDATOR_DB_COLUMN]; +pub const COLUMNS: [&str; 4] = [ + BLOCKS_DB_COLUMN, + STATES_DB_COLUMN, + POW_CHAIN_DB_COLUMN, + VALIDATOR_DB_COLUMN, +]; diff --git a/lighthouse/db/src/stores/pow_chain_store.rs b/lighthouse/db/src/stores/pow_chain_store.rs index aa2b267f70..a7c77bab5a 100644 --- a/lighthouse/db/src/stores/pow_chain_store.rs +++ b/lighthouse/db/src/stores/pow_chain_store.rs @@ -26,9 +26,9 @@ impl PoWChainStore { #[cfg(test)] mod tests { extern crate types; - - use super::*; + use super::super::super::MemoryDB; + use super::*; use self::types::Hash256; From ad91648f30248e0e62146de14112e9fd50987cd4 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 4 Jan 2019 18:10:14 +1100 Subject: [PATCH 08/15] Tidy up SSZ testing on types crate --- beacon_chain/types/src/attestation.rs | 3 +-- beacon_chain/types/src/attestation_data.rs | 3 +-- beacon_chain/types/src/beacon_block.rs | 3 +-- beacon_chain/types/src/beacon_block_body.rs | 3 +-- beacon_chain/types/src/beacon_state.rs | 18 ++++++++++++++++++ .../src/candidate_pow_receipt_root_record.rs | 3 +-- beacon_chain/types/src/casper_slashing.rs | 3 +-- beacon_chain/types/src/crosslink_record.rs | 3 +-- beacon_chain/types/src/deposit_data.rs | 3 +-- beacon_chain/types/src/deposit_input.rs | 3 +-- beacon_chain/types/src/exit.rs | 3 +-- beacon_chain/types/src/fork_data.rs | 3 +-- .../types/src/pending_attestation_record.rs | 3 +-- beacon_chain/types/src/proposal_signed_data.rs | 3 +-- beacon_chain/types/src/proposer_slashing.rs | 3 +-- beacon_chain/types/src/readers/block_reader.rs | 6 +++--- beacon_chain/types/src/readers/state_reader.rs | 6 +++--- beacon_chain/types/src/shard_committee.rs | 3 +-- .../types/src/shard_reassignment_record.rs | 3 +-- beacon_chain/types/src/slashable_vote_data.rs | 3 +-- beacon_chain/types/src/validator_record.rs | 3 +-- 21 files changed, 42 insertions(+), 42 deletions(-) diff --git a/beacon_chain/types/src/attestation.rs b/beacon_chain/types/src/attestation.rs index 7a94eee51c..fb8b946a5c 100644 --- a/beacon_chain/types/src/attestation.rs +++ b/beacon_chain/types/src/attestation.rs @@ -64,8 +64,7 @@ impl TestRandom for Attestation { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/attestation_data.rs b/beacon_chain/types/src/attestation_data.rs index 616c52d790..e1d70e2b01 100644 --- a/beacon_chain/types/src/attestation_data.rs +++ b/beacon_chain/types/src/attestation_data.rs @@ -104,8 +104,7 @@ impl TestRandom for AttestationData { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/beacon_block.rs b/beacon_chain/types/src/beacon_block.rs index 826667d5ef..30a1bd83b0 100644 --- a/beacon_chain/types/src/beacon_block.rs +++ b/beacon_chain/types/src/beacon_block.rs @@ -79,8 +79,7 @@ impl TestRandom for BeaconBlock { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/beacon_block_body.rs b/beacon_chain/types/src/beacon_block_body.rs index f5ac223432..8bcf1af334 100644 --- a/beacon_chain/types/src/beacon_block_body.rs +++ b/beacon_chain/types/src/beacon_block_body.rs @@ -59,8 +59,7 @@ impl TestRandom for BeaconBlockBody { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/beacon_state.rs b/beacon_chain/types/src/beacon_state.rs index f2d8f6aeb5..7bd0283c59 100644 --- a/beacon_chain/types/src/beacon_state.rs +++ b/beacon_chain/types/src/beacon_state.rs @@ -166,3 +166,21 @@ impl TestRandom for BeaconState { } } } + +#[cfg(test)] +mod tests { + use super::super::ssz::ssz_encode; + use super::*; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; + + #[test] + pub fn test_ssz_round_trip() { + let mut rng = XorShiftRng::from_seed([42; 16]); + let original = BeaconState::random_for_test(&mut rng); + + let bytes = ssz_encode(&original); + let (decoded, _) = <_>::ssz_decode(&bytes, 0).unwrap(); + + assert_eq!(original, decoded); + } +} diff --git a/beacon_chain/types/src/candidate_pow_receipt_root_record.rs b/beacon_chain/types/src/candidate_pow_receipt_root_record.rs index 663c3e1dbf..5f260c4234 100644 --- a/beacon_chain/types/src/candidate_pow_receipt_root_record.rs +++ b/beacon_chain/types/src/candidate_pow_receipt_root_record.rs @@ -44,8 +44,7 @@ impl TestRandom for CandidatePoWReceiptRootRecord { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/casper_slashing.rs b/beacon_chain/types/src/casper_slashing.rs index f3c1b5d180..08dbd9ff33 100644 --- a/beacon_chain/types/src/casper_slashing.rs +++ b/beacon_chain/types/src/casper_slashing.rs @@ -44,8 +44,7 @@ impl TestRandom for CasperSlashing { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/crosslink_record.rs b/beacon_chain/types/src/crosslink_record.rs index 9e525664a0..ae6c5bf929 100644 --- a/beacon_chain/types/src/crosslink_record.rs +++ b/beacon_chain/types/src/crosslink_record.rs @@ -54,8 +54,7 @@ impl TestRandom for CrosslinkRecord { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/deposit_data.rs b/beacon_chain/types/src/deposit_data.rs index b236709d20..5dcd20c7cb 100644 --- a/beacon_chain/types/src/deposit_data.rs +++ b/beacon_chain/types/src/deposit_data.rs @@ -49,8 +49,7 @@ impl TestRandom for DepositData { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/deposit_input.rs b/beacon_chain/types/src/deposit_input.rs index 3a9a08412c..d7e2272e91 100644 --- a/beacon_chain/types/src/deposit_input.rs +++ b/beacon_chain/types/src/deposit_input.rs @@ -55,8 +55,7 @@ impl TestRandom for DepositInput { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/exit.rs b/beacon_chain/types/src/exit.rs index 6a6d573051..eeac11ce75 100644 --- a/beacon_chain/types/src/exit.rs +++ b/beacon_chain/types/src/exit.rs @@ -49,8 +49,7 @@ impl TestRandom for Exit { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/fork_data.rs b/beacon_chain/types/src/fork_data.rs index e779c797e7..7cd4dc3ac5 100644 --- a/beacon_chain/types/src/fork_data.rs +++ b/beacon_chain/types/src/fork_data.rs @@ -48,8 +48,7 @@ impl TestRandom for ForkData { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/pending_attestation_record.rs b/beacon_chain/types/src/pending_attestation_record.rs index aa289e9048..3bebf5676d 100644 --- a/beacon_chain/types/src/pending_attestation_record.rs +++ b/beacon_chain/types/src/pending_attestation_record.rs @@ -54,8 +54,7 @@ impl TestRandom for PendingAttestationRecord { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/proposal_signed_data.rs b/beacon_chain/types/src/proposal_signed_data.rs index 7a01fc10a5..e38a9cadb2 100644 --- a/beacon_chain/types/src/proposal_signed_data.rs +++ b/beacon_chain/types/src/proposal_signed_data.rs @@ -49,8 +49,7 @@ impl TestRandom for ProposalSignedData { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/proposer_slashing.rs b/beacon_chain/types/src/proposer_slashing.rs index 0ae1c6e663..3754c3b328 100644 --- a/beacon_chain/types/src/proposer_slashing.rs +++ b/beacon_chain/types/src/proposer_slashing.rs @@ -60,8 +60,7 @@ impl TestRandom for ProposerSlashing { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/readers/block_reader.rs b/beacon_chain/types/src/readers/block_reader.rs index e1cba224d2..1698a0380a 100644 --- a/beacon_chain/types/src/readers/block_reader.rs +++ b/beacon_chain/types/src/readers/block_reader.rs @@ -5,7 +5,7 @@ pub trait BeaconBlockReader { fn parent_root(&self) -> Hash256; fn state_root(&self) -> Hash256; fn canonical_root(&self) -> Hash256; - fn to_beacon_block(self) -> BeaconBlock; + fn into_beacon_block(self) -> Option; } impl BeaconBlockReader for BeaconBlock { @@ -25,7 +25,7 @@ impl BeaconBlockReader for BeaconBlock { self.canonical_root() } - fn to_beacon_block(self) -> BeaconBlock { - self + fn into_beacon_block(self) -> Option { + Some(self) } } diff --git a/beacon_chain/types/src/readers/state_reader.rs b/beacon_chain/types/src/readers/state_reader.rs index a43ef9930e..cbbf0854cb 100644 --- a/beacon_chain/types/src/readers/state_reader.rs +++ b/beacon_chain/types/src/readers/state_reader.rs @@ -3,7 +3,7 @@ use crate::{BeaconState, Hash256}; pub trait BeaconStateReader { fn slot(&self) -> u64; fn canonical_root(&self) -> Hash256; - fn to_beacon_state(self) -> BeaconState; + fn into_beacon_state(self) -> Option; } impl BeaconStateReader for BeaconState { @@ -15,7 +15,7 @@ impl BeaconStateReader for BeaconState { self.canonical_root() } - fn to_beacon_state(self) -> BeaconState { - self + fn into_beacon_state(self) -> Option { + Some(self) } } diff --git a/beacon_chain/types/src/shard_committee.rs b/beacon_chain/types/src/shard_committee.rs index 073dd01769..d920f7db3a 100644 --- a/beacon_chain/types/src/shard_committee.rs +++ b/beacon_chain/types/src/shard_committee.rs @@ -37,8 +37,7 @@ impl TestRandom for ShardCommittee { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/shard_reassignment_record.rs b/beacon_chain/types/src/shard_reassignment_record.rs index e990afeac8..9bc014689a 100644 --- a/beacon_chain/types/src/shard_reassignment_record.rs +++ b/beacon_chain/types/src/shard_reassignment_record.rs @@ -48,8 +48,7 @@ impl TestRandom for ShardReassignmentRecord { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/slashable_vote_data.rs b/beacon_chain/types/src/slashable_vote_data.rs index 4d8e2eab30..5703853b2f 100644 --- a/beacon_chain/types/src/slashable_vote_data.rs +++ b/beacon_chain/types/src/slashable_vote_data.rs @@ -55,8 +55,7 @@ impl TestRandom for SlashableVoteData { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { diff --git a/beacon_chain/types/src/validator_record.rs b/beacon_chain/types/src/validator_record.rs index 45b6c301af..374d956239 100644 --- a/beacon_chain/types/src/validator_record.rs +++ b/beacon_chain/types/src/validator_record.rs @@ -150,8 +150,7 @@ impl TestRandom for ValidatorRecord { mod tests { use super::super::ssz::ssz_encode; use super::*; - use crate::test_utils::TestRandom; - use rand::{prng::XorShiftRng, SeedableRng}; + use crate::test_utils::{SeedableRng, TestRandom, XorShiftRng}; #[test] pub fn test_ssz_round_trip() { From 62640ad6919268a12360a423991ff5c4265a91de Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 4 Jan 2019 18:12:32 +1100 Subject: [PATCH 09/15] Remove old chain maps and stores mods --- lighthouse/beacon_chain/src/lib.rs | 4 +- lighthouse/beacon_chain/src/maps.rs | 127 -------------------------- lighthouse/beacon_chain/src/stores.rs | 9 -- 3 files changed, 1 insertion(+), 139 deletions(-) delete mode 100644 lighthouse/beacon_chain/src/maps.rs delete mode 100644 lighthouse/beacon_chain/src/stores.rs diff --git a/lighthouse/beacon_chain/src/lib.rs b/lighthouse/beacon_chain/src/lib.rs index 632aa2c852..6acdc3ef69 100644 --- a/lighthouse/beacon_chain/src/lib.rs +++ b/lighthouse/beacon_chain/src/lib.rs @@ -1,14 +1,12 @@ mod block_processing; mod block_production; -mod maps; -mod stores; use db::{ stores::{BeaconBlockStore, BeaconStateStore}, ClientDB, DBError, }; use genesis::{genesis_beacon_block, genesis_beacon_state, GenesisError}; -use slot_clock::{SlotClock, TestingSlotClockError}; +use slot_clock::SlotClock; use spec::ChainSpec; use ssz::ssz_encode; use std::collections::HashSet; diff --git a/lighthouse/beacon_chain/src/maps.rs b/lighthouse/beacon_chain/src/maps.rs deleted file mode 100644 index 4c737cb447..0000000000 --- a/lighthouse/beacon_chain/src/maps.rs +++ /dev/null @@ -1,127 +0,0 @@ -use types::{AttesterMap, ProposerMap, ShardCommittee}; - -#[derive(Debug, PartialEq)] -pub enum AttesterAndProposerMapError { - NoShardCommitteeForSlot, - NoAvailableProposer, -} - -/// Generate a map of `(slot, shard) |--> committee`. -/// -/// The attester map is used to optimise the lookup of a committee. -pub fn generate_attester_and_proposer_maps( - shard_and_committee_for_slots: &Vec>, - start_slot: u64, -) -> Result<(AttesterMap, ProposerMap), AttesterAndProposerMapError> { - let mut attester_map = AttesterMap::new(); - let mut proposer_map = ProposerMap::new(); - for (i, slot) in shard_and_committee_for_slots.iter().enumerate() { - /* - * Store the proposer for the block. - */ - let slot_number = (i as u64).saturating_add(start_slot); - let first_committee = &slot - .get(0) - .ok_or(AttesterAndProposerMapError::NoShardCommitteeForSlot)? - .committee; - let proposer_index = (slot_number as usize) - .checked_rem(first_committee.len()) - .ok_or(AttesterAndProposerMapError::NoAvailableProposer)?; - proposer_map.insert(slot_number, first_committee[proposer_index]); - - /* - * Loop through the shards and extend the attester map. - */ - for shard_and_committee in slot { - let committee = shard_and_committee.committee.clone(); - attester_map.insert((slot_number, shard_and_committee.shard), committee); - } - } - Ok((attester_map, proposer_map)) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sac_generator( - shard_count: u64, - slot_count: usize, - sac_per_slot: usize, - committee_size: usize, - ) -> Vec> { - let mut shard = 0; - let mut validator = 0; - let mut cycle = vec![]; - - for _ in 0..slot_count { - let mut slot: Vec = vec![]; - for _ in 0..sac_per_slot { - let mut sac = ShardCommittee { - shard: shard % shard_count, - committee: vec![], - }; - for _ in 0..committee_size { - sac.committee.push(validator); - validator += 1; - } - slot.push(sac); - shard += 1; - } - cycle.push(slot); - } - cycle - } - - #[test] - fn test_attester_proposer_maps_empty_slots() { - let sac = sac_generator(4, 4, 0, 1); - let result = generate_attester_and_proposer_maps(&sac, 0); - assert_eq!( - result, - Err(AttesterAndProposerMapError::NoShardCommitteeForSlot) - ); - } - - #[test] - fn test_attester_proposer_maps_empty_committees() { - let sac = sac_generator(4, 4, 1, 0); - let result = generate_attester_and_proposer_maps(&sac, 0); - assert_eq!( - result, - Err(AttesterAndProposerMapError::NoAvailableProposer) - ); - } - - #[test] - fn test_attester_proposer_maps_scenario_a() { - let sac = sac_generator(4, 4, 1, 1); - let (a, p) = generate_attester_and_proposer_maps(&sac, 0).unwrap(); - - assert_eq!(*p.get(&0).unwrap(), 0); - assert_eq!(*p.get(&1).unwrap(), 1); - assert_eq!(*p.get(&2).unwrap(), 2); - assert_eq!(*p.get(&3).unwrap(), 3); - - assert_eq!(*a.get(&(0, 0)).unwrap(), vec![0]); - assert_eq!(*a.get(&(1, 1)).unwrap(), vec![1]); - assert_eq!(*a.get(&(2, 2)).unwrap(), vec![2]); - assert_eq!(*a.get(&(3, 3)).unwrap(), vec![3]); - } - - #[test] - fn test_attester_proposer_maps_scenario_b() { - let sac = sac_generator(4, 4, 1, 4); - let (a, p) = generate_attester_and_proposer_maps(&sac, 0).unwrap(); - - assert_eq!(*p.get(&0).unwrap(), 0); - assert_eq!(*p.get(&1).unwrap(), 5); - assert_eq!(*p.get(&2).unwrap(), 10); - assert_eq!(*p.get(&3).unwrap(), 15); - - assert_eq!(*a.get(&(0, 0)).unwrap(), vec![0, 1, 2, 3]); - assert_eq!(*a.get(&(1, 1)).unwrap(), vec![4, 5, 6, 7]); - assert_eq!(*a.get(&(2, 2)).unwrap(), vec![8, 9, 10, 11]); - assert_eq!(*a.get(&(3, 3)).unwrap(), vec![12, 13, 14, 15]); - } -} diff --git a/lighthouse/beacon_chain/src/stores.rs b/lighthouse/beacon_chain/src/stores.rs deleted file mode 100644 index fae097899e..0000000000 --- a/lighthouse/beacon_chain/src/stores.rs +++ /dev/null @@ -1,9 +0,0 @@ -use db::stores::{BeaconBlockStore, PoWChainStore, ValidatorStore}; -use db::ClientDB; -use std::sync::Arc; - -pub struct BeaconChainStore { - pub block: Arc>, - pub pow_chain: Arc>, - pub validator: Arc>, -} From 3876e29f6e0d8d20e2cb1f4df3b4dd92080bbe4c Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Fri, 4 Jan 2019 18:30:24 +1100 Subject: [PATCH 10/15] Fix clippy lints --- beacon_chain/genesis/src/beacon_block.rs | 2 +- beacon_chain/naive_fork_choice/src/lib.rs | 6 ++--- beacon_chain/spec/src/foundation.rs | 2 +- beacon_chain/types/src/validator_record.rs | 4 ++-- .../utils/bls/src/aggregate_signature.rs | 2 +- .../slot_clock/src/system_time_slot_clock.rs | 2 +- .../beacon_chain/src/block_production.rs | 22 ++++++++++++------- lighthouse/beacon_chain/src/lib.rs | 2 +- lighthouse/db/src/stores/macros.rs | 1 + 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/beacon_chain/genesis/src/beacon_block.rs b/beacon_chain/genesis/src/beacon_block.rs index 65adc1cf76..696ac6499f 100644 --- a/beacon_chain/genesis/src/beacon_block.rs +++ b/beacon_chain/genesis/src/beacon_block.rs @@ -8,7 +8,7 @@ pub fn genesis_beacon_block(state_root: Hash256, spec: &ChainSpec) -> BeaconBloc BeaconBlock { slot: spec.initial_slot_number, parent_root: spec.zero_hash, - state_root: state_root, + state_root, randao_reveal: spec.zero_hash, candidate_pow_receipt_root: spec.zero_hash, signature: genesis_signature(), diff --git a/beacon_chain/naive_fork_choice/src/lib.rs b/beacon_chain/naive_fork_choice/src/lib.rs index 385c9c50d0..9a75786078 100644 --- a/beacon_chain/naive_fork_choice/src/lib.rs +++ b/beacon_chain/naive_fork_choice/src/lib.rs @@ -15,8 +15,8 @@ pub enum ForkChoiceError { } pub fn naive_fork_choice( - head_block_hashes: &Vec, - block_store: Arc>, + head_block_hashes: &[Hash256], + block_store: &Arc>, ) -> Result, ForkChoiceError> where T: ClientDB + Sized, @@ -28,7 +28,7 @@ where */ for (index, block_hash) in head_block_hashes.iter().enumerate() { let ssz = block_store - .get(&block_hash.to_vec()[..])? + .get(&block_hash)? .ok_or(ForkChoiceError::MissingBlock)?; let (block, _) = BeaconBlock::ssz_decode(&ssz, 0)?; head_blocks.push((index, block)); diff --git a/beacon_chain/spec/src/foundation.rs b/beacon_chain/spec/src/foundation.rs index b80187281a..4dd1931d22 100644 --- a/beacon_chain/spec/src/foundation.rs +++ b/beacon_chain/spec/src/foundation.rs @@ -64,7 +64,7 @@ impl ChainSpec { * Intialization parameters */ initial_validators: initial_validators_for_testing(), - genesis_time: 1544672897, + genesis_time: 1_544_672_897, processed_pow_receipt_root: Hash256::from("pow_root".as_bytes()), } } diff --git a/beacon_chain/types/src/validator_record.rs b/beacon_chain/types/src/validator_record.rs index 374d956239..1168296f7b 100644 --- a/beacon_chain/types/src/validator_record.rs +++ b/beacon_chain/types/src/validator_record.rs @@ -5,7 +5,7 @@ use rand::RngCore; use ssz::{Decodable, DecodeError, Encodable, SszStream}; use std::convert; -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Copy)] pub enum ValidatorStatus { PendingActivation, Active, @@ -87,7 +87,7 @@ impl TestRandom for ValidatorStatus { ValidatorStatus::Withdrawn, ValidatorStatus::Penalized, ]; - options[(rng.next_u32() as usize) % options.len()].clone() + options[(rng.next_u32() as usize) % options.len()] } } diff --git a/beacon_chain/utils/bls/src/aggregate_signature.rs b/beacon_chain/utils/bls/src/aggregate_signature.rs index 2e36302688..90bf44702b 100644 --- a/beacon_chain/utils/bls/src/aggregate_signature.rs +++ b/beacon_chain/utils/bls/src/aggregate_signature.rs @@ -6,7 +6,7 @@ use bls_aggregates::AggregateSignature as RawAggregateSignature; /// /// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ /// serialization). -#[derive(Debug, PartialEq, Clone)] +#[derive(Debug, PartialEq, Clone, Default)] pub struct AggregateSignature(RawAggregateSignature); impl AggregateSignature { diff --git a/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs b/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs index be7265ae8f..8c1e2f66c0 100644 --- a/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs +++ b/beacon_chain/utils/slot_clock/src/system_time_slot_clock.rs @@ -26,7 +26,7 @@ impl SystemTimeSlotClock { if slot_duration_seconds == 0 { Err(Error::SlotDurationIsZero) } else { - Ok(SystemTimeSlotClock { + Ok(Self { genesis_seconds, slot_duration_seconds, }) diff --git a/lighthouse/beacon_chain/src/block_production.rs b/lighthouse/beacon_chain/src/block_production.rs index 9754b4a54c..b83669cd21 100644 --- a/lighthouse/beacon_chain/src/block_production.rs +++ b/lighthouse/beacon_chain/src/block_production.rs @@ -23,25 +23,31 @@ where .present_slot()? .ok_or(Error::PresentSlotIsNone)?; let parent_root = self.canonical_leaf_block; - let parent_block = self + let parent_block_reader = self .block_store - .get_deserialized(&parent_root)? - .ok_or(Error::DBError("Block not found.".to_string()))?; - let parent_state = self + .get_reader(&parent_root)? + .ok_or_else(|| Error::DBError("Block not found.".to_string()))?; + let parent_state_reader = self .state_store - .get_deserialized(&parent_block.state_root())? - .ok_or(Error::DBError("State not found.".to_string()))?; + .get_reader(&parent_block_reader.state_root())? + .ok_or_else(|| Error::DBError("State not found.".to_string()))?; + let parent_block = parent_block_reader + .into_beacon_block() + .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?; let mut block = BeaconBlock { slot: present_slot, parent_root, state_root: Hash256::zero(), // Updated after the state is calculated. - ..parent_block.to_beacon_block() + ..parent_block }; + let parent_state = parent_state_reader + .into_beacon_state() + .ok_or_else(|| Error::DBError("Bad parent block SSZ.".to_string()))?; let state = BeaconState { slot: present_slot, - ..parent_state.to_beacon_state() + ..parent_state }; let state_root = state.canonical_root(); diff --git a/lighthouse/beacon_chain/src/lib.rs b/lighthouse/beacon_chain/src/lib.rs index 6acdc3ef69..2a3d415386 100644 --- a/lighthouse/beacon_chain/src/lib.rs +++ b/lighthouse/beacon_chain/src/lib.rs @@ -55,7 +55,7 @@ where block_store.put(&block_root, &ssz_encode(&genesis_block)[..])?; let mut leaf_blocks = HashSet::new(); - leaf_blocks.insert(block_root.clone()); + leaf_blocks.insert(block_root); Ok(Self { block_store, diff --git a/lighthouse/db/src/stores/macros.rs b/lighthouse/db/src/stores/macros.rs index 2086dcbacd..36b8aef8e3 100644 --- a/lighthouse/db/src/stores/macros.rs +++ b/lighthouse/db/src/stores/macros.rs @@ -20,6 +20,7 @@ macro_rules! impl_crud_for_store { }; } +#[allow(unused_macros)] macro_rules! test_crud_for_store { ($store: ident, $db_column: expr) => { #[test] From 96f4a30f64612faf1ddea8a0e218de39f1349777 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Jan 2019 17:04:15 +1100 Subject: [PATCH 11/15] Force `Debug` and `PartialEq` on reader traits. --- beacon_chain/types/src/readers/block_reader.rs | 11 ++++++++++- beacon_chain/types/src/readers/state_reader.rs | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/beacon_chain/types/src/readers/block_reader.rs b/beacon_chain/types/src/readers/block_reader.rs index 1698a0380a..91a2852acd 100644 --- a/beacon_chain/types/src/readers/block_reader.rs +++ b/beacon_chain/types/src/readers/block_reader.rs @@ -1,6 +1,15 @@ use crate::{BeaconBlock, Hash256}; +use std::fmt::Debug; -pub trait BeaconBlockReader { +/// The `BeaconBlockReader` provides interfaces for reading a subset of fields of a `BeaconBlock`. +/// +/// The purpose of this trait is to allow reading from either; +/// - a standard `BeaconBlock` struct, or +/// - a SSZ serialized byte array. +/// +/// Note: presently, direct SSZ reading has not been implemented so this trait is being used for +/// "future proofing". +pub trait BeaconBlockReader: Debug + PartialEq { fn slot(&self) -> u64; fn parent_root(&self) -> Hash256; fn state_root(&self) -> Hash256; diff --git a/beacon_chain/types/src/readers/state_reader.rs b/beacon_chain/types/src/readers/state_reader.rs index cbbf0854cb..47266bd364 100644 --- a/beacon_chain/types/src/readers/state_reader.rs +++ b/beacon_chain/types/src/readers/state_reader.rs @@ -1,6 +1,15 @@ use crate::{BeaconState, Hash256}; +use std::fmt::Debug; -pub trait BeaconStateReader { +/// The `BeaconStateReader` provides interfaces for reading a subset of fields of a `BeaconState`. +/// +/// The purpose of this trait is to allow reading from either; +/// - a standard `BeaconState` struct, or +/// - a SSZ serialized byte array. +/// +/// Note: presently, direct SSZ reading has not been implemented so this trait is being used for +/// "future proofing". +pub trait BeaconStateReader: Debug + PartialEq { fn slot(&self) -> u64; fn canonical_root(&self) -> Hash256; fn into_beacon_state(self) -> Option; From feff7d36605d862c39eb1d022cc77e7ff7de62e6 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Jan 2019 17:04:49 +1100 Subject: [PATCH 12/15] Update block_store `block_at_slot` to return trait --- .../db/src/stores/beacon_block_store.rs | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/lighthouse/db/src/stores/beacon_block_store.rs b/lighthouse/db/src/stores/beacon_block_store.rs index ff7247b1c4..73fc9125ba 100644 --- a/lighthouse/db/src/stores/beacon_block_store.rs +++ b/lighthouse/db/src/stores/beacon_block_store.rs @@ -52,23 +52,29 @@ impl BeaconBlockStore { /// slot number. If the slot is skipped, the function will return None. /// /// If a block is found, a tuple of (block_hash, serialized_block) is returned. + /// + /// Note: this function uses a loop instead of recursion as the compiler is over-strict when it + /// comes to recursion and the `impl Trait` pattern. See: + /// https://stackoverflow.com/questions/54032940/using-impl-trait-in-a-recursive-function pub fn block_at_slot( &self, head_hash: &Hash256, slot: u64, - ) -> Result, BeaconBlockAtSlotError> { - match self.get_reader(head_hash)? { - None => Err(BeaconBlockAtSlotError::UnknownBeaconBlock), - Some(block_reader) => match block_reader.slot() { - s if s == slot => { - let block = block_reader - .into_beacon_block() - .ok_or(BeaconBlockAtSlotError::InvalidBeaconBlock)?; - Ok(Some((*head_hash, block))) + ) -> Result, BeaconBlockAtSlotError> { + let mut current_hash = *head_hash; + + loop { + if let Some(block_reader) = self.get_reader(¤t_hash)? { + if block_reader.slot() == slot { + break Ok(Some((current_hash, block_reader))); + } else if block_reader.slot() < slot { + break Ok(None); + } else { + current_hash = block_reader.parent_root(); } - s if s < slot => Ok(None), - _ => self.block_at_slot(&block_reader.parent_root(), slot), - }, + } else { + break Err(BeaconBlockAtSlotError::UnknownBeaconBlock); + } } } } @@ -207,11 +213,12 @@ mod tests { // Test that certain slots can be reached from certain hashes. let test_cases = vec![(4, 4), (4, 3), (4, 2), (4, 1), (4, 0)]; for (hashes_index, slot_index) in test_cases { - let (matched_block_hash, _) = bs + let (matched_block_hash, reader) = bs .block_at_slot(&hashes[hashes_index], slots[slot_index]) .unwrap() .unwrap(); assert_eq!(matched_block_hash, hashes[slot_index]); + assert_eq!(reader.slot(), slots[slot_index]); } let ssz = bs.block_at_slot(&hashes[4], 2).unwrap(); From 9ea64b47dc4e7a8e12510eff42246f8b21fd08e0 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Jan 2019 17:37:34 +1100 Subject: [PATCH 13/15] Add additional block_store test. --- lighthouse/db/src/stores/beacon_block_store.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lighthouse/db/src/stores/beacon_block_store.rs b/lighthouse/db/src/stores/beacon_block_store.rs index 73fc9125ba..6477573e8e 100644 --- a/lighthouse/db/src/stores/beacon_block_store.rs +++ b/lighthouse/db/src/stores/beacon_block_store.rs @@ -100,6 +100,22 @@ mod tests { test_crud_for_store!(BeaconBlockStore, DB_COLUMN); + #[test] + fn head_hash_slot_too_low() { + let db = Arc::new(MemoryDB::open()); + let bs = Arc::new(BeaconBlockStore::new(db.clone())); + let mut rng = XorShiftRng::from_seed([42; 16]); + + let mut block = BeaconBlock::random_for_test(&mut rng); + block.slot = 10; + + let block_root = block.canonical_root(); + bs.put(&block_root, &ssz_encode(&block)).unwrap(); + + let result = bs.block_at_slot(&block_root, 11).unwrap(); + assert_eq!(result, None); + } + #[test] fn test_invalid_block_at_slot() { let db = Arc::new(MemoryDB::open()); From 355edeae2c5c3edba82a96430f4e86fcfe66a114 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Jan 2019 17:37:48 +1100 Subject: [PATCH 14/15] Fix formatting on types/src/lib.rs --- beacon_chain/types/src/lib.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/beacon_chain/types/src/lib.rs b/beacon_chain/types/src/lib.rs index 7863944d70..8a7fee0f97 100644 --- a/beacon_chain/types/src/lib.rs +++ b/beacon_chain/types/src/lib.rs @@ -6,8 +6,8 @@ extern crate ssz; pub mod test_utils; pub mod active_state; -pub mod attestation_data; pub mod attestation; +pub mod attestation_data; pub mod beacon_block; pub mod beacon_block_body; pub mod beacon_state; @@ -26,20 +26,19 @@ pub mod proposal_signed_data; pub mod proposer_slashing; pub mod shard_committee; pub mod shard_reassignment_record; -pub mod special_record; pub mod slashable_vote_data; +pub mod special_record; pub mod validator_record; pub mod validator_registration; pub mod readers; - use self::ethereum_types::{H160, H256, U256}; use std::collections::HashMap; pub use crate::active_state::ActiveState; -pub use crate::attestation_data::AttestationData; pub use crate::attestation::Attestation; +pub use crate::attestation_data::AttestationData; pub use crate::beacon_block::BeaconBlock; pub use crate::beacon_block_body::BeaconBlockBody; pub use crate::beacon_state::BeaconState; @@ -55,8 +54,8 @@ pub use crate::fork_data::ForkData; pub use crate::pending_attestation_record::PendingAttestationRecord; pub use crate::proposal_signed_data::ProposalSignedData; pub use crate::proposer_slashing::ProposerSlashing; -pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::shard_committee::ShardCommittee; +pub use crate::slashable_vote_data::SlashableVoteData; pub use crate::special_record::{SpecialRecord, SpecialRecordKind}; pub use crate::validator_record::{ValidatorRecord, ValidatorStatus}; pub use crate::validator_registration::ValidatorRegistration; From 09f1db0282f5290acacc8cd43283a7f81b888705 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Jan 2019 18:04:42 +1100 Subject: [PATCH 15/15] Add stub message to chain block_production --- lighthouse/beacon_chain/src/block_production.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lighthouse/beacon_chain/src/block_production.rs b/lighthouse/beacon_chain/src/block_production.rs index b83669cd21..ba781d6e92 100644 --- a/lighthouse/beacon_chain/src/block_production.rs +++ b/lighthouse/beacon_chain/src/block_production.rs @@ -18,6 +18,11 @@ where Error: From<::Error>, { pub fn produce_block(&mut self) -> Result<(BeaconBlock, BeaconState), Error> { + /* + * Important: this code is a big stub and only exists to ensure that tests pass. + * + * https://github.com/sigp/lighthouse/issues/107 + */ let present_slot = self .slot_clock .present_slot()?