From 0fbe4179b31938bf565a5cff1813dd2563a39f94 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 2 Oct 2018 16:41:10 +1000 Subject: [PATCH] Heavily restructure repo Separate most modules into crates --- Cargo.toml | 20 +- beacon_chain/types/Cargo.toml | 11 + .../types/src}/active_state.rs | 2 +- .../types/src/attestation_record.rs | 2 +- .../types/src/block.rs | 2 +- .../types/src}/chain_config.rs | 0 .../src}/common/delegation/block_hash.rs | 0 .../types/src}/common/delegation/mod.rs | 0 .../types/src}/common/maps.rs | 0 .../types/src}/common/mod.rs | 0 .../types/src}/common/shuffling/README.md | 0 .../types/src}/common/shuffling/mod.rs | 0 .../types/src}/common/shuffling/rng.rs | 0 .../types/src}/crosslink_record.rs | 2 +- .../types/src}/crystallized_state.rs | 2 +- beacon_chain/types/src/lib.rs | 49 +++ .../state => beacon_chain/types/src}/mod.rs | 0 .../types/src}/shard_and_committee.rs | 1 - .../types/src}/validator_record.rs | 17 +- beacon_chain/utils/bls/Cargo.toml | 7 + .../utils/bls/src/lib.rs | 0 .../utils/boolean-bitfield}/Cargo.toml | 0 .../utils/boolean-bitfield}/README.md | 0 .../utils/boolean-bitfield}/src/lib.rs | 2 + beacon_chain/utils/hashing/Cargo.toml | 7 + .../utils/hashing/src/lib.rs | 4 +- {ssz => beacon_chain/utils/ssz}/Cargo.toml | 0 {ssz => beacon_chain/utils/ssz}/README.md | 0 {ssz => beacon_chain/utils/ssz}/src/decode.rs | 0 {ssz => beacon_chain/utils/ssz}/src/encode.rs | 0 .../utils/ssz}/src/impl_decode.rs | 2 +- .../utils/ssz}/src/impl_encode.rs | 0 {ssz => beacon_chain/utils/ssz}/src/lib.rs | 0 beacon_chain/utils/ssz_helpers/Cargo.toml | 10 + .../src/attestation_ssz_splitter.rs | 6 +- beacon_chain/utils/ssz_helpers/src/lib.rs | 7 + .../utils/ssz_helpers/src}/ssz_block.rs | 14 +- benches/block_validation.rs | 148 ------- benches/main.rs | 5 - lighthouse/db/Cargo.toml | 10 + lighthouse/db/{ => src}/disk_db.rs | 0 lighthouse/db/{mod.rs => src/lib.rs} | 4 +- lighthouse/db/{ => src}/memory_db.rs | 0 lighthouse/db/{ => src}/stores/block_store.rs | 0 lighthouse/db/{ => src}/stores/mod.rs | 0 .../db/{ => src}/stores/pow_chain_store.rs | 0 .../db/{ => src}/stores/validator_store.rs | 0 lighthouse/db/{ => src}/traits.rs | 0 lighthouse/lib.rs | 22 -- lighthouse/main.rs | 14 +- lighthouse/state/attestation_record/mod.rs | 24 -- .../validation/attestation_validation.rs | 208 ---------- .../validation/message_generation.rs | 70 ---- .../attestation_record/validation/mod.rs | 16 - .../validation/signature_verification.rs | 183 --------- lighthouse/state/block/mod.rs | 17 - .../block/validation/block_validation.rs | 370 ------------------ lighthouse/state/block/validation/mod.rs | 21 - .../state/common/attestation_parent_hashes.rs | 226 ----------- lighthouse/utils/errors.rs | 8 - lighthouse/utils/logging.rs | 22 -- lighthouse/utils/macros.rs | 8 - lighthouse/utils/mod.rs | 11 - lighthouse/utils/types.rs | 12 - tests/attestation_validation/helpers.rs | 192 --------- tests/attestation_validation/mod.rs | 8 - tests/attestation_validation/tests.rs | 127 ------ tests/block_validation/helpers.rs | 230 ----------- tests/block_validation/mod.rs | 10 - tests/block_validation/tests.rs | 245 ------------ tests/main.rs | 12 - 71 files changed, 150 insertions(+), 2240 deletions(-) create mode 100644 beacon_chain/types/Cargo.toml rename {lighthouse/state => beacon_chain/types/src}/active_state.rs (94%) rename lighthouse/state/attestation_record/structs.rs => beacon_chain/types/src/attestation_record.rs (98%) rename lighthouse/state/block/structs.rs => beacon_chain/types/src/block.rs (98%) rename {lighthouse/state => beacon_chain/types/src}/chain_config.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/delegation/block_hash.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/delegation/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/maps.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/README.md (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/common/shuffling/rng.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/crosslink_record.rs (93%) rename {lighthouse/state => beacon_chain/types/src}/crystallized_state.rs (98%) create mode 100644 beacon_chain/types/src/lib.rs rename {lighthouse/state => beacon_chain/types/src}/mod.rs (100%) rename {lighthouse/state => beacon_chain/types/src}/shard_and_committee.rs (99%) rename {lighthouse/state => beacon_chain/types/src}/validator_record.rs (89%) create mode 100644 beacon_chain/utils/bls/Cargo.toml rename lighthouse/bls/mod.rs => beacon_chain/utils/bls/src/lib.rs (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/Cargo.toml (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/README.md (100%) rename {boolean-bitfield => beacon_chain/utils/boolean-bitfield}/src/lib.rs (99%) create mode 100644 beacon_chain/utils/hashing/Cargo.toml rename lighthouse/utils/hash.rs => beacon_chain/utils/hashing/src/lib.rs (66%) rename {ssz => beacon_chain/utils/ssz}/Cargo.toml (100%) rename {ssz => beacon_chain/utils/ssz}/README.md (100%) rename {ssz => beacon_chain/utils/ssz}/src/decode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/encode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/impl_decode.rs (99%) rename {ssz => beacon_chain/utils/ssz}/src/impl_encode.rs (100%) rename {ssz => beacon_chain/utils/ssz}/src/lib.rs (100%) create mode 100644 beacon_chain/utils/ssz_helpers/Cargo.toml rename lighthouse/state/attestation_record/ssz_splitter.rs => beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs (96%) create mode 100644 beacon_chain/utils/ssz_helpers/src/lib.rs rename {lighthouse/state/block => beacon_chain/utils/ssz_helpers/src}/ssz_block.rs (97%) delete mode 100644 benches/block_validation.rs delete mode 100644 benches/main.rs create mode 100644 lighthouse/db/Cargo.toml rename lighthouse/db/{ => src}/disk_db.rs (100%) rename lighthouse/db/{mod.rs => src/lib.rs} (93%) rename lighthouse/db/{ => src}/memory_db.rs (100%) rename lighthouse/db/{ => src}/stores/block_store.rs (100%) rename lighthouse/db/{ => src}/stores/mod.rs (100%) rename lighthouse/db/{ => src}/stores/pow_chain_store.rs (100%) rename lighthouse/db/{ => src}/stores/validator_store.rs (100%) rename lighthouse/db/{ => src}/traits.rs (100%) delete mode 100644 lighthouse/lib.rs delete mode 100644 lighthouse/state/attestation_record/mod.rs delete mode 100644 lighthouse/state/attestation_record/validation/attestation_validation.rs delete mode 100644 lighthouse/state/attestation_record/validation/message_generation.rs delete mode 100644 lighthouse/state/attestation_record/validation/mod.rs delete mode 100644 lighthouse/state/attestation_record/validation/signature_verification.rs delete mode 100644 lighthouse/state/block/mod.rs delete mode 100644 lighthouse/state/block/validation/block_validation.rs delete mode 100644 lighthouse/state/block/validation/mod.rs delete mode 100644 lighthouse/state/common/attestation_parent_hashes.rs delete mode 100644 lighthouse/utils/errors.rs delete mode 100644 lighthouse/utils/logging.rs delete mode 100644 lighthouse/utils/macros.rs delete mode 100644 lighthouse/utils/mod.rs delete mode 100644 lighthouse/utils/types.rs delete mode 100644 tests/attestation_validation/helpers.rs delete mode 100644 tests/attestation_validation/mod.rs delete mode 100644 tests/attestation_validation/tests.rs delete mode 100644 tests/block_validation/helpers.rs delete mode 100644 tests/block_validation/mod.rs delete mode 100644 tests/block_validation/tests.rs delete mode 100644 tests/main.rs diff --git a/Cargo.toml b/Cargo.toml index 50c7360bd8..d9f9d14bc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,26 +4,20 @@ version = "0.0.1" authors = ["Paul Hauner "] [dependencies] -# TODO: remove "blake2" in favor of "blake2-rfc" -blake2 = "^0.7.1" blake2-rfc = "0.2.18" bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } -boolean-bitfield = { path = "boolean-bitfield" } bytes = "" crypto-mac = "^0.6.2" clap = "2.32.0" +db = { path = "lighthouse/db" } dirs = "1.0.3" -ethereum-types = "0.4.0" futures = "0.1.23" network-libp2p = { path = "network-libp2p" } rand = "0.3" -rayon = "1.0.2" -rocksdb = "0.10.1" rlp = { git = "https://github.com/paritytech/parity-common" } slog = "^2.2.3" slog-term = "^2.4.0" slog-async = "^2.3.0" -ssz = { path = "ssz" } tokio = "0.1" [dependencies.pairing] @@ -37,5 +31,13 @@ ring = { git = "https://github.com/paritytech/ring" } path = "lighthouse/main.rs" name = "lighthouse" -[lib] -path = "lighthouse/lib.rs" +[workspace] +members = [ + "beacon_chain/types", + "beacon_chain/utils/bls", + "beacon_chain/utils/boolean-bitfield", + "beacon_chain/utils/hashing", + "beacon_chain/utils/ssz", + "beacon_chain/utils/ssz_helpers", + "lighthouse/db", +] diff --git a/beacon_chain/types/Cargo.toml b/beacon_chain/types/Cargo.toml new file mode 100644 index 0000000000..cf20f69c00 --- /dev/null +++ b/beacon_chain/types/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "types" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../utils/bls" } +boolean-bitfield = { path = "../utils/boolean-bitfield" } +ethereum-types = "0.4.0" +rand = "0.3" +ssz = { path = "../utils/ssz" } diff --git a/lighthouse/state/active_state.rs b/beacon_chain/types/src/active_state.rs similarity index 94% rename from lighthouse/state/active_state.rs rename to beacon_chain/types/src/active_state.rs index d107bc2992..6c71cadb88 100644 --- a/lighthouse/state/active_state.rs +++ b/beacon_chain/types/src/active_state.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; use super::attestation_record::AttestationRecord; pub struct ActiveState { diff --git a/lighthouse/state/attestation_record/structs.rs b/beacon_chain/types/src/attestation_record.rs similarity index 98% rename from lighthouse/state/attestation_record/structs.rs rename to beacon_chain/types/src/attestation_record.rs index 9cfa535164..77631372b5 100644 --- a/lighthouse/state/attestation_record/structs.rs +++ b/beacon_chain/types/src/attestation_record.rs @@ -1,4 +1,4 @@ -use super::utils::types::{ Hash256, Bitfield }; +use super::{ Hash256, Bitfield }; use super::bls::{ AggregateSignature, BLS_AGG_SIG_BYTE_SIZE, diff --git a/lighthouse/state/block/structs.rs b/beacon_chain/types/src/block.rs similarity index 98% rename from lighthouse/state/block/structs.rs rename to beacon_chain/types/src/block.rs index eb8833a408..ee84773d01 100644 --- a/lighthouse/state/block/structs.rs +++ b/beacon_chain/types/src/block.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; use super::attestation_record::AttestationRecord; use super::ssz::{ Encodable, SszStream }; diff --git a/lighthouse/state/chain_config.rs b/beacon_chain/types/src/chain_config.rs similarity index 100% rename from lighthouse/state/chain_config.rs rename to beacon_chain/types/src/chain_config.rs diff --git a/lighthouse/state/common/delegation/block_hash.rs b/beacon_chain/types/src/common/delegation/block_hash.rs similarity index 100% rename from lighthouse/state/common/delegation/block_hash.rs rename to beacon_chain/types/src/common/delegation/block_hash.rs diff --git a/lighthouse/state/common/delegation/mod.rs b/beacon_chain/types/src/common/delegation/mod.rs similarity index 100% rename from lighthouse/state/common/delegation/mod.rs rename to beacon_chain/types/src/common/delegation/mod.rs diff --git a/lighthouse/state/common/maps.rs b/beacon_chain/types/src/common/maps.rs similarity index 100% rename from lighthouse/state/common/maps.rs rename to beacon_chain/types/src/common/maps.rs diff --git a/lighthouse/state/common/mod.rs b/beacon_chain/types/src/common/mod.rs similarity index 100% rename from lighthouse/state/common/mod.rs rename to beacon_chain/types/src/common/mod.rs diff --git a/lighthouse/state/common/shuffling/README.md b/beacon_chain/types/src/common/shuffling/README.md similarity index 100% rename from lighthouse/state/common/shuffling/README.md rename to beacon_chain/types/src/common/shuffling/README.md diff --git a/lighthouse/state/common/shuffling/mod.rs b/beacon_chain/types/src/common/shuffling/mod.rs similarity index 100% rename from lighthouse/state/common/shuffling/mod.rs rename to beacon_chain/types/src/common/shuffling/mod.rs diff --git a/lighthouse/state/common/shuffling/rng.rs b/beacon_chain/types/src/common/shuffling/rng.rs similarity index 100% rename from lighthouse/state/common/shuffling/rng.rs rename to beacon_chain/types/src/common/shuffling/rng.rs diff --git a/lighthouse/state/crosslink_record.rs b/beacon_chain/types/src/crosslink_record.rs similarity index 93% rename from lighthouse/state/crosslink_record.rs rename to beacon_chain/types/src/crosslink_record.rs index 37dfe75004..a28284681b 100644 --- a/lighthouse/state/crosslink_record.rs +++ b/beacon_chain/types/src/crosslink_record.rs @@ -1,4 +1,4 @@ -use super::utils::types::Hash256; +use super::Hash256; #[derive(Clone)] pub struct CrosslinkRecord { diff --git a/lighthouse/state/crystallized_state.rs b/beacon_chain/types/src/crystallized_state.rs similarity index 98% rename from lighthouse/state/crystallized_state.rs rename to beacon_chain/types/src/crystallized_state.rs index 8769bdcaae..e6d901495f 100644 --- a/lighthouse/state/crystallized_state.rs +++ b/beacon_chain/types/src/crystallized_state.rs @@ -2,7 +2,7 @@ use super::validator_record::ValidatorRecord; use super::crosslink_record::CrosslinkRecord; use super::shard_and_committee::ShardAndCommittee; use super::ethereum_types::U256; -use super::utils::types::{ Hash256 }; +use super::Hash256; pub struct CrystallizedState { diff --git a/beacon_chain/types/src/lib.rs b/beacon_chain/types/src/lib.rs new file mode 100644 index 0000000000..683560ab0c --- /dev/null +++ b/beacon_chain/types/src/lib.rs @@ -0,0 +1,49 @@ +extern crate ethereum_types; +extern crate bls; +extern crate boolean_bitfield; +extern crate ssz; + +pub mod active_state; +pub mod attestation_record; +pub mod crystallized_state; +pub mod chain_config; +pub mod block; +pub mod crosslink_record; +pub mod shard_and_committee; +pub mod validator_record; + +use self::ethereum_types::{ + H256, + H160, + U256 +}; +use self::boolean_bitfield::BooleanBitfield; +use std::collections::HashMap; + +pub use active_state::ActiveState; +pub use attestation_record::AttestationRecord; +pub use crystallized_state::CrystallizedState; +pub use chain_config::ChainConfig; +pub use block::Block; +pub use crosslink_record::CrosslinkRecord; +pub use shard_and_committee::ShardAndCommittee; +pub use validator_record::ValidatorRecord; + +pub type Hash256 = H256; +pub type Address = H160; +pub type EthBalance = U256; +pub type Bitfield = BooleanBitfield; + +/// Maps a (slot, shard_id) to attestation_indices. +pub type AttesterMap = HashMap<(u64, u16), Vec>; + +/// Maps a slot to a block proposer. +pub type ProposerMap = HashMap; + +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/lighthouse/state/mod.rs b/beacon_chain/types/src/mod.rs similarity index 100% rename from lighthouse/state/mod.rs rename to beacon_chain/types/src/mod.rs diff --git a/lighthouse/state/shard_and_committee.rs b/beacon_chain/types/src/shard_and_committee.rs similarity index 99% rename from lighthouse/state/shard_and_committee.rs rename to beacon_chain/types/src/shard_and_committee.rs index 7f57fa6c83..3b2bcd0fd3 100644 --- a/lighthouse/state/shard_and_committee.rs +++ b/beacon_chain/types/src/shard_and_committee.rs @@ -1,4 +1,3 @@ - #[derive(Clone,Debug)] pub struct ShardAndCommittee { pub shard_id: u16, diff --git a/lighthouse/state/validator_record.rs b/beacon_chain/types/src/validator_record.rs similarity index 89% rename from lighthouse/state/validator_record.rs rename to beacon_chain/types/src/validator_record.rs index 323c9e88d4..4949e13f43 100644 --- a/lighthouse/state/validator_record.rs +++ b/beacon_chain/types/src/validator_record.rs @@ -1,14 +1,19 @@ -extern crate rand; - -use super::utils::types::{ Hash256, Address, U256 }; -use super::bls::{ PublicKey, Keypair }; +use super::{ + Hash256, + Address, + EthBalance, +}; +use super::bls::{ + PublicKey, + Keypair +}; pub struct ValidatorRecord { pub pubkey: PublicKey, pub withdrawal_shard: u16, pub withdrawal_address: Address, pub randao_commitment: Hash256, - pub balance: U256, + pub balance: EthBalance, pub start_dynasty: u64, pub end_dynasty: u64, } @@ -25,7 +30,7 @@ impl ValidatorRecord { withdrawal_shard: 0, withdrawal_address: Address::zero(), randao_commitment: Hash256::zero(), - balance: U256::zero(), + balance: EthBalance::zero(), start_dynasty: 0, end_dynasty: 0, }; diff --git a/beacon_chain/utils/bls/Cargo.toml b/beacon_chain/utils/bls/Cargo.toml new file mode 100644 index 0000000000..eef0ec0aac --- /dev/null +++ b/beacon_chain/utils/bls/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "bls" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls-aggregates = { git = "https://github.com/sigp/signature-schemes" } diff --git a/lighthouse/bls/mod.rs b/beacon_chain/utils/bls/src/lib.rs similarity index 100% rename from lighthouse/bls/mod.rs rename to beacon_chain/utils/bls/src/lib.rs diff --git a/boolean-bitfield/Cargo.toml b/beacon_chain/utils/boolean-bitfield/Cargo.toml similarity index 100% rename from boolean-bitfield/Cargo.toml rename to beacon_chain/utils/boolean-bitfield/Cargo.toml diff --git a/boolean-bitfield/README.md b/beacon_chain/utils/boolean-bitfield/README.md similarity index 100% rename from boolean-bitfield/README.md rename to beacon_chain/utils/boolean-bitfield/README.md diff --git a/boolean-bitfield/src/lib.rs b/beacon_chain/utils/boolean-bitfield/src/lib.rs similarity index 99% rename from boolean-bitfield/src/lib.rs rename to beacon_chain/utils/boolean-bitfield/src/lib.rs index 60d877e652..c9a4382f4b 100644 --- a/boolean-bitfield/src/lib.rs +++ b/beacon_chain/utils/boolean-bitfield/src/lib.rs @@ -218,6 +218,7 @@ mod tests { assert_eq!(stream.drain(), vec![0, 0, 0, 2, 0, 1]); } + /* #[test] fn test_ssz_decoding() { /* @@ -246,6 +247,7 @@ mod tests { let res = BooleanBitfield::ssz_decode(&input, 0); assert_eq!(res, Err(ssz::DecodeError::TooShort)); } + */ #[test] fn test_new_bitfield_len() { diff --git a/beacon_chain/utils/hashing/Cargo.toml b/beacon_chain/utils/hashing/Cargo.toml new file mode 100644 index 0000000000..36cbc41ef1 --- /dev/null +++ b/beacon_chain/utils/hashing/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "hashing" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +blake2-rfc = "0.2.18" diff --git a/lighthouse/utils/hash.rs b/beacon_chain/utils/hashing/src/lib.rs similarity index 66% rename from lighthouse/utils/hash.rs rename to beacon_chain/utils/hashing/src/lib.rs index 7b52eb9d13..8e2bd37a9e 100644 --- a/lighthouse/utils/hash.rs +++ b/beacon_chain/utils/hashing/src/lib.rs @@ -1,4 +1,6 @@ -use super::blake2::blake2b::blake2b; +extern crate blake2_rfc; + +use self::blake2_rfc::blake2b::blake2b; pub fn canonical_hash(input: &[u8]) -> Vec { let result = blake2b(64, &[], input); diff --git a/ssz/Cargo.toml b/beacon_chain/utils/ssz/Cargo.toml similarity index 100% rename from ssz/Cargo.toml rename to beacon_chain/utils/ssz/Cargo.toml diff --git a/ssz/README.md b/beacon_chain/utils/ssz/README.md similarity index 100% rename from ssz/README.md rename to beacon_chain/utils/ssz/README.md diff --git a/ssz/src/decode.rs b/beacon_chain/utils/ssz/src/decode.rs similarity index 100% rename from ssz/src/decode.rs rename to beacon_chain/utils/ssz/src/decode.rs diff --git a/ssz/src/encode.rs b/beacon_chain/utils/ssz/src/encode.rs similarity index 100% rename from ssz/src/encode.rs rename to beacon_chain/utils/ssz/src/encode.rs diff --git a/ssz/src/impl_decode.rs b/beacon_chain/utils/ssz/src/impl_decode.rs similarity index 99% rename from ssz/src/impl_decode.rs rename to beacon_chain/utils/ssz/src/impl_decode.rs index 12e3e04c9c..2833037f31 100644 --- a/ssz/src/impl_decode.rs +++ b/beacon_chain/utils/ssz/src/impl_decode.rs @@ -214,7 +214,7 @@ mod tests { &vec![1], 2 ); - assert_eq!(err, Err(DecodeError::OutOfBounds)); + assert_eq!(err, Err(DecodeError::TooShort)); let err: Result<(u16,usize), DecodeError> = decode_ssz( &vec![0, 0, 0, 0], diff --git a/ssz/src/impl_encode.rs b/beacon_chain/utils/ssz/src/impl_encode.rs similarity index 100% rename from ssz/src/impl_encode.rs rename to beacon_chain/utils/ssz/src/impl_encode.rs diff --git a/ssz/src/lib.rs b/beacon_chain/utils/ssz/src/lib.rs similarity index 100% rename from ssz/src/lib.rs rename to beacon_chain/utils/ssz/src/lib.rs diff --git a/beacon_chain/utils/ssz_helpers/Cargo.toml b/beacon_chain/utils/ssz_helpers/Cargo.toml new file mode 100644 index 0000000000..a8189ce60e --- /dev/null +++ b/beacon_chain/utils/ssz_helpers/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "ssz_helpers" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../bls" } +hashing = { path = "../hashing" } +types = { path = "../../types" } +ssz = { path = "../ssz" } diff --git a/lighthouse/state/attestation_record/ssz_splitter.rs b/beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs similarity index 96% rename from lighthouse/state/attestation_record/ssz_splitter.rs rename to beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs index e9e1b5ff81..ba11e1815d 100644 --- a/lighthouse/state/attestation_record/ssz_splitter.rs +++ b/beacon_chain/utils/ssz_helpers/src/attestation_ssz_splitter.rs @@ -1,4 +1,4 @@ -use super::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH; +use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH as MIN_LENGTH; use super::ssz::LENGTH_BYTES; use super::ssz::decode::decode_length; @@ -54,8 +54,8 @@ pub fn split_one_attestation(full_ssz: &[u8], index: usize) #[cfg(test)] mod tests { use super::*; - use super::super::AttestationRecord; - use super::super::utils::types::{ + use super::super::types::{ + AttestationRecord, Hash256, Bitfield, }; diff --git a/beacon_chain/utils/ssz_helpers/src/lib.rs b/beacon_chain/utils/ssz_helpers/src/lib.rs new file mode 100644 index 0000000000..6f313e8a01 --- /dev/null +++ b/beacon_chain/utils/ssz_helpers/src/lib.rs @@ -0,0 +1,7 @@ +extern crate bls; +extern crate hashing; +extern crate types; +extern crate ssz; + +pub mod attestation_ssz_splitter; +pub mod ssz_block; diff --git a/lighthouse/state/block/ssz_block.rs b/beacon_chain/utils/ssz_helpers/src/ssz_block.rs similarity index 97% rename from lighthouse/state/block/ssz_block.rs rename to beacon_chain/utils/ssz_helpers/src/ssz_block.rs index e92291f800..7dd2ddc6d7 100644 --- a/lighthouse/state/block/ssz_block.rs +++ b/beacon_chain/utils/ssz_helpers/src/ssz_block.rs @@ -2,12 +2,12 @@ use super::ssz::decode::{ decode_length, Decodable, }; -use super::utils::hash::canonical_hash; -use super::structs::{ +use super::hashing::canonical_hash; +use super::types::block::{ MIN_SSZ_BLOCK_LENGTH, MAX_SSZ_BLOCK_LENGTH, }; -use super::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; +use super::types::attestation_record::MIN_SSZ_ATTESTION_RECORD_LENGTH; #[derive(Debug, PartialEq)] pub enum SszBlockError { @@ -146,10 +146,12 @@ impl<'a> SszBlock<'a> { #[cfg(test)] mod tests { use super::*; - use super::super::structs::Block; - use super::super::attestation_record::AttestationRecord; + use super::super::types::{ + AttestationRecord, + Block, + }; use super::super::ssz::SszStream; - use super::super::utils::types::Hash256; + use super::super::types::Hash256; fn get_block_ssz(b: &Block) -> Vec { let mut ssz_stream = SszStream::new(); diff --git a/benches/block_validation.rs b/benches/block_validation.rs deleted file mode 100644 index 03e9df4984..0000000000 --- a/benches/block_validation.rs +++ /dev/null @@ -1,148 +0,0 @@ -#![feature(test)] - -extern crate test; - -use self::test::Bencher; - -use std::sync::Arc; - -use super::{ - BlockValidationContext, - AttesterMap, - ProposerMap, -}; - -use super::tests::{ - TestStore, - TestParams, - setup_block_validation_scenario, - serialize_block, -}; - -use super::super::{ - Block, - SszBlock, -}; - -fn bench_block_validation_scenario( - b: &mut Bencher, - params: &TestParams, - mutator_func: F) - where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) - -> (Block, AttesterMap, ProposerMap, TestStore) -{ - let (block, - parent_hashes, - attester_map, - proposer_map, - stores) = setup_block_validation_scenario(¶ms); - - let (block, - attester_map, - proposer_map, - stores) = mutator_func(block, attester_map, proposer_map, stores); - - let ssz_bytes = serialize_block(&block); - let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) - .unwrap(); - - let parent_hashes = Arc::new(parent_hashes); - let proposer_map = Arc::new(proposer_map); - let attester_map = Arc::new(attester_map); - b.iter(|| { - let context = BlockValidationContext { - present_slot: params.validation_context_slot, - cycle_length: params.cycle_length, - last_justified_slot: params.validation_context_justified_slot, - last_finalized_slot: params.validation_context_finalized_slot, - parent_hashes: parent_hashes.clone(), - proposer_map: proposer_map.clone(), - attester_map: attester_map.clone(), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - pow_store: stores.pow_chain.clone() - }; - let result = context.validate_ssz_block(&ssz_block); - assert!(result.is_ok()); - }); -} - -#[bench] -#[ignore] -fn bench_block_validation_10m_eth(b: &mut Bencher) { - let total_validators: usize = 10_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} - -#[bench] -#[ignore] -fn bench_block_validation_100m_eth(b: &mut Bencher) { - let total_validators: usize = 100_000_000 / 32; - let cycle_length: u8 = 64; - let shard_count: u16 = 1024; - let shards_per_slot: u16 = 1024 / u16::from(cycle_length); - let validators_per_shard: usize = total_validators / usize::from(shard_count); - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - let params = TestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - }; - - let no_mutate = |block, attester_map, proposer_map, stores| { - (block, attester_map, proposer_map, stores) - }; - - bench_block_validation_scenario( - b, - ¶ms, - no_mutate); -} diff --git a/benches/main.rs b/benches/main.rs deleted file mode 100644 index 0faf89f6d8..0000000000 --- a/benches/main.rs +++ /dev/null @@ -1,5 +0,0 @@ -#![feature(test)] -extern crate lighthouse; -extern crate tests; - -mod block_validation; diff --git a/lighthouse/db/Cargo.toml b/lighthouse/db/Cargo.toml new file mode 100644 index 0000000000..898df0c9b6 --- /dev/null +++ b/lighthouse/db/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "db" +version = "0.1.0" +authors = ["Paul Hauner "] + +[dependencies] +bls = { path = "../../beacon_chain/utils/bls" } +bytes = "0.4.10" +rocksdb = "0.10.1" +blake2-rfc = "0.2.18" diff --git a/lighthouse/db/disk_db.rs b/lighthouse/db/src/disk_db.rs similarity index 100% rename from lighthouse/db/disk_db.rs rename to lighthouse/db/src/disk_db.rs diff --git a/lighthouse/db/mod.rs b/lighthouse/db/src/lib.rs similarity index 93% rename from lighthouse/db/mod.rs rename to lighthouse/db/src/lib.rs index 40cab486d3..be3d986b9b 100644 --- a/lighthouse/db/mod.rs +++ b/lighthouse/db/src/lib.rs @@ -1,12 +1,12 @@ -extern crate rocksdb; extern crate blake2_rfc as blake2; +extern crate bls; +extern crate rocksdb; mod disk_db; mod memory_db; mod traits; pub mod stores; -use super::bls; use self::stores::COLUMNS; pub use self::disk_db::DiskDB; diff --git a/lighthouse/db/memory_db.rs b/lighthouse/db/src/memory_db.rs similarity index 100% rename from lighthouse/db/memory_db.rs rename to lighthouse/db/src/memory_db.rs diff --git a/lighthouse/db/stores/block_store.rs b/lighthouse/db/src/stores/block_store.rs similarity index 100% rename from lighthouse/db/stores/block_store.rs rename to lighthouse/db/src/stores/block_store.rs diff --git a/lighthouse/db/stores/mod.rs b/lighthouse/db/src/stores/mod.rs similarity index 100% rename from lighthouse/db/stores/mod.rs rename to lighthouse/db/src/stores/mod.rs diff --git a/lighthouse/db/stores/pow_chain_store.rs b/lighthouse/db/src/stores/pow_chain_store.rs similarity index 100% rename from lighthouse/db/stores/pow_chain_store.rs rename to lighthouse/db/src/stores/pow_chain_store.rs diff --git a/lighthouse/db/stores/validator_store.rs b/lighthouse/db/src/stores/validator_store.rs similarity index 100% rename from lighthouse/db/stores/validator_store.rs rename to lighthouse/db/src/stores/validator_store.rs diff --git a/lighthouse/db/traits.rs b/lighthouse/db/src/traits.rs similarity index 100% rename from lighthouse/db/traits.rs rename to lighthouse/db/src/traits.rs diff --git a/lighthouse/lib.rs b/lighthouse/lib.rs deleted file mode 100644 index 9abf4184ec..0000000000 --- a/lighthouse/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[macro_use] -extern crate slog; -extern crate slog_term; -extern crate slog_async; -extern crate ssz; -extern crate clap; -extern crate network_libp2p; -extern crate futures; - -#[macro_use] -#[allow(dead_code)] -pub mod utils; -#[allow(dead_code)] -pub mod bls; -#[allow(dead_code)] -pub mod db; -pub mod client; -#[allow(dead_code)] -pub mod state; -#[allow(dead_code)] -mod sync; -mod config; diff --git a/lighthouse/main.rs b/lighthouse/main.rs index c90bcdaea5..ee90649f59 100644 --- a/lighthouse/main.rs +++ b/lighthouse/main.rs @@ -2,22 +2,14 @@ extern crate slog; extern crate slog_term; extern crate slog_async; -extern crate ssz; +// extern crate ssz; extern crate clap; extern crate network_libp2p; extern crate futures; -#[macro_use] -#[allow(dead_code)] -mod utils; -#[allow(dead_code)] -mod bls; -#[allow(dead_code)] -mod db; +extern crate db; + mod client; -#[allow(dead_code)] -mod state; -#[allow(dead_code)] mod sync; mod config; diff --git a/lighthouse/state/attestation_record/mod.rs b/lighthouse/state/attestation_record/mod.rs deleted file mode 100644 index d1813b2a54..0000000000 --- a/lighthouse/state/attestation_record/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -use super::bls; -use super::common; -use super::db; -use super::ssz; -use super::utils; - - -mod structs; -mod ssz_splitter; -mod validation; - -pub use self::structs::{ - AttestationRecord, - MIN_SSZ_ATTESTION_RECORD_LENGTH, -}; -pub use self::ssz_splitter::{ - split_all_attestations, - split_one_attestation, - AttestationSplitError, -}; -pub use self::validation::{ - AttestationValidationContext, - AttestationValidationError, -}; diff --git a/lighthouse/state/attestation_record/validation/attestation_validation.rs b/lighthouse/state/attestation_record/validation/attestation_validation.rs deleted file mode 100644 index c96dfb4799..0000000000 --- a/lighthouse/state/attestation_record/validation/attestation_validation.rs +++ /dev/null @@ -1,208 +0,0 @@ -use std::collections::HashSet; -use std::sync::Arc; -use super::structs::AttestationRecord; -use super::AttesterMap; -use super::attestation_parent_hashes::{ - attestation_parent_hashes, - ParentHashesError, -}; -use super::db::{ - ClientDB, - DBError -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, -}; -use super::utils::types::{ - Hash256, -}; -use super::message_generation::generate_signed_message; -use super::signature_verification::{ - verify_aggregate_signature_for_indices, - SignatureVerificationError, -}; - -#[derive(Debug,PartialEq)] -pub enum AttestationValidationError { - SlotTooHigh, - SlotTooLow, - JustifiedSlotIncorrect, - UnknownJustifiedBlock, - TooManyObliqueHashes, - BadCurrentHashes, - BadObliqueHashes, - BadAttesterMap, - IntWrapping, - PublicKeyCorrupt, - NoPublicKeyForValidator, - BadBitfieldLength, - InvalidBitfield, - InvalidBitfieldEndBits, - NoSignatures, - NonZeroTrailingBits, - BadAggregateSignature, - DBError(String), -} - -pub struct AttestationValidationContext - where T: ClientDB + Sized -{ - pub block_slot: u64, - pub cycle_length: u8, - pub last_justified_slot: u64, - pub parent_hashes: Arc>, - pub block_store: Arc>, - pub validator_store: Arc>, - pub attester_map: Arc, -} - -impl AttestationValidationContext - where T: ClientDB -{ - pub fn validate_attestation(&self, a: &AttestationRecord) - -> Result, AttestationValidationError> - { - /* - * The attesation slot must not be higher than the block that contained it. - */ - if a.slot > self.block_slot { - return Err(AttestationValidationError::SlotTooHigh); - } - - /* - * The slot of this attestation must not be more than cycle_length + 1 distance - * from the block that contained it. - */ - if a.slot < self.block_slot - .saturating_sub(u64::from(self.cycle_length).saturating_add(1)) { - return Err(AttestationValidationError::SlotTooLow); - } - - /* - * The attestation must indicate that its last justified slot is the same as the last justified - * slot known to us. - */ - if a.justified_slot != self.last_justified_slot { - return Err(AttestationValidationError::JustifiedSlotIncorrect); - } - - /* - * There is no need to include more oblique parents hashes than there are blocks - * in a cycle. - */ - if a.oblique_parent_hashes.len() > usize::from(self.cycle_length) { - return Err(AttestationValidationError::TooManyObliqueHashes); - } - - /* - * Retrieve the set of attestation indices for this slot and shard id. - * - * This is an array mapping the order that validators will appear in the bitfield to the - * canonincal index of a validator. - */ - let attestation_indices = self.attester_map.get(&(a.slot, a.shard_id)) - .ok_or(AttestationValidationError::BadAttesterMap)?; - - /* - * The bitfield must be no longer than the minimum required to represent each validator in the - * attestation indicies for this slot and shard id. - */ - if a.attester_bitfield.num_bytes() != - bytes_for_bits(attestation_indices.len()) - { - return Err(AttestationValidationError::BadBitfieldLength); - } - - /* - * If there are excess bits in the bitfield because the number of a validators in not a - * multiple of 8, reject this attestation record. - * - * Allow extra set bits would permit mutliple different byte layouts (and therefore hashes) to - * refer to the same AttesationRecord. - */ - if a.attester_bitfield.len() > attestation_indices.len() { - return Err(AttestationValidationError::InvalidBitfieldEndBits) - } - - /* - * The specified justified block hash must be known to us - */ - if !self.block_store.block_exists(&a.justified_block_hash)? { - return Err(AttestationValidationError::UnknownJustifiedBlock) - } - - let signed_message = { - let parent_hashes = attestation_parent_hashes( - self.cycle_length, - self.block_slot, - a.slot, - &self.parent_hashes, - &a.oblique_parent_hashes)?; - generate_signed_message( - a.slot, - &parent_hashes, - a.shard_id, - &a.shard_block_hash, - a.justified_slot) - }; - - let voted_hashmap = - verify_aggregate_signature_for_indices( - &signed_message, - &a.aggregate_sig, - &attestation_indices, - &a.attester_bitfield, - &self.validator_store)?; - - /* - * If the hashmap of voters is None, the signature verification failed. - */ - match voted_hashmap { - None => Err(AttestationValidationError::BadAggregateSignature), - Some(hashmap) => Ok(hashmap), - } - } -} - -fn bytes_for_bits(bits: usize) -> usize { - (bits.saturating_sub(1) / 8) + 1 -} - -impl From for AttestationValidationError { - fn from(e: ParentHashesError) -> Self { - match e { - ParentHashesError::BadCurrentHashes - => AttestationValidationError::BadCurrentHashes, - ParentHashesError::BadObliqueHashes - => AttestationValidationError::BadObliqueHashes, - ParentHashesError::SlotTooLow - => AttestationValidationError::SlotTooLow, - ParentHashesError::SlotTooHigh - => AttestationValidationError::SlotTooHigh, - ParentHashesError::IntWrapping - => AttestationValidationError::IntWrapping - } - } -} - -impl From for AttestationValidationError { - fn from(e: DBError) -> Self { - AttestationValidationError::DBError(e.message) - } -} - -impl From for AttestationValidationError { - fn from(e: SignatureVerificationError) -> Self { - match e { - SignatureVerificationError::BadValidatorIndex - => AttestationValidationError::BadAttesterMap, - SignatureVerificationError::PublicKeyCorrupt - => AttestationValidationError::PublicKeyCorrupt, - SignatureVerificationError::NoPublicKeyForValidator - => AttestationValidationError::NoPublicKeyForValidator, - SignatureVerificationError::DBError(s) - => AttestationValidationError::DBError(s), - } - } -} diff --git a/lighthouse/state/attestation_record/validation/message_generation.rs b/lighthouse/state/attestation_record/validation/message_generation.rs deleted file mode 100644 index 6299802e93..0000000000 --- a/lighthouse/state/attestation_record/validation/message_generation.rs +++ /dev/null @@ -1,70 +0,0 @@ -use super::ssz::SszStream; -use super::utils::hash::canonical_hash; -use super::utils::types::Hash256; - -/// Generates the message used to validate the signature provided with an AttestationRecord. -/// -/// Ensures that the signer of the message has a view of the chain that is compatible with ours. -pub fn generate_signed_message( - slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - /* - * Note: it's a little risky here to use SSZ, because the encoding is not necessarily SSZ - * (for example, SSZ might change whilst this doesn't). - * - * I have suggested switching this to ssz here: - * https://github.com/ethereum/eth2.0-specs/issues/5 - * - * If this doesn't happen, it would be safer to not use SSZ at all. - */ - let mut ssz_stream = SszStream::new(); - ssz_stream.append(&slot); - ssz_stream.append_vec(&parent_hashes.to_vec()); - ssz_stream.append(&shard_id); - ssz_stream.append(shard_block_hash); - ssz_stream.append(&justified_slot); - let bytes = ssz_stream.drain(); - canonical_hash(&bytes) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_generate_signed_message() { - let slot = 93; - let parent_hashes: Vec = (0..12) - .map(|i| Hash256::from(i as u64)) - .collect(); - let shard_id = 15; - let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); - let justified_slot = 18; - - let output = generate_signed_message( - slot, - &parent_hashes, - shard_id, - &shard_block_hash, - justified_slot); - - /* - * Note: this is not some well-known test vector, it's simply the result of running - * this and printing the output. - * - * Once well-known test vectors are established, they should be placed here. - */ - let expected = vec![ - 149, 99, 94, 229, 72, 144, 233, 14, 164, 16, 143, 53, 94, 48, - 118, 179, 33, 181, 172, 215, 2, 191, 176, 18, 188, 172, 137, - 178, 236, 66, 74, 120 - ]; - - assert_eq!(output, expected); - } -} diff --git a/lighthouse/state/attestation_record/validation/mod.rs b/lighthouse/state/attestation_record/validation/mod.rs deleted file mode 100644 index dd3c589c5c..0000000000 --- a/lighthouse/state/attestation_record/validation/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -use super::common::maps::AttesterMap; -use super::db; -use super::bls; -use super::structs; -use super::ssz; -use super::common::attestation_parent_hashes; -use super::utils; - -mod attestation_validation; -mod signature_verification; -mod message_generation; - -pub use self::attestation_validation::{ - AttestationValidationContext, - AttestationValidationError, -}; diff --git a/lighthouse/state/attestation_record/validation/signature_verification.rs b/lighthouse/state/attestation_record/validation/signature_verification.rs deleted file mode 100644 index c33dc28063..0000000000 --- a/lighthouse/state/attestation_record/validation/signature_verification.rs +++ /dev/null @@ -1,183 +0,0 @@ -use std::collections::HashSet; -use super::bls::{ - AggregateSignature, - AggregatePublicKey, -}; -use super::db::ClientDB; -use super::db::stores::{ - ValidatorStore, - ValidatorStoreError, -}; -use super::utils::types::Bitfield; - -#[derive(Debug, PartialEq)] -pub enum SignatureVerificationError { - BadValidatorIndex, - PublicKeyCorrupt, - NoPublicKeyForValidator, - DBError(String), -} - -/// Verify an aggregate signature across the supplied message. -/// -/// The public keys used for verification are collected by mapping -/// each true bitfield bit to canonical ValidatorRecord index through -/// the attestation_indicies map. -/// -/// Each public key is loaded from the store on-demand. -pub fn verify_aggregate_signature_for_indices( - message: &[u8], - agg_sig: &AggregateSignature, - attestation_indices: &[usize], - bitfield: &Bitfield, - validator_store: &ValidatorStore) - -> Result>, SignatureVerificationError> - where T: ClientDB + Sized -{ - let mut voters = HashSet::new(); - let mut agg_pub_key = AggregatePublicKey::new(); - - for i in 0..attestation_indices.len() { - let voted = bitfield.get_bit(i); - if voted { - /* - * De-reference the attestation index into a canonical ValidatorRecord index. - */ - let validator = *attestation_indices.get(i) - .ok_or(SignatureVerificationError::BadValidatorIndex)?; - /* - * Load the validators public key from our store. - */ - let pub_key = validator_store - .get_public_key_by_index(validator)? - .ok_or(SignatureVerificationError::NoPublicKeyForValidator)?; - /* - * Add the validators public key to the aggregate public key. - */ - agg_pub_key.add(&pub_key); - /* - * Add to the validator to the set of voters for this attestation record. - */ - voters.insert(validator); - } - } - /* - * Verify the aggregate public key against the aggregate signature. - * - * This verification will only succeed if the exact set of public keys - * were added to the aggregate public key as those that signed the aggregate signature. - */ - if agg_sig.verify(&message, &agg_pub_key) { - Ok(Some(voters)) - } else { - Ok(None) - } -} - -impl From for SignatureVerificationError { - fn from(error: ValidatorStoreError) -> Self { - match error { - ValidatorStoreError::DBError(s) => - SignatureVerificationError::DBError(s), - ValidatorStoreError::DecodeError => - SignatureVerificationError::PublicKeyCorrupt, - } - } -} - - -#[cfg(test)] -mod tests { - use super::*; - use super::super::bls::{ - Keypair, - Signature, - }; - use super::super::db::MemoryDB; - use std::sync::Arc; - - /* - * Cases that still need testing: - * - * - No signatures. - * - Database failure. - * - Unknown validator index. - * - Extra validator on signature. - */ - - #[test] - fn test_signature_verification() { - let message = "cats".as_bytes(); - let signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - let non_signing_keypairs = vec![ - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - Keypair::random(), - ]; - /* - * Signing keypairs first, then non-signing - */ - let mut all_keypairs = signing_keypairs.clone(); - all_keypairs.append(&mut non_signing_keypairs.clone()); - - let attestation_indices: Vec = (0..all_keypairs.len()) - .collect(); - let mut bitfield = Bitfield::new(); - for i in 0..signing_keypairs.len() { - bitfield.set_bit(i, true); - } - - let db = Arc::new(MemoryDB::open()); - let store = ValidatorStore::new(db); - - for (i, keypair) in all_keypairs.iter().enumerate() { - store.put_public_key_by_index(i, &keypair.pk).unwrap(); - } - - let mut agg_sig = AggregateSignature::new(); - for keypair in &signing_keypairs { - let sig = Signature::new(&message, &keypair.sk); - agg_sig.add(&sig); - } - - /* - * Test using all valid parameters. - */ - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - let voters = voters.unwrap(); - (0..signing_keypairs.len()) - .for_each(|i| assert!(voters.contains(&i))); - (signing_keypairs.len()..non_signing_keypairs.len()) - .for_each(|i| assert!(!voters.contains(&i))); - - /* - * Add another validator to the bitfield, run validation will all other - * parameters the same and assert that it fails. - */ - bitfield.set_bit(signing_keypairs.len() + 1, true); - let voters = verify_aggregate_signature_for_indices( - &message, - &agg_sig, - &attestation_indices, - &bitfield, - &store).unwrap(); - - assert_eq!(voters, None); - } -} diff --git a/lighthouse/state/block/mod.rs b/lighthouse/state/block/mod.rs deleted file mode 100644 index eb6365bdc9..0000000000 --- a/lighthouse/state/block/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -extern crate blake2_rfc; - -use super::attestation_record; -use super::common; -use super::db; -use super::ssz; -use super::utils; - -mod structs; -mod ssz_block; -pub mod validation; - -pub use self::structs::Block; -pub use self::ssz_block::{ - SszBlock, - SszBlockError, -}; diff --git a/lighthouse/state/block/validation/block_validation.rs b/lighthouse/state/block/validation/block_validation.rs deleted file mode 100644 index 2b6a8906f3..0000000000 --- a/lighthouse/state/block/validation/block_validation.rs +++ /dev/null @@ -1,370 +0,0 @@ -extern crate rayon; - -use self::rayon::prelude::*; - -use std::sync::{ - Arc, - RwLock, -}; -use super::attestation_record::{ - AttestationValidationContext, - AttestationValidationError, -}; -use super::attestation_record::{ - AttestationRecord, - split_one_attestation, - split_all_attestations, - AttestationSplitError, -}; -use super::{ - AttesterMap, - ProposerMap, -}; -use super::{ - SszBlock, - SszBlockError, - Block, -}; -use super::db::{ - ClientDB, - DBError, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::ssz::{ - Decodable, - DecodeError, -}; -use super::utils::types::Hash256; - -#[derive(Debug, PartialEq)] -pub enum BlockStatus { - NewBlock, - KnownBlock, -} - -#[derive(Debug, PartialEq)] -pub enum SszBlockValidationError { - FutureSlot, - SlotAlreadyFinalized, - UnknownPoWChainRef, - UnknownParentHash, - BadAttestationSsz, - AttestationValidationError(AttestationValidationError), - AttestationSignatureFailed, - ProposerAttestationHasObliqueHashes, - NoProposerSignature, - BadProposerMap, - RwLockPoisoned, - DBError(String), -} - -/// The context against which a block should be validated. -pub struct BlockValidationContext - where T: ClientDB + Sized -{ - /// The slot as determined by the system time. - pub present_slot: u64, - /// The cycle_length as determined by the chain configuration. - pub cycle_length: u8, - /// The last justified slot as per the client's view of the canonical chain. - pub last_justified_slot: u64, - /// The last finalized slot as per the client's view of the canonical chain. - pub last_finalized_slot: u64, - /// A vec of the hashes of the blocks preceeding the present slot. - pub parent_hashes: Arc>, - /// A map of slots to a block proposer validation index. - pub proposer_map: Arc, - /// A map of (slot, shard_id) to the attestation set of validation indices. - pub attester_map: Arc, - /// The store containing block information. - pub block_store: Arc>, - /// The store containing validator information. - pub validator_store: Arc>, - /// The store containing information about the proof-of-work chain. - pub pow_store: Arc>, -} - -impl BlockValidationContext - where T: ClientDB -{ - /// Validate some SszBlock against a block validation context. An SszBlock varies from a Block in - /// that is a read-only structure that reads directly from encoded SSZ. - /// - /// The reason to validate an SzzBlock is to avoid decoding it in its entirety if there is - /// a suspicion that the block might be invalid. Such a suspicion should be applied to - /// all blocks coming from the network. - /// - /// This function will determine if the block is new, already known or invalid (either - /// intrinsically or due to some application error.) - /// - /// Note: this function does not implement randao_reveal checking as it is not in the - /// specification. - #[allow(dead_code)] - pub fn validate_ssz_block(&self, b: &SszBlock) - -> Result<(BlockStatus, Option), SszBlockValidationError> - where T: ClientDB + Sized - { - - /* - * If this block is already known, return immediately and indicate the the block is - * known. Don't attempt to deserialize the block. - */ - let block_hash = &b.block_hash(); - if self.block_store.block_exists(&block_hash)? { - return Ok((BlockStatus::KnownBlock, None)); - } - - /* - * If the block slot corresponds to a slot in the future, drop it. - */ - let block_slot = b.slot_number(); - if block_slot > self.present_slot { - return Err(SszBlockValidationError::FutureSlot); - } - - /* - * If the block is unknown (assumed unknown because we checked the db earlier in this - * function) and it comes from a slot that is already finalized, drop the block. - * - * If a slot is finalized, there's no point in considering any other blocks for that slot. - */ - if block_slot <= self.last_finalized_slot { - return Err(SszBlockValidationError::SlotAlreadyFinalized); - } - - /* - * If the PoW chain hash is not known to us, drop it. - * - * We only accept blocks that reference a known PoW hash. - * - * Note: it is not clear what a "known" PoW chain ref is. Likely it means the block hash is - * "sufficienty deep in the canonical PoW chain". This should be clarified as the spec - * crystallizes. - */ - let pow_chain_ref = b.pow_chain_ref(); - if !self.pow_store.block_hash_exists(b.pow_chain_ref())? { - return Err(SszBlockValidationError::UnknownPoWChainRef); - } - - /* - * Store a slice of the serialized attestations from the block SSZ. - */ - let attestations_ssz = &b.attestations(); - - /* - * Get a slice of the first serialized attestation (the 0'th) and decode it into - * a full AttestationRecord object. - * - * The first attestation must be validated separately as it must contain a signature of the - * proposer of the previous block (this is checked later in this function). - */ - let (first_attestation_ssz, next_index) = split_one_attestation( - &attestations_ssz, - 0)?; - let (first_attestation, _) = AttestationRecord::ssz_decode( - &first_attestation_ssz, 0)?; - - /* - * The first attestation may not have oblique hashes. - * - * The presence of oblique hashes in the first attestation would indicate that the proposer - * of the previous block is attesting to some other block than the one they produced. - */ - if first_attestation.oblique_parent_hashes.len() > 0 { - return Err(SszBlockValidationError::ProposerAttestationHasObliqueHashes); - } - - /* - * Generate the context in which attestations will be validated. - */ - let attestation_validation_context = Arc::new(AttestationValidationContext { - block_slot, - cycle_length: self.cycle_length, - last_justified_slot: self.last_justified_slot, - parent_hashes: self.parent_hashes.clone(), - block_store: self.block_store.clone(), - validator_store: self.validator_store.clone(), - attester_map: self.attester_map.clone(), - }); - - /* - * Validate this first attestation. - */ - let attestation_voters = attestation_validation_context - .validate_attestation(&first_attestation)?; - - /* - * Read the parent hash from the block we are validating then attempt to load - * the parent block ssz from the database. If that parent doesn't exist in - * the database, reject the block. - * - * If the parent does exist in the database, read the slot of that parent. Then, - * determine the proposer of that slot (the parent slot) by looking it up - * in the proposer map. - * - * If that proposer (the proposer of the parent block) was not present in the first (0'th) - * attestation of this block, reject the block. - */ - let parent_hash = b.parent_hash(); - match self.block_store.get_serialized_block(&parent_hash)? { - None => return Err(SszBlockValidationError::UnknownParentHash), - Some(ssz) => { - let parent_block = SszBlock::from_slice(&ssz[..])?; - let proposer = self.proposer_map.get(&parent_block.slot_number()) - .ok_or(SszBlockValidationError::BadProposerMap)?; - if !attestation_voters.contains(&proposer) { - return Err(SszBlockValidationError::NoProposerSignature); - } - } - } - - /* - * Split the remaining attestations into a vector of slices, each containing - * a single serialized attestation record. - */ - let other_attestations = split_all_attestations(attestations_ssz, - next_index)?; - - /* - * Verify each other AttestationRecord. - * - * This uses the `rayon` library to do "sometimes" parallelization. Put simply, - * if there are some spare threads, the verification of attestation records will happen - * concurrently. - * - * There is a thread-safe `failure` variable which is set whenever an attestation fails - * validation. This is so all attestation validation is halted if a single bad attestation - * is found. - */ - let failure: RwLock> = RwLock::new(None); - let mut deserialized_attestations: Vec = other_attestations - .par_iter() - .filter_map(|attestation_ssz| { - /* - * If some thread has set the `failure` variable to `Some(error)` the abandon - * attestation serialization and validation. - */ - if let Some(_) = *failure.read().unwrap() { - return None; - } - /* - * If there has not been a failure yet, attempt to serialize and validate the - * attestation. - */ - match AttestationRecord::ssz_decode(&attestation_ssz, 0) { - /* - * Deserialization failed, therefore the block is invalid. - */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None - } - /* - * Deserialization succeeded and the attestation should be validated. - */ - Ok((attestation, _)) => { - match attestation_validation_context.validate_attestation(&attestation) { - /* - * Attestation validation failed with some error. - */ - Err(e) => { - let mut failure = failure.write().unwrap(); - *failure = Some(SszBlockValidationError::from(e)); - None - } - /* - * Attestation validation succeded. - */ - Ok(_) => Some(attestation) - } - } - } - }) - .collect(); - - match failure.into_inner() { - Err(_) => return Err(SszBlockValidationError::RwLockPoisoned), - Ok(failure) => { - match failure { - Some(error) => return Err(error), - _ => () - } - - } - } - - /* - * Add the first attestation to the vec of deserialized attestations at - * index 0. - */ - deserialized_attestations.insert(0, first_attestation); - - /* - * If we have reached this point, the block is a new valid block that is worthy of - * processing. - */ - let block = Block { - parent_hash: Hash256::from(parent_hash), - slot_number: block_slot, - randao_reveal: Hash256::from(b.randao_reveal()), - attestations: deserialized_attestations, - pow_chain_ref: Hash256::from(pow_chain_ref), - active_state_root: Hash256::from(b.act_state_root()), - crystallized_state_root: Hash256::from(b.cry_state_root()), - }; - Ok((BlockStatus::NewBlock, Some(block))) - } -} - -impl From for SszBlockValidationError { - fn from(e: DBError) -> Self { - SszBlockValidationError::DBError(e.message) - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationSplitError) -> Self { - match e { - AttestationSplitError::TooShort => - SszBlockValidationError::BadAttestationSsz - } - } -} - -impl From for SszBlockValidationError { - fn from(e: SszBlockError) -> Self { - match e { - SszBlockError::TooShort => - SszBlockValidationError::DBError("Bad parent block in db.".to_string()), - SszBlockError::TooLong => - SszBlockValidationError::DBError("Bad parent block in db.".to_string()), - } - } -} - -impl From for SszBlockValidationError { - fn from(e: DecodeError) -> Self { - match e { - DecodeError::TooShort => - SszBlockValidationError::BadAttestationSsz, - DecodeError::TooLong => - SszBlockValidationError::BadAttestationSsz, - } - } -} - -impl From for SszBlockValidationError { - fn from(e: AttestationValidationError) -> Self { - SszBlockValidationError::AttestationValidationError(e) - } -} - -/* - * Tests for block validation are contained in the root directory "tests" directory (AKA - * "integration tests directory"). - */ diff --git a/lighthouse/state/block/validation/mod.rs b/lighthouse/state/block/validation/mod.rs deleted file mode 100644 index a18d3934f5..0000000000 --- a/lighthouse/state/block/validation/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -mod block_validation; - -use super::attestation_record; -use super::{ - SszBlock, - SszBlockError, - Block, -}; -use super::db; -use super::ssz; -use super::utils; - -pub use super::common::maps::{ - AttesterMap, - ProposerMap, -}; -pub use self::block_validation::{ - BlockValidationContext, - SszBlockValidationError, - BlockStatus, -}; diff --git a/lighthouse/state/common/attestation_parent_hashes.rs b/lighthouse/state/common/attestation_parent_hashes.rs deleted file mode 100644 index ff54233bc2..0000000000 --- a/lighthouse/state/common/attestation_parent_hashes.rs +++ /dev/null @@ -1,226 +0,0 @@ -use super::Hash256; - -#[derive(Debug)] -pub enum ParentHashesError { - BadCurrentHashes, - BadObliqueHashes, - SlotTooHigh, - SlotTooLow, - IntWrapping, -} - -/// This function is used to select the hashes used in -/// the signing of an AttestationRecord. -/// -/// It either returns Result with a vector of length `cycle_length,` or -/// returns an Error. -/// -/// This function corresponds to the `get_signed_parent_hashes` function -/// in the Python reference implentation. -/// -/// See this slide for more information: -/// https://tinyurl.com/ybzn2spw -pub fn attestation_parent_hashes( - cycle_length: u8, - block_slot: u64, - attestation_slot: u64, - current_hashes: &[Hash256], - oblique_hashes: &[Hash256]) - -> Result, ParentHashesError> -{ - // This cast places a limit on cycle_length. If you change it, check math - // for overflow. - let cycle_length: u64 = u64::from(cycle_length); - - if current_hashes.len() as u64 != (cycle_length * 2) { - return Err(ParentHashesError::BadCurrentHashes); - } - if oblique_hashes.len() as u64 > cycle_length { - return Err(ParentHashesError::BadObliqueHashes); - } - if attestation_slot >= block_slot { - return Err(ParentHashesError::SlotTooHigh); - } - - /* - * Cannot underflow as block_slot cannot be less - * than attestation_slot. - */ - let attestation_distance = block_slot - attestation_slot; - - if attestation_distance > cycle_length { - return Err(ParentHashesError::SlotTooLow); - } - - /* - * Cannot underflow because attestation_distance cannot - * be larger than cycle_length. - */ - let start = cycle_length - attestation_distance; - - /* - * Overflow is potentially impossible, but proof is complicated - * enough to just use checked math. - * - * Arithmetic is: - * start + cycle_length - oblique_hashes.len() - */ - let end = start.checked_add(cycle_length) - .and_then(|x| x.checked_sub(oblique_hashes.len() as u64)) - .ok_or(ParentHashesError::IntWrapping)?; - - - let mut hashes = Vec::new(); - hashes.extend_from_slice( - ¤t_hashes[(start as usize)..(end as usize)]); - hashes.extend_from_slice(oblique_hashes); - - Ok(hashes) -} - - -#[cfg(test)] -mod tests { - use super::*; - - fn get_range_of_hashes(from: usize, to: usize) -> Vec { - (from..to).map(|i| get_hash(&vec![i as u8])).collect() - } - - fn get_hash(value: &[u8]) -> Hash256 { - Hash256::from_slice(value) - } - - #[test] - fn test_get_signed_hashes_oblique_scenario_1() { - /* - * Two oblique hashes. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = get_range_of_hashes(100, 102); - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - - let mut expected_result = get_range_of_hashes(7, 13); - expected_result.append(&mut get_range_of_hashes(100, 102)); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_oblique_scenario_2() { - /* - * All oblique hashes. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = get_range_of_hashes(100, 108); - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - - let expected_result = get_range_of_hashes(100, 108); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_1() { - /* - * Google Slides example. - * https://tinyurl.com/ybzn2spw - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 19; - let attestation_slot: u64 = 15; - let current_hashes = get_range_of_hashes(3, 19); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_ok()); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - let expected_result = get_range_of_hashes(7, 15); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_2() { - /* - * Block 1, attestation 0. - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 1; - let attestation_slot: u64 = 0; - let current_hashes = get_range_of_hashes(0, 16); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - let result = result.unwrap(); - assert_eq!(result.len(), cycle_length as usize); - let expected_result = get_range_of_hashes(7, 15); - assert_eq!(result, expected_result); - } - - #[test] - fn test_get_signed_hashes_scenario_3() { - /* - * attestation_slot too large - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 100; - let attestation_slot: u64 = 100; - let current_hashes = get_range_of_hashes(0, 16); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_err()); - } - - #[test] - fn test_get_signed_hashes_scenario_4() { - /* - * Current hashes too small - */ - let cycle_length: u8 = 8; - let block_slot: u64 = 100; - let attestation_slot: u64 = 99; - let current_hashes = get_range_of_hashes(0, 15); - let oblique_hashes = vec![]; - let result = attestation_parent_hashes( - cycle_length, - block_slot, - attestation_slot, - ¤t_hashes, - &oblique_hashes); - assert!(result.is_err()); - } -} diff --git a/lighthouse/utils/errors.rs b/lighthouse/utils/errors.rs deleted file mode 100644 index b464f6a485..0000000000 --- a/lighthouse/utils/errors.rs +++ /dev/null @@ -1,8 +0,0 @@ -// Collection of custom errors - -#[derive(Debug,PartialEq)] -pub enum ParameterError { - IntWrapping, - OutOfBounds, - InvalidInput(String), -} diff --git a/lighthouse/utils/logging.rs b/lighthouse/utils/logging.rs deleted file mode 100644 index 7dc186d302..0000000000 --- a/lighthouse/utils/logging.rs +++ /dev/null @@ -1,22 +0,0 @@ -extern crate slog; -extern crate slog_term; -extern crate slog_async; - -use slog::*; -pub use slog::Logger; - -pub fn test_logger() -> slog::Logger { - let plain = slog_term::PlainSyncDecorator::new(slog_term::TestStdoutWriter); - Logger::root( - slog_term::FullFormat::new(plain) - .build().fuse(), o!() - ) -} - -pub fn get_logger() -> slog::Logger { - let decorator = slog_term::TermDecorator::new().build(); - let drain = slog_term::CompactFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - - slog::Logger::root(drain, o!()) -} diff --git a/lighthouse/utils/macros.rs b/lighthouse/utils/macros.rs deleted file mode 100644 index e9a8c416d9..0000000000 --- a/lighthouse/utils/macros.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[macro_export] -macro_rules! assert_error { - ($exp: expr, $err: expr) => { - if !$exp { - return Err($err); - } - } -} diff --git a/lighthouse/utils/mod.rs b/lighthouse/utils/mod.rs deleted file mode 100644 index 16e986e227..0000000000 --- a/lighthouse/utils/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -extern crate ethereum_types; -extern crate blake2_rfc as blake2; -extern crate crypto_mac; -extern crate boolean_bitfield; - -#[macro_use] -pub mod macros; -pub mod hash; -pub mod types; -pub mod logging; -pub mod errors; diff --git a/lighthouse/utils/types.rs b/lighthouse/utils/types.rs deleted file mode 100644 index 949b7a549d..0000000000 --- a/lighthouse/utils/types.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate boolean_bitfield; - -use super::ethereum_types::{ H256, H160 }; -use self::boolean_bitfield::BooleanBitfield; - -pub use super::ethereum_types::U256; - -pub type Hash256 = H256; - -pub type Address = H160; - -pub type Bitfield = BooleanBitfield; diff --git a/tests/attestation_validation/helpers.rs b/tests/attestation_validation/helpers.rs deleted file mode 100644 index 3a42db9e1b..0000000000 --- a/tests/attestation_validation/helpers.rs +++ /dev/null @@ -1,192 +0,0 @@ -use std::sync::Arc; - -use super::db::{ - MemoryDB, -}; -use super::db::stores::{ - BlockStore, - ValidatorStore, -}; -use super::state::attestation_record::{ - AttestationRecord, - AttestationValidationContext, -}; -use super::state::block::validation::AttesterMap; -use super::bls::{ - AggregateSignature, - Keypair, - SecretKey, - Signature, -}; -use super::ssz::SszStream; -use super::utils::types::{ - Hash256, - Bitfield, -}; -use super::utils::hash::{ - canonical_hash, -}; - - -pub struct TestStore { - pub db: Arc, - pub block: Arc>, - pub validator: Arc>, -} - -impl TestStore { - pub fn new() -> Self { - let db = Arc::new(MemoryDB::open()); - let block = Arc::new(BlockStore::new(db.clone())); - let validator = Arc::new(ValidatorStore::new(db.clone())); - Self { - db, - block, - validator, - } - } -} - -pub struct TestRig { - pub attestation: AttestationRecord, - pub context: AttestationValidationContext, - pub stores: TestStore, - pub attester_count: usize, -} - -fn generate_message_hash(slot: u64, - parent_hashes: &[Hash256], - shard_id: u16, - shard_block_hash: &Hash256, - justified_slot: u64) - -> Vec -{ - let mut stream = SszStream::new(); - stream.append(&slot); - stream.append_vec(&parent_hashes.to_vec()); - stream.append(&shard_id); - stream.append(shard_block_hash); - stream.append(&justified_slot); - let bytes = stream.drain(); - canonical_hash(&bytes) -} - -pub fn generate_attestation(shard_id: u16, - shard_block_hash: &Hash256, - block_slot: u64, - attestation_slot: u64, - justified_slot: u64, - justified_block_hash: &Hash256, - cycle_length: u8, - parent_hashes: &[Hash256], - signing_keys: &[Option]) - -> AttestationRecord -{ - let mut attester_bitfield = Bitfield::new(); - let mut aggregate_sig = AggregateSignature::new(); - - let parent_hashes_slice = { - let distance: usize = (block_slot - attestation_slot) as usize; - let last: usize = parent_hashes.len() - distance; - let first: usize = last - usize::from(cycle_length); - &parent_hashes[first..last] - }; - - /* - * Generate the message that will be signed across for this attr record. - */ - let attestation_message = generate_message_hash( - attestation_slot, - parent_hashes_slice, - shard_id, - shard_block_hash, - justified_slot); - - for (i, secret_key) in signing_keys.iter().enumerate() { - /* - * If the signing key is Some, set the bitfield bit to true - * and sign the aggregate sig. - */ - if let Some(sk) = secret_key { - attester_bitfield.set_bit(i, true); - let sig = Signature::new(&attestation_message, sk); - aggregate_sig.add(&sig); - } - } - - AttestationRecord { - slot: attestation_slot, - shard_id, - oblique_parent_hashes: vec![], - shard_block_hash: shard_block_hash.clone(), - attester_bitfield, - justified_slot, - justified_block_hash: justified_block_hash.clone(), - aggregate_sig, - } -} - -pub fn setup_attestation_validation_test(shard_id: u16, attester_count: usize) - -> TestRig -{ - let stores = TestStore::new(); - - let block_slot = 10000; - let cycle_length: u8 = 64; - let last_justified_slot = block_slot - u64::from(cycle_length); - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let parent_hashes = Arc::new(parent_hashes); - let justified_block_hash = Hash256::from("justified_block".as_bytes()); - let shard_block_hash = Hash256::from("shard_block".as_bytes()); - - stores.block.put_serialized_block(&justified_block_hash.as_ref(), &[42]).unwrap(); - - let attestation_slot = block_slot - 1; - - let mut keypairs = vec![]; - let mut signing_keys = vec![]; - let mut attester_map = AttesterMap::new(); - let mut attesters = vec![]; - - /* - * Generate a random keypair for each validator and clone it into the - * list of keypairs. Store it in the database. - */ - for i in 0..attester_count { - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - signing_keys.push(Some(keypair.sk.clone())); - attesters.push(i); - } - attester_map.insert((attestation_slot, shard_id), attesters); - - let context: AttestationValidationContext = AttestationValidationContext { - block_slot, - cycle_length, - last_justified_slot, - parent_hashes: parent_hashes.clone(), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - attester_map: Arc::new(attester_map), - }; - let attestation = generate_attestation( - shard_id, - &shard_block_hash, - block_slot, - attestation_slot, - last_justified_slot, - &justified_block_hash, - cycle_length, - &parent_hashes.clone(), - &signing_keys); - - TestRig { - attestation, - context, - stores, - attester_count, - } -} diff --git a/tests/attestation_validation/mod.rs b/tests/attestation_validation/mod.rs deleted file mode 100644 index 0555fbe3ac..0000000000 --- a/tests/attestation_validation/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod helpers; -mod tests; - -use super::bls; -use super::db; -use super::state; -use super::ssz; -use super::utils; diff --git a/tests/attestation_validation/tests.rs b/tests/attestation_validation/tests.rs deleted file mode 100644 index f486cab37c..0000000000 --- a/tests/attestation_validation/tests.rs +++ /dev/null @@ -1,127 +0,0 @@ -use std::sync::Arc; - -use super::helpers::{ - TestRig, - setup_attestation_validation_test, -}; -use super::state::attestation_record::{ - AttestationValidationError, -}; -use super::state::block::validation::AttesterMap; -use super::bls::{ - AggregateSignature, -}; -use super::utils::types::{ - Hash256, -}; - -fn generic_rig() -> TestRig { - let shard_id = 10; - let validator_count = 2; - setup_attestation_validation_test(shard_id, validator_count) -} - -#[test] -fn test_attestation_validation_valid() { - let rig = generic_rig(); - - let result = rig.context.validate_attestation(&rig.attestation); - - let voter_map = result.unwrap(); - assert_eq!(voter_map.len(), 2); -} - -#[test] -fn test_attestation_validation_invalid_slot_too_high() { - let mut rig = generic_rig(); - - rig.attestation.slot = rig.context.block_slot + 1; - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::SlotTooHigh)); -} - -#[test] -fn test_attestation_validation_invalid_slot_too_low() { - let mut rig = generic_rig(); - - rig.attestation.slot = rig.context.block_slot - u64::from(rig.context.cycle_length) - 2; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::SlotTooLow)); -} - -#[test] -fn test_attestation_validation_invalid_justified_slot_incorrect() { - let mut rig = generic_rig(); - - let original = rig.attestation.justified_slot; - rig.attestation.justified_slot = original - 1; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); - - rig.attestation.justified_slot = original + 1; - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::JustifiedSlotIncorrect)); -} - -#[test] -fn test_attestation_validation_invalid_too_many_oblique() { - let mut rig = generic_rig(); - - let obliques: Vec = (0..(rig.context.cycle_length + 1)) - .map(|i| Hash256::from((i * 2) as u64)) - .collect(); - - rig.attestation.oblique_parent_hashes = obliques; - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::TooManyObliqueHashes)); -} - -#[test] -fn test_attestation_validation_invalid_bad_attester_map() { - let mut rig = generic_rig(); - - rig.context.attester_map = Arc::new(AttesterMap::new()); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadAttesterMap)); -} - -#[test] -fn test_attestation_validation_invalid_bad_bitfield_length() { - let mut rig = generic_rig(); - - /* - * Extend the bitfield by one byte - * - * This is a little hacky and makes assumptions about the internals - * of the bitfield. - */ - let one_byte_higher = rig.attester_count + 8; - rig.attestation.attester_bitfield.set_bit(one_byte_higher, true); - rig.attestation.attester_bitfield.set_bit(one_byte_higher, false); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadBitfieldLength)); -} - -#[test] -fn test_attestation_validation_invalid_unknown_justfied_block_hash() { - let mut rig = generic_rig(); - - rig.attestation.justified_block_hash = Hash256::from("unknown block hash".as_bytes()); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::UnknownJustifiedBlock)); -} - -#[test] -fn test_attestation_validation_invalid_empty_signature() { - let mut rig = generic_rig(); - - rig.attestation.aggregate_sig = AggregateSignature::new(); - - let result = rig.context.validate_attestation(&rig.attestation); - assert_eq!(result, Err(AttestationValidationError::BadAggregateSignature)); -} diff --git a/tests/block_validation/helpers.rs b/tests/block_validation/helpers.rs deleted file mode 100644 index 7e085320db..0000000000 --- a/tests/block_validation/helpers.rs +++ /dev/null @@ -1,230 +0,0 @@ -use std::sync::Arc; - -use super::generate_attestation; -use super::bls::{ - Keypair, -}; -use super::db::{ - MemoryDB, -}; -use super::db::stores::{ - BlockStore, - PoWChainStore, - ValidatorStore, -}; -use super::state::attestation_record::{ - AttestationRecord, -}; -use super::state::block::{ - SszBlock, - Block, -}; -use super::state::block::validation::{ - BlockValidationContext, - SszBlockValidationError, - BlockStatus, - AttesterMap, - ProposerMap, -}; -use super::ssz::{ - SszStream, -}; -use super::utils::types::{ - Hash256, -}; - -#[derive(Debug)] -pub struct BlockTestParams { - pub total_validators: usize, - pub cycle_length: u8, - pub shard_count: u16, - pub shards_per_slot: u16, - pub validators_per_shard: usize, - pub block_slot: u64, - pub attestations_justified_slot: u64, - pub parent_proposer_index: usize, - pub validation_context_slot: u64, - pub validation_context_justified_slot: u64, - pub validation_context_finalized_slot: u64, -} - -pub struct TestStore { - pub db: Arc, - pub block: Arc>, - pub pow_chain: Arc>, - pub validator: Arc>, -} - -impl TestStore { - pub fn new() -> Self { - let db = Arc::new(MemoryDB::open()); - let block = Arc::new(BlockStore::new(db.clone())); - let pow_chain = Arc::new(PoWChainStore::new(db.clone())); - let validator = Arc::new(ValidatorStore::new(db.clone())); - Self { - db, - block, - pow_chain, - validator, - } - } -} - -type ParentHashes = Vec; - -/// Setup for a block validation function, without actually executing the -/// block validation function. -pub fn setup_block_validation_scenario(params: &BlockTestParams) - -> (Block, ParentHashes, AttesterMap, ProposerMap, TestStore) -{ - let stores = TestStore::new(); - - let cycle_length = params.cycle_length; - let shards_per_slot = params.shards_per_slot; - let validators_per_shard = params.validators_per_shard; - let block_slot = params.block_slot; - let attestations_justified_slot = params.attestations_justified_slot; - - let parent_hashes: Vec = (0..(cycle_length * 2)) - .map(|i| Hash256::from(i as u64)) - .collect(); - let parent_hash = Hash256::from("parent_hash".as_bytes()); - let randao_reveal = Hash256::from("randao_reveal".as_bytes()); - let justified_block_hash = Hash256::from("justified_hash".as_bytes()); - let pow_chain_ref = Hash256::from("pow_chain".as_bytes()); - let active_state_root = Hash256::from("active_state".as_bytes()); - let crystallized_state_root = Hash256::from("cry_state".as_bytes()); - let shard_block_hash = Hash256::from("shard_block_hash".as_bytes()); - - stores.pow_chain.put_block_hash(pow_chain_ref.as_ref()).unwrap(); - stores.block.put_serialized_block(justified_block_hash.as_ref(), &vec![42]).unwrap(); - - /* - * Generate a minimum viable parent block and store it in the database. - */ - let mut parent_block = Block::zero(); - let parent_attestation = AttestationRecord::zero(); - parent_block.slot_number = block_slot - 1; - parent_block.attestations.push(parent_attestation); - let parent_block_ssz = serialize_block(&parent_block); - stores.block.put_serialized_block(parent_hash.as_ref(), &parent_block_ssz).unwrap(); - - let proposer_map = { - let mut proposer_map = ProposerMap::new(); - proposer_map.insert(parent_block.slot_number, params.parent_proposer_index); - proposer_map - }; - - let (attester_map, attestations, _keypairs) = { - let mut i = 0; - let attestation_slot = block_slot - 1; - let mut attester_map = AttesterMap::new(); - let mut attestations = vec![]; - let mut keypairs = vec![]; - /* - * For each shard in this slot, generate an attestation. - */ - for shard in 0..shards_per_slot { - let mut signing_keys = vec![]; - let mut attesters = vec![]; - /* - * Generate a random keypair for each validator and clone it into the - * list of keypairs. Store it in the database. - */ - for _ in 0..validators_per_shard { - let keypair = Keypair::random(); - keypairs.push(keypair.clone()); - stores.validator.put_public_key_by_index(i, &keypair.pk).unwrap(); - signing_keys.push(Some(keypair.sk.clone())); - attesters.push(i); - i += 1; - } - attester_map.insert((attestation_slot, shard), attesters); - - let attestation = generate_attestation( - shard, - &shard_block_hash, - block_slot, - attestation_slot, - attestations_justified_slot, - &justified_block_hash, - cycle_length, - &parent_hashes, - &signing_keys[..]); - attestations.push(attestation); - } - (attester_map, attestations, keypairs) - }; - - let block = Block { - parent_hash, - slot_number: block_slot, - randao_reveal, - attestations, - pow_chain_ref, - active_state_root, - crystallized_state_root, - }; - - (block, - parent_hashes, - attester_map, - proposer_map, - stores) -} - -/// Helper function to take some Block and SSZ serialize it. -pub fn serialize_block(b: &Block) -> Vec { - let mut stream = SszStream::new(); - stream.append(b); - stream.drain() -} - -/// Setup and run a block validation scenario, given some parameters. -/// -/// Returns the Result returned from the block validation function. -pub fn run_block_validation_scenario( - params: &BlockTestParams, - mutator_func: F) - -> Result<(BlockStatus, Option), SszBlockValidationError> - where F: FnOnce(Block, AttesterMap, ProposerMap, TestStore) - -> (Block, AttesterMap, ProposerMap, TestStore) -{ - let (block, - parent_hashes, - attester_map, - proposer_map, - stores) = setup_block_validation_scenario(¶ms); - - let (block, - attester_map, - proposer_map, - stores) = mutator_func(block, attester_map, proposer_map, stores); - - let ssz_bytes = serialize_block(&block); - let ssz_block = SszBlock::from_slice(&ssz_bytes[..]) - .unwrap(); - - let context = BlockValidationContext { - present_slot: params.validation_context_slot, - cycle_length: params.cycle_length, - last_justified_slot: params.validation_context_justified_slot, - last_finalized_slot: params.validation_context_finalized_slot, - parent_hashes: Arc::new(parent_hashes), - proposer_map: Arc::new(proposer_map), - attester_map: Arc::new(attester_map), - block_store: stores.block.clone(), - validator_store: stores.validator.clone(), - pow_store: stores.pow_chain.clone() - }; - let validation_status = context.validate_ssz_block(&ssz_block); - /* - * If validation returned a block, make sure it's the same block we supplied to it. - * - * I.e., there were no errors during the serialization -> deserialization process. - */ - if let Ok((_, Some(returned_block))) = &validation_status { - assert_eq!(*returned_block, block); - }; - validation_status -} diff --git a/tests/block_validation/mod.rs b/tests/block_validation/mod.rs deleted file mode 100644 index 3ff5741cb9..0000000000 --- a/tests/block_validation/mod.rs +++ /dev/null @@ -1,10 +0,0 @@ -mod helpers; -mod tests; - -use super::bls; -use super::db; -use super::ssz; -use super::state; -use super::utils; - -use super::attestation_validation::helpers::generate_attestation; diff --git a/tests/block_validation/tests.rs b/tests/block_validation/tests.rs deleted file mode 100644 index d3c18af849..0000000000 --- a/tests/block_validation/tests.rs +++ /dev/null @@ -1,245 +0,0 @@ -use super::bls::{ - AggregateSignature, -}; -use super::helpers::{ - BlockTestParams, - TestStore, - run_block_validation_scenario, - serialize_block, -}; -use super::state::block::{ - SszBlock, - Block, -}; -use super::state::block::validation::{ - SszBlockValidationError, - BlockStatus, - ProposerMap, -}; -use super::state::attestation_record::{ - AttestationValidationError, -}; -use super::utils::hash::canonical_hash; -use super::utils::types::{ - Hash256, -}; - -fn get_simple_params() -> BlockTestParams { - let validators_per_shard: usize = 5; - let cycle_length: u8 = 2; - let shard_count: u16 = 4; - let shards_per_slot: u16 = shard_count / u16::from(cycle_length); - let total_validators: usize = validators_per_shard * shard_count as usize; - let block_slot = u64::from(cycle_length) * 10000; - let attestations_justified_slot = block_slot - u64::from(cycle_length); - let parent_proposer_index = 0; - - let validation_context_slot = block_slot; - let validation_context_justified_slot = attestations_justified_slot; - let validation_context_finalized_slot = 0; - - BlockTestParams { - total_validators, - cycle_length, - shard_count, - shards_per_slot, - validators_per_shard, - parent_proposer_index, - block_slot, - attestations_justified_slot, - validation_context_slot, - validation_context_justified_slot, - validation_context_finalized_slot, - } -} - -// TODO: test bad ssz serialization - -#[test] -fn test_block_validation_valid() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap().0, BlockStatus::NewBlock); -} - -#[test] -fn test_block_validation_valid_known_block() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, proposer_map, stores: TestStore| { - /* - * Pre-store the block in the database - */ - let block_ssz = serialize_block(&block); - let block_hash = canonical_hash(&block_ssz); - stores.block.put_serialized_block(&block_hash, &block_ssz).unwrap(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status.unwrap(), (BlockStatus::KnownBlock, None)); -} - -#[test] -fn test_block_validation_invalid_future_slot() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.slot_number = block.slot_number + 1; - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::FutureSlot)); -} - -#[test] -fn test_block_validation_invalid_slot_already_finalized() { - let mut params = get_simple_params(); - - params.validation_context_finalized_slot = params.block_slot; - params.validation_context_justified_slot = params.validation_context_finalized_slot + - u64::from(params.cycle_length); - - let mutator = |block, attester_map, proposer_map, stores| { - /* - * Do not mutate - */ - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::SlotAlreadyFinalized)); -} - -#[test] -fn test_block_validation_invalid_unknown_pow_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.pow_chain_ref = Hash256::from("unknown pow hash".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownPoWChainRef)); -} - -#[test] -fn test_block_validation_invalid_unknown_parent_hash() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - block.parent_hash = Hash256::from("unknown parent block".as_bytes()); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::UnknownParentHash)); -} - -#[test] -fn test_block_validation_invalid_1st_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[0].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( - AttestationValidationError::BadAggregateSignature))); -} - -#[test] -fn test_block_validation_invalid_no_parent_proposer_signature() { - let params = get_simple_params(); - - let mutator = |block: Block, attester_map, mut proposer_map: ProposerMap, stores: TestStore| { - /* - * Set the proposer for this slot to be a validator that does not exist. - */ - let ssz = stores.block.get_serialized_block(&block.parent_hash.as_ref()).unwrap().unwrap(); - let parent_block_slot = SszBlock::from_slice(&ssz[..]).unwrap().slot_number(); - proposer_map.insert(parent_block_slot, params.total_validators + 1); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::NoProposerSignature)); -} - -#[test] -fn test_block_validation_invalid_bad_proposer_map() { - let params = get_simple_params(); - - let mutator = |block, attester_map, _, stores| { - /* - * Initialize a new, empty proposer map - */ - let proposer_map = ProposerMap::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::BadProposerMap)); -} - -#[test] -fn test_block_validation_invalid_2nd_attestation_signature() { - let params = get_simple_params(); - - let mutator = |mut block: Block, attester_map, proposer_map, stores| { - /* - * Set the second attestaion record to have an invalid signature. - */ - block.attestations[1].aggregate_sig = AggregateSignature::new(); - (block, attester_map, proposer_map, stores) - }; - - let status = run_block_validation_scenario( - ¶ms, - mutator); - - assert_eq!(status, Err(SszBlockValidationError::AttestationValidationError( - AttestationValidationError::BadAggregateSignature))); -} diff --git a/tests/main.rs b/tests/main.rs deleted file mode 100644 index caa601105c..0000000000 --- a/tests/main.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate lighthouse; -extern crate ssz; - -#[cfg(test)] -mod attestation_validation; -#[cfg(test)] -mod block_validation; - -use lighthouse::bls; -use lighthouse::db; -use lighthouse::state; -use lighthouse::utils;