gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 10:22:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into 215-migrate-ssz-little-endian

Browse Source
This commit is contained in:
Kirk Baird
2019-03-18 10:47:40 +11:00
parent 2961edcbb4 3cf2359244
commit 2c95d531ea
146 changed files with 7502 additions and 3642 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
eth2/utils/bls/Cargo.toml
View File

@@ -5,7 +5,7 @@ authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
[dependencies]
bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "v0.3.0" }
bls-aggregates = { git = "https://github.com/sigp/signature-schemes", tag = "0.5.2" }
hashing = { path = "../hashing" }
hex = "0.3"
serde = "1.0"

24
eth2/utils/bls/src/aggregate_public_key.rs Normal file
View File

@@ -0,0 +1,24 @@
use super::PublicKey;
use bls_aggregates::AggregatePublicKey as RawAggregatePublicKey;
/// A single BLS signature.
///
/// This struct is a wrapper upon a base type and provides helper functions (e.g., SSZ
/// serialization).
#[derive(Debug, Clone, Default)]
pub struct AggregatePublicKey(RawAggregatePublicKey);
impl AggregatePublicKey {
pub fn new() -> Self {
AggregatePublicKey(RawAggregatePublicKey::new())
}
pub fn add(&mut self, public_key: &PublicKey) {
self.0.add(public_key.as_raw())
}
/// Returns the underlying signature.
pub fn as_raw(&self) -> &RawAggregatePublicKey {
&self.0
}
}

46
eth2/utils/bls/src/aggregate_signature.rs
View File

@@ -1,5 +1,7 @@
use super::{AggregatePublicKey, Signature};
use bls_aggregates::AggregateSignature as RawAggregateSignature;
use bls_aggregates::{
AggregatePublicKey as RawAggregatePublicKey, AggregateSignature as RawAggregateSignature,
};
use serde::ser::{Serialize, Serializer};
use ssz::{
decode_ssz_list, hash, ssz_encode, Decodable, DecodeError, Encodable, SszStream, TreeHash,
@@ -27,8 +29,44 @@ impl AggregateSignature {
///
/// Only returns `true` if the set of keys in the `AggregatePublicKey` match the set of keys
/// that signed the `AggregateSignature`.
pub fn verify(&self, msg: &[u8], aggregate_public_key: &AggregatePublicKey) -> bool {
self.0.verify(msg, aggregate_public_key)
pub fn verify(
&self,
msg: &[u8],
domain: u64,
aggregate_public_key: &AggregatePublicKey,
) -> bool {
self.0.verify(msg, domain, aggregate_public_key.as_raw())
}
/// Verify this AggregateSignature against multiple AggregatePublickeys with multiple Messages.
///
/// All PublicKeys related to a Message should be aggregated into one AggregatePublicKey.
/// Each AggregatePublicKey has a 1:1 ratio with a 32 byte Message.
pub fn verify_multiple(
&self,
messages: &[&[u8]],
domain: u64,
aggregate_public_keys: &[&AggregatePublicKey],
) -> bool {
// TODO: the API for `RawAggregatePublicKey` shoudn't need to take an owned
// `AggregatePublicKey`. There is an issue to fix this, but in the meantime we need to
// clone.
//
// https://github.com/sigp/signature-schemes/issues/10
let aggregate_public_keys: Vec<RawAggregatePublicKey> = aggregate_public_keys
.iter()
.map(|pk| pk.as_raw())
.cloned()
.collect();
// Messages are concatenated into one long message.
let mut msg: Vec<u8> = vec![];
for message in messages {
msg.extend_from_slice(message);
}
self.0
.verify_multiple(&msg[..], domain, &aggregate_public_keys[..])
}
}
@@ -73,7 +111,7 @@ mod tests {
let keypair = Keypair::random();
let mut original = AggregateSignature::new();
original.add(&Signature::new(&[42, 42], &keypair.sk));
original.add(&Signature::new(&[42, 42], 0, &keypair.sk));
let bytes = ssz_encode(&original);
let (decoded, _) = AggregateSignature::ssz_decode(&bytes, 0).unwrap();

39
eth2/utils/bls/src/lib.rs
View File

@@ -1,52 +1,53 @@
extern crate bls_aggregates;
extern crate hashing;
extern crate ssz;
mod aggregate_public_key;
mod aggregate_signature;
mod keypair;
mod public_key;
mod secret_key;
mod signature;
pub use crate::aggregate_public_key::AggregatePublicKey;
pub use crate::aggregate_signature::AggregateSignature;
pub use crate::keypair::Keypair;
pub use crate::public_key::PublicKey;
pub use crate::secret_key::SecretKey;
pub use crate::signature::Signature;
pub use self::bls_aggregates::AggregatePublicKey;
pub const BLS_AGG_SIG_BYTE_SIZE: usize = 97;
pub const BLS_AGG_SIG_BYTE_SIZE: usize = 96;
use hashing::hash;
use ssz::ssz_encode;
use std::default::Default;
fn extend_if_needed(hash: &mut Vec<u8>) {
// NOTE: bls_aggregates crate demands 48 bytes, this may be removed as we get closer to production
hash.resize(48, Default::default())
}
/// For some signature and public key, ensure that the signature message was the public key and it
/// was signed by the secret key that corresponds to that public key.
pub fn verify_proof_of_possession(sig: &Signature, pubkey: &PublicKey) -> bool {
let mut hash = hash(&ssz_encode(pubkey));
extend_if_needed(&mut hash);
sig.verify_hashed(&hash, &pubkey)
// TODO: replace this function with state.validate_proof_of_possession
// https://github.com/sigp/lighthouse/issues/239
sig.verify(&ssz_encode(pubkey), 0, &pubkey)
}
// TODO: Update this method
// https://github.com/sigp/lighthouse/issues/239
pub fn create_proof_of_possession(keypair: &Keypair) -> Signature {
let mut hash = hash(&ssz_encode(&keypair.pk));
extend_if_needed(&mut hash);
Signature::new_hashed(&hash, &keypair.sk)
Signature::new(&ssz_encode(&keypair.pk), 0, &keypair.sk)
}
/// Returns the withdrawal credentials for a given public key.
pub fn get_withdrawal_credentials(pubkey: &PublicKey, prefix_byte: u8) -> Vec<u8> {
let hashed = hash(&ssz_encode(pubkey));
let mut prefixed = vec![prefix_byte];
prefixed.extend_from_slice(&hashed[1..]);
prefixed
}
pub fn bls_verify_aggregate(
pubkey: &AggregatePublicKey,
message: &[u8],
signature: &AggregateSignature,
_domain: u64,
domain: u64,
) -> bool {
// TODO: add domain
signature.verify(message, pubkey)
signature.verify(message, domain, pubkey)
}

42
eth2/utils/bls/src/signature.rs
View File

@@ -14,24 +14,34 @@ pub struct Signature(RawSignature);
impl Signature {
/// Instantiate a new Signature from a message and a SecretKey.
pub fn new(msg: &[u8], sk: &SecretKey) -> Self {
Signature(RawSignature::new(msg, sk.as_raw()))
pub fn new(msg: &[u8], domain: u64, sk: &SecretKey) -> Self {
Signature(RawSignature::new(msg, domain, sk.as_raw()))
}
/// Instantiate a new Signature from a message and a SecretKey, where the message has already
/// been hashed.
pub fn new_hashed(msg_hashed: &[u8], sk: &SecretKey) -> Self {
Signature(RawSignature::new_hashed(msg_hashed, sk.as_raw()))
pub fn new_hashed(x_real_hashed: &[u8], x_imaginary_hashed: &[u8], sk: &SecretKey) -> Self {
Signature(RawSignature::new_hashed(
x_real_hashed,
x_imaginary_hashed,
sk.as_raw(),
))
}
/// Verify the Signature against a PublicKey.
pub fn verify(&self, msg: &[u8], pk: &PublicKey) -> bool {
self.0.verify(msg, pk.as_raw())
pub fn verify(&self, msg: &[u8], domain: u64, pk: &PublicKey) -> bool {
self.0.verify(msg, domain, pk.as_raw())
}
/// Verify the Signature against a PublicKey, where the message has already been hashed.
pub fn verify_hashed(&self, msg_hash: &[u8], pk: &PublicKey) -> bool {
self.0.verify_hashed(msg_hash, pk.as_raw())
pub fn verify_hashed(
&self,
x_real_hashed: &[u8],
x_imaginary_hashed: &[u8],
pk: &PublicKey,
) -> bool {
self.0
.verify_hashed(x_real_hashed, x_imaginary_hashed, pk.as_raw())
}
/// Returns the underlying signature.
@@ -41,7 +51,9 @@ impl Signature {
/// Returns a new empty signature.
pub fn empty_signature() -> Self {
let empty: Vec<u8> = vec![0; 97];
let mut empty: Vec<u8> = vec![0; 96];
// TODO: Modify the way flags are used (b_flag should not be used for empty_signature in the future)
empty[0] += u8::pow(2, 6);
Signature(RawSignature::from_bytes(&empty).unwrap())
}
}
@@ -85,7 +97,7 @@ mod tests {
pub fn test_ssz_round_trip() {
let keypair = Keypair::random();
let original = Signature::new(&[42, 42], &keypair.sk);
let original = Signature::new(&[42, 42], 0, &keypair.sk);
let bytes = ssz_encode(&original);
let (decoded, _) = Signature::ssz_decode(&bytes, 0).unwrap();
@@ -99,9 +111,13 @@ mod tests {
let sig_as_bytes: Vec<u8> = sig.as_raw().as_bytes();
assert_eq!(sig_as_bytes.len(), 97);
for one_byte in sig_as_bytes.iter() {
assert_eq!(*one_byte, 0);
assert_eq!(sig_as_bytes.len(), 96);
for (i, one_byte) in sig_as_bytes.iter().enumerate() {
if i == 0 {
assert_eq!(*one_byte, u8::pow(2, 6));
} else {
assert_eq!(*one_byte, 0);
}
}
}
}

9
eth2/utils/fisher_yates_shuffle/Cargo.toml
View File

@@ -4,8 +4,13 @@ version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
[dependencies]
hashing = { path = "../hashing" }
[[bench]]
name = "benches"
harness = false
[dev-dependencies]
criterion = "0.2"
yaml-rust = "0.4.2"
[dependencies]
hashing = { path = "../hashing" }

55
eth2/utils/fisher_yates_shuffle/benches/benches.rs Normal file
View File

@@ -0,0 +1,55 @@
use criterion::Criterion;
use criterion::{black_box, criterion_group, criterion_main, Benchmark};
use fisher_yates_shuffle::shuffle;
fn get_list(n: usize) -> Vec<usize> {
let mut list = Vec::with_capacity(n);
for i in 0..n {
list.push(i)
}
assert_eq!(list.len(), n);
list
}
fn shuffles(c: &mut Criterion) {
c.bench(
"whole list shuffle",
Benchmark::new("8 elements", move |b| {
let seed = vec![42; 32];
let list = get_list(8);
b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list)))
}),
);
c.bench(
"whole list shuffle",
Benchmark::new("16 elements", move |b| {
let seed = vec![42; 32];
let list = get_list(16);
b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list)))
}),
);
c.bench(
"whole list shuffle",
Benchmark::new("512 elements", move |b| {
let seed = vec![42; 32];
let list = get_list(512);
b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list)))
})
.sample_size(10),
);
c.bench(
"whole list shuffle",
Benchmark::new("16384 elements", move |b| {
let seed = vec![42; 32];
let list = get_list(16_384);
b.iter_with_setup(|| list.clone(), |list| black_box(shuffle(&seed, list)))
})
.sample_size(10),
);
}
criterion_group!(benches, shuffles);
criterion_main!(benches);

9
eth2/utils/merkle_proof/Cargo.toml Normal file
View File

@@ -0,0 +1,9 @@
[package]
name = "merkle_proof"
version = "0.1.0"
authors = ["Michael Sproul <michael@sigmaprime.io>"]
edition = "2018"
[dependencies]
ethereum-types = "0.5"
hashing = { path = "../hashing" }

148
eth2/utils/merkle_proof/src/lib.rs Normal file
View File

@@ -0,0 +1,148 @@
use ethereum_types::H256;
use hashing::hash;
/// Verify a proof that `leaf` exists at `index` in a Merkle tree rooted at `root`.
///
/// The `branch` argument is the main component of the proof: it should be a list of internal
/// node hashes such that the root can be reconstructed (in bottom-up order).
pub fn verify_merkle_proof(
leaf: H256,
branch: &[H256],
depth: usize,
index: usize,
root: H256,
) -> bool {
if branch.len() == depth {
merkle_root_from_branch(leaf, branch, depth, index) == root
} else {
false
}
}
/// Compute a root hash from a leaf and a Merkle proof.
fn merkle_root_from_branch(leaf: H256, branch: &[H256], depth: usize, index: usize) -> H256 {
assert_eq!(branch.len(), depth, "proof length should equal depth");
let mut merkle_root = leaf.as_bytes().to_vec();
for (i, leaf) in branch.iter().enumerate().take(depth) {
let ith_bit = (index >> i) & 0x01;
if ith_bit == 1 {
let input = concat(leaf.as_bytes().to_vec(), merkle_root);
merkle_root = hash(&input);
} else {
let mut input = merkle_root;
input.extend_from_slice(leaf.as_bytes());
merkle_root = hash(&input);
}
}
H256::from_slice(&merkle_root)
}
/// Concatenate two vectors.
fn concat(mut vec1: Vec<u8>, mut vec2: Vec<u8>) -> Vec<u8> {
vec1.append(&mut vec2);
vec1
}
#[cfg(test)]
mod tests {
use super::*;
fn hash_concat(h1: H256, h2: H256) -> H256 {
H256::from_slice(&hash(&concat(
h1.as_bytes().to_vec(),
h2.as_bytes().to_vec(),
)))
}
#[test]
fn verify_small_example() {
// Construct a small merkle tree manually
let leaf_b00 = H256::from([0xAA; 32]);
let leaf_b01 = H256::from([0xBB; 32]);
let leaf_b10 = H256::from([0xCC; 32]);
let leaf_b11 = H256::from([0xDD; 32]);
let node_b0x = hash_concat(leaf_b00, leaf_b01);
let node_b1x = hash_concat(leaf_b10, leaf_b11);
let root = hash_concat(node_b0x, node_b1x);
// Run some proofs
assert!(verify_merkle_proof(
leaf_b00,
&[leaf_b01, node_b1x],
2,
0b00,
root
));
assert!(verify_merkle_proof(
leaf_b01,
&[leaf_b00, node_b1x],
2,
0b01,
root
));
assert!(verify_merkle_proof(
leaf_b10,
&[leaf_b11, node_b0x],
2,
0b10,
root
));
assert!(verify_merkle_proof(
leaf_b11,
&[leaf_b10, node_b0x],
2,
0b11,
root
));
assert!(verify_merkle_proof(
leaf_b11,
&[leaf_b10],
1,
0b11,
node_b1x
));
// Ensure that incorrect proofs fail
// Zero-length proof
assert!(!verify_merkle_proof(leaf_b01, &[], 2, 0b01, root));
// Proof in reverse order
assert!(!verify_merkle_proof(
leaf_b01,
&[node_b1x, leaf_b00],
2,
0b01,
root
));
// Proof too short
assert!(!verify_merkle_proof(leaf_b01, &[leaf_b00], 2, 0b01, root));
// Wrong index
assert!(!verify_merkle_proof(
leaf_b01,
&[leaf_b00, node_b1x],
2,
0b10,
root
));
// Wrong root
assert!(!verify_merkle_proof(
leaf_b01,
&[leaf_b00, node_b1x],
2,
0b01,
node_b1x
));
}
#[test]
fn verify_zero_depth() {
let leaf = H256::from([0xD6; 32]);
let junk = H256::from([0xD7; 32]);
assert!(verify_merkle_proof(leaf, &[], 0, 0, leaf));
assert!(!verify_merkle_proof(leaf, &[], 0, 7, junk));
}
}

2
eth2/utils/ssz/Cargo.toml
View File

@@ -6,5 +6,5 @@ edition = "2018"
[dependencies]
bytes = "0.4.9"
ethereum-types = "0.4.0"
ethereum-types = "0.5"
hashing = { path = "../hashing" }

8
eth2/utils/ssz/src/impl_decode.rs
View File

@@ -59,7 +59,7 @@ impl Decodable for H256 {
if bytes.len() < 32 || bytes.len() - 32 < index {
Err(DecodeError::TooShort)
} else {
Ok((H256::from(&bytes[index..(index + 32)]), index + 32))
Ok((H256::from_slice(&bytes[index..(index + 32)]), index + 32))
}
}
}
@@ -69,7 +69,7 @@ impl Decodable for Address {
if bytes.len() < 20 || bytes.len() - 20 < index {
Err(DecodeError::TooShort)
} else {
Ok((Address::from(&bytes[index..(index + 20)]), index + 20))
Ok((Address::from_slice(&bytes[index..(index + 20)]), index + 20))
}
}
}
@@ -95,7 +95,7 @@ mod tests {
*/
let input = vec![42_u8; 32];
let (decoded, i) = H256::ssz_decode(&input, 0).unwrap();
assert_eq!(decoded.to_vec(), input);
assert_eq!(decoded.as_bytes(), &input[..]);
assert_eq!(i, 32);
/*
@@ -104,7 +104,7 @@ mod tests {
let mut input = vec![42_u8; 32];
input.push(12);
let (decoded, i) = H256::ssz_decode(&input, 0).unwrap();
assert_eq!(decoded.to_vec()[..], input[0..32]);
assert_eq!(decoded.as_bytes(), &input[0..32]);
assert_eq!(i, 32);
/*

4
eth2/utils/ssz/src/impl_encode.rs
View File

@@ -55,13 +55,13 @@ impl Encodable for bool {
impl Encodable for H256 {
fn ssz_append(&self, s: &mut SszStream) {
s.append_encoded_raw(&self.to_vec());
s.append_encoded_raw(self.as_bytes());
}
}
impl Encodable for Address {
fn ssz_append(&self, s: &mut SszStream) {
s.append_encoded_raw(&self.to_vec());
s.append_encoded_raw(self.as_bytes());
}
}

6
eth2/utils/ssz/src/impl_tree_hash.rs
View File

@@ -32,6 +32,12 @@ impl TreeHash for usize {
}
}
impl TreeHash for bool {
fn hash_tree_root_internal(&self) -> Vec<u8> {
ssz_encode(self)
}
}
impl TreeHash for Address {
fn hash_tree_root_internal(&self) -> Vec<u8> {
ssz_encode(self)

2
eth2/utils/ssz/src/lib.rs
View File

@@ -12,6 +12,7 @@ extern crate ethereum_types;
pub mod decode;
pub mod encode;
mod signed_root;
pub mod tree_hash;
mod impl_decode;
@@ -20,6 +21,7 @@ mod impl_tree_hash;
pub use crate::decode::{decode_ssz, decode_ssz_list, Decodable, DecodeError};
pub use crate::encode::{Encodable, SszStream};
pub use crate::signed_root::SignedRoot;
pub use crate::tree_hash::{merkle_hash, TreeHash};
pub use hashing::hash;

5
eth2/utils/ssz/src/signed_root.rs Normal file
View File

@@ -0,0 +1,5 @@
use crate::TreeHash;
pub trait SignedRoot: TreeHash {
fn signed_root(&self) -> Vec<u8>;
}

4
eth2/utils/ssz/src/tree_hash.rs
View File

@@ -7,9 +7,7 @@ pub trait TreeHash {
fn hash_tree_root_internal(&self) -> Vec<u8>;
fn hash_tree_root(&self) -> Vec<u8> {
let mut result = self.hash_tree_root_internal();
if result.len() < HASHSIZE {
zpad(&mut result, HASHSIZE);
}
zpad(&mut result, HASHSIZE);
result
}
}

107
eth2/utils/ssz_derive/src/lib.rs
View File

@@ -2,6 +2,7 @@
//!
//! - `#[derive(Encode)]`
//! - `#[derive(Decode)]`
//! - `#[derive(TreeHash)]`
//!
//! These macros provide SSZ encoding/decoding for a `struct`. Fields are encoded/decoded in the
//! order they are defined.
@@ -126,3 +127,109 @@ pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
};
output.into()
}
/// Implements `ssz::TreeHash` for some `struct`.
///
/// Fields are processed in the order they are defined.
#[proc_macro_derive(TreeHash)]
pub fn ssz_tree_hash_derive(input: TokenStream) -> TokenStream {
let item = parse_macro_input!(input as DeriveInput);
let name = &item.ident;
let struct_data = match &item.data {
syn::Data::Struct(s) => s,
_ => panic!("ssz_derive only supports structs."),
};
let field_idents = get_named_field_idents(&struct_data);
let output = quote! {
impl ssz::TreeHash for #name {
fn hash_tree_root_internal(&self) -> Vec<u8> {
let mut list: Vec<Vec<u8>> = Vec::new();
#(
list.push(self.#field_idents.hash_tree_root_internal());
)*
ssz::merkle_hash(&mut list)
}
}
};
output.into()
}
/// Returns `true` if some `Ident` should be considered to be a signature type.
fn type_ident_is_signature(ident: &syn::Ident) -> bool {
match ident.to_string().as_ref() {
"Signature" => true,
"AggregateSignature" => true,
_ => false,
}
}
/// Takes a `Field` where the type (`ty`) portion is a path (e.g., `types::Signature`) and returns
/// the final `Ident` in that path.
///
/// E.g., for `types::Signature` returns `Signature`.
fn final_type_ident(field: &syn::Field) -> &syn::Ident {
match &field.ty {
syn::Type::Path(path) => &path.path.segments.last().unwrap().value().ident,
_ => panic!("ssz_derive only supports Path types."),
}
}
/// Implements `ssz::TreeHash` for some `struct`, whilst excluding any fields following and
/// including a field that is of type "Signature" or "AggregateSignature".
///
/// See:
/// https://github.com/ethereum/eth2.0-specs/blob/master/specs/simple-serialize.md#signed-roots
///
/// This is a rather horrendous macro, it will read the type of the object as a string and decide
/// if it's a signature by matching that string against "Signature" or "AggregateSignature". So,
/// it's important that you use those exact words as your type -- don't alias it to something else.
///
/// If you can think of a better way to do this, please make an issue!
///
/// Fields are processed in the order they are defined.
#[proc_macro_derive(SignedRoot)]
pub fn ssz_signed_root_derive(input: TokenStream) -> TokenStream {
let item = parse_macro_input!(input as DeriveInput);
let name = &item.ident;
let struct_data = match &item.data {
syn::Data::Struct(s) => s,
_ => panic!("ssz_derive only supports structs."),
};
let mut field_idents: Vec<&syn::Ident> = vec![];
for field in struct_data.fields.iter() {
let final_type_ident = final_type_ident(&field);
if type_ident_is_signature(final_type_ident) {
break;
} else {
let ident = field
.ident
.as_ref()
.expect("ssz_derive only supports named_struct fields.");
field_idents.push(ident);
}
}
let output = quote! {
impl ssz::SignedRoot for #name {
fn signed_root(&self) -> Vec<u8> {
let mut list: Vec<Vec<u8>> = Vec::new();
#(
list.push(self.#field_idents.hash_tree_root_internal());
)*
ssz::merkle_hash(&mut list)
}
}
};
output.into()
}

15
eth2/utils/swap_or_not_shuffle/Cargo.toml
View File

@@ -4,12 +4,17 @@ version = "0.1.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2018"
[[bench]]
name = "benches"
harness = false
[dev-dependencies]
criterion = "0.2"
yaml-rust = "0.4.2"
hex = "0.3"
ethereum-types = "0.5"
[dependencies]
bytes = "0.4"
hashing = { path = "../hashing" }
int_to_bytes = { path = "../int_to_bytes" }
[dev-dependencies]
yaml-rust = "0.4.2"
hex = "0.3"
ethereum-types = "0.5"

92
eth2/utils/swap_or_not_shuffle/benches/benches.rs Normal file
View File

@@ -0,0 +1,92 @@
use criterion::Criterion;
use criterion::{black_box, criterion_group, criterion_main, Benchmark};
use swap_or_not_shuffle::{get_permutated_index, shuffle_list as fast_shuffle};
const SHUFFLE_ROUND_COUNT: u8 = 90;
fn shuffle_list(seed: &[u8], list_size: usize) -> Vec<usize> {
let mut output = Vec::with_capacity(list_size);
for i in 0..list_size {
output.push(get_permutated_index(i, list_size, seed, SHUFFLE_ROUND_COUNT).unwrap());
}
output
}
fn shuffles(c: &mut Criterion) {
c.bench_function("single swap", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(get_permutated_index(0, 10, &seed, SHUFFLE_ROUND_COUNT)))
});
c.bench_function("whole list of size 8", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(shuffle_list(&seed, 8)))
});
c.bench(
"whole list shuffle",
Benchmark::new("8 elements", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(shuffle_list(&seed, 8)))
}),
);
c.bench(
"whole list shuffle",
Benchmark::new("16 elements", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(shuffle_list(&seed, 16)))
}),
);
c.bench(
"whole list shuffle",
Benchmark::new("512 elements", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(shuffle_list(&seed, 512)))
})
.sample_size(10),
);
c.bench(
"_fast_ whole list shuffle",
Benchmark::new("512 elements", move |b| {
let seed = vec![42; 32];
let list: Vec<usize> = (0..512).collect();
b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
})
.sample_size(10),
);
c.bench(
"whole list shuffle",
Benchmark::new("16384 elements", move |b| {
let seed = vec![42; 32];
b.iter(|| black_box(shuffle_list(&seed, 16_384)))
})
.sample_size(10),
);
c.bench(
"_fast_ whole list shuffle",
Benchmark::new("16384 elements", move |b| {
let seed = vec![42; 32];
let list: Vec<usize> = (0..16384).collect();
b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
})
.sample_size(10),
);
c.bench(
"_fast_ whole list shuffle",
Benchmark::new("4m elements", move |b| {
let seed = vec![42; 32];
let list: Vec<usize> = (0..4_000_000).collect();
b.iter(|| black_box(fast_shuffle(list.clone(), SHUFFLE_ROUND_COUNT, &seed, true)))
})
.sample_size(10),
);
}
criterion_group!(benches, shuffles,);
criterion_main!(benches);

187
eth2/utils/swap_or_not_shuffle/src/get_permutated_index.rs Normal file
View File

@@ -0,0 +1,187 @@
use bytes::Buf;
use hashing::hash;
use int_to_bytes::{int_to_bytes1, int_to_bytes4};
use std::cmp::max;
use std::io::Cursor;
/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
///
/// Utilizes 'swap or not' shuffling found in
/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
/// See the 'generalized domain' algorithm on page 3.
///
/// Note: this function is significantly slower than the `shuffle_list` function in this crate.
/// Using `get_permutated_list` to shuffle an entire list, index by index, has been observed to be
/// 250x slower than `shuffle_list`. Therefore, this function is only useful when shuffling a small
/// portion of a much larger list.
///
/// Returns `None` under any of the following conditions:
/// - `list_size == 0`
/// - `index >= list_size`
/// - `list_size > 2**24`
/// - `list_size > usize::max_value() / 2`
pub fn get_permutated_index(
index: usize,
list_size: usize,
seed: &[u8],
shuffle_round_count: u8,
) -> Option<usize> {
if list_size == 0
|| index >= list_size
|| list_size > usize::max_value() / 2
|| list_size > 2_usize.pow(24)
{
return None;
}
let mut index = index;
for round in 0..shuffle_round_count {
let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size;
index = do_round(seed, index, pivot, round, list_size)?;
}
Some(index)
}
fn do_round(seed: &[u8], index: usize, pivot: usize, round: u8, list_size: usize) -> Option<usize> {
let flip = (pivot + list_size - index) % list_size;
let position = max(index, flip);
let source = hash_with_round_and_position(seed, round, position)?;
let byte = source[(position % 256) / 8];
let bit = (byte >> (position % 8)) % 2;
Some(if bit == 1 { flip } else { index })
}
fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option<Vec<u8>> {
let mut seed = seed.to_vec();
seed.append(&mut int_to_bytes1(round));
/*
* Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 <
* 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`.
*/
seed.append(&mut int_to_bytes4((position / 256) as u32));
Some(hash(&seed[..]))
}
fn hash_with_round(seed: &[u8], round: u8) -> Vec<u8> {
let mut seed = seed.to_vec();
seed.append(&mut int_to_bytes1(round));
hash(&seed[..])
}
fn bytes_to_int64(bytes: &[u8]) -> u64 {
let mut cursor = Cursor::new(bytes);
cursor.get_u64_le()
}
#[cfg(test)]
mod tests {
use super::*;
use ethereum_types::H256 as Hash256;
use hex;
use std::{fs::File, io::prelude::*, path::PathBuf};
use yaml_rust::yaml;
#[test]
#[ignore]
fn fuzz_test() {
let max_list_size = 2_usize.pow(24);
let test_runs = 1000;
// Test at max list_size with the end index.
for _ in 0..test_runs {
let index = max_list_size - 1;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
// Test at max list_size low indices.
for i in 0..test_runs {
let index = i;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
// Test at max list_size high indices.
for i in 0..test_runs {
let index = max_list_size - 1 - i;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
}
#[test]
fn returns_none_for_zero_length_list() {
assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90));
}
#[test]
fn returns_none_for_out_of_bounds_index() {
assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90));
}
#[test]
fn returns_none_for_too_large_list() {
assert_eq!(
None,
get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90)
);
}
#[test]
fn test_vectors() {
/*
* Test vectors are generated here:
*
* https://github.com/ethereum/eth2.0-test-generators
*/
let mut file = {
let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
file_path_buf.push("src/specs/test_vector_permutated_index.yml");
File::open(file_path_buf).unwrap()
};
let mut yaml_str = String::new();
file.read_to_string(&mut yaml_str).unwrap();
let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
let doc = &docs[0];
let test_cases = doc["test_cases"].as_vec().unwrap();
for (i, test_case) in test_cases.iter().enumerate() {
let index = test_case["index"].as_i64().unwrap() as usize;
let list_size = test_case["list_size"].as_i64().unwrap() as usize;
let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize;
let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap();
let seed_string = test_case["seed"].clone().into_string().unwrap();
let seed = hex::decode(seed_string.replace("0x", "")).unwrap();
let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) {
shuffle_round_count as u8
} else {
panic!("shuffle_round_count must be a u8")
};
assert_eq!(
Some(permutated_index),
get_permutated_index(index, list_size, &seed[..], shuffle_round_count),
"Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}",
i,
index,
list_size,
shuffle_round_count,
seed_string,
);
}
}
}

195
eth2/utils/swap_or_not_shuffle/src/lib.rs
View File

@@ -1,178 +1,21 @@
use bytes::Buf;
use hashing::hash;
use int_to_bytes::{int_to_bytes1, int_to_bytes4};
use std::cmp::max;
use std::io::Cursor;
//! Provides list-shuffling functions matching the Ethereum 2.0 specification.
//!
//! See
//! [get_permutated_index](https://github.com/ethereum/eth2.0-specs/blob/0.4.0/specs/core/0_beacon-chain.md#get_permuted_index)
//! for specifications.
//!
//! There are two functions exported by this crate:
//!
//! - `get_permutated_index`: given a single index, computes the index resulting from a shuffle.
//! Runs in less time than it takes to run `shuffle_list`.
//! - `shuffle_list`: shuffles an entire list in-place. Runs in less time than it takes to run
//! `get_permutated_index` on each index.
//!
//! In general, use `get_permutated_list` to calculate the shuffling of a small subset of a much
//! larger list (~250x larger is a good guide, but solid figures yet to be calculated).
/// Return `p(index)` in a pseudorandom permutation `p` of `0...list_size-1` with ``seed`` as entropy.
///
/// Utilizes 'swap or not' shuffling found in
/// https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf
/// See the 'generalized domain' algorithm on page 3.
///
/// Returns `None` under any of the following conditions:
/// - `list_size == 0`
/// - `index >= list_size`
/// - `list_size > 2**24`
/// - `list_size > usize::max_value() / 2`
pub fn get_permutated_index(
index: usize,
list_size: usize,
seed: &[u8],
shuffle_round_count: u8,
) -> Option<usize> {
if list_size == 0
|| index >= list_size
|| list_size > usize::max_value() / 2
|| list_size > 2_usize.pow(24)
{
return None;
}
mod get_permutated_index;
mod shuffle_list;
let mut index = index;
for round in 0..shuffle_round_count {
let pivot = bytes_to_int64(&hash_with_round(seed, round)[..]) as usize % list_size;
let flip = (pivot + list_size - index) % list_size;
let position = max(index, flip);
let source = hash_with_round_and_position(seed, round, position)?;
let byte = source[(position % 256) / 8];
let bit = (byte >> (position % 8)) % 2;
index = if bit == 1 { flip } else { index }
}
Some(index)
}
fn hash_with_round_and_position(seed: &[u8], round: u8, position: usize) -> Option<Vec<u8>> {
let mut seed = seed.to_vec();
seed.append(&mut int_to_bytes1(round));
/*
* Note: the specification has an implicit assertion in `int_to_bytes4` that `position / 256 <
* 2**24`. For efficiency, we do not check for that here as it is checked in `get_permutated_index`.
*/
seed.append(&mut int_to_bytes4((position / 256) as u32));
Some(hash(&seed[..]))
}
fn hash_with_round(seed: &[u8], round: u8) -> Vec<u8> {
let mut seed = seed.to_vec();
seed.append(&mut int_to_bytes1(round));
hash(&seed[..])
}
fn bytes_to_int64(bytes: &[u8]) -> u64 {
let mut cursor = Cursor::new(bytes);
cursor.get_u64_le()
}
#[cfg(test)]
mod tests {
use super::*;
use ethereum_types::H256 as Hash256;
use hex;
use std::{fs::File, io::prelude::*, path::PathBuf};
use yaml_rust::yaml;
#[test]
#[ignore]
fn fuzz_test() {
let max_list_size = 2_usize.pow(24);
let test_runs = 1000;
// Test at max list_size with the end index.
for _ in 0..test_runs {
let index = max_list_size - 1;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
// Test at max list_size low indices.
for i in 0..test_runs {
let index = i;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
// Test at max list_size high indices.
for i in 0..test_runs {
let index = max_list_size - 1 - i;
let list_size = max_list_size;
let seed = Hash256::random();
let shuffle_rounds = 90;
assert!(get_permutated_index(index, list_size, &seed[..], shuffle_rounds).is_some());
}
}
#[test]
fn returns_none_for_zero_length_list() {
assert_eq!(None, get_permutated_index(100, 0, &[42, 42], 90));
}
#[test]
fn returns_none_for_out_of_bounds_index() {
assert_eq!(None, get_permutated_index(100, 100, &[42, 42], 90));
}
#[test]
fn returns_none_for_too_large_list() {
assert_eq!(
None,
get_permutated_index(100, usize::max_value() / 2, &[42, 42], 90)
);
}
#[test]
fn test_vectors() {
/*
* Test vectors are generated here:
*
* https://github.com/ethereum/eth2.0-test-generators
*/
let mut file = {
let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
file_path_buf.push("src/specs/test_vector_permutated_index.yml");
File::open(file_path_buf).unwrap()
};
let mut yaml_str = String::new();
file.read_to_string(&mut yaml_str).unwrap();
let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
let doc = &docs[0];
let test_cases = doc["test_cases"].as_vec().unwrap();
for (i, test_case) in test_cases.iter().enumerate() {
let index = test_case["index"].as_i64().unwrap() as usize;
let list_size = test_case["list_size"].as_i64().unwrap() as usize;
let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize;
let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap();
let seed_string = test_case["seed"].clone().into_string().unwrap();
let seed = hex::decode(seed_string.replace("0x", "")).unwrap();
let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) {
shuffle_round_count as u8
} else {
panic!("shuffle_round_count must be a u8")
};
assert_eq!(
Some(permutated_index),
get_permutated_index(index, list_size, &seed[..], shuffle_round_count),
"Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}",
i,
index,
list_size,
shuffle_round_count,
seed_string,
);
}
}
}
pub use get_permutated_index::get_permutated_index;
pub use shuffle_list::shuffle_list;

178
eth2/utils/swap_or_not_shuffle/src/shuffle_list.rs Normal file
View File

@@ -0,0 +1,178 @@
use bytes::Buf;
use hashing::hash;
use int_to_bytes::int_to_bytes4;
use std::io::Cursor;
const SEED_SIZE: usize = 32;
const ROUND_SIZE: usize = 1;
const POSITION_WINDOW_SIZE: usize = 4;
const PIVOT_VIEW_SIZE: usize = SEED_SIZE + ROUND_SIZE;
const TOTAL_SIZE: usize = SEED_SIZE + ROUND_SIZE + POSITION_WINDOW_SIZE;
/// Shuffles an entire list in-place.
///
/// Note: this is equivalent to the `get_permutated_index` function, except it shuffles an entire
/// list not just a single index. With large lists this function has been observed to be 250x
/// faster than running `get_permutated_index` across an entire list.
///
/// Credits to [@protolambda](https://github.com/protolambda) for defining this algorithm.
///
/// Shuffles if `forwards == true`, otherwise un-shuffles.
///
/// Returns `None` under any of the following conditions:
/// - `list_size == 0`
/// - `list_size > 2**24`
/// - `list_size > usize::max_value() / 2`
pub fn shuffle_list(
mut input: Vec<usize>,
rounds: u8,
seed: &[u8],
forwards: bool,
) -> Option<Vec<usize>> {
let list_size = input.len();
if input.is_empty()
|| list_size > usize::max_value() / 2
|| list_size > 2_usize.pow(24)
|| rounds == 0
{
return None;
}
let mut buf: Vec<u8> = Vec::with_capacity(TOTAL_SIZE);
let mut r = if forwards { 0 } else { rounds - 1 };
buf.extend_from_slice(seed);
loop {
buf.splice(SEED_SIZE.., vec![r]);
let pivot = bytes_to_int64(&hash(&buf[0..PIVOT_VIEW_SIZE])[0..8]) as usize % list_size;
let mirror = (pivot + 1) >> 1;
buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((pivot >> 8) as u32));
let mut source = hash(&buf[..]);
let mut byte_v = source[(pivot & 0xff) >> 3];
for i in 0..mirror {
let j = pivot - i;
if j & 0xff == 0xff {
buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32));
source = hash(&buf[..]);
}
if j & 0x07 == 0x07 {
byte_v = source[(j & 0xff) >> 3];
}
let bit_v = (byte_v >> (j & 0x07)) & 0x01;
if bit_v == 1 {
input.swap(i, j);
}
}
let mirror = (pivot + list_size + 1) >> 1;
let end = list_size - 1;
buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((end >> 8) as u32));
let mut source = hash(&buf[..]);
let mut byte_v = source[(end & 0xff) >> 3];
for (loop_iter, i) in ((pivot + 1)..mirror).enumerate() {
let j = end - loop_iter;
if j & 0xff == 0xff {
buf.splice(PIVOT_VIEW_SIZE.., int_to_bytes4((j >> 8) as u32));
source = hash(&buf[..]);
}
if j & 0x07 == 0x07 {
byte_v = source[(j & 0xff) >> 3];
}
let bit_v = (byte_v >> (j & 0x07)) & 0x01;
if bit_v == 1 {
input.swap(i, j);
}
}
if forwards {
r += 1;
if r == rounds {
break;
}
} else {
if r == 0 {
break;
}
r -= 1;
}
}
Some(input)
}
fn bytes_to_int64(bytes: &[u8]) -> u64 {
let mut cursor = Cursor::new(bytes);
cursor.get_u64_le()
}
#[cfg(test)]
mod tests {
use super::*;
use hex;
use std::{fs::File, io::prelude::*, path::PathBuf};
use yaml_rust::yaml;
#[test]
fn returns_none_for_zero_length_list() {
assert_eq!(None, shuffle_list(vec![], 90, &[42, 42], true));
}
#[test]
fn test_vectors() {
let mut file = {
let mut file_path_buf = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
file_path_buf.push("src/specs/test_vector_permutated_index.yml");
File::open(file_path_buf).unwrap()
};
let mut yaml_str = String::new();
file.read_to_string(&mut yaml_str).unwrap();
let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap();
let doc = &docs[0];
let test_cases = doc["test_cases"].as_vec().unwrap();
for (i, test_case) in test_cases.iter().enumerate() {
let index = test_case["index"].as_i64().unwrap() as usize;
let list_size = test_case["list_size"].as_i64().unwrap() as usize;
let permutated_index = test_case["permutated_index"].as_i64().unwrap() as usize;
let shuffle_round_count = test_case["shuffle_round_count"].as_i64().unwrap();
let seed_string = test_case["seed"].clone().into_string().unwrap();
let seed = hex::decode(seed_string.replace("0x", "")).unwrap();
let shuffle_round_count = if shuffle_round_count < (u8::max_value() as i64) {
shuffle_round_count as u8
} else {
panic!("shuffle_round_count must be a u8")
};
let list: Vec<usize> = (0..list_size).collect();
let shuffled =
shuffle_list(list.clone(), shuffle_round_count, &seed[..], true).unwrap();
assert_eq!(
list[index], shuffled[permutated_index],
"Failure on case #{} index: {}, list_size: {}, round_count: {}, seed: {}",
i, index, list_size, shuffle_round_count, seed_string
);
}
}
}

13
eth2/utils/test_random_derive/Cargo.toml Normal file
View File

@@ -0,0 +1,13 @@
[package]
name = "test_random_derive"
version = "0.1.0"
authors = ["thojest <thojest@gmail.com>"]
edition = "2018"
description = "Procedural derive macros for implementation of TestRandom trait"
[lib]
proc-macro = true
[dependencies]
syn = "0.15"
quote = "0.6"

43
eth2/utils/test_random_derive/src/lib.rs Normal file
View File

@@ -0,0 +1,43 @@
extern crate proc_macro;
use crate::proc_macro::TokenStream;
use quote::quote;
use syn::{parse_macro_input, DeriveInput};
#[proc_macro_derive(TestRandom)]
pub fn test_random_derive(input: TokenStream) -> TokenStream {
let derived_input = parse_macro_input!(input as DeriveInput);
let name = &derived_input.ident;
let struct_data = match &derived_input.data {
syn::Data::Struct(s) => s,
_ => panic!("test_random_derive only supports structs."),
};
let field_names = get_named_field_idents(&struct_data);
let output = quote! {
impl<T: RngCore> TestRandom<T> for #name {
fn random_for_test(rng: &mut T) -> Self {
Self {
#(
#field_names: <_>::random_for_test(rng),
)*
}
}
}
};
output.into()
}
fn get_named_field_idents(struct_data: &syn::DataStruct) -> Vec<(&syn::Ident)> {
struct_data
.fields
.iter()
.map(|f| match &f.ident {
Some(ref ident) => ident,
_ => panic!("test_random_derive only supports named struct fields."),
})
.collect()
}