gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-07 08:52:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Delete crates!

Browse Source
This commit is contained in:
Michael Sproul
2023-01-16 17:21:03 +11:00
parent dd9ee38084
commit 7cae5d99d7
51 changed files with 0 additions and 9385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
consensus/serde_utils/Cargo.toml
View File

@@ -1,14 +0,0 @@
[package]
name = "eth2_serde_utils"
version = "0.1.1"
authors = ["Paul Hauner <paul@paulhauner.com", "Michael Sproul <michael@sigmaprime.io>"]
edition = "2021"
description = "Serialization and deserialization utilities useful for JSON representations of Ethereum 2.0 types."
license = "Apache-2.0"
[dependencies]
serde = { version = "1.0.116", features = ["derive"] }
serde_derive = "1.0.116"
serde_json = "1.0.58"
hex = "0.4.2"
ethereum-types = "0.14.1"

52
consensus/serde_utils/src/fixed_bytes_hex.rs
View File

@@ -1,52 +0,0 @@
//! Formats `[u8; n]` as a 0x-prefixed hex string.
//!
//! E.g., `[0, 1, 2, 3]` serializes as `"0x00010203"`.
use crate::hex::PrefixedHexVisitor;
use serde::de::Error;
use serde::{Deserializer, Serializer};
macro_rules! bytes_hex {
($num_bytes: tt) => {
use super::*;
const BYTES_LEN: usize = $num_bytes;
pub fn serialize<S>(bytes: &[u8; BYTES_LEN], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut hex_string: String = "0x".to_string();
hex_string.push_str(&hex::encode(&bytes));
serializer.serialize_str(&hex_string)
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<[u8; BYTES_LEN], D::Error>
where
D: Deserializer<'de>,
{
let decoded = deserializer.deserialize_str(PrefixedHexVisitor)?;
if decoded.len() != BYTES_LEN {
return Err(D::Error::custom(format!(
"expected {} bytes for array, got {}",
BYTES_LEN,
decoded.len()
)));
}
let mut array = [0; BYTES_LEN];
array.copy_from_slice(&decoded);
Ok(array)
}
};
}
pub mod bytes_4_hex {
bytes_hex!(4);
}
pub mod bytes_8_hex {
bytes_hex!(8);
}

77
consensus/serde_utils/src/hex.rs
View File

@@ -1,77 +0,0 @@
//! Provides utilities for parsing 0x-prefixed hex strings.
use serde::de::{self, Visitor};
use std::fmt;
/// Encode `data` as a 0x-prefixed hex string.
pub fn encode<T: AsRef<[u8]>>(data: T) -> String {
let hex = hex::encode(data);
let mut s = "0x".to_string();
s.push_str(hex.as_str());
s
}
/// Decode `data` from a 0x-prefixed hex string.
pub fn decode(s: &str) -> Result<Vec<u8>, String> {
if let Some(stripped) = s.strip_prefix("0x") {
hex::decode(stripped).map_err(|e| format!("invalid hex: {:?}", e))
} else {
Err("hex must have 0x prefix".to_string())
}
}
pub struct PrefixedHexVisitor;
impl<'de> Visitor<'de> for PrefixedHexVisitor {
type Value = Vec<u8>;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a hex string with 0x prefix")
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
decode(value).map_err(de::Error::custom)
}
}
pub struct HexVisitor;
impl<'de> Visitor<'de> for HexVisitor {
type Value = Vec<u8>;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a hex string (irrelevant of prefix)")
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
hex::decode(value.trim_start_matches("0x"))
.map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn encoding() {
let bytes = vec![0, 255];
let hex = encode(bytes);
assert_eq!(hex.as_str(), "0x00ff");
let bytes = vec![];
let hex = encode(bytes);
assert_eq!(hex.as_str(), "0x");
let bytes = vec![1, 2, 3];
let hex = encode(bytes);
assert_eq!(hex.as_str(), "0x010203");
}
}

23
consensus/serde_utils/src/hex_vec.rs
View File

@@ -1,23 +0,0 @@
//! Formats `Vec<u8>` as a 0x-prefixed hex string.
//!
//! E.g., `vec![0, 1, 2, 3]` serializes as `"0x00010203"`.
use crate::hex::PrefixedHexVisitor;
use serde::{Deserializer, Serializer};
pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut hex_string: String = "0x".to_string();
hex_string.push_str(&hex::encode(bytes));
serializer.serialize_str(&hex_string)
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_str(PrefixedHexVisitor)
}

25
consensus/serde_utils/src/json_str.rs
View File

@@ -1,25 +0,0 @@
//! Serialize a datatype as a JSON-blob within a single string.
use serde::{
de::{DeserializeOwned, Error as _},
ser::Error as _,
Deserialize, Deserializer, Serialize, Serializer,
};
/// Serialize as a JSON object within a string.
pub fn serialize<T, S>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
T: Serialize,
{
serializer.serialize_str(&serde_json::to_string(value).map_err(S::Error::custom)?)
}
/// Deserialize a JSON object embedded in a string.
pub fn deserialize<'de, T, D>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: DeserializeOwned,
{
let json_str = String::deserialize(deserializer)?;
serde_json::from_str(&json_str).map_err(D::Error::custom)
}

15
consensus/serde_utils/src/lib.rs
View File

@@ -1,15 +0,0 @@
mod quoted_int;
pub mod fixed_bytes_hex;
pub mod hex;
pub mod hex_vec;
pub mod json_str;
pub mod list_of_bytes_lists;
pub mod quoted_u64_vec;
pub mod u256_hex_be;
pub mod u32_hex;
pub mod u64_hex_be;
pub mod u8_hex;
pub use fixed_bytes_hex::{bytes_4_hex, bytes_8_hex};
pub use quoted_int::{quoted_u256, quoted_u32, quoted_u64, quoted_u8};

49
consensus/serde_utils/src/list_of_bytes_lists.rs
View File

@@ -1,49 +0,0 @@
//! Formats `Vec<u64>` using quotes.
//!
//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`.
//!
//! Quotes can be optional during decoding.
use crate::hex;
use serde::ser::SerializeSeq;
use serde::{de, Deserializer, Serializer};
pub struct ListOfBytesListVisitor;
impl<'a> serde::de::Visitor<'a> for ListOfBytesListVisitor {
type Value = Vec<Vec<u8>>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a list of 0x-prefixed byte lists")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let mut vec = vec![];
while let Some(val) = seq.next_element::<String>()? {
vec.push(hex::decode(&val).map_err(de::Error::custom)?);
}
Ok(vec)
}
}
pub fn serialize<S>(value: &[Vec<u8>], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(value.len()))?;
for val in value {
seq.serialize_element(&hex::encode(val))?;
}
seq.end()
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<Vec<u8>>, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(ListOfBytesListVisitor)
}

219
consensus/serde_utils/src/quoted_int.rs
View File

@@ -1,219 +0,0 @@
//! Formats some integer types using quotes.
//!
//! E.g., `1` serializes as `"1"`.
//!
//! Quotes can be optional during decoding.
use ethereum_types::U256;
use serde::{Deserializer, Serializer};
use serde_derive::{Deserialize, Serialize};
use std::convert::TryFrom;
use std::marker::PhantomData;
macro_rules! define_mod {
($int: ty, $visit_fn: ident) => {
/// Serde support for deserializing quoted integers.
///
/// Configurable so that quotes are either required or optional.
pub struct QuotedIntVisitor<T> {
require_quotes: bool,
_phantom: PhantomData<T>,
}
impl<'a, T> serde::de::Visitor<'a> for QuotedIntVisitor<T>
where
T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
{
type Value = T;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
if self.require_quotes {
write!(formatter, "a quoted integer")
} else {
write!(formatter, "a quoted or unquoted integer")
}
}
fn visit_str<E>(self, s: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
s.parse::<$int>()
.map(T::from)
.map_err(serde::de::Error::custom)
}
fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
if self.require_quotes {
Err(serde::de::Error::custom(
"received unquoted integer when quotes are required",
))
} else {
T::try_from(v).map_err(|_| serde::de::Error::custom("invalid integer"))
}
}
}
/// Compositional wrapper type that allows quotes or no quotes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
#[serde(transparent)]
pub struct MaybeQuoted<T>
where
T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
{
#[serde(with = "self")]
pub value: T,
}
/// Wrapper type for requiring quotes on a `$int`-like type.
///
/// Unlike using `serde(with = "quoted_$int::require_quotes")` this is composable, and can be nested
/// inside types like `Option`, `Result` and `Vec`.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
#[serde(transparent)]
pub struct Quoted<T>
where
T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
{
#[serde(with = "require_quotes")]
pub value: T,
}
/// Serialize with quotes.
pub fn serialize<S, T>(value: &T, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
T: From<$int> + Into<$int> + Copy,
{
let v: $int = (*value).into();
serializer.serialize_str(&format!("{}", v))
}
/// Deserialize with or without quotes.
pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
{
deserializer.deserialize_any(QuotedIntVisitor {
require_quotes: false,
_phantom: PhantomData,
})
}
/// Requires quotes when deserializing.
///
/// Usage: `#[serde(with = "quoted_u64::require_quotes")]`.
pub mod require_quotes {
pub use super::serialize;
use super::*;
pub fn deserialize<'de, D, T>(deserializer: D) -> Result<T, D::Error>
where
D: Deserializer<'de>,
T: From<$int> + Into<$int> + Copy + TryFrom<u64>,
{
deserializer.deserialize_any(QuotedIntVisitor {
require_quotes: true,
_phantom: PhantomData,
})
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn require_quotes() {
let x = serde_json::from_str::<Quoted<$int>>("\"8\"").unwrap();
assert_eq!(x.value, 8);
serde_json::from_str::<Quoted<$int>>("8").unwrap_err();
}
}
};
}
pub mod quoted_u8 {
use super::*;
define_mod!(u8, visit_u8);
}
pub mod quoted_u32 {
use super::*;
define_mod!(u32, visit_u32);
}
pub mod quoted_u64 {
use super::*;
define_mod!(u64, visit_u64);
}
pub mod quoted_u256 {
use super::*;
struct U256Visitor;
impl<'de> serde::de::Visitor<'de> for U256Visitor {
type Value = U256;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a quoted U256 integer")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
U256::from_dec_str(v).map_err(serde::de::Error::custom)
}
}
/// Serialize with quotes.
pub fn serialize<S>(value: &U256, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(&format!("{}", value))
}
/// Deserialize with quotes.
pub fn deserialize<'de, D>(deserializer: D) -> Result<U256, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_str(U256Visitor)
}
}
#[cfg(test)]
mod test {
use super::*;
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
struct WrappedU256(#[serde(with = "quoted_u256")] U256);
#[test]
fn u256_with_quotes() {
assert_eq!(
&serde_json::to_string(&WrappedU256(U256::one())).unwrap(),
"\"1\""
);
assert_eq!(
serde_json::from_str::<WrappedU256>("\"1\"").unwrap(),
WrappedU256(U256::one())
);
}
#[test]
fn u256_without_quotes() {
serde_json::from_str::<WrappedU256>("1").unwrap_err();
}
}

97
consensus/serde_utils/src/quoted_u64_vec.rs
View File

@@ -1,97 +0,0 @@
//! Formats `Vec<u64>` using quotes.
//!
//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`.
//!
//! Quotes can be optional during decoding.
use serde::ser::SerializeSeq;
use serde::{Deserializer, Serializer};
use serde_derive::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
#[serde(transparent)]
pub struct QuotedIntWrapper {
#[serde(with = "crate::quoted_u64")]
pub int: u64,
}
pub struct QuotedIntVecVisitor;
impl<'a> serde::de::Visitor<'a> for QuotedIntVecVisitor {
type Value = Vec<u64>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a list of quoted or unquoted integers")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let mut vec = vec![];
while let Some(val) = seq.next_element()? {
let val: QuotedIntWrapper = val;
vec.push(val.int);
}
Ok(vec)
}
}
pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(value.len()))?;
for &int in value {
seq.serialize_element(&QuotedIntWrapper { int })?;
}
seq.end()
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u64>, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_any(QuotedIntVecVisitor)
}
#[cfg(test)]
mod test {
use super::*;
#[derive(Debug, Serialize, Deserialize)]
struct Obj {
#[serde(with = "crate::quoted_u64_vec")]
values: Vec<u64>,
}
#[test]
fn quoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
assert_eq!(obj.values, vec![1, 2, 3, 4]);
}
#[test]
fn unquoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
assert_eq!(obj.values, vec![1, 2, 3, 4]);
}
#[test]
fn mixed_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
assert_eq!(obj.values, vec![1, 2, 3, 4]);
}
#[test]
fn empty_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [] }"#).unwrap();
assert!(obj.values.is_empty());
}
#[test]
fn whole_list_quoted_err() {
serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
}
}

144
consensus/serde_utils/src/u256_hex_be.rs
View File

@@ -1,144 +0,0 @@
use ethereum_types::U256;
use serde::de::Visitor;
use serde::{de, Deserializer, Serialize, Serializer};
use std::fmt;
use std::str::FromStr;
pub fn serialize<S>(num: &U256, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
num.serialize(serializer)
}
pub struct U256Visitor;
impl<'de> Visitor<'de> for U256Visitor {
type Value = String;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a well formatted hex string")
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
if !value.starts_with("0x") {
return Err(de::Error::custom("must start with 0x"));
}
let stripped = &value[2..];
if stripped.is_empty() {
Err(de::Error::custom(format!(
"quantity cannot be {:?}",
stripped
)))
} else if stripped == "0" {
Ok(value.to_string())
} else if stripped.starts_with('0') {
Err(de::Error::custom("cannot have leading zero"))
} else {
Ok(value.to_string())
}
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<U256, D::Error>
where
D: Deserializer<'de>,
{
let decoded = deserializer.deserialize_string(U256Visitor)?;
U256::from_str(&decoded).map_err(|e| de::Error::custom(format!("Invalid U256 string: {}", e)))
}
#[cfg(test)]
mod test {
use ethereum_types::U256;
use serde::{Deserialize, Serialize};
use serde_json;
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
struct Wrapper {
#[serde(with = "super")]
val: U256,
}
#[test]
fn encoding() {
assert_eq!(
&serde_json::to_string(&Wrapper { val: 0.into() }).unwrap(),
"\"0x0\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 1.into() }).unwrap(),
"\"0x1\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 256.into() }).unwrap(),
"\"0x100\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 65.into() }).unwrap(),
"\"0x41\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 1024.into() }).unwrap(),
"\"0x400\""
);
assert_eq!(
&serde_json::to_string(&Wrapper {
val: U256::max_value() - 1
})
.unwrap(),
"\"0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe\""
);
assert_eq!(
&serde_json::to_string(&Wrapper {
val: U256::max_value()
})
.unwrap(),
"\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\""
);
}
#[test]
fn decoding() {
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x0\"").unwrap(),
Wrapper { val: 0.into() },
);
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x41\"").unwrap(),
Wrapper { val: 65.into() },
);
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x400\"").unwrap(),
Wrapper { val: 1024.into() },
);
assert_eq!(
serde_json::from_str::<Wrapper>(
"\"0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe\""
)
.unwrap(),
Wrapper {
val: U256::max_value() - 1
},
);
assert_eq!(
serde_json::from_str::<Wrapper>(
"\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\""
)
.unwrap(),
Wrapper {
val: U256::max_value()
},
);
serde_json::from_str::<Wrapper>("\"0x\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"0x0400\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"400\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"ff\"").unwrap_err();
}
}

21
consensus/serde_utils/src/u32_hex.rs
View File

@@ -1,21 +0,0 @@
//! Formats `u32` as a 0x-prefixed, little-endian hex string.
//!
//! E.g., `0` serializes as `"0x00000000"`.
use crate::bytes_4_hex;
use serde::{Deserializer, Serializer};
pub fn serialize<S>(num: &u32, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let hex = format!("0x{}", hex::encode(num.to_le_bytes()));
serializer.serialize_str(&hex)
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<u32, D::Error>
where
D: Deserializer<'de>,
{
bytes_4_hex::deserialize(deserializer).map(u32::from_le_bytes)
}

134
consensus/serde_utils/src/u64_hex_be.rs
View File

@@ -1,134 +0,0 @@
//! Formats `u64` as a 0x-prefixed, big-endian hex string.
//!
//! E.g., `0` serializes as `"0x0000000000000000"`.
use serde::de::{self, Error, Visitor};
use serde::{Deserializer, Serializer};
use std::fmt;
const BYTES_LEN: usize = 8;
pub struct QuantityVisitor;
impl<'de> Visitor<'de> for QuantityVisitor {
type Value = Vec<u8>;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a hex string")
}
fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
if !value.starts_with("0x") {
return Err(de::Error::custom("must start with 0x"));
}
let stripped = value.trim_start_matches("0x");
if stripped.is_empty() {
Err(de::Error::custom(format!(
"quantity cannot be {}",
stripped
)))
} else if stripped == "0" {
Ok(vec![0])
} else if stripped.starts_with('0') {
Err(de::Error::custom("cannot have leading zero"))
} else if stripped.len() % 2 != 0 {
hex::decode(format!("0{}", stripped))
.map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
} else {
hex::decode(stripped).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))
}
}
}
pub fn serialize<S>(num: &u64, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let raw = hex::encode(num.to_be_bytes());
let trimmed = raw.trim_start_matches('0');
let hex = if trimmed.is_empty() { "0" } else { trimmed };
serializer.serialize_str(&format!("0x{}", &hex))
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<u64, D::Error>
where
D: Deserializer<'de>,
{
let decoded = deserializer.deserialize_str(QuantityVisitor)?;
// TODO: this is not strict about byte length like other methods.
if decoded.len() > BYTES_LEN {
return Err(D::Error::custom(format!(
"expected max {} bytes for array, got {}",
BYTES_LEN,
decoded.len()
)));
}
let mut array = [0; BYTES_LEN];
array[BYTES_LEN - decoded.len()..].copy_from_slice(&decoded);
Ok(u64::from_be_bytes(array))
}
#[cfg(test)]
mod test {
use serde::{Deserialize, Serialize};
use serde_json;
#[derive(Debug, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
struct Wrapper {
#[serde(with = "super")]
val: u64,
}
#[test]
fn encoding() {
assert_eq!(
&serde_json::to_string(&Wrapper { val: 0 }).unwrap(),
"\"0x0\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 1 }).unwrap(),
"\"0x1\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 256 }).unwrap(),
"\"0x100\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 65 }).unwrap(),
"\"0x41\""
);
assert_eq!(
&serde_json::to_string(&Wrapper { val: 1024 }).unwrap(),
"\"0x400\""
);
}
#[test]
fn decoding() {
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x0\"").unwrap(),
Wrapper { val: 0 },
);
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x41\"").unwrap(),
Wrapper { val: 65 },
);
assert_eq!(
serde_json::from_str::<Wrapper>("\"0x400\"").unwrap(),
Wrapper { val: 1024 },
);
serde_json::from_str::<Wrapper>("\"0x\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"0x0400\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"400\"").unwrap_err();
serde_json::from_str::<Wrapper>("\"ff\"").unwrap_err();
}
}

29
consensus/serde_utils/src/u8_hex.rs
View File

@@ -1,29 +0,0 @@
//! Formats `u8` as a 0x-prefixed hex string.
//!
//! E.g., `0` serializes as `"0x00"`.
use crate::hex::PrefixedHexVisitor;
use serde::de::Error;
use serde::{Deserializer, Serializer};
pub fn serialize<S>(byte: &u8, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let hex = format!("0x{}", hex::encode([*byte]));
serializer.serialize_str(&hex)
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<u8, D::Error>
where
D: Deserializer<'de>,
{
let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
if bytes.len() != 1 {
return Err(D::Error::custom(format!(
"expected 1 byte for u8, got {}",
bytes.len()
)));
}
Ok(bytes[0])
}

21
consensus/ssz/Cargo.toml
View File

@@ -1,21 +0,0 @@
[package]
name = "eth2_ssz"
version = "0.4.1"
authors = ["Paul Hauner <paul@sigmaprime.io>"]
edition = "2021"
description = "SimpleSerialize (SSZ) as used in Ethereum 2.0"
license = "Apache-2.0"
[lib]
name = "ssz"
[dev-dependencies]
ethereum_ssz_derive = "1.0.0-beta.2"
[dependencies]
ethereum-types = "0.14.1"
smallvec = { version = "1.6.1", features = ["const_generics"] }
itertools = "0.10.3"
[features]
arbitrary = ["ethereum-types/arbitrary"]

3
consensus/ssz/README.md
View File

@@ -1,3 +0,0 @@
# simpleserialize (ssz)
[<img src="https://img.shields.io/crates/v/eth2_ssz">](https://crates.io/crates/eth2_ssz)

15
consensus/ssz/examples/large_list.rs
View File

@@ -1,15 +0,0 @@
//! Encode and decode a list many times.
//!
//! Useful for `cargo flamegraph`.
use ssz::{Decode, Encode};
fn main() {
let vec: Vec<u64> = vec![4242; 8196];
let output: Vec<Vec<u64>> = (0..40_000)
.map(|_| Vec::from_ssz_bytes(&vec.as_ssz_bytes()).unwrap())
.collect();
println!("{}", output.len());
}

31
consensus/ssz/examples/large_list_of_structs.rs
View File

@@ -1,31 +0,0 @@
//! Encode and decode a list many times.
//!
//! Useful for `cargo flamegraph`.
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
#[derive(Clone, Copy, Encode, Decode)]
pub struct FixedLen {
a: u64,
b: u64,
c: u64,
d: u64,
}
fn main() {
let fixed_len = FixedLen {
a: 42,
b: 42,
c: 42,
d: 42,
};
let vec: Vec<FixedLen> = vec![fixed_len; 8196];
let output: Vec<Vec<u64>> = (0..40_000)
.map(|_| Vec::from_ssz_bytes(&vec.as_ssz_bytes()).unwrap())
.collect();
println!("{}", output.len());
}

73
consensus/ssz/examples/struct_definition.rs
View File

@@ -1,73 +0,0 @@
use ssz::{Decode, DecodeError, Encode, SszDecoderBuilder, SszEncoder};
#[derive(Debug, PartialEq)]
pub struct Foo {
a: u16,
b: Vec<u8>,
c: u16,
}
impl Encode for Foo {
fn is_ssz_fixed_len() -> bool {
<u16 as Encode>::is_ssz_fixed_len() && <Vec<u16> as Encode>::is_ssz_fixed_len()
}
fn ssz_bytes_len(&self) -> usize {
<u16 as Encode>::ssz_fixed_len()
+ ssz::BYTES_PER_LENGTH_OFFSET
+ <u16 as Encode>::ssz_fixed_len()
+ self.b.ssz_bytes_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
let offset = <u16 as Encode>::ssz_fixed_len()
+ <Vec<u16> as Encode>::ssz_fixed_len()
+ <u16 as Encode>::ssz_fixed_len();
let mut encoder = SszEncoder::container(buf, offset);
encoder.append(&self.a);
encoder.append(&self.b);
encoder.append(&self.c);
encoder.finalize();
}
}
impl Decode for Foo {
fn is_ssz_fixed_len() -> bool {
<u16 as Decode>::is_ssz_fixed_len() && <Vec<u16> as Decode>::is_ssz_fixed_len()
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let mut builder = SszDecoderBuilder::new(bytes);
builder.register_type::<u16>()?;
builder.register_type::<Vec<u8>>()?;
builder.register_type::<u16>()?;
let mut decoder = builder.build()?;
Ok(Self {
a: decoder.decode_next()?,
b: decoder.decode_next()?,
c: decoder.decode_next()?,
})
}
}
fn main() {
let my_foo = Foo {
a: 42,
b: vec![0, 1, 2, 3],
c: 11,
};
let bytes = vec![42, 0, 8, 0, 0, 0, 11, 0, 0, 1, 2, 3];
assert_eq!(my_foo.as_ssz_bytes(), bytes);
let decoded_foo = Foo::from_ssz_bytes(&bytes).unwrap();
assert_eq!(my_foo, decoded_foo);
}

374
consensus/ssz/src/decode.rs
View File

@@ -1,374 +0,0 @@
use super::*;
use smallvec::{smallvec, SmallVec};
use std::cmp::Ordering;
type SmallVec8<T> = SmallVec<[T; 8]>;
pub mod impls;
pub mod try_from_iter;
/// Returned when SSZ decoding fails.
#[derive(Debug, PartialEq, Clone)]
pub enum DecodeError {
/// The bytes supplied were too short to be decoded into the specified type.
InvalidByteLength { len: usize, expected: usize },
/// The given bytes were too short to be read as a length prefix.
InvalidLengthPrefix { len: usize, expected: usize },
/// A length offset pointed to a byte that was out-of-bounds (OOB).
///
/// A bytes may be OOB for the following reasons:
///
/// - It is `>= bytes.len()`.
/// - When decoding variable length items, the 1st offset points "backwards" into the fixed
/// length items (i.e., `length[0] < BYTES_PER_LENGTH_OFFSET`).
/// - When decoding variable-length items, the `n`'th offset was less than the `n-1`'th offset.
OutOfBoundsByte { i: usize },
/// An offset points “backwards” into the fixed-bytes portion of the message, essentially
/// double-decoding bytes that will also be decoded as fixed-length.
///
/// https://notes.ethereum.org/ruKvDXl6QOW3gnqVYb8ezA?view#1-Offset-into-fixed-portion
OffsetIntoFixedPortion(usize),
/// The first offset does not point to the byte that follows the fixed byte portion,
/// essentially skipping a variable-length byte.
///
/// https://notes.ethereum.org/ruKvDXl6QOW3gnqVYb8ezA?view#2-Skip-first-variable-byte
OffsetSkipsVariableBytes(usize),
/// An offset points to bytes prior to the previous offset. Depending on how you look at it,
/// this either double-decodes bytes or makes the first offset a negative-length.
///
/// https://notes.ethereum.org/ruKvDXl6QOW3gnqVYb8ezA?view#3-Offsets-are-decreasing
OffsetsAreDecreasing(usize),
/// An offset references byte indices that do not exist in the source bytes.
///
/// https://notes.ethereum.org/ruKvDXl6QOW3gnqVYb8ezA?view#4-Offsets-are-out-of-bounds
OffsetOutOfBounds(usize),
/// A variable-length list does not have a fixed portion that is cleanly divisible by
/// `BYTES_PER_LENGTH_OFFSET`.
InvalidListFixedBytesLen(usize),
/// Some item has a `ssz_fixed_len` of zero. This is illegal.
ZeroLengthItem,
/// The given bytes were invalid for some application-level reason.
BytesInvalid(String),
/// The given union selector is out of bounds.
UnionSelectorInvalid(u8),
}
/// Performs checks on the `offset` based upon the other parameters provided.
///
/// ## Detail
///
/// - `offset`: the offset bytes (e.g., result of `read_offset(..)`).
/// - `previous_offset`: unless this is the first offset in the SSZ object, the value of the
/// previously-read offset. Used to ensure offsets are not decreasing.
/// - `num_bytes`: the total number of bytes in the SSZ object. Used to ensure the offset is not
/// out of bounds.
/// - `num_fixed_bytes`: the number of fixed-bytes in the struct, if it is known. Used to ensure
/// that the first offset doesn't skip any variable bytes.
///
/// ## References
///
/// The checks here are derived from this document:
///
/// https://notes.ethereum.org/ruKvDXl6QOW3gnqVYb8ezA?view
pub fn sanitize_offset(
offset: usize,
previous_offset: Option<usize>,
num_bytes: usize,
num_fixed_bytes: Option<usize>,
) -> Result<usize, DecodeError> {
if num_fixed_bytes.map_or(false, |fixed_bytes| offset < fixed_bytes) {
Err(DecodeError::OffsetIntoFixedPortion(offset))
} else if previous_offset.is_none()
&& num_fixed_bytes.map_or(false, |fixed_bytes| offset != fixed_bytes)
{
Err(DecodeError::OffsetSkipsVariableBytes(offset))
} else if offset > num_bytes {
Err(DecodeError::OffsetOutOfBounds(offset))
} else if previous_offset.map_or(false, |prev| prev > offset) {
Err(DecodeError::OffsetsAreDecreasing(offset))
} else {
Ok(offset)
}
}
/// Provides SSZ decoding (de-serialization) via the `from_ssz_bytes(&bytes)` method.
///
/// See `examples/` for manual implementations or the crate root for implementations using
/// `#[derive(Decode)]`.
pub trait Decode: Sized {
/// Returns `true` if this object has a fixed-length.
///
/// I.e., there are no variable length items in this object or any of it's contained objects.
fn is_ssz_fixed_len() -> bool;
/// The number of bytes this object occupies in the fixed-length portion of the SSZ bytes.
///
/// By default, this is set to `BYTES_PER_LENGTH_OFFSET` which is suitable for variable length
/// objects, but not fixed-length objects. Fixed-length objects _must_ return a value which
/// represents their length.
fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
/// Attempts to decode `Self` from `bytes`, returning a `DecodeError` on failure.
///
/// The supplied bytes must be the exact length required to decode `Self`, excess bytes will
/// result in an error.
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError>;
}
#[derive(Copy, Clone, Debug)]
pub struct Offset {
position: usize,
offset: usize,
}
/// Builds an `SszDecoder`.
///
/// The purpose of this struct is to split some SSZ bytes into individual slices. The builder is
/// then converted into a `SszDecoder` which decodes those values into object instances.
///
/// See [`SszDecoder`](struct.SszDecoder.html) for usage examples.
pub struct SszDecoderBuilder<'a> {
bytes: &'a [u8],
items: SmallVec8<&'a [u8]>,
offsets: SmallVec8<Offset>,
items_index: usize,
}
impl<'a> SszDecoderBuilder<'a> {
/// Instantiate a new builder that should build a `SszDecoder` over the given `bytes` which
/// are assumed to be the SSZ encoding of some object.
pub fn new(bytes: &'a [u8]) -> Self {
Self {
bytes,
items: smallvec![],
offsets: smallvec![],
items_index: 0,
}
}
/// Registers a variable-length object as the next item in `bytes`, without specifying the
/// actual type.
///
/// ## Notes
///
/// Use of this function is generally discouraged since it cannot detect if some type changes
/// from variable to fixed length.
///
/// Use `Self::register_type` wherever possible.
pub fn register_anonymous_variable_length_item(&mut self) -> Result<(), DecodeError> {
struct Anonymous;
impl Decode for Anonymous {
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(_bytes: &[u8]) -> Result<Self, DecodeError> {
unreachable!("Anonymous should never be decoded")
}
}
self.register_type::<Anonymous>()
}
/// Declares that some type `T` is the next item in `bytes`.
pub fn register_type<T: Decode>(&mut self) -> Result<(), DecodeError> {
self.register_type_parameterized(T::is_ssz_fixed_len(), T::ssz_fixed_len())
}
/// Declares that a type with the given parameters is the next item in `bytes`.
pub fn register_type_parameterized(
&mut self,
is_ssz_fixed_len: bool,
ssz_fixed_len: usize,
) -> Result<(), DecodeError> {
if is_ssz_fixed_len {
let start = self.items_index;
self.items_index += ssz_fixed_len;
let slice =
self.bytes
.get(start..self.items_index)
.ok_or(DecodeError::InvalidByteLength {
len: self.bytes.len(),
expected: self.items_index,
})?;
self.items.push(slice);
} else {
self.offsets.push(Offset {
position: self.items.len(),
offset: sanitize_offset(
read_offset(&self.bytes[self.items_index..])?,
self.offsets.last().map(|o| o.offset),
self.bytes.len(),
None,
)?,
});
// Push an empty slice into items; it will be replaced later.
self.items.push(&[]);
self.items_index += BYTES_PER_LENGTH_OFFSET;
}
Ok(())
}
fn finalize(&mut self) -> Result<(), DecodeError> {
if let Some(first_offset) = self.offsets.first().map(|o| o.offset) {
// Check to ensure the first offset points to the byte immediately following the
// fixed-length bytes.
match first_offset.cmp(&self.items_index) {
Ordering::Less => return Err(DecodeError::OffsetIntoFixedPortion(first_offset)),
Ordering::Greater => {
return Err(DecodeError::OffsetSkipsVariableBytes(first_offset))
}
Ordering::Equal => (),
}
// Iterate through each pair of offsets, grabbing the slice between each of the offsets.
for pair in self.offsets.windows(2) {
let a = pair[0];
let b = pair[1];
self.items[a.position] = &self.bytes[a.offset..b.offset];
}
// Handle the last offset, pushing a slice from it's start through to the end of
// `self.bytes`.
if let Some(last) = self.offsets.last() {
self.items[last.position] = &self.bytes[last.offset..]
}
} else {
// If the container is fixed-length, ensure there are no excess bytes.
if self.items_index != self.bytes.len() {
return Err(DecodeError::InvalidByteLength {
len: self.bytes.len(),
expected: self.items_index,
});
}
}
Ok(())
}
/// Finalizes the builder, returning a `SszDecoder` that may be used to instantiate objects.
pub fn build(mut self) -> Result<SszDecoder<'a>, DecodeError> {
self.finalize()?;
Ok(SszDecoder { items: self.items })
}
}
/// Decodes some slices of SSZ into object instances. Should be instantiated using
/// [`SszDecoderBuilder`](struct.SszDecoderBuilder.html).
///
/// ## Example
///
/// ```rust
/// use ssz_derive::{Encode, Decode};
/// use ssz::{Decode, Encode, SszDecoder, SszDecoderBuilder};
///
/// #[derive(PartialEq, Debug, Encode, Decode)]
/// struct Foo {
/// a: u64,
/// b: Vec<u16>,
/// }
///
/// fn ssz_decoding_example() {
/// let foo = Foo {
/// a: 42,
/// b: vec![1, 3, 3, 7]
/// };
///
/// let bytes = foo.as_ssz_bytes();
///
/// let mut builder = SszDecoderBuilder::new(&bytes);
///
/// builder.register_type::<u64>().unwrap();
/// builder.register_type::<Vec<u16>>().unwrap();
///
/// let mut decoder = builder.build().unwrap();
///
/// let decoded_foo = Foo {
/// a: decoder.decode_next().unwrap(),
/// b: decoder.decode_next().unwrap(),
/// };
///
/// assert_eq!(foo, decoded_foo);
/// }
///
/// ```
pub struct SszDecoder<'a> {
items: SmallVec8<&'a [u8]>,
}
impl<'a> SszDecoder<'a> {
/// Decodes the next item.
///
/// # Panics
///
/// Panics when attempting to decode more items than actually exist.
pub fn decode_next<T: Decode>(&mut self) -> Result<T, DecodeError> {
self.decode_next_with(|slice| T::from_ssz_bytes(slice))
}
/// Decodes the next item using the provided function.
pub fn decode_next_with<T, F>(&mut self, f: F) -> Result<T, DecodeError>
where
F: FnOnce(&'a [u8]) -> Result<T, DecodeError>,
{
f(self.items.remove(0))
}
}
/// Takes `bytes`, assuming it is the encoding for a SSZ union, and returns the union-selector and
/// the body (trailing bytes).
///
/// ## Errors
///
/// Returns an error if:
///
/// - `bytes` is empty.
/// - the union selector is not a valid value (i.e., larger than the maximum number of variants.
pub fn split_union_bytes(bytes: &[u8]) -> Result<(UnionSelector, &[u8]), DecodeError> {
let selector = bytes
.first()
.copied()
.ok_or(DecodeError::OutOfBoundsByte { i: 0 })
.and_then(UnionSelector::new)?;
let body = bytes
.get(1..)
.ok_or(DecodeError::OutOfBoundsByte { i: 1 })?;
Ok((selector, body))
}
/// Reads a `BYTES_PER_LENGTH_OFFSET`-byte length from `bytes`, where `bytes.len() >=
/// BYTES_PER_LENGTH_OFFSET`.
pub fn read_offset(bytes: &[u8]) -> Result<usize, DecodeError> {
decode_offset(bytes.get(0..BYTES_PER_LENGTH_OFFSET).ok_or(
DecodeError::InvalidLengthPrefix {
len: bytes.len(),
expected: BYTES_PER_LENGTH_OFFSET,
},
)?)
}
/// Decode bytes as a little-endian usize, returning an `Err` if `bytes.len() !=
/// BYTES_PER_LENGTH_OFFSET`.
fn decode_offset(bytes: &[u8]) -> Result<usize, DecodeError> {
let len = bytes.len();
let expected = BYTES_PER_LENGTH_OFFSET;
if len != expected {
Err(DecodeError::InvalidLengthPrefix { len, expected })
} else {
let mut array: [u8; BYTES_PER_LENGTH_OFFSET] = std::default::Default::default();
array.clone_from_slice(bytes);
Ok(u32::from_le_bytes(array) as usize)
}
}

775
consensus/ssz/src/decode/impls.rs
View File

@@ -1,775 +0,0 @@
use super::*;
use crate::decode::try_from_iter::{TryCollect, TryFromIter};
use core::num::NonZeroUsize;
use ethereum_types::{H160, H256, U128, U256};
use itertools::process_results;
use smallvec::SmallVec;
use std::collections::{BTreeMap, BTreeSet};
use std::iter::{self, FromIterator};
use std::sync::Arc;
macro_rules! impl_decodable_for_uint {
($type: ident, $bit_size: expr) => {
impl Decode for $type {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
$bit_size / 8
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
let mut array: [u8; $bit_size / 8] = std::default::Default::default();
array.clone_from_slice(bytes);
Ok(Self::from_le_bytes(array))
}
}
}
};
}
impl_decodable_for_uint!(u8, 8);
impl_decodable_for_uint!(u16, 16);
impl_decodable_for_uint!(u32, 32);
impl_decodable_for_uint!(u64, 64);
#[cfg(target_pointer_width = "32")]
impl_decodable_for_uint!(usize, 32);
#[cfg(target_pointer_width = "64")]
impl_decodable_for_uint!(usize, 64);
macro_rules! impl_decode_for_tuples {
($(
$Tuple:ident {
$(($idx:tt) -> $T:ident)+
}
)+) => {
$(
impl<$($T: Decode),+> Decode for ($($T,)+) {
fn is_ssz_fixed_len() -> bool {
$(
<$T as Decode>::is_ssz_fixed_len() &&
)*
true
}
fn ssz_fixed_len() -> usize {
if <Self as Decode>::is_ssz_fixed_len() {
$(
<$T as Decode>::ssz_fixed_len() +
)*
0
} else {
BYTES_PER_LENGTH_OFFSET
}
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let mut builder = SszDecoderBuilder::new(bytes);
$(
builder.register_type::<$T>()?;
)*
let mut decoder = builder.build()?;
Ok(($(
decoder.decode_next::<$T>()?,
)*
))
}
}
)+
}
}
impl_decode_for_tuples! {
Tuple2 {
(0) -> A
(1) -> B
}
Tuple3 {
(0) -> A
(1) -> B
(2) -> C
}
Tuple4 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
}
Tuple5 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
}
Tuple6 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
}
Tuple7 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
}
Tuple8 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
}
Tuple9 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
}
Tuple10 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
}
Tuple11 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
(10) -> K
}
Tuple12 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
(10) -> K
(11) -> L
}
}
impl Decode for bool {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
match bytes[0] {
0b0000_0000 => Ok(false),
0b0000_0001 => Ok(true),
_ => Err(DecodeError::BytesInvalid(format!(
"Out-of-range for boolean: {}",
bytes[0]
))),
}
}
}
}
impl Decode for NonZeroUsize {
fn is_ssz_fixed_len() -> bool {
<usize as Decode>::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
<usize as Decode>::ssz_fixed_len()
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let x = usize::from_ssz_bytes(bytes)?;
if x == 0 {
Err(DecodeError::BytesInvalid(
"NonZeroUsize cannot be zero.".to_string(),
))
} else {
// `unwrap` is safe here as `NonZeroUsize::new()` succeeds if `x > 0` and this path
// never executes when `x == 0`.
Ok(NonZeroUsize::new(x).unwrap())
}
}
}
impl<T: Decode> Decode for Option<T> {
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let (selector, body) = split_union_bytes(bytes)?;
match selector.into() {
0u8 => Ok(None),
1u8 => <T as Decode>::from_ssz_bytes(body).map(Option::Some),
other => Err(DecodeError::UnionSelectorInvalid(other)),
}
}
}
impl<T: Decode> Decode for Arc<T> {
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
T::ssz_fixed_len()
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
T::from_ssz_bytes(bytes).map(Arc::new)
}
}
impl Decode for H160 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
20
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
Ok(Self::from_slice(bytes))
}
}
}
impl Decode for H256 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
32
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
Ok(H256::from_slice(bytes))
}
}
}
impl Decode for U256 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
32
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
Ok(U256::from_little_endian(bytes))
}
}
}
impl Decode for U128 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
16
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
Ok(U128::from_little_endian(bytes))
}
}
}
macro_rules! impl_decodable_for_u8_array {
($len: expr) => {
impl Decode for [u8; $len] {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
$len
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
let len = bytes.len();
let expected = <Self as Decode>::ssz_fixed_len();
if len != expected {
Err(DecodeError::InvalidByteLength { len, expected })
} else {
let mut array: [u8; $len] = [0; $len];
array.copy_from_slice(bytes);
Ok(array)
}
}
}
};
}
impl_decodable_for_u8_array!(4);
impl_decodable_for_u8_array!(32);
macro_rules! impl_for_vec {
($type: ty, $max_len: expr) => {
impl<T: Decode> Decode for $type {
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
if bytes.is_empty() {
Ok(Self::from_iter(iter::empty()))
} else if T::is_ssz_fixed_len() {
bytes
.chunks(T::ssz_fixed_len())
.map(T::from_ssz_bytes)
.collect()
} else {
decode_list_of_variable_length_items(bytes, $max_len)
}
}
}
};
}
impl_for_vec!(Vec<T>, None);
impl_for_vec!(SmallVec<[T; 1]>, None);
impl_for_vec!(SmallVec<[T; 2]>, None);
impl_for_vec!(SmallVec<[T; 3]>, None);
impl_for_vec!(SmallVec<[T; 4]>, None);
impl_for_vec!(SmallVec<[T; 5]>, None);
impl_for_vec!(SmallVec<[T; 6]>, None);
impl_for_vec!(SmallVec<[T; 7]>, None);
impl_for_vec!(SmallVec<[T; 8]>, None);
impl<K, V> Decode for BTreeMap<K, V>
where
K: Decode + Ord,
V: Decode,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
if bytes.is_empty() {
Ok(Self::from_iter(iter::empty()))
} else if <(K, V)>::is_ssz_fixed_len() {
bytes
.chunks(<(K, V)>::ssz_fixed_len())
.map(<(K, V)>::from_ssz_bytes)
.collect()
} else {
decode_list_of_variable_length_items(bytes, None)
}
}
}
impl<T> Decode for BTreeSet<T>
where
T: Decode + Ord,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
if bytes.is_empty() {
Ok(Self::from_iter(iter::empty()))
} else if T::is_ssz_fixed_len() {
bytes
.chunks(T::ssz_fixed_len())
.map(T::from_ssz_bytes)
.collect()
} else {
decode_list_of_variable_length_items(bytes, None)
}
}
}
/// Decodes `bytes` as if it were a list of variable-length items.
///
/// The `ssz::SszDecoder` can also perform this functionality, however this function is
/// significantly faster as it is optimized to read same-typed items whilst `ssz::SszDecoder`
/// supports reading items of differing types.
pub fn decode_list_of_variable_length_items<T: Decode, Container: TryFromIter<T>>(
bytes: &[u8],
max_len: Option<usize>,
) -> Result<Container, DecodeError> {
if bytes.is_empty() {
return Container::try_from_iter(iter::empty()).map_err(|e| {
DecodeError::BytesInvalid(format!("Error trying to collect empty list: {:?}", e))
});
}
let first_offset = read_offset(bytes)?;
sanitize_offset(first_offset, None, bytes.len(), Some(first_offset))?;
if first_offset % BYTES_PER_LENGTH_OFFSET != 0 || first_offset < BYTES_PER_LENGTH_OFFSET {
return Err(DecodeError::InvalidListFixedBytesLen(first_offset));
}
let num_items = first_offset / BYTES_PER_LENGTH_OFFSET;
if max_len.map_or(false, |max| num_items > max) {
return Err(DecodeError::BytesInvalid(format!(
"Variable length list of {} items exceeds maximum of {:?}",
num_items, max_len
)));
}
let mut offset = first_offset;
process_results(
(1..=num_items).map(|i| {
let slice_option = if i == num_items {
bytes.get(offset..)
} else {
let start = offset;
let next_offset = read_offset(&bytes[(i * BYTES_PER_LENGTH_OFFSET)..])?;
offset =
sanitize_offset(next_offset, Some(offset), bytes.len(), Some(first_offset))?;
bytes.get(start..offset)
};
let slice = slice_option.ok_or(DecodeError::OutOfBoundsByte { i: offset })?;
T::from_ssz_bytes(slice)
}),
|iter| iter.try_collect(),
)?
.map_err(|e| DecodeError::BytesInvalid(format!("Error collecting into container: {:?}", e)))
}
#[cfg(test)]
mod tests {
use super::*;
// Note: decoding of valid bytes is generally tested "indirectly" in the `/tests` dir, by
// encoding then decoding the element.
#[test]
fn invalid_u8_array_4() {
assert_eq!(
<[u8; 4]>::from_ssz_bytes(&[0; 3]),
Err(DecodeError::InvalidByteLength {
len: 3,
expected: 4
})
);
assert_eq!(
<[u8; 4]>::from_ssz_bytes(&[0; 5]),
Err(DecodeError::InvalidByteLength {
len: 5,
expected: 4
})
);
}
#[test]
fn invalid_bool() {
assert_eq!(
bool::from_ssz_bytes(&[0; 2]),
Err(DecodeError::InvalidByteLength {
len: 2,
expected: 1
})
);
assert_eq!(
bool::from_ssz_bytes(&[]),
Err(DecodeError::InvalidByteLength {
len: 0,
expected: 1
})
);
if let Err(DecodeError::BytesInvalid(_)) = bool::from_ssz_bytes(&[2]) {
// Success.
} else {
panic!("Did not return error on invalid bool val")
}
}
#[test]
fn invalid_h256() {
assert_eq!(
H256::from_ssz_bytes(&[0; 33]),
Err(DecodeError::InvalidByteLength {
len: 33,
expected: 32
})
);
assert_eq!(
H256::from_ssz_bytes(&[0; 31]),
Err(DecodeError::InvalidByteLength {
len: 31,
expected: 32
})
);
}
#[test]
fn empty_list() {
let vec: Vec<Vec<u16>> = vec![];
let bytes = vec.as_ssz_bytes();
assert!(bytes.is_empty());
assert_eq!(Vec::from_ssz_bytes(&bytes), Ok(vec),);
}
#[test]
fn first_length_points_backwards() {
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[0, 0, 0, 0]),
Err(DecodeError::InvalidListFixedBytesLen(0))
);
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[1, 0, 0, 0]),
Err(DecodeError::InvalidListFixedBytesLen(1))
);
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[2, 0, 0, 0]),
Err(DecodeError::InvalidListFixedBytesLen(2))
);
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[3, 0, 0, 0]),
Err(DecodeError::InvalidListFixedBytesLen(3))
);
}
#[test]
fn lengths_are_decreasing() {
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[12, 0, 0, 0, 14, 0, 0, 0, 12, 0, 0, 0, 1, 0, 1, 0]),
Err(DecodeError::OffsetsAreDecreasing(12))
);
}
#[test]
fn awkward_fixed_length_portion() {
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[10, 0, 0, 0, 10, 0, 0, 0, 0, 0]),
Err(DecodeError::InvalidListFixedBytesLen(10))
);
}
#[test]
fn length_out_of_bounds() {
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[5, 0, 0, 0]),
Err(DecodeError::OffsetOutOfBounds(5))
);
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[8, 0, 0, 0, 9, 0, 0, 0]),
Err(DecodeError::OffsetOutOfBounds(9))
);
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[8, 0, 0, 0, 16, 0, 0, 0]),
Err(DecodeError::OffsetOutOfBounds(16))
);
}
#[test]
fn vec_of_vec_of_u16() {
assert_eq!(
<Vec<Vec<u16>>>::from_ssz_bytes(&[4, 0, 0, 0]),
Ok(vec![vec![]])
);
assert_eq!(
<Vec<u16>>::from_ssz_bytes(&[0, 0, 1, 0, 2, 0, 3, 0]),
Ok(vec![0, 1, 2, 3])
);
assert_eq!(<u16>::from_ssz_bytes(&[16, 0]), Ok(16));
assert_eq!(<u16>::from_ssz_bytes(&[0, 1]), Ok(256));
assert_eq!(<u16>::from_ssz_bytes(&[255, 255]), Ok(65535));
assert_eq!(
<u16>::from_ssz_bytes(&[255]),
Err(DecodeError::InvalidByteLength {
len: 1,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[]),
Err(DecodeError::InvalidByteLength {
len: 0,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[0, 1, 2]),
Err(DecodeError::InvalidByteLength {
len: 3,
expected: 2
})
);
}
#[test]
fn vec_of_u16() {
assert_eq!(<Vec<u16>>::from_ssz_bytes(&[0, 0, 0, 0]), Ok(vec![0, 0]));
assert_eq!(
<Vec<u16>>::from_ssz_bytes(&[0, 0, 1, 0, 2, 0, 3, 0]),
Ok(vec![0, 1, 2, 3])
);
assert_eq!(<u16>::from_ssz_bytes(&[16, 0]), Ok(16));
assert_eq!(<u16>::from_ssz_bytes(&[0, 1]), Ok(256));
assert_eq!(<u16>::from_ssz_bytes(&[255, 255]), Ok(65535));
assert_eq!(
<u16>::from_ssz_bytes(&[255]),
Err(DecodeError::InvalidByteLength {
len: 1,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[]),
Err(DecodeError::InvalidByteLength {
len: 0,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[0, 1, 2]),
Err(DecodeError::InvalidByteLength {
len: 3,
expected: 2
})
);
}
#[test]
fn u16() {
assert_eq!(<u16>::from_ssz_bytes(&[0, 0]), Ok(0));
assert_eq!(<u16>::from_ssz_bytes(&[16, 0]), Ok(16));
assert_eq!(<u16>::from_ssz_bytes(&[0, 1]), Ok(256));
assert_eq!(<u16>::from_ssz_bytes(&[255, 255]), Ok(65535));
assert_eq!(
<u16>::from_ssz_bytes(&[255]),
Err(DecodeError::InvalidByteLength {
len: 1,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[]),
Err(DecodeError::InvalidByteLength {
len: 0,
expected: 2
})
);
assert_eq!(
<u16>::from_ssz_bytes(&[0, 1, 2]),
Err(DecodeError::InvalidByteLength {
len: 3,
expected: 2
})
);
}
#[test]
fn tuple() {
assert_eq!(<(u16, u16)>::from_ssz_bytes(&[0, 0, 0, 0]), Ok((0, 0)));
assert_eq!(<(u16, u16)>::from_ssz_bytes(&[16, 0, 17, 0]), Ok((16, 17)));
assert_eq!(<(u16, u16)>::from_ssz_bytes(&[0, 1, 2, 0]), Ok((256, 2)));
assert_eq!(
<(u16, u16)>::from_ssz_bytes(&[255, 255, 0, 0]),
Ok((65535, 0))
);
}
}

103
consensus/ssz/src/decode/try_from_iter.rs
View File

@@ -1,103 +0,0 @@
use smallvec::SmallVec;
use std::collections::{BTreeMap, BTreeSet};
use std::convert::Infallible;
use std::fmt::Debug;
/// Partial variant of `std::iter::FromIterator`.
///
/// This trait is implemented for types which can be constructed from an iterator of decoded SSZ
/// values, but which may refuse values once a length limit is reached.
pub trait TryFromIter<T>: Sized {
type Error: Debug;
fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
where
I: IntoIterator<Item = T>;
}
// It would be nice to be able to do a blanket impl, e.g.
//
// `impl TryFromIter<T> for C where C: FromIterator<T>`
//
// However this runs into trait coherence issues due to the type parameter `T` on `TryFromIter`.
//
// E.g. If we added an impl downstream for `List<T, N>` then another crate downstream of that
// could legally add an impl of `FromIterator<Local> for List<Local, N>` which would create
// two conflicting implementations for `List<Local, N>`. Hence the `List<T, N>` impl is disallowed
// by the compiler in the presence of the blanket impl. That's obviously annoying, so we opt to
// abandon the blanket impl in favour of impls for selected types.
impl<T> TryFromIter<T> for Vec<T> {
type Error = Infallible;
fn try_from_iter<I>(values: I) -> Result<Self, Self::Error>
where
I: IntoIterator<Item = T>,
{
// Pre-allocate the expected size of the Vec, which is parsed from the SSZ input bytes as
// `num_items`. This length has already been checked to be less than or equal to the type's
// maximum length in `decode_list_of_variable_length_items`.
let iter = values.into_iter();
let (_, opt_max_len) = iter.size_hint();
let mut vec = Vec::with_capacity(opt_max_len.unwrap_or(0));
vec.extend(iter);
Ok(vec)
}
}
impl<T, const N: usize> TryFromIter<T> for SmallVec<[T; N]> {
type Error = Infallible;
fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
where
I: IntoIterator<Item = T>,
{
Ok(Self::from_iter(iter))
}
}
impl<K, V> TryFromIter<(K, V)> for BTreeMap<K, V>
where
K: Ord,
{
type Error = Infallible;
fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
where
I: IntoIterator<Item = (K, V)>,
{
Ok(Self::from_iter(iter))
}
}
impl<T> TryFromIter<T> for BTreeSet<T>
where
T: Ord,
{
type Error = Infallible;
fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
where
I: IntoIterator<Item = T>,
{
Ok(Self::from_iter(iter))
}
}
/// Partial variant of `collect`.
pub trait TryCollect: Iterator {
fn try_collect<C>(self) -> Result<C, C::Error>
where
C: TryFromIter<Self::Item>;
}
impl<I> TryCollect for I
where
I: Iterator,
{
fn try_collect<C>(self) -> Result<C, C::Error>
where
C: TryFromIter<Self::Item>,
{
C::try_from_iter(self)
}
}

196
consensus/ssz/src/encode.rs
View File

@@ -1,196 +0,0 @@
use super::*;
mod impls;
/// Provides SSZ encoding (serialization) via the `as_ssz_bytes(&self)` method.
///
/// See `examples/` for manual implementations or the crate root for implementations using
/// `#[derive(Encode)]`.
pub trait Encode {
/// Returns `true` if this object has a fixed-length.
///
/// I.e., there are no variable length items in this object or any of it's contained objects.
fn is_ssz_fixed_len() -> bool;
/// Append the encoding `self` to `buf`.
///
/// Note, variable length objects need only to append their "variable length" portion, they do
/// not need to provide their offset.
fn ssz_append(&self, buf: &mut Vec<u8>);
/// The number of bytes this object occupies in the fixed-length portion of the SSZ bytes.
///
/// By default, this is set to `BYTES_PER_LENGTH_OFFSET` which is suitable for variable length
/// objects, but not fixed-length objects. Fixed-length objects _must_ return a value which
/// represents their length.
fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
/// Returns the size (in bytes) when `self` is serialized.
///
/// Returns the same value as `self.as_ssz_bytes().len()` but this method is significantly more
/// efficient.
fn ssz_bytes_len(&self) -> usize;
/// Returns the full-form encoding of this object.
///
/// The default implementation of this method should suffice for most cases.
fn as_ssz_bytes(&self) -> Vec<u8> {
let mut buf = vec![];
self.ssz_append(&mut buf);
buf
}
}
/// Allow for encoding an ordered series of distinct or indistinct objects as SSZ bytes.
///
/// **You must call `finalize(..)` after the final `append(..)` call** to ensure the bytes are
/// written to `buf`.
///
/// ## Example
///
/// Use `SszEncoder` to produce identical output to `foo.as_ssz_bytes()`:
///
/// ```rust
/// use ssz_derive::{Encode, Decode};
/// use ssz::{Decode, Encode, SszEncoder};
///
/// #[derive(PartialEq, Debug, Encode, Decode)]
/// struct Foo {
/// a: u64,
/// b: Vec<u16>,
/// }
///
/// fn ssz_encode_example() {
/// let foo = Foo {
/// a: 42,
/// b: vec![1, 3, 3, 7]
/// };
///
/// let mut buf: Vec<u8> = vec![];
/// let offset = <u64 as Encode>::ssz_fixed_len() + <Vec<u16> as Encode>::ssz_fixed_len();
///
/// let mut encoder = SszEncoder::container(&mut buf, offset);
///
/// encoder.append(&foo.a);
/// encoder.append(&foo.b);
///
/// encoder.finalize();
///
/// assert_eq!(foo.as_ssz_bytes(), buf);
/// }
///
/// ```
pub struct SszEncoder<'a> {
offset: usize,
buf: &'a mut Vec<u8>,
variable_bytes: Vec<u8>,
}
impl<'a> SszEncoder<'a> {
/// Instantiate a new encoder for encoding a SSZ container.
pub fn container(buf: &'a mut Vec<u8>, num_fixed_bytes: usize) -> Self {
buf.reserve(num_fixed_bytes);
Self {
offset: num_fixed_bytes,
buf,
variable_bytes: vec![],
}
}
/// Append some `item` to the SSZ bytes.
pub fn append<T: Encode>(&mut self, item: &T) {
self.append_parameterized(T::is_ssz_fixed_len(), |buf| item.ssz_append(buf))
}
/// Uses `ssz_append` to append the encoding of some item to the SSZ bytes.
pub fn append_parameterized<F>(&mut self, is_ssz_fixed_len: bool, ssz_append: F)
where
F: Fn(&mut Vec<u8>),
{
if is_ssz_fixed_len {
ssz_append(self.buf);
} else {
self.buf
.extend_from_slice(&encode_length(self.offset + self.variable_bytes.len()));
ssz_append(&mut self.variable_bytes);
}
}
/// Write the variable bytes to `self.bytes`.
///
/// This method must be called after the final `append(..)` call when serializing
/// variable-length items.
pub fn finalize(&mut self) -> &mut Vec<u8> {
self.buf.append(&mut self.variable_bytes);
self.buf
}
}
/// Encode `len` as a little-endian byte array of `BYTES_PER_LENGTH_OFFSET` length.
///
/// If `len` is larger than `2 ^ BYTES_PER_LENGTH_OFFSET`, a `debug_assert` is raised.
pub fn encode_length(len: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
// Note: it is possible for `len` to be larger than what can be encoded in
// `BYTES_PER_LENGTH_OFFSET` bytes, triggering this debug assertion.
//
// These are the alternatives to using a `debug_assert` here:
//
// 1. Use `assert`.
// 2. Push an error to the caller (e.g., `Option` or `Result`).
// 3. Ignore it completely.
//
// I have avoided (1) because it's basically a choice between "produce invalid SSZ" or "kill
// the entire program". I figure it may be possible for an attacker to trigger this assert and
// take the program down -- I think producing invalid SSZ is a better option than this.
//
// I have avoided (2) because this error will need to be propagated upstream, making encoding a
// function which may fail. I don't think this is ergonomic and the upsides don't outweigh the
// downsides.
//
// I figure a `debug_assertion` is better than (3) as it will give us a change to detect the
// error during testing.
//
// If you have a different opinion, feel free to start an issue and tag @paulhauner.
debug_assert!(len <= MAX_LENGTH_VALUE);
let mut bytes = [0; BYTES_PER_LENGTH_OFFSET];
bytes.copy_from_slice(&len.to_le_bytes()[0..BYTES_PER_LENGTH_OFFSET]);
bytes
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_encode_length() {
assert_eq!(encode_length(0), [0; 4]);
assert_eq!(encode_length(1), [1, 0, 0, 0]);
assert_eq!(
encode_length(MAX_LENGTH_VALUE),
[255; BYTES_PER_LENGTH_OFFSET]
);
}
#[test]
#[should_panic]
#[cfg(debug_assertions)]
fn test_encode_length_above_max_debug_panics() {
encode_length(MAX_LENGTH_VALUE + 1);
}
#[test]
#[cfg(not(debug_assertions))]
fn test_encode_length_above_max_not_debug_does_not_panic() {
assert_eq!(&encode_length(MAX_LENGTH_VALUE + 1)[..], &[0; 4]);
}
}

632
consensus/ssz/src/encode/impls.rs
View File

@@ -1,632 +0,0 @@
use super::*;
use core::num::NonZeroUsize;
use ethereum_types::{H160, H256, U128, U256};
use smallvec::SmallVec;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
macro_rules! impl_encodable_for_uint {
($type: ident, $bit_size: expr) => {
impl Encode for $type {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
$bit_size / 8
}
fn ssz_bytes_len(&self) -> usize {
$bit_size / 8
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(&self.to_le_bytes());
}
}
};
}
impl_encodable_for_uint!(u8, 8);
impl_encodable_for_uint!(u16, 16);
impl_encodable_for_uint!(u32, 32);
impl_encodable_for_uint!(u64, 64);
#[cfg(target_pointer_width = "32")]
impl_encodable_for_uint!(usize, 32);
#[cfg(target_pointer_width = "64")]
impl_encodable_for_uint!(usize, 64);
// Based on the `tuple_impls` macro from the standard library.
macro_rules! impl_encode_for_tuples {
($(
$Tuple:ident {
$(($idx:tt) -> $T:ident)+
}
)+) => {
$(
impl<$($T: Encode),+> Encode for ($($T,)+) {
fn is_ssz_fixed_len() -> bool {
$(
<$T as Encode>::is_ssz_fixed_len() &&
)*
true
}
fn ssz_fixed_len() -> usize {
if <Self as Encode>::is_ssz_fixed_len() {
$(
<$T as Encode>::ssz_fixed_len() +
)*
0
} else {
BYTES_PER_LENGTH_OFFSET
}
}
fn ssz_bytes_len(&self) -> usize {
if <Self as Encode>::is_ssz_fixed_len() {
<Self as Encode>::ssz_fixed_len()
} else {
let mut len = 0;
$(
len += if <$T as Encode>::is_ssz_fixed_len() {
<$T as Encode>::ssz_fixed_len()
} else {
BYTES_PER_LENGTH_OFFSET +
self.$idx.ssz_bytes_len()
};
)*
len
}
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
let offset = $(
<$T as Encode>::ssz_fixed_len() +
)*
0;
let mut encoder = SszEncoder::container(buf, offset);
$(
encoder.append(&self.$idx);
)*
encoder.finalize();
}
}
)+
}
}
impl_encode_for_tuples! {
Tuple2 {
(0) -> A
(1) -> B
}
Tuple3 {
(0) -> A
(1) -> B
(2) -> C
}
Tuple4 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
}
Tuple5 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
}
Tuple6 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
}
Tuple7 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
}
Tuple8 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
}
Tuple9 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
}
Tuple10 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
}
Tuple11 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
(10) -> K
}
Tuple12 {
(0) -> A
(1) -> B
(2) -> C
(3) -> D
(4) -> E
(5) -> F
(6) -> G
(7) -> H
(8) -> I
(9) -> J
(10) -> K
(11) -> L
}
}
impl<T: Encode> Encode for Option<T> {
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
Option::None => {
let union_selector: u8 = 0u8;
buf.push(union_selector);
}
Option::Some(ref inner) => {
let union_selector: u8 = 1u8;
buf.push(union_selector);
inner.ssz_append(buf);
}
}
}
fn ssz_bytes_len(&self) -> usize {
match self {
Option::None => 1usize,
Option::Some(ref inner) => inner
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max_value"),
}
}
}
impl<T: Encode> Encode for Arc<T> {
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
T::ssz_fixed_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
self.as_ref().ssz_append(buf)
}
fn ssz_bytes_len(&self) -> usize {
self.as_ref().ssz_bytes_len()
}
}
// Encode transparently through references.
impl<'a, T: Encode> Encode for &'a T {
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
T::ssz_fixed_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
T::ssz_append(self, buf)
}
fn ssz_bytes_len(&self) -> usize {
T::ssz_bytes_len(self)
}
}
/// Compute the encoded length of a vector-like sequence of `T`.
pub fn sequence_ssz_bytes_len<I, T>(iter: I) -> usize
where
I: Iterator<Item = T> + ExactSizeIterator,
T: Encode,
{
// Compute length before doing any iteration.
let length = iter.len();
if <T as Encode>::is_ssz_fixed_len() {
<T as Encode>::ssz_fixed_len() * length
} else {
let mut len = iter.map(|item| item.ssz_bytes_len()).sum();
len += BYTES_PER_LENGTH_OFFSET * length;
len
}
}
/// Encode a vector-like sequence of `T`.
pub fn sequence_ssz_append<I, T>(iter: I, buf: &mut Vec<u8>)
where
I: Iterator<Item = T> + ExactSizeIterator,
T: Encode,
{
if T::is_ssz_fixed_len() {
buf.reserve(T::ssz_fixed_len() * iter.len());
for item in iter {
item.ssz_append(buf);
}
} else {
let mut encoder = SszEncoder::container(buf, iter.len() * BYTES_PER_LENGTH_OFFSET);
for item in iter {
encoder.append(&item);
}
encoder.finalize();
}
}
macro_rules! impl_for_vec {
($type: ty) => {
impl<T: Encode> Encode for $type {
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_bytes_len(&self) -> usize {
sequence_ssz_bytes_len(self.iter())
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
sequence_ssz_append(self.iter(), buf)
}
}
};
}
impl_for_vec!(Vec<T>);
impl_for_vec!(SmallVec<[T; 1]>);
impl_for_vec!(SmallVec<[T; 2]>);
impl_for_vec!(SmallVec<[T; 3]>);
impl_for_vec!(SmallVec<[T; 4]>);
impl_for_vec!(SmallVec<[T; 5]>);
impl_for_vec!(SmallVec<[T; 6]>);
impl_for_vec!(SmallVec<[T; 7]>);
impl_for_vec!(SmallVec<[T; 8]>);
impl<K, V> Encode for BTreeMap<K, V>
where
K: Encode + Ord,
V: Encode,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_bytes_len(&self) -> usize {
sequence_ssz_bytes_len(self.iter())
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
sequence_ssz_append(self.iter(), buf)
}
}
impl<T> Encode for BTreeSet<T>
where
T: Encode + Ord,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_bytes_len(&self) -> usize {
sequence_ssz_bytes_len(self.iter())
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
sequence_ssz_append(self.iter(), buf)
}
}
impl Encode for bool {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
1
}
fn ssz_bytes_len(&self) -> usize {
1
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(&(*self as u8).to_le_bytes());
}
}
impl Encode for NonZeroUsize {
fn is_ssz_fixed_len() -> bool {
<usize as Encode>::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
<usize as Encode>::ssz_fixed_len()
}
fn ssz_bytes_len(&self) -> usize {
std::mem::size_of::<usize>()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
self.get().ssz_append(buf)
}
}
impl Encode for H160 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
20
}
fn ssz_bytes_len(&self) -> usize {
20
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self.as_bytes());
}
}
impl Encode for H256 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
32
}
fn ssz_bytes_len(&self) -> usize {
32
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(self.as_bytes());
}
}
impl Encode for U256 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
32
}
fn ssz_bytes_len(&self) -> usize {
32
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
let n = <Self as Encode>::ssz_fixed_len();
let s = buf.len();
buf.resize(s + n, 0);
self.to_little_endian(&mut buf[s..]);
}
}
impl Encode for U128 {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
16
}
fn ssz_bytes_len(&self) -> usize {
16
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
let n = <Self as Encode>::ssz_fixed_len();
let s = buf.len();
buf.resize(s + n, 0);
self.to_little_endian(&mut buf[s..]);
}
}
macro_rules! impl_encodable_for_u8_array {
($len: expr) => {
impl Encode for [u8; $len] {
fn is_ssz_fixed_len() -> bool {
true
}
fn ssz_fixed_len() -> usize {
$len
}
fn ssz_bytes_len(&self) -> usize {
$len
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
buf.extend_from_slice(&self[..]);
}
}
};
}
impl_encodable_for_u8_array!(4);
impl_encodable_for_u8_array!(32);
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn vec_of_u8() {
let vec: Vec<u8> = vec![];
assert_eq!(vec.as_ssz_bytes(), vec![]);
let vec: Vec<u8> = vec![1];
assert_eq!(vec.as_ssz_bytes(), vec![1]);
let vec: Vec<u8> = vec![0, 1, 2, 3];
assert_eq!(vec.as_ssz_bytes(), vec![0, 1, 2, 3]);
}
#[test]
fn vec_of_vec_of_u8() {
let vec: Vec<Vec<u8>> = vec![];
assert_eq!(vec.as_ssz_bytes(), vec![]);
let vec: Vec<Vec<u8>> = vec![vec![]];
assert_eq!(vec.as_ssz_bytes(), vec![4, 0, 0, 0]);
let vec: Vec<Vec<u8>> = vec![vec![], vec![]];
assert_eq!(vec.as_ssz_bytes(), vec![8, 0, 0, 0, 8, 0, 0, 0]);
let vec: Vec<Vec<u8>> = vec![vec![0, 1, 2], vec![11, 22, 33]];
assert_eq!(
vec.as_ssz_bytes(),
vec![8, 0, 0, 0, 11, 0, 0, 0, 0, 1, 2, 11, 22, 33]
);
}
#[test]
fn ssz_encode_u8() {
assert_eq!(0_u8.as_ssz_bytes(), vec![0]);
assert_eq!(1_u8.as_ssz_bytes(), vec![1]);
assert_eq!(100_u8.as_ssz_bytes(), vec![100]);
assert_eq!(255_u8.as_ssz_bytes(), vec![255]);
}
#[test]
fn ssz_encode_u16() {
assert_eq!(1_u16.as_ssz_bytes(), vec![1, 0]);
assert_eq!(100_u16.as_ssz_bytes(), vec![100, 0]);
assert_eq!((1_u16 << 8).as_ssz_bytes(), vec![0, 1]);
assert_eq!(65535_u16.as_ssz_bytes(), vec![255, 255]);
}
#[test]
fn ssz_encode_u32() {
assert_eq!(1_u32.as_ssz_bytes(), vec![1, 0, 0, 0]);
assert_eq!(100_u32.as_ssz_bytes(), vec![100, 0, 0, 0]);
assert_eq!((1_u32 << 16).as_ssz_bytes(), vec![0, 0, 1, 0]);
assert_eq!((1_u32 << 24).as_ssz_bytes(), vec![0, 0, 0, 1]);
assert_eq!((!0_u32).as_ssz_bytes(), vec![255, 255, 255, 255]);
}
#[test]
fn ssz_encode_u64() {
assert_eq!(1_u64.as_ssz_bytes(), vec![1, 0, 0, 0, 0, 0, 0, 0]);
assert_eq!(
(!0_u64).as_ssz_bytes(),
vec![255, 255, 255, 255, 255, 255, 255, 255]
);
}
#[test]
fn ssz_encode_usize() {
assert_eq!(1_usize.as_ssz_bytes(), vec![1, 0, 0, 0, 0, 0, 0, 0]);
assert_eq!(
(!0_usize).as_ssz_bytes(),
vec![255, 255, 255, 255, 255, 255, 255, 255]
);
}
#[test]
fn ssz_encode_option_u8() {
let opt: Option<u8> = None;
assert_eq!(opt.as_ssz_bytes(), vec![0]);
let opt: Option<u8> = Some(2);
assert_eq!(opt.as_ssz_bytes(), vec![1, 2]);
}
#[test]
fn ssz_encode_bool() {
assert_eq!(true.as_ssz_bytes(), vec![1]);
assert_eq!(false.as_ssz_bytes(), vec![0]);
}
#[test]
fn ssz_encode_h256() {
assert_eq!(H256::from(&[0; 32]).as_ssz_bytes(), vec![0; 32]);
assert_eq!(H256::from(&[1; 32]).as_ssz_bytes(), vec![1; 32]);
let bytes = vec![
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0,
];
assert_eq!(H256::from_slice(&bytes).as_ssz_bytes(), bytes);
}
#[test]
fn ssz_encode_u8_array_4() {
assert_eq!([0, 0, 0, 0].as_ssz_bytes(), vec![0; 4]);
assert_eq!([1, 0, 0, 0].as_ssz_bytes(), vec![1, 0, 0, 0]);
assert_eq!([1, 2, 3, 4].as_ssz_bytes(), vec![1, 2, 3, 4]);
}
#[test]
fn tuple() {
assert_eq!((10u8, 11u8).as_ssz_bytes(), vec![10, 11]);
assert_eq!((10u32, 11u8).as_ssz_bytes(), vec![10, 0, 0, 0, 11]);
assert_eq!((10u8, 11u8, 12u8).as_ssz_bytes(), vec![10, 11, 12]);
}
}

265
consensus/ssz/src/legacy.rs
View File

@@ -1,265 +0,0 @@
//! Provides a "legacy" version of SSZ encoding for `Option<T> where T: Encode + Decode`.
//!
//! The SSZ specification changed in 2021 to use a 1-byte union selector, instead of a 4-byte one
//! which was used in the Lighthouse database.
//!
//! Users can use the `four_byte_option_impl` macro to define a module that can be used with the
//! `#[ssz(with = "module")]`.
//!
//! ## Example
//!
//! ```rust
//! use ssz_derive::{Encode, Decode};
//! use ssz::four_byte_option_impl;
//!
//! four_byte_option_impl!(impl_for_u64, u64);
//!
//! #[derive(Encode, Decode)]
//! struct Foo {
//! #[ssz(with = "impl_for_u64")]
//! a: Option<u64>,
//! }
//! ```
use crate::*;
#[macro_export]
macro_rules! four_byte_option_impl {
($mod_name: ident, $type: ty) => {
#[allow(dead_code)]
mod $mod_name {
use super::*;
pub mod encode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn ssz_bytes_len(opt: &Option<$type>) -> usize {
if let Some(some) = opt {
let len = if <$type as Encode>::is_ssz_fixed_len() {
<$type as Encode>::ssz_fixed_len()
} else {
<$type as Encode>::ssz_bytes_len(some)
};
len + BYTES_PER_LENGTH_OFFSET
} else {
BYTES_PER_LENGTH_OFFSET
}
}
pub fn ssz_append(opt: &Option<$type>, buf: &mut Vec<u8>) {
match opt {
None => buf.extend_from_slice(&legacy::encode_four_byte_union_selector(0)),
Some(t) => {
buf.extend_from_slice(&legacy::encode_four_byte_union_selector(1));
t.ssz_append(buf);
}
}
}
pub fn as_ssz_bytes(opt: &Option<$type>) -> Vec<u8> {
let mut buf = vec![];
ssz_append(opt, &mut buf);
buf
}
}
pub mod decode {
use super::*;
#[allow(unused_imports)]
use ssz::*;
pub fn is_ssz_fixed_len() -> bool {
false
}
pub fn ssz_fixed_len() -> usize {
BYTES_PER_LENGTH_OFFSET
}
pub fn from_ssz_bytes(bytes: &[u8]) -> Result<Option<$type>, DecodeError> {
if bytes.len() < BYTES_PER_LENGTH_OFFSET {
return Err(DecodeError::InvalidByteLength {
len: bytes.len(),
expected: BYTES_PER_LENGTH_OFFSET,
});
}
let (index_bytes, value_bytes) = bytes.split_at(BYTES_PER_LENGTH_OFFSET);
let index = legacy::read_four_byte_union_selector(index_bytes)?;
if index == 0 {
Ok(None)
} else if index == 1 {
Ok(Some(<$type as ssz::Decode>::from_ssz_bytes(value_bytes)?))
} else {
Err(DecodeError::BytesInvalid(format!(
"{} is not a valid union index for Option<T>",
index
)))
}
}
}
}
};
}
pub fn encode_four_byte_union_selector(selector: usize) -> [u8; BYTES_PER_LENGTH_OFFSET] {
encode_length(selector)
}
pub fn read_four_byte_union_selector(bytes: &[u8]) -> Result<usize, DecodeError> {
read_offset(bytes)
}
#[cfg(test)]
mod test {
use super::*;
use crate as ssz;
use ssz_derive::{Decode, Encode};
type VecU16 = Vec<u16>;
four_byte_option_impl!(impl_u16, u16);
four_byte_option_impl!(impl_vec_u16, VecU16);
#[test]
fn ssz_encode_option_u16() {
let item = Some(65535_u16);
let bytes = vec![1, 0, 0, 0, 255, 255];
assert_eq!(impl_u16::encode::as_ssz_bytes(&item), bytes);
assert_eq!(impl_u16::decode::from_ssz_bytes(&bytes).unwrap(), item);
let item = None;
let bytes = vec![0, 0, 0, 0];
assert_eq!(impl_u16::encode::as_ssz_bytes(&item), bytes);
assert_eq!(impl_u16::decode::from_ssz_bytes(&bytes).unwrap(), None);
}
#[test]
fn ssz_encode_option_vec_u16() {
let item = Some(vec![0_u16, 1]);
let bytes = vec![1, 0, 0, 0, 0, 0, 1, 0];
assert_eq!(impl_vec_u16::encode::as_ssz_bytes(&item), bytes);
assert_eq!(impl_vec_u16::decode::from_ssz_bytes(&bytes).unwrap(), item);
let item = None;
let bytes = vec![0, 0, 0, 0];
assert_eq!(impl_vec_u16::encode::as_ssz_bytes(&item), bytes);
assert_eq!(impl_vec_u16::decode::from_ssz_bytes(&bytes).unwrap(), item);
}
fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(items: Vec<T>) {
for item in items {
let encoded = &item.as_ssz_bytes();
assert_eq!(item.ssz_bytes_len(), encoded.len());
assert_eq!(T::from_ssz_bytes(encoded), Ok(item));
}
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct TwoVariableLenOptions {
a: u16,
#[ssz(with = "impl_u16")]
b: Option<u16>,
#[ssz(with = "impl_vec_u16")]
c: Option<Vec<u16>>,
#[ssz(with = "impl_vec_u16")]
d: Option<Vec<u16>>,
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn two_variable_len_options_encoding() {
let s = TwoVariableLenOptions {
a: 42,
b: None,
c: Some(vec![0]),
d: None,
};
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
// | option<u16> | offset | offset | option<u16 | 1st list
42, 00, 14, 00, 00, 00, 18, 00, 00, 00, 24, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00,
// 23 24 25 26 27
// | 2nd list
00, 00, 00, 00, 00, 00,
];
assert_eq!(s.as_ssz_bytes(), bytes);
}
#[test]
fn two_variable_len_options_round_trip() {
let vec: Vec<TwoVariableLenOptions> = vec![
TwoVariableLenOptions {
a: 42,
b: Some(12),
c: Some(vec![0]),
d: Some(vec![1]),
},
TwoVariableLenOptions {
a: 42,
b: Some(12),
c: Some(vec![0]),
d: None,
},
TwoVariableLenOptions {
a: 42,
b: None,
c: Some(vec![0]),
d: None,
},
TwoVariableLenOptions {
a: 42,
b: None,
c: None,
d: None,
},
];
round_trip(vec);
}
#[test]
fn tuple_u8_u16() {
let vec: Vec<(u8, u16)> = vec![
(0, 0),
(0, 1),
(1, 0),
(u8::max_value(), u16::max_value()),
(0, u16::max_value()),
(u8::max_value(), 0),
(42, 12301),
];
round_trip(vec);
}
#[test]
fn tuple_vec_vec() {
let vec: Vec<(u64, Vec<u8>, Vec<Vec<u16>>)> = vec![
(0, vec![], vec![vec![]]),
(99, vec![101], vec![vec![], vec![]]),
(
42,
vec![12, 13, 14],
vec![vec![99, 98, 97, 96], vec![42, 44, 46, 48, 50]],
),
];
round_trip(vec);
}
}

71
consensus/ssz/src/lib.rs
View File

@@ -1,71 +0,0 @@
//! Provides encoding (serialization) and decoding (deserialization) in the SimpleSerialize (SSZ)
//! format designed for use in Ethereum 2.0.
//!
//! Adheres to the Ethereum 2.0 [SSZ
//! specification](https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/ssz/simple-serialize.md)
//! at v0.12.1.
//!
//! ## Example
//!
//! ```rust
//! use ssz_derive::{Encode, Decode};
//! use ssz::{Decode, Encode};
//!
//! #[derive(PartialEq, Debug, Encode, Decode)]
//! struct Foo {
//! a: u64,
//! b: Vec<u16>,
//! }
//!
//! fn ssz_encode_decode_example() {
//! let foo = Foo {
//! a: 42,
//! b: vec![1, 3, 3, 7]
//! };
//!
//! let ssz_bytes: Vec<u8> = foo.as_ssz_bytes();
//!
//! let decoded_foo = Foo::from_ssz_bytes(&ssz_bytes).unwrap();
//!
//! assert_eq!(foo, decoded_foo);
//! }
//!
//! ```
//!
//! See `examples/` for manual implementations of the `Encode` and `Decode` traits.
mod decode;
mod encode;
pub mod legacy;
mod union_selector;
pub use decode::{
impls::decode_list_of_variable_length_items, read_offset, split_union_bytes,
try_from_iter::TryFromIter, Decode, DecodeError, SszDecoder, SszDecoderBuilder,
};
pub use encode::{encode_length, Encode, SszEncoder};
pub use union_selector::UnionSelector;
/// The number of bytes used to represent an offset.
pub const BYTES_PER_LENGTH_OFFSET: usize = 4;
/// The maximum value that can be represented using `BYTES_PER_LENGTH_OFFSET`.
#[cfg(target_pointer_width = "32")]
pub const MAX_LENGTH_VALUE: usize = (std::u32::MAX >> (8 * (4 - BYTES_PER_LENGTH_OFFSET))) as usize;
#[cfg(target_pointer_width = "64")]
pub const MAX_LENGTH_VALUE: usize = (std::u64::MAX >> (8 * (8 - BYTES_PER_LENGTH_OFFSET))) as usize;
/// The number of bytes used to indicate the variant of a union.
pub const BYTES_PER_UNION_SELECTOR: usize = 1;
/// The highest possible union selector value (higher values are reserved for backwards compatible
/// extensions).
pub const MAX_UNION_SELECTOR: u8 = 127;
/// Convenience function to SSZ encode an object supporting ssz::Encode.
///
/// Equivalent to `val.as_ssz_bytes()`.
pub fn ssz_encode<T>(val: &T) -> Vec<u8>
where
T: Encode,
{
val.as_ssz_bytes()
}

29
consensus/ssz/src/union_selector.rs
View File

@@ -1,29 +0,0 @@
use crate::*;
/// Provides the one-byte "selector" from the SSZ union specification:
///
/// https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union
#[derive(Copy, Clone)]
pub struct UnionSelector(u8);
impl From<UnionSelector> for u8 {
fn from(union_selector: UnionSelector) -> u8 {
union_selector.0
}
}
impl PartialEq<u8> for UnionSelector {
fn eq(&self, other: &u8) -> bool {
self.0 == *other
}
}
impl UnionSelector {
/// Instantiate `self`, returning an error if `selector > MAX_UNION_SELECTOR`.
pub fn new(selector: u8) -> Result<Self, DecodeError> {
Some(selector)
.filter(|_| selector <= MAX_UNION_SELECTOR)
.map(Self)
.ok_or(DecodeError::UnionSelectorInvalid(selector))
}
}

532
consensus/ssz/tests/tests.rs
View File

@@ -1,532 +0,0 @@
use ethereum_types::H256;
use ssz::{Decode, DecodeError, Encode};
use ssz_derive::{Decode, Encode};
mod round_trip {
use super::*;
use std::collections::BTreeMap;
use std::iter::FromIterator;
fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(items: Vec<T>) {
for item in items {
let encoded = &item.as_ssz_bytes();
assert_eq!(item.ssz_bytes_len(), encoded.len());
assert_eq!(T::from_ssz_bytes(encoded), Ok(item));
}
}
#[test]
fn bool() {
let items: Vec<bool> = vec![true, false];
round_trip(items);
}
#[test]
fn option_u16() {
let items: Vec<Option<u16>> = vec![None, Some(2u16)];
round_trip(items);
}
#[test]
fn u8_array_4() {
let items: Vec<[u8; 4]> = vec![[0, 0, 0, 0], [1, 0, 0, 0], [1, 2, 3, 4], [1, 2, 0, 4]];
round_trip(items);
}
#[test]
fn h256() {
let items: Vec<H256> = vec![H256::zero(), H256::from([1; 32]), H256::random()];
round_trip(items);
}
#[test]
fn vec_of_h256() {
let items: Vec<Vec<H256>> = vec![
vec![],
vec![H256::zero(), H256::from([1; 32]), H256::random()],
];
round_trip(items);
}
#[test]
fn option_vec_h256() {
let items: Vec<Option<Vec<H256>>> = vec![
None,
Some(vec![]),
Some(vec![H256::zero(), H256::from([1; 32]), H256::random()]),
];
round_trip(items);
}
#[test]
fn vec_u16() {
let items: Vec<Vec<u16>> = vec![
vec![],
vec![255],
vec![0, 1, 2],
vec![100; 64],
vec![255, 0, 255],
];
round_trip(items);
}
#[test]
fn vec_of_vec_u16() {
let items: Vec<Vec<Vec<u16>>> = vec![
vec![],
vec![vec![]],
vec![vec![1, 2, 3]],
vec![vec![], vec![]],
vec![vec![], vec![1, 2, 3]],
vec![vec![1, 2, 3], vec![1, 2, 3]],
vec![vec![1, 2, 3], vec![], vec![1, 2, 3]],
vec![vec![], vec![], vec![1, 2, 3]],
vec![vec![], vec![1], vec![1, 2, 3]],
vec![vec![], vec![1], vec![1, 2, 3]],
];
round_trip(items);
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct FixedLen {
a: u16,
b: u64,
c: u32,
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn fixed_len_struct_encoding() {
let items: Vec<FixedLen> = vec![
FixedLen { a: 0, b: 0, c: 0 },
FixedLen { a: 1, b: 1, c: 1 },
FixedLen { a: 1, b: 0, c: 1 },
];
let expected_encodings = vec![
// | u16--| u64----------------------------| u32----------|
vec![00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00],
vec![01, 00, 01, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00],
vec![01, 00, 00, 00, 00, 00, 00, 00, 00, 00, 01, 00, 00, 00],
];
for i in 0..items.len() {
assert_eq!(
items[i].as_ssz_bytes(),
expected_encodings[i],
"Failed on {}",
i
);
}
}
#[test]
fn fixed_len_excess_bytes() {
let fixed = FixedLen { a: 1, b: 2, c: 3 };
let mut bytes = fixed.as_ssz_bytes();
bytes.append(&mut vec![0]);
assert_eq!(
FixedLen::from_ssz_bytes(&bytes),
Err(DecodeError::InvalidByteLength {
len: 15,
expected: 14,
})
);
}
#[test]
fn vec_of_fixed_len_struct() {
let items: Vec<FixedLen> = vec![
FixedLen { a: 0, b: 0, c: 0 },
FixedLen { a: 1, b: 1, c: 1 },
FixedLen { a: 1, b: 0, c: 1 },
];
round_trip(items);
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct VariableLen {
a: u16,
b: Vec<u16>,
c: u32,
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn offset_into_fixed_bytes() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | u32 | variable
01, 00, 09, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
];
assert_eq!(
VariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetIntoFixedPortion(9))
);
}
#[test]
fn variable_len_excess_bytes() {
let variable = VariableLen {
a: 1,
b: vec![2],
c: 3,
};
let mut bytes = variable.as_ssz_bytes();
bytes.append(&mut vec![0]);
// The error message triggered is not so helpful, it's caught by a side-effect. Just
// checking there is _some_ error is fine.
assert!(VariableLen::from_ssz_bytes(&bytes).is_err());
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn first_offset_skips_byte() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | u32 | variable
01, 00, 11, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
];
assert_eq!(
VariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetSkipsVariableBytes(11))
);
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn variable_len_struct_encoding() {
let items: Vec<VariableLen> = vec![
VariableLen {
a: 0,
b: vec![],
c: 0,
},
VariableLen {
a: 1,
b: vec![0],
c: 1,
},
VariableLen {
a: 1,
b: vec![0, 1, 2],
c: 1,
},
];
let expected_encodings = vec![
// 00..................................09
// | u16--| vec offset-----| u32------------| vec payload --------|
vec![00, 00, 10, 00, 00, 00, 00, 00, 00, 00],
vec![01, 00, 10, 00, 00, 00, 01, 00, 00, 00, 00, 00],
vec![
01, 00, 10, 00, 00, 00, 01, 00, 00, 00, 00, 00, 01, 00, 02, 00,
],
];
for i in 0..items.len() {
assert_eq!(
items[i].as_ssz_bytes(),
expected_encodings[i],
"Failed on {}",
i
);
}
}
#[test]
fn vec_of_variable_len_struct() {
let items: Vec<VariableLen> = vec![
VariableLen {
a: 0,
b: vec![],
c: 0,
},
VariableLen {
a: 255,
b: vec![0, 1, 2, 3],
c: 99,
},
VariableLen {
a: 255,
b: vec![0],
c: 99,
},
VariableLen {
a: 50,
b: vec![0],
c: 0,
},
];
round_trip(items);
}
#[derive(Debug, PartialEq, Encode, Decode)]
struct ThreeVariableLen {
a: u16,
b: Vec<u16>,
c: Vec<u16>,
d: Vec<u16>,
}
#[test]
fn three_variable_len() {
let vec: Vec<ThreeVariableLen> = vec![ThreeVariableLen {
a: 42,
b: vec![0],
c: vec![1],
d: vec![2],
}];
round_trip(vec);
}
#[test]
#[allow(clippy::zero_prefixed_literal)]
fn offsets_decreasing() {
let bytes = vec![
// 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
// | offset | offset | offset | variable
01, 00, 14, 00, 00, 00, 15, 00, 00, 00, 14, 00, 00, 00, 00, 00,
];
assert_eq!(
ThreeVariableLen::from_ssz_bytes(&bytes),
Err(DecodeError::OffsetsAreDecreasing(14))
);
}
#[test]
fn tuple_u8_u16() {
let vec: Vec<(u8, u16)> = vec![
(0, 0),
(0, 1),
(1, 0),
(u8::max_value(), u16::max_value()),
(0, u16::max_value()),
(u8::max_value(), 0),
(42, 12301),
];
round_trip(vec);
}
#[test]
fn tuple_vec_vec() {
let vec: Vec<(u64, Vec<u8>, Vec<Vec<u16>>)> = vec![
(0, vec![], vec![vec![]]),
(99, vec![101], vec![vec![], vec![]]),
(
42,
vec![12, 13, 14],
vec![vec![99, 98, 97, 96], vec![42, 44, 46, 48, 50]],
),
];
round_trip(vec);
}
#[test]
fn btree_map_fixed() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![(0u8, 0u16), (1, 2), (2, 4), (4, 6)]),
];
round_trip(data);
}
#[test]
fn btree_map_variable_value() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![
(
0u64,
ThreeVariableLen {
a: 1,
b: vec![3, 5, 7],
c: vec![],
d: vec![0, 0],
},
),
(
1,
ThreeVariableLen {
a: 99,
b: vec![1],
c: vec![2, 3, 4, 5, 6, 7, 8, 9, 10],
d: vec![4, 5, 6, 7, 8],
},
),
(
2,
ThreeVariableLen {
a: 0,
b: vec![],
c: vec![],
d: vec![],
},
),
]),
];
round_trip(data);
}
}
mod derive_macro {
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use std::fmt::Debug;
fn assert_encode<T: Encode>(item: &T, bytes: &[u8]) {
assert_eq!(item.as_ssz_bytes(), bytes);
}
fn assert_encode_decode<T: Encode + Decode + PartialEq + Debug>(item: &T, bytes: &[u8]) {
assert_encode(item, bytes);
assert_eq!(T::from_ssz_bytes(bytes).unwrap(), *item);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoFixedUnion {
U8(u8),
U16(u16),
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct TwoFixedUnionStruct {
a: TwoFixedUnion,
}
#[test]
fn two_fixed_union() {
let eight = TwoFixedUnion::U8(1);
let sixteen = TwoFixedUnion::U16(1);
assert_encode_decode(&eight, &[0, 1]);
assert_encode_decode(&sixteen, &[1, 1, 0]);
assert_encode_decode(&TwoFixedUnionStruct { a: eight }, &[4, 0, 0, 0, 0, 1]);
assert_encode_decode(&TwoFixedUnionStruct { a: sixteen }, &[4, 0, 0, 0, 1, 1, 0]);
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct VariableA {
a: u8,
b: Vec<u8>,
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct VariableB {
a: Vec<u8>,
b: u8,
}
#[derive(PartialEq, Debug, Encode)]
#[ssz(enum_behaviour = "transparent")]
enum TwoVariableTrans {
A(VariableA),
B(VariableB),
}
#[derive(PartialEq, Debug, Encode)]
struct TwoVariableTransStruct {
a: TwoVariableTrans,
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVariableUnion {
A(VariableA),
B(VariableB),
}
#[derive(PartialEq, Debug, Encode, Decode)]
struct TwoVariableUnionStruct {
a: TwoVariableUnion,
}
#[test]
fn two_variable_trans() {
let trans_a = TwoVariableTrans::A(VariableA {
a: 1,
b: vec![2, 3],
});
let trans_b = TwoVariableTrans::B(VariableB {
a: vec![1, 2],
b: 3,
});
assert_encode(&trans_a, &[1, 5, 0, 0, 0, 2, 3]);
assert_encode(&trans_b, &[5, 0, 0, 0, 3, 1, 2]);
assert_encode(
&TwoVariableTransStruct { a: trans_a },
&[4, 0, 0, 0, 1, 5, 0, 0, 0, 2, 3],
);
assert_encode(
&TwoVariableTransStruct { a: trans_b },
&[4, 0, 0, 0, 5, 0, 0, 0, 3, 1, 2],
);
}
#[test]
fn two_variable_union() {
let union_a = TwoVariableUnion::A(VariableA {
a: 1,
b: vec![2, 3],
});
let union_b = TwoVariableUnion::B(VariableB {
a: vec![1, 2],
b: 3,
});
assert_encode_decode(&union_a, &[0, 1, 5, 0, 0, 0, 2, 3]);
assert_encode_decode(&union_b, &[1, 5, 0, 0, 0, 3, 1, 2]);
assert_encode_decode(
&TwoVariableUnionStruct { a: union_a },
&[4, 0, 0, 0, 0, 1, 5, 0, 0, 0, 2, 3],
);
assert_encode_decode(
&TwoVariableUnionStruct { a: union_b },
&[4, 0, 0, 0, 1, 5, 0, 0, 0, 3, 1, 2],
);
}
#[derive(PartialEq, Debug, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
enum TwoVecUnion {
A(Vec<u8>),
B(Vec<u8>),
}
#[test]
fn two_vec_union() {
assert_encode_decode(&TwoVecUnion::A(vec![]), &[0]);
assert_encode_decode(&TwoVecUnion::B(vec![]), &[1]);
assert_encode_decode(&TwoVecUnion::A(vec![0]), &[0, 0]);
assert_encode_decode(&TwoVecUnion::B(vec![0]), &[1, 0]);
assert_encode_decode(&TwoVecUnion::A(vec![0, 1]), &[0, 0, 1]);
assert_encode_decode(&TwoVecUnion::B(vec![0, 1]), &[1, 0, 1]);
}
}

17
consensus/ssz_derive/Cargo.toml
View File

@@ -1,17 +0,0 @@
[package]
name = "eth2_ssz_derive"
version = "0.3.0"
authors = ["Paul Hauner <paul@sigmaprime.io>"]
edition = "2021"
description = "Procedural derive macros to accompany the eth2_ssz crate."
license = "Apache-2.0"
[lib]
name = "ssz_derive"
proc-macro = true
[dependencies]
syn = "1.0.42"
proc-macro2 = "1.0.23"
quote = "1.0.7"
darling = "0.13.0"

622
consensus/ssz_derive/src/lib.rs
View File

@@ -1,622 +0,0 @@
#![recursion_limit = "256"]
//! Provides procedural derive macros for the `Encode` and `Decode` traits of the `eth2_ssz` crate.
//!
//! Supports field attributes, see each derive macro for more information.
use darling::{FromDeriveInput, FromMeta};
use proc_macro::TokenStream;
use quote::quote;
use std::convert::TryInto;
use syn::{parse_macro_input, DataEnum, DataStruct, DeriveInput, Ident};
/// The highest possible union selector value (higher values are reserved for backwards compatible
/// extensions).
const MAX_UNION_SELECTOR: u8 = 127;
#[derive(Debug, FromDeriveInput)]
#[darling(attributes(ssz))]
struct StructOpts {
#[darling(default)]
enum_behaviour: Option<String>,
}
/// Field-level configuration.
#[derive(Debug, Default, FromMeta)]
struct FieldOpts {
#[darling(default)]
with: Option<Ident>,
#[darling(default)]
skip_serializing: bool,
#[darling(default)]
skip_deserializing: bool,
}
const ENUM_TRANSPARENT: &str = "transparent";
const ENUM_UNION: &str = "union";
const ENUM_VARIANTS: &[&str] = &[ENUM_TRANSPARENT, ENUM_UNION];
const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute, \
e.g., #[ssz(enum_behaviour = \"transparent\")]";
enum EnumBehaviour {
Transparent,
Union,
}
impl EnumBehaviour {
pub fn new(s: Option<String>) -> Option<Self> {
s.map(|s| match s.as_ref() {
ENUM_TRANSPARENT => EnumBehaviour::Transparent,
ENUM_UNION => EnumBehaviour::Union,
other => panic!(
"{} is an invalid enum_behaviour, use either {:?}",
other, ENUM_VARIANTS
),
})
}
}
fn parse_ssz_fields(struct_data: &syn::DataStruct) -> Vec<(&syn::Type, &syn::Ident, FieldOpts)> {
struct_data
.fields
.iter()
.map(|field| {
let ty = &field.ty;
let ident = match &field.ident {
Some(ref ident) => ident,
_ => panic!("ssz_derive only supports named struct fields."),
};
let field_opts_candidates = field
.attrs
.iter()
.filter(|attr| attr.path.get_ident().map_or(false, |ident| *ident == "ssz"))
.collect::<Vec<_>>();
if field_opts_candidates.len() > 1 {
panic!("more than one field-level \"ssz\" attribute provided")
}
let field_opts = field_opts_candidates
.first()
.map(|attr| {
let meta = attr.parse_meta().unwrap();
FieldOpts::from_meta(&meta).unwrap()
})
.unwrap_or_default();
(ty, ident, field_opts)
})
.collect()
}
/// Implements `ssz::Encode` for some `struct` or `enum`.
#[proc_macro_derive(Encode, attributes(ssz))]
pub fn ssz_encode_derive(input: TokenStream) -> TokenStream {
let item = parse_macro_input!(input as DeriveInput);
let opts = StructOpts::from_derive_input(&item).unwrap();
let enum_opt = EnumBehaviour::new(opts.enum_behaviour);
match &item.data {
syn::Data::Struct(s) => {
if enum_opt.is_some() {
panic!("enum_behaviour is invalid for structs");
}
ssz_encode_derive_struct(&item, s)
}
syn::Data::Enum(s) => match enum_opt.expect(NO_ENUM_BEHAVIOUR_ERROR) {
EnumBehaviour::Transparent => ssz_encode_derive_enum_transparent(&item, s),
EnumBehaviour::Union => ssz_encode_derive_enum_union(&item, s),
},
_ => panic!("ssz_derive only supports structs and enums"),
}
}
/// Derive `ssz::Encode` for a struct.
///
/// Fields are encoded in the order they are defined.
///
/// ## Field attributes
///
/// - `#[ssz(skip_serializing)]`: the field will not be serialized.
fn ssz_encode_derive_struct(derive_input: &DeriveInput, struct_data: &DataStruct) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let field_is_ssz_fixed_len = &mut vec![];
let field_fixed_len = &mut vec![];
let field_ssz_bytes_len = &mut vec![];
let field_encoder_append = &mut vec![];
for (ty, ident, field_opts) in parse_ssz_fields(struct_data) {
if field_opts.skip_serializing {
continue;
}
if let Some(module) = field_opts.with {
let module = quote! { #module::encode };
field_is_ssz_fixed_len.push(quote! { #module::is_ssz_fixed_len() });
field_fixed_len.push(quote! { #module::ssz_fixed_len() });
field_ssz_bytes_len.push(quote! { #module::ssz_bytes_len(&self.#ident) });
field_encoder_append.push(quote! {
encoder.append_parameterized(
#module::is_ssz_fixed_len(),
|buf| #module::ssz_append(&self.#ident, buf)
)
});
} else {
field_is_ssz_fixed_len.push(quote! { <#ty as ssz::Encode>::is_ssz_fixed_len() });
field_fixed_len.push(quote! { <#ty as ssz::Encode>::ssz_fixed_len() });
field_ssz_bytes_len.push(quote! { self.#ident.ssz_bytes_len() });
field_encoder_append.push(quote! { encoder.append(&self.#ident) });
}
}
let output = quote! {
impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
#(
#field_is_ssz_fixed_len &&
)*
true
}
fn ssz_fixed_len() -> usize {
if <Self as ssz::Encode>::is_ssz_fixed_len() {
let mut len: usize = 0;
#(
len = len
.checked_add(#field_fixed_len)
.expect("encode ssz_fixed_len length overflow");
)*
len
} else {
ssz::BYTES_PER_LENGTH_OFFSET
}
}
fn ssz_bytes_len(&self) -> usize {
if <Self as ssz::Encode>::is_ssz_fixed_len() {
<Self as ssz::Encode>::ssz_fixed_len()
} else {
let mut len: usize = 0;
#(
if #field_is_ssz_fixed_len {
len = len
.checked_add(#field_fixed_len)
.expect("encode ssz_bytes_len length overflow");
} else {
len = len
.checked_add(ssz::BYTES_PER_LENGTH_OFFSET)
.expect("encode ssz_bytes_len length overflow for offset");
len = len
.checked_add(#field_ssz_bytes_len)
.expect("encode ssz_bytes_len length overflow for bytes");
}
)*
len
}
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
let mut offset: usize = 0;
#(
offset = offset
.checked_add(#field_fixed_len)
.expect("encode ssz_append offset overflow");
)*
let mut encoder = ssz::SszEncoder::container(buf, offset);
#(
#field_encoder_append;
)*
encoder.finalize();
}
}
};
output.into()
}
/// Derive `ssz::Encode` for an enum in the "transparent" method.
///
/// The "transparent" method is distinct from the "union" method specified in the SSZ specification.
/// When using "transparent", the enum will be ignored and the contained field will be serialized as
/// if the enum does not exist. Since an union variant "selector" is not serialized, it is not
/// possible to reliably decode an enum that is serialized transparently.
///
/// ## Limitations
///
/// Only supports:
/// - Enums with a single field per variant, where
/// - All fields are variably sized from an SSZ-perspective (not fixed size).
///
/// ## Panics
///
/// Will panic at compile-time if the single field requirement isn't met, but will panic *at run
/// time* if the variable-size requirement isn't met.
fn ssz_encode_derive_enum_transparent(
derive_input: &DeriveInput,
enum_data: &DataEnum,
) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let (patterns, assert_exprs): (Vec<_>, Vec<_>) = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if variant.fields.len() != 1 {
panic!("ssz::Encode can only be derived for enums with 1 field per variant");
}
let pattern = quote! {
#name::#variant_name(ref inner)
};
let ty = &(&variant.fields).into_iter().next().unwrap().ty;
let type_assert = quote! {
!<#ty as ssz::Encode>::is_ssz_fixed_len()
};
(pattern, type_assert)
})
.unzip();
let output = quote! {
impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
assert!(
#(
#assert_exprs &&
)* true,
"not all enum variants are variably-sized"
);
false
}
fn ssz_bytes_len(&self) -> usize {
match self {
#(
#patterns => inner.ssz_bytes_len(),
)*
}
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
#(
#patterns => inner.ssz_append(buf),
)*
}
}
}
};
output.into()
}
/// Derive `ssz::Encode` for an `enum` following the "union" SSZ spec.
///
/// The union selector will be determined based upon the order in which the enum variants are
/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant
/// beneath it will have a selector of `1` and so on.
///
/// # Limitations
///
/// Only supports enums where each variant has a single field.
fn ssz_encode_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if variant.fields.len() != 1 {
panic!("ssz::Encode can only be derived for enums with 1 field per variant");
}
let pattern = quote! {
#name::#variant_name(ref inner)
};
pattern
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics ssz::Encode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_bytes_len(&self) -> usize {
match self {
#(
#patterns => inner
.ssz_bytes_len()
.checked_add(1)
.expect("encoded length must be less than usize::max_value"),
)*
}
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
match self {
#(
#patterns => {
let union_selector: u8 = #union_selectors;
debug_assert!(union_selector <= ssz::MAX_UNION_SELECTOR);
buf.push(union_selector);
inner.ssz_append(buf)
},
)*
}
}
}
};
output.into()
}
/// Derive `ssz::Decode` for a struct or enum.
#[proc_macro_derive(Decode, attributes(ssz))]
pub fn ssz_decode_derive(input: TokenStream) -> TokenStream {
let item = parse_macro_input!(input as DeriveInput);
let opts = StructOpts::from_derive_input(&item).unwrap();
let enum_opt = EnumBehaviour::new(opts.enum_behaviour);
match &item.data {
syn::Data::Struct(s) => {
if enum_opt.is_some() {
panic!("enum_behaviour is invalid for structs");
}
ssz_decode_derive_struct(&item, s)
}
syn::Data::Enum(s) => match enum_opt.expect(NO_ENUM_BEHAVIOUR_ERROR) {
EnumBehaviour::Transparent => panic!(
"Decode cannot be derived for enum_behaviour \"{}\", only \"{}\" is valid.",
ENUM_TRANSPARENT, ENUM_UNION
),
EnumBehaviour::Union => ssz_decode_derive_enum_union(&item, s),
},
_ => panic!("ssz_derive only supports structs and enums"),
}
}
/// Implements `ssz::Decode` for some `struct`.
///
/// Fields are decoded in the order they are defined.
///
/// ## Field attributes
///
/// - `#[ssz(skip_deserializing)]`: during de-serialization the field will be instantiated from a
/// `Default` implementation. The decoder will assume that the field was not serialized at all
/// (e.g., if it has been serialized, an error will be raised instead of `Default` overriding it).
fn ssz_decode_derive_struct(item: &DeriveInput, struct_data: &DataStruct) -> TokenStream {
let name = &item.ident;
let (impl_generics, ty_generics, where_clause) = &item.generics.split_for_impl();
let mut register_types = vec![];
let mut field_names = vec![];
let mut fixed_decodes = vec![];
let mut decodes = vec![];
let mut is_fixed_lens = vec![];
let mut fixed_lens = vec![];
for (ty, ident, field_opts) in parse_ssz_fields(struct_data) {
field_names.push(quote! {
#ident
});
// Field should not be deserialized; use a `Default` impl to instantiate.
if field_opts.skip_deserializing {
decodes.push(quote! {
let #ident = <_>::default();
});
fixed_decodes.push(quote! {
let #ident = <_>::default();
});
continue;
}
let is_ssz_fixed_len;
let ssz_fixed_len;
let from_ssz_bytes;
if let Some(module) = field_opts.with {
let module = quote! { #module::decode };
is_ssz_fixed_len = quote! { #module::is_ssz_fixed_len() };
ssz_fixed_len = quote! { #module::ssz_fixed_len() };
from_ssz_bytes = quote! { #module::from_ssz_bytes(slice) };
register_types.push(quote! {
builder.register_type_parameterized(#is_ssz_fixed_len, #ssz_fixed_len)?;
});
decodes.push(quote! {
let #ident = decoder.decode_next_with(|slice| #module::from_ssz_bytes(slice))?;
});
} else {
is_ssz_fixed_len = quote! { <#ty as ssz::Decode>::is_ssz_fixed_len() };
ssz_fixed_len = quote! { <#ty as ssz::Decode>::ssz_fixed_len() };
from_ssz_bytes = quote! { <#ty as ssz::Decode>::from_ssz_bytes(slice) };
register_types.push(quote! {
builder.register_type::<#ty>()?;
});
decodes.push(quote! {
let #ident = decoder.decode_next()?;
});
}
fixed_decodes.push(quote! {
let #ident = {
start = end;
end = end
.checked_add(#ssz_fixed_len)
.ok_or_else(|| ssz::DecodeError::OutOfBoundsByte {
i: usize::max_value()
})?;
let slice = bytes.get(start..end)
.ok_or_else(|| ssz::DecodeError::InvalidByteLength {
len: bytes.len(),
expected: end
})?;
#from_ssz_bytes?
};
});
is_fixed_lens.push(is_ssz_fixed_len);
fixed_lens.push(ssz_fixed_len);
}
let output = quote! {
impl #impl_generics ssz::Decode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
#(
#is_fixed_lens &&
)*
true
}
fn ssz_fixed_len() -> usize {
if <Self as ssz::Decode>::is_ssz_fixed_len() {
let mut len: usize = 0;
#(
len = len
.checked_add(#fixed_lens)
.expect("decode ssz_fixed_len overflow");
)*
len
} else {
ssz::BYTES_PER_LENGTH_OFFSET
}
}
fn from_ssz_bytes(bytes: &[u8]) -> std::result::Result<Self, ssz::DecodeError> {
if <Self as ssz::Decode>::is_ssz_fixed_len() {
if bytes.len() != <Self as ssz::Decode>::ssz_fixed_len() {
return Err(ssz::DecodeError::InvalidByteLength {
len: bytes.len(),
expected: <Self as ssz::Decode>::ssz_fixed_len(),
});
}
let mut start: usize = 0;
let mut end = start;
#(
#fixed_decodes
)*
Ok(Self {
#(
#field_names,
)*
})
} else {
let mut builder = ssz::SszDecoderBuilder::new(bytes);
#(
#register_types
)*
let mut decoder = builder.build()?;
#(
#decodes
)*
Ok(Self {
#(
#field_names,
)*
})
}
}
}
};
output.into()
}
/// Derive `ssz::Decode` for an `enum` following the "union" SSZ spec.
fn ssz_decode_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let (constructors, var_types): (Vec<_>, Vec<_>) = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if variant.fields.len() != 1 {
panic!("ssz::Encode can only be derived for enums with 1 field per variant");
}
let constructor = quote! {
#name::#variant_name
};
let ty = &(&variant.fields).into_iter().next().unwrap().ty;
(constructor, ty)
})
.unzip();
let union_selectors = compute_union_selectors(constructors.len());
let output = quote! {
impl #impl_generics ssz::Decode for #name #ty_generics #where_clause {
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
// Sanity check to ensure the definition here does not drift from the one defined in
// `ssz`.
debug_assert_eq!(#MAX_UNION_SELECTOR, ssz::MAX_UNION_SELECTOR);
let (selector, body) = ssz::split_union_bytes(bytes)?;
match selector.into() {
#(
#union_selectors => {
<#var_types as ssz::Decode>::from_ssz_bytes(body).map(#constructors)
},
)*
other => Err(ssz::DecodeError::UnionSelectorInvalid(other))
}
}
}
};
output.into()
}
fn compute_union_selectors(num_variants: usize) -> Vec<u8> {
let union_selectors = (0..num_variants)
.map(|i| {
i.try_into()
.expect("union selector exceeds u8::max_value, union has too many variants")
})
.collect::<Vec<u8>>();
let highest_selector = union_selectors
.last()
.copied()
.expect("0-variant union is not permitted");
assert!(
highest_selector <= MAX_UNION_SELECTOR,
"union selector {} exceeds limit of {}, enum has too many variants",
highest_selector,
MAX_UNION_SELECTOR
);
union_selectors
}

25
consensus/ssz_types/Cargo.toml
View File

@@ -1,25 +0,0 @@
[package]
name = "eth2_ssz_types"
version = "0.2.2"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
description = "Provides types with unique properties required for SSZ serialization and Merklization."
license = "Apache-2.0"
[lib]
name = "ssz_types"
[dependencies]
tree_hash = "0.4.1"
serde = "1.0.116"
serde_derive = "1.0.116"
ethereum_serde_utils = "1.0.0-beta.0"
ethereum_ssz = "1.0.0-beta.2"
typenum = "1.12.0"
arbitrary = { version = "1.0", features = ["derive"], optional = true }
derivative = "2.1.1"
smallvec = "1.8.0"
[dev-dependencies]
serde_json = "1.0.58"
tree_hash_derive = "0.4.0"

1332
consensus/ssz_types/src/bitfield.rs
View File

File diff suppressed because it is too large Load Diff

446
consensus/ssz_types/src/fixed_vector.rs
View File

@@ -1,446 +0,0 @@
use crate::tree_hash::vec_tree_hash_root;
use crate::Error;
use derivative::Derivative;
use serde_derive::{Deserialize, Serialize};
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut, Index, IndexMut};
use std::slice::SliceIndex;
use tree_hash::Hash256;
use typenum::Unsigned;
pub use typenum;
/// Emulates a SSZ `Vector` (distinct from a Rust `Vec`).
///
/// An ordered, heap-allocated, fixed-length, homogeneous collection of `T`, with `N` values.
///
/// This struct is backed by a Rust `Vec` but constrained such that it must be instantiated with a
/// fixed number of elements and you may not add or remove elements, only modify.
///
/// The length of this struct is fixed at the type-level using
/// [typenum](https://crates.io/crates/typenum).
///
/// ## Note
///
/// Whilst it is possible with this library, SSZ declares that a `FixedVector` with a length of `0`
/// is illegal.
///
/// ## Example
///
/// ```
/// use ssz_types::{FixedVector, typenum};
///
/// let base: Vec<u64> = vec![1, 2, 3, 4];
///
/// // Create a `FixedVector` from a `Vec` that has the expected length.
/// let exact: FixedVector<_, typenum::U4> = FixedVector::from(base.clone());
/// assert_eq!(&exact[..], &[1, 2, 3, 4]);
///
/// // Create a `FixedVector` from a `Vec` that is too long and the `Vec` is truncated.
/// let short: FixedVector<_, typenum::U3> = FixedVector::from(base.clone());
/// assert_eq!(&short[..], &[1, 2, 3]);
///
/// // Create a `FixedVector` from a `Vec` that is too short and the missing values are created
/// // using `std::default::Default`.
/// let long: FixedVector<_, typenum::U5> = FixedVector::from(base);
/// assert_eq!(&long[..], &[1, 2, 3, 4, 0]);
/// ```
#[derive(Debug, Clone, Serialize, Deserialize, Derivative)]
#[derivative(PartialEq, Hash(bound = "T: std::hash::Hash"))]
#[serde(transparent)]
pub struct FixedVector<T, N> {
vec: Vec<T>,
_phantom: PhantomData<N>,
}
impl<T, N: Unsigned> FixedVector<T, N> {
/// Returns `Ok` if the given `vec` equals the fixed length of `Self`. Otherwise returns
/// `Err`.
pub fn new(vec: Vec<T>) -> Result<Self, Error> {
if vec.len() == Self::capacity() {
Ok(Self {
vec,
_phantom: PhantomData,
})
} else {
Err(Error::OutOfBounds {
i: vec.len(),
len: Self::capacity(),
})
}
}
/// Create a new vector filled with clones of `elem`.
pub fn from_elem(elem: T) -> Self
where
T: Clone,
{
Self {
vec: vec![elem; N::to_usize()],
_phantom: PhantomData,
}
}
/// Identical to `self.capacity`, returns the type-level constant length.
///
/// Exists for compatibility with `Vec`.
pub fn len(&self) -> usize {
self.vec.len()
}
/// True if the type-level constant length of `self` is zero.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Returns the type-level constant length.
pub fn capacity() -> usize {
N::to_usize()
}
}
impl<T: Default, N: Unsigned> From<Vec<T>> for FixedVector<T, N> {
fn from(mut vec: Vec<T>) -> Self {
vec.resize_with(Self::capacity(), Default::default);
Self {
vec,
_phantom: PhantomData,
}
}
}
impl<T, N: Unsigned> From<FixedVector<T, N>> for Vec<T> {
fn from(vector: FixedVector<T, N>) -> Vec<T> {
vector.vec
}
}
impl<T: Default, N: Unsigned> Default for FixedVector<T, N> {
fn default() -> Self {
Self {
vec: (0..N::to_usize()).map(|_| T::default()).collect(),
_phantom: PhantomData,
}
}
}
impl<T, N: Unsigned, I: SliceIndex<[T]>> Index<I> for FixedVector<T, N> {
type Output = I::Output;
#[inline]
fn index(&self, index: I) -> &Self::Output {
Index::index(&self.vec, index)
}
}
impl<T, N: Unsigned, I: SliceIndex<[T]>> IndexMut<I> for FixedVector<T, N> {
#[inline]
fn index_mut(&mut self, index: I) -> &mut Self::Output {
IndexMut::index_mut(&mut self.vec, index)
}
}
impl<T, N: Unsigned> Deref for FixedVector<T, N> {
type Target = [T];
fn deref(&self) -> &[T] {
&self.vec[..]
}
}
// This implementation is required to use `get_mut` to access elements.
//
// It's safe because none of the methods on mutable slices allow changing the length
// of the backing vec.
impl<T, N: Unsigned> DerefMut for FixedVector<T, N> {
fn deref_mut(&mut self) -> &mut [T] {
&mut self.vec[..]
}
}
impl<T, N: Unsigned> tree_hash::TreeHash for FixedVector<T, N>
where
T: tree_hash::TreeHash,
{
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::Vector
}
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("Vector should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("Vector should never be packed.")
}
fn tree_hash_root(&self) -> Hash256 {
vec_tree_hash_root::<T, N>(&self.vec)
}
}
impl<T, N: Unsigned> ssz::Encode for FixedVector<T, N>
where
T: ssz::Encode,
{
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
if <Self as ssz::Encode>::is_ssz_fixed_len() {
T::ssz_fixed_len() * N::to_usize()
} else {
ssz::BYTES_PER_LENGTH_OFFSET
}
}
fn ssz_bytes_len(&self) -> usize {
self.vec.ssz_bytes_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
if T::is_ssz_fixed_len() {
buf.reserve(T::ssz_fixed_len() * self.len());
for item in &self.vec {
item.ssz_append(buf);
}
} else {
let mut encoder =
ssz::SszEncoder::container(buf, self.len() * ssz::BYTES_PER_LENGTH_OFFSET);
for item in &self.vec {
encoder.append(item);
}
encoder.finalize();
}
}
}
impl<T, N: Unsigned> ssz::Decode for FixedVector<T, N>
where
T: ssz::Decode,
{
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
if <Self as ssz::Decode>::is_ssz_fixed_len() {
T::ssz_fixed_len() * N::to_usize()
} else {
ssz::BYTES_PER_LENGTH_OFFSET
}
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
let fixed_len = N::to_usize();
if bytes.is_empty() {
Err(ssz::DecodeError::InvalidByteLength {
len: 0,
expected: 1,
})
} else if T::is_ssz_fixed_len() {
let num_items = bytes
.len()
.checked_div(T::ssz_fixed_len())
.ok_or(ssz::DecodeError::ZeroLengthItem)?;
if num_items != fixed_len {
return Err(ssz::DecodeError::BytesInvalid(format!(
"FixedVector of {} items has {} items",
num_items, fixed_len
)));
}
bytes
.chunks(T::ssz_fixed_len())
.map(|chunk| T::from_ssz_bytes(chunk))
.collect::<Result<Vec<T>, _>>()
.and_then(|vec| {
Self::new(vec).map_err(|e| {
ssz::DecodeError::BytesInvalid(format!(
"Wrong number of FixedVector elements: {:?}",
e
))
})
})
} else {
let vec = ssz::decode_list_of_variable_length_items(bytes, Some(fixed_len))?;
Self::new(vec).map_err(|e| {
ssz::DecodeError::BytesInvalid(format!(
"Wrong number of FixedVector elements: {:?}",
e
))
})
}
}
}
#[cfg(feature = "arbitrary")]
impl<'a, T: arbitrary::Arbitrary<'a>, N: 'static + Unsigned> arbitrary::Arbitrary<'a>
for FixedVector<T, N>
{
fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
let size = N::to_usize();
let mut vec: Vec<T> = Vec::with_capacity(size);
for _ in 0..size {
vec.push(<T>::arbitrary(u)?);
}
Ok(Self::new(vec).map_err(|_| arbitrary::Error::IncorrectFormat)?)
}
}
#[cfg(test)]
mod test {
use super::*;
use ssz::*;
use tree_hash::{merkle_root, TreeHash};
use tree_hash_derive::TreeHash;
use typenum::*;
#[test]
fn new() {
let vec = vec![42; 5];
let fixed: Result<FixedVector<u64, U4>, _> = FixedVector::new(vec);
assert!(fixed.is_err());
let vec = vec![42; 3];
let fixed: Result<FixedVector<u64, U4>, _> = FixedVector::new(vec);
assert!(fixed.is_err());
let vec = vec![42; 4];
let fixed: Result<FixedVector<u64, U4>, _> = FixedVector::new(vec);
assert!(fixed.is_ok());
}
#[test]
fn indexing() {
let vec = vec![1, 2];
let mut fixed: FixedVector<u64, U8192> = vec.clone().into();
assert_eq!(fixed[0], 1);
assert_eq!(&fixed[0..1], &vec[0..1]);
assert_eq!((fixed[..]).len(), 8192);
fixed[1] = 3;
assert_eq!(fixed[1], 3);
}
#[test]
fn length() {
let vec = vec![42; 5];
let fixed: FixedVector<u64, U4> = FixedVector::from(vec.clone());
assert_eq!(&fixed[..], &vec[0..4]);
let vec = vec![42; 3];
let fixed: FixedVector<u64, U4> = FixedVector::from(vec.clone());
assert_eq!(&fixed[0..3], &vec[..]);
assert_eq!(&fixed[..], &vec![42, 42, 42, 0][..]);
let vec = vec![];
let fixed: FixedVector<u64, U4> = FixedVector::from(vec);
assert_eq!(&fixed[..], &vec![0, 0, 0, 0][..]);
}
#[test]
fn deref() {
let vec = vec![0, 2, 4, 6];
let fixed: FixedVector<u64, U4> = FixedVector::from(vec);
assert_eq!(fixed.first(), Some(&0));
assert_eq!(fixed.get(3), Some(&6));
assert_eq!(fixed.get(4), None);
}
#[test]
fn ssz_encode() {
let vec: FixedVector<u16, U2> = vec![0; 2].into();
assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]);
assert_eq!(<FixedVector<u16, U2> as Encode>::ssz_fixed_len(), 4);
}
fn ssz_round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(item: T) {
let encoded = &item.as_ssz_bytes();
assert_eq!(item.ssz_bytes_len(), encoded.len());
assert_eq!(T::from_ssz_bytes(encoded), Ok(item));
}
#[test]
fn ssz_round_trip_u16_len_8() {
ssz_round_trip::<FixedVector<u16, U8>>(vec![42; 8].into());
ssz_round_trip::<FixedVector<u16, U8>>(vec![0; 8].into());
}
#[test]
fn tree_hash_u8() {
let fixed: FixedVector<u8, U0> = FixedVector::from(vec![]);
assert_eq!(fixed.tree_hash_root(), merkle_root(&[0; 8], 0));
let fixed: FixedVector<u8, U1> = FixedVector::from(vec![0; 1]);
assert_eq!(fixed.tree_hash_root(), merkle_root(&[0; 8], 0));
let fixed: FixedVector<u8, U8> = FixedVector::from(vec![0; 8]);
assert_eq!(fixed.tree_hash_root(), merkle_root(&[0; 8], 0));
let fixed: FixedVector<u8, U16> = FixedVector::from(vec![42; 16]);
assert_eq!(fixed.tree_hash_root(), merkle_root(&[42; 16], 0));
let source: Vec<u8> = (0..16).collect();
let fixed: FixedVector<u8, U16> = FixedVector::from(source.clone());
assert_eq!(fixed.tree_hash_root(), merkle_root(&source, 0));
}
#[derive(Clone, Copy, TreeHash, Default)]
struct A {
a: u32,
b: u32,
}
fn repeat(input: &[u8], n: usize) -> Vec<u8> {
let mut output = vec![];
for _ in 0..n {
output.append(&mut input.to_vec());
}
output
}
#[test]
fn tree_hash_composite() {
let a = A { a: 0, b: 1 };
let fixed: FixedVector<A, U0> = FixedVector::from(vec![]);
assert_eq!(fixed.tree_hash_root(), merkle_root(&[0; 32], 0));
let fixed: FixedVector<A, U1> = FixedVector::from(vec![a]);
assert_eq!(
fixed.tree_hash_root(),
merkle_root(a.tree_hash_root().as_bytes(), 0)
);
let fixed: FixedVector<A, U8> = FixedVector::from(vec![a; 8]);
assert_eq!(
fixed.tree_hash_root(),
merkle_root(&repeat(a.tree_hash_root().as_bytes(), 8), 0)
);
let fixed: FixedVector<A, U13> = FixedVector::from(vec![a; 13]);
assert_eq!(
fixed.tree_hash_root(),
merkle_root(&repeat(a.tree_hash_root().as_bytes(), 13), 0)
);
let fixed: FixedVector<A, U16> = FixedVector::from(vec![a; 16]);
assert_eq!(
fixed.tree_hash_root(),
merkle_root(&repeat(a.tree_hash_root().as_bytes(), 16), 0)
);
}
}

72
consensus/ssz_types/src/lib.rs
View File

@@ -1,72 +0,0 @@
//! Provides types with unique properties required for SSZ serialization and Merklization:
//!
//! - `FixedVector`: A heap-allocated list with a size that is fixed at compile time.
//! - `VariableList`: A heap-allocated list that cannot grow past a type-level maximum length.
//! - `BitList`: A heap-allocated bitfield that with a type-level _maximum_ length.
//! - `BitVector`: A heap-allocated bitfield that with a type-level _fixed__ length.
//!
//! These structs are required as SSZ serialization and Merklization rely upon type-level lengths
//! for padding and verification.
//!
//! Adheres to the Ethereum 2.0 [SSZ
//! specification](https://github.com/ethereum/eth2.0-specs/blob/v0.12.1/ssz/simple-serialize.md)
//! at v0.12.1.
//!
//! ## Example
//! ```
//! use ssz_types::*;
//!
//! pub struct Example {
//! bit_vector: BitVector<typenum::U8>,
//! bit_list: BitList<typenum::U8>,
//! variable_list: VariableList<u64, typenum::U8>,
//! fixed_vector: FixedVector<u64, typenum::U8>,
//! }
//!
//! let mut example = Example {
//! bit_vector: Bitfield::new(),
//! bit_list: Bitfield::with_capacity(4).unwrap(),
//! variable_list: <_>::from(vec![0, 1]),
//! fixed_vector: <_>::from(vec![2, 3]),
//! };
//!
//! assert_eq!(example.bit_vector.len(), 8);
//! assert_eq!(example.bit_list.len(), 4);
//! assert_eq!(&example.variable_list[..], &[0, 1]);
//! assert_eq!(&example.fixed_vector[..], &[2, 3, 0, 0, 0, 0, 0, 0]);
//!
//! ```
#[macro_use]
mod bitfield;
mod fixed_vector;
pub mod serde_utils;
mod tree_hash;
mod variable_list;
pub use bitfield::{BitList, BitVector, Bitfield};
pub use fixed_vector::FixedVector;
pub use typenum;
pub use variable_list::VariableList;
pub mod length {
pub use crate::bitfield::{Fixed, Variable};
}
/// Returned when an item encounters an error.
#[derive(PartialEq, Debug, Clone)]
pub enum Error {
OutOfBounds {
i: usize,
len: usize,
},
/// A `BitList` does not have a set bit, therefore it's length is unknowable.
MissingLengthInformation,
/// A `BitList` has excess bits set to true.
ExcessBits,
/// A `BitList` has an invalid number of bytes for a given bit length.
InvalidByteCount {
given: usize,
expected: usize,
},
}

22
consensus/ssz_types/src/serde_utils/hex_fixed_vec.rs
View File

@@ -1,22 +0,0 @@
use crate::FixedVector;
use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
use serde::{Deserializer, Serializer};
use typenum::Unsigned;
pub fn serialize<S, U>(bytes: &FixedVector<u8, U>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
U: Unsigned,
{
serializer.serialize_str(&hex::encode(&bytes[..]))
}
pub fn deserialize<'de, D, U>(deserializer: D) -> Result<FixedVector<u8, U>, D::Error>
where
D: Deserializer<'de>,
U: Unsigned,
{
let vec = deserializer.deserialize_string(PrefixedHexVisitor)?;
FixedVector::new(vec)
.map_err(|e| serde::de::Error::custom(format!("invalid fixed vector: {:?}", e)))
}

23
consensus/ssz_types/src/serde_utils/hex_var_list.rs
View File

@@ -1,23 +0,0 @@
//! Serialize `VariableList<u8, N>` as 0x-prefixed hex string.
use crate::VariableList;
use eth2_serde_utils::hex::{self, PrefixedHexVisitor};
use serde::{Deserializer, Serializer};
use typenum::Unsigned;
pub fn serialize<S, N>(bytes: &VariableList<u8, N>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
N: Unsigned,
{
serializer.serialize_str(&hex::encode(&**bytes))
}
pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u8, N>, D::Error>
where
D: Deserializer<'de>,
N: Unsigned,
{
let bytes = deserializer.deserialize_str(PrefixedHexVisitor)?;
VariableList::new(bytes)
.map_err(|e| serde::de::Error::custom(format!("invalid variable list: {:?}", e)))
}

77
consensus/ssz_types/src/serde_utils/list_of_hex_var_list.rs
View File

@@ -1,77 +0,0 @@
//! Serialize `VaraibleList<VariableList<u8, M>, N>` as list of 0x-prefixed hex string.
use crate::VariableList;
use serde::{ser::SerializeSeq, Deserialize, Deserializer, Serialize, Serializer};
use std::marker::PhantomData;
use typenum::Unsigned;
#[derive(Deserialize)]
#[serde(transparent)]
pub struct WrappedListOwned<N: Unsigned>(
#[serde(with = "crate::serde_utils::hex_var_list")] VariableList<u8, N>,
);
#[derive(Serialize)]
#[serde(transparent)]
pub struct WrappedListRef<'a, N: Unsigned>(
#[serde(with = "crate::serde_utils::hex_var_list")] &'a VariableList<u8, N>,
);
pub fn serialize<S, M, N>(
list: &VariableList<VariableList<u8, M>, N>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: Serializer,
M: Unsigned,
N: Unsigned,
{
let mut seq = serializer.serialize_seq(Some(list.len()))?;
for bytes in list {
seq.serialize_element(&WrappedListRef(bytes))?;
}
seq.end()
}
#[derive(Default)]
pub struct Visitor<M, N> {
_phantom_m: PhantomData<M>,
_phantom_n: PhantomData<N>,
}
impl<'a, M, N> serde::de::Visitor<'a> for Visitor<M, N>
where
M: Unsigned,
N: Unsigned,
{
type Value = VariableList<VariableList<u8, M>, N>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a list of 0x-prefixed hex bytes")
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let mut list: VariableList<VariableList<u8, M>, N> = <_>::default();
while let Some(val) = seq.next_element::<WrappedListOwned<M>>()? {
list.push(val.0).map_err(|e| {
serde::de::Error::custom(format!("failed to push value to list: {:?}.", e))
})?;
}
Ok(list)
}
}
pub fn deserialize<'de, D, M, N>(
deserializer: D,
) -> Result<VariableList<VariableList<u8, M>, N>, D::Error>
where
D: Deserializer<'de>,
M: Unsigned,
N: Unsigned,
{
deserializer.deserialize_seq(Visitor::default())
}

5
consensus/ssz_types/src/serde_utils/mod.rs
View File

@@ -1,5 +0,0 @@
pub mod hex_fixed_vec;
pub mod hex_var_list;
pub mod list_of_hex_var_list;
pub mod quoted_u64_fixed_vec;
pub mod quoted_u64_var_list;

113
consensus/ssz_types/src/serde_utils/quoted_u64_fixed_vec.rs
View File

@@ -1,113 +0,0 @@
//! Formats `FixedVector<u64,N>` using quotes.
//!
//! E.g., `FixedVector::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
//!
//! Quotes can be optional during decoding. If `N` does not equal the length deserialization will fail.
use crate::serde_utils::quoted_u64_var_list::deserialize_max;
use crate::FixedVector;
use eth2_serde_utils::quoted_u64_vec::QuotedIntWrapper;
use serde::ser::SerializeSeq;
use serde::{Deserializer, Serializer};
use std::marker::PhantomData;
use typenum::Unsigned;
pub struct QuotedIntFixedVecVisitor<N> {
_phantom: PhantomData<N>,
}
impl<'a, N> serde::de::Visitor<'a> for QuotedIntFixedVecVisitor<N>
where
N: Unsigned,
{
type Value = FixedVector<u64, N>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a list of quoted or unquoted integers")
}
fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let vec = deserialize_max(seq, N::to_usize())?;
let fix: FixedVector<u64, N> = FixedVector::new(vec)
.map_err(|e| serde::de::Error::custom(format!("FixedVector: {:?}", e)))?;
Ok(fix)
}
}
pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(value.len()))?;
for &int in value {
seq.serialize_element(&QuotedIntWrapper { int })?;
}
seq.end()
}
pub fn deserialize<'de, D, N>(deserializer: D) -> Result<FixedVector<u64, N>, D::Error>
where
D: Deserializer<'de>,
N: Unsigned,
{
deserializer.deserialize_any(QuotedIntFixedVecVisitor {
_phantom: PhantomData,
})
}
#[cfg(test)]
mod test {
use super::*;
use serde_derive::{Deserialize, Serialize};
use typenum::U4;
#[derive(Debug, Serialize, Deserialize)]
struct Obj {
#[serde(with = "crate::serde_utils::quoted_u64_fixed_vec")]
values: FixedVector<u64, U4>,
}
#[test]
fn quoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn unquoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn mixed_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
let expected: FixedVector<u64, U4> = FixedVector::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn empty_list_err() {
serde_json::from_str::<Obj>(r#"{ "values": [] }"#).unwrap_err();
}
#[test]
fn short_list_err() {
serde_json::from_str::<Obj>(r#"{ "values": [1, 2] }"#).unwrap_err();
}
#[test]
fn long_list_err() {
serde_json::from_str::<Obj>(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap_err();
}
#[test]
fn whole_list_quoted_err() {
serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
}
}

139
consensus/ssz_types/src/serde_utils/quoted_u64_var_list.rs
View File

@@ -1,139 +0,0 @@
//! Formats `VariableList<u64,N>` using quotes.
//!
//! E.g., `VariableList::from(vec![0, 1, 2])` serializes as `["0", "1", "2"]`.
//!
//! Quotes can be optional during decoding. If the length of the `Vec` is greater than `N`, deserialization fails.
use crate::VariableList;
use eth2_serde_utils::quoted_u64_vec::QuotedIntWrapper;
use serde::ser::SerializeSeq;
use serde::{Deserializer, Serializer};
use std::marker::PhantomData;
use typenum::Unsigned;
pub struct QuotedIntVarListVisitor<N> {
_phantom: PhantomData<N>,
}
impl<'a, N> serde::de::Visitor<'a> for QuotedIntVarListVisitor<N>
where
N: Unsigned,
{
type Value = VariableList<u64, N>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(formatter, "a list of quoted or unquoted integers")
}
fn visit_seq<A>(self, seq: A) -> Result<Self::Value, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let vec = deserialize_max(seq, N::to_usize())?;
let list: VariableList<u64, N> = VariableList::new(vec)
.map_err(|e| serde::de::Error::custom(format!("VariableList: {:?}", e)))?;
Ok(list)
}
}
pub fn serialize<S>(value: &[u64], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(value.len()))?;
for &int in value {
seq.serialize_element(&QuotedIntWrapper { int })?;
}
seq.end()
}
pub fn deserialize<'de, D, N>(deserializer: D) -> Result<VariableList<u64, N>, D::Error>
where
D: Deserializer<'de>,
N: Unsigned,
{
deserializer.deserialize_any(QuotedIntVarListVisitor {
_phantom: PhantomData,
})
}
/// Returns a `Vec` of no more than `max_items` length.
pub(crate) fn deserialize_max<'a, A>(mut seq: A, max_items: usize) -> Result<Vec<u64>, A::Error>
where
A: serde::de::SeqAccess<'a>,
{
let mut vec = vec![];
let mut counter = 0;
while let Some(val) = seq.next_element()? {
let val: QuotedIntWrapper = val;
counter += 1;
if counter > max_items {
return Err(serde::de::Error::custom(format!(
"Deserialization failed. Length cannot be greater than {}.",
max_items
)));
}
vec.push(val.int);
}
Ok(vec)
}
#[cfg(test)]
mod test {
use super::*;
use serde_derive::{Deserialize, Serialize};
use typenum::U4;
#[derive(Debug, Serialize, Deserialize)]
struct Obj {
#[serde(with = "crate::serde_utils::quoted_u64_var_list")]
values: VariableList<u64, U4>,
}
#[test]
fn quoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", "2", "3", "4"] }"#).unwrap();
let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn unquoted_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2, 3, 4] }"#).unwrap();
let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn mixed_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": ["1", 2, "3", "4"] }"#).unwrap();
let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2, 3, 4]);
assert_eq!(obj.values, expected);
}
#[test]
fn empty_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [] }"#).unwrap();
assert!(obj.values.is_empty());
}
#[test]
fn short_list_success() {
let obj: Obj = serde_json::from_str(r#"{ "values": [1, 2] }"#).unwrap();
let expected: VariableList<u64, U4> = VariableList::from(vec![1, 2]);
assert_eq!(obj.values, expected);
}
#[test]
fn long_list_err() {
serde_json::from_str::<Obj>(r#"{ "values": [1, 2, 3, 4, 5] }"#).unwrap_err();
}
#[test]
fn whole_list_quoted_err() {
serde_json::from_str::<Obj>(r#"{ "values": "[1, 2, 3, 4]" }"#).unwrap_err();
}
}

58
consensus/ssz_types/src/tree_hash.rs
View File

@@ -1,58 +0,0 @@
use tree_hash::{Hash256, MerkleHasher, TreeHash, TreeHashType, BYTES_PER_CHUNK};
use typenum::Unsigned;
/// A helper function providing common functionality between the `TreeHash` implementations for
/// `FixedVector` and `VariableList`.
pub fn vec_tree_hash_root<T, N>(vec: &[T]) -> Hash256
where
T: TreeHash,
N: Unsigned,
{
match T::tree_hash_type() {
TreeHashType::Basic => {
let mut hasher = MerkleHasher::with_leaves(
(N::to_usize() + T::tree_hash_packing_factor() - 1) / T::tree_hash_packing_factor(),
);
for item in vec {
hasher
.write(&item.tree_hash_packed_encoding())
.expect("ssz_types variable vec should not contain more elements than max");
}
hasher
.finish()
.expect("ssz_types variable vec should not have a remaining buffer")
}
TreeHashType::Container | TreeHashType::List | TreeHashType::Vector => {
let mut hasher = MerkleHasher::with_leaves(N::to_usize());
for item in vec {
hasher
.write(item.tree_hash_root().as_bytes())
.expect("ssz_types vec should not contain more elements than max");
}
hasher
.finish()
.expect("ssz_types vec should not have a remaining buffer")
}
}
}
/// A helper function providing common functionality for finding the Merkle root of some bytes that
/// represent a bitfield.
pub fn bitfield_bytes_tree_hash_root<N: Unsigned>(bytes: &[u8]) -> Hash256 {
let byte_size = (N::to_usize() + 7) / 8;
let leaf_count = (byte_size + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK;
let mut hasher = MerkleHasher::with_leaves(leaf_count);
hasher
.write(bytes)
.expect("bitfield should not exceed tree hash leaf limit");
hasher
.finish()
.expect("bitfield tree hash buffer should not exceed leaf limit")
}

468
consensus/ssz_types/src/variable_list.rs
View File

@@ -1,468 +0,0 @@
use crate::tree_hash::vec_tree_hash_root;
use crate::Error;
use derivative::Derivative;
use serde_derive::{Deserialize, Serialize};
use std::marker::PhantomData;
use std::ops::{Deref, DerefMut, Index, IndexMut};
use std::slice::SliceIndex;
use tree_hash::Hash256;
use typenum::Unsigned;
pub use typenum;
/// Emulates a SSZ `List`.
///
/// An ordered, heap-allocated, variable-length, homogeneous collection of `T`, with no more than
/// `N` values.
///
/// This struct is backed by a Rust `Vec` but constrained such that it must be instantiated with a
/// fixed number of elements and you may not add or remove elements, only modify.
///
/// The length of this struct is fixed at the type-level using
/// [typenum](https://crates.io/crates/typenum).
///
/// ## Example
///
/// ```
/// use ssz_types::{VariableList, typenum};
///
/// let base: Vec<u64> = vec![1, 2, 3, 4];
///
/// // Create a `VariableList` from a `Vec` that has the expected length.
/// let exact: VariableList<_, typenum::U4> = VariableList::from(base.clone());
/// assert_eq!(&exact[..], &[1, 2, 3, 4]);
///
/// // Create a `VariableList` from a `Vec` that is too long and the `Vec` is truncated.
/// let short: VariableList<_, typenum::U3> = VariableList::from(base.clone());
/// assert_eq!(&short[..], &[1, 2, 3]);
///
/// // Create a `VariableList` from a `Vec` that is shorter than the maximum.
/// let mut long: VariableList<_, typenum::U5> = VariableList::from(base);
/// assert_eq!(&long[..], &[1, 2, 3, 4]);
///
/// // Push a value to if it does not exceed the maximum
/// long.push(5).unwrap();
/// assert_eq!(&long[..], &[1, 2, 3, 4, 5]);
///
/// // Push a value to if it _does_ exceed the maximum.
/// assert!(long.push(6).is_err());
/// ```
#[derive(Debug, Clone, Serialize, Deserialize, Derivative)]
#[derivative(PartialEq, Eq, Hash(bound = "T: std::hash::Hash"))]
#[serde(transparent)]
pub struct VariableList<T, N> {
vec: Vec<T>,
_phantom: PhantomData<N>,
}
impl<T, N: Unsigned> VariableList<T, N> {
/// Returns `Some` if the given `vec` equals the fixed length of `Self`. Otherwise returns
/// `None`.
pub fn new(vec: Vec<T>) -> Result<Self, Error> {
if vec.len() <= N::to_usize() {
Ok(Self {
vec,
_phantom: PhantomData,
})
} else {
Err(Error::OutOfBounds {
i: vec.len(),
len: Self::max_len(),
})
}
}
/// Create an empty list.
pub fn empty() -> Self {
Self {
vec: vec![],
_phantom: PhantomData,
}
}
/// Returns the number of values presently in `self`.
pub fn len(&self) -> usize {
self.vec.len()
}
/// True if `self` does not contain any values.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Returns the type-level maximum length.
pub fn max_len() -> usize {
N::to_usize()
}
/// Appends `value` to the back of `self`.
///
/// Returns `Err(())` when appending `value` would exceed the maximum length.
pub fn push(&mut self, value: T) -> Result<(), Error> {
if self.vec.len() < Self::max_len() {
self.vec.push(value);
Ok(())
} else {
Err(Error::OutOfBounds {
i: self.vec.len() + 1,
len: Self::max_len(),
})
}
}
}
impl<T, N: Unsigned> From<Vec<T>> for VariableList<T, N> {
fn from(mut vec: Vec<T>) -> Self {
vec.truncate(N::to_usize());
Self {
vec,
_phantom: PhantomData,
}
}
}
impl<T, N: Unsigned> From<VariableList<T, N>> for Vec<T> {
fn from(list: VariableList<T, N>) -> Vec<T> {
list.vec
}
}
impl<T, N: Unsigned> Default for VariableList<T, N> {
fn default() -> Self {
Self {
vec: Vec::default(),
_phantom: PhantomData,
}
}
}
impl<T, N: Unsigned, I: SliceIndex<[T]>> Index<I> for VariableList<T, N> {
type Output = I::Output;
#[inline]
fn index(&self, index: I) -> &Self::Output {
Index::index(&self.vec, index)
}
}
impl<T, N: Unsigned, I: SliceIndex<[T]>> IndexMut<I> for VariableList<T, N> {
#[inline]
fn index_mut(&mut self, index: I) -> &mut Self::Output {
IndexMut::index_mut(&mut self.vec, index)
}
}
impl<T, N: Unsigned> Deref for VariableList<T, N> {
type Target = [T];
fn deref(&self) -> &[T] {
&self.vec[..]
}
}
impl<T, N: Unsigned> DerefMut for VariableList<T, N> {
fn deref_mut(&mut self) -> &mut [T] {
&mut self.vec[..]
}
}
impl<'a, T, N: Unsigned> IntoIterator for &'a VariableList<T, N> {
type Item = &'a T;
type IntoIter = std::slice::Iter<'a, T>;
fn into_iter(self) -> Self::IntoIter {
self.iter()
}
}
impl<T, N: Unsigned> tree_hash::TreeHash for VariableList<T, N>
where
T: tree_hash::TreeHash,
{
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::List
}
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("List should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("List should never be packed.")
}
fn tree_hash_root(&self) -> Hash256 {
let root = vec_tree_hash_root::<T, N>(&self.vec);
tree_hash::mix_in_length(&root, self.len())
}
}
impl<T, N: Unsigned> ssz::Encode for VariableList<T, N>
where
T: ssz::Encode,
{
fn is_ssz_fixed_len() -> bool {
<Vec<T>>::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
<Vec<T>>::ssz_fixed_len()
}
fn ssz_bytes_len(&self) -> usize {
self.vec.ssz_bytes_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
self.vec.ssz_append(buf)
}
}
impl<T, N> ssz::Decode for VariableList<T, N>
where
T: ssz::Decode,
N: Unsigned,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, ssz::DecodeError> {
let max_len = N::to_usize();
if bytes.is_empty() {
Ok(vec![].into())
} else if T::is_ssz_fixed_len() {
let num_items = bytes
.len()
.checked_div(T::ssz_fixed_len())
.ok_or(ssz::DecodeError::ZeroLengthItem)?;
if num_items > max_len {
return Err(ssz::DecodeError::BytesInvalid(format!(
"VariableList of {} items exceeds maximum of {}",
num_items, max_len
)));
}
bytes
.chunks(T::ssz_fixed_len())
.try_fold(Vec::with_capacity(num_items), |mut vec, chunk| {
vec.push(T::from_ssz_bytes(chunk)?);
Ok(vec)
})
.map(Into::into)
} else {
ssz::decode_list_of_variable_length_items(bytes, Some(max_len))
.map(|vec: Vec<_>| vec.into())
}
}
}
#[cfg(feature = "arbitrary")]
impl<'a, T: arbitrary::Arbitrary<'a>, N: 'static + Unsigned> arbitrary::Arbitrary<'a>
for VariableList<T, N>
{
fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
let max_size = N::to_usize();
let rand = usize::arbitrary(u)?;
let size = std::cmp::min(rand, max_size);
let mut vec: Vec<T> = Vec::with_capacity(size);
for _ in 0..size {
vec.push(<T>::arbitrary(u)?);
}
Ok(Self::new(vec).map_err(|_| arbitrary::Error::IncorrectFormat)?)
}
}
#[cfg(test)]
mod test {
use super::*;
use ssz::*;
use tree_hash::{merkle_root, TreeHash};
use tree_hash_derive::TreeHash;
use typenum::*;
#[test]
fn new() {
let vec = vec![42; 5];
let fixed: Result<VariableList<u64, U4>, _> = VariableList::new(vec);
assert!(fixed.is_err());
let vec = vec![42; 3];
let fixed: Result<VariableList<u64, U4>, _> = VariableList::new(vec);
assert!(fixed.is_ok());
let vec = vec![42; 4];
let fixed: Result<VariableList<u64, U4>, _> = VariableList::new(vec);
assert!(fixed.is_ok());
}
#[test]
fn indexing() {
let vec = vec![1, 2];
let mut fixed: VariableList<u64, U8192> = vec.clone().into();
assert_eq!(fixed[0], 1);
assert_eq!(&fixed[0..1], &vec[0..1]);
assert_eq!((fixed[..]).len(), 2);
fixed[1] = 3;
assert_eq!(fixed[1], 3);
}
#[test]
fn length() {
let vec = vec![42; 5];
let fixed: VariableList<u64, U4> = VariableList::from(vec.clone());
assert_eq!(&fixed[..], &vec[0..4]);
let vec = vec![42; 3];
let fixed: VariableList<u64, U4> = VariableList::from(vec.clone());
assert_eq!(&fixed[0..3], &vec[..]);
assert_eq!(&fixed[..], &vec![42, 42, 42][..]);
let vec = vec![];
let fixed: VariableList<u64, U4> = VariableList::from(vec);
assert_eq!(&fixed[..], &[] as &[u64]);
}
#[test]
fn deref() {
let vec = vec![0, 2, 4, 6];
let fixed: VariableList<u64, U4> = VariableList::from(vec);
assert_eq!(fixed.first(), Some(&0));
assert_eq!(fixed.get(3), Some(&6));
assert_eq!(fixed.get(4), None);
}
#[test]
fn encode() {
let vec: VariableList<u16, U2> = vec![0; 2].into();
assert_eq!(vec.as_ssz_bytes(), vec![0, 0, 0, 0]);
assert_eq!(<VariableList<u16, U2> as Encode>::ssz_fixed_len(), 4);
}
fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(item: T) {
let encoded = &item.as_ssz_bytes();
assert_eq!(item.ssz_bytes_len(), encoded.len());
assert_eq!(T::from_ssz_bytes(encoded), Ok(item));
}
#[test]
fn u16_len_8() {
round_trip::<VariableList<u16, U8>>(vec![42; 8].into());
round_trip::<VariableList<u16, U8>>(vec![0; 8].into());
}
fn root_with_length(bytes: &[u8], len: usize) -> Hash256 {
let root = merkle_root(bytes, 0);
tree_hash::mix_in_length(&root, len)
}
#[test]
fn tree_hash_u8() {
let fixed: VariableList<u8, U0> = VariableList::from(vec![]);
assert_eq!(fixed.tree_hash_root(), root_with_length(&[0; 8], 0));
for i in 0..=1 {
let fixed: VariableList<u8, U1> = VariableList::from(vec![0; i]);
assert_eq!(fixed.tree_hash_root(), root_with_length(&vec![0; i], i));
}
for i in 0..=8 {
let fixed: VariableList<u8, U8> = VariableList::from(vec![0; i]);
assert_eq!(fixed.tree_hash_root(), root_with_length(&vec![0; i], i));
}
for i in 0..=13 {
let fixed: VariableList<u8, U13> = VariableList::from(vec![0; i]);
assert_eq!(fixed.tree_hash_root(), root_with_length(&vec![0; i], i));
}
for i in 0..=16 {
let fixed: VariableList<u8, U16> = VariableList::from(vec![0; i]);
assert_eq!(fixed.tree_hash_root(), root_with_length(&vec![0; i], i));
}
let source: Vec<u8> = (0..16).collect();
let fixed: VariableList<u8, U16> = VariableList::from(source.clone());
assert_eq!(fixed.tree_hash_root(), root_with_length(&source, 16));
}
#[derive(Clone, Copy, TreeHash, Default)]
struct A {
a: u32,
b: u32,
}
fn repeat(input: &[u8], n: usize) -> Vec<u8> {
let mut output = vec![];
for _ in 0..n {
output.append(&mut input.to_vec());
}
output
}
fn padded_root_with_length(bytes: &[u8], len: usize, min_nodes: usize) -> Hash256 {
let root = merkle_root(bytes, min_nodes);
tree_hash::mix_in_length(&root, len)
}
#[test]
fn tree_hash_composite() {
let a = A { a: 0, b: 1 };
let fixed: VariableList<A, U0> = VariableList::from(vec![]);
assert_eq!(
fixed.tree_hash_root(),
padded_root_with_length(&[0; 32], 0, 0),
);
for i in 0..=1 {
let fixed: VariableList<A, U1> = VariableList::from(vec![a; i]);
assert_eq!(
fixed.tree_hash_root(),
padded_root_with_length(&repeat(a.tree_hash_root().as_bytes(), i), i, 1),
"U1 {}",
i
);
}
for i in 0..=8 {
let fixed: VariableList<A, U8> = VariableList::from(vec![a; i]);
assert_eq!(
fixed.tree_hash_root(),
padded_root_with_length(&repeat(a.tree_hash_root().as_bytes(), i), i, 8),
"U8 {}",
i
);
}
for i in 0..=13 {
let fixed: VariableList<A, U13> = VariableList::from(vec![a; i]);
assert_eq!(
fixed.tree_hash_root(),
padded_root_with_length(&repeat(a.tree_hash_root().as_bytes(), i), i, 13),
"U13 {}",
i
);
}
for i in 0..=16 {
let fixed: VariableList<A, U16> = VariableList::from(vec![a; i]);
assert_eq!(
fixed.tree_hash_root(),
padded_root_with_length(&repeat(a.tree_hash_root().as_bytes(), i), i, 16),
"U16 {}",
i
);
}
}
}

23
consensus/tree_hash/Cargo.toml
View File

@@ -1,23 +0,0 @@
[package]
name = "tree_hash"
version = "0.4.1"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
license = "Apache-2.0"
description = "Efficient Merkle-hashing as used in Ethereum 2.0"
[dev-dependencies]
rand = "0.8.5"
tree_hash_derive = "0.4.0"
types = { path = "../types" }
beacon_chain = { path = "../../beacon_node/beacon_chain" }
ethereum_ssz = "1.0.0-beta.2"
ethereum_ssz_derive = "1.0.0-beta.2"
[dependencies]
ethereum-types = "0.14.1"
eth2_hashing = "0.3.0"
smallvec = "1.6.1"
[features]
arbitrary = ["ethereum-types/arbitrary"]

50
consensus/tree_hash/examples/flamegraph_beacon_state.rs
View File

@@ -1,50 +0,0 @@
use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType};
use types::{BeaconState, EthSpec, MainnetEthSpec};
const TREE_HASH_LOOPS: usize = 1_000;
const VALIDATOR_COUNT: usize = 1_000;
fn get_harness<T: EthSpec>() -> BeaconChainHarness<EphemeralHarnessType<T>> {
let harness = BeaconChainHarness::builder(T::default())
.default_spec()
.deterministic_keypairs(VALIDATOR_COUNT)
.fresh_ephemeral_store()
.build();
harness.advance_slot();
harness
}
fn build_state<T: EthSpec>() -> BeaconState<T> {
let state = get_harness::<T>().chain.head_beacon_state_cloned();
assert_eq!(state.as_base().unwrap().validators.len(), VALIDATOR_COUNT);
assert_eq!(state.as_base().unwrap().balances.len(), VALIDATOR_COUNT);
assert!(state
.as_base()
.unwrap()
.previous_epoch_attestations
.is_empty());
assert!(state
.as_base()
.unwrap()
.current_epoch_attestations
.is_empty());
assert!(state.as_base().unwrap().eth1_data_votes.is_empty());
assert!(state.as_base().unwrap().historical_roots.is_empty());
state
}
fn main() {
let state = build_state::<MainnetEthSpec>();
// This vec is an attempt to ensure the compiler doesn't optimize-out the hashing.
let mut vec = Vec::with_capacity(TREE_HASH_LOOPS);
for _ in 0..TREE_HASH_LOOPS {
let root = state.canonical_root();
vec.push(root[0]);
}
}

202
consensus/tree_hash/src/impls.rs
View File

@@ -1,202 +0,0 @@
use super::*;
use ethereum_types::{H160, H256, U128, U256};
fn int_to_hash256(int: u64) -> Hash256 {
let mut bytes = [0; HASHSIZE];
bytes[0..8].copy_from_slice(&int.to_le_bytes());
Hash256::from_slice(&bytes)
}
macro_rules! impl_for_bitsize {
($type: ident, $bit_size: expr) => {
impl TreeHash for $type {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Basic
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
PackedEncoding::from_slice(&self.to_le_bytes())
}
fn tree_hash_packing_factor() -> usize {
HASHSIZE / ($bit_size / 8)
}
#[allow(clippy::cast_lossless)] // Lint does not apply to all uses of this macro.
fn tree_hash_root(&self) -> Hash256 {
int_to_hash256(*self as u64)
}
}
};
}
impl_for_bitsize!(u8, 8);
impl_for_bitsize!(u16, 16);
impl_for_bitsize!(u32, 32);
impl_for_bitsize!(u64, 64);
impl_for_bitsize!(usize, 64);
impl TreeHash for bool {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Basic
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
(*self as u8).tree_hash_packed_encoding()
}
fn tree_hash_packing_factor() -> usize {
u8::tree_hash_packing_factor()
}
fn tree_hash_root(&self) -> Hash256 {
int_to_hash256(*self as u64)
}
}
/// Only valid for byte types less than 32 bytes.
macro_rules! impl_for_lt_32byte_u8_array {
($len: expr) => {
impl TreeHash for [u8; $len] {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Vector
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
unreachable!("bytesN should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("bytesN should never be packed.")
}
fn tree_hash_root(&self) -> Hash256 {
let mut result = [0; 32];
result[0..$len].copy_from_slice(&self[..]);
Hash256::from_slice(&result)
}
}
};
}
impl_for_lt_32byte_u8_array!(4);
impl_for_lt_32byte_u8_array!(32);
impl TreeHash for U128 {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Basic
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
let mut result = [0; 16];
self.to_little_endian(&mut result);
PackedEncoding::from_slice(&result)
}
fn tree_hash_packing_factor() -> usize {
2
}
fn tree_hash_root(&self) -> Hash256 {
let mut result = [0; HASHSIZE];
self.to_little_endian(&mut result[0..16]);
Hash256::from_slice(&result)
}
}
impl TreeHash for U256 {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Basic
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
let mut result = [0; 32];
self.to_little_endian(&mut result);
PackedEncoding::from_slice(&result)
}
fn tree_hash_packing_factor() -> usize {
1
}
fn tree_hash_root(&self) -> Hash256 {
let mut result = [0; 32];
self.to_little_endian(&mut result[..]);
Hash256::from_slice(&result)
}
}
impl TreeHash for H160 {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Vector
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
let mut result = [0; 32];
result[0..20].copy_from_slice(self.as_bytes());
PackedEncoding::from_slice(&result)
}
fn tree_hash_packing_factor() -> usize {
1
}
fn tree_hash_root(&self) -> Hash256 {
let mut result = [0; 32];
result[0..20].copy_from_slice(self.as_bytes());
Hash256::from_slice(&result)
}
}
impl TreeHash for H256 {
fn tree_hash_type() -> TreeHashType {
TreeHashType::Vector
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
PackedEncoding::from_slice(self.as_bytes())
}
fn tree_hash_packing_factor() -> usize {
1
}
fn tree_hash_root(&self) -> Hash256 {
*self
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn bool() {
let mut true_bytes: Vec<u8> = vec![1];
true_bytes.append(&mut vec![0; 31]);
let false_bytes: Vec<u8> = vec![0; 32];
assert_eq!(true.tree_hash_root().as_bytes(), true_bytes.as_slice());
assert_eq!(false.tree_hash_root().as_bytes(), false_bytes.as_slice());
}
#[test]
fn int_to_bytes() {
assert_eq!(int_to_hash256(0).as_bytes(), &[0; 32]);
assert_eq!(
int_to_hash256(1).as_bytes(),
&[
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0
]
);
assert_eq!(
int_to_hash256(u64::max_value()).as_bytes(),
&[
255, 255, 255, 255, 255, 255, 255, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0
]
);
}
}

208
consensus/tree_hash/src/lib.rs
View File

@@ -1,208 +0,0 @@
pub mod impls;
mod merkle_hasher;
mod merkleize_padded;
mod merkleize_standard;
pub use merkle_hasher::{Error, MerkleHasher};
pub use merkleize_padded::merkleize_padded;
pub use merkleize_standard::merkleize_standard;
use eth2_hashing::{hash_fixed, ZERO_HASHES, ZERO_HASHES_MAX_INDEX};
use smallvec::SmallVec;
pub const BYTES_PER_CHUNK: usize = 32;
pub const HASHSIZE: usize = 32;
pub const MERKLE_HASH_CHUNK: usize = 2 * BYTES_PER_CHUNK;
pub const MAX_UNION_SELECTOR: u8 = 127;
pub const SMALLVEC_SIZE: usize = 32;
pub type Hash256 = ethereum_types::H256;
pub type PackedEncoding = SmallVec<[u8; SMALLVEC_SIZE]>;
/// Convenience method for `MerkleHasher` which also provides some fast-paths for small trees.
///
/// `minimum_leaf_count` will only be used if it is greater than or equal to the minimum number of leaves that can be created from `bytes`.
pub fn merkle_root(bytes: &[u8], minimum_leaf_count: usize) -> Hash256 {
let leaves = std::cmp::max(
(bytes.len() + (HASHSIZE - 1)) / HASHSIZE,
minimum_leaf_count,
);
if leaves == 0 {
// If there are no bytes then the hash is always zero.
Hash256::zero()
} else if leaves == 1 {
// If there is only one leaf, the hash is always those leaf bytes padded out to 32-bytes.
let mut hash = [0; HASHSIZE];
hash[0..bytes.len()].copy_from_slice(bytes);
Hash256::from_slice(&hash)
} else if leaves == 2 {
// If there are only two leaves (this is common with BLS pubkeys), we can avoid some
// overhead with `MerkleHasher` and just do a simple 3-node tree here.
let mut leaves = [0; HASHSIZE * 2];
leaves[0..bytes.len()].copy_from_slice(bytes);
Hash256::from_slice(&hash_fixed(&leaves))
} else {
// If there are 3 or more leaves, use `MerkleHasher`.
let mut hasher = MerkleHasher::with_leaves(leaves);
hasher
.write(bytes)
.expect("the number of leaves is adequate for the number of bytes");
hasher
.finish()
.expect("the number of leaves is adequate for the number of bytes")
}
}
/// Returns the node created by hashing `root` and `length`.
///
/// Used in `TreeHash` for inserting the length of a list above it's root.
pub fn mix_in_length(root: &Hash256, length: usize) -> Hash256 {
let usize_len = std::mem::size_of::<usize>();
let mut length_bytes = [0; BYTES_PER_CHUNK];
length_bytes[0..usize_len].copy_from_slice(&length.to_le_bytes());
Hash256::from_slice(&eth2_hashing::hash32_concat(root.as_bytes(), &length_bytes)[..])
}
/// Returns `Some(root)` created by hashing `root` and `selector`, if `selector <=
/// MAX_UNION_SELECTOR`. Otherwise, returns `None`.
///
/// Used in `TreeHash` for the "union" type.
///
/// ## Specification
///
/// ```ignore,text
/// mix_in_selector: Given a Merkle root root and a type selector selector ("uint256" little-endian
/// serialization) return hash(root + selector).
/// ```
///
/// https://github.com/ethereum/consensus-specs/blob/v1.1.0-beta.3/ssz/simple-serialize.md#union
pub fn mix_in_selector(root: &Hash256, selector: u8) -> Option<Hash256> {
if selector > MAX_UNION_SELECTOR {
return None;
}
let mut chunk = [0; BYTES_PER_CHUNK];
chunk[0] = selector;
let root = eth2_hashing::hash32_concat(root.as_bytes(), &chunk);
Some(Hash256::from_slice(&root))
}
/// Returns a cached padding node for a given height.
fn get_zero_hash(height: usize) -> &'static [u8] {
if height <= ZERO_HASHES_MAX_INDEX {
&ZERO_HASHES[height]
} else {
panic!("Tree exceeds MAX_TREE_DEPTH of {}", ZERO_HASHES_MAX_INDEX)
}
}
#[derive(Debug, PartialEq, Clone)]
pub enum TreeHashType {
Basic,
Vector,
List,
Container,
}
pub trait TreeHash {
fn tree_hash_type() -> TreeHashType;
fn tree_hash_packed_encoding(&self) -> PackedEncoding;
fn tree_hash_packing_factor() -> usize;
fn tree_hash_root(&self) -> Hash256;
}
/// Punch through references.
impl<'a, T> TreeHash for &'a T
where
T: TreeHash,
{
fn tree_hash_type() -> TreeHashType {
T::tree_hash_type()
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
T::tree_hash_packed_encoding(*self)
}
fn tree_hash_packing_factor() -> usize {
T::tree_hash_packing_factor()
}
fn tree_hash_root(&self) -> Hash256 {
T::tree_hash_root(*self)
}
}
#[macro_export]
macro_rules! tree_hash_ssz_encoding_as_vector {
($type: ident) => {
impl tree_hash::TreeHash for $type {
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::Vector
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
unreachable!("Vector should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("Vector should never be packed.")
}
fn tree_hash_root(&self) -> Vec<u8> {
tree_hash::merkle_root(&ssz::ssz_encode(self))
}
}
};
}
#[macro_export]
macro_rules! tree_hash_ssz_encoding_as_list {
($type: ident) => {
impl tree_hash::TreeHash for $type {
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::List
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
unreachable!("List should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("List should never be packed.")
}
fn tree_hash_root(&self) -> Vec<u8> {
ssz::ssz_encode(self).tree_hash_root()
}
}
};
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn mix_length() {
let hash = {
let mut preimage = vec![42; BYTES_PER_CHUNK];
preimage.append(&mut vec![42]);
preimage.append(&mut vec![0; BYTES_PER_CHUNK - 1]);
eth2_hashing::hash(&preimage)
};
assert_eq!(
mix_in_length(&Hash256::from_slice(&[42; BYTES_PER_CHUNK]), 42).as_bytes(),
&hash[..]
);
}
}

573
consensus/tree_hash/src/merkle_hasher.rs
View File

@@ -1,573 +0,0 @@
use crate::{get_zero_hash, Hash256, HASHSIZE};
use eth2_hashing::{Context, Sha256Context, HASH_LEN};
use smallvec::{smallvec, SmallVec};
use std::mem;
type SmallVec8<T> = SmallVec<[T; 8]>;
#[derive(Clone, Debug, PartialEq)]
pub enum Error {
/// The maximum number of leaves defined by the initialization `depth` has been exceed.
MaximumLeavesExceeded { max_leaves: usize },
}
/// Helper struct to store either a hash digest or a slice.
///
/// Should be used as a left or right value for some node.
enum Preimage<'a> {
Digest([u8; HASH_LEN]),
Slice(&'a [u8]),
}
impl<'a> Preimage<'a> {
/// Returns a 32-byte slice.
fn as_bytes(&self) -> &[u8] {
match self {
Preimage::Digest(digest) => digest.as_ref(),
Preimage::Slice(slice) => slice,
}
}
}
/// A node that has had a left child supplied, but not a right child.
struct HalfNode {
/// The hasher context.
context: Context,
/// The tree id of the node. The root node has in id of `1` and ids increase moving down the
/// tree from left to right.
id: usize,
}
impl HalfNode {
/// Create a new half-node from the given `left` value.
fn new(id: usize, left: Preimage) -> Self {
let mut context = Context::new();
context.update(left.as_bytes());
Self { context, id }
}
/// Complete the half-node by providing a `right` value. Returns a digest of the left and right
/// nodes.
fn finish(mut self, right: Preimage) -> [u8; HASH_LEN] {
self.context.update(right.as_bytes());
self.context.finalize()
}
}
/// Provides a Merkle-root hasher that allows for streaming bytes (i.e., providing any-length byte
/// slices without need to separate into leaves). Efficiently handles cases where not all leaves
/// have been provided by assuming all non-provided leaves are `[0; 32]` and pre-computing the
/// zero-value hashes at all depths of the tree.
///
/// This algorithm aims to allocate as little memory as possible and it does this by "folding" up
/// the tree as each leaf is provided. Consider this step-by-step functional diagram of hashing a
/// tree with depth three:
///
/// ## Functional Diagram
///
/// Nodes that are `-` have not been defined and do not occupy memory. Nodes that are `L` are
/// leaves that are provided but are not stored. Nodes that have integers (`1`, `2`) are stored in
/// our struct. Finally, nodes that are `X` were stored, but are now removed.
///
/// ### Start
///
/// ```ignore
/// -
/// / \
/// - -
/// / \ / \
/// - - - -
/// ```
///
/// ### Provide first leaf
///
/// ```ignore
/// -
/// / \
/// 2 -
/// / \ / \
/// L - - -
/// ```
///
/// ### Provide second leaf
///
/// ```ignore
/// 1
/// / \
/// X -
/// / \ / \
/// L L - -
/// ```
///
/// ### Provide third leaf
///
/// ```ignore
/// 1
/// / \
/// X 3
/// / \ / \
/// L L L -
/// ```
///
/// ### Provide fourth and final leaf
///
/// ```ignore
/// 1
/// / \
/// X X
/// / \ / \
/// L L L L
/// ```
///
pub struct MerkleHasher {
/// Stores the nodes that are half-complete and awaiting a right node.
///
/// A smallvec of size 8 means we can hash a tree with 256 leaves without allocating on the
/// heap. Each half-node is 232 bytes, so this smallvec may store 1856 bytes on the stack.
half_nodes: SmallVec8<HalfNode>,
/// The depth of the tree that will be produced.
///
/// Depth is counted top-down (i.e., the root node is at depth 0). A tree with 1 leaf has a
/// depth of 1, a tree with 4 leaves has a depth of 3.
depth: usize,
/// The next leaf that we are expecting to process.
next_leaf: usize,
/// A buffer of bytes that are waiting to be written to a leaf.
buffer: SmallVec<[u8; 32]>,
/// Set to Some(root) when the root of the tree is known.
root: Option<Hash256>,
}
/// Returns the parent of node with id `i`.
fn get_parent(i: usize) -> usize {
i / 2
}
/// Gets the depth of a node with an id of `i`.
///
/// It is a logic error to provide `i == 0`.
///
/// E.g., if `i` is 1, depth is 0. If `i` is is 1, depth is 1.
fn get_depth(i: usize) -> usize {
let total_bits = mem::size_of::<usize>() * 8;
total_bits - i.leading_zeros() as usize - 1
}
impl MerkleHasher {
/// Instantiate a hasher for a tree with a given number of leaves.
///
/// `num_leaves` will be rounded to the next power of two. E.g., if `num_leaves == 6`, then the
/// tree will _actually_ be able to accomodate 8 leaves and the resulting hasher is exactly the
/// same as one that was instantiated with `Self::with_leaves(8)`.
///
/// ## Notes
///
/// If `num_leaves == 0`, a tree of depth 1 will be created. If no leaves are provided it will
/// return a root of `[0; 32]`.
pub fn with_leaves(num_leaves: usize) -> Self {
let depth = get_depth(num_leaves.next_power_of_two()) + 1;
Self::with_depth(depth)
}
/// Instantiates a new, empty hasher for a tree with `depth` layers which will have capacity
/// for `1 << (depth - 1)` leaf nodes.
///
/// It is not possible to grow the depth of the tree after instantiation.
///
/// ## Panics
///
/// Panics if `depth == 0`.
fn with_depth(depth: usize) -> Self {
assert!(depth > 0, "merkle tree cannot have a depth of zero");
Self {
half_nodes: SmallVec::with_capacity(depth - 1),
depth,
next_leaf: 1 << (depth - 1),
buffer: SmallVec::with_capacity(32),
root: None,
}
}
/// Write some bytes to the hasher.
///
/// ## Errors
///
/// Returns an error if the given bytes would create a leaf that would exceed the maximum
/// permissible number of leaves defined by the initialization `depth`. E.g., a tree of `depth
/// == 2` can only accept 2 leaves. A tree of `depth == 14` can only accept 8,192 leaves.
pub fn write(&mut self, bytes: &[u8]) -> Result<(), Error> {
let mut ptr = 0;
while ptr <= bytes.len() {
let slice = &bytes[ptr..std::cmp::min(bytes.len(), ptr + HASHSIZE)];
if self.buffer.is_empty() && slice.len() == HASHSIZE {
self.process_leaf(slice)?;
ptr += HASHSIZE
} else if self.buffer.len() + slice.len() < HASHSIZE {
self.buffer.extend_from_slice(slice);
ptr += HASHSIZE
} else {
let buf_len = self.buffer.len();
let required = HASHSIZE - buf_len;
let mut leaf = [0; HASHSIZE];
leaf[..buf_len].copy_from_slice(&self.buffer);
leaf[buf_len..].copy_from_slice(&slice[0..required]);
self.process_leaf(&leaf)?;
self.buffer = smallvec![];
ptr += required
}
}
Ok(())
}
/// Process the next leaf in the tree.
///
/// ## Errors
///
/// Returns an error if the given leaf would exceed the maximum permissible number of leaves
/// defined by the initialization `depth`. E.g., a tree of `depth == 2` can only accept 2
/// leaves. A tree of `depth == 14` can only accept 8,192 leaves.
fn process_leaf(&mut self, leaf: &[u8]) -> Result<(), Error> {
assert_eq!(leaf.len(), HASHSIZE, "a leaf must be 32 bytes");
let max_leaves = 1 << (self.depth + 1);
if self.next_leaf > max_leaves {
return Err(Error::MaximumLeavesExceeded { max_leaves });
} else if self.next_leaf == 1 {
// A tree of depth one has a root that is equal to the first given leaf.
self.root = Some(Hash256::from_slice(leaf))
} else if self.next_leaf % 2 == 0 {
self.process_left_node(self.next_leaf, Preimage::Slice(leaf))
} else {
self.process_right_node(self.next_leaf, Preimage::Slice(leaf))
}
self.next_leaf += 1;
Ok(())
}
/// Returns the root of the Merkle tree.
///
/// If not all leaves have been provided, the tree will be efficiently completed under the
/// assumption that all not-yet-provided leaves are equal to `[0; 32]`.
///
/// ## Errors
///
/// Returns an error if the bytes remaining in the buffer would create a leaf that would exceed
/// the maximum permissible number of leaves defined by the initialization `depth`.
pub fn finish(mut self) -> Result<Hash256, Error> {
if !self.buffer.is_empty() {
let mut leaf = [0; HASHSIZE];
leaf[..self.buffer.len()].copy_from_slice(&self.buffer);
self.process_leaf(&leaf)?
}
// If the tree is incomplete, we must complete it by providing zero-hashes.
loop {
if let Some(root) = self.root {
break Ok(root);
} else if let Some(node) = self.half_nodes.last() {
let right_child = node.id * 2 + 1;
self.process_right_node(right_child, self.zero_hash(right_child));
} else if self.next_leaf == 1 {
// The next_leaf can only be 1 if the tree has a depth of one. If have been no
// leaves supplied, assume a root of zero.
break Ok(Hash256::zero());
} else {
// The only scenario where there are (a) no half nodes and (b) a tree of depth
// two or more is where no leaves have been supplied at all.
//
// Once we supply this first zero-hash leaf then all future operations will be
// triggered via the `process_right_node` branch.
self.process_left_node(self.next_leaf, self.zero_hash(self.next_leaf))
}
}
}
/// Process a node that will become the left-hand node of some parent. The supplied `id` is
/// that of the node (not the parent). The `preimage` is the value of the node (i.e., if this
/// is a leaf node it will be the value of that leaf).
///
/// In this scenario, the only option is to push a new half-node.
fn process_left_node(&mut self, id: usize, preimage: Preimage) {
self.half_nodes
.push(HalfNode::new(get_parent(id), preimage))
}
/// Process a node that will become the right-hand node of some parent. The supplied `id` is
/// that of the node (not the parent). The `preimage` is the value of the node (i.e., if this
/// is a leaf node it will be the value of that leaf).
///
/// This operation will always complete one node, then it will attempt to crawl up the tree and
/// collapse all other completed nodes. For example, consider a tree of depth 3 (see diagram
/// below). When providing the node with id `7`, the node with id `3` will be completed which
/// will also provide the right-node for the `1` node. This function will complete both of
/// those nodes and ultimately find the root of the tree.
///
/// ```ignore
/// 1 <-- completed
/// / \
/// 2 3 <-- completed
/// / \ / \
/// 4 5 6 7 <-- supplied right node
/// ```
fn process_right_node(&mut self, id: usize, mut preimage: Preimage) {
let mut parent = get_parent(id);
loop {
match self.half_nodes.last() {
Some(node) if node.id == parent => {
preimage = Preimage::Digest(
self.half_nodes
.pop()
.expect("if .last() is Some then .pop() must succeed")
.finish(preimage),
);
if parent == 1 {
self.root = Some(Hash256::from_slice(preimage.as_bytes()));
break;
} else {
parent = get_parent(parent);
}
}
_ => {
self.half_nodes.push(HalfNode::new(parent, preimage));
break;
}
}
}
}
/// Returns a "zero hash" from a pre-computed set for the given node.
///
/// Note: this node is not always zero, instead it is the result of hashing up a tree where the
/// leaves are all zeros. E.g., in a tree of depth 2, the `zero_hash` of a node at depth 1
/// will be `[0; 32]`. However, the `zero_hash` for a node at depth 0 will be
/// `hash(concat([0; 32], [0; 32])))`.
fn zero_hash(&self, id: usize) -> Preimage<'static> {
Preimage::Slice(get_zero_hash(self.depth - (get_depth(id) + 1)))
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::merkleize_padded;
/// This test is just to ensure that the stack size of the `Context` remains the same. We choose
/// our smallvec size based upon this, so it's good to know if it suddenly changes in size.
#[test]
fn context_size() {
assert_eq!(
mem::size_of::<HalfNode>(),
224,
"Halfnode size should be as expected"
);
}
fn compare_with_reference(leaves: &[Hash256], depth: usize) {
let reference_bytes = leaves
.iter()
.flat_map(|hash| hash.as_bytes())
.copied()
.collect::<Vec<_>>();
let reference_root = merkleize_padded(&reference_bytes, 1 << (depth - 1));
let merklizer_root_32_bytes = {
let mut m = MerkleHasher::with_depth(depth);
for leaf in leaves.iter() {
m.write(leaf.as_bytes()).expect("should process leaf");
}
m.finish().expect("should finish")
};
assert_eq!(
reference_root, merklizer_root_32_bytes,
"32 bytes should match reference root"
);
let merklizer_root_individual_3_bytes = {
let mut m = MerkleHasher::with_depth(depth);
for bytes in reference_bytes.chunks(3) {
m.write(bytes).expect("should process byte");
}
m.finish().expect("should finish")
};
assert_eq!(
reference_root, merklizer_root_individual_3_bytes,
"3 bytes should match reference root"
);
let merklizer_root_individual_single_bytes = {
let mut m = MerkleHasher::with_depth(depth);
for byte in reference_bytes.iter() {
m.write(&[*byte]).expect("should process byte");
}
m.finish().expect("should finish")
};
assert_eq!(
reference_root, merklizer_root_individual_single_bytes,
"single bytes should match reference root"
);
}
/// A simple wrapper to compare MerkleHasher to the reference function by just giving a number
/// of leaves and a depth.
fn compare_reference_with_len(leaves: u64, depth: usize) {
let leaves = (0..leaves)
.map(Hash256::from_low_u64_be)
.collect::<Vec<_>>();
compare_with_reference(&leaves, depth)
}
/// Compares the `MerkleHasher::with_depth` and `MerkleHasher::with_leaves` generate consistent
/// results.
fn compare_new_with_leaf_count(num_leaves: u64, depth: usize) {
let leaves = (0..num_leaves)
.map(Hash256::from_low_u64_be)
.collect::<Vec<_>>();
let from_depth = {
let mut m = MerkleHasher::with_depth(depth);
for leaf in leaves.iter() {
m.write(leaf.as_bytes()).expect("should process leaf");
}
m.finish()
};
let from_num_leaves = {
let mut m = MerkleHasher::with_leaves(num_leaves as usize);
for leaf in leaves.iter() {
m.process_leaf(leaf.as_bytes())
.expect("should process leaf");
}
m.finish()
};
assert_eq!(
from_depth, from_num_leaves,
"hash generated by depth should match that from num leaves"
);
}
#[test]
fn with_leaves() {
compare_new_with_leaf_count(1, 1);
compare_new_with_leaf_count(2, 2);
compare_new_with_leaf_count(3, 3);
compare_new_with_leaf_count(4, 3);
compare_new_with_leaf_count(5, 4);
compare_new_with_leaf_count(6, 4);
compare_new_with_leaf_count(7, 4);
compare_new_with_leaf_count(8, 4);
compare_new_with_leaf_count(9, 5);
compare_new_with_leaf_count(10, 5);
compare_new_with_leaf_count(11, 5);
compare_new_with_leaf_count(12, 5);
compare_new_with_leaf_count(13, 5);
compare_new_with_leaf_count(14, 5);
compare_new_with_leaf_count(15, 5);
}
#[test]
fn depth() {
assert_eq!(get_depth(1), 0);
assert_eq!(get_depth(2), 1);
assert_eq!(get_depth(3), 1);
assert_eq!(get_depth(4), 2);
assert_eq!(get_depth(5), 2);
assert_eq!(get_depth(6), 2);
assert_eq!(get_depth(7), 2);
assert_eq!(get_depth(8), 3);
}
#[test]
fn with_0_leaves() {
let hasher = MerkleHasher::with_leaves(0);
assert_eq!(hasher.finish().unwrap(), Hash256::zero());
}
#[test]
#[should_panic]
fn too_many_leaves() {
compare_reference_with_len(2, 1);
}
#[test]
fn full_trees() {
compare_reference_with_len(1, 1);
compare_reference_with_len(2, 2);
compare_reference_with_len(4, 3);
compare_reference_with_len(8, 4);
compare_reference_with_len(16, 5);
compare_reference_with_len(32, 6);
compare_reference_with_len(64, 7);
compare_reference_with_len(128, 8);
compare_reference_with_len(256, 9);
compare_reference_with_len(256, 9);
compare_reference_with_len(8192, 14);
}
#[test]
fn incomplete_trees() {
compare_reference_with_len(0, 1);
compare_reference_with_len(0, 2);
compare_reference_with_len(1, 2);
for i in 0..=4 {
compare_reference_with_len(i, 3);
}
for i in 0..=7 {
compare_reference_with_len(i, 4);
}
for i in 0..=15 {
compare_reference_with_len(i, 5);
}
for i in 0..=32 {
compare_reference_with_len(i, 6);
}
for i in 0..=64 {
compare_reference_with_len(i, 7);
}
compare_reference_with_len(0, 14);
compare_reference_with_len(13, 14);
compare_reference_with_len(8191, 14);
}
#[test]
fn remaining_buffer() {
let a = {
let mut m = MerkleHasher::with_leaves(2);
m.write(&[1]).expect("should write");
m.finish().expect("should finish")
};
let b = {
let mut m = MerkleHasher::with_leaves(2);
let mut leaf = vec![1];
leaf.extend_from_slice(&[0; 31]);
m.write(&leaf).expect("should write");
m.write(&[0; 32]).expect("should write");
m.finish().expect("should finish")
};
assert_eq!(a, b, "should complete buffer");
}
}

330
consensus/tree_hash/src/merkleize_padded.rs
View File

@@ -1,330 +0,0 @@
use super::{get_zero_hash, Hash256, BYTES_PER_CHUNK};
use eth2_hashing::{hash32_concat, hash_fixed};
/// Merkleize `bytes` and return the root, optionally padding the tree out to `min_leaves` number of
/// leaves.
///
/// **Note**: This function is generally worse than using the `crate::merkle_root` which uses
/// `MerkleHasher`. We only keep this function around for reference testing.
///
/// First all nodes are extracted from `bytes` and then a padding node is added until the number of
/// leaf chunks is greater than or equal to `min_leaves`. Callers may set `min_leaves` to `0` if no
/// adding additional chunks should be added to the given `bytes`.
///
/// If `bytes.len() <= BYTES_PER_CHUNK`, no hashing is done and `bytes` is returned, potentially
/// padded out to `BYTES_PER_CHUNK` length with `0`.
///
/// ## CPU Performance
///
/// A cache of `MAX_TREE_DEPTH` hashes are stored to avoid re-computing the hashes of padding nodes
/// (or their parents). Therefore, adding padding nodes only incurs one more hash per additional
/// height of the tree.
///
/// ## Memory Performance
///
/// This algorithm has two interesting memory usage properties:
///
/// 1. The maximum memory footprint is roughly `O(V / 2)` memory, where `V` is the number of leaf
/// chunks with values (i.e., leaves that are not padding). The means adding padding nodes to
/// the tree does not increase the memory footprint.
/// 2. At each height of the tree half of the memory is freed until only a single chunk is stored.
/// 3. The input `bytes` are not copied into another list before processing.
///
/// _Note: there are some minor memory overheads, including a handful of usizes and a list of
/// `MAX_TREE_DEPTH` hashes as `lazy_static` constants._
pub fn merkleize_padded(bytes: &[u8], min_leaves: usize) -> Hash256 {
// If the bytes are just one chunk or less, pad to one chunk and return without hashing.
if bytes.len() <= BYTES_PER_CHUNK && min_leaves <= 1 {
let mut o = bytes.to_vec();
o.resize(BYTES_PER_CHUNK, 0);
return Hash256::from_slice(&o);
}
assert!(
bytes.len() > BYTES_PER_CHUNK || min_leaves > 1,
"Merkle hashing only needs to happen if there is more than one chunk"
);
// The number of leaves that can be made directly from `bytes`.
let leaves_with_values = (bytes.len() + (BYTES_PER_CHUNK - 1)) / BYTES_PER_CHUNK;
// The number of parents that have at least one non-padding leaf.
//
// Since there is more than one node in this tree (see prior assertion), there should always be
// one or more initial parent nodes.
let initial_parents_with_values = std::cmp::max(1, next_even_number(leaves_with_values) / 2);
// The number of leaves in the full tree (including padding nodes).
let num_leaves = std::cmp::max(leaves_with_values, min_leaves).next_power_of_two();
// The number of levels in the tree.
//
// A tree with a single node has `height == 1`.
let height = num_leaves.trailing_zeros() as usize + 1;
assert!(height >= 2, "The tree should have two or more heights");
// A buffer/scratch-space used for storing each round of hashes at each height.
//
// This buffer is kept as small as possible; it will shrink so it never stores a padding node.
let mut chunks = ChunkStore::with_capacity(initial_parents_with_values);
// Create a parent in the `chunks` buffer for every two chunks in `bytes`.
//
// I.e., do the first round of hashing, hashing from the `bytes` slice and filling the `chunks`
// struct.
for i in 0..initial_parents_with_values {
let start = i * BYTES_PER_CHUNK * 2;
// Hash two chunks, creating a parent chunk.
let hash = match bytes.get(start..start + BYTES_PER_CHUNK * 2) {
// All bytes are available, hash as usual.
Some(slice) => hash_fixed(slice),
// Unable to get all the bytes, get a small slice and pad it out.
None => {
let mut preimage = bytes
.get(start..)
.expect("`i` can only be larger than zero if there are bytes to read")
.to_vec();
preimage.resize(BYTES_PER_CHUNK * 2, 0);
hash_fixed(&preimage)
}
};
assert_eq!(
hash.len(),
BYTES_PER_CHUNK,
"Hashes should be exactly one chunk"
);
// Store the parent node.
chunks
.set(i, &hash)
.expect("Buffer should always have capacity for parent nodes")
}
// Iterate through all heights above the leaf nodes and either (a) hash two children or, (b)
// hash a left child and a right padding node.
//
// Skip the 0'th height because the leaves have already been processed. Skip the highest-height
// in the tree as it is the root does not require hashing.
//
// The padding nodes for each height are cached via `lazy static` to simulate non-adjacent
// padding nodes (i.e., avoid doing unnecessary hashing).
for height in 1..height - 1 {
let child_nodes = chunks.len();
let parent_nodes = next_even_number(child_nodes) / 2;
// For each pair of nodes stored in `chunks`:
//
// - If two nodes are available, hash them to form a parent.
// - If one node is available, hash it and a cached padding node to form a parent.
for i in 0..parent_nodes {
let (left, right) = match (chunks.get(i * 2), chunks.get(i * 2 + 1)) {
(Ok(left), Ok(right)) => (left, right),
(Ok(left), Err(_)) => (left, get_zero_hash(height)),
// Deriving `parent_nodes` from `chunks.len()` has ensured that we never encounter the
// scenario where we expect two nodes but there are none.
(Err(_), Err(_)) => unreachable!("Parent must have one child"),
// `chunks` is a contiguous array so it is impossible for an index to be missing
// when a higher index is present.
(Err(_), Ok(_)) => unreachable!("Parent must have a left child"),
};
assert!(
left.len() == right.len() && right.len() == BYTES_PER_CHUNK,
"Both children should be `BYTES_PER_CHUNK` bytes."
);
let hash = hash32_concat(left, right);
// Store a parent node.
chunks
.set(i, &hash)
.expect("Buf is adequate size for parent");
}
// Shrink the buffer so it neatly fits the number of new nodes created in this round.
//
// The number of `parent_nodes` is either decreasing or stable. It never increases.
chunks.truncate(parent_nodes);
}
// There should be a single chunk left in the buffer and it is the Merkle root.
let root = chunks.into_vec();
assert_eq!(root.len(), BYTES_PER_CHUNK, "Only one chunk should remain");
Hash256::from_slice(&root)
}
/// A helper struct for storing words of `BYTES_PER_CHUNK` size in a flat byte array.
#[derive(Debug)]
struct ChunkStore(Vec<u8>);
impl ChunkStore {
/// Creates a new instance with `chunks` padding nodes.
fn with_capacity(chunks: usize) -> Self {
Self(vec![0; chunks * BYTES_PER_CHUNK])
}
/// Set the `i`th chunk to `value`.
///
/// Returns `Err` if `value.len() != BYTES_PER_CHUNK` or `i` is out-of-bounds.
fn set(&mut self, i: usize, value: &[u8]) -> Result<(), ()> {
if i < self.len() && value.len() == BYTES_PER_CHUNK {
let slice = &mut self.0[i * BYTES_PER_CHUNK..i * BYTES_PER_CHUNK + BYTES_PER_CHUNK];
slice.copy_from_slice(value);
Ok(())
} else {
Err(())
}
}
/// Gets the `i`th chunk.
///
/// Returns `Err` if `i` is out-of-bounds.
fn get(&self, i: usize) -> Result<&[u8], ()> {
if i < self.len() {
Ok(&self.0[i * BYTES_PER_CHUNK..i * BYTES_PER_CHUNK + BYTES_PER_CHUNK])
} else {
Err(())
}
}
/// Returns the number of chunks presently stored in `self`.
fn len(&self) -> usize {
self.0.len() / BYTES_PER_CHUNK
}
/// Truncates 'self' to `num_chunks` chunks.
///
/// Functionally identical to `Vec::truncate`.
fn truncate(&mut self, num_chunks: usize) {
self.0.truncate(num_chunks * BYTES_PER_CHUNK)
}
/// Consumes `self`, returning the underlying byte array.
fn into_vec(self) -> Vec<u8> {
self.0
}
}
/// Returns the next even number following `n`. If `n` is even, `n` is returned.
fn next_even_number(n: usize) -> usize {
n + n % 2
}
#[cfg(test)]
mod test {
use super::*;
use crate::ZERO_HASHES_MAX_INDEX;
pub fn reference_root(bytes: &[u8]) -> Hash256 {
crate::merkleize_standard(bytes)
}
macro_rules! common_tests {
($get_bytes: ident) => {
#[test]
fn zero_value_0_nodes() {
test_against_reference(&$get_bytes(0 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_1_nodes() {
test_against_reference(&$get_bytes(1 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_2_nodes() {
test_against_reference(&$get_bytes(2 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_3_nodes() {
test_against_reference(&$get_bytes(3 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_4_nodes() {
test_against_reference(&$get_bytes(4 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_8_nodes() {
test_against_reference(&$get_bytes(8 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_9_nodes() {
test_against_reference(&$get_bytes(9 * BYTES_PER_CHUNK), 0);
}
#[test]
fn zero_value_8_nodes_varying_min_length() {
for i in 0..64 {
test_against_reference(&$get_bytes(8 * BYTES_PER_CHUNK), i);
}
}
#[test]
fn zero_value_range_of_nodes() {
for i in 0..32 * BYTES_PER_CHUNK {
test_against_reference(&$get_bytes(i), 0);
}
}
#[test]
fn max_tree_depth_min_nodes() {
let input = vec![0; 10 * BYTES_PER_CHUNK];
let min_nodes = 2usize.pow(ZERO_HASHES_MAX_INDEX as u32);
assert_eq!(
merkleize_padded(&input, min_nodes).as_bytes(),
get_zero_hash(ZERO_HASHES_MAX_INDEX)
);
}
};
}
mod zero_value {
use super::*;
fn zero_bytes(bytes: usize) -> Vec<u8> {
vec![0; bytes]
}
common_tests!(zero_bytes);
}
mod random_value {
use super::*;
use rand::RngCore;
fn random_bytes(bytes: usize) -> Vec<u8> {
let mut bytes = Vec::with_capacity(bytes);
rand::thread_rng().fill_bytes(&mut bytes);
bytes
}
common_tests!(random_bytes);
}
fn test_against_reference(input: &[u8], min_nodes: usize) {
let mut reference_input = input.to_vec();
reference_input.resize(
std::cmp::max(
reference_input.len(),
min_nodes.next_power_of_two() * BYTES_PER_CHUNK,
),
0,
);
assert_eq!(
reference_root(&reference_input),
merkleize_padded(input, min_nodes),
"input.len(): {:?}",
input.len()
);
}
}

81
consensus/tree_hash/src/merkleize_standard.rs
View File

@@ -1,81 +0,0 @@
use super::*;
use eth2_hashing::hash;
/// Merkleizes bytes and returns the root, using a simple algorithm that does not optimize to avoid
/// processing or storing padding bytes.
///
/// **Note**: This function is generally worse than using the `crate::merkle_root` which uses
/// `MerkleHasher`. We only keep this function around for reference testing.
///
/// The input `bytes` will be padded to ensure that the number of leaves is a power-of-two.
///
/// ## CPU Performance
///
/// Will hash all nodes in the tree, even if they are padding and pre-determined.
///
/// ## Memory Performance
///
/// - Duplicates the input `bytes`.
/// - Stores all internal nodes, even if they are padding.
/// - Does not free up unused memory during operation.
pub fn merkleize_standard(bytes: &[u8]) -> Hash256 {
// If the bytes are just one chunk (or less than one chunk) just return them.
if bytes.len() <= HASHSIZE {
let mut o = bytes.to_vec();
o.resize(HASHSIZE, 0);
return Hash256::from_slice(&o[0..HASHSIZE]);
}
let leaves = num_sanitized_leaves(bytes.len());
let nodes = num_nodes(leaves);
let internal_nodes = nodes - leaves;
let num_bytes = std::cmp::max(internal_nodes, 1) * HASHSIZE + bytes.len();
let mut o: Vec<u8> = vec![0; internal_nodes * HASHSIZE];
o.append(&mut bytes.to_vec());
assert_eq!(o.len(), num_bytes);
let empty_chunk_hash = hash(&[0; MERKLE_HASH_CHUNK]);
let mut i = nodes * HASHSIZE;
let mut j = internal_nodes * HASHSIZE;
while i >= MERKLE_HASH_CHUNK {
i -= MERKLE_HASH_CHUNK;
j -= HASHSIZE;
let hash = match o.get(i..i + MERKLE_HASH_CHUNK) {
// All bytes are available, hash as usual.
Some(slice) => hash(slice),
// Unable to get all the bytes.
None => {
match o.get(i..) {
// Able to get some of the bytes, pad them out.
Some(slice) => {
let mut bytes = slice.to_vec();
bytes.resize(MERKLE_HASH_CHUNK, 0);
hash(&bytes)
}
// Unable to get any bytes, use the empty-chunk hash.
None => empty_chunk_hash.clone(),
}
}
};
o[j..j + HASHSIZE].copy_from_slice(&hash);
}
Hash256::from_slice(&o[0..HASHSIZE])
}
fn num_sanitized_leaves(num_bytes: usize) -> usize {
let leaves = (num_bytes + HASHSIZE - 1) / HASHSIZE;
leaves.next_power_of_two()
}
fn num_nodes(num_leaves: usize) -> usize {
2 * num_leaves - 1
}

128
consensus/tree_hash/tests/tests.rs
View File

@@ -1,128 +0,0 @@
use ssz_derive::Encode;
use tree_hash::{Hash256, MerkleHasher, PackedEncoding, TreeHash, BYTES_PER_CHUNK};
use tree_hash_derive::TreeHash;
#[derive(Encode)]
struct HashVec {
vec: Vec<u8>,
}
impl From<Vec<u8>> for HashVec {
fn from(vec: Vec<u8>) -> Self {
Self { vec }
}
}
impl tree_hash::TreeHash for HashVec {
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::List
}
fn tree_hash_packed_encoding(&self) -> PackedEncoding {
unreachable!("List should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("List should never be packed.")
}
fn tree_hash_root(&self) -> Hash256 {
let mut hasher =
MerkleHasher::with_leaves((self.vec.len() + BYTES_PER_CHUNK - 1) / BYTES_PER_CHUNK);
for item in &self.vec {
hasher.write(&item.tree_hash_packed_encoding()).unwrap()
}
let root = hasher.finish().unwrap();
tree_hash::mix_in_length(&root, self.vec.len())
}
}
fn mix_in_selector(a: Hash256, selector: u8) -> Hash256 {
let mut b = [0; 32];
b[0] = selector;
Hash256::from_slice(&eth2_hashing::hash32_concat(a.as_bytes(), &b))
}
fn u8_hash_concat(v1: u8, v2: u8) -> Hash256 {
let mut a = [0; 32];
let mut b = [0; 32];
a[0] = v1;
b[0] = v2;
Hash256::from_slice(&eth2_hashing::hash32_concat(&a, &b))
}
fn u8_hash(x: u8) -> Hash256 {
let mut a = [0; 32];
a[0] = x;
Hash256::from_slice(&a)
}
#[derive(TreeHash)]
#[tree_hash(enum_behaviour = "transparent")]
enum FixedTrans {
A(u8),
B(u8),
}
#[test]
fn fixed_trans() {
assert_eq!(FixedTrans::A(2).tree_hash_root(), u8_hash(2));
assert_eq!(FixedTrans::B(2).tree_hash_root(), u8_hash(2));
}
#[derive(TreeHash)]
#[tree_hash(enum_behaviour = "union")]
enum FixedUnion {
A(u8),
B(u8),
}
#[test]
fn fixed_union() {
assert_eq!(FixedUnion::A(2).tree_hash_root(), u8_hash_concat(2, 0));
assert_eq!(FixedUnion::B(2).tree_hash_root(), u8_hash_concat(2, 1));
}
#[derive(TreeHash)]
#[tree_hash(enum_behaviour = "transparent")]
enum VariableTrans {
A(HashVec),
B(HashVec),
}
#[test]
fn variable_trans() {
assert_eq!(
VariableTrans::A(HashVec::from(vec![2])).tree_hash_root(),
u8_hash_concat(2, 1)
);
assert_eq!(
VariableTrans::B(HashVec::from(vec![2])).tree_hash_root(),
u8_hash_concat(2, 1)
);
}
#[derive(TreeHash)]
#[tree_hash(enum_behaviour = "union")]
enum VariableUnion {
A(HashVec),
B(HashVec),
}
#[test]
fn variable_union() {
assert_eq!(
VariableUnion::A(HashVec::from(vec![2])).tree_hash_root(),
mix_in_selector(u8_hash_concat(2, 1), 0)
);
assert_eq!(
VariableUnion::B(HashVec::from(vec![2])).tree_hash_root(),
mix_in_selector(u8_hash_concat(2, 1), 1)
);
}

15
consensus/tree_hash_derive/Cargo.toml
View File

@@ -1,15 +0,0 @@
[package]
name = "tree_hash_derive"
version = "0.4.0"
authors = ["Paul Hauner <paul@paulhauner.com>"]
edition = "2021"
description = "Procedural derive macros to accompany the tree_hash crate."
license = "Apache-2.0"
[lib]
proc-macro = true
[dependencies]
syn = "1.0.42"
quote = "1.0.7"
darling = "0.13.0"

337
consensus/tree_hash_derive/src/lib.rs
View File

@@ -1,337 +0,0 @@
#![recursion_limit = "256"]
use darling::FromDeriveInput;
use proc_macro::TokenStream;
use quote::quote;
use std::convert::TryInto;
use syn::{parse_macro_input, Attribute, DataEnum, DataStruct, DeriveInput, Meta};
/// The highest possible union selector value (higher values are reserved for backwards compatible
/// extensions).
const MAX_UNION_SELECTOR: u8 = 127;
#[derive(Debug, FromDeriveInput)]
#[darling(attributes(tree_hash))]
struct StructOpts {
#[darling(default)]
enum_behaviour: Option<String>,
}
const ENUM_TRANSPARENT: &str = "transparent";
const ENUM_UNION: &str = "union";
const ENUM_VARIANTS: &[&str] = &[ENUM_TRANSPARENT, ENUM_UNION];
const NO_ENUM_BEHAVIOUR_ERROR: &str = "enums require an \"enum_behaviour\" attribute, \
e.g., #[tree_hash(enum_behaviour = \"transparent\")]";
enum EnumBehaviour {
Transparent,
Union,
}
impl EnumBehaviour {
pub fn new(s: Option<String>) -> Option<Self> {
s.map(|s| match s.as_ref() {
ENUM_TRANSPARENT => EnumBehaviour::Transparent,
ENUM_UNION => EnumBehaviour::Union,
other => panic!(
"{} is an invalid enum_behaviour, use either {:?}",
other, ENUM_VARIANTS
),
})
}
}
/// Return a Vec of `syn::Ident` for each named field in the struct, whilst filtering out fields
/// that should not be hashed.
///
/// # Panics
/// Any unnamed struct field (like in a tuple struct) will raise a panic at compile time.
fn get_hashable_fields(struct_data: &syn::DataStruct) -> Vec<&syn::Ident> {
get_hashable_fields_and_their_caches(struct_data)
.into_iter()
.map(|(ident, _, _)| ident)
.collect()
}
/// Return a Vec of the hashable fields of a struct, and each field's type and optional cache field.
fn get_hashable_fields_and_their_caches(
struct_data: &syn::DataStruct,
) -> Vec<(&syn::Ident, syn::Type, Option<syn::Ident>)> {
struct_data
.fields
.iter()
.filter_map(|f| {
if should_skip_hashing(f) {
None
} else {
let ident = f
.ident
.as_ref()
.expect("tree_hash_derive only supports named struct fields");
let opt_cache_field = get_cache_field_for(f);
Some((ident, f.ty.clone(), opt_cache_field))
}
})
.collect()
}
/// Parse the cached_tree_hash attribute for a field.
///
/// Extract the cache field name from `#[cached_tree_hash(cache_field_name)]`
///
/// Return `Some(cache_field_name)` if the field has a cached tree hash attribute,
/// or `None` otherwise.
fn get_cache_field_for(field: &syn::Field) -> Option<syn::Ident> {
use syn::{MetaList, NestedMeta};
let parsed_attrs = cached_tree_hash_attr_metas(&field.attrs);
if let [Meta::List(MetaList { nested, .. })] = &parsed_attrs[..] {
nested.iter().find_map(|x| match x {
NestedMeta::Meta(Meta::Path(path)) => path.get_ident().cloned(),
_ => None,
})
} else {
None
}
}
/// Process the `cached_tree_hash` attributes from a list of attributes into structured `Meta`s.
fn cached_tree_hash_attr_metas(attrs: &[Attribute]) -> Vec<Meta> {
attrs
.iter()
.filter(|attr| attr.path.is_ident("cached_tree_hash"))
.flat_map(|attr| attr.parse_meta())
.collect()
}
/// Returns true if some field has an attribute declaring it should not be hashed.
///
/// The field attribute is: `#[tree_hash(skip_hashing)]`
fn should_skip_hashing(field: &syn::Field) -> bool {
field.attrs.iter().any(|attr| {
attr.path.is_ident("tree_hash")
&& attr.tokens.to_string().replace(' ', "") == "(skip_hashing)"
})
}
/// Implements `tree_hash::TreeHash` for some `struct`.
///
/// Fields are hashed in the order they are defined.
#[proc_macro_derive(TreeHash, attributes(tree_hash))]
pub fn tree_hash_derive(input: TokenStream) -> TokenStream {
let item = parse_macro_input!(input as DeriveInput);
let opts = StructOpts::from_derive_input(&item).unwrap();
let enum_opt = EnumBehaviour::new(opts.enum_behaviour);
match &item.data {
syn::Data::Struct(s) => {
if enum_opt.is_some() {
panic!("enum_behaviour is invalid for structs");
}
tree_hash_derive_struct(&item, s)
}
syn::Data::Enum(s) => match enum_opt.expect(NO_ENUM_BEHAVIOUR_ERROR) {
EnumBehaviour::Transparent => tree_hash_derive_enum_transparent(&item, s),
EnumBehaviour::Union => tree_hash_derive_enum_union(&item, s),
},
_ => panic!("tree_hash_derive only supports structs and enums."),
}
}
fn tree_hash_derive_struct(item: &DeriveInput, struct_data: &DataStruct) -> TokenStream {
let name = &item.ident;
let (impl_generics, ty_generics, where_clause) = &item.generics.split_for_impl();
let idents = get_hashable_fields(struct_data);
let num_leaves = idents.len();
let output = quote! {
impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause {
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::Container
}
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("Struct should never be packed.")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("Struct should never be packed.")
}
fn tree_hash_root(&self) -> tree_hash::Hash256 {
let mut hasher = tree_hash::MerkleHasher::with_leaves(#num_leaves);
#(
hasher.write(self.#idents.tree_hash_root().as_bytes())
.expect("tree hash derive should not apply too many leaves");
)*
hasher.finish().expect("tree hash derive should not have a remaining buffer")
}
}
};
output.into()
}
/// Derive `TreeHash` for an enum in the "transparent" method.
///
/// The "transparent" method is distinct from the "union" method specified in the SSZ specification.
/// When using "transparent", the enum will be ignored and the contained field will be hashed as if
/// the enum does not exist.
///
///## Limitations
///
/// Only supports:
/// - Enums with a single field per variant, where
/// - All fields are "container" types.
///
/// ## Panics
///
/// Will panic at compile-time if the single field requirement isn't met, but will panic *at run
/// time* if the container type requirement isn't met.
fn tree_hash_derive_enum_transparent(
derive_input: &DeriveInput,
enum_data: &DataEnum,
) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let (patterns, type_exprs): (Vec<_>, Vec<_>) = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if variant.fields.len() != 1 {
panic!("TreeHash can only be derived for enums with 1 field per variant");
}
let pattern = quote! {
#name::#variant_name(ref inner)
};
let ty = &(&variant.fields).into_iter().next().unwrap().ty;
let type_expr = quote! {
<#ty as tree_hash::TreeHash>::tree_hash_type()
};
(pattern, type_expr)
})
.unzip();
let output = quote! {
impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause {
fn tree_hash_type() -> tree_hash::TreeHashType {
#(
assert_eq!(
#type_exprs,
tree_hash::TreeHashType::Container,
"all variants must be of container type"
);
)*
tree_hash::TreeHashType::Container
}
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("Enum should never be packed")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("Enum should never be packed")
}
fn tree_hash_root(&self) -> tree_hash::Hash256 {
match self {
#(
#patterns => inner.tree_hash_root(),
)*
}
}
}
};
output.into()
}
/// Derive `TreeHash` for an `enum` following the "union" SSZ spec.
///
/// The union selector will be determined based upon the order in which the enum variants are
/// defined. E.g., the top-most variant in the enum will have a selector of `0`, the variant
/// beneath it will have a selector of `1` and so on.
///
/// # Limitations
///
/// Only supports enums where each variant has a single field.
fn tree_hash_derive_enum_union(derive_input: &DeriveInput, enum_data: &DataEnum) -> TokenStream {
let name = &derive_input.ident;
let (impl_generics, ty_generics, where_clause) = &derive_input.generics.split_for_impl();
let patterns: Vec<_> = enum_data
.variants
.iter()
.map(|variant| {
let variant_name = &variant.ident;
if variant.fields.len() != 1 {
panic!("TreeHash can only be derived for enums with 1 field per variant");
}
quote! {
#name::#variant_name(ref inner)
}
})
.collect();
let union_selectors = compute_union_selectors(patterns.len());
let output = quote! {
impl #impl_generics tree_hash::TreeHash for #name #ty_generics #where_clause {
fn tree_hash_type() -> tree_hash::TreeHashType {
tree_hash::TreeHashType::Container
}
fn tree_hash_packed_encoding(&self) -> tree_hash::PackedEncoding {
unreachable!("Enum should never be packed")
}
fn tree_hash_packing_factor() -> usize {
unreachable!("Enum should never be packed")
}
fn tree_hash_root(&self) -> tree_hash::Hash256 {
match self {
#(
#patterns => {
let root = inner.tree_hash_root();
let selector = #union_selectors;
tree_hash::mix_in_selector(&root, selector)
.expect("derive macro should prevent out-of-bounds selectors")
},
)*
}
}
}
};
output.into()
}
fn compute_union_selectors(num_variants: usize) -> Vec<u8> {
let union_selectors = (0..num_variants)
.map(|i| {
i.try_into()
.expect("union selector exceeds u8::max_value, union has too many variants")
})
.collect::<Vec<u8>>();
let highest_selector = union_selectors
.last()
.copied()
.expect("0-variant union is not permitted");
assert!(
highest_selector <= MAX_UNION_SELECTOR,
"union selector {} exceeds limit of {}, enum has too many variants",
highest_selector,
MAX_UNION_SELECTOR
);
union_selectors
}