Beacon state diffs!

This commit is contained in:
Michael Sproul
2022-02-24 08:46:03 +11:00
parent 0a4dcdd4e3
commit 143cf59504
38 changed files with 860 additions and 277 deletions

View File

@@ -2,6 +2,8 @@ use super::*;
use core::num::NonZeroUsize;
use ethereum_types::{H160, H256, U128, U256};
use smallvec::SmallVec;
use std::collections::BTreeMap;
use std::iter::{self, FromIterator};
use std::sync::Arc;
macro_rules! impl_decodable_for_uint {
@@ -380,14 +382,14 @@ macro_rules! impl_for_vec {
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
if bytes.is_empty() {
Ok(vec![].into())
Ok(Self::from_iter(iter::empty()))
} else if T::is_ssz_fixed_len() {
bytes
.chunks(T::ssz_fixed_len())
.map(|chunk| T::from_ssz_bytes(chunk))
.map(T::from_ssz_bytes)
.collect()
} else {
decode_list_of_variable_length_items(bytes, $max_len).map(|vec| vec.into())
decode_list_of_variable_length_items(bytes, $max_len)
}
}
}
@@ -404,17 +406,40 @@ impl_for_vec!(SmallVec<[T; 6]>, Some(6));
impl_for_vec!(SmallVec<[T; 7]>, Some(7));
impl_for_vec!(SmallVec<[T; 8]>, Some(8));
impl<K, V> Decode for BTreeMap<K, V>
where
K: Decode + Ord,
V: Decode,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn from_ssz_bytes(bytes: &[u8]) -> Result<Self, DecodeError> {
if bytes.is_empty() {
Ok(Self::from_iter(iter::empty()))
} else if <(K, V)>::is_ssz_fixed_len() {
bytes
.chunks(<(K, V)>::ssz_fixed_len())
.map(<(K, V)>::from_ssz_bytes)
.collect()
} else {
decode_list_of_variable_length_items(bytes, None)
}
}
}
/// Decodes `bytes` as if it were a list of variable-length items.
///
/// The `ssz::SszDecoder` can also perform this functionality, however it it significantly faster
/// as it is optimized to read same-typed items whilst `ssz::SszDecoder` supports reading items of
/// differing types.
pub fn decode_list_of_variable_length_items<T: Decode>(
/// The `ssz::SszDecoder` can also perform this functionality, however this function is
/// significantly faster as it is optimized to read same-typed items whilst `ssz::SszDecoder`
/// supports reading items of differing types.
pub fn decode_list_of_variable_length_items<T: Decode, Container: FromIterator<T>>(
bytes: &[u8],
max_len: Option<usize>,
) -> Result<Vec<T>, DecodeError> {
) -> Result<Container, DecodeError> {
if bytes.is_empty() {
return Ok(vec![]);
return Ok(Container::from_iter(iter::empty()));
}
let first_offset = read_offset(bytes)?;
@@ -433,35 +458,25 @@ pub fn decode_list_of_variable_length_items<T: Decode>(
)));
}
// Only initialize the vec with a capacity if a maximum length is provided.
//
// We assume that if a max length is provided then the application is able to handle an
// allocation of this size.
let mut values = if max_len.is_some() {
Vec::with_capacity(num_items)
} else {
vec![]
};
let mut offset = first_offset;
for i in 1..=num_items {
let slice_option = if i == num_items {
bytes.get(offset..)
} else {
let start = offset;
(1..=num_items)
.map(|i| {
let slice_option = if i == num_items {
bytes.get(offset..)
} else {
let start = offset;
let next_offset = read_offset(&bytes[(i * BYTES_PER_LENGTH_OFFSET)..])?;
offset = sanitize_offset(next_offset, Some(offset), bytes.len(), Some(first_offset))?;
let next_offset = read_offset(&bytes[(i * BYTES_PER_LENGTH_OFFSET)..])?;
offset =
sanitize_offset(next_offset, Some(offset), bytes.len(), Some(first_offset))?;
bytes.get(start..offset)
};
bytes.get(start..offset)
};
let slice = slice_option.ok_or(DecodeError::OutOfBoundsByte { i: offset })?;
values.push(T::from_ssz_bytes(slice)?);
}
Ok(values)
let slice = slice_option.ok_or(DecodeError::OutOfBoundsByte { i: offset })?;
T::from_ssz_bytes(slice)
})
.collect()
}
#[cfg(test)]

View File

@@ -2,6 +2,7 @@ use super::*;
use core::num::NonZeroUsize;
use ethereum_types::{H160, H256, U128, U256};
use smallvec::SmallVec;
use std::collections::BTreeMap;
use std::sync::Arc;
macro_rules! impl_encodable_for_uint {
@@ -220,6 +221,65 @@ impl<T: Encode> Encode for Arc<T> {
}
}
// Encode transparently through references.
impl<'a, T: Encode> Encode for &'a T {
fn is_ssz_fixed_len() -> bool {
T::is_ssz_fixed_len()
}
fn ssz_fixed_len() -> usize {
T::ssz_fixed_len()
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
T::ssz_append(self, buf)
}
fn ssz_bytes_len(&self) -> usize {
T::ssz_bytes_len(self)
}
}
/// Compute the encoded length of a vector-like sequence of `T`.
pub fn sequence_ssz_bytes_len<I, T>(iter: I) -> usize
where
I: Iterator<Item = T> + ExactSizeIterator,
T: Encode,
{
// Compute length before doing any iteration.
let length = iter.len();
if <T as Encode>::is_ssz_fixed_len() {
<T as Encode>::ssz_fixed_len() * length
} else {
let mut len = iter.map(|item| item.ssz_bytes_len()).sum();
len += BYTES_PER_LENGTH_OFFSET * length;
len
}
}
/// Encode a vector-like sequence of `T`.
pub fn sequence_ssz_append<I, T>(iter: I, buf: &mut Vec<u8>)
where
I: Iterator<Item = T> + ExactSizeIterator,
T: Encode,
{
if T::is_ssz_fixed_len() {
buf.reserve(T::ssz_fixed_len() * iter.len());
for item in iter {
item.ssz_append(buf);
}
} else {
let mut encoder = SszEncoder::container(buf, iter.len() * BYTES_PER_LENGTH_OFFSET);
for item in iter {
encoder.append(&item);
}
encoder.finalize();
}
}
macro_rules! impl_for_vec {
($type: ty) => {
impl<T: Encode> Encode for $type {
@@ -228,32 +288,11 @@ macro_rules! impl_for_vec {
}
fn ssz_bytes_len(&self) -> usize {
if <T as Encode>::is_ssz_fixed_len() {
<T as Encode>::ssz_fixed_len() * self.len()
} else {
let mut len = self.iter().map(|item| item.ssz_bytes_len()).sum();
len += BYTES_PER_LENGTH_OFFSET * self.len();
len
}
sequence_ssz_bytes_len(self.iter())
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
if T::is_ssz_fixed_len() {
buf.reserve(T::ssz_fixed_len() * self.len());
for item in self {
item.ssz_append(buf);
}
} else {
let mut encoder =
SszEncoder::container(buf, self.len() * BYTES_PER_LENGTH_OFFSET);
for item in self {
encoder.append(item);
}
encoder.finalize();
}
sequence_ssz_append(self.iter(), buf)
}
}
};
@@ -269,6 +308,24 @@ impl_for_vec!(SmallVec<[T; 6]>);
impl_for_vec!(SmallVec<[T; 7]>);
impl_for_vec!(SmallVec<[T; 8]>);
impl<K, V> Encode for BTreeMap<K, V>
where
K: Encode + Ord,
V: Encode,
{
fn is_ssz_fixed_len() -> bool {
false
}
fn ssz_bytes_len(&self) -> usize {
sequence_ssz_bytes_len(self.iter())
}
fn ssz_append(&self, buf: &mut Vec<u8>) {
sequence_ssz_append(self.iter(), buf)
}
}
impl Encode for bool {
fn is_ssz_fixed_len() -> bool {
true

View File

@@ -4,6 +4,8 @@ use ssz_derive::{Decode, Encode};
mod round_trip {
use super::*;
use std::collections::BTreeMap;
use std::iter::FromIterator;
fn round_trip<T: Encode + Decode + std::fmt::Debug + PartialEq>(items: Vec<T>) {
for item in items {
@@ -321,6 +323,52 @@ mod round_trip {
round_trip(vec);
}
#[test]
fn btree_map_fixed() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![(0u8, 0u16), (1, 2), (2, 4), (4, 6)]),
];
round_trip(data);
}
#[test]
fn btree_map_variable_value() {
let data = vec![
BTreeMap::new(),
BTreeMap::from_iter(vec![
(
0u64,
ThreeVariableLen {
a: 1,
b: vec![3, 5, 7],
c: vec![],
d: vec![0, 0],
},
),
(
1,
ThreeVariableLen {
a: 99,
b: vec![1],
c: vec![2, 3, 4, 5, 6, 7, 8, 9, 10],
d: vec![4, 5, 6, 7, 8],
},
),
(
2,
ThreeVariableLen {
a: 0,
b: vec![],
c: vec![],
d: vec![],
},
),
]),
];
round_trip(data);
}
}
mod derive_macro {

View File

@@ -19,6 +19,7 @@ typenum = "1.12.0"
arbitrary = { version = "1.0", features = ["derive"], optional = true }
derivative = "2.1.1"
smallvec = "1.8.0"
milhouse = { path = "../../../milhouse" }
[dev-dependencies]
serde_json = "1.0.58"

View File

@@ -255,7 +255,8 @@ where
})
.map(Into::into)
} else {
ssz::decode_list_of_variable_length_items(bytes, Some(max_len)).map(|vec| vec.into())
ssz::decode_list_of_variable_length_items(bytes, Some(max_len))
.map(|vec: Vec<_>| vec.into())
}
}
}

View File

@@ -71,7 +71,7 @@ impl<T: EthSpec> ConsensusContext<T> {
return Ok(current_block_root);
}
let current_block_root = block.tree_hash_root();
let current_block_root = block.message().tree_hash_root();
self.current_block_root = Some(current_block_root);
Ok(current_block_root)
}

View File

@@ -26,6 +26,7 @@ pub use self::committee_cache::{
CommitteeCache,
};
pub use clone_config::CloneConfig;
pub use diff::BeaconStateDiff;
pub use eth_spec::*;
pub use iter::BlockRootsIter;
@@ -40,6 +41,7 @@ pub use {
#[macro_use]
mod committee_cache;
mod clone_config;
mod diff;
mod exit_cache;
mod iter;
mod pubkey_cache;
@@ -1577,7 +1579,10 @@ impl<T: EthSpec> BeaconState<T> {
|| self.randao_mixes().has_pending_updates()
|| self.slashings().has_pending_updates()
|| self
.inactivity_scores()
.previous_epoch_attestations()
.map_or(false, VList::has_pending_updates)
|| self
.current_epoch_attestations()
.map_or(false, VList::has_pending_updates)
|| self
.previous_epoch_participation()
@@ -1585,6 +1590,9 @@ impl<T: EthSpec> BeaconState<T> {
|| self
.current_epoch_participation()
.map_or(false, VList::has_pending_updates)
|| self
.inactivity_scores()
.map_or(false, VList::has_pending_updates)
}
// FIXME(sproul): automate this somehow
@@ -1598,7 +1606,12 @@ impl<T: EthSpec> BeaconState<T> {
self.randao_mixes_mut().apply_updates()?;
self.slashings_mut().apply_updates()?;
// FIXME(sproul): phase0 fields
if let Ok(previous_epoch_attestations) = self.previous_epoch_attestations_mut() {
previous_epoch_attestations.apply_updates()?;
}
if let Ok(current_epoch_attestations) = self.current_epoch_attestations_mut() {
current_epoch_attestations.apply_updates()?;
}
if let Ok(inactivity_scores) = self.inactivity_scores_mut() {
inactivity_scores.apply_updates()?;
}

View File

@@ -0,0 +1,255 @@
use crate::{
BeaconBlockHeader, BeaconState, BeaconStateError as Error, BitVector, Checkpoint, Eth1Data,
EthSpec, ExecutionPayloadHeader, Fork, Hash256, ParticipationFlags, PendingAttestation, Slot,
SyncCommittee, Validator,
};
use milhouse::{CloneDiff, Diff, ListDiff, ResetListDiff, VectorDiff};
use ssz::{Decode, Encode};
use ssz_derive::{Decode, Encode};
use std::sync::Arc;
/// `Option`-like type implementing SSZ encode/decode.
///
/// Uses a succinct 1 byte union selector.
#[derive(Debug, PartialEq, Encode, Decode)]
#[ssz(enum_behaviour = "union")]
pub enum Maybe<T: Encode + Decode> {
Nothing(u8),
Just(T),
}
impl<T: Encode + Decode> Maybe<T> {
fn nothing() -> Self {
Self::Nothing(0)
}
}
#[derive(Debug, PartialEq, Encode, Decode)]
pub struct BeaconStateDiff<T: EthSpec> {
// Versioning
genesis_time: CloneDiff<u64>,
genesis_validators_root: CloneDiff<Hash256>,
slot: CloneDiff<Slot>,
fork: CloneDiff<Fork>,
// History
latest_block_header: CloneDiff<BeaconBlockHeader>,
block_roots: VectorDiff<Hash256, T::SlotsPerHistoricalRoot>,
state_roots: VectorDiff<Hash256, T::SlotsPerHistoricalRoot>,
historical_roots: ListDiff<Hash256, T::HistoricalRootsLimit>,
// Ethereum 1.0 chain data
eth1_data: CloneDiff<Eth1Data>,
eth1_data_votes: ResetListDiff<Eth1Data, T::SlotsPerEth1VotingPeriod>,
eth1_deposit_index: CloneDiff<u64>,
// Registry
validators: ListDiff<Validator, T::ValidatorRegistryLimit>,
balances: ListDiff<u64, T::ValidatorRegistryLimit>,
// Randomness
randao_mixes: VectorDiff<Hash256, T::EpochsPerHistoricalVector>,
// Slashings
slashings: VectorDiff<u64, T::EpochsPerSlashingsVector>,
// Attestations (genesis fork only)
// FIXME(sproul): do some clever diffing of prev against former current
previous_epoch_attestations:
Maybe<ResetListDiff<PendingAttestation<T>, T::MaxPendingAttestations>>,
current_epoch_attestations:
Maybe<ResetListDiff<PendingAttestation<T>, T::MaxPendingAttestations>>,
// Participation (Altair and later)
previous_epoch_participation: Maybe<ListDiff<ParticipationFlags, T::ValidatorRegistryLimit>>,
current_epoch_participation: Maybe<ListDiff<ParticipationFlags, T::ValidatorRegistryLimit>>,
// Finality
justification_bits: CloneDiff<BitVector<T::JustificationBitsLength>>,
previous_justified_checkpoint: CloneDiff<Checkpoint>,
current_justified_checkpoint: CloneDiff<Checkpoint>,
finalized_checkpoint: CloneDiff<Checkpoint>,
// Inactivity
inactivity_scores: Maybe<ListDiff<u64, T::ValidatorRegistryLimit>>,
// Light-client sync committees
current_sync_committee: Maybe<CloneDiff<Arc<SyncCommittee<T>>>>,
next_sync_committee: Maybe<CloneDiff<Arc<SyncCommittee<T>>>>,
// Execution
latest_execution_payload_header: Maybe<CloneDiff<ExecutionPayloadHeader<T>>>,
}
fn optional_field_diff<
T: EthSpec,
X,
D: Diff<Target = X, Error = milhouse::Error> + Encode + Decode,
>(
old: &BeaconState<T>,
new: &BeaconState<T>,
field: impl Fn(&BeaconState<T>) -> Result<&X, Error>,
) -> Result<Maybe<D>, Error> {
if let Ok(new_value) = field(new) {
let old_value = field(old)?;
Ok(Maybe::Just(D::compute_diff(old_value, new_value)?))
} else {
Ok(Maybe::nothing())
}
}
fn apply_optional_diff<X, D: Diff<Target = X, Error = milhouse::Error> + Encode + Decode>(
diff: Maybe<D>,
field: Result<&mut X, Error>,
) -> Result<(), Error> {
if let Maybe::Just(diff) = diff {
diff.apply_diff(field?)?;
}
Ok(())
}
impl<T: EthSpec> Diff for BeaconStateDiff<T> {
type Target = BeaconState<T>;
type Error = Error;
// FIXME(sproul): proc macro
fn compute_diff(orig: &Self::Target, other: &Self::Target) -> Result<Self, Error> {
// FIXME(sproul): consider cross-variant diffs
Ok(BeaconStateDiff {
genesis_time: <_>::compute_diff(&orig.genesis_time(), &other.genesis_time())?,
genesis_validators_root: <_>::compute_diff(
&orig.genesis_validators_root(),
&other.genesis_validators_root(),
)?,
slot: <_>::compute_diff(&orig.slot(), &other.slot())?,
fork: <_>::compute_diff(&orig.fork(), &other.fork())?,
latest_block_header: <_>::compute_diff(
orig.latest_block_header(),
other.latest_block_header(),
)?,
block_roots: <_>::compute_diff(orig.block_roots(), other.block_roots())?,
state_roots: <_>::compute_diff(orig.state_roots(), other.state_roots())?,
historical_roots: <_>::compute_diff(orig.historical_roots(), other.historical_roots())?,
eth1_data: <_>::compute_diff(orig.eth1_data(), other.eth1_data())?,
eth1_data_votes: <_>::compute_diff(orig.eth1_data_votes(), other.eth1_data_votes())?,
eth1_deposit_index: <_>::compute_diff(
&orig.eth1_deposit_index(),
&other.eth1_deposit_index(),
)?,
validators: <_>::compute_diff(orig.validators(), other.validators())?,
balances: <_>::compute_diff(orig.balances(), other.balances())?,
randao_mixes: <_>::compute_diff(orig.randao_mixes(), other.randao_mixes())?,
slashings: <_>::compute_diff(orig.slashings(), other.slashings())?,
previous_epoch_attestations: optional_field_diff(
orig,
other,
BeaconState::previous_epoch_attestations,
)?,
current_epoch_attestations: optional_field_diff(
orig,
other,
BeaconState::current_epoch_attestations,
)?,
previous_epoch_participation: optional_field_diff(
orig,
other,
BeaconState::previous_epoch_participation,
)?,
current_epoch_participation: optional_field_diff(
orig,
other,
BeaconState::current_epoch_participation,
)?,
justification_bits: <_>::compute_diff(
orig.justification_bits(),
other.justification_bits(),
)?,
previous_justified_checkpoint: <_>::compute_diff(
&orig.previous_justified_checkpoint(),
&other.previous_justified_checkpoint(),
)?,
current_justified_checkpoint: <_>::compute_diff(
&orig.current_justified_checkpoint(),
&other.current_justified_checkpoint(),
)?,
finalized_checkpoint: <_>::compute_diff(
&orig.finalized_checkpoint(),
&other.finalized_checkpoint(),
)?,
inactivity_scores: optional_field_diff(orig, other, BeaconState::inactivity_scores)?,
current_sync_committee: optional_field_diff(
orig,
other,
BeaconState::current_sync_committee,
)?,
next_sync_committee: optional_field_diff(
orig,
other,
BeaconState::next_sync_committee,
)?,
latest_execution_payload_header: optional_field_diff(
orig,
other,
BeaconState::latest_execution_payload_header,
)?,
})
}
fn apply_diff(self, target: &mut BeaconState<T>) -> Result<(), Error> {
self.genesis_time.apply_diff(target.genesis_time_mut())?;
self.genesis_validators_root
.apply_diff(target.genesis_validators_root_mut())?;
self.slot.apply_diff(target.slot_mut())?;
self.fork.apply_diff(target.fork_mut())?;
self.latest_block_header
.apply_diff(target.latest_block_header_mut())?;
self.block_roots.apply_diff(target.block_roots_mut())?;
self.state_roots.apply_diff(target.state_roots_mut())?;
self.historical_roots
.apply_diff(target.historical_roots_mut())?;
self.eth1_data.apply_diff(target.eth1_data_mut())?;
self.eth1_data_votes
.apply_diff(target.eth1_data_votes_mut())?;
self.eth1_deposit_index
.apply_diff(target.eth1_deposit_index_mut())?;
self.validators.apply_diff(target.validators_mut())?;
self.balances.apply_diff(target.balances_mut())?;
self.randao_mixes.apply_diff(target.randao_mixes_mut())?;
self.slashings.apply_diff(target.slashings_mut())?;
apply_optional_diff(
self.previous_epoch_attestations,
target.previous_epoch_attestations_mut(),
)?;
apply_optional_diff(
self.current_epoch_attestations,
target.current_epoch_attestations_mut(),
)?;
apply_optional_diff(
self.previous_epoch_participation,
target.previous_epoch_participation_mut(),
)?;
apply_optional_diff(
self.current_epoch_participation,
target.current_epoch_participation_mut(),
)?;
self.justification_bits
.apply_diff(target.justification_bits_mut())?;
self.previous_justified_checkpoint
.apply_diff(target.previous_justified_checkpoint_mut())?;
self.current_justified_checkpoint
.apply_diff(target.current_justified_checkpoint_mut())?;
self.finalized_checkpoint
.apply_diff(target.finalized_checkpoint_mut())?;
apply_optional_diff(self.inactivity_scores, target.inactivity_scores_mut())?;
apply_optional_diff(
self.current_sync_committee,
target.current_sync_committee_mut(),
)?;
apply_optional_diff(self.next_sync_committee, target.next_sync_committee_mut())?;
apply_optional_diff(
self.latest_execution_payload_header,
target.latest_execution_payload_header_mut(),
)?;
Ok(())
}
}

View File

@@ -226,6 +226,13 @@ impl ChainSpec {
}
}
/// Return the name of the fork activated at `slot`, if any.
pub fn fork_activated_at_slot<E: EthSpec>(&self, slot: Slot) -> Option<ForkName> {
let prev_slot_fork = self.fork_name_at_slot::<E>(slot - 1);
let slot_fork = self.fork_name_at_slot::<E>(slot);
(slot_fork != prev_slot_fork).then(|| slot_fork)
}
/// Returns the fork version for a named fork.
pub fn fork_version_for_name(&self, fork_name: ForkName) -> [u8; 4] {
match fork_name {