gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-22 22:34:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add error handling to iterators (#1243)

Browse Source 
* Add error handling to iterators

* Review feedback

* Leverage itertools::process_results() in few places
This commit is contained in:
Adam Szkoda
2020-06-10 01:55:44 +02:00
committed by GitHub
parent ed4b3ef471
commit 7f036a6e95
17 changed files with 193 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
beacon_node/store/src/errors.rs
View File

@@ -3,6 +3,8 @@ use crate::hot_cold_store::HotColdDBError;
use ssz::DecodeError;
use types::{BeaconStateError, Hash256};
pub type Result<T> = std::result::Result<T, Error>;
#[derive(Debug)]
pub enum Error {
SszDecodeError(DecodeError),
@@ -13,6 +15,7 @@ pub enum Error {
DBError { message: String },
RlpError(String),
BlockNotFound(Hash256),
NoContinuationData,
}
impl From<DecodeError> for Error {

63
beacon_node/store/src/forwards_iter.rs
View File

@@ -1,8 +1,9 @@
use crate::chunked_iter::ChunkedVectorIter;
use crate::chunked_vector::BlockRoots;
use crate::errors::{Error, Result};
use crate::iter::BlockRootsIterator;
use crate::{HotColdDB, Store};
use slog::error;
use itertools::process_results;
use std::sync::Arc;
use types::{BeaconState, ChainSpec, EthSpec, Hash256, Slot};
@@ -63,22 +64,26 @@ impl SimpleForwardsBlockRootsIterator {
start_slot: Slot,
end_state: BeaconState<E>,
end_block_root: Hash256,
) -> Self {
) -> Result<Self> {
// Iterate backwards from the end state, stopping at the start slot.
let iter = std::iter::once((end_block_root, end_state.slot))
.chain(BlockRootsIterator::owned(store, end_state));
Self {
values: iter.take_while(|(_, slot)| *slot >= start_slot).collect(),
}
let values = process_results(
std::iter::once(Ok((end_block_root, end_state.slot)))
.chain(BlockRootsIterator::owned(store, end_state)),
|iter| {
iter.take_while(|(_, slot)| *slot >= start_slot)
.collect::<Vec<_>>()
},
)?;
Ok(Self { values: values })
}
}
impl Iterator for SimpleForwardsBlockRootsIterator {
type Item = (Hash256, Slot);
type Item = Result<(Hash256, Slot)>;
fn next(&mut self) -> Option<Self::Item> {
// Pop from the end of the vector to get the block roots in slot-ascending order.
self.values.pop()
Ok(self.values.pop()).transpose()
}
}
@@ -89,12 +94,12 @@ impl<E: EthSpec> HybridForwardsBlockRootsIterator<E> {
end_state: BeaconState<E>,
end_block_root: Hash256,
spec: &ChainSpec,
) -> Self {
) -> Result<Self> {
use HybridForwardsBlockRootsIterator::*;
let latest_restore_point_slot = store.get_latest_restore_point_slot();
if start_slot < latest_restore_point_slot {
let result = if start_slot < latest_restore_point_slot {
PreFinalization {
iter: Box::new(FrozenForwardsBlockRootsIterator::new(
store,
@@ -111,16 +116,14 @@ impl<E: EthSpec> HybridForwardsBlockRootsIterator<E> {
start_slot,
end_state,
end_block_root,
),
)?,
}
}
};
Ok(result)
}
}
impl<E: EthSpec> Iterator for HybridForwardsBlockRootsIterator<E> {
type Item = (Hash256, Slot);
fn next(&mut self) -> Option<Self::Item> {
fn do_next(&mut self) -> Result<Option<(Hash256, Slot)>> {
use HybridForwardsBlockRootsIterator::*;
match self {
@@ -129,19 +132,13 @@ impl<E: EthSpec> Iterator for HybridForwardsBlockRootsIterator<E> {
continuation_data,
} => {
match iter.next() {
Some(x) => Some(x),
Some(x) => Ok(Some(x)),
// Once the pre-finalization iterator is consumed, transition
// to a post-finalization iterator beginning from the last slot
// of the pre iterator.
None => {
let (end_state, end_block_root) =
continuation_data.take().or_else(|| {
error!(
iter.inner.store.log,
"HybridForwardsBlockRootsIterator: logic error"
);
None
})?;
continuation_data.take().ok_or(Error::NoContinuationData)?;
*self = PostFinalization {
iter: SimpleForwardsBlockRootsIterator::new(
@@ -149,13 +146,21 @@ impl<E: EthSpec> Iterator for HybridForwardsBlockRootsIterator<E> {
Slot::from(iter.inner.end_vindex),
end_state,
end_block_root,
),
)?,
};
self.next()
self.do_next()
}
}
}
PostFinalization { iter } => iter.next(),
PostFinalization { iter } => iter.next().transpose(),
}
}
}
impl<E: EthSpec> Iterator for HybridForwardsBlockRootsIterator<E> {
type Item = Result<(Hash256, Slot)>;
fn next(&mut self) -> Option<Self::Item> {
self.do_next().transpose()
}
}

8
beacon_node/store/src/hot_cold_store.rs
View File

@@ -190,7 +190,7 @@ impl<E: EthSpec> Store<E> for HotColdDB<E> {
end_state: BeaconState<E>,
end_block_root: Hash256,
spec: &ChainSpec,
) -> Self::ForwardsBlockRootsIterator {
) -> Result<Self::ForwardsBlockRootsIterator, Error> {
HybridForwardsBlockRootsIterator::new(store, start_slot, end_state, end_block_root, spec)
}
@@ -708,7 +708,11 @@ pub fn process_finalization<E: EthSpec>(
let state_root_iter = StateRootsIterator::new(store.clone(), frozen_head);
let mut to_delete = vec![];
for (state_root, slot) in state_root_iter.take_while(|&(_, slot)| slot >= current_split_slot) {
for maybe_pair in state_root_iter.take_while(|result| match result {
Ok((_, slot)) => slot >= &current_split_slot,
Err(_) => true,
}) {
let (state_root, slot) = maybe_pair?;
if slot % store.config.slots_per_restore_point == 0 {
let state: BeaconState<E> = get_full_state(&store.hot_db, &state_root)?
.ok_or_else(|| HotColdDBError::MissingStateToFreeze(state_root))?;

69
beacon_node/store/src/iter.rs
View File

@@ -69,12 +69,12 @@ impl<'a, T: EthSpec, U: Store<T>> StateRootsIterator<'a, T, U> {
}
impl<'a, T: EthSpec, U: Store<T>> Iterator for StateRootsIterator<'a, T, U> {
type Item = (Hash256, Slot);
type Item = Result<(Hash256, Slot), Error>;
fn next(&mut self) -> Option<Self::Item> {
self.inner
.next()
.map(|(_, state_root, slot)| (state_root, slot))
.map(|result| result.map(|(_, state_root, slot)| (state_root, slot)))
}
}
@@ -115,12 +115,12 @@ impl<'a, T: EthSpec, U: Store<T>> BlockRootsIterator<'a, T, U> {
}
impl<'a, T: EthSpec, U: Store<T>> Iterator for BlockRootsIterator<'a, T, U> {
type Item = (Hash256, Slot);
type Item = Result<(Hash256, Slot), Error>;
fn next(&mut self) -> Option<Self::Item> {
self.inner
.next()
.map(|(block_root, _, slot)| (block_root, slot))
.map(|result| result.map(|(block_root, _, slot)| (block_root, slot)))
}
}
@@ -167,15 +167,10 @@ impl<'a, T: EthSpec, U: Store<T>> RootsIterator<'a, T, U> {
.ok_or_else(|| BeaconStateError::MissingBeaconState(block.state_root().into()))?;
Ok(Self::owned(store, state))
}
}
impl<'a, T: EthSpec, U: Store<T>> Iterator for RootsIterator<'a, T, U> {
/// (block_root, state_root, slot)
type Item = (Hash256, Hash256, Slot);
fn next(&mut self) -> Option<Self::Item> {
fn do_next(&mut self) -> Result<Option<(Hash256, Hash256, Slot)>, Error> {
if self.slot == 0 || self.slot > self.beacon_state.slot {
return None;
return Ok(None);
}
self.slot -= 1;
@@ -184,7 +179,7 @@ impl<'a, T: EthSpec, U: Store<T>> Iterator for RootsIterator<'a, T, U> {
self.beacon_state.get_block_root(self.slot),
self.beacon_state.get_state_root(self.slot),
) {
(Ok(block_root), Ok(state_root)) => Some((*block_root, *state_root, self.slot)),
(Ok(block_root), Ok(state_root)) => Ok(Some((*block_root, *state_root, self.slot))),
(Err(BeaconStateError::SlotOutOfBounds), Err(BeaconStateError::SlotOutOfBounds)) => {
// Read a `BeaconState` from the store that has access to prior historical roots.
let beacon_state =
@@ -192,16 +187,26 @@ impl<'a, T: EthSpec, U: Store<T>> Iterator for RootsIterator<'a, T, U> {
self.beacon_state = Cow::Owned(beacon_state);
let block_root = *self.beacon_state.get_block_root(self.slot).ok()?;
let state_root = *self.beacon_state.get_state_root(self.slot).ok()?;
let block_root = *self.beacon_state.get_block_root(self.slot)?;
let state_root = *self.beacon_state.get_state_root(self.slot)?;
Some((block_root, state_root, self.slot))
Ok(Some((block_root, state_root, self.slot)))
}
_ => None,
(Err(e), _) => Err(e.into()),
(Ok(_), Err(e)) => Err(e.into()),
}
}
}
impl<'a, T: EthSpec, U: Store<T>> Iterator for RootsIterator<'a, T, U> {
/// (block_root, state_root, slot)
type Item = Result<(Hash256, Hash256, Slot), Error>;
fn next(&mut self) -> Option<Self::Item> {
self.do_next().transpose()
}
}
/// Block iterator that uses the `parent_root` of each block to backtrack.
pub struct ParentRootBlockIterator<'a, E: EthSpec, S: Store<E>> {
store: &'a S,
@@ -263,14 +268,22 @@ impl<'a, T: EthSpec, U: Store<T>> BlockIterator<'a, T, U> {
roots: BlockRootsIterator::owned(store, beacon_state),
}
}
fn do_next(&mut self) -> Result<Option<SignedBeaconBlock<T>>, Error> {
if let Some(result) = self.roots.next() {
let (root, _slot) = result?;
self.roots.inner.store.get_block(&root)
} else {
Ok(None)
}
}
}
impl<'a, T: EthSpec, U: Store<T>> Iterator for BlockIterator<'a, T, U> {
type Item = SignedBeaconBlock<T>;
type Item = Result<SignedBeaconBlock<T>, Error>;
fn next(&mut self) -> Option<Self::Item> {
let (root, _slot) = self.roots.next()?;
self.roots.inner.store.get_block(&root).ok()?
self.do_next().transpose()
}
}
@@ -278,14 +291,16 @@ impl<'a, T: EthSpec, U: Store<T>> Iterator for BlockIterator<'a, T, U> {
fn next_historical_root_backtrack_state<E: EthSpec, S: Store<E>>(
store: &S,
current_state: &BeaconState<E>,
) -> Option<BeaconState<E>> {
) -> Result<BeaconState<E>, Error> {
// For compatibility with the freezer database's restore points, we load a state at
// a restore point slot (thus avoiding replaying blocks). In the case where we're
// not frozen, this just means we might not jump back by the maximum amount on
// our first jump (i.e. at most 1 extra state load).
let new_state_slot = slot_of_prev_restore_point::<E>(current_state.slot);
let new_state_root = current_state.get_state_root(new_state_slot).ok()?;
store.get_state(new_state_root, Some(new_state_slot)).ok()?
let new_state_root = current_state.get_state_root(new_state_slot)?;
Ok(store
.get_state(new_state_root, Some(new_state_slot))?
.ok_or_else(|| BeaconStateError::MissingBeaconState((*new_state_root).into()))?)
}
/// Compute the slot of the last guaranteed restore point in the freezer database.
@@ -337,11 +352,12 @@ mod test {
let iter = BlockRootsIterator::new(store, &state_b);
assert!(
iter.clone().any(|(_root, slot)| slot == 0),
iter.clone()
.any(|result| result.map(|(_root, slot)| slot == 0).unwrap()),
"iter should contain zero slot"
);
let mut collected: Vec<(Hash256, Slot)> = iter.collect();
let mut collected: Vec<(Hash256, Slot)> = iter.collect::<Result<Vec<_>, _>>().unwrap();
collected.reverse();
let expected_len = 2 * MainnetEthSpec::slots_per_historical_root();
@@ -386,11 +402,12 @@ mod test {
let iter = StateRootsIterator::new(store, &state_b);
assert!(
iter.clone().any(|(_root, slot)| slot == 0),
iter.clone()
.any(|result| result.map(|(_root, slot)| slot == 0).unwrap()),
"iter should contain zero slot"
);
let mut collected: Vec<(Hash256, Slot)> = iter.collect();
let mut collected: Vec<(Hash256, Slot)> = iter.collect::<Result<Vec<_>, _>>().unwrap();
collected.reverse();
let expected_len = MainnetEthSpec::slots_per_historical_root() * 2;

6
beacon_node/store/src/lib.rs
View File

@@ -13,7 +13,7 @@ extern crate lazy_static;
pub mod chunked_iter;
pub mod chunked_vector;
pub mod config;
mod errors;
pub mod errors;
mod forwards_iter;
pub mod hot_cold_store;
mod impls;
@@ -99,7 +99,7 @@ pub trait ItemStore<E: EthSpec>: KeyValueStore<E> + Sync + Send + Sized + 'stati
/// columns. A simple column implementation might involve prefixing a key with some bytes unique to
/// each column.
pub trait Store<E: EthSpec>: Sync + Send + Sized + 'static {
type ForwardsBlockRootsIterator: Iterator<Item = (Hash256, Slot)>;
type ForwardsBlockRootsIterator: Iterator<Item = Result<(Hash256, Slot), Error>>;
/// Store a block in the store.
fn put_block(&self, block_root: &Hash256, block: SignedBeaconBlock<E>) -> Result<(), Error>;
@@ -146,7 +146,7 @@ pub trait Store<E: EthSpec>: Sync + Send + Sized + 'static {
end_state: BeaconState<E>,
end_block_root: Hash256,
spec: &ChainSpec,
) -> Self::ForwardsBlockRootsIterator;
) -> Result<Self::ForwardsBlockRootsIterator, Error>;
fn load_epoch_boundary_state(
&self,

2
beacon_node/store/src/memory_store.rs
View File

@@ -149,7 +149,7 @@ impl<E: EthSpec> Store<E> for MemoryStore<E> {
end_state: BeaconState<E>,
end_block_root: Hash256,
_: &ChainSpec,
) -> Self::ForwardsBlockRootsIterator {
) -> Result<Self::ForwardsBlockRootsIterator, Error> {
SimpleForwardsBlockRootsIterator::new(store, start_slot, end_state, end_block_root)
}