gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-14 02:12:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix failing tests (#4423)

Browse Source 
* Get tests passing

* Get benchmarks compiling

* Fix EF withdrawals test

* Remove unused deps

* Fix tree_hash panic in tests

* Fix slasher compilation

* Fix ssz_generic test

* Get more tests passing

* Fix EF tests for real

* Fix local testnet scripts
This commit is contained in:
Michael Sproul
2023-06-27 15:04:39 +10:00
committed by GitHub
parent ca412ab3f0
commit 88e30b6dcf
26 changed files with 404 additions and 500 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
beacon_node/store/src/forwards_iter.rs
View File

@@ -67,8 +67,8 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
/// Forwards root iterator that makes use of a flat field table in the freezer DB.
pub struct FrozenForwardsIterator<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> {
inner: ColumnIter<'a, Vec<u8>>,
limit: Slot,
finished: bool,
next_slot: Slot,
end_slot: Slot,
_phantom: PhantomData<(E, Hot, Cold)>,
}
@@ -88,8 +88,8 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
let start = start_slot.as_u64().to_be_bytes();
Self {
inner: store.cold_db.iter_column_from(column, &start),
limit: end_slot,
finished: false,
next_slot: start_slot,
end_slot,
_phantom: PhantomData,
}
}
@@ -101,7 +101,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator
type Item = Result<(Hash256, Slot)>;
fn next(&mut self) -> Option<Self::Item> {
if self.finished {
if self.next_slot == self.end_slot {
return None;
}
@@ -114,9 +114,9 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Iterator
let slot = Slot::new(u64::from_be_bytes(slot_bytes.try_into().unwrap()));
let root = Hash256::from_slice(&root_bytes);
if slot + 1 == self.limit {
self.finished = true;
}
assert_eq!(slot, self.next_slot);
self.next_slot += 1;
Ok(Some((root, slot)))
}
})
@@ -144,6 +144,7 @@ pub enum HybridForwardsIterator<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemSto
PreFinalization {
iter: Box<FrozenForwardsIterator<'a, E, Hot, Cold>>,
store: &'a HotColdDB<E, Hot, Cold>,
end_slot: Option<Slot>,
/// Data required by the `PostFinalization` iterator when we get to it.
continuation_data: Option<Box<(BeaconState<E>, Hash256)>>,
column: DBColumn,
@@ -157,6 +158,7 @@ pub enum HybridForwardsIterator<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemSto
PostFinalization {
iter: SimpleForwardsIterator,
},
Finished,
}
impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
@@ -183,7 +185,6 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
) -> Result<Self> {
use HybridForwardsIterator::*;
// FIXME(sproul): consider whether this is 100% correct
let split_slot = store.get_split_slot();
let result = if start_slot < split_slot {
@@ -202,6 +203,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
PreFinalization {
iter,
store,
end_slot,
continuation_data,
column,
}
@@ -224,6 +226,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
PreFinalization {
iter,
store,
end_slot,
continuation_data,
column,
} => {
@@ -233,8 +236,15 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
// to a post-finalization iterator beginning from the last slot
// of the pre iterator.
None => {
// If the iterator has an end slot (inclusive) which has already been
// covered by the (exclusive) frozen forwards iterator, then we're done!
if end_slot.map_or(false, |end_slot| iter.end_slot == end_slot + 1) {
*self = Finished;
return Ok(None);
}
let continuation_data = continuation_data.take();
let start_slot = iter.limit;
let start_slot = iter.end_slot;
*self = PostFinalizationLazy {
continuation_data,
@@ -266,6 +276,7 @@ impl<'a, E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>
self.do_next()
}
PostFinalization { iter } => iter.next().transpose(),
Finished => Ok(None),
}
}
}

10
beacon_node/store/src/hot_cold_store.rs
View File

@@ -628,6 +628,16 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
.map(|payload| payload.is_some())
}
/// Store an execution payload in the hot database.
pub fn put_execution_payload(
&self,
block_root: &Hash256,
execution_payload: &ExecutionPayload<E>,
) -> Result<(), Error> {
self.hot_db
.do_atomically(vec![execution_payload.as_kv_store_op(*block_root)?])
}
/// Determine whether a block exists in the database (hot *or* cold).
pub fn block_exists(&self, block_root: &Hash256) -> Result<bool, Error> {
Ok(self

128
beacon_node/store/src/iter.rs
View File

@@ -378,131 +378,3 @@ fn slot_of_prev_restore_point<E: EthSpec>(current_slot: Slot) -> Slot {
let slots_per_historical_root = E::SlotsPerHistoricalRoot::to_u64();
(current_slot - 1) / slots_per_historical_root * slots_per_historical_root
}
#[cfg(test)]
mod test {
use super::*;
use crate::HotColdDB;
use crate::StoreConfig as Config;
use beacon_chain::test_utils::BeaconChainHarness;
use beacon_chain::types::{ChainSpec, MainnetEthSpec};
use sloggers::{null::NullLoggerBuilder, Build};
fn get_state<T: EthSpec>() -> BeaconState<T> {
let harness = BeaconChainHarness::builder(T::default())
.default_spec()
.deterministic_keypairs(1)
.fresh_ephemeral_store()
.build();
harness.advance_slot();
harness.get_current_state()
}
#[test]
fn block_root_iter() {
let log = NullLoggerBuilder.build().unwrap();
let store =
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap();
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state();
let mut state_b: BeaconState<MainnetEthSpec> = get_state();
*state_a.slot_mut() = Slot::from(slots_per_historical_root);
*state_b.slot_mut() = Slot::from(slots_per_historical_root * 2);
let mut hashes = (0..).map(Hash256::from_low_u64_be);
let roots_a = state_a.block_roots_mut();
for i in 0..roots_a.len() {
*roots_a.get_mut(i).unwrap() = hashes.next().unwrap()
}
let roots_b = state_b.block_roots_mut();
for i in 0..roots_b.len() {
*roots_b.get_mut(i).unwrap() = hashes.next().unwrap()
}
let state_a_root = hashes.next().unwrap();
*state_b.state_roots_mut().get_mut(0).unwrap() = state_a_root;
store.put_state(&state_a_root, &state_a).unwrap();
let iter = BlockRootsIterator::new(&store, &state_b);
assert!(
iter.clone()
.any(|result| result.map(|(_root, slot)| slot == 0).unwrap()),
"iter should contain zero slot"
);
let mut collected: Vec<(Hash256, Slot)> = iter.collect::<Result<Vec<_>, _>>().unwrap();
collected.reverse();
let expected_len = 2 * MainnetEthSpec::slots_per_historical_root();
assert_eq!(collected.len(), expected_len);
for (i, item) in collected.iter().enumerate() {
assert_eq!(item.0, Hash256::from_low_u64_be(i as u64));
}
}
#[test]
fn state_root_iter() {
let log = NullLoggerBuilder.build().unwrap();
let store =
HotColdDB::open_ephemeral(Config::default(), ChainSpec::minimal(), log).unwrap();
let slots_per_historical_root = MainnetEthSpec::slots_per_historical_root();
let mut state_a: BeaconState<MainnetEthSpec> = get_state();
let mut state_b: BeaconState<MainnetEthSpec> = get_state();
*state_a.slot_mut() = Slot::from(slots_per_historical_root);
*state_b.slot_mut() = Slot::from(slots_per_historical_root * 2);
let mut hashes = (0..).map(Hash256::from_low_u64_be);
for slot in 0..slots_per_historical_root {
state_a
.set_state_root(Slot::from(slot), hashes.next().unwrap())
.unwrap_or_else(|_| panic!("should set state_a slot {}", slot));
}
for slot in slots_per_historical_root..slots_per_historical_root * 2 {
state_b
.set_state_root(Slot::from(slot), hashes.next().unwrap())
.unwrap_or_else(|_| panic!("should set state_b slot {}", slot));
}
let state_a_root = Hash256::from_low_u64_be(slots_per_historical_root as u64);
let state_b_root = Hash256::from_low_u64_be(slots_per_historical_root as u64 * 2);
store.put_state(&state_a_root, &state_a).unwrap();
store.put_state(&state_b_root, &state_b).unwrap();
let iter = StateRootsIterator::new(&store, &state_b);
assert!(
iter.clone()
.any(|result| result.map(|(_root, slot)| slot == 0).unwrap()),
"iter should contain zero slot"
);
let mut collected: Vec<(Hash256, Slot)> = iter.collect::<Result<Vec<_>, _>>().unwrap();
collected.reverse();
let expected_len = MainnetEthSpec::slots_per_historical_root() * 2;
assert_eq!(collected.len(), expected_len, "collection length incorrect");
for (i, item) in collected.iter().enumerate() {
let (hash, slot) = *item;
assert_eq!(slot, i as u64, "slot mismatch at {}: {} vs {}", i, slot, i);
assert_eq!(
hash,
Hash256::from_low_u64_be(i as u64),
"hash mismatch at {}",
i
);
}
}
}

7
beacon_node/store/src/leveldb_store.rs
View File

@@ -177,11 +177,6 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
Ok(())
}
/// Iterate through all keys and values in a particular column.
fn iter_column<K: Key>(&self, column: DBColumn) -> ColumnIter<K> {
self.iter_column_from(column, &vec![0; column.key_size()])
}
fn iter_column_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnIter<K> {
let start_key = BytesKey::from_vec(get_key_for_col(column.into(), from));
@@ -227,7 +222,7 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
impl<E: EthSpec> ItemStore<E> for LevelDB<E> {}
/// Used for keying leveldb.
#[derive(Debug, Clone, PartialEq)]
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub struct BytesKey {
key: Vec<u8>,
}

10
beacon_node/store/src/lib.rs
View File

@@ -83,16 +83,10 @@ pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
self.iter_column_from(column, &vec![0; column.key_size()])
}
fn iter_column_from<K: Key>(&self, _column: DBColumn, _from: &[u8]) -> ColumnIter<K> {
// Default impl for non LevelDB databases
Box::new(std::iter::empty())
}
fn iter_column_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnIter<K>;
/// Iterate through all keys in a particular column.
fn iter_column_keys(&self, _column: DBColumn) -> ColumnKeyIter {
// Default impl for non LevelDB databases
Box::new(std::iter::empty())
}
fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter;
}
pub trait Key: Sized + 'static {

85
beacon_node/store/src/memory_store.rs
View File

@@ -1,16 +1,17 @@
use crate::{ColumnIter, DBColumn, Error, ItemStore, Key, KeyValueStore, KeyValueStoreOp};
use crate::{
get_key_for_col, leveldb_store::BytesKey, ColumnIter, ColumnKeyIter, DBColumn, Error,
ItemStore, Key, KeyValueStore, KeyValueStoreOp,
};
use parking_lot::{Mutex, MutexGuard, RwLock};
use std::collections::{HashMap, HashSet};
use std::collections::BTreeMap;
use std::marker::PhantomData;
use types::*;
type DBHashMap = HashMap<Vec<u8>, Vec<u8>>;
type DBKeyMap = HashMap<Vec<u8>, HashSet<Vec<u8>>>;
type DBMap = BTreeMap<BytesKey, Vec<u8>>;
/// A thread-safe `HashMap` wrapper.
/// A thread-safe `BTreeMap` wrapper.
pub struct MemoryStore<E: EthSpec> {
db: RwLock<DBHashMap>,
col_keys: RwLock<DBKeyMap>,
db: RwLock<DBMap>,
transaction_mutex: Mutex<()>,
_phantom: PhantomData<E>,
}
@@ -19,36 +20,24 @@ impl<E: EthSpec> MemoryStore<E> {
/// Create a new, empty database.
pub fn open() -> Self {
Self {
db: RwLock::new(HashMap::new()),
col_keys: RwLock::new(HashMap::new()),
db: RwLock::new(BTreeMap::new()),
transaction_mutex: Mutex::new(()),
_phantom: PhantomData,
}
}
fn get_key_for_col(col: &str, key: &[u8]) -> Vec<u8> {
let mut col = col.as_bytes().to_vec();
col.append(&mut key.to_vec());
col
}
}
impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
/// Get the value of some key from the database. Returns `None` if the key does not exist.
fn get_bytes(&self, col: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error> {
let column_key = Self::get_key_for_col(col, key);
let column_key = BytesKey::from_vec(get_key_for_col(col, key));
Ok(self.db.read().get(&column_key).cloned())
}
/// Puts a key in the database.
fn put_bytes(&self, col: &str, key: &[u8], val: &[u8]) -> Result<(), Error> {
let column_key = Self::get_key_for_col(col, key);
let column_key = BytesKey::from_vec(get_key_for_col(col, key));
self.db.write().insert(column_key, val.to_vec());
self.col_keys
.write()
.entry(col.as_bytes().to_vec())
.or_insert_with(HashSet::new)
.insert(key.to_vec());
Ok(())
}
@@ -63,18 +52,14 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
/// Return true if some key exists in some column.
fn key_exists(&self, col: &str, key: &[u8]) -> Result<bool, Error> {
let column_key = Self::get_key_for_col(col, key);
let column_key = BytesKey::from_vec(get_key_for_col(col, key));
Ok(self.db.read().contains_key(&column_key))
}
/// Delete some key from the database.
fn key_delete(&self, col: &str, key: &[u8]) -> Result<(), Error> {
let column_key = Self::get_key_for_col(col, key);
let column_key = BytesKey::from_vec(get_key_for_col(col, key));
self.db.write().remove(&column_key);
self.col_keys
.write()
.get_mut(&col.as_bytes().to_vec())
.map(|set| set.remove(key));
Ok(())
}
@@ -82,35 +67,41 @@ impl<E: EthSpec> KeyValueStore<E> for MemoryStore<E> {
for op in batch {
match op {
KeyValueStoreOp::PutKeyValue(key, value) => {
self.db.write().insert(key, value);
self.db.write().insert(BytesKey::from_vec(key), value);
}
KeyValueStoreOp::DeleteKey(hash) => {
self.db.write().remove(&hash);
KeyValueStoreOp::DeleteKey(key) => {
self.db.write().remove(&BytesKey::from_vec(key));
}
}
}
Ok(())
}
fn iter_column<K: Key>(&self, column: DBColumn) -> ColumnIter<K> {
fn iter_column_from<K: Key>(&self, column: DBColumn, from: &[u8]) -> ColumnIter<K> {
// We use this awkward pattern because we can't lock the `self.db` field *and* maintain a
// reference to the lock guard across calls to `.next()`. This would be require a
// struct with a field (the iterator) which references another field (the lock guard).
let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), from));
let col = column.as_str();
if let Some(keys) = self
.col_keys
let keys = self
.db
.read()
.get(col.as_bytes())
.map(|set| set.iter().cloned().collect::<Vec<_>>())
{
Box::new(keys.into_iter().filter_map(move |key| {
self.get_bytes(col, &key).transpose().map(|res| {
let k = K::from_bytes(&key)?;
let v = res?;
Ok((k, v))
})
}))
} else {
Box::new(std::iter::empty())
}
.range(start_key..)
.take_while(|(k, _)| k.remove_column_variable(column).is_some())
.filter_map(|(k, _)| k.remove_column_variable(column).map(|k| k.to_vec()))
.collect::<Vec<_>>();
Box::new(keys.into_iter().filter_map(move |key| {
self.get_bytes(col, &key).transpose().map(|res| {
let k = K::from_bytes(&key)?;
let v = res?;
Ok((k, v))
})
}))
}
fn iter_column_keys(&self, column: DBColumn) -> ColumnKeyIter {
Box::new(self.iter_column(column).map(|res| res.map(|(k, _)| k)))
}
fn begin_rw_transaction(&self) -> MutexGuard<()> {