gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-10 04:01:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement freezer database (#508)

Browse Source 
* Implement freezer database for state vectors

* Improve BeaconState safe accessors

And fix a bug in the compact committees accessor.

* Banish dodgy type bounds back to gRPC

* Clean up

* Switch to exclusive end points in chunked vec

* Cleaning up and start of tests

* Randao fix, more tests

* Fix unsightly hack

* Resolve test FIXMEs

* Config file support

* More clean-ups, migrator beginnings

* Finish migrator, integrate into BeaconChain

* Fixups

* Fix store tests

* Fix BeaconChain tests

* Fix LMD GHOST tests

* Address review comments, delete 'static bounds

* Cargo format

* Address review comments

* Fix LMD ghost tests

* Update to spec v0.9.0

* Update to v0.9.1

* Bump spec tags for v0.9.1

* Formatting, fix CI failures

* Resolve accidental KeyPair merge conflict

* Document new BeaconState functions

* Fix incorrect cache drops in `advance_caches`

* Update fork choice for v0.9.1

* Clean up some FIXMEs

* Fix a few docs/logs

* Update for new builder paradigm, spec changes

* Freezer DB integration into BeaconNode

* Cleaning up

* This works, clean it up

* Cleanups

* Fix and improve store tests

* Refine store test

* Delete unused beacon_chain_builder.rs

* Fix CLI

* Store state at split slot in hot database

* Make fork choice lookup fast again

* Store freezer DB split slot in the database

* Handle potential div by 0 in chunked_vector

* Exclude committee caches from freezer DB

* Remove FIXME about long-running test
This commit is contained in:
Michael Sproul
2019-11-27 10:54:46 +11:00
committed by Paul Hauner
parent a514968155
commit bf2eeae3f2
36 changed files with 2486 additions and 580 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
beacon_node/client/src/builder.rs
View File

@@ -5,7 +5,10 @@ use beacon_chain::{
eth1_chain::CachingEth1Backend,
lmd_ghost::ThreadSafeReducedTree,
slot_clock::{SlotClock, SystemTimeSlotClock},
store::{DiskStore, MemoryStore, Store},
store::{
migrate::{BackgroundMigrator, Migrate, NullMigrator},
DiskStore, MemoryStore, SimpleDiskStore, Store,
},
BeaconChain, BeaconChainTypes, Eth1ChainBackend, EventHandler,
};
use environment::RuntimeContext;
@@ -52,6 +55,7 @@ pub const ETH1_GENESIS_UPDATE_INTERVAL_MILLIS: u64 = 500;
pub struct ClientBuilder<T: BeaconChainTypes> {
slot_clock: Option<T::SlotClock>,
store: Option<Arc<T::Store>>,
store_migrator: Option<T::StoreMigrator>,
runtime_context: Option<RuntimeContext<T::EthSpec>>,
chain_spec: Option<ChainSpec>,
beacon_chain_builder: Option<BeaconChainBuilder<T>>,
@@ -66,10 +70,21 @@ pub struct ClientBuilder<T: BeaconChainTypes> {
eth_spec_instance: T::EthSpec,
}
impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec>,
TSlotClock: SlotClock + Clone + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -83,6 +98,7 @@ where
Self {
slot_clock: None,
store: None,
store_migrator: None,
runtime_context: None,
chain_spec: None,
beacon_chain_builder: None,
@@ -118,6 +134,7 @@ where
config: Eth1Config,
) -> impl Future<Item = Self, Error = String> {
let store = self.store.clone();
let store_migrator = self.store_migrator.take();
let chain_spec = self.chain_spec.clone();
let runtime_context = self.runtime_context.clone();
let eth_spec_instance = self.eth_spec_instance.clone();
@@ -126,6 +143,9 @@ where
.and_then(move |()| {
let store = store
.ok_or_else(|| "beacon_chain_start_method requires a store".to_string())?;
let store_migrator = store_migrator.ok_or_else(|| {
"beacon_chain_start_method requires a store migrator".to_string()
})?;
let context = runtime_context
.ok_or_else(|| "beacon_chain_start_method requires a log".to_string())?
.service_context("beacon");
@@ -135,6 +155,7 @@ where
let builder = BeaconChainBuilder::new(eth_spec_instance)
.logger(context.log.clone())
.store(store.clone())
.store_migrator(store_migrator)
.custom_spec(spec.clone());
Ok((builder, spec, context))
@@ -300,7 +321,9 @@ where
&context.executor,
beacon_chain.clone(),
network_info,
client_config.db_path().expect("unable to read datadir"),
client_config
.create_db_path()
.expect("unable to read datadir"),
eth2_config.clone(),
context.log,
)
@@ -420,7 +443,17 @@ where
/// If type inference errors are being raised, see the comment on the definition of `Self`.
pub fn build(
self,
) -> Client<Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>> {
) -> Client<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
> {
Client {
beacon_chain: self.beacon_chain,
libp2p_network: self.libp2p_network,
@@ -431,10 +464,11 @@ where
}
}
impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
ThreadSafeReducedTree<TStore, TEthSpec>,
TEth1Backend,
@@ -444,6 +478,7 @@ impl<TStore, TSlotClock, TEth1Backend, TEthSpec, TEventHandler>
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec>,
TSlotClock: SlotClock + Clone + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
@@ -476,12 +511,21 @@ where
}
}
impl<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec>
ClientBuilder<
Witness<TStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, WebSocketSender<TEthSpec>>,
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
WebSocketSender<TEthSpec>,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec>,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
@@ -517,18 +561,68 @@ where
}
}
impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<Witness<DiskStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>>
impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
DiskStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TSlotClock: SlotClock + 'static,
TStoreMigrator: store::Migrate<DiskStore, TEthSpec> + 'static,
TLmdGhost: LmdGhost<DiskStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
/// Specifies that the `Client` should use a `DiskStore` database.
pub fn disk_store(mut self, path: &Path) -> Result<Self, String> {
let store = DiskStore::open(path)
pub fn disk_store(mut self, hot_path: &Path, cold_path: &Path) -> Result<Self, String> {
let context = self
.runtime_context
.as_ref()
.ok_or_else(|| "disk_store requires a log".to_string())?
.service_context("freezer_db");
let spec = self
.chain_spec
.clone()
.ok_or_else(|| "disk_store requires a chain spec".to_string())?;
let store = DiskStore::open(hot_path, cold_path, spec, context.log)
.map_err(|e| format!("Unable to open database: {:?}", e).to_string())?;
self.store = Some(Arc::new(store));
Ok(self)
}
}
impl<TStoreMigrator, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
SimpleDiskStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TSlotClock: SlotClock + 'static,
TStoreMigrator: store::Migrate<SimpleDiskStore, TEthSpec> + 'static,
TLmdGhost: LmdGhost<SimpleDiskStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
/// Specifies that the `Client` should use a `DiskStore` database.
pub fn simple_disk_store(mut self, path: &Path) -> Result<Self, String> {
let store = SimpleDiskStore::open(path)
.map_err(|e| format!("Unable to open database: {:?}", e).to_string())?;
self.store = Some(Arc::new(store));
Ok(self)
@@ -537,7 +631,15 @@ where
impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<MemoryStore, TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
MemoryStore,
NullMigrator,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TSlotClock: SlotClock + 'static,
@@ -547,17 +649,49 @@ where
TEventHandler: EventHandler<TEthSpec> + 'static,
{
/// Specifies that the `Client` should use a `MemoryStore` database.
///
/// Also sets the `store_migrator` to the `NullMigrator`, as that's the only viable choice.
pub fn memory_store(mut self) -> Self {
let store = MemoryStore::open();
self.store = Some(Arc::new(store));
self.store_migrator = Some(NullMigrator);
self
}
}
impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
impl<TSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
DiskStore,
BackgroundMigrator<TEthSpec>,
TSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<DiskStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
TEventHandler: EventHandler<TEthSpec> + 'static,
{
pub fn background_migrator(mut self) -> Result<Self, String> {
let store = self.store.clone().ok_or_else(|| {
"background_migrator requires the store to be initialized".to_string()
})?;
self.store_migrator = Some(BackgroundMigrator::new(store));
Ok(self)
}
}
impl<TStore, TStoreMigrator, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
ClientBuilder<
Witness<
TStore,
TStoreMigrator,
TSlotClock,
TLmdGhost,
CachingEth1Backend<TEthSpec, TStore>,
@@ -567,6 +701,7 @@ impl<TStore, TSlotClock, TLmdGhost, TEthSpec, TEventHandler>
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec>,
TSlotClock: SlotClock + 'static,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,
@@ -643,12 +778,21 @@ where
}
}
impl<TStore, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
impl<TStore, TStoreMigrator, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>
ClientBuilder<
Witness<TStore, SystemTimeSlotClock, TLmdGhost, TEth1Backend, TEthSpec, TEventHandler>,
Witness<
TStore,
TStoreMigrator,
SystemTimeSlotClock,
TLmdGhost,
TEth1Backend,
TEthSpec,
TEventHandler,
>,
>
where
TStore: Store + 'static,
TStoreMigrator: store::Migrate<TStore, TEthSpec>,
TLmdGhost: LmdGhost<TStore, TEthSpec> + 'static,
TEth1Backend: Eth1ChainBackend<TEthSpec> + 'static,
TEthSpec: EthSpec + 'static,