gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-29 18:53:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimise slasher DB layout and switch to MDBX (#2776)

Browse Source 
## Issue Addressed

Closes #2286
Closes #2538
Closes #2342

## Proposed Changes

Part II of major slasher optimisations after #2767

These changes will be backwards-incompatible due to the move to MDBX (and the schema change) 😱 

* [x] Shrink attester keys from 16 bytes to 7 bytes.
* [x] Shrink attester records from 64 bytes to 6 bytes.
* [x] Separate `DiskConfig` from regular `Config`.
* [x] Add configuration for the LRU cache size.
* [x] Add a "migration" that deletes any legacy LMDB database.
This commit is contained in:
Michael Sproul
2021-12-21 08:23:17 +00:00
parent a290a3c537
commit 3b61ac9cbf
26 changed files with 963 additions and 566 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
slasher/src/array.rs
View File

@@ -1,8 +1,9 @@
use crate::metrics::{self, SLASHER_COMPRESSION_RATIO, SLASHER_NUM_CHUNKS_UPDATED};
use crate::RwTransaction;
use crate::{AttesterSlashingStatus, Config, Error, IndexedAttesterRecord, SlasherDB};
use flate2::bufread::{ZlibDecoder, ZlibEncoder};
use lmdb::{RwTransaction, Transaction};
use serde_derive::{Deserialize, Serialize};
use std::borrow::{Borrow, Cow};
use std::collections::{btree_map::Entry, BTreeMap, HashSet};
use std::convert::TryFrom;
use std::io::Read;
@@ -146,7 +147,10 @@ pub trait TargetArrayChunk: Sized + serde::Serialize + serde::de::DeserializeOwn
fn next_start_epoch(start_epoch: Epoch, config: &Config) -> Epoch;
fn select_db<E: EthSpec>(db: &SlasherDB<E>) -> lmdb::Database;
fn select_db<'txn, E: EthSpec>(
db: &SlasherDB<E>,
txn: &'txn RwTransaction<'txn>,
) -> Result<mdbx::Database<'txn>, Error>;
fn load<E: EthSpec>(
db: &SlasherDB<E>,
@@ -156,13 +160,13 @@ pub trait TargetArrayChunk: Sized + serde::Serialize + serde::de::DeserializeOwn
config: &Config,
) -> Result<Option<Self>, Error> {
let disk_key = config.disk_key(validator_chunk_index, chunk_index);
let chunk_bytes = match txn.get(Self::select_db(db), &disk_key.to_be_bytes()) {
Ok(chunk_bytes) => chunk_bytes,
Err(lmdb::Error::NotFound) => return Ok(None),
Err(e) => return Err(e.into()),
};
let chunk_bytes: Cow<[u8]> =
match txn.get(&Self::select_db(db, txn)?, &disk_key.to_be_bytes())? {
Some(chunk_bytes) => chunk_bytes,
None => return Ok(None),
};
let chunk = bincode::deserialize_from(ZlibDecoder::new(chunk_bytes))?;
let chunk = bincode::deserialize_from(ZlibDecoder::new(chunk_bytes.borrow()))?;
Ok(Some(chunk))
}
@@ -185,7 +189,7 @@ pub trait TargetArrayChunk: Sized + serde::Serialize + serde::de::DeserializeOwn
metrics::set_float_gauge(&SLASHER_COMPRESSION_RATIO, compression_ratio);
txn.put(
Self::select_db(db),
&Self::select_db(db, txn)?,
&disk_key.to_be_bytes(),
&compressed_value,
SlasherDB::<E>::write_flags(),
@@ -292,8 +296,11 @@ impl TargetArrayChunk for MinTargetChunk {
start_epoch / chunk_size * chunk_size - 1
}
fn select_db<E: EthSpec>(db: &SlasherDB<E>) -> lmdb::Database {
db.min_targets_db
fn select_db<'txn, E: EthSpec>(
db: &SlasherDB<E>,
txn: &'txn RwTransaction<'txn>,
) -> Result<mdbx::Database<'txn>, Error> {
db.min_targets_db(txn)
}
}
@@ -391,8 +398,11 @@ impl TargetArrayChunk for MaxTargetChunk {
(start_epoch / chunk_size + 1) * chunk_size
}
fn select_db<E: EthSpec>(db: &SlasherDB<E>) -> lmdb::Database {
db.max_targets_db
fn select_db<'txn, E: EthSpec>(
db: &SlasherDB<E>,
txn: &'txn RwTransaction<'txn>,
) -> Result<mdbx::Database<'txn>, Error> {
db.max_targets_db(txn)
}
}