mirror of
https://github.com/sigp/lighthouse.git
synced 2026-03-10 04:01:51 +00:00
## Issue Addressed Closes #2052 ## Proposed Changes - Refactor the attester/proposer duties endpoints in the BN - Performance improvements - Fixes some potential inconsistencies with the dependent root fields. - Removes `http_api::beacon_proposer_cache` and just uses the one on the `BeaconChain` instead. - Move the code for the proposer/attester duties endpoints into separate files, for readability. - Refactor the `DutiesService` in the VC - Required to reduce the delay on broadcasting new blocks. - Gets rid of the `ValidatorDuty` shim struct that came about when we adopted the standard API. - Separate block/attestation duty tasks so that they don't block each other when one is slow. - In the VC, use `PublicKeyBytes` to represent validators instead of `PublicKey`. `PublicKey` is a legit crypto object whilst `PublicKeyBytes` is just a byte-array, it's much faster to clone/hash `PublicKeyBytes` and this change has had a significant impact on runtimes. - Unfortunately this has created lots of dust changes. - In the BN, store `PublicKeyBytes` in the `beacon_proposer_cache` and allow access to them. The HTTP API always sends `PublicKeyBytes` over the wire and the conversion from `PublicKey` -> `PublickeyBytes` is non-trivial, especially when queries have 100s/1000s of validators (like Pyrmont). - Add the `state_processing::state_advance` mod which dedups a lot of the "apply `n` skip slots to the state" code. - This also fixes a bug with some functions which were failing to include a state root as per [this comment](072695284f/consensus/state_processing/src/state_advance.rs (L69-L74)). I couldn't find any instance of this bug that resulted in anything more severe than keying a shuffling cache by the wrong block root. - Swap the VC block service to use `mpsc` from `tokio` instead of `futures`. This is consistent with the rest of the code base. ~~This PR *reduces* the size of the codebase 🎉~~ It *used* to reduce the size of the code base before I added more comments. ## Observations on Prymont - Proposer duties times down from peaks of 450ms to consistent <1ms. - Current epoch attester duties times down from >1s peaks to a consistent 20-30ms. - Block production down from +600ms to 100-200ms. ## Additional Info - ~~Blocked on #2241~~ - ~~Blocked on #2234~~ ## TODO - [x] ~~Refactor this into some smaller PRs?~~ Leaving this as-is for now. - [x] Address `per_slot_processing` roots. - [x] Investigate slow next epoch times. Not getting added to cache on block processing? - [x] Consider [this](072695284f/beacon_node/store/src/hot_cold_store.rs (L811-L812)) in the scenario of replacing the state roots Co-authored-by: pawan <pawandhananjay@gmail.com> Co-authored-by: Michael Sproul <michael@sigmaprime.io>
178 lines
6.4 KiB
Rust
178 lines
6.4 KiB
Rust
use serde_derive::{Deserialize, Serialize};
|
|
use std::collections::HashMap;
|
|
use std::fs::File;
|
|
use std::io::{prelude::*, BufReader};
|
|
use std::path::PathBuf;
|
|
use std::str::FromStr;
|
|
|
|
use bls::blst_implementations::PublicKeyBytes;
|
|
use types::{graffiti::GraffitiString, Graffiti};
|
|
|
|
#[derive(Debug)]
|
|
pub enum Error {
|
|
InvalidFile(std::io::Error),
|
|
InvalidLine(String),
|
|
InvalidPublicKey(String),
|
|
InvalidGraffiti(String),
|
|
}
|
|
|
|
/// Struct to load validator graffitis from file.
|
|
/// The graffiti file is expected to have the following structure
|
|
///
|
|
/// default: Lighthouse
|
|
/// public_key1: graffiti1
|
|
/// public_key2: graffiti2
|
|
/// ...
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct GraffitiFile {
|
|
graffiti_path: PathBuf,
|
|
graffitis: HashMap<PublicKeyBytes, Graffiti>,
|
|
default: Option<Graffiti>,
|
|
}
|
|
|
|
impl GraffitiFile {
|
|
pub fn new(graffiti_path: PathBuf) -> Self {
|
|
Self {
|
|
graffiti_path,
|
|
graffitis: HashMap::new(),
|
|
default: None,
|
|
}
|
|
}
|
|
|
|
/// Loads the graffiti file and populates the default graffiti and `graffitis` hashmap.
|
|
/// Returns the graffiti corresponding to the given public key if present, else returns the
|
|
/// default graffiti.
|
|
///
|
|
/// Returns an error if loading from the graffiti file fails.
|
|
pub fn load_graffiti(
|
|
&mut self,
|
|
public_key: &PublicKeyBytes,
|
|
) -> Result<Option<Graffiti>, Error> {
|
|
self.read_graffiti_file()?;
|
|
Ok(self.graffitis.get(public_key).copied().or(self.default))
|
|
}
|
|
|
|
/// Reads from a graffiti file with the specified format and populates the default value
|
|
/// and the hashmap.
|
|
///
|
|
/// Returns an error if the file does not exist, or if the format is invalid.
|
|
pub fn read_graffiti_file(&mut self) -> Result<(), Error> {
|
|
let file = File::open(self.graffiti_path.as_path()).map_err(Error::InvalidFile)?;
|
|
let reader = BufReader::new(file);
|
|
|
|
let lines = reader.lines();
|
|
|
|
for line in lines {
|
|
let line = line.map_err(|e| Error::InvalidLine(e.to_string()))?;
|
|
let (pk_opt, graffiti) = read_line(&line)?;
|
|
match pk_opt {
|
|
Some(pk) => {
|
|
self.graffitis.insert(pk, graffiti);
|
|
}
|
|
None => self.default = Some(graffiti),
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// Parses a line from the graffiti file.
|
|
///
|
|
/// `Ok((None, graffiti))` represents the graffiti for the default key.
|
|
/// `Ok((Some(pk), graffiti))` represents graffiti for the public key `pk`.
|
|
/// Returns an error if the line is in the wrong format or does not contain a valid public key or graffiti.
|
|
fn read_line(line: &str) -> Result<(Option<PublicKeyBytes>, Graffiti), Error> {
|
|
if let Some(i) = line.find(':') {
|
|
let (key, value) = line.split_at(i);
|
|
// Note: `value.len() >=1` so `value[1..]` is safe
|
|
let graffiti = GraffitiString::from_str(value[1..].trim())
|
|
.map_err(Error::InvalidGraffiti)?
|
|
.into();
|
|
if key == "default" {
|
|
Ok((None, graffiti))
|
|
} else {
|
|
let pk = PublicKeyBytes::from_str(&key).map_err(Error::InvalidPublicKey)?;
|
|
Ok((Some(pk), graffiti))
|
|
}
|
|
} else {
|
|
Err(Error::InvalidLine(format!("Missing delimiter: {}", line)))
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use bls::Keypair;
|
|
use std::io::LineWriter;
|
|
use tempfile::TempDir;
|
|
|
|
const DEFAULT_GRAFFITI: &str = "lighthouse";
|
|
const CUSTOM_GRAFFITI1: &str = "custom-graffiti1";
|
|
const CUSTOM_GRAFFITI2: &str = "graffitiwall:720:641:#ffff00";
|
|
const EMPTY_GRAFFITI: &str = "";
|
|
const PK1: &str = "0x800012708dc03f611751aad7a43a082142832b5c1aceed07ff9b543cf836381861352aa923c70eeb02018b638aa306aa";
|
|
const PK2: &str = "0x80001866ce324de7d80ec73be15e2d064dcf121adf1b34a0d679f2b9ecbab40ce021e03bb877e1a2fe72eaaf475e6e21";
|
|
const PK3: &str = "0x9035d41a8bc11b08c17d0d93d876087958c9d055afe86fce558e3b988d92434769c8d50b0b463708db80c6aae1160c02";
|
|
|
|
// Create a graffiti file in the required format and return a path to the file.
|
|
fn create_graffiti_file() -> PathBuf {
|
|
let temp = TempDir::new().unwrap();
|
|
let pk1 = PublicKeyBytes::deserialize(&hex::decode(&PK1[2..]).unwrap()).unwrap();
|
|
let pk2 = PublicKeyBytes::deserialize(&hex::decode(&PK2[2..]).unwrap()).unwrap();
|
|
let pk3 = PublicKeyBytes::deserialize(&hex::decode(&PK3[2..]).unwrap()).unwrap();
|
|
|
|
let file_name = temp.into_path().join("graffiti.txt");
|
|
|
|
let file = File::create(&file_name).unwrap();
|
|
let mut graffiti_file = LineWriter::new(file);
|
|
graffiti_file
|
|
.write_all(format!("default: {}\n", DEFAULT_GRAFFITI).as_bytes())
|
|
.unwrap();
|
|
graffiti_file
|
|
.write_all(format!("{}: {}\n", pk1.to_hex_string(), CUSTOM_GRAFFITI1).as_bytes())
|
|
.unwrap();
|
|
graffiti_file
|
|
.write_all(format!("{}: {}\n", pk2.to_hex_string(), CUSTOM_GRAFFITI2).as_bytes())
|
|
.unwrap();
|
|
graffiti_file
|
|
.write_all(format!("{}:{}\n", pk3.to_hex_string(), EMPTY_GRAFFITI).as_bytes())
|
|
.unwrap();
|
|
graffiti_file.flush().unwrap();
|
|
file_name
|
|
}
|
|
|
|
#[test]
|
|
fn test_load_graffiti() {
|
|
let graffiti_file_path = create_graffiti_file();
|
|
let mut gf = GraffitiFile::new(graffiti_file_path);
|
|
|
|
let pk1 = PublicKeyBytes::deserialize(&hex::decode(&PK1[2..]).unwrap()).unwrap();
|
|
let pk2 = PublicKeyBytes::deserialize(&hex::decode(&PK2[2..]).unwrap()).unwrap();
|
|
let pk3 = PublicKeyBytes::deserialize(&hex::decode(&PK3[2..]).unwrap()).unwrap();
|
|
|
|
// Read once
|
|
gf.read_graffiti_file().unwrap();
|
|
|
|
assert_eq!(
|
|
gf.load_graffiti(&pk1).unwrap().unwrap(),
|
|
GraffitiString::from_str(CUSTOM_GRAFFITI1).unwrap().into()
|
|
);
|
|
assert_eq!(
|
|
gf.load_graffiti(&pk2).unwrap().unwrap(),
|
|
GraffitiString::from_str(CUSTOM_GRAFFITI2).unwrap().into()
|
|
);
|
|
|
|
assert_eq!(
|
|
gf.load_graffiti(&pk3).unwrap().unwrap(),
|
|
GraffitiString::from_str(EMPTY_GRAFFITI).unwrap().into()
|
|
);
|
|
|
|
// Random pk should return the default graffiti
|
|
let random_pk = Keypair::random().pk.compress();
|
|
assert_eq!(
|
|
gf.load_graffiti(&random_pk).unwrap().unwrap(),
|
|
GraffitiString::from_str(DEFAULT_GRAFFITI).unwrap().into()
|
|
);
|
|
}
|
|
}
|