diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 416179445f..86f99b53e1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -258,8 +258,8 @@ jobs: | | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-unknown-linux-gnu-portable.tar.gz.asc) | | | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) | | | aarch64 | [lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-aarch64-unknown-linux-gnu-portable.tar.gz.asc) | - | | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz.asc) | - | | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz.asc) | + | | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows.tar.gz.asc) | + | | x86_64 | [lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/lighthouse-${{ env.VERSION }}-x86_64-windows-portable.tar.gz.asc) | | | | | | | **System** | **Option** | - | **Resource** | | | Docker | [${{ env.VERSION }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}/tags?page=1&ordering=last_updated&name=${{ env.VERSION }}) | [${{ env.IMAGE_NAME }}](https://hub.docker.com/r/${{ env.IMAGE_NAME }}) | diff --git a/Cargo.lock b/Cargo.lock index cc6c1a9fd6..431a272be8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1340,6 +1340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -1355,6 +1356,18 @@ dependencies = [ "terminal_size", ] +[[package]] +name = "clap_derive" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "clap_lex" version = "0.7.0" @@ -1824,6 +1837,7 @@ dependencies = [ "clap_utils", "environment", "hex", + "serde", "slog", "store", "strum", @@ -2069,7 +2083,7 @@ dependencies = [ "enr", "fnv", "futures", - "hashlink", + "hashlink 0.8.4", "hex", "hkdf", "lazy_static", @@ -3358,6 +3372,7 @@ dependencies = [ "futures-ticker", "futures-timer", "getrandom", + "hashlink 0.9.0", "hex_fmt", "libp2p", "prometheus-client", @@ -3473,6 +3488,15 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hashlink" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "headers" version = "0.3.9" @@ -6957,7 +6981,7 @@ dependencies = [ "bitflags 1.3.2", "fallible-iterator", "fallible-streaming-iterator", - "hashlink", + "hashlink 0.8.4", "libsqlite3-sys", "smallvec", ] @@ -9774,7 +9798,7 @@ checksum = "498f4d102a79ea1c9d4dd27573c0fc96ad74c023e8da38484e47883076da25fb" dependencies = [ "arraydeque", "encoding_rs", - "hashlink", + "hashlink 0.8.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eedc47470e..fad5fbead1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,7 @@ bincode = "1" bitvec = "1" byteorder = "1" bytes = "1" -clap = { version = "4.5.4", features = ["cargo", "wrap_help"] } +clap = { version = "4.5.4", features = ["derive", "cargo", "wrap_help"] } # Turn off c-kzg's default features which include `blst/portable`. We can turn on blst's portable # feature ourselves when desired. c-kzg = { version = "1", default-features = false } @@ -126,6 +126,7 @@ fnv = "1" fs2 = "0.4" futures = "0.3" hex = "0.4" +hashlink = "0.9.0" hyper = "1" itertools = "0.10" lazy_static = "1" @@ -153,7 +154,7 @@ serde_json = "1" serde_repr = "0.1" serde_yaml = "0.9" sha2 = "0.9" -slog = { version = "2", features = ["max_level_trace", "release_max_level_trace", "nested-values"] } +slog = { version = "2", features = ["max_level_debug", "release_max_level_debug", "nested-values"] } slog-async = "2" slog-term = "2" sloggers = { version = "2", features = ["json"] } diff --git a/Dockerfile b/Dockerfile index e0c48699bf..ff7f14d534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,8 +4,8 @@ COPY . lighthouse ARG FEATURES ARG PROFILE=release ARG CARGO_USE_GIT_CLI=true -ENV FEATURES $FEATURES -ENV PROFILE $PROFILE +ENV FEATURES=$FEATURES +ENV PROFILE=$PROFILE ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_USE_GIT_CLI RUN cd lighthouse && make diff --git a/Makefile b/Makefile index 7d144e55fb..d18a673880 100644 --- a/Makefile +++ b/Makefile @@ -174,8 +174,9 @@ test-network-%: # Run the tests in the `slasher` crate for all supported database backends. test-slasher: cargo nextest run --release -p slasher --features "lmdb,$(TEST_FEATURES)" + cargo nextest run --release -p slasher --no-default-features --features "redb,$(TEST_FEATURES)" cargo nextest run --release -p slasher --no-default-features --features "mdbx,$(TEST_FEATURES)" - cargo nextest run --release -p slasher --features "lmdb,mdbx,$(TEST_FEATURES)" # both backends enabled + cargo nextest run --release -p slasher --features "lmdb,mdbx,redb,$(TEST_FEATURES)" # all backends enabled # Runs only the tests/state_transition_vectors tests. run-state-transition-tests: diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 7f09430227..19ee3d116c 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -723,13 +723,6 @@ impl BeaconChain { Ok(()) } - pub fn persist_data_availability_checker(&self) -> Result<(), Error> { - let _timer = metrics::start_timer(&metrics::PERSIST_DATA_AVAILABILITY_CHECKER); - self.data_availability_checker.persist_all()?; - - Ok(()) - } - /// Returns the slot _right now_ according to `self.slot_clock`. Returns `Err` if the slot is /// unavailable. /// @@ -3095,14 +3088,21 @@ impl BeaconChain { notify_execution_layer, )?; publish_fn()?; + + // Record the time it took to complete consensus verification. + if let Some(timestamp) = self.slot_clock.now_duration() { + self.block_times_cache + .write() + .set_time_consensus_verified(block_root, block_slot, timestamp) + } + let executed_block = chain.into_executed_block(execution_pending).await?; - // Record the time it took to ask the execution layer. - if let Some(seen_timestamp) = self.slot_clock.now_duration() { - self.block_times_cache.write().set_execution_time( - block_root, - block_slot, - seen_timestamp, - ) + + // Record the *additional* time it took to wait for execution layer verification. + if let Some(timestamp) = self.slot_clock.now_duration() { + self.block_times_cache + .write() + .set_time_executed(block_root, block_slot, timestamp) } match executed_block { @@ -6753,7 +6753,6 @@ impl Drop for BeaconChain { let drop = || -> Result<(), Error> { self.persist_head_and_fork_choice()?; self.persist_op_pool()?; - self.persist_data_availability_checker()?; self.persist_eth1_cache() }; diff --git a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs index f7f389c543..f746b68996 100644 --- a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs +++ b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs @@ -20,26 +20,12 @@ use types::{ Hash256, Slot, }; -/// Ensure this justified checkpoint has an epoch of 0 so that it is never -/// greater than the justified checkpoint and enshrined as the actual justified -/// checkpoint. -const JUNK_BEST_JUSTIFIED_CHECKPOINT: Checkpoint = Checkpoint { - epoch: Epoch::new(0), - root: Hash256::repeat_byte(0), -}; - #[derive(Debug)] pub enum Error { - UnableToReadSlot, - UnableToReadTime, - InvalidGenesisSnapshot(Slot), - AncestorUnknown { ancestor_slot: Slot }, - UninitializedBestJustifiedBalances, FailedToReadBlock(StoreError), MissingBlock(Hash256), FailedToReadState(StoreError), MissingState(Hash256), - InvalidPersistedBytes(ssz::DecodeError), BeaconStateError(BeaconStateError), Arith(ArithError), } @@ -66,7 +52,6 @@ const MAX_BALANCE_CACHE_SIZE: usize = 4; )] pub(crate) struct CacheItem { pub(crate) block_root: Hash256, - #[superstruct(only(V8))] pub(crate) epoch: Epoch, pub(crate) balances: Vec, } @@ -79,7 +64,6 @@ pub(crate) type CacheItem = CacheItemV8; no_enum )] pub struct BalancesCache { - #[superstruct(only(V8))] pub(crate) items: Vec, } @@ -365,59 +349,15 @@ where pub type PersistedForkChoiceStore = PersistedForkChoiceStoreV17; /// A container which allows persisting the `BeaconForkChoiceStore` to the on-disk database. -#[superstruct( - variants(V11, V17), - variant_attributes(derive(Encode, Decode)), - no_enum -)] +#[superstruct(variants(V17), variant_attributes(derive(Encode, Decode)), no_enum)] pub struct PersistedForkChoiceStore { - #[superstruct(only(V11, V17))] pub balances_cache: BalancesCacheV8, pub time: Slot, pub finalized_checkpoint: Checkpoint, pub justified_checkpoint: Checkpoint, pub justified_balances: Vec, - #[superstruct(only(V11))] - pub best_justified_checkpoint: Checkpoint, - #[superstruct(only(V11, V17))] pub unrealized_justified_checkpoint: Checkpoint, - #[superstruct(only(V11, V17))] pub unrealized_finalized_checkpoint: Checkpoint, - #[superstruct(only(V11, V17))] pub proposer_boost_root: Hash256, - #[superstruct(only(V11, V17))] pub equivocating_indices: BTreeSet, } - -impl From for PersistedForkChoiceStore { - fn from(from: PersistedForkChoiceStoreV11) -> PersistedForkChoiceStore { - PersistedForkChoiceStore { - balances_cache: from.balances_cache, - time: from.time, - finalized_checkpoint: from.finalized_checkpoint, - justified_checkpoint: from.justified_checkpoint, - justified_balances: from.justified_balances, - unrealized_justified_checkpoint: from.unrealized_justified_checkpoint, - unrealized_finalized_checkpoint: from.unrealized_finalized_checkpoint, - proposer_boost_root: from.proposer_boost_root, - equivocating_indices: from.equivocating_indices, - } - } -} - -impl From for PersistedForkChoiceStoreV11 { - fn from(from: PersistedForkChoiceStore) -> PersistedForkChoiceStoreV11 { - PersistedForkChoiceStoreV11 { - balances_cache: from.balances_cache, - time: from.time, - finalized_checkpoint: from.finalized_checkpoint, - justified_checkpoint: from.justified_checkpoint, - justified_balances: from.justified_balances, - best_justified_checkpoint: JUNK_BEST_JUSTIFIED_CHECKPOINT, - unrealized_justified_checkpoint: from.unrealized_justified_checkpoint, - unrealized_finalized_checkpoint: from.unrealized_finalized_checkpoint, - proposer_boost_root: from.proposer_boost_root, - equivocating_indices: from.equivocating_indices, - } - } -} diff --git a/beacon_node/beacon_chain/src/block_times_cache.rs b/beacon_node/beacon_chain/src/block_times_cache.rs index db547a1186..3b75046f3a 100644 --- a/beacon_node/beacon_chain/src/block_times_cache.rs +++ b/beacon_node/beacon_chain/src/block_times_cache.rs @@ -19,7 +19,9 @@ type BlockRoot = Hash256; pub struct Timestamps { pub observed: Option, pub all_blobs_observed: Option, - pub execution_time: Option, + pub consensus_verified: Option, + pub started_execution: Option, + pub executed: Option, pub attestable: Option, pub imported: Option, pub set_as_head: Option, @@ -32,7 +34,9 @@ pub struct BlockDelays { pub observed: Option, /// The time after the start of the slot we saw all blobs. pub all_blobs_observed: Option, - /// The time it took to get verification from the EL for the block. + /// The time it took to complete consensus verification of the block. + pub consensus_verification_time: Option, + /// The time it took to complete execution verification of the block. pub execution_time: Option, /// The delay from the start of the slot before the block became available /// @@ -58,13 +62,16 @@ impl BlockDelays { let all_blobs_observed = times .all_blobs_observed .and_then(|all_blobs_observed| all_blobs_observed.checked_sub(slot_start_time)); + let consensus_verification_time = times + .consensus_verified + .and_then(|consensus_verified| consensus_verified.checked_sub(times.observed?)); let execution_time = times - .execution_time - .and_then(|execution_time| execution_time.checked_sub(times.observed?)); + .executed + .and_then(|executed| executed.checked_sub(times.started_execution?)); // Duration since UNIX epoch at which block became available. - let available_time = times.execution_time.map(|execution_time| { - std::cmp::max(execution_time, times.all_blobs_observed.unwrap_or_default()) - }); + let available_time = times + .executed + .map(|executed| std::cmp::max(executed, times.all_blobs_observed.unwrap_or_default())); // Duration from the start of the slot until the block became available. let available_delay = available_time.and_then(|available_time| available_time.checked_sub(slot_start_time)); @@ -80,6 +87,7 @@ impl BlockDelays { BlockDelays { observed, all_blobs_observed, + consensus_verification_time, execution_time, available: available_delay, attestable, @@ -155,6 +163,9 @@ impl BlockTimesCache { slot: Slot, timestamp: Duration, ) { + // Unlike other functions in this file, we update the blob observed time only if it is + // *greater* than existing blob observation times. This allows us to know the observation + // time of the last blob to arrive. let block_times = self .cache .entry(block_root) @@ -168,48 +179,89 @@ impl BlockTimesCache { } } - pub fn set_execution_time(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { + /// Set the timestamp for `field` if that timestamp is less than any previously known value. + /// + /// If no previous value is known for the field, then the supplied timestamp will always be + /// stored. + pub fn set_time_if_less( + &mut self, + block_root: BlockRoot, + slot: Slot, + field: impl Fn(&mut Timestamps) -> &mut Option, + timestamp: Duration, + ) { let block_times = self .cache .entry(block_root) .or_insert_with(|| BlockTimesCacheValue::new(slot)); - if block_times - .timestamps - .execution_time - .map_or(true, |prev| timestamp < prev) - { - block_times.timestamps.execution_time = Some(timestamp); + let existing_timestamp = field(&mut block_times.timestamps); + if existing_timestamp.map_or(true, |prev| timestamp < prev) { + *existing_timestamp = Some(timestamp); } } + pub fn set_time_consensus_verified( + &mut self, + block_root: BlockRoot, + slot: Slot, + timestamp: Duration, + ) { + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.consensus_verified, + timestamp, + ) + } + + pub fn set_time_executed(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.executed, + timestamp, + ) + } + + pub fn set_time_started_execution( + &mut self, + block_root: BlockRoot, + slot: Slot, + timestamp: Duration, + ) { + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.started_execution, + timestamp, + ) + } + pub fn set_time_attestable(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { - let block_times = self - .cache - .entry(block_root) - .or_insert_with(|| BlockTimesCacheValue::new(slot)); - if block_times - .timestamps - .attestable - .map_or(true, |prev| timestamp < prev) - { - block_times.timestamps.attestable = Some(timestamp); - } + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.attestable, + timestamp, + ) } pub fn set_time_imported(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { - let block_times = self - .cache - .entry(block_root) - .or_insert_with(|| BlockTimesCacheValue::new(slot)); - block_times.timestamps.imported = Some(timestamp); + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.imported, + timestamp, + ) } pub fn set_time_set_as_head(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { - let block_times = self - .cache - .entry(block_root) - .or_insert_with(|| BlockTimesCacheValue::new(slot)); - block_times.timestamps.set_as_head = Some(timestamp); + self.set_time_if_less( + block_root, + slot, + |timestamps| &mut timestamps.set_as_head, + timestamp, + ) } pub fn get_block_delays( diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index b24921e317..d906518ff5 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -67,7 +67,7 @@ use crate::{ metrics, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; -use eth2::types::{EventKind, PublishBlockRequest}; +use eth2::types::{BlockGossip, EventKind, PublishBlockRequest}; use execution_layer::PayloadStatus; pub use fork_choice::{AttestationFromBlock, PayloadVerificationStatus}; use parking_lot::RwLockReadGuard; @@ -93,7 +93,6 @@ use std::io::Write; use std::sync::Arc; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; -use tree_hash::TreeHash; use types::{ BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, SignedBeaconBlock, @@ -975,6 +974,16 @@ impl GossipVerifiedBlock { // Validate the block's execution_payload (if any). validate_execution_payload_for_gossip(&parent_block, block.message(), chain)?; + // Beacon API block_gossip events + if let Some(event_handler) = chain.event_handler.as_ref() { + if event_handler.has_block_gossip_subscribers() { + event_handler.register(EventKind::BlockGossip(Box::new(BlockGossip { + slot: block.slot(), + block: block_root, + }))); + } + } + // Having checked the proposer index and the block root we can cache them. let consensus_context = ConsensusContext::new(block.slot()) .set_current_block_root(block_root) @@ -1335,6 +1344,13 @@ impl ExecutionPendingBlock { // The specification declares that this should be run *inside* `per_block_processing`, // however we run it here to keep `per_block_processing` pure (i.e., no calls to external // servers). + if let Some(started_execution) = chain.slot_clock.now_duration() { + chain.block_times_cache.write().set_time_started_execution( + block_root, + block.slot(), + started_execution, + ); + } let payload_verification_status = payload_notifier.notify_new_payload().await?; // If the payload did not validate or invalidate the block, check to see if this block is @@ -2107,7 +2123,14 @@ pub fn verify_header_signature( fn write_state(prefix: &str, state: &BeaconState, log: &Logger) { if WRITE_BLOCK_PROCESSING_SSZ { - let root = state.tree_hash_root(); + let mut state = state.clone(); + let Ok(root) = state.canonical_root() else { + error!( + log, + "Unable to hash state for writing"; + ); + return; + }; let filename = format!("{}_slot_{}_root_{}.ssz", prefix, state.slot(), root); let mut path = std::env::temp_dir().join("lighthouse"); let _ = fs::create_dir_all(path.clone()); diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 90461b8f03..7217f2c640 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1195,7 +1195,7 @@ mod test { let head = chain.head_snapshot(); - let state = &head.beacon_state; + let mut state = head.beacon_state.clone(); let block = &head.beacon_block; assert_eq!(state.slot(), Slot::new(0), "should start from genesis"); @@ -1206,7 +1206,7 @@ mod test { ); assert_eq!( block.state_root(), - state.canonical_root(), + state.canonical_root().unwrap(), "block should have correct state root" ); assert_eq!( diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index a84cfab298..84e1544451 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1385,6 +1385,15 @@ fn observe_head_block_delays( .as_millis() as i64, ); + // The time it took to check the validity within Lighthouse + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_CONSENSUS_VERIFICATION_TIME, + block_delays + .consensus_verification_time + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + // The time it took to check the validity with the EL metrics::set_gauge( &metrics::BEACON_BLOCK_DELAY_EXECUTION_TIME, @@ -1447,6 +1456,7 @@ fn observe_head_block_delays( "total_delay_ms" => block_delay_total.as_millis(), "observed_delay_ms" => format_delay(&block_delays.observed), "blob_delay_ms" => format_delay(&block_delays.all_blobs_observed), + "consensus_time_ms" => format_delay(&block_delays.consensus_verification_time), "execution_time_ms" => format_delay(&block_delays.execution_time), "available_delay_ms" => format_delay(&block_delays.available), "attestable_delay_ms" => format_delay(&block_delays.attestable), @@ -1463,6 +1473,7 @@ fn observe_head_block_delays( "total_delay_ms" => block_delay_total.as_millis(), "observed_delay_ms" => format_delay(&block_delays.observed), "blob_delay_ms" => format_delay(&block_delays.all_blobs_observed), + "consensus_time_ms" => format_delay(&block_delays.consensus_verification_time), "execution_time_ms" => format_delay(&block_delays.execution_time), "available_delay_ms" => format_delay(&block_delays.available), "attestable_delay_ms" => format_delay(&block_delays.attestable), diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index e0347d81c3..2431769ddb 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -2,7 +2,7 @@ use crate::blob_verification::{verify_kzg_for_blob_list, GossipVerifiedBlob, Kzg use crate::block_verification_types::{ AvailabilityPendingExecutedBlock, AvailableExecutedBlock, RpcBlock, }; -use crate::data_availability_checker::overflow_lru_cache::OverflowLRUCache; +use crate::data_availability_checker::overflow_lru_cache::DataAvailabilityCheckerInner; use crate::{BeaconChain, BeaconChainTypes, BeaconStore}; use kzg::Kzg; use slog::{debug, error, Logger}; @@ -33,12 +33,32 @@ pub const OVERFLOW_LRU_CAPACITY: NonZeroUsize = new_non_zero_usize(1024); pub const STATE_LRU_CAPACITY_NON_ZERO: NonZeroUsize = new_non_zero_usize(2); pub const STATE_LRU_CAPACITY: usize = STATE_LRU_CAPACITY_NON_ZERO.get(); -/// This includes a cache for any blocks or blobs that have been received over gossip or RPC -/// and are awaiting more components before they can be imported. Additionally the -/// `DataAvailabilityChecker` is responsible for KZG verification of block components as well as -/// checking whether a "availability check" is required at all. +/// Cache to hold fully valid data that can't be imported to fork-choice yet. After Dencun hard-fork +/// blocks have a sidecar of data that is received separately from the network. We call the concept +/// of a block "becoming available" when all of its import dependencies are inserted into this +/// cache. +/// +/// Usually a block becomes available on its slot within a second of receiving its first component +/// over gossip. However, a block may never become available if a malicious proposer does not +/// publish its data, or there are network issues that prevent us from receiving it. If the block +/// does not become available after some time we can safely forget about it. Consider these two +/// cases: +/// +/// - Global unavailability: If nobody has received the block components it's likely that the +/// proposer never made the block available. So we can safely forget about the block as it will +/// never become available. +/// - Local unavailability: Some fraction of the network has received all block components, but not us. +/// Some of our peers will eventually attest to a descendant of that block and lookup sync will +/// fetch its components. Therefore it's not strictly necessary to hold to the partially available +/// block for too long as we can recover from other peers. +/// +/// Even in periods of non-finality, the proposer is expected to publish the block's data +/// immediately. Because this cache only holds fully valid data, its capacity is bound to 1 block +/// per slot and fork: before inserting into this cache we check the proposer signature and correct +/// proposer. Having a capacity > 1 is an optimization to prevent sync lookup from having re-fetch +/// data during moments of unstable network conditions. pub struct DataAvailabilityChecker { - availability_cache: Arc>, + availability_cache: Arc>, slot_clock: T::SlotClock, kzg: Option>, log: Logger, @@ -74,7 +94,8 @@ impl DataAvailabilityChecker { log: &Logger, spec: ChainSpec, ) -> Result { - let overflow_cache = OverflowLRUCache::new(OVERFLOW_LRU_CAPACITY, store, spec.clone())?; + let overflow_cache = + DataAvailabilityCheckerInner::new(OVERFLOW_LRU_CAPACITY, store, spec.clone())?; Ok(Self { availability_cache: Arc::new(overflow_cache), slot_clock, @@ -329,15 +350,9 @@ impl DataAvailabilityChecker { }) } - /// Persist all in memory components to disk - pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> { - self.availability_cache.write_all_to_disk() - } - /// Collects metrics from the data availability checker. pub fn metrics(&self) -> DataAvailabilityCheckerMetrics { DataAvailabilityCheckerMetrics { - num_store_entries: self.availability_cache.num_store_entries(), state_cache_size: self.availability_cache.state_cache_size(), block_cache_size: self.availability_cache.block_cache_size(), } @@ -346,7 +361,6 @@ impl DataAvailabilityChecker { /// Helper struct to group data availability checker metrics. pub struct DataAvailabilityCheckerMetrics { - pub num_store_entries: usize, pub state_cache_size: usize, pub block_cache_size: usize, } @@ -372,7 +386,7 @@ pub fn start_availability_cache_maintenance_service( async fn availability_cache_maintenance_service( chain: Arc>, - overflow_cache: Arc>, + overflow_cache: Arc>, ) { let epoch_duration = chain.slot_clock.slot_duration() * T::EthSpec::slots_per_epoch() as u32; loop { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index adc1a1e202..5e0513c8d3 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1,32 +1,3 @@ -//! This module implements a LRU cache for storing partially available blocks and blobs. -//! When the cache overflows, the least recently used items are persisted to the database. -//! This prevents lighthouse from using too much memory storing unfinalized blocks and blobs -//! if the chain were to lose finality. -//! -//! ## Deadlock safety -//! -//! The main object in this module is the `OverflowLruCache`. It contains two locks: -//! -//! - `self.critical` is an `RwLock` that protects content stored in memory. -//! - `self.maintenance_lock` is held when moving data between memory and disk. -//! -//! You mostly need to ensure that you don't try to hold the critical lock more than once -//! -//! ## Basic Algorithm -//! -//! As blocks and blobs come in from the network, their components are stored in memory in -//! this cache. When a block becomes fully available, it is removed from the cache and -//! imported into fork-choice. Blocks/blobs that remain unavailable will linger in the -//! cache until they are older than the finalized epoch or older than the data availability -//! cutoff. In the event the chain is not finalizing, the cache will eventually overflow and -//! the least recently used items will be persisted to disk. When this happens, we will still -//! store the hash of the block in memory so we always know we have data for that block -//! without needing to check the database. -//! -//! When the client is shut down, all pending components are persisted in the database. -//! On startup, the keys of these components are stored in memory and will be loaded in -//! the cache when they are accessed. - use super::state_lru_cache::{DietAvailabilityPendingExecutedBlock, StateLRUCache}; use crate::beacon_chain::BeaconStore; use crate::blob_verification::KzgVerifiedBlob; @@ -34,15 +5,13 @@ use crate::block_verification_types::{ AvailabilityPendingExecutedBlock, AvailableBlock, AvailableExecutedBlock, }; use crate::data_availability_checker::{Availability, AvailabilityCheckError}; -use crate::store::{DBColumn, KeyValueStore}; use crate::BeaconChainTypes; use lru::LruCache; -use parking_lot::{Mutex, RwLock, RwLockUpgradableReadGuard}; -use ssz::{Decode, Encode}; +use parking_lot::RwLock; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; use std::num::NonZeroUsize; -use std::{collections::HashSet, sync::Arc}; +use std::sync::Arc; use types::blob_sidecar::BlobIdentifier; use types::{BlobSidecar, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock}; @@ -243,318 +212,27 @@ impl PendingComponents { AvailableExecutedBlock::new(available_block, import_data, payload_verification_outcome), ))) } - - /// Returns the epoch of the block if it is cached, otherwise returns the epoch of the first blob. - pub fn epoch(&self) -> Option { - self.executed_block - .as_ref() - .map(|pending_block| pending_block.as_block().epoch()) - .or_else(|| { - for maybe_blob in self.verified_blobs.iter() { - if maybe_blob.is_some() { - return maybe_blob.as_ref().map(|kzg_verified_blob| { - kzg_verified_blob - .as_blob() - .slot() - .epoch(E::slots_per_epoch()) - }); - } - } - None - }) - } -} - -/// Blocks and blobs are stored in the database sequentially so that it's -/// fast to iterate over all the data for a particular block. -#[derive(Debug, PartialEq)] -enum OverflowKey { - Block(Hash256), - Blob(Hash256, u8), -} - -impl OverflowKey { - pub fn from_block_root(block_root: Hash256) -> Self { - Self::Block(block_root) - } - - pub fn from_blob_id( - blob_id: BlobIdentifier, - ) -> Result { - if blob_id.index > E::max_blobs_per_block() as u64 || blob_id.index > u8::MAX as u64 { - return Err(AvailabilityCheckError::BlobIndexInvalid(blob_id.index)); - } - Ok(Self::Blob(blob_id.block_root, blob_id.index as u8)) - } - - pub fn root(&self) -> &Hash256 { - match self { - Self::Block(root) => root, - Self::Blob(root, _) => root, - } - } -} - -/// A wrapper around BeaconStore that implements various -/// methods used for saving and retrieving blocks / blobs -/// from the store (for organization) -struct OverflowStore(BeaconStore); - -impl OverflowStore { - /// Store pending components in the database - pub fn persist_pending_components( - &self, - block_root: Hash256, - mut pending_components: PendingComponents, - ) -> Result<(), AvailabilityCheckError> { - let col = DBColumn::OverflowLRUCache; - - if let Some(block) = pending_components.executed_block.take() { - let key = OverflowKey::from_block_root(block_root); - self.0 - .hot_db - .put_bytes(col.as_str(), &key.as_ssz_bytes(), &block.as_ssz_bytes())? - } - - for blob in Vec::from(pending_components.verified_blobs) - .into_iter() - .flatten() - { - let key = OverflowKey::from_blob_id::(BlobIdentifier { - block_root, - index: blob.blob_index(), - })?; - - self.0 - .hot_db - .put_bytes(col.as_str(), &key.as_ssz_bytes(), &blob.as_ssz_bytes())? - } - - Ok(()) - } - - /// Load the pending components that we have in the database for a given block root - pub fn load_pending_components( - &self, - block_root: Hash256, - ) -> Result>, AvailabilityCheckError> { - // read everything from disk and reconstruct - let mut maybe_pending_components = None; - for res in self - .0 - .hot_db - .iter_raw_entries(DBColumn::OverflowLRUCache, block_root.as_bytes()) - { - let (key_bytes, value_bytes) = res?; - match OverflowKey::from_ssz_bytes(&key_bytes)? { - OverflowKey::Block(_) => { - maybe_pending_components - .get_or_insert_with(|| PendingComponents::empty(block_root)) - .executed_block = - Some(DietAvailabilityPendingExecutedBlock::from_ssz_bytes( - value_bytes.as_slice(), - )?); - } - OverflowKey::Blob(_, index) => { - *maybe_pending_components - .get_or_insert_with(|| PendingComponents::empty(block_root)) - .verified_blobs - .get_mut(index as usize) - .ok_or(AvailabilityCheckError::BlobIndexInvalid(index as u64))? = - Some(KzgVerifiedBlob::from_ssz_bytes(value_bytes.as_slice())?); - } - } - } - - Ok(maybe_pending_components) - } - - /// Returns the hashes of all the blocks we have any data for on disk - pub fn read_keys_on_disk(&self) -> Result, AvailabilityCheckError> { - let mut disk_keys = HashSet::new(); - for res in self.0.hot_db.iter_raw_keys(DBColumn::OverflowLRUCache, &[]) { - let key_bytes = res?; - disk_keys.insert(*OverflowKey::from_ssz_bytes(&key_bytes)?.root()); - } - Ok(disk_keys) - } - - /// Load a single blob from the database - pub fn load_blob( - &self, - blob_id: &BlobIdentifier, - ) -> Result>>, AvailabilityCheckError> { - let key = OverflowKey::from_blob_id::(*blob_id)?; - - self.0 - .hot_db - .get_bytes(DBColumn::OverflowLRUCache.as_str(), &key.as_ssz_bytes())? - .map(|blob_bytes| Arc::>::from_ssz_bytes(blob_bytes.as_slice())) - .transpose() - .map_err(|e| e.into()) - } - - /// Delete a set of keys from the database - pub fn delete_keys(&self, keys: &Vec) -> Result<(), AvailabilityCheckError> { - for key in keys { - self.0 - .hot_db - .key_delete(DBColumn::OverflowLRUCache.as_str(), &key.as_ssz_bytes())?; - } - Ok(()) - } -} - -/// This data stores the *critical* data that we need to keep in memory -/// protected by the RWLock -struct Critical { - /// This is the LRU cache of pending components - pub in_memory: LruCache>, - /// This holds all the roots of the blocks for which we have - /// `PendingComponents` in the database. - pub store_keys: HashSet, -} - -impl Critical { - pub fn new(capacity: NonZeroUsize) -> Self { - Self { - in_memory: LruCache::new(capacity), - store_keys: HashSet::new(), - } - } - - pub fn reload_store_keys( - &mut self, - overflow_store: &OverflowStore, - ) -> Result<(), AvailabilityCheckError> { - let disk_keys = overflow_store.read_keys_on_disk()?; - self.store_keys = disk_keys; - Ok(()) - } - - /// This only checks for the blobs in memory - pub fn peek_blob( - &self, - blob_id: &BlobIdentifier, - ) -> Result>>, AvailabilityCheckError> { - if let Some(pending_components) = self.in_memory.peek(&blob_id.block_root) { - Ok(pending_components - .verified_blobs - .get(blob_id.index as usize) - .ok_or(AvailabilityCheckError::BlobIndexInvalid(blob_id.index))? - .as_ref() - .map(|blob| blob.clone_blob())) - } else { - Ok(None) - } - } - - pub fn peek_pending_components( - &self, - block_root: &Hash256, - ) -> Option<&PendingComponents> { - self.in_memory.peek(block_root) - } - - /// Puts the pending components in the LRU cache. If the cache - /// is at capacity, the LRU entry is written to the store first - pub fn put_pending_components( - &mut self, - block_root: Hash256, - pending_components: PendingComponents, - overflow_store: &OverflowStore, - ) -> Result<(), AvailabilityCheckError> { - if self.in_memory.len() == self.in_memory.cap().get() { - // cache will overflow, must write lru entry to disk - if let Some((lru_key, lru_value)) = self.in_memory.pop_lru() { - overflow_store.persist_pending_components(lru_key, lru_value)?; - self.store_keys.insert(lru_key); - } - } - self.in_memory.put(block_root, pending_components); - Ok(()) - } - - /// Removes and returns the pending_components corresponding to - /// the `block_root` or `None` if it does not exist - pub fn pop_pending_components( - &mut self, - block_root: Hash256, - store: &OverflowStore, - ) -> Result>, AvailabilityCheckError> { - match self.in_memory.pop_entry(&block_root) { - Some((_, pending_components)) => Ok(Some(pending_components)), - None => { - // not in memory, is it in the store? - if self.store_keys.remove(&block_root) { - // We don't need to remove the data from the store as we have removed it from - // `store_keys` so we won't go looking for it on disk. The maintenance thread - // will remove it from disk the next time it runs. - store.load_pending_components(block_root) - } else { - Ok(None) - } - } - } - } - - /// Removes and returns the pending_components corresponding to - /// the `block_root` or `None` if it does not exist - pub fn remove_pending_components(&mut self, block_root: Hash256) { - match self.in_memory.pop_entry(&block_root) { - Some { .. } => {} - None => { - // not in memory, is it in the store? - // We don't need to remove the data from the store as we have removed it from - // `store_keys` so we won't go looking for it on disk. The maintenance thread - // will remove it from disk the next time it runs. - self.store_keys.remove(&block_root); - } - } - } - - /// Returns the number of pending component entries in memory. - pub fn num_blocks(&self) -> usize { - self.in_memory.len() - } - - /// Returns the number of entries that have overflowed to disk. - pub fn num_store_entries(&self) -> usize { - self.store_keys.len() - } } /// This is the main struct for this module. Outside methods should /// interact with the cache through this. -pub struct OverflowLRUCache { +pub struct DataAvailabilityCheckerInner { /// Contains all the data we keep in memory, protected by an RwLock - critical: RwLock>, - /// This is how we read and write components to the disk - overflow_store: OverflowStore, + critical: RwLock>>, /// This cache holds a limited number of states in memory and reconstructs them /// from disk when necessary. This is necessary until we merge tree-states state_cache: StateLRUCache, - /// Mutex to guard maintenance methods which move data between disk and memory - maintenance_lock: Mutex<()>, - /// The capacity of the LRU cache - capacity: NonZeroUsize, } -impl OverflowLRUCache { +impl DataAvailabilityCheckerInner { pub fn new( capacity: NonZeroUsize, beacon_store: BeaconStore, spec: ChainSpec, ) -> Result { - let overflow_store = OverflowStore(beacon_store.clone()); - let mut critical = Critical::new(capacity); - critical.reload_store_keys(&overflow_store)?; Ok(Self { - critical: RwLock::new(critical), - overflow_store, + critical: RwLock::new(LruCache::new(capacity)), state_cache: StateLRUCache::new(beacon_store, spec), - maintenance_lock: Mutex::new(()), - capacity, }) } @@ -565,7 +243,7 @@ impl OverflowLRUCache { ) -> Option>> { self.critical .read() - .peek_pending_components(block_root) + .peek(block_root) .and_then(|pending_components| { pending_components .executed_block @@ -579,12 +257,13 @@ impl OverflowLRUCache { &self, blob_id: &BlobIdentifier, ) -> Result>>, AvailabilityCheckError> { - let read_lock = self.critical.read(); - if let Some(blob) = read_lock.peek_blob(blob_id)? { - Ok(Some(blob)) - } else if read_lock.store_keys.contains(&blob_id.block_root) { - drop(read_lock); - self.overflow_store.load_blob(blob_id) + if let Some(pending_components) = self.critical.read().peek(&blob_id.block_root) { + Ok(pending_components + .verified_blobs + .get(blob_id.index as usize) + .ok_or(AvailabilityCheckError::BlobIndexInvalid(blob_id.index))? + .as_ref() + .map(|blob| blob.clone_blob())) } else { Ok(None) } @@ -595,7 +274,7 @@ impl OverflowLRUCache { block_root: &Hash256, f: F, ) -> R { - f(self.critical.read().peek_pending_components(block_root)) + f(self.critical.read().peek(block_root)) } pub fn put_kzg_verified_blobs>>( @@ -615,29 +294,22 @@ impl OverflowLRUCache { // Grab existing entry or create a new entry. let mut pending_components = write_lock - .pop_pending_components(block_root, &self.overflow_store)? + .pop_entry(&block_root) + .map(|(_, v)| v) .unwrap_or_else(|| PendingComponents::empty(block_root)); // Merge in the blobs. pending_components.merge_blobs(fixed_blobs); if pending_components.is_available() { - write_lock.put_pending_components( - block_root, - pending_components.clone(), - &self.overflow_store, - )?; + write_lock.put(block_root, pending_components.clone()); // No need to hold the write lock anymore drop(write_lock); pending_components.make_available(|diet_block| { self.state_cache.recover_pending_executed_block(diet_block) }) } else { - write_lock.put_pending_components( - block_root, - pending_components, - &self.overflow_store, - )?; + write_lock.put(block_root, pending_components); Ok(Availability::MissingComponents(block_root)) } } @@ -658,7 +330,8 @@ impl OverflowLRUCache { // Grab existing entry or create a new entry. let mut pending_components = write_lock - .pop_pending_components(block_root, &self.overflow_store)? + .pop_entry(&block_root) + .map(|(_, v)| v) .unwrap_or_else(|| PendingComponents::empty(block_root)); // Merge in the block. @@ -666,203 +339,29 @@ impl OverflowLRUCache { // Check if we have all components and entire set is consistent. if pending_components.is_available() { - write_lock.put_pending_components( - block_root, - pending_components.clone(), - &self.overflow_store, - )?; + write_lock.put(block_root, pending_components.clone()); // No need to hold the write lock anymore drop(write_lock); pending_components.make_available(|diet_block| { self.state_cache.recover_pending_executed_block(diet_block) }) } else { - write_lock.put_pending_components( - block_root, - pending_components, - &self.overflow_store, - )?; + write_lock.put(block_root, pending_components); Ok(Availability::MissingComponents(block_root)) } } pub fn remove_pending_components(&self, block_root: Hash256) { - self.critical.write().remove_pending_components(block_root); - } - - /// write all in memory objects to disk - pub fn write_all_to_disk(&self) -> Result<(), AvailabilityCheckError> { - let maintenance_lock = self.maintenance_lock.lock(); - let mut critical_lock = self.critical.write(); - - let mut swap_lru = LruCache::new(self.capacity); - std::mem::swap(&mut swap_lru, &mut critical_lock.in_memory); - - for (root, pending_components) in swap_lru.into_iter() { - self.overflow_store - .persist_pending_components(root, pending_components)?; - critical_lock.store_keys.insert(root); - } - - drop(critical_lock); - drop(maintenance_lock); - Ok(()) + self.critical.write().pop_entry(&block_root); } /// maintain the cache pub fn do_maintenance(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> { - // ensure memory usage is below threshold - let threshold = self.capacity.get() * 3 / 4; - self.maintain_threshold(threshold, cutoff_epoch)?; - // clean up any keys on the disk that shouldn't be there - self.prune_disk(cutoff_epoch)?; // clean up any lingering states in the state cache self.state_cache.do_maintenance(cutoff_epoch); Ok(()) } - /// Enforce that the size of the cache is below a given threshold by - /// moving the least recently used items to disk. - fn maintain_threshold( - &self, - threshold: usize, - cutoff_epoch: Epoch, - ) -> Result<(), AvailabilityCheckError> { - // ensure only one thread at a time can be deleting things from the disk or - // moving things between memory and storage - let maintenance_lock = self.maintenance_lock.lock(); - - let mut stored = self.critical.read().in_memory.len(); - while stored > threshold { - let read_lock = self.critical.upgradable_read(); - let lru_entry = read_lock - .in_memory - .peek_lru() - .map(|(key, value)| (*key, value.clone())); - - let Some((lru_root, lru_pending_components)) = lru_entry else { - break; - }; - - if lru_pending_components - .epoch() - .map(|epoch| epoch < cutoff_epoch) - .unwrap_or(true) - { - // this data is no longer needed -> delete it - let mut write_lock = RwLockUpgradableReadGuard::upgrade(read_lock); - write_lock.in_memory.pop_entry(&lru_root); - stored = write_lock.in_memory.len(); - continue; - } else { - drop(read_lock); - } - - // write the lru entry to disk (we aren't holding any critical locks while we do this) - self.overflow_store - .persist_pending_components(lru_root, lru_pending_components)?; - // now that we've written to disk, grab the critical write lock - let mut write_lock = self.critical.write(); - if let Some((new_lru_root_ref, _)) = write_lock.in_memory.peek_lru() { - // need to ensure the entry we just wrote to disk wasn't updated - // while we were writing and is still the LRU entry - if *new_lru_root_ref == lru_root { - // it is still LRU entry -> delete it from memory & record that it's on disk - write_lock.in_memory.pop_entry(&lru_root); - write_lock.store_keys.insert(lru_root); - } - } - stored = write_lock.in_memory.len(); - drop(write_lock); - } - - drop(maintenance_lock); - Ok(()) - } - - /// Delete any data on disk that shouldn't be there. This can happen if - /// 1. The entry has been moved back to memory (or become fully available) - /// 2. The entry belongs to a block beyond the cutoff epoch - fn prune_disk(&self, cutoff_epoch: Epoch) -> Result<(), AvailabilityCheckError> { - // ensure only one thread at a time can be deleting things from the disk or - // moving things between memory and storage - let maintenance_lock = self.maintenance_lock.lock(); - - struct BlockData { - keys: Vec, - root: Hash256, - epoch: Epoch, - } - - let delete_if_outdated = |cache: &OverflowLRUCache, - block_data: Option| - -> Result<(), AvailabilityCheckError> { - let Some(block_data) = block_data else { - return Ok(()); - }; - let not_in_store_keys = !cache.critical.read().store_keys.contains(&block_data.root); - if not_in_store_keys { - // these keys aren't supposed to be on disk - cache.overflow_store.delete_keys(&block_data.keys)?; - } else { - // check this data is still relevant - if block_data.epoch < cutoff_epoch { - // this data is no longer needed -> delete it - self.overflow_store.delete_keys(&block_data.keys)?; - } - } - Ok(()) - }; - - let mut current_block_data: Option = None; - for res in self - .overflow_store - .0 - .hot_db - .iter_raw_entries(DBColumn::OverflowLRUCache, &[]) - { - let (key_bytes, value_bytes) = res?; - let overflow_key = OverflowKey::from_ssz_bytes(&key_bytes)?; - let current_root = *overflow_key.root(); - - match &mut current_block_data { - Some(block_data) if block_data.root == current_root => { - // still dealing with the same block - block_data.keys.push(overflow_key); - } - _ => { - // first time encountering data for this block - delete_if_outdated(self, current_block_data)?; - let current_epoch = match &overflow_key { - OverflowKey::Block(_) => { - DietAvailabilityPendingExecutedBlock::::from_ssz_bytes( - value_bytes.as_slice(), - )? - .as_block() - .epoch() - } - OverflowKey::Blob(_, _) => { - KzgVerifiedBlob::::from_ssz_bytes(value_bytes.as_slice())? - .as_blob() - .slot() - .epoch(T::EthSpec::slots_per_epoch()) - } - }; - current_block_data = Some(BlockData { - keys: vec![overflow_key], - root: current_root, - epoch: current_epoch, - }); - } - } - } - // can't fall off the end - delete_if_outdated(self, current_block_data)?; - - drop(maintenance_lock); - Ok(()) - } - #[cfg(test)] /// get the state cache for inspection (used only for tests) pub fn state_lru_cache(&self) -> &StateLRUCache { @@ -876,74 +375,7 @@ impl OverflowLRUCache { /// Number of pending component entries in memory in the cache. pub fn block_cache_size(&self) -> usize { - self.critical.read().num_blocks() - } - - /// Returns the number of entries in the cache that have overflowed to disk. - pub fn num_store_entries(&self) -> usize { - self.critical.read().num_store_entries() - } -} - -impl ssz::Encode for OverflowKey { - fn is_ssz_fixed_len() -> bool { - true - } - - fn ssz_append(&self, buf: &mut Vec) { - match self { - OverflowKey::Block(block_hash) => { - block_hash.ssz_append(buf); - buf.push(0u8) - } - OverflowKey::Blob(block_hash, index) => { - block_hash.ssz_append(buf); - buf.push(*index + 1) - } - } - } - - fn ssz_fixed_len() -> usize { - ::ssz_fixed_len() + 1 - } - - fn ssz_bytes_len(&self) -> usize { - match self { - Self::Block(root) => root.ssz_bytes_len() + 1, - Self::Blob(root, _) => root.ssz_bytes_len() + 1, - } - } -} - -impl ssz::Decode for OverflowKey { - fn is_ssz_fixed_len() -> bool { - true - } - - fn ssz_fixed_len() -> usize { - ::ssz_fixed_len() + 1 - } - - fn from_ssz_bytes(bytes: &[u8]) -> Result { - let len = bytes.len(); - let h256_len = ::ssz_fixed_len(); - let expected = h256_len + 1; - - if len != expected { - Err(ssz::DecodeError::InvalidByteLength { len, expected }) - } else { - let root_bytes = bytes - .get(..h256_len) - .ok_or(ssz::DecodeError::OutOfBoundsByte { i: 0 })?; - let block_root = Hash256::from_ssz_bytes(root_bytes)?; - let id_byte = *bytes - .get(h256_len) - .ok_or(ssz::DecodeError::OutOfBoundsByte { i: h256_len })?; - match id_byte { - 0 => Ok(OverflowKey::Block(block_root)), - n => Ok(OverflowKey::Blob(block_root, n - 1)), - } - } + self.critical.read().len() } } @@ -962,8 +394,7 @@ mod test { use logging::test_logger; use slog::{info, Logger}; use state_processing::ConsensusContext; - use std::collections::{BTreeMap, HashMap, VecDeque}; - use std::ops::AddAssign; + use std::collections::VecDeque; use store::{HotColdDB, ItemStore, LevelDB, StoreConfig}; use tempfile::{tempdir, TempDir}; use types::non_zero_usize::new_non_zero_usize; @@ -1047,39 +478,6 @@ mod test { harness } - #[test] - fn overflow_key_encode_decode_equality() { - type E = types::MainnetEthSpec; - let key_block = OverflowKey::Block(Hash256::random()); - let key_blob_0 = OverflowKey::from_blob_id::(BlobIdentifier { - block_root: Hash256::random(), - index: 0, - }) - .expect("should create overflow key 0"); - let key_blob_1 = OverflowKey::from_blob_id::(BlobIdentifier { - block_root: Hash256::random(), - index: 1, - }) - .expect("should create overflow key 1"); - let key_blob_2 = OverflowKey::from_blob_id::(BlobIdentifier { - block_root: Hash256::random(), - index: 2, - }) - .expect("should create overflow key 2"); - let key_blob_3 = OverflowKey::from_blob_id::(BlobIdentifier { - block_root: Hash256::random(), - index: 3, - }) - .expect("should create overflow key 3"); - - let keys = vec![key_block, key_blob_0, key_blob_1, key_blob_2, key_blob_3]; - for key in keys { - let encoded = key.as_ssz_bytes(); - let decoded = OverflowKey::from_ssz_bytes(&encoded).expect("should decode"); - assert_eq!(key, decoded, "Encoded and decoded keys should be equal"); - } - } - async fn availability_pending_block( harness: &BeaconChainHarness>, ) -> ( @@ -1176,7 +574,7 @@ mod test { capacity: usize, ) -> ( BeaconChainHarness>, - Arc>, + Arc>, TempDir, ) where @@ -1190,7 +588,7 @@ mod test { let test_store = harness.chain.store.clone(); let capacity_non_zero = new_non_zero_usize(capacity); let cache = Arc::new( - OverflowLRUCache::::new(capacity_non_zero, test_store, spec.clone()) + DataAvailabilityCheckerInner::::new(capacity_non_zero, test_store, spec.clone()) .expect("should create cache"), ); (harness, cache, chain_db_path) @@ -1212,10 +610,7 @@ mod test { blobs_expected, "should have expected number of blobs" ); - assert!( - cache.critical.read().in_memory.is_empty(), - "cache should be empty" - ); + assert!(cache.critical.read().is_empty(), "cache should be empty"); let availability = cache .put_pending_executed_block(pending_block) .expect("should put block"); @@ -1225,14 +620,14 @@ mod test { "block doesn't have blobs, should be available" ); assert_eq!( - cache.critical.read().in_memory.len(), + cache.critical.read().len(), 1, "cache should still have block as it hasn't been imported yet" ); // remove the blob to simulate successful import cache.remove_pending_components(root); assert_eq!( - cache.critical.read().in_memory.len(), + cache.critical.read().len(), 0, "cache should be empty now that block has been imported" ); @@ -1242,12 +637,12 @@ mod test { "should be pending blobs" ); assert_eq!( - cache.critical.read().in_memory.len(), + cache.critical.read().len(), 1, "cache should have one block" ); assert!( - cache.critical.read().in_memory.peek(&root).is_some(), + cache.critical.read().peek(&root).is_some(), "newly inserted block should exist in memory" ); } @@ -1262,11 +657,11 @@ mod test { assert!(matches!(availability, Availability::Available(_))); } else { assert!(matches!(availability, Availability::MissingComponents(_))); - assert_eq!(cache.critical.read().in_memory.len(), 1); + assert_eq!(cache.critical.read().len(), 1); } } assert!( - cache.critical.read().in_memory.is_empty(), + cache.critical.read().is_empty(), "cache should be empty now that all components available" ); @@ -1289,7 +684,7 @@ mod test { Availability::MissingComponents(root), "should be pending block" ); - assert_eq!(cache.critical.read().in_memory.len(), 1); + assert_eq!(cache.critical.read().len(), 1); } let availability = cache .put_pending_executed_block(pending_block) @@ -1300,457 +695,17 @@ mod test { availability ); assert!( - cache.critical.read().in_memory.len() == 1, + cache.critical.read().len() == 1, "cache should still have available block until import" ); // remove the blob to simulate successful import cache.remove_pending_components(root); assert!( - cache.critical.read().in_memory.is_empty(), + cache.critical.read().is_empty(), "cache should be empty now that all components available" ); } - #[tokio::test] - async fn overflow_cache_test_overflow() { - type E = MinimalEthSpec; - type T = DiskHarnessType; - let capacity = 4; - let (harness, cache, _path) = setup_harness_and_cache::(capacity).await; - - let mut pending_blocks = VecDeque::new(); - let mut pending_blobs = VecDeque::new(); - let mut roots = VecDeque::new(); - while pending_blobs.len() < capacity + 1 { - let (pending_block, blobs) = availability_pending_block(&harness).await; - if pending_block.num_blobs_expected() == 0 { - // we need blocks with blobs - continue; - } - let root = pending_block.block.canonical_root(); - pending_blocks.push_back(pending_block); - pending_blobs.push_back(blobs); - roots.push_back(root); - } - - for i in 0..capacity { - cache - .put_pending_executed_block(pending_blocks.pop_front().expect("should have block")) - .expect("should put block"); - assert_eq!(cache.critical.read().in_memory.len(), i + 1); - } - for root in roots.iter().take(capacity) { - assert!(cache.critical.read().in_memory.peek(root).is_some()); - } - assert_eq!( - cache.critical.read().in_memory.len(), - capacity, - "cache should be full" - ); - // the first block should be the lru entry - assert_eq!( - *cache - .critical - .read() - .in_memory - .peek_lru() - .expect("should exist") - .0, - roots[0], - "first block should be lru" - ); - - cache - .put_pending_executed_block(pending_blocks.pop_front().expect("should have block")) - .expect("should put block"); - assert_eq!( - cache.critical.read().in_memory.len(), - capacity, - "cache should be full" - ); - assert!( - cache.critical.read().in_memory.peek(&roots[0]).is_none(), - "first block should be evicted" - ); - assert_eq!( - *cache - .critical - .read() - .in_memory - .peek_lru() - .expect("should exist") - .0, - roots[1], - "second block should be lru" - ); - - assert!(cache - .overflow_store - .load_pending_components(roots[0]) - .expect("should exist") - .is_some()); - - let threshold = capacity * 3 / 4; - cache - .maintain_threshold(threshold, Epoch::new(0)) - .expect("should maintain threshold"); - assert_eq!( - cache.critical.read().in_memory.len(), - threshold, - "cache should have been maintained" - ); - - let store_keys = cache - .overflow_store - .read_keys_on_disk() - .expect("should read keys"); - assert_eq!(store_keys.len(), 2); - assert!(store_keys.contains(&roots[0])); - assert!(store_keys.contains(&roots[1])); - assert!(cache.critical.read().store_keys.contains(&roots[0])); - assert!(cache.critical.read().store_keys.contains(&roots[1])); - - let blobs_0 = pending_blobs.pop_front().expect("should have blobs"); - let expected_blobs = blobs_0.len(); - let mut kzg_verified_blobs = vec![]; - for (blob_index, gossip_blob) in blobs_0.into_iter().enumerate() { - kzg_verified_blobs.push(gossip_blob.into_inner()); - let availability = cache - .put_kzg_verified_blobs(roots[0], kzg_verified_blobs.clone()) - .expect("should put blob"); - if blob_index == expected_blobs - 1 { - assert!(matches!(availability, Availability::Available(_))); - // remove the block from the cache to simulate import - cache.remove_pending_components(roots[0]); - } else { - // the first block should be brought back into memory - assert!( - cache.critical.read().in_memory.peek(&roots[0]).is_some(), - "first block should be in memory" - ); - assert!(matches!(availability, Availability::MissingComponents(_))); - } - } - assert_eq!( - cache.critical.read().in_memory.len(), - threshold, - "cache should no longer have the first block" - ); - cache.prune_disk(Epoch::new(0)).expect("should prune disk"); - assert!( - cache - .overflow_store - .load_pending_components(roots[1]) - .expect("no error") - .is_some(), - "second block should still be on disk" - ); - assert!( - cache - .overflow_store - .load_pending_components(roots[0]) - .expect("no error") - .is_none(), - "first block should not be on disk" - ); - } - - #[tokio::test] - async fn overflow_cache_test_maintenance() { - type E = MinimalEthSpec; - type T = DiskHarnessType; - let capacity = E::slots_per_epoch() as usize; - let (harness, cache, _path) = setup_harness_and_cache::(capacity).await; - - let n_epochs = 4; - let mut pending_blocks = VecDeque::new(); - let mut pending_blobs = VecDeque::new(); - let mut epoch_count = BTreeMap::new(); - while pending_blobs.len() < n_epochs * capacity { - let (pending_block, blobs) = availability_pending_block(&harness).await; - if pending_block.num_blobs_expected() == 0 { - // we need blocks with blobs - continue; - } - let epoch = pending_block - .block - .as_block() - .slot() - .epoch(E::slots_per_epoch()); - epoch_count.entry(epoch).or_insert_with(|| 0).add_assign(1); - - pending_blocks.push_back(pending_block); - pending_blobs.push_back(blobs); - } - - for _ in 0..(n_epochs * capacity) { - let pending_block = pending_blocks.pop_front().expect("should have block"); - let mut pending_block_blobs = pending_blobs.pop_front().expect("should have blobs"); - let block_root = pending_block.block.as_block().canonical_root(); - let expected_blobs = pending_block.num_blobs_expected(); - if expected_blobs > 1 { - // might as well add a blob too - let one_blob = pending_block_blobs - .pop() - .expect("should have at least one blob"); - let kzg_verified_blobs = vec![one_blob.into_inner()]; - // generate random boolean - let block_first = (rand::random::() % 2) == 0; - if block_first { - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should have pending blobs" - ); - let availability = cache - .put_kzg_verified_blobs(block_root, kzg_verified_blobs) - .expect("should put blob"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "availabilty should be pending blobs: {:?}", - availability - ); - } else { - let availability = cache - .put_kzg_verified_blobs(block_root, kzg_verified_blobs) - .expect("should put blob"); - let root = pending_block.block.as_block().canonical_root(); - assert_eq!( - availability, - Availability::MissingComponents(root), - "should be pending block" - ); - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should have pending blobs" - ); - } - } else { - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should be pending blobs" - ); - } - } - - // now we should have a full cache spanning multiple epochs - // run the maintenance routine for increasing epochs and ensure that the cache is pruned - assert_eq!( - cache.critical.read().in_memory.len(), - capacity, - "cache memory should be full" - ); - let store_keys = cache - .overflow_store - .read_keys_on_disk() - .expect("should read keys"); - assert_eq!( - store_keys.len(), - capacity * (n_epochs - 1), - "cache disk should have the rest" - ); - let mut expected_length = n_epochs * capacity; - for (epoch, count) in epoch_count { - cache - .do_maintenance(epoch + 1) - .expect("should run maintenance"); - let disk_keys = cache - .overflow_store - .read_keys_on_disk() - .expect("should read keys") - .len(); - let mem_keys = cache.critical.read().in_memory.len(); - expected_length -= count; - info!( - harness.chain.log, - "EPOCH: {} DISK KEYS: {} MEM KEYS: {} TOTAL: {} EXPECTED: {}", - epoch, - disk_keys, - mem_keys, - (disk_keys + mem_keys), - std::cmp::max(expected_length, capacity * 3 / 4), - ); - assert_eq!( - (disk_keys + mem_keys), - std::cmp::max(expected_length, capacity * 3 / 4), - "cache should be pruned" - ); - } - } - - #[tokio::test] - async fn overflow_cache_test_persist_recover() { - type E = MinimalEthSpec; - type T = DiskHarnessType; - let capacity = E::slots_per_epoch() as usize; - let (harness, cache, _path) = setup_harness_and_cache::(capacity).await; - - let n_epochs = 4; - let mut pending_blocks = VecDeque::new(); - let mut pending_blobs = VecDeque::new(); - let mut epoch_count = BTreeMap::new(); - while pending_blobs.len() < n_epochs * capacity { - let (pending_block, blobs) = availability_pending_block(&harness).await; - if pending_block.num_blobs_expected() == 0 { - // we need blocks with blobs - continue; - } - let epoch = pending_block - .block - .as_block() - .slot() - .epoch(E::slots_per_epoch()); - epoch_count.entry(epoch).or_insert_with(|| 0).add_assign(1); - - pending_blocks.push_back(pending_block); - pending_blobs.push_back(blobs); - } - - let mut remaining_blobs = HashMap::new(); - for _ in 0..(n_epochs * capacity) { - let pending_block = pending_blocks.pop_front().expect("should have block"); - let mut pending_block_blobs = pending_blobs.pop_front().expect("should have blobs"); - let block_root = pending_block.block.as_block().canonical_root(); - let expected_blobs = pending_block.num_blobs_expected(); - if expected_blobs > 1 { - // might as well add a blob too - let one_blob = pending_block_blobs - .pop() - .expect("should have at least one blob"); - let kzg_verified_blobs = vec![one_blob.into_inner()]; - // generate random boolean - let block_first = (rand::random::() % 2) == 0; - if block_first { - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should have pending blobs" - ); - let availability = cache - .put_kzg_verified_blobs(block_root, kzg_verified_blobs) - .expect("should put blob"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "availabilty should be pending blobs: {:?}", - availability - ); - } else { - let availability = cache - .put_kzg_verified_blobs(block_root, kzg_verified_blobs) - .expect("should put blob"); - let root = pending_block.block.as_block().canonical_root(); - assert_eq!( - availability, - Availability::MissingComponents(root), - "should be pending block" - ); - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should have pending blobs" - ); - } - } else { - let availability = cache - .put_pending_executed_block(pending_block) - .expect("should put block"); - assert!( - matches!(availability, Availability::MissingComponents(_)), - "should be pending blobs" - ); - } - remaining_blobs.insert(block_root, pending_block_blobs); - } - - // now we should have a full cache spanning multiple epochs - // cache should be at capacity - assert_eq!( - cache.critical.read().in_memory.len(), - capacity, - "cache memory should be full" - ); - // write all components to disk - cache.write_all_to_disk().expect("should write all to disk"); - // everything should be on disk now - assert_eq!( - cache - .overflow_store - .read_keys_on_disk() - .expect("should read keys") - .len(), - capacity * n_epochs, - "cache disk should have the rest" - ); - assert_eq!( - cache.critical.read().in_memory.len(), - 0, - "cache memory should be empty" - ); - assert_eq!( - cache.critical.read().store_keys.len(), - n_epochs * capacity, - "cache store should have the rest" - ); - drop(cache); - - // create a new cache with the same store - let recovered_cache = OverflowLRUCache::::new( - new_non_zero_usize(capacity), - harness.chain.store.clone(), - harness.chain.spec.clone(), - ) - .expect("should recover cache"); - // again, everything should be on disk - assert_eq!( - recovered_cache - .overflow_store - .read_keys_on_disk() - .expect("should read keys") - .len(), - capacity * n_epochs, - "cache disk should have the rest" - ); - assert_eq!( - recovered_cache.critical.read().in_memory.len(), - 0, - "cache memory should be empty" - ); - assert_eq!( - recovered_cache.critical.read().store_keys.len(), - n_epochs * capacity, - "cache store should have the rest" - ); - - // now lets insert the remaining blobs until the cache is empty - for (root, blobs) in remaining_blobs { - let additional_blobs = blobs.len(); - let mut kzg_verified_blobs = vec![]; - for (i, gossip_blob) in blobs.into_iter().enumerate() { - kzg_verified_blobs.push(gossip_blob.into_inner()); - let availability = recovered_cache - .put_kzg_verified_blobs(root, kzg_verified_blobs.clone()) - .expect("should put blob"); - if i == additional_blobs - 1 { - assert!(matches!(availability, Availability::Available(_))) - } else { - assert!(matches!(availability, Availability::MissingComponents(_))); - } - } - } - } - #[tokio::test] // ensure the state cache keeps memory usage low and that it can properly recover states // THIS TEST CAN BE DELETED ONCE TREE STATES IS MERGED AND WE RIP OUT THE STATE CACHE @@ -1765,12 +720,12 @@ mod test { let mut state_roots = Vec::new(); // Get enough blocks to fill the cache to capacity, ensuring all blocks have blobs while pending_blocks.len() < capacity { - let (pending_block, _) = availability_pending_block(&harness).await; + let (mut pending_block, _) = availability_pending_block(&harness).await; if pending_block.num_blobs_expected() == 0 { // we need blocks with blobs continue; } - let state_root = pending_block.import_data.state.canonical_root(); + let state_root = pending_block.import_data.state.canonical_root().unwrap(); states.push(pending_block.import_data.state.clone()); pending_blocks.push_back(pending_block); state_roots.push(state_root); @@ -1807,7 +762,6 @@ mod test { let diet_block = cache .critical .read() - .in_memory .peek(&block_root) .map(|pending_components| { pending_components @@ -1836,7 +790,7 @@ mod test { // reconstruct the pending block by replaying the block on the parent state let recovered_pending_block = cache .state_lru_cache() - .reconstruct_pending_executed_block(diet_block) + .recover_pending_executed_block(diet_block) .expect("should reconstruct pending block"); // assert the recovered state is the same as the original diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index 9775d54c02..cf6eb669d5 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -114,38 +114,12 @@ impl StateLRUCache { &self, diet_executed_block: DietAvailabilityPendingExecutedBlock, ) -> Result, AvailabilityCheckError> { - let maybe_state = self.states.write().pop(&diet_executed_block.state_root); - if let Some(state) = maybe_state { - let block_root = diet_executed_block.block.canonical_root(); - Ok(AvailabilityPendingExecutedBlock { - block: diet_executed_block.block, - import_data: BlockImportData { - block_root, - state, - parent_block: diet_executed_block.parent_block, - parent_eth1_finalization_data: diet_executed_block - .parent_eth1_finalization_data, - confirmed_state_roots: diet_executed_block.confirmed_state_roots, - consensus_context: diet_executed_block - .consensus_context - .into_consensus_context(), - }, - payload_verification_outcome: diet_executed_block.payload_verification_outcome, - }) + let state = if let Some(state) = self.states.write().pop(&diet_executed_block.state_root) { + state } else { - self.reconstruct_pending_executed_block(diet_executed_block) - } - } - - /// Reconstruct the `AvailabilityPendingExecutedBlock` by loading the parent - /// state from disk and replaying the block. This function does NOT check the - /// LRU cache. - pub fn reconstruct_pending_executed_block( - &self, - diet_executed_block: DietAvailabilityPendingExecutedBlock, - ) -> Result, AvailabilityCheckError> { + self.reconstruct_state(&diet_executed_block)? + }; let block_root = diet_executed_block.block.canonical_root(); - let state = self.reconstruct_state(&diet_executed_block)?; Ok(AvailabilityPendingExecutedBlock { block: diet_executed_block.block, import_data: BlockImportData { @@ -235,10 +209,10 @@ impl StateLRUCache { impl From> for DietAvailabilityPendingExecutedBlock { - fn from(value: AvailabilityPendingExecutedBlock) -> Self { + fn from(mut value: AvailabilityPendingExecutedBlock) -> Self { Self { block: value.block, - state_root: value.import_data.state.canonical_root(), + state_root: value.import_data.state.canonical_root().unwrap(), parent_block: value.import_data.parent_block, parent_eth1_finalization_data: value.import_data.parent_eth1_finalization_data, confirmed_state_roots: value.import_data.confirmed_state_roots, diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index ee50e3b384..62aad558ee 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -546,12 +546,20 @@ impl Eth1ChainBackend for CachingEth1Backend { state.eth1_data().deposit_count }; - match deposit_index.cmp(&deposit_count) { + // [New in Electra:EIP6110] + let deposit_index_limit = + if let Ok(deposit_receipts_start_index) = state.deposit_requests_start_index() { + std::cmp::min(deposit_count, deposit_receipts_start_index) + } else { + deposit_count + }; + + match deposit_index.cmp(&deposit_index_limit) { Ordering::Greater => Err(Error::DepositIndexTooHigh), Ordering::Equal => Ok(vec![]), Ordering::Less => { let next = deposit_index; - let last = std::cmp::min(deposit_count, next + E::MaxDeposits::to_u64()); + let last = std::cmp::min(deposit_index_limit, next + E::MaxDeposits::to_u64()); self.core .deposits() diff --git a/beacon_node/beacon_chain/src/events.rs b/beacon_node/beacon_chain/src/events.rs index 5f91fe5d0c..267d56220c 100644 --- a/beacon_node/beacon_chain/src/events.rs +++ b/beacon_node/beacon_chain/src/events.rs @@ -23,6 +23,7 @@ pub struct ServerSentEventHandler { proposer_slashing_tx: Sender>, attester_slashing_tx: Sender>, bls_to_execution_change_tx: Sender>, + block_gossip_tx: Sender>, log: Logger, } @@ -51,6 +52,7 @@ impl ServerSentEventHandler { let (proposer_slashing_tx, _) = broadcast::channel(capacity); let (attester_slashing_tx, _) = broadcast::channel(capacity); let (bls_to_execution_change_tx, _) = broadcast::channel(capacity); + let (block_gossip_tx, _) = broadcast::channel(capacity); Self { attestation_tx, @@ -69,6 +71,7 @@ impl ServerSentEventHandler { proposer_slashing_tx, attester_slashing_tx, bls_to_execution_change_tx, + block_gossip_tx, log, } } @@ -147,6 +150,10 @@ impl ServerSentEventHandler { .bls_to_execution_change_tx .send(kind) .map(|count| log_count("bls to execution change", count)), + EventKind::BlockGossip(_) => self + .block_gossip_tx + .send(kind) + .map(|count| log_count("block gossip", count)), }; if let Err(SendError(event)) = result { trace!(self.log, "No receivers registered to listen for event"; "event" => ?event); @@ -217,6 +224,10 @@ impl ServerSentEventHandler { self.bls_to_execution_change_tx.subscribe() } + pub fn subscribe_block_gossip(&self) -> Receiver> { + self.block_gossip_tx.subscribe() + } + pub fn has_attestation_subscribers(&self) -> bool { self.attestation_tx.receiver_count() > 0 } @@ -272,4 +283,8 @@ impl ServerSentEventHandler { pub fn has_bls_to_execution_change_subscribers(&self) -> bool { self.bls_to_execution_change_tx.receiver_count() > 0 } + + pub fn has_block_gossip_subscribers(&self) -> bool { + self.block_gossip_tx.receiver_count() > 0 + } } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 4ceaf675ce..064b2b199f 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -401,8 +401,6 @@ lazy_static! { try_create_histogram("beacon_persist_eth1_cache", "Time taken to persist the eth1 caches"); pub static ref PERSIST_FORK_CHOICE: Result = try_create_histogram("beacon_persist_fork_choice", "Time taken to persist the fork choice struct"); - pub static ref PERSIST_DATA_AVAILABILITY_CHECKER: Result = - try_create_histogram("beacon_persist_data_availability_checker", "Time taken to persist the data availability checker"); /* * Eth1 @@ -859,6 +857,11 @@ lazy_static! { "Duration between the start of the block's slot and the time the block was observed.", ); + pub static ref BEACON_BLOCK_DELAY_CONSENSUS_VERIFICATION_TIME: Result = try_create_int_gauge( + "beacon_block_delay_consensus_verification_time", + "The time taken to verify the block within Lighthouse", + ); + pub static ref BEACON_BLOCK_DELAY_EXECUTION_TIME: Result = try_create_int_gauge( "beacon_block_delay_execution_time", "The duration in verifying the block with the execution layer.", @@ -1213,10 +1216,6 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { &DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE, da_checker_metrics.state_cache_size, ); - set_gauge_by_usize( - &DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE, - da_checker_metrics.num_store_entries, - ); if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() { set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size); diff --git a/beacon_node/beacon_chain/src/persisted_fork_choice.rs b/beacon_node/beacon_chain/src/persisted_fork_choice.rs index 5e50c3433a..8961a74c3d 100644 --- a/beacon_node/beacon_chain/src/persisted_fork_choice.rs +++ b/beacon_node/beacon_chain/src/persisted_fork_choice.rs @@ -1,4 +1,4 @@ -use crate::beacon_fork_choice_store::{PersistedForkChoiceStoreV11, PersistedForkChoiceStoreV17}; +use crate::beacon_fork_choice_store::PersistedForkChoiceStoreV17; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use store::{DBColumn, Error, StoreItem}; @@ -7,37 +7,12 @@ use superstruct::superstruct; // If adding a new version you should update this type alias and fix the breakages. pub type PersistedForkChoice = PersistedForkChoiceV17; -#[superstruct( - variants(V11, V17), - variant_attributes(derive(Encode, Decode)), - no_enum -)] +#[superstruct(variants(V17), variant_attributes(derive(Encode, Decode)), no_enum)] pub struct PersistedForkChoice { pub fork_choice: fork_choice::PersistedForkChoice, - #[superstruct(only(V11))] - pub fork_choice_store: PersistedForkChoiceStoreV11, - #[superstruct(only(V17))] pub fork_choice_store: PersistedForkChoiceStoreV17, } -impl From for PersistedForkChoice { - fn from(from: PersistedForkChoiceV11) -> PersistedForkChoice { - PersistedForkChoice { - fork_choice: from.fork_choice, - fork_choice_store: from.fork_choice_store.into(), - } - } -} - -impl From for PersistedForkChoiceV11 { - fn from(from: PersistedForkChoice) -> PersistedForkChoiceV11 { - PersistedForkChoiceV11 { - fork_choice: from.fork_choice, - fork_choice_store: from.fork_choice_store.into(), - } - } -} - macro_rules! impl_store_item { ($type:ty) => { impl StoreItem for $type { @@ -56,5 +31,4 @@ macro_rules! impl_store_item { }; } -impl_store_item!(PersistedForkChoiceV11); impl_store_item!(PersistedForkChoiceV17); diff --git a/beacon_node/beacon_chain/src/schema_change.rs b/beacon_node/beacon_chain/src/schema_change.rs index 3fe75e348c..4f7770e22c 100644 --- a/beacon_node/beacon_chain/src/schema_change.rs +++ b/beacon_node/beacon_chain/src/schema_change.rs @@ -1,7 +1,4 @@ //! Utilities for managing database schema changes. -mod migration_schema_v17; -mod migration_schema_v18; -mod migration_schema_v19; mod migration_schema_v20; mod migration_schema_v21; @@ -54,32 +51,8 @@ pub fn migrate_schema( } // - // Migrations from before SchemaVersion(16) are deprecated. + // Migrations from before SchemaVersion(19) are deprecated. // - (SchemaVersion(16), SchemaVersion(17)) => { - let ops = migration_schema_v17::upgrade_to_v17::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } - (SchemaVersion(17), SchemaVersion(16)) => { - let ops = migration_schema_v17::downgrade_from_v17::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } - (SchemaVersion(17), SchemaVersion(18)) => { - let ops = migration_schema_v18::upgrade_to_v18::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } - (SchemaVersion(18), SchemaVersion(17)) => { - let ops = migration_schema_v18::downgrade_from_v18::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } - (SchemaVersion(18), SchemaVersion(19)) => { - let ops = migration_schema_v19::upgrade_to_v19::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } - (SchemaVersion(19), SchemaVersion(18)) => { - let ops = migration_schema_v19::downgrade_from_v19::(db.clone(), log)?; - db.store_schema_version_atomically(to, ops) - } (SchemaVersion(19), SchemaVersion(20)) => { let ops = migration_schema_v20::upgrade_to_v20::(db.clone(), log)?; db.store_schema_version_atomically(to, ops) diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v17.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v17.rs deleted file mode 100644 index 770cbb8ab5..0000000000 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v17.rs +++ /dev/null @@ -1,88 +0,0 @@ -use crate::beacon_chain::{BeaconChainTypes, FORK_CHOICE_DB_KEY}; -use crate::persisted_fork_choice::{PersistedForkChoiceV11, PersistedForkChoiceV17}; -use proto_array::core::{SszContainerV16, SszContainerV17}; -use slog::{debug, Logger}; -use ssz::{Decode, Encode}; -use std::sync::Arc; -use store::{Error, HotColdDB, KeyValueStoreOp, StoreItem}; - -pub fn upgrade_fork_choice( - mut fork_choice: PersistedForkChoiceV11, -) -> Result { - let ssz_container_v16 = SszContainerV16::from_ssz_bytes( - &fork_choice.fork_choice.proto_array_bytes, - ) - .map_err(|e| { - Error::SchemaMigrationError(format!( - "Failed to decode ProtoArrayForkChoice during schema migration: {:?}", - e - )) - })?; - - let ssz_container_v17: SszContainerV17 = ssz_container_v16.try_into().map_err(|e| { - Error::SchemaMigrationError(format!( - "Missing checkpoint during schema migration: {:?}", - e - )) - })?; - fork_choice.fork_choice.proto_array_bytes = ssz_container_v17.as_ssz_bytes(); - - Ok(fork_choice.into()) -} - -pub fn downgrade_fork_choice( - mut fork_choice: PersistedForkChoiceV17, -) -> Result { - let ssz_container_v17 = SszContainerV17::from_ssz_bytes( - &fork_choice.fork_choice.proto_array_bytes, - ) - .map_err(|e| { - Error::SchemaMigrationError(format!( - "Failed to decode ProtoArrayForkChoice during schema migration: {:?}", - e - )) - })?; - - let ssz_container_v16: SszContainerV16 = ssz_container_v17.into(); - fork_choice.fork_choice.proto_array_bytes = ssz_container_v16.as_ssz_bytes(); - - Ok(fork_choice.into()) -} - -pub fn upgrade_to_v17( - db: Arc>, - log: Logger, -) -> Result, Error> { - // Get persisted_fork_choice. - let v11 = db - .get_item::(&FORK_CHOICE_DB_KEY)? - .ok_or_else(|| Error::SchemaMigrationError("fork choice missing from database".into()))?; - - let v17 = upgrade_fork_choice(v11)?; - - debug!( - log, - "Removing unused best_justified_checkpoint from fork choice store." - ); - - Ok(vec![v17.as_kv_store_op(FORK_CHOICE_DB_KEY)]) -} - -pub fn downgrade_from_v17( - db: Arc>, - log: Logger, -) -> Result, Error> { - // Get persisted_fork_choice. - let v17 = db - .get_item::(&FORK_CHOICE_DB_KEY)? - .ok_or_else(|| Error::SchemaMigrationError("fork choice missing from database".into()))?; - - let v11 = downgrade_fork_choice(v17)?; - - debug!( - log, - "Adding junk best_justified_checkpoint to fork choice store." - ); - - Ok(vec![v11.as_kv_store_op(FORK_CHOICE_DB_KEY)]) -} diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v18.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v18.rs deleted file mode 100644 index 04a9da8412..0000000000 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v18.rs +++ /dev/null @@ -1,119 +0,0 @@ -use crate::beacon_chain::BeaconChainTypes; -use slog::{error, info, warn, Logger}; -use slot_clock::SlotClock; -use std::sync::Arc; -use std::time::Duration; -use store::{ - get_key_for_col, metadata::BLOB_INFO_KEY, DBColumn, Error, HotColdDB, KeyValueStoreOp, -}; -use types::{Epoch, EthSpec, Hash256, Slot}; - -/// The slot clock isn't usually available before the database is initialized, so we construct a -/// temporary slot clock by reading the genesis state. It should always exist if the database is -/// initialized at a prior schema version, however we still handle the lack of genesis state -/// gracefully. -fn get_slot_clock( - db: &HotColdDB, - log: &Logger, -) -> Result, Error> { - let spec = db.get_chain_spec(); - let Some(genesis_block) = db.get_blinded_block(&Hash256::zero())? else { - error!(log, "Missing genesis block"); - return Ok(None); - }; - let Some(genesis_state) = db.get_state(&genesis_block.state_root(), Some(Slot::new(0)))? else { - error!(log, "Missing genesis state"; "state_root" => ?genesis_block.state_root()); - return Ok(None); - }; - Ok(Some(T::SlotClock::new( - spec.genesis_slot, - Duration::from_secs(genesis_state.genesis_time()), - Duration::from_secs(spec.seconds_per_slot), - ))) -} - -fn get_current_epoch( - db: &Arc>, - log: &Logger, -) -> Result { - get_slot_clock::(db, log)? - .and_then(|clock| clock.now()) - .map(|slot| slot.epoch(T::EthSpec::slots_per_epoch())) - .ok_or(Error::SlotClockUnavailableForMigration) -} - -pub fn upgrade_to_v18( - db: Arc>, - log: Logger, -) -> Result, Error> { - db.heal_freezer_block_roots_at_split()?; - db.heal_freezer_block_roots_at_genesis()?; - info!(log, "Healed freezer block roots"); - - // No-op, even if Deneb has already occurred. The database is probably borked in this case, but - // *maybe* the fork recovery will revert the minority fork and succeed. - if let Some(deneb_fork_epoch) = db.get_chain_spec().deneb_fork_epoch { - let current_epoch = get_current_epoch::(&db, &log)?; - if current_epoch >= deneb_fork_epoch { - warn!( - log, - "Attempting upgrade to v18 schema"; - "info" => "this may not work as Deneb has already been activated" - ); - } else { - info!( - log, - "Upgrading to v18 schema"; - "info" => "ready for Deneb", - "epochs_until_deneb" => deneb_fork_epoch - current_epoch - ); - } - } else { - info!( - log, - "Upgrading to v18 schema"; - "info" => "ready for Deneb once it is scheduled" - ); - } - Ok(vec![]) -} - -pub fn downgrade_from_v18( - db: Arc>, - log: Logger, -) -> Result, Error> { - // We cannot downgrade from V18 once the Deneb fork has been activated, because there will - // be blobs and blob metadata in the database that aren't understood by the V17 schema. - if let Some(deneb_fork_epoch) = db.get_chain_spec().deneb_fork_epoch { - let current_epoch = get_current_epoch::(&db, &log)?; - if current_epoch >= deneb_fork_epoch { - error!( - log, - "Deneb already active: v18+ is mandatory"; - "current_epoch" => current_epoch, - "deneb_fork_epoch" => deneb_fork_epoch, - ); - return Err(Error::UnableToDowngrade); - } else { - info!( - log, - "Downgrading to v17 schema"; - "info" => "you will need to upgrade before Deneb", - "epochs_until_deneb" => deneb_fork_epoch - current_epoch - ); - } - } else { - info!( - log, - "Downgrading to v17 schema"; - "info" => "you need to upgrade before Deneb", - ); - } - - let ops = vec![KeyValueStoreOp::DeleteKey(get_key_for_col( - DBColumn::BeaconMeta.into(), - BLOB_INFO_KEY.as_bytes(), - ))]; - - Ok(ops) -} diff --git a/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs b/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs deleted file mode 100644 index 578e9bad31..0000000000 --- a/beacon_node/beacon_chain/src/schema_change/migration_schema_v19.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::beacon_chain::BeaconChainTypes; -use slog::{debug, info, Logger}; -use std::sync::Arc; -use store::{get_key_for_col, DBColumn, Error, HotColdDB, KeyValueStore, KeyValueStoreOp}; - -pub fn upgrade_to_v19( - db: Arc>, - log: Logger, -) -> Result, Error> { - let mut hot_delete_ops = vec![]; - let mut blob_keys = vec![]; - let column = DBColumn::BeaconBlob; - - debug!(log, "Migrating from v18 to v19"); - // Iterate through the blobs on disk. - for res in db.hot_db.iter_column_keys::>(column) { - let key = res?; - let key_col = get_key_for_col(column.as_str(), &key); - hot_delete_ops.push(KeyValueStoreOp::DeleteKey(key_col)); - blob_keys.push(key); - } - - let num_blobs = blob_keys.len(); - debug!(log, "Collected {} blob lists to migrate", num_blobs); - - let batch_size = 500; - let mut batch = Vec::with_capacity(batch_size); - - for key in blob_keys { - let next_blob = db.hot_db.get_bytes(column.as_str(), &key)?; - if let Some(next_blob) = next_blob { - let key_col = get_key_for_col(column.as_str(), &key); - batch.push(KeyValueStoreOp::PutKeyValue(key_col, next_blob)); - - if batch.len() >= batch_size { - db.blobs_db.do_atomically(batch.clone())?; - batch.clear(); - } - } - } - - // Process the remaining batch if it's not empty - if !batch.is_empty() { - db.blobs_db.do_atomically(batch)?; - } - - debug!(log, "Wrote {} blobs to the blobs db", num_blobs); - - // Delete all the blobs - info!(log, "Upgrading to v19 schema"); - Ok(hot_delete_ops) -} - -pub fn downgrade_from_v19( - _db: Arc>, - log: Logger, -) -> Result, Error> { - // No-op - info!( - log, - "Downgrading to v18 schema"; - ); - - Ok(vec![]) -} diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index bd98f19af6..6b85d7aadf 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -2246,7 +2246,7 @@ where .unwrap(); state = new_state; block_hash_from_slot.insert(*slot, block_hash); - state_hash_from_slot.insert(*slot, state.tree_hash_root().into()); + state_hash_from_slot.insert(*slot, state.canonical_root().unwrap().into()); latest_block_hash = Some(block_hash); } ( diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index c2468102ee..e675d6956e 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -35,7 +35,6 @@ use store::{ }; use tempfile::{tempdir, TempDir}; use tokio::time::sleep; -use tree_hash::TreeHash; use types::test_utils::{SeedableRng, XorShiftRng}; use types::*; @@ -199,8 +198,8 @@ async fn heal_freezer_block_roots_with_skip_slots() { ); let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT); - let current_state = harness.get_current_state(); - let state_root = harness.get_current_state().tree_hash_root(); + let mut current_state = harness.get_current_state(); + let state_root = current_state.canonical_root().unwrap(); let all_validators = &harness.get_all_validators(); harness .add_attested_blocks_at_slots( @@ -611,12 +610,13 @@ async fn epoch_boundary_state_attestation_processing() { .get_blinded_block(&block_root) .unwrap() .expect("block exists"); - let epoch_boundary_state = store + let mut epoch_boundary_state = store .load_epoch_boundary_state(&block.state_root()) .expect("no error") .expect("epoch boundary state exists"); + let ebs_state_root = epoch_boundary_state.canonical_root().unwrap(); let ebs_of_ebs = store - .load_epoch_boundary_state(&epoch_boundary_state.canonical_root()) + .load_epoch_boundary_state(&ebs_state_root) .expect("no error") .expect("ebs of ebs exists"); assert_eq!(epoch_boundary_state, ebs_of_ebs); @@ -2604,9 +2604,9 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .unwrap() .map(Result::unwrap) { - let state = store.get_state(&state_root, Some(slot)).unwrap().unwrap(); + let mut state = store.get_state(&state_root, Some(slot)).unwrap().unwrap(); assert_eq!(state.slot(), slot); - assert_eq!(state.canonical_root(), state_root); + assert_eq!(state.canonical_root().unwrap(), state_root); } // Anchor slot is still set to the slot of the checkpoint block. @@ -3050,13 +3050,7 @@ async fn schema_downgrade_to_min_version() { ) .await; - let min_version = if harness.spec.deneb_fork_epoch.is_some() { - // Can't downgrade beyond V18 once Deneb is reached, for simplicity don't test that - // at all if Deneb is enabled. - SchemaVersion(18) - } else { - SchemaVersion(16) - }; + let min_version = SchemaVersion(19); // Save the slot clock so that the new harness doesn't revert in time. let slot_clock = harness.chain.slot_clock.clone(); diff --git a/beacon_node/eth1/src/deposit_cache.rs b/beacon_node/eth1/src/deposit_cache.rs index 75391e58a0..b443f739e8 100644 --- a/beacon_node/eth1/src/deposit_cache.rs +++ b/beacon_node/eth1/src/deposit_cache.rs @@ -54,7 +54,7 @@ pub enum Error { pub type SszDepositCache = SszDepositCacheV13; #[superstruct( - variants(V1, V13), + variants(V13), variant_attributes(derive(Encode, Decode, Clone)), no_enum )] @@ -62,11 +62,8 @@ pub struct SszDepositCache { pub logs: Vec, pub leaves: Vec, pub deposit_contract_deploy_block: u64, - #[superstruct(only(V13))] pub finalized_deposit_count: u64, - #[superstruct(only(V13))] pub finalized_block_height: u64, - #[superstruct(only(V13))] pub deposit_tree_snapshot: Option, pub deposit_roots: Vec, } diff --git a/beacon_node/eth1/src/inner.rs b/beacon_node/eth1/src/inner.rs index 0468a02d2e..452922b173 100644 --- a/beacon_node/eth1/src/inner.rs +++ b/beacon_node/eth1/src/inner.rs @@ -2,7 +2,7 @@ use crate::service::endpoint_from_config; use crate::Config; use crate::{ block_cache::{BlockCache, Eth1Block}, - deposit_cache::{DepositCache, SszDepositCache, SszDepositCacheV1, SszDepositCacheV13}, + deposit_cache::{DepositCache, SszDepositCache, SszDepositCacheV13}, }; use execution_layer::HttpJsonRpc; use parking_lot::RwLock; @@ -90,15 +90,12 @@ impl Inner { pub type SszEth1Cache = SszEth1CacheV13; #[superstruct( - variants(V1, V13), + variants(V13), variant_attributes(derive(Encode, Decode, Clone)), no_enum )] pub struct SszEth1Cache { pub block_cache: BlockCache, - #[superstruct(only(V1))] - pub deposit_cache: SszDepositCacheV1, - #[superstruct(only(V13))] pub deposit_cache: SszDepositCacheV13, #[ssz(with = "four_byte_option_u64")] pub last_processed_block: Option, diff --git a/beacon_node/eth1/src/lib.rs b/beacon_node/eth1/src/lib.rs index 9d6cb7c847..9c4f9a1d8d 100644 --- a/beacon_node/eth1/src/lib.rs +++ b/beacon_node/eth1/src/lib.rs @@ -5,9 +5,9 @@ mod metrics; mod service; pub use block_cache::{BlockCache, Eth1Block}; -pub use deposit_cache::{DepositCache, SszDepositCache, SszDepositCacheV1, SszDepositCacheV13}; +pub use deposit_cache::{DepositCache, SszDepositCache, SszDepositCacheV13}; pub use execution_layer::http::deposit_log::DepositLog; -pub use inner::{SszEth1Cache, SszEth1CacheV1, SszEth1CacheV13}; +pub use inner::{SszEth1Cache, SszEth1CacheV13}; pub use service::{ BlockCacheUpdateOutcome, Config, DepositCacheUpdateOutcome, Error, Eth1Endpoint, Service, DEFAULT_CHAIN_ID, diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 7c8f64a722..2d50dc6c63 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -31,6 +31,7 @@ mod validators; mod version; use crate::produce_block::{produce_blinded_block_v2, produce_block_v2, produce_block_v3}; +use crate::version::fork_versioned_response; use beacon_chain::{ attestation_verification::VerifiedAttestation, observed_operations::ObservationOutcome, validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, @@ -96,7 +97,7 @@ use warp::hyper::Body; use warp::sse::Event; use warp::Reply; use warp::{http::Response, Filter, Rejection}; -use warp_utils::{query::multi_key_query, uor::UnifyingOrFilter}; +use warp_utils::{query::multi_key_query, reject::convert_rejection, uor::UnifyingOrFilter}; const API_PREFIX: &str = "eth"; @@ -256,12 +257,15 @@ pub fn prometheus_metrics() -> warp::filters::log::Log( ); // GET beacon/blocks/{block_id}/attestations - let get_beacon_block_attestations = beacon_blocks_path_v1 + let get_beacon_block_attestations = beacon_blocks_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .then( - |block_id: BlockId, + |endpoint_version: EndpointVersion, + block_id: BlockId, task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; - Ok(api_types::GenericResponse::from( - block - .message() - .body() - .attestations() - .map(|att| att.clone_as_attestation()) - .collect::>(), - ) - .add_execution_optimistic_finalized(execution_optimistic, finalized)) + let fork_name = block + .fork_name(&chain.spec) + .map_err(inconsistent_fork_rejection)?; + let atts = block + .message() + .body() + .attestations() + .map(|att| att.clone_as_attestation()) + .collect::>(); + let res = execution_optimistic_finalized_fork_versioned_response( + endpoint_version, + fork_name, + execution_optimistic, + finalized, + &atts, + )?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); @@ -1750,8 +1766,14 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()); + let beacon_pool_path_any = any_version + .and(warp::path("beacon")) + .and(warp::path("pool")) + .and(task_spawner_filter.clone()) + .and(chain_filter.clone()); + // POST beacon/pool/attestations - let post_beacon_pool_attestations = beacon_pool_path + let post_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) @@ -1760,7 +1782,11 @@ pub fn serve( .and(reprocess_send_filter) .and(log_filter.clone()) .then( - |task_spawner: TaskSpawner, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, @@ -1776,21 +1802,22 @@ pub fn serve( ) .await .map(|()| warp::reply::json(&())); - task_spawner::convert_rejection(result).await + convert_rejection(result).await }, ); // GET beacon/pool/attestations?committee_index,slot - let get_beacon_pool_attestations = beacon_pool_path + let get_beacon_pool_attestations = beacon_pool_path_any .clone() .and(warp::path("attestations")) .and(warp::path::end()) .and(warp::query::()) .then( - |task_spawner: TaskSpawner, + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, query: api_types::AttestationPoolQuery| { - task_spawner.blocking_json_task(Priority::P1, move || { + task_spawner.blocking_response_task(Priority::P1, move || { let query_filter = |data: &AttestationData| { query.slot.map_or(true, |slot| slot == data.slot) && query @@ -1807,20 +1834,48 @@ pub fn serve( .filter(|&att| query_filter(att.data())) .cloned(), ); - Ok(api_types::GenericResponse::from(attestations)) + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = + chain + .slot_clock + .now() + .ok_or(warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ))?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + let attestations = attestations + .into_iter() + .filter(|att| { + (fork_name.electra_enabled() && matches!(att, Attestation::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(att, Attestation::Base(_))) + }) + .collect::>(); + + let res = fork_versioned_response(endpoint_version, fork_name, &attestations)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) }) }, ); // POST beacon/pool/attester_slashings - let post_beacon_pool_attester_slashings = beacon_pool_path + let post_beacon_pool_attester_slashings = beacon_pool_path_any .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) .then( - |task_spawner: TaskSpawner, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, chain: Arc>, slashing: AttesterSlashing, network_tx: UnboundedSender>| { @@ -1857,18 +1912,45 @@ pub fn serve( ); // GET beacon/pool/attester_slashings - let get_beacon_pool_attester_slashings = beacon_pool_path - .clone() - .and(warp::path("attester_slashings")) - .and(warp::path::end()) - .then( - |task_spawner: TaskSpawner, chain: Arc>| { - task_spawner.blocking_json_task(Priority::P1, move || { - let attestations = chain.op_pool.get_all_attester_slashings(); - Ok(api_types::GenericResponse::from(attestations)) - }) - }, - ); + let get_beacon_pool_attester_slashings = + beacon_pool_path_any + .clone() + .and(warp::path("attester_slashings")) + .and(warp::path::end()) + .then( + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + let slashings = chain.op_pool.get_all_attester_slashings(); + + // Use the current slot to find the fork version, and convert all messages to the + // current fork's format. This is to ensure consistent message types matching + // `Eth-Consensus-Version`. + let current_slot = chain.slot_clock.now().ok_or( + warp_utils::reject::custom_server_error( + "unable to read slot clock".to_string(), + ), + )?; + let fork_name = chain.spec.fork_name_at_slot::(current_slot); + let slashings = slashings + .into_iter() + .filter(|slashing| { + (fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Electra(_))) + || (!fork_name.electra_enabled() + && matches!(slashing, AttesterSlashing::Base(_))) + }) + .collect::>(); + + let res = fork_versioned_response(endpoint_version, fork_name, &slashings)?; + Ok(add_consensus_version_header( + warp::reply::json(&res).into_response(), + fork_name, + )) + }) + }, + ); // POST beacon/pool/proposer_slashings let post_beacon_pool_proposer_slashings = beacon_pool_path @@ -3175,7 +3257,7 @@ pub fn serve( ); // GET validator/aggregate_attestation?attestation_data_root,slot - let get_validator_aggregate_attestation = eth_v1 + let get_validator_aggregate_attestation = any_version .and(warp::path("validator")) .and(warp::path("aggregate_attestation")) .and(warp::path::end()) @@ -3184,29 +3266,45 @@ pub fn serve( .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .then( - |query: api_types::ValidatorAggregateAttestationQuery, + |endpoint_version: EndpointVersion, + query: api_types::ValidatorAggregateAttestationQuery, not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { not_synced_filter?; - chain - .get_pre_electra_aggregated_attestation_by_slot_and_root( + let res = if endpoint_version == V2 { + let Some(committee_index) = query.committee_index else { + return Err(warp_utils::reject::custom_bad_request( + "missing committee index".to_string(), + )); + }; + chain.get_aggregated_attestation_electra( + query.slot, + &query.attestation_data_root, + committee_index, + ) + } else if endpoint_version == V1 { + // Do nothing + chain.get_pre_electra_aggregated_attestation_by_slot_and_root( query.slot, &query.attestation_data_root, ) - .map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "unable to fetch aggregate: {:?}", - e - )) - })? - .map(api_types::GenericResponse::from) - .ok_or_else(|| { - warp_utils::reject::custom_not_found( - "no matching aggregate found".to_string(), - ) - }) + } else { + return Err(unsupported_version_rejection(endpoint_version)); + }; + res.map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "unable to fetch aggregate: {:?}", + e + )) + })? + .map(api_types::GenericResponse::from) + .ok_or_else(|| { + warp_utils::reject::custom_not_found( + "no matching aggregate found".to_string(), + ) + }) }) }, ); @@ -3302,7 +3400,7 @@ pub fn serve( ); // POST validator/aggregate_and_proofs - let post_validator_aggregate_and_proofs = eth_v1 + let post_validator_aggregate_and_proofs = any_version .and(warp::path("validator")) .and(warp::path("aggregate_and_proofs")) .and(warp::path::end()) @@ -3313,7 +3411,11 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( - |not_synced_filter: Result<(), Rejection>, + // V1 and V2 are identical except V2 has a consensus version header in the request. + // We only require this header for SSZ deserialization, which isn't supported for + // this endpoint presently. + |_endpoint_version: EndpointVersion, + not_synced_filter: Result<(), Rejection>, task_spawner: TaskSpawner, chain: Arc>, aggregates: Vec>, @@ -3715,12 +3817,12 @@ pub fn serve( .await; if initial_result.is_err() { - return task_spawner::convert_rejection(initial_result).await; + return convert_rejection(initial_result).await; } // Await a response from the builder without blocking a // `BeaconProcessor` worker. - task_spawner::convert_rejection(rx.await.unwrap_or_else(|_| { + convert_rejection(rx.await.unwrap_or_else(|_| { Ok(warp::reply::with_status( warp::reply::json(&"No response from channel"), eth2::StatusCode::INTERNAL_SERVER_ERROR, @@ -4372,6 +4474,9 @@ pub fn serve( api_types::EventTopic::BlsToExecutionChange => { event_handler.subscribe_bls_to_execution_change() } + api_types::EventTopic::BlockGossip => { + event_handler.subscribe_block_gossip() + } }; receivers.push( diff --git a/beacon_node/http_api/src/task_spawner.rs b/beacon_node/http_api/src/task_spawner.rs index cfee5e01ca..a679b294f6 100644 --- a/beacon_node/http_api/src/task_spawner.rs +++ b/beacon_node/http_api/src/task_spawner.rs @@ -4,6 +4,7 @@ use std::future::Future; use tokio::sync::{mpsc::error::TrySendError, oneshot}; use types::EthSpec; use warp::reply::{Reply, Response}; +use warp_utils::reject::convert_rejection; /// Maps a request to a queue in the `BeaconProcessor`. #[derive(Clone, Copy)] @@ -35,24 +36,6 @@ pub struct TaskSpawner { beacon_processor_send: Option>, } -/// Convert a warp `Rejection` into a `Response`. -/// -/// This function should *always* be used to convert rejections into responses. This prevents warp -/// from trying to backtrack in strange ways. See: https://github.com/sigp/lighthouse/issues/3404 -pub async fn convert_rejection(res: Result) -> Response { - match res { - Ok(response) => response.into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - warp::reply::json(&"unhandled error"), - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } -} - impl TaskSpawner { pub fn new(beacon_processor_send: Option>) -> Self { Self { diff --git a/beacon_node/http_api/tests/broadcast_validation_tests.rs b/beacon_node/http_api/tests/broadcast_validation_tests.rs index 6a3f7947e6..78f9c81988 100644 --- a/beacon_node/http_api/tests/broadcast_validation_tests.rs +++ b/beacon_node/http_api/tests/broadcast_validation_tests.rs @@ -7,7 +7,6 @@ use eth2::types::{BroadcastValidation, PublishBlockRequest}; use http_api::test_utils::InteractiveTester; use http_api::{publish_blinded_block, publish_block, reconstruct_block, ProvenancedBlock}; use std::sync::Arc; -use tree_hash::TreeHash; use types::{Epoch, EthSpec, ForkName, Hash256, MainnetEthSpec, Slot}; use warp::Rejection; use warp_utils::reject::CustomBadRequest; @@ -353,13 +352,20 @@ pub async fn consensus_partial_pass_only_consensus() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, _), state_after_a) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), state_after_b) = tester.harness.make_block(state_a, slot_b).await; + let ((block_a, _), mut state_after_a) = + tester.harness.make_block(state_a.clone(), slot_b).await; + let ((block_b, blobs_b), mut state_after_b) = tester.harness.make_block(state_a, slot_b).await; let block_b_root = block_b.canonical_root(); /* check for `make_block` curios */ - assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); - assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_eq!( + block_a.state_root(), + state_after_a.canonical_root().unwrap() + ); + assert_eq!( + block_b.state_root(), + state_after_b.canonical_root().unwrap() + ); assert_ne!(block_a.state_root(), block_b.state_root()); let gossip_block_contents_b = PublishBlockRequest::new(block_b, blobs_b) @@ -516,13 +522,19 @@ pub async fn equivocation_consensus_early_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, blobs_a), state_after_a) = + let ((block_a, blobs_a), mut state_after_a) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), state_after_b) = tester.harness.make_block(state_a, slot_b).await; + let ((block_b, blobs_b), mut state_after_b) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ - assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); - assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_eq!( + block_a.state_root(), + state_after_a.canonical_root().unwrap() + ); + assert_eq!( + block_b.state_root(), + state_after_b.canonical_root().unwrap() + ); assert_ne!(block_a.state_root(), block_b.state_root()); /* submit `block_a` as valid */ @@ -642,13 +654,19 @@ pub async fn equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let ((block_a, blobs_a), state_after_a) = + let ((block_a, blobs_a), mut state_after_a) = tester.harness.make_block(state_a.clone(), slot_b).await; - let ((block_b, blobs_b), state_after_b) = tester.harness.make_block(state_a, slot_b).await; + let ((block_b, blobs_b), mut state_after_b) = tester.harness.make_block(state_a, slot_b).await; /* check for `make_block` curios */ - assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); - assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_eq!( + block_a.state_root(), + state_after_a.canonical_root().unwrap() + ); + assert_eq!( + block_b.state_root(), + state_after_b.canonical_root().unwrap() + ); assert_ne!(block_a.state_root(), block_b.state_root()); let gossip_block_contents_b = PublishBlockRequest::new(block_b, blobs_b) @@ -1135,15 +1153,21 @@ pub async fn blinded_equivocation_consensus_early_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a) = tester + let (block_a, mut state_after_a) = tester .harness .make_blinded_block(state_a.clone(), slot_b) .await; - let (block_b, state_after_b) = tester.harness.make_blinded_block(state_a, slot_b).await; + let (block_b, mut state_after_b) = tester.harness.make_blinded_block(state_a, slot_b).await; /* check for `make_blinded_block` curios */ - assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); - assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_eq!( + block_a.state_root(), + state_after_a.canonical_root().unwrap() + ); + assert_eq!( + block_b.state_root(), + state_after_b.canonical_root().unwrap() + ); assert_ne!(block_a.state_root(), block_b.state_root()); /* submit `block_a` as valid */ @@ -1259,16 +1283,22 @@ pub async fn blinded_equivocation_consensus_late_equivocation() { let slot_b = slot_a + 1; let state_a = tester.harness.get_current_state(); - let (block_a, state_after_a) = tester + let (block_a, mut state_after_a) = tester .harness .make_blinded_block(state_a.clone(), slot_b) .await; - let (block_b, state_after_b) = tester.harness.make_blinded_block(state_a, slot_b).await; + let (block_b, mut state_after_b) = tester.harness.make_blinded_block(state_a, slot_b).await; let block_b = Arc::new(block_b); /* check for `make_blinded_block` curios */ - assert_eq!(block_a.state_root(), state_after_a.tree_hash_root()); - assert_eq!(block_b.state_root(), state_after_b.tree_hash_root()); + assert_eq!( + block_a.state_root(), + state_after_a.canonical_root().unwrap() + ); + assert_eq!( + block_b.state_root(), + state_after_b.canonical_root().unwrap() + ); assert_ne!(block_a.state_root(), block_b.state_root()); let unblinded_block_a = reconstruct_block( diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index ad32ff1d57..db8a0ab2b5 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -55,7 +55,7 @@ async fn sync_committee_duties_across_fork() { // though the head state hasn't transitioned yet. let fork_slot = fork_epoch.start_slot(E::slots_per_epoch()); let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); - let (_, state) = harness + let (_, mut state) = harness .add_attested_block_at_slot( fork_slot - 1, genesis_state, @@ -76,7 +76,7 @@ async fn sync_committee_duties_across_fork() { assert_eq!(sync_duties.len(), E::sync_committee_size()); // After applying a block at the fork slot the duties should remain unchanged. - let state_root = state.canonical_root(); + let state_root = state.canonical_root().unwrap(); harness .add_attested_block_at_slot(fork_slot, state, state_root, &all_validators) .await @@ -150,8 +150,13 @@ async fn attestations_across_fork_with_skip_slots() { .collect::>(); assert!(!unaggregated_attestations.is_empty()); + let fork_name = harness.spec.fork_name_at_slot::(fork_slot); client - .post_beacon_pool_attestations(&unaggregated_attestations) + .post_beacon_pool_attestations_v1(&unaggregated_attestations) + .await + .unwrap(); + client + .post_beacon_pool_attestations_v2(&unaggregated_attestations, fork_name) .await .unwrap(); @@ -162,7 +167,11 @@ async fn attestations_across_fork_with_skip_slots() { assert!(!signed_aggregates.is_empty()); client - .post_validator_aggregate_and_proof(&signed_aggregates) + .post_validator_aggregate_and_proof_v1(&signed_aggregates) + .await + .unwrap(); + client + .post_validator_aggregate_and_proof_v2(&signed_aggregates, fork_name) .await .unwrap(); } @@ -257,7 +266,7 @@ async fn sync_committee_indices_across_fork() { // applied. let fork_slot = fork_epoch.start_slot(E::slots_per_epoch()); let (genesis_state, genesis_state_root) = harness.get_current_state_and_root(); - let (_, state) = harness + let (_, mut state) = harness .add_attested_block_at_slot( fork_slot - 1, genesis_state, @@ -295,7 +304,7 @@ async fn sync_committee_indices_across_fork() { // Once the head is updated it should be useable for requests, including in the next sync // committee period. - let state_root = state.canonical_root(); + let state_root = state.canonical_root().unwrap(); harness .add_attested_block_at_slot(fork_slot + 1, state, state_root, &all_validators) .await diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 711820ccac..2f417cf7ba 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -17,7 +17,6 @@ use state_processing::{ use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; -use tree_hash::TreeHash; use types::{ Address, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, MainnetEthSpec, MinimalEthSpec, ProposerPreparationData, Slot, @@ -515,16 +514,17 @@ pub async fn proposer_boost_re_org_test( } harness.advance_slot(); - let (block_a_root, block_a, state_a) = harness + let (block_a_root, block_a, mut state_a) = harness .add_block_at_slot(slot_a, harness.get_current_state()) .await .unwrap(); + let state_a_root = state_a.canonical_root().unwrap(); // Attest to block A during slot A. let (block_a_parent_votes, _) = harness.make_attestations_with_limit( &all_validators, &state_a, - state_a.canonical_root(), + state_a_root, block_a_root, slot_a, num_parent_votes, @@ -538,7 +538,7 @@ pub async fn proposer_boost_re_org_test( let (block_a_empty_votes, block_a_attesters) = harness.make_attestations_with_limit( &all_validators, &state_a, - state_a.canonical_root(), + state_a_root, block_a_root, slot_b, num_empty_votes, @@ -553,6 +553,7 @@ pub async fn proposer_boost_re_org_test( // Produce block B and process it halfway through the slot. let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; + let state_b_root = state_b.canonical_root().unwrap(); let block_b_root = block_b.0.canonical_root(); let obs_time = slot_clock.start_of(slot_b).unwrap() + slot_clock.slot_duration() / 2; @@ -570,7 +571,7 @@ pub async fn proposer_boost_re_org_test( let (block_b_head_votes, _) = harness.make_attestations_with_limit( &remaining_attesters, &state_b, - state_b.canonical_root(), + state_b_root, block_b_root.into(), slot_b, num_head_votes, @@ -774,32 +775,34 @@ pub async fn fork_choice_before_proposal() { let slot_d = slot_a + 3; let state_a = harness.get_current_state(); - let (block_b, state_b) = harness.make_block(state_a.clone(), slot_b).await; + let (block_b, mut state_b) = harness.make_block(state_a.clone(), slot_b).await; let block_root_b = harness .process_block(slot_b, block_b.0.canonical_root(), block_b) .await .unwrap(); + let state_root_b = state_b.canonical_root().unwrap(); // Create attestations to B but keep them in reserve until after C has been processed. let attestations_b = harness.make_attestations( &all_validators, &state_b, - state_b.tree_hash_root(), + state_root_b, block_root_b, slot_b, ); - let (block_c, state_c) = harness.make_block(state_a, slot_c).await; + let (block_c, mut state_c) = harness.make_block(state_a, slot_c).await; let block_root_c = harness .process_block(slot_c, block_c.0.canonical_root(), block_c.clone()) .await .unwrap(); + let state_root_c = state_c.canonical_root().unwrap(); // Create attestations to C from a small number of validators and process them immediately. let attestations_c = harness.make_attestations( &all_validators[..validator_count / 2], &state_c, - state_c.tree_hash_root(), + state_root_c, block_root_c, slot_c, ); @@ -890,9 +893,10 @@ async fn queue_attestations_from_http() { .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) .collect::>(); + let fork_name = tester.harness.spec.fork_name_at_slot::(attestation_slot); let attestation_future = tokio::spawn(async move { client - .post_beacon_pool_attestations(&attestations) + .post_beacon_pool_attestations_v2(&attestations, fork_name) .await .expect("attestations should be processed successfully") }); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 4213fd4ab8..633baaf6f4 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -10,7 +10,9 @@ use eth2::{ types::{ BlockId as CoreBlockId, ForkChoiceNode, ProduceBlockV3Response, StateId as CoreStateId, *, }, - BeaconNodeHttpClient, Error, StatusCode, Timeouts, + BeaconNodeHttpClient, Error, + Error::ServerMessage, + StatusCode, Timeouts, }; use execution_layer::test_utils::{ MockBuilder, Operation, DEFAULT_BUILDER_PAYLOAD_VALUE_WEI, DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI, @@ -807,6 +809,39 @@ impl ApiTester { self } + pub async fn post_beacon_states_validator_balances_unsupported_media_failure(self) -> Self { + for state_id in self.interesting_state_ids() { + for validator_indices in self.interesting_validator_indices() { + let validator_index_ids = validator_indices + .iter() + .cloned() + .map(|i| ValidatorId::Index(i)) + .collect::>(); + + let unsupported_media_response = self + .client + .post_beacon_states_validator_balances_with_ssz_header( + state_id.0, + validator_index_ids, + ) + .await; + + if let Err(unsupported_media_response) = unsupported_media_response { + match unsupported_media_response { + ServerMessage(error) => { + assert_eq!(error.code, 415) + } + _ => panic!("Should error with unsupported media response"), + } + } else { + panic!("Should error with unsupported media response"); + } + } + } + + self + } + pub async fn test_beacon_states_validator_balances(self) -> Self { for state_id in self.interesting_state_ids() { for validator_indices in self.interesting_validator_indices() { @@ -1633,7 +1668,7 @@ impl ApiTester { for block_id in self.interesting_block_ids() { let result = self .client - .get_beacon_blocks_attestations(block_id.0) + .get_beacon_blocks_attestations_v2(block_id.0) .await .unwrap() .map(|res| res.data); @@ -1664,9 +1699,9 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid_v1(mut self) -> Self { self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -1678,7 +1713,25 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attestations_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attestations_valid_v2(mut self) -> Self { + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + self.client + .post_beacon_pool_attestations_v2(self.attestations.as_slice(), fork_name) + .await + .unwrap(); + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attestation should be sent to network" + ); + + self + } + + pub async fn test_post_beacon_pool_attestations_invalid_v1(mut self) -> Self { let mut attestations = Vec::new(); for attestation in &self.attestations { let mut invalid_attestation = attestation.clone(); @@ -1691,7 +1744,7 @@ impl ApiTester { let err = self .client - .post_beacon_pool_attestations(attestations.as_slice()) + .post_beacon_pool_attestations_v1(attestations.as_slice()) .await .unwrap_err(); @@ -1714,6 +1767,48 @@ impl ApiTester { self } + pub async fn test_post_beacon_pool_attestations_invalid_v2(mut self) -> Self { + let mut attestations = Vec::new(); + for attestation in &self.attestations { + let mut invalid_attestation = attestation.clone(); + invalid_attestation.data_mut().slot += 1; + + // add both to ensure we only fail on invalid attestations + attestations.push(attestation.clone()); + attestations.push(invalid_attestation); + } + + let fork_name = self + .attestations + .first() + .map(|att| self.chain.spec.fork_name_at_slot::(att.data().slot)) + .unwrap(); + + let err_v2 = self + .client + .post_beacon_pool_attestations_v2(attestations.as_slice(), fork_name) + .await + .unwrap_err(); + + match err_v2 { + Error::ServerIndexedMessage(IndexedErrorMessage { + code, + message: _, + failures, + }) => { + assert_eq!(code, 400); + assert_eq!(failures.len(), self.attestations.len()); + } + _ => panic!("query did not fail correctly"), + } + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "if some attestations are valid, we should send them to the network" + ); + + self + } pub async fn test_get_beacon_light_client_bootstrap(self) -> Self { let block_id = BlockId(CoreBlockId::Finalized); @@ -1777,7 +1872,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attestations(self) -> Self { let result = self .client - .get_beacon_pool_attestations(None, None) + .get_beacon_pool_attestations_v1(None, None) .await .unwrap() .data; @@ -1787,12 +1882,20 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attestations_v2(None, None) + .await + .unwrap() + .data; + assert_eq!(result, expected); + self } - pub async fn test_post_beacon_pool_attester_slashings_valid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_valid_v1(mut self) -> Self { self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) .await .unwrap(); @@ -1804,7 +1907,25 @@ impl ApiTester { self } - pub async fn test_post_beacon_pool_attester_slashings_invalid(mut self) -> Self { + pub async fn test_post_beacon_pool_attester_slashings_valid_v2(mut self) -> Self { + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); + self.client + .post_beacon_pool_attester_slashings_v2(&self.attester_slashing, fork_name) + .await + .unwrap(); + + assert!( + self.network_rx.network_recv.recv().await.is_some(), + "valid attester slashing should be sent to network" + ); + + self + } + + pub async fn test_post_beacon_pool_attester_slashings_invalid_v1(mut self) -> Self { let mut slashing = self.attester_slashing.clone(); match &mut slashing { AttesterSlashing::Base(ref mut slashing) => { @@ -1816,7 +1937,35 @@ impl ApiTester { } self.client - .post_beacon_pool_attester_slashings(&slashing) + .post_beacon_pool_attester_slashings_v1(&slashing) + .await + .unwrap_err(); + + assert!( + self.network_rx.network_recv.recv().now_or_never().is_none(), + "invalid attester slashing should not be sent to network" + ); + + self + } + + pub async fn test_post_beacon_pool_attester_slashings_invalid_v2(mut self) -> Self { + let mut slashing = self.attester_slashing.clone(); + match &mut slashing { + AttesterSlashing::Base(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + AttesterSlashing::Electra(ref mut slashing) => { + slashing.attestation_1.data.slot += 1; + } + } + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(self.attester_slashing.attestation_1().data().slot); + self.client + .post_beacon_pool_attester_slashings_v2(&slashing, fork_name) .await .unwrap_err(); @@ -1831,7 +1980,7 @@ impl ApiTester { pub async fn test_get_beacon_pool_attester_slashings(self) -> Self { let result = self .client - .get_beacon_pool_attester_slashings() + .get_beacon_pool_attester_slashings_v1() .await .unwrap() .data; @@ -1840,6 +1989,14 @@ impl ApiTester { assert_eq!(result, expected); + let result = self + .client + .get_beacon_pool_attester_slashings_v2() + .await + .unwrap() + .data; + assert_eq!(result, expected); + self } @@ -3198,30 +3355,52 @@ impl ApiTester { } pub async fn test_get_validator_aggregate_attestation(self) -> Self { - let attestation = self + if self .chain - .head_beacon_block() - .message() - .body() - .attestations() - .next() - .unwrap() - .clone_as_attestation(); + .spec + .fork_name_at_slot::(self.chain.slot().unwrap()) + .electra_enabled() + { + for attestation in self.chain.naive_aggregation_pool.read().iter() { + let result = self + .client + .get_validator_aggregate_attestation_v2( + attestation.data().slot, + attestation.data().tree_hash_root(), + attestation.committee_index().unwrap(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - let result = self - .client - .get_validator_aggregate_attestation( - attestation.data().slot, - attestation.data().tree_hash_root(), - ) - .await - .unwrap() - .unwrap() - .data; + assert_eq!(&result, expected); + } + } else { + let attestation = self + .chain + .head_beacon_block() + .message() + .body() + .attestations() + .next() + .unwrap() + .clone_as_attestation(); + let result = self + .client + .get_validator_aggregate_attestation_v1( + attestation.data().slot, + attestation.data().tree_hash_root(), + ) + .await + .unwrap() + .unwrap() + .data; + let expected = attestation; - let expected = attestation; - - assert_eq!(result, expected); + assert_eq!(result, expected); + } self } @@ -3320,11 +3499,11 @@ impl ApiTester { ) } - pub async fn test_get_validator_aggregate_and_proofs_valid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_valid_v1(mut self) -> Self { let aggregate = self.get_aggregate().await; self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate]) .await .unwrap(); @@ -3333,7 +3512,7 @@ impl ApiTester { self } - pub async fn test_get_validator_aggregate_and_proofs_invalid(mut self) -> Self { + pub async fn test_get_validator_aggregate_and_proofs_invalid_v1(mut self) -> Self { let mut aggregate = self.get_aggregate().await; match &mut aggregate { SignedAggregateAndProof::Base(ref mut aggregate) => { @@ -3345,7 +3524,7 @@ impl ApiTester { } self.client - .post_validator_aggregate_and_proof::(&[aggregate]) + .post_validator_aggregate_and_proof_v1::(&[aggregate.clone()]) .await .unwrap_err(); @@ -3354,6 +3533,46 @@ impl ApiTester { self } + pub async fn test_get_validator_aggregate_and_proofs_valid_v2(mut self) -> Self { + let aggregate = self.get_aggregate().await; + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) + .await + .unwrap(); + + assert!(self.network_rx.network_recv.recv().await.is_some()); + + self + } + + pub async fn test_get_validator_aggregate_and_proofs_invalid_v2(mut self) -> Self { + let mut aggregate = self.get_aggregate().await; + match &mut aggregate { + SignedAggregateAndProof::Base(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + SignedAggregateAndProof::Electra(ref mut aggregate) => { + aggregate.message.aggregate.data.slot += 1; + } + } + + let fork_name = self + .chain + .spec + .fork_name_at_slot::(aggregate.message().aggregate().data().slot); + self.client + .post_validator_aggregate_and_proof_v2::(&[aggregate], fork_name) + .await + .unwrap_err(); + assert!(self.network_rx.network_recv.recv().now_or_never().is_none()); + + self + } + pub async fn test_get_validator_beacon_committee_subscriptions(mut self) -> Self { let subscription = BeaconCommitteeSubscription { validator_index: 0, @@ -3449,7 +3668,7 @@ impl ApiTester { pub async fn test_post_validator_register_validator_slashed(self) -> Self { // slash a validator self.client - .post_beacon_pool_attester_slashings(&self.attester_slashing) + .post_beacon_pool_attester_slashings_v1(&self.attester_slashing) .await .unwrap(); @@ -3562,7 +3781,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5202,7 +5421,7 @@ impl ApiTester { // Attest to the current slot self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5242,6 +5461,7 @@ impl ApiTester { EventTopic::Attestation, EventTopic::VoluntaryExit, EventTopic::Block, + EventTopic::BlockGossip, EventTopic::Head, EventTopic::FinalizedCheckpoint, EventTopic::AttesterSlashing, @@ -5257,7 +5477,7 @@ impl ApiTester { let expected_attestation_len = self.attestations.len(); self.client - .post_beacon_pool_attestations(self.attestations.as_slice()) + .post_beacon_pool_attestations_v1(self.attestations.as_slice()) .await .unwrap(); @@ -5357,10 +5577,20 @@ impl ApiTester { .await .unwrap(); - let block_events = poll_events(&mut events_future, 3, Duration::from_millis(10000)).await; + let expected_gossip = EventKind::BlockGossip(Box::new(BlockGossip { + slot: next_slot, + block: block_root, + })); + + let block_events = poll_events(&mut events_future, 4, Duration::from_millis(10000)).await; assert_eq!( block_events.as_slice(), - &[expected_block, expected_head, expected_finalized] + &[ + expected_gossip, + expected_block, + expected_head, + expected_finalized + ] ); // Test a reorg event @@ -5660,6 +5890,14 @@ async fn get_events_from_genesis() { .await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_unsupported_media_response() { + ApiTester::new() + .await + .post_beacon_states_validator_balances_unsupported_media_failure() + .await; +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn beacon_get() { ApiTester::new() @@ -5758,34 +5996,66 @@ async fn post_beacon_blocks_duplicate() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_valid() { +async fn beacon_pools_post_attestations_valid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attestations_valid() + .test_post_beacon_pool_attestations_valid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attestations_invalid() { +async fn beacon_pools_post_attestations_invalid_v1() { ApiTester::new() .await - .test_post_beacon_pool_attestations_invalid() + .test_post_beacon_pool_attestations_invalid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_valid() { +async fn beacon_pools_post_attestations_valid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_valid() + .test_post_beacon_pool_attestations_valid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn beacon_pools_post_attester_slashings_invalid() { +async fn beacon_pools_post_attestations_invalid_v2() { ApiTester::new() .await - .test_post_beacon_pool_attester_slashings_invalid() + .test_post_beacon_pool_attestations_invalid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_valid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_valid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_invalid_v1() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_valid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn beacon_pools_post_attester_slashings_invalid_v2() { + ApiTester::new() + .await + .test_post_beacon_pool_attester_slashings_invalid_v2() .await; } @@ -6113,36 +6383,70 @@ async fn get_validator_aggregate_attestation_with_skip_slots() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid() { +async fn get_validator_aggregate_and_proofs_valid_v1() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_valid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_valid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v1() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_valid() + .test_get_validator_aggregate_and_proofs_valid_v1() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid() { +async fn get_validator_aggregate_and_proofs_valid_v2() { ApiTester::new() .await - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_valid_v2() .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots() { +async fn get_validator_aggregate_and_proofs_valid_with_skip_slots_v2() { ApiTester::new() .await .skip_slots(E::slots_per_epoch() * 2) - .test_get_validator_aggregate_and_proofs_invalid() + .test_get_validator_aggregate_and_proofs_valid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_v1() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v1() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_invalid_v1() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_v2() { + ApiTester::new() + .await + .test_get_validator_aggregate_and_proofs_invalid_v2() + .await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn get_validator_aggregate_and_proofs_invalid_with_skip_slots_v2() { + ApiTester::new() + .await + .skip_slots(E::slots_per_epoch() * 2) + .test_get_validator_aggregate_and_proofs_invalid_v2() .await; } diff --git a/beacon_node/lighthouse_network/gossipsub/CHANGELOG.md b/beacon_node/lighthouse_network/gossipsub/CHANGELOG.md index 448e224cb6..7ec10af741 100644 --- a/beacon_node/lighthouse_network/gossipsub/CHANGELOG.md +++ b/beacon_node/lighthouse_network/gossipsub/CHANGELOG.md @@ -1,5 +1,8 @@ ## 0.5 Sigma Prime fork +- Implement IDONTWANT messages as per [spec](https://github.com/libp2p/specs/pull/548). + See [PR 5422](https://github.com/sigp/lighthouse/pull/5422) + - Attempt to publish to at least mesh_n peers when publishing a message when flood publish is disabled. See [PR 5357](https://github.com/sigp/lighthouse/pull/5357). - Drop `Publish` and `Forward` gossipsub stale messages when polling ConnectionHandler. diff --git a/beacon_node/lighthouse_network/gossipsub/Cargo.toml b/beacon_node/lighthouse_network/gossipsub/Cargo.toml index d8fa445e63..56c42d2992 100644 --- a/beacon_node/lighthouse_network/gossipsub/Cargo.toml +++ b/beacon_node/lighthouse_network/gossipsub/Cargo.toml @@ -24,6 +24,7 @@ futures = "0.3.30" futures-ticker = "0.0.3" futures-timer = "3.0.2" getrandom = "0.2.12" +hashlink.workspace = true hex_fmt = "0.3.0" libp2p = { version = "0.53", default-features = false } quick-protobuf = "0.8" diff --git a/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs b/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs index ccebb4e267..0a3b7a9f52 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/behaviour.rs @@ -31,6 +31,7 @@ use std::{ use futures::StreamExt; use futures_ticker::Ticker; +use hashlink::LinkedHashMap; use prometheus_client::registry::Registry; use rand::{seq::SliceRandom, thread_rng}; @@ -45,6 +46,8 @@ use libp2p::swarm::{ }; use web_time::{Instant, SystemTime}; +use crate::types::IDontWant; + use super::gossip_promises::GossipPromises; use super::handler::{Handler, HandlerEvent, HandlerIn}; use super::mcache::MessageCache; @@ -73,6 +76,12 @@ use std::{cmp::Ordering::Equal, fmt::Debug}; #[cfg(test)] mod tests; +/// IDONTWANT cache capacity. +const IDONTWANT_CAP: usize = 10_000; + +/// IDONTWANT timeout before removal. +const IDONTWANT_TIMEOUT: Duration = Duration::new(3, 0); + /// Determines if published messages should be signed or not. /// /// Without signing, a number of privacy preserving modes can be selected. @@ -304,9 +313,8 @@ pub struct Behaviour { /// discovery and not by PX). outbound_peers: HashSet, - /// Stores optional peer score data together with thresholds, decay interval and gossip - /// promises. - peer_score: Option<(PeerScore, PeerScoreThresholds, Ticker, GossipPromises)>, + /// Stores optional peer score data together with thresholds and decay interval. + peer_score: Option<(PeerScore, PeerScoreThresholds, Ticker)>, /// Counts the number of `IHAVE` received from each peer since the last heartbeat. count_received_ihave: HashMap, @@ -331,6 +339,9 @@ pub struct Behaviour { /// Tracks the numbers of failed messages per peer-id. failed_messages: HashMap, + + /// Tracks recently sent `IWANT` messages and checks if peers respond to them. + gossip_promises: GossipPromises, } impl Behaviour @@ -467,6 +478,7 @@ where subscription_filter, data_transform, failed_messages: Default::default(), + gossip_promises: Default::default(), }) } } @@ -919,7 +931,7 @@ where let interval = Ticker::new(params.decay_interval); let peer_score = PeerScore::new_with_message_delivery_time_callback(params, callback); - self.peer_score = Some((peer_score, threshold, interval, GossipPromises::default())); + self.peer_score = Some((peer_score, threshold, interval)); Ok(()) } @@ -1187,7 +1199,7 @@ where } fn score_below_threshold_from_scores( - peer_score: &Option<(PeerScore, PeerScoreThresholds, Ticker, GossipPromises)>, + peer_score: &Option<(PeerScore, PeerScoreThresholds, Ticker)>, peer_id: &PeerId, threshold: impl Fn(&PeerScoreThresholds) -> f64, ) -> (bool, f64) { @@ -1248,10 +1260,7 @@ where return false; } - self.peer_score - .as_ref() - .map(|(_, _, _, promises)| !promises.contains(id)) - .unwrap_or(true) + !self.gossip_promises.contains(id) }; for (topic, ids) in ihave_msgs { @@ -1298,13 +1307,11 @@ where iwant_ids_vec.truncate(iask); *iasked += iask; - if let Some((_, _, _, gossip_promises)) = &mut self.peer_score { - gossip_promises.add_promise( - *peer_id, - &iwant_ids_vec, - Instant::now() + self.config.iwant_followup_time(), - ); - } + self.gossip_promises.add_promise( + *peer_id, + &iwant_ids_vec, + Instant::now() + self.config.iwant_followup_time(), + ); if let Some(peer) = &mut self.connected_peers.get_mut(peer_id) { tracing::trace!( @@ -1369,6 +1376,11 @@ where "IWANT: Peer has asked for message too many times; ignoring request" ); } else if let Some(peer) = &mut self.connected_peers.get_mut(peer_id) { + if peer.dont_send.get(&id).is_some() { + tracing::debug!(%peer_id, message=%id, "Peer already sent IDONTWANT for this message"); + continue; + } + tracing::debug!(peer=%peer_id, "IWANT: Sending cached messages to peer"); if peer .sender @@ -1706,14 +1718,15 @@ where peer=%propagation_source, "Rejecting message from blacklisted peer" ); - if let Some((peer_score, .., gossip_promises)) = &mut self.peer_score { + self.gossip_promises + .reject_message(msg_id, &RejectReason::BlackListedPeer); + if let Some((peer_score, ..)) = &mut self.peer_score { peer_score.reject_message( propagation_source, msg_id, &raw_message.topic, RejectReason::BlackListedPeer, ); - gossip_promises.reject_message(msg_id, &RejectReason::BlackListedPeer); } return false; } @@ -1795,6 +1808,9 @@ where // Calculate the message id on the transformed data. let msg_id = self.config.message_id(&message); + // Broadcast IDONTWANT messages. + self.send_idontwant(&raw_message, &msg_id, propagation_source); + // Check the validity of the message // Peers get penalized if this message is invalid. We don't add it to the duplicate cache // and instead continually penalize peers that repeatedly send this message. @@ -1820,11 +1836,12 @@ where metrics.msg_recvd(&message.topic); } - // Tells score that message arrived (but is maybe not fully validated yet). // Consider the message as delivered for gossip promises. - if let Some((peer_score, .., gossip_promises)) = &mut self.peer_score { + self.gossip_promises.message_delivered(&msg_id); + + // Tells score that message arrived (but is maybe not fully validated yet). + if let Some((peer_score, ..)) = &mut self.peer_score { peer_score.validate_message(propagation_source, &msg_id, &message.topic); - gossip_promises.message_delivered(&msg_id); } // Add the message to our memcache @@ -1871,7 +1888,7 @@ where raw_message: &RawMessage, reject_reason: RejectReason, ) { - if let Some((peer_score, .., gossip_promises)) = &mut self.peer_score { + if let Some((peer_score, ..)) = &mut self.peer_score { if let Some(metrics) = self.metrics.as_mut() { metrics.register_invalid_message(&raw_message.topic); } @@ -1886,7 +1903,8 @@ where reject_reason, ); - gossip_promises.reject_message(&message_id, &reject_reason); + self.gossip_promises + .reject_message(&message_id, &reject_reason); } else { // The message is invalid, we reject it ignoring any gossip promises. If a peer is // advertising this message via an IHAVE and it's invalid it will be double @@ -1959,7 +1977,7 @@ where } // if the mesh needs peers add the peer to the mesh if !self.explicit_peers.contains(propagation_source) - && matches!(peer.kind, PeerKind::Gossipsubv1_1 | PeerKind::Gossipsub) + && peer.kind.is_gossipsub() && !Self::score_below_threshold_from_scores( &self.peer_score, propagation_source, @@ -2066,8 +2084,8 @@ where /// Applies penalties to peers that did not respond to our IWANT requests. fn apply_iwant_penalties(&mut self) { - if let Some((peer_score, .., gossip_promises)) = &mut self.peer_score { - for (peer, count) in gossip_promises.get_broken_promises() { + if let Some((peer_score, ..)) = &mut self.peer_score { + for (peer, count) in self.gossip_promises.get_broken_promises() { peer_score.add_penalty(&peer, count); if let Some(metrics) = self.metrics.as_mut() { metrics.register_score_penalty(Penalty::BrokenPromise); @@ -2288,7 +2306,7 @@ where && peers.len() > 1 && self.peer_score.is_some() { - if let Some((_, thresholds, _, _)) = &self.peer_score { + if let Some((_, thresholds, _)) = &self.peer_score { // Opportunistic grafting works as follows: we check the median score of peers // in the mesh; if this score is below the opportunisticGraftThreshold, we // select a few peers at random with score over the median. @@ -2381,7 +2399,7 @@ where for (topic_hash, peers) in self.fanout.iter_mut() { let mut to_remove_peers = Vec::new(); let publish_threshold = match &self.peer_score { - Some((_, thresholds, _, _)) => thresholds.publish_threshold, + Some((_, thresholds, _)) => thresholds.publish_threshold, _ => 0.0, }; for peer_id in peers.iter() { @@ -2474,6 +2492,17 @@ where } self.failed_messages.shrink_to_fit(); + // Flush stale IDONTWANTs. + for peer in self.connected_peers.values_mut() { + while let Some((_front, instant)) = peer.dont_send.front() { + if (*instant + IDONTWANT_TIMEOUT) >= Instant::now() { + break; + } else { + peer.dont_send.pop_front(); + } + } + } + tracing::debug!("Completed Heartbeat"); if let Some(metrics) = self.metrics.as_mut() { let duration = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX); @@ -2655,6 +2684,59 @@ where } } + /// Helper function which sends an IDONTWANT message to mesh\[topic\] peers. + fn send_idontwant( + &mut self, + message: &RawMessage, + msg_id: &MessageId, + propagation_source: &PeerId, + ) { + let Some(mesh_peers) = self.mesh.get(&message.topic) else { + return; + }; + + let iwant_peers = self.gossip_promises.peers_for_message(msg_id); + + let recipient_peers = mesh_peers + .iter() + .chain(iwant_peers.iter()) + .filter(|peer_id| { + *peer_id != propagation_source && Some(*peer_id) != message.source.as_ref() + }); + + for peer_id in recipient_peers { + let Some(peer) = self.connected_peers.get_mut(peer_id) else { + tracing::error!(peer = %peer_id, + "Could not IDONTWANT, peer doesn't exist in connected peer list"); + continue; + }; + + // Only gossipsub 1.2 peers support IDONTWANT. + if peer.kind != PeerKind::Gossipsubv1_2_beta { + continue; + } + + if peer + .sender + .idontwant(IDontWant { + message_ids: vec![msg_id.clone()], + }) + .is_err() + { + tracing::warn!(peer=%peer_id, "Send Queue full. Could not send IDONTWANT"); + + if let Some((peer_score, ..)) = &mut self.peer_score { + peer_score.failed_message_slow_peer(peer_id); + } + // Increment failed message count + self.failed_messages + .entry(*peer_id) + .or_default() + .non_priority += 1; + } + } + } + /// Helper function which forwards a message to mesh\[topic\] peers. /// /// Returns true if at least one peer was messaged. @@ -2708,6 +2790,11 @@ where if !recipient_peers.is_empty() { for peer_id in recipient_peers.iter() { if let Some(peer) = self.connected_peers.get_mut(peer_id) { + if peer.dont_send.get(msg_id).is_some() { + tracing::debug!(%peer_id, message=%msg_id, "Peer doesn't want message"); + continue; + } + tracing::debug!(%peer_id, message=%msg_id, "Sending message to peer"); if peer .sender @@ -3057,6 +3144,7 @@ where connections: vec![], sender: RpcSender::new(self.config.connection_handler_queue_len()), topics: Default::default(), + dont_send: LinkedHashMap::new(), }); // Add the new connection connected_peer.connections.push(connection_id); @@ -3087,6 +3175,7 @@ where connections: vec![], sender: RpcSender::new(self.config.connection_handler_queue_len()), topics: Default::default(), + dont_send: LinkedHashMap::new(), }); // Add the new connection connected_peer.connections.push(connection_id); @@ -3136,7 +3225,7 @@ where } HandlerEvent::MessageDropped(rpc) => { // Account for this in the scoring logic - if let Some((peer_score, _, _, _)) = &mut self.peer_score { + if let Some((peer_score, _, _)) = &mut self.peer_score { peer_score.failed_message_slow_peer(&propagation_source); } @@ -3245,6 +3334,24 @@ where peers, backoff, }) => prune_msgs.push((topic_hash, peers, backoff)), + ControlAction::IDontWant(IDontWant { message_ids }) => { + let Some(peer) = self.connected_peers.get_mut(&propagation_source) + else { + tracing::error!(peer = %propagation_source, + "Could not handle IDONTWANT, peer doesn't exist in connected peer list"); + continue; + }; + if let Some(metrics) = self.metrics.as_mut() { + metrics.register_idontwant(message_ids.len()); + } + for message_id in message_ids { + peer.dont_send.insert(message_id, Instant::now()); + // Don't exceed capacity. + if peer.dont_send.len() > IDONTWANT_CAP { + peer.dont_send.pop_front(); + } + } + } } } if !ihave_msgs.is_empty() { @@ -3270,7 +3377,7 @@ where } // update scores - if let Some((peer_score, _, interval, _)) = &mut self.peer_score { + if let Some((peer_score, _, interval)) = &mut self.peer_score { while let Poll::Ready(Some(_)) = interval.poll_next_unpin(cx) { peer_score.refresh_scores(); } @@ -3395,7 +3502,7 @@ fn get_random_peers_dynamic( .iter() .filter(|(_, p)| p.topics.contains(topic_hash)) .filter(|(peer_id, _)| f(peer_id)) - .filter(|(_, p)| p.kind == PeerKind::Gossipsub || p.kind == PeerKind::Gossipsubv1_1) + .filter(|(_, p)| p.kind.is_gossipsub()) .map(|(peer_id, _)| *peer_id) .collect::>(); diff --git a/beacon_node/lighthouse_network/gossipsub/src/behaviour/tests.rs b/beacon_node/lighthouse_network/gossipsub/src/behaviour/tests.rs index 2af0199ec9..a378198be3 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/behaviour/tests.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/behaviour/tests.rs @@ -31,13 +31,7 @@ use std::net::Ipv4Addr; use std::thread::sleep; #[derive(Default, Debug)] -struct InjectNodes -// TODO: remove trait bound Default when this issue is fixed: -// https://github.com/colin-kiegel/rust-derive-builder/issues/93 -where - D: DataTransform + Default + Clone + Send + 'static, - F: TopicSubscriptionFilter + Clone + Default + Send + 'static, -{ +struct InjectNodes { peer_no: usize, topics: Vec, to_subscribe: bool, @@ -47,6 +41,7 @@ where scoring: Option<(PeerScoreParams, PeerScoreThresholds)>, data_transform: D, subscription_filter: F, + peer_kind: Option, } impl InjectNodes @@ -94,7 +89,7 @@ where let empty = vec![]; for i in 0..self.peer_no { - let (peer, receiver) = add_peer( + let (peer, receiver) = add_peer_with_addr_and_kind( &mut gs, if self.to_subscribe { &topic_hashes @@ -103,6 +98,8 @@ where }, i < self.outbound, i < self.explicit, + Multiaddr::empty(), + self.peer_kind.clone().or(Some(PeerKind::Gossipsubv1_1)), ); peers.push(peer); receivers.insert(peer, receiver); @@ -151,6 +148,11 @@ where self.subscription_filter = subscription_filter; self } + + fn peer_kind(mut self, peer_kind: PeerKind) -> Self { + self.peer_kind = Some(peer_kind); + self + } } fn inject_nodes() -> InjectNodes @@ -235,6 +237,7 @@ where kind: kind.clone().unwrap_or(PeerKind::Floodsub), connections: vec![connection_id], topics: Default::default(), + dont_send: LinkedHashMap::new(), sender, }, ); @@ -620,6 +623,7 @@ fn test_join() { kind: PeerKind::Floodsub, connections: vec![connection_id], topics: Default::default(), + dont_send: LinkedHashMap::new(), sender, }, ); @@ -1015,6 +1019,7 @@ fn test_get_random_peers() { connections: vec![ConnectionId::new_unchecked(0)], topics: topics.clone(), sender: RpcSender::new(gs.config.connection_handler_queue_len()), + dont_send: LinkedHashMap::new(), }, ); } @@ -4580,9 +4585,9 @@ fn test_ignore_too_many_messages_in_ihave() { let (peer, receiver) = add_peer(&mut gs, &topics, false, false); receivers.insert(peer, receiver); - //peer has 20 messages + //peer has 30 messages let mut seq = 0; - let message_ids: Vec<_> = (0..20) + let message_ids: Vec<_> = (0..30) .map(|_| random_message(&mut seq, &topics)) .map(|msg| gs.data_transform.inbound_transform(msg).unwrap()) .map(|msg| config.message_id(&msg)) @@ -4624,7 +4629,7 @@ fn test_ignore_too_many_messages_in_ihave() { gs.heartbeat(); gs.handle_ihave( &peer, - vec![(topics[0].clone(), message_ids[10..20].to_vec())], + vec![(topics[0].clone(), message_ids[20..30].to_vec())], ); //we sent 10 iwant messages ids via a IWANT rpc. @@ -5236,3 +5241,191 @@ fn test_graft_without_subscribe() { // We unsubscribe from the topic. let _ = gs.unsubscribe(&Topic::new(topic)); } + +/// Test that a node sends IDONTWANT messages to the mesh peers +/// that run Gossipsub v1.2. +#[test] +fn sends_idontwant() { + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + .peer_no(5) + .topics(vec![String::from("topic1")]) + .to_subscribe(true) + .gs_config(Config::default()) + .explicit(1) + .peer_kind(PeerKind::Gossipsubv1_2_beta) + .create_network(); + + let local_id = PeerId::random(); + + let message = RawMessage { + source: Some(peers[1]), + data: vec![12], + sequence_number: Some(0), + topic: topic_hashes[0].clone(), + signature: None, + key: None, + validated: true, + }; + gs.handle_received_message(message.clone(), &local_id); + assert_eq!( + receivers + .into_iter() + .fold(0, |mut idontwants, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::IDontWant(_)) = non_priority.try_recv() { + assert_ne!(peer_id, peers[1]); + idontwants += 1; + } + } + idontwants + }), + 3, + "IDONTWANT was not sent" + ); +} + +/// Test that a node doesn't send IDONTWANT messages to the mesh peers +/// that don't run Gossipsub v1.2. +#[test] +fn doesnt_send_idontwant() { + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + .peer_no(5) + .topics(vec![String::from("topic1")]) + .to_subscribe(true) + .gs_config(Config::default()) + .explicit(1) + .peer_kind(PeerKind::Gossipsubv1_1) + .create_network(); + + let local_id = PeerId::random(); + + let message = RawMessage { + source: Some(peers[1]), + data: vec![12], + sequence_number: Some(0), + topic: topic_hashes[0].clone(), + signature: None, + key: None, + validated: true, + }; + gs.handle_received_message(message.clone(), &local_id); + assert_eq!( + receivers + .into_iter() + .fold(0, |mut idontwants, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if matches!(non_priority.try_recv(), Ok(RpcOut::IDontWant(_)) if peer_id != peers[1]) { + idontwants += 1; + } + } + idontwants + }), + 0, + "IDONTWANT were sent" + ); +} + +/// Test that a node doesn't forward a messages to the mesh peers +/// that sent IDONTWANT. +#[test] +fn doesnt_forward_idontwant() { + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + .peer_no(4) + .topics(vec![String::from("topic1")]) + .to_subscribe(true) + .gs_config(Config::default()) + .explicit(1) + .peer_kind(PeerKind::Gossipsubv1_2_beta) + .create_network(); + + let local_id = PeerId::random(); + + let raw_message = RawMessage { + source: Some(peers[1]), + data: vec![12], + sequence_number: Some(0), + topic: topic_hashes[0].clone(), + signature: None, + key: None, + validated: true, + }; + let message = gs + .data_transform + .inbound_transform(raw_message.clone()) + .unwrap(); + let message_id = gs.config.message_id(&message); + let peer = gs.connected_peers.get_mut(&peers[2]).unwrap(); + peer.dont_send.insert(message_id, Instant::now()); + + gs.handle_received_message(raw_message.clone(), &local_id); + assert_eq!( + receivers.into_iter().fold(0, |mut fwds, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { .. }) = non_priority.try_recv() { + assert_ne!(peer_id, peers[2]); + fwds += 1; + } + } + fwds + }), + 2, + "IDONTWANT was not sent" + ); +} + +/// Test that a node parses an +/// IDONTWANT message to the respective peer. +#[test] +fn parses_idontwant() { + let (mut gs, peers, _receivers, _topic_hashes) = inject_nodes1() + .peer_no(2) + .topics(vec![String::from("topic1")]) + .to_subscribe(true) + .gs_config(Config::default()) + .explicit(1) + .peer_kind(PeerKind::Gossipsubv1_2_beta) + .create_network(); + + let message_id = MessageId::new(&[0, 1, 2, 3]); + let rpc = Rpc { + messages: vec![], + subscriptions: vec![], + control_msgs: vec![ControlAction::IDontWant(IDontWant { + message_ids: vec![message_id.clone()], + })], + }; + gs.on_connection_handler_event( + peers[1], + ConnectionId::new_unchecked(0), + HandlerEvent::Message { + rpc, + invalid_messages: vec![], + }, + ); + let peer = gs.connected_peers.get_mut(&peers[1]).unwrap(); + assert!(peer.dont_send.get(&message_id).is_some()); +} + +/// Test that a node clears stale IDONTWANT messages. +#[test] +fn clear_stale_idontwant() { + let (mut gs, peers, _receivers, _topic_hashes) = inject_nodes1() + .peer_no(4) + .topics(vec![String::from("topic1")]) + .to_subscribe(true) + .gs_config(Config::default()) + .explicit(1) + .peer_kind(PeerKind::Gossipsubv1_2_beta) + .create_network(); + + let peer = gs.connected_peers.get_mut(&peers[2]).unwrap(); + peer.dont_send + .insert(MessageId::new(&[1, 2, 3, 4]), Instant::now()); + std::thread::sleep(Duration::from_secs(3)); + gs.heartbeat(); + let peer = gs.connected_peers.get_mut(&peers[2]).unwrap(); + assert!(peer.dont_send.is_empty()); +} diff --git a/beacon_node/lighthouse_network/gossipsub/src/generated/gossipsub/pb.rs b/beacon_node/lighthouse_network/gossipsub/src/generated/gossipsub/pb.rs index 9a074fd61f..24ac80d275 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/generated/gossipsub/pb.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/generated/gossipsub/pb.rs @@ -154,6 +154,7 @@ pub struct ControlMessage { pub iwant: Vec, pub graft: Vec, pub prune: Vec, + pub idontwant: Vec, } impl<'a> MessageRead<'a> for ControlMessage { @@ -165,6 +166,7 @@ impl<'a> MessageRead<'a> for ControlMessage { Ok(18) => msg.iwant.push(r.read_message::(bytes)?), Ok(26) => msg.graft.push(r.read_message::(bytes)?), Ok(34) => msg.prune.push(r.read_message::(bytes)?), + Ok(42) => msg.idontwant.push(r.read_message::(bytes)?), Ok(t) => { r.read_unknown(bytes, t)?; } Err(e) => return Err(e), } @@ -180,6 +182,7 @@ impl MessageWrite for ControlMessage { + self.iwant.iter().map(|s| 1 + sizeof_len((s).get_size())).sum::() + self.graft.iter().map(|s| 1 + sizeof_len((s).get_size())).sum::() + self.prune.iter().map(|s| 1 + sizeof_len((s).get_size())).sum::() + + self.idontwant.iter().map(|s| 1 + sizeof_len((s).get_size())).sum::() } fn write_message(&self, w: &mut Writer) -> Result<()> { @@ -187,6 +190,7 @@ impl MessageWrite for ControlMessage { for s in &self.iwant { w.write_with_tag(18, |w| w.write_message(s))?; } for s in &self.graft { w.write_with_tag(26, |w| w.write_message(s))?; } for s in &self.prune { w.write_with_tag(34, |w| w.write_message(s))?; } + for s in &self.idontwant { w.write_with_tag(42, |w| w.write_message(s))?; } Ok(()) } } @@ -331,6 +335,38 @@ impl MessageWrite for ControlPrune { } } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, Default, PartialEq, Clone)] +pub struct ControlIDontWant { + pub message_ids: Vec>, +} + +impl<'a> MessageRead<'a> for ControlIDontWant { + fn from_reader(r: &mut BytesReader, bytes: &'a [u8]) -> Result { + let mut msg = Self::default(); + while !r.is_eof() { + match r.next_tag(bytes) { + Ok(10) => msg.message_ids.push(r.read_bytes(bytes)?.to_owned()), + Ok(t) => { r.read_unknown(bytes, t)?; } + Err(e) => return Err(e), + } + } + Ok(msg) + } +} + +impl MessageWrite for ControlIDontWant { + fn get_size(&self) -> usize { + 0 + + self.message_ids.iter().map(|s| 1 + sizeof_len((s).len())).sum::() + } + + fn write_message(&self, w: &mut Writer) -> Result<()> { + for s in &self.message_ids { w.write_with_tag(10, |w| w.write_bytes(&**s))?; } + Ok(()) + } +} + #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Debug, Default, PartialEq, Clone)] pub struct PeerInfo { diff --git a/beacon_node/lighthouse_network/gossipsub/src/generated/rpc.proto b/beacon_node/lighthouse_network/gossipsub/src/generated/rpc.proto index 2ce12f3f37..e3b5888d2c 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/generated/rpc.proto +++ b/beacon_node/lighthouse_network/gossipsub/src/generated/rpc.proto @@ -28,6 +28,7 @@ message ControlMessage { repeated ControlIWant iwant = 2; repeated ControlGraft graft = 3; repeated ControlPrune prune = 4; + repeated ControlIDontWant idontwant = 5; } message ControlIHave { @@ -49,6 +50,10 @@ message ControlPrune { optional uint64 backoff = 3; // gossipsub v1.1 backoff time (in seconds) } +message ControlIDontWant { + repeated bytes message_ids = 1; +} + message PeerInfo { optional bytes peer_id = 1; optional bytes signed_peer_record = 2; diff --git a/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs b/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs index 2bfb20595a..3f72709245 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/gossip_promises.rs @@ -41,6 +41,14 @@ impl GossipPromises { self.promises.contains_key(message) } + ///Get the peers we sent IWANT the input message id. + pub(crate) fn peers_for_message(&self, message_id: &MessageId) -> Vec { + self.promises + .get(message_id) + .map(|peers| peers.keys().copied().collect()) + .unwrap_or_default() + } + /// Track a promise to deliver a message from a list of [`MessageId`]s we are requesting. pub(crate) fn add_promise(&mut self, peer: PeerId, messages: &[MessageId], expires: Instant) { for message_id in messages { diff --git a/beacon_node/lighthouse_network/gossipsub/src/metrics.rs b/beacon_node/lighthouse_network/gossipsub/src/metrics.rs index 91bcd5f54b..7e1cdac18b 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/metrics.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/metrics.rs @@ -179,6 +179,12 @@ pub(crate) struct Metrics { /// topic. A very high metric might indicate an underperforming network. topic_iwant_msgs: Family, + /// The number of times we have received an IDONTWANT control message. + idontwant_msgs: Counter, + + /// The number of msg_id's we have received in every IDONTWANT control message. + idontwant_msgs_ids: Counter, + /// The size of the priority queue. priority_queue_size: Histogram, /// The size of the non-priority queue. @@ -311,6 +317,27 @@ impl Metrics { "topic_iwant_msgs", "Number of times we have decided an IWANT is required for this topic" ); + + let idontwant_msgs = { + let metric = Counter::default(); + registry.register( + "idontwant_msgs", + "The number of times we have received an IDONTWANT control message", + metric.clone(), + ); + metric + }; + + let idontwant_msgs_ids = { + let metric = Counter::default(); + registry.register( + "idontwant_msgs_ids", + "The number of msg_id's we have received in every IDONTWANT control message.", + metric.clone(), + ); + metric + }; + let memcache_misses = { let metric = Counter::default(); registry.register( @@ -362,6 +389,8 @@ impl Metrics { heartbeat_duration, memcache_misses, topic_iwant_msgs, + idontwant_msgs, + idontwant_msgs_ids, priority_queue_size, non_priority_queue_size, } @@ -560,6 +589,12 @@ impl Metrics { } } + /// Register receiving an IDONTWANT msg for this topic. + pub(crate) fn register_idontwant(&mut self, msgs: usize) { + self.idontwant_msgs.inc(); + self.idontwant_msgs_ids.inc_by(msgs as u64); + } + /// Observes a heartbeat duration. pub(crate) fn observe_heartbeat_duration(&mut self, millis: u64) { self.heartbeat_duration.observe(millis as f64); diff --git a/beacon_node/lighthouse_network/gossipsub/src/protocol.rs b/beacon_node/lighthouse_network/gossipsub/src/protocol.rs index ba84ae0aa7..5611ae32c9 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/protocol.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/protocol.rs @@ -23,8 +23,8 @@ use super::handler::HandlerEvent; use super::rpc_proto::proto; use super::topic::TopicHash; use super::types::{ - ControlAction, Graft, IHave, IWant, MessageId, PeerInfo, PeerKind, Prune, RawMessage, Rpc, - Subscription, SubscriptionAction, + ControlAction, Graft, IDontWant, IHave, IWant, MessageId, PeerInfo, PeerKind, Prune, + RawMessage, Rpc, Subscription, SubscriptionAction, }; use super::ValidationError; use asynchronous_codec::{Decoder, Encoder, Framed}; @@ -40,6 +40,10 @@ use void::Void; pub(crate) const SIGNING_PREFIX: &[u8] = b"libp2p-pubsub:"; +pub(crate) const GOSSIPSUB_1_2_0_BETA_PROTOCOL: ProtocolId = ProtocolId { + protocol: StreamProtocol::new("/meshsub/1.2.0"), + kind: PeerKind::Gossipsubv1_2_beta, +}; pub(crate) const GOSSIPSUB_1_1_0_PROTOCOL: ProtocolId = ProtocolId { protocol: StreamProtocol::new("/meshsub/1.1.0"), kind: PeerKind::Gossipsubv1_1, @@ -69,7 +73,11 @@ impl Default for ProtocolConfig { Self { max_transmit_size: 65536, validation_mode: ValidationMode::Strict, - protocol_ids: vec![GOSSIPSUB_1_1_0_PROTOCOL, GOSSIPSUB_1_0_0_PROTOCOL], + protocol_ids: vec![ + GOSSIPSUB_1_2_0_BETA_PROTOCOL, + GOSSIPSUB_1_1_0_PROTOCOL, + GOSSIPSUB_1_0_0_PROTOCOL, + ], } } } @@ -476,10 +484,25 @@ impl Decoder for GossipsubCodec { })); } + let idontwant_msgs: Vec = rpc_control + .idontwant + .into_iter() + .map(|idontwant| { + ControlAction::IDontWant(IDontWant { + message_ids: idontwant + .message_ids + .into_iter() + .map(MessageId::from) + .collect::>(), + }) + }) + .collect(); + control_msgs.extend(ihave_msgs); control_msgs.extend(iwant_msgs); control_msgs.extend(graft_msgs); control_msgs.extend(prune_msgs); + control_msgs.extend(idontwant_msgs); } Ok(Some(HandlerEvent::Message { diff --git a/beacon_node/lighthouse_network/gossipsub/src/types.rs b/beacon_node/lighthouse_network/gossipsub/src/types.rs index 84bdfb786f..8df307d470 100644 --- a/beacon_node/lighthouse_network/gossipsub/src/types.rs +++ b/beacon_node/lighthouse_network/gossipsub/src/types.rs @@ -25,6 +25,7 @@ use async_channel::{Receiver, Sender}; use futures::stream::Peekable; use futures::{Future, Stream, StreamExt}; use futures_timer::Delay; +use hashlink::LinkedHashMap; use libp2p::identity::PeerId; use libp2p::swarm::ConnectionId; use prometheus_client::encoding::EncodeLabelValue; @@ -34,6 +35,7 @@ use std::fmt::Debug; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::task::{Context, Poll}; +use std::time::Instant; use std::{fmt, pin::Pin}; use web_time::Duration; @@ -121,11 +123,16 @@ pub(crate) struct PeerConnections { pub(crate) sender: RpcSender, /// Subscribed topics. pub(crate) topics: BTreeSet, + /// Don't send messages. + pub(crate) dont_send: LinkedHashMap, } /// Describes the types of peers that can exist in the gossipsub context. #[derive(Debug, Clone, PartialEq, Hash, EncodeLabelValue, Eq)] +#[allow(non_camel_case_types)] pub enum PeerKind { + /// A gossipsub 1.2 peer. + Gossipsubv1_2_beta, /// A gossipsub 1.1 peer. Gossipsubv1_1, /// A gossipsub 1.0 peer. @@ -136,6 +143,16 @@ pub enum PeerKind { NotSupported, } +impl PeerKind { + /// Returns true if peer speaks any gossipsub version. + pub(crate) fn is_gossipsub(&self) -> bool { + matches!( + self, + Self::Gossipsubv1_2_beta | Self::Gossipsubv1_1 | Self::Gossipsub + ) + } +} + /// A message received by the gossipsub system and stored locally in caches.. #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub struct RawMessage { @@ -257,6 +274,8 @@ pub enum ControlAction { Graft(Graft), /// The node has been removed from the mesh - Prune control message. Prune(Prune), + /// The node requests us to not forward message ids (peer_id + sequence _number) - IDontWant control message. + IDontWant(IDontWant), } /// Node broadcasts known messages per topic - IHave control message. @@ -293,6 +312,13 @@ pub struct Prune { pub(crate) backoff: Option, } +/// The node requests us to not forward message ids - IDontWant control message. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct IDontWant { + /// A list of known message ids. + pub(crate) message_ids: Vec, +} + /// A Gossipsub RPC message sent. #[derive(Debug)] pub enum RpcOut { @@ -314,6 +340,8 @@ pub enum RpcOut { IHave(IHave), /// Send a IWant control message. IWant(IWant), + /// Send a IDontWant control message. + IDontWant(IDontWant), } impl RpcOut { @@ -374,6 +402,7 @@ impl From for proto::RPC { iwant: vec![], graft: vec![], prune: vec![], + idontwant: vec![], }), }, RpcOut::IWant(IWant { message_ids }) => proto::RPC { @@ -386,6 +415,7 @@ impl From for proto::RPC { }], graft: vec![], prune: vec![], + idontwant: vec![], }), }, RpcOut::Graft(Graft { topic_hash }) => proto::RPC { @@ -398,6 +428,7 @@ impl From for proto::RPC { topic_id: Some(topic_hash.into_string()), }], prune: vec![], + idontwant: vec![], }), }, RpcOut::Prune(Prune { @@ -424,9 +455,23 @@ impl From for proto::RPC { .collect(), backoff, }], + idontwant: vec![], }), } } + RpcOut::IDontWant(IDontWant { message_ids }) => proto::RPC { + publish: Vec::new(), + subscriptions: Vec::new(), + control: Some(proto::ControlMessage { + ihave: vec![], + iwant: vec![], + graft: vec![], + prune: vec![], + idontwant: vec![proto::ControlIDontWant { + message_ids: message_ids.into_iter().map(|msg_id| msg_id.0).collect(), + }], + }), + }, } } } @@ -485,6 +530,7 @@ impl From for proto::RPC { iwant: Vec::new(), graft: Vec::new(), prune: Vec::new(), + idontwant: Vec::new(), }; let empty_control_msg = rpc.control_msgs.is_empty(); @@ -533,6 +579,12 @@ impl From for proto::RPC { }; control.prune.push(rpc_prune); } + ControlAction::IDontWant(IDontWant { message_ids }) => { + let rpc_idontwant = proto::ControlIDontWant { + message_ids: message_ids.into_iter().map(|msg_id| msg_id.0).collect(), + }; + control.idontwant.push(rpc_idontwant); + } } } @@ -571,6 +623,7 @@ impl PeerKind { Self::Floodsub => "Floodsub", Self::Gossipsub => "Gossipsub v1.0", Self::Gossipsubv1_1 => "Gossipsub v1.1", + Self::Gossipsubv1_2_beta => "Gossipsub v1.2-beta", } } } @@ -657,6 +710,15 @@ impl RpcSender { .map_err(|err| err.into_inner()) } + /// Send a `RpcOut::IWant` message to the `RpcReceiver` + /// this is low priority, if the queue is full an Err is returned. + #[allow(clippy::result_large_err)] + pub(crate) fn idontwant(&mut self, idontwant: IDontWant) -> Result<(), RpcOut> { + self.non_priority_sender + .try_send(RpcOut::IDontWant(idontwant)) + .map_err(|err| err.into_inner()) + } + /// Send a `RpcOut::Subscribe` message to the `RpcReceiver` /// this is high priority. pub(crate) fn subscribe(&mut self, topic: TopicHash) { diff --git a/beacon_node/lighthouse_network/src/rpc/config.rs b/beacon_node/lighthouse_network/src/rpc/config.rs index 08b81c7eae..d17fa112a1 100644 --- a/beacon_node/lighthouse_network/src/rpc/config.rs +++ b/beacon_node/lighthouse_network/src/rpc/config.rs @@ -103,8 +103,13 @@ impl RateLimiterConfig { pub const DEFAULT_GOODBYE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_BLOCKS_BY_RANGE_QUOTA: Quota = Quota::n_every(1024, 10); pub const DEFAULT_BLOCKS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); - pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(768, 10); - pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(128, 10); + // `BlocksByRange` and `BlobsByRange` are sent together during range sync. + // It makes sense for blocks and blobs quotas to be equivalent in terms of the number of blocks: + // 1024 blocks * 6 max blobs per block. + // This doesn't necessarily mean that we are sending this many blobs, because the quotas are + // measured against the maximum request size. + pub const DEFAULT_BLOBS_BY_RANGE_QUOTA: Quota = Quota::n_every(6144, 10); + pub const DEFAULT_BLOBS_BY_ROOT_QUOTA: Quota = Quota::n_every(768, 10); pub const DEFAULT_LIGHT_CLIENT_BOOTSTRAP_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_OPTIMISTIC_UPDATE_QUOTA: Quota = Quota::one_every(10); pub const DEFAULT_LIGHT_CLIENT_FINALITY_UPDATE_QUOTA: Quota = Quota::one_every(10); diff --git a/beacon_node/lighthouse_network/src/rpc/self_limiter.rs b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs index 115ae45a98..77caecb16d 100644 --- a/beacon_node/lighthouse_network/src/rpc/self_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/self_limiter.rs @@ -212,7 +212,7 @@ mod tests { use crate::rpc::rate_limiter::Quota; use crate::rpc::self_limiter::SelfRateLimiter; use crate::rpc::{OutboundRequest, Ping, Protocol}; - use crate::service::api_types::RequestId; + use crate::service::api_types::{AppRequestId, RequestId, SyncRequestId}; use libp2p::PeerId; use std::time::Duration; use types::MainnetEthSpec; @@ -225,15 +225,17 @@ mod tests { ping_quota: Quota::n_every(1, 2), ..Default::default() }); - let mut limiter: SelfRateLimiter, MainnetEthSpec> = + let mut limiter: SelfRateLimiter = SelfRateLimiter::new(config, log).unwrap(); let peer_id = PeerId::random(); - for i in 1..=5 { + for i in 1..=5u32 { let _ = limiter.allows( peer_id, - RequestId::Application(i), - OutboundRequest::Ping(Ping { data: i }), + RequestId::Application(AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { + id: i, + })), + OutboundRequest::Ping(Ping { data: i as u64 }), ); } @@ -246,8 +248,13 @@ mod tests { // Check that requests in the queue are ordered in the sequence 2, 3, 4, 5. let mut iter = queue.iter(); - for i in 2..=5 { - assert_eq!(iter.next().unwrap().request_id, RequestId::Application(i)); + for i in 2..=5u32 { + assert!(matches!( + iter.next().unwrap().request_id, + RequestId::Application(AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { + id, + })) if id == i + )); } assert_eq!(limiter.ready_requests.len(), 0); @@ -267,7 +274,12 @@ mod tests { // Check that requests in the queue are ordered in the sequence 3, 4, 5. let mut iter = queue.iter(); for i in 3..=5 { - assert_eq!(iter.next().unwrap().request_id, RequestId::Application(i)); + assert!(matches!( + iter.next().unwrap().request_id, + RequestId::Application(AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { + id + })) if id == i + )); } assert_eq!(limiter.ready_requests.len(), 1); diff --git a/beacon_node/lighthouse_network/src/service/api_types.rs b/beacon_node/lighthouse_network/src/service/api_types.rs index 2ea4150248..376ac34dee 100644 --- a/beacon_node/lighthouse_network/src/service/api_types.rs +++ b/beacon_node/lighthouse_network/src/service/api_types.rs @@ -19,10 +19,36 @@ use crate::rpc::{ /// Identifier of requests sent by a peer. pub type PeerRequestId = (ConnectionId, SubstreamId); -/// Identifier of a request. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum RequestId { - Application(AppReqId), +pub type Id = u32; + +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +pub struct SingleLookupReqId { + pub lookup_id: Id, + pub req_id: Id, +} + +/// Id of rpc requests sent by sync to the network. +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +pub enum SyncRequestId { + /// Request searching for a block given a hash. + SingleBlock { id: SingleLookupReqId }, + /// Request searching for a set of blobs given a hash. + SingleBlob { id: SingleLookupReqId }, + /// Range request that is composed by both a block range request and a blob range request. + RangeBlockAndBlobs { id: Id }, +} + +/// Application level requests sent to the network. +#[derive(Debug, Clone, Copy)] +pub enum AppRequestId { + Sync(SyncRequestId), + Router, +} + +/// Global identifier of a request. +#[derive(Debug, Clone, Copy)] +pub enum RequestId { + Application(AppRequestId), Internal, } @@ -142,7 +168,7 @@ impl std::convert::From> for RPCCodedResponse { } } -impl slog::Value for RequestId { +impl slog::Value for RequestId { fn serialize( &self, record: &slog::Record, diff --git a/beacon_node/lighthouse_network/src/service/behaviour.rs b/beacon_node/lighthouse_network/src/service/behaviour.rs index 90121ffbfb..ab2e43630b 100644 --- a/beacon_node/lighthouse_network/src/service/behaviour.rs +++ b/beacon_node/lighthouse_network/src/service/behaviour.rs @@ -1,6 +1,6 @@ use crate::discovery::Discovery; use crate::peer_manager::PeerManager; -use crate::rpc::{ReqId, RPC}; +use crate::rpc::RPC; use crate::types::SnappyTransform; use libp2p::identify; @@ -16,9 +16,8 @@ pub type SubscriptionFilter = pub type Gossipsub = gossipsub::Behaviour; #[derive(NetworkBehaviour)] -pub(crate) struct Behaviour +pub(crate) struct Behaviour where - AppReqId: ReqId, E: EthSpec, { /// Keep track of active and pending connections to enforce hard limits. @@ -26,7 +25,7 @@ where /// The peer manager that keeps track of peer's reputation and status. pub peer_manager: PeerManager, /// The Eth2 RPC specified in the wire-0 protocol. - pub eth2_rpc: RPC, E>, + pub eth2_rpc: RPC, /// Discv5 Discovery protocol. pub discovery: Discovery, /// Keep regular connection to peers and disconnect if absent. diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index dbf7c38226..2868c616bd 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -21,7 +21,7 @@ use crate::types::{ use crate::EnrExt; use crate::Eth2Enr; use crate::{error, metrics, Enr, NetworkGlobals, PubsubMessage, TopicHash}; -use api_types::{PeerRequestId, Request, RequestId, Response}; +use api_types::{AppRequestId, PeerRequestId, Request, RequestId, Response}; use futures::stream::StreamExt; use gossipsub::{ IdentTopic as Topic, MessageAcceptance, MessageAuthenticity, MessageId, PublishError, @@ -57,7 +57,7 @@ const MAX_IDENTIFY_ADDRESSES: usize = 10; /// The types of events than can be obtained from polling the behaviour. #[derive(Debug)] -pub enum NetworkEvent { +pub enum NetworkEvent { /// We have successfully dialed and connected to a peer. PeerConnectedOutgoing(PeerId), /// A peer has successfully dialed and connected to us. @@ -67,7 +67,7 @@ pub enum NetworkEvent { /// An RPC Request that was sent failed. RPCFailed { /// The id of the failed request. - id: AppReqId, + id: AppRequestId, /// The peer to which this request was sent. peer_id: PeerId, /// The error of the failed request. @@ -85,7 +85,7 @@ pub enum NetworkEvent { /// Peer that sent the response. peer_id: PeerId, /// Id of the request to which the peer is responding. - id: AppReqId, + id: AppRequestId, /// Response the peer sent. response: Response, }, @@ -108,8 +108,8 @@ pub enum NetworkEvent { /// Builds the network behaviour that manages the core protocols of eth2. /// This core behaviour is managed by `Behaviour` which adds peer management to all core /// behaviours. -pub struct Network { - swarm: libp2p::swarm::Swarm>, +pub struct Network { + swarm: libp2p::swarm::Swarm>, /* Auxiliary Fields */ /// A collections of variables accessible outside the network service. network_globals: Arc>, @@ -132,7 +132,7 @@ pub struct Network { } /// Implements the combined behaviour for the libp2p service. -impl Network { +impl Network { pub async fn new( executor: task_executor::TaskExecutor, mut ctx: ServiceContext<'_>, @@ -592,7 +592,7 @@ impl Network { &mut self.swarm.behaviour_mut().gossipsub } /// The Eth2 RPC specified in the wire-0 protocol. - pub fn eth2_rpc_mut(&mut self) -> &mut RPC, E> { + pub fn eth2_rpc_mut(&mut self) -> &mut RPC { &mut self.swarm.behaviour_mut().eth2_rpc } /// Discv5 Discovery protocol. @@ -613,7 +613,7 @@ impl Network { &self.swarm.behaviour().gossipsub } /// The Eth2 RPC specified in the wire-0 protocol. - pub fn eth2_rpc(&self) -> &RPC, E> { + pub fn eth2_rpc(&self) -> &RPC { &self.swarm.behaviour().eth2_rpc } /// Discv5 Discovery protocol. @@ -920,9 +920,9 @@ impl Network { pub fn send_request( &mut self, peer_id: PeerId, - request_id: AppReqId, + request_id: AppRequestId, request: Request, - ) -> Result<(), (AppReqId, RPCError)> { + ) -> Result<(), (AppRequestId, RPCError)> { // Check if the peer is connected before sending an RPC request if !self.swarm.is_connected(&peer_id) { return Err((request_id, RPCError::Disconnected)); @@ -1157,10 +1157,10 @@ impl Network { #[must_use = "return the response"] fn build_response( &mut self, - id: RequestId, + id: RequestId, peer_id: PeerId, response: Response, - ) -> Option> { + ) -> Option> { match id { RequestId::Application(id) => Some(NetworkEvent::ResponseReceived { peer_id, @@ -1178,7 +1178,7 @@ impl Network { id: PeerRequestId, peer_id: PeerId, request: Request, - ) -> NetworkEvent { + ) -> NetworkEvent { // Increment metrics match &request { Request::Status(_) => { @@ -1244,7 +1244,7 @@ impl Network { /* Sub-behaviour event handling functions */ /// Handle a gossipsub event. - fn inject_gs_event(&mut self, event: gossipsub::Event) -> Option> { + fn inject_gs_event(&mut self, event: gossipsub::Event) -> Option> { match event { gossipsub::Event::Message { propagation_source, @@ -1383,10 +1383,7 @@ impl Network { } /// Handle an RPC event. - fn inject_rpc_event( - &mut self, - event: RPCMessage, E>, - ) -> Option> { + fn inject_rpc_event(&mut self, event: RPCMessage) -> Option> { let peer_id = event.peer_id; // Do not permit Inbound events from peers that are being disconnected, or RPC requests. @@ -1619,10 +1616,7 @@ impl Network { } /// Handle an identify event. - fn inject_identify_event( - &mut self, - event: identify::Event, - ) -> Option> { + fn inject_identify_event(&mut self, event: identify::Event) -> Option> { match event { identify::Event::Received { peer_id, mut info } => { if info.listen_addrs.len() > MAX_IDENTIFY_ADDRESSES { @@ -1643,7 +1637,7 @@ impl Network { } /// Handle a peer manager event. - fn inject_pm_event(&mut self, event: PeerManagerEvent) -> Option> { + fn inject_pm_event(&mut self, event: PeerManagerEvent) -> Option> { match event { PeerManagerEvent::PeerConnectedIncoming(peer_id) => { Some(NetworkEvent::PeerConnectedIncoming(peer_id)) @@ -1747,7 +1741,7 @@ impl Network { /// Poll the p2p networking stack. /// /// This will poll the swarm and do maintenance routines. - pub fn poll_network(&mut self, cx: &mut Context) -> Poll> { + pub fn poll_network(&mut self, cx: &mut Context) -> Poll> { while let Poll::Ready(Some(swarm_event)) = self.swarm.poll_next_unpin(cx) { let maybe_event = match swarm_event { SwarmEvent::Behaviour(behaviour_event) => match behaviour_event { @@ -1889,7 +1883,7 @@ impl Network { Poll::Pending } - pub async fn next_event(&mut self) -> NetworkEvent { + pub async fn next_event(&mut self) -> NetworkEvent { futures::future::poll_fn(|cx| self.poll_network(cx)).await } } diff --git a/beacon_node/lighthouse_network/src/service/utils.rs b/beacon_node/lighthouse_network/src/service/utils.rs index c6dbee1d2e..80187efc10 100644 --- a/beacon_node/lighthouse_network/src/service/utils.rs +++ b/beacon_node/lighthouse_network/src/service/utils.rs @@ -5,6 +5,7 @@ use crate::types::{ }; use crate::{GossipTopic, NetworkConfig}; use futures::future::Either; +use gossipsub; use libp2p::core::{multiaddr::Multiaddr, muxing::StreamMuxerBox, transport::Boxed}; use libp2p::identity::{secp256k1, Keypair}; use libp2p::{core, noise, yamux, PeerId, Transport}; diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 32e3a03466..25431226ca 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -13,7 +13,6 @@ use types::{ }; type E = MinimalEthSpec; -type ReqId = usize; use tempfile::Builder as TempBuilder; @@ -44,14 +43,14 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { } pub struct Libp2pInstance( - LibP2PService, + LibP2PService, #[allow(dead_code)] // This field is managed for lifetime purposes may not be used directly, hence the `#[allow(dead_code)]` attribute. async_channel::Sender<()>, ); impl std::ops::Deref for Libp2pInstance { - type Target = LibP2PService; + type Target = LibP2PService; fn deref(&self) -> &Self::Target { &self.0 } @@ -125,7 +124,7 @@ pub async fn build_libp2p_instance( } #[allow(dead_code)] -pub fn get_enr(node: &LibP2PService) -> Enr { +pub fn get_enr(node: &LibP2PService) -> Enr { node.local_enr() } diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 527b853dc3..12a1c59393 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -4,6 +4,7 @@ mod common; use common::Protocol; use lighthouse_network::rpc::methods::*; +use lighthouse_network::service::api_types::AppRequestId; use lighthouse_network::{rpc::max_rpc_size, NetworkEvent, ReportSource, Request, Response}; use slog::{debug, warn, Level}; use ssz::Encode; @@ -99,12 +100,12 @@ fn test_tcp_status_rpc() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, 10, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { peer_id: _, - id: 10, + id: AppRequestId::Router, response, } => { // Should receive the RPC response @@ -196,7 +197,6 @@ fn test_tcp_blocks_by_range_chunked_rpc() { // keep count of the number of messages received let mut messages_received = 0; - let request_id = messages_to_send as usize; // build the sender future let sender_future = async { loop { @@ -205,7 +205,7 @@ fn test_tcp_blocks_by_range_chunked_rpc() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, request_id, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { @@ -323,7 +323,6 @@ fn test_blobs_by_range_chunked_rpc() { // keep count of the number of messages received let mut messages_received = 0; - let request_id = messages_to_send as usize; // build the sender future let sender_future = async { loop { @@ -332,7 +331,7 @@ fn test_blobs_by_range_chunked_rpc() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, request_id, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { @@ -433,7 +432,6 @@ fn test_tcp_blocks_by_range_over_limit() { let rpc_response_bellatrix_large = Response::BlocksByRange(Some(Arc::new(signed_full_block))); - let request_id = messages_to_send as usize; // build the sender future let sender_future = async { loop { @@ -442,12 +440,12 @@ fn test_tcp_blocks_by_range_over_limit() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, request_id, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } // The request will fail because the sender will refuse to send anything > MAX_RPC_SIZE NetworkEvent::RPCFailed { id, .. } => { - assert_eq!(id, request_id); + assert!(matches!(id, AppRequestId::Router)); return; } _ => {} // Ignore other behaviour events @@ -528,7 +526,6 @@ fn test_tcp_blocks_by_range_chunked_rpc_terminates_correctly() { // keep count of the number of messages received let mut messages_received: u64 = 0; - let request_id = messages_to_send as usize; // build the sender future let sender_future = async { loop { @@ -537,7 +534,7 @@ fn test_tcp_blocks_by_range_chunked_rpc_terminates_correctly() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, request_id, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { @@ -668,12 +665,12 @@ fn test_tcp_blocks_by_range_single_empty_rpc() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, 10, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { peer_id: _, - id: 10, + id: AppRequestId::Router, response, } => match response { Response::BlocksByRange(Some(_)) => { @@ -793,12 +790,12 @@ fn test_tcp_blocks_by_root_chunked_rpc() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, 6, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { peer_id: _, - id: 6, + id: AppRequestId::Router, response, } => match response { Response::BlocksByRoot(Some(_)) => { @@ -926,12 +923,12 @@ fn test_tcp_blocks_by_root_chunked_rpc_terminates_correctly() { // Send a STATUS message debug!(log, "Sending RPC"); sender - .send_request(peer_id, 10, rpc_request.clone()) + .send_request(peer_id, AppRequestId::Router, rpc_request.clone()) .unwrap(); } NetworkEvent::ResponseReceived { peer_id: _, - id: 10, + id: AppRequestId::Router, response, } => { debug!(log, "Sender received a response"); diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index 32e57da8ae..f0dba8d965 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -237,6 +237,36 @@ lazy_static! { "Number of Syncing chains in range, per range type", &["range_type"] ); + pub static ref SYNCING_CHAINS_REMOVED: Result = try_create_int_counter_vec( + "sync_range_removed_chains_total", + "Total count of range syncing chains removed per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_ADDED: Result = try_create_int_counter_vec( + "sync_range_added_chains_total", + "Total count of range syncing chains added per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_DROPPED_BLOCKS: Result = try_create_int_counter_vec( + "sync_range_chains_dropped_blocks_total", + "Total count of dropped blocks when removing a syncing chain per range type", + &["range_type"] + ); + pub static ref SYNCING_CHAINS_IGNORED_BLOCKS: Result = try_create_int_counter_vec( + "sync_range_chains_ignored_blocks_total", + "Total count of ignored blocks when processing a syncing chain batch per chain type", + &["chain_type"] + ); + pub static ref SYNCING_CHAINS_PROCESSED_BATCHES: Result = try_create_int_counter_vec( + "sync_range_chains_processed_batches_total", + "Total count of processed batches in a syncing chain batch per chain type", + &["chain_type"] + ); + pub static ref SYNCING_CHAIN_BATCH_AWAITING_PROCESSING: Result = try_create_histogram_with_buckets( + "sync_range_chain_batch_awaiting_processing_seconds", + "Time range sync batches spend in AwaitingProcessing state", + Ok(vec![0.01,0.02,0.05,0.1,0.2,0.5,1.0,2.0,5.0,10.0,20.0]) + ); pub static ref SYNC_SINGLE_BLOCK_LOOKUPS: Result = try_create_int_gauge( "sync_single_block_lookups", "Number of single block lookups underway" @@ -262,6 +292,16 @@ lazy_static! { "sync_lookups_stuck_total", "Total count of sync lookups that are stuck and dropped", ); + pub static ref SYNC_ACTIVE_NETWORK_REQUESTS: Result = try_create_int_gauge_vec( + "sync_active_network_requests", + "Current count of active network requests from sync", + &["type"], + ); + pub static ref SYNC_UNKNOWN_NETWORK_REQUESTS: Result = try_create_int_counter_vec( + "sync_unknwon_network_request", + "Total count of network messages received for unknown active requests", + &["type"], + ); /* * Block Delay Metrics diff --git a/beacon_node/network/src/network_beacon_processor/sync_methods.rs b/beacon_node/network/src/network_beacon_processor/sync_methods.rs index acd02ab6ad..68bd674514 100644 --- a/beacon_node/network/src/network_beacon_processor/sync_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/sync_methods.rs @@ -326,7 +326,7 @@ impl NetworkBeaconProcessor { .process_blocks(downloaded_blocks.iter(), notify_execution_layer) .await { - (_, Ok(_)) => { + (imported_blocks, Ok(_)) => { debug!(self.log, "Batch processed"; "batch_epoch" => epoch, "first_block_slot" => start_slot, @@ -335,7 +335,8 @@ impl NetworkBeaconProcessor { "processed_blocks" => sent_blocks, "service"=> "sync"); BatchProcessResult::Success { - was_non_empty: sent_blocks > 0, + sent_blocks, + imported_blocks, } } (imported_blocks, Err(e)) => { @@ -349,7 +350,7 @@ impl NetworkBeaconProcessor { "service" => "sync"); match e.peer_action { Some(penalty) => BatchProcessResult::FaultyFailure { - imported_blocks: imported_blocks > 0, + imported_blocks, penalty, }, None => BatchProcessResult::NonFaultyFailure, @@ -368,7 +369,7 @@ impl NetworkBeaconProcessor { .sum::(); match self.process_backfill_blocks(downloaded_blocks) { - (_, Ok(_)) => { + (imported_blocks, Ok(_)) => { debug!(self.log, "Backfill batch processed"; "batch_epoch" => epoch, "first_block_slot" => start_slot, @@ -377,7 +378,8 @@ impl NetworkBeaconProcessor { "processed_blobs" => n_blobs, "service"=> "sync"); BatchProcessResult::Success { - was_non_empty: sent_blocks > 0, + sent_blocks, + imported_blocks, } } (_, Err(e)) => { @@ -390,7 +392,7 @@ impl NetworkBeaconProcessor { "service" => "sync"); match e.peer_action { Some(penalty) => BatchProcessResult::FaultyFailure { - imported_blocks: false, + imported_blocks: 0, penalty, }, None => BatchProcessResult::NonFaultyFailure, diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 1937fc11cf..e125c13f4c 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -7,9 +7,8 @@ use crate::error; use crate::network_beacon_processor::{InvalidBlockStorage, NetworkBeaconProcessor}; -use crate::service::{NetworkMessage, RequestId}; +use crate::service::NetworkMessage; use crate::status::status_message; -use crate::sync::manager::RequestId as SyncId; use crate::sync::SyncMessage; use beacon_chain::{BeaconChain, BeaconChainTypes}; use beacon_processor::{ @@ -18,6 +17,7 @@ use beacon_processor::{ use futures::prelude::*; use lighthouse_network::rpc::*; use lighthouse_network::{ + service::api_types::{AppRequestId, SyncRequestId}, MessageId, NetworkGlobals, PeerId, PeerRequestId, PubsubMessage, Request, Response, }; use logging::TimeLatch; @@ -61,13 +61,13 @@ pub enum RouterMessage { /// An RPC response has been received. RPCResponseReceived { peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, response: Response, }, /// An RPC request failed RPCFailed { peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, error: RPCError, }, /// A gossip message has been received. The fields are: message id, the peer that sent us this @@ -235,7 +235,7 @@ impl Router { fn handle_rpc_response( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, response: Response, ) { match response { @@ -448,9 +448,9 @@ impl Router { /// An error occurred during an RPC request. The state is maintained by the sync manager, so /// this function notifies the sync manager of the error. - pub fn on_rpc_error(&mut self, peer_id: PeerId, request_id: RequestId, error: RPCError) { + pub fn on_rpc_error(&mut self, peer_id: PeerId, request_id: AppRequestId, error: RPCError) { // Check if the failed RPC belongs to sync - if let RequestId::Sync(request_id) = request_id { + if let AppRequestId::Sync(request_id) = request_id { self.send_to_sync(SyncMessage::RpcError { peer_id, request_id, @@ -488,18 +488,18 @@ impl Router { pub fn on_blocks_by_range_response( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, beacon_block: Option>>, ) { let request_id = match request_id { - RequestId::Sync(sync_id) => match sync_id { - SyncId::SingleBlock { .. } | SyncId::SingleBlob { .. } => { + AppRequestId::Sync(sync_id) => match sync_id { + SyncRequestId::SingleBlock { .. } | SyncRequestId::SingleBlob { .. } => { crit!(self.log, "Block lookups do not request BBRange requests"; "peer_id" => %peer_id); return; } - id @ SyncId::RangeBlockAndBlobs { .. } => id, + id @ SyncRequestId::RangeBlockAndBlobs { .. } => id, }, - RequestId::Router => { + AppRequestId::Router => { crit!(self.log, "All BBRange requests belong to sync"; "peer_id" => %peer_id); return; } @@ -522,7 +522,7 @@ impl Router { pub fn on_blobs_by_range_response( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, blob_sidecar: Option>>, ) { trace!( @@ -531,7 +531,7 @@ impl Router { "peer" => %peer_id, ); - if let RequestId::Sync(id) = request_id { + if let AppRequestId::Sync(id) = request_id { self.send_to_sync(SyncMessage::RpcBlob { peer_id, request_id: id, @@ -550,22 +550,22 @@ impl Router { pub fn on_blocks_by_root_response( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, beacon_block: Option>>, ) { let request_id = match request_id { - RequestId::Sync(sync_id) => match sync_id { - id @ SyncId::SingleBlock { .. } => id, - SyncId::RangeBlockAndBlobs { .. } => { + AppRequestId::Sync(sync_id) => match sync_id { + id @ SyncRequestId::SingleBlock { .. } => id, + SyncRequestId::RangeBlockAndBlobs { .. } => { crit!(self.log, "Batch syncing do not request BBRoot requests"; "peer_id" => %peer_id); return; } - SyncId::SingleBlob { .. } => { + SyncRequestId::SingleBlob { .. } => { crit!(self.log, "Blob response to block by roots request"; "peer_id" => %peer_id); return; } }, - RequestId::Router => { + AppRequestId::Router => { crit!(self.log, "All BBRoot requests belong to sync"; "peer_id" => %peer_id); return; } @@ -588,22 +588,22 @@ impl Router { pub fn on_blobs_by_root_response( &mut self, peer_id: PeerId, - request_id: RequestId, + request_id: AppRequestId, blob_sidecar: Option>>, ) { let request_id = match request_id { - RequestId::Sync(sync_id) => match sync_id { - id @ SyncId::SingleBlob { .. } => id, - SyncId::SingleBlock { .. } => { + AppRequestId::Sync(sync_id) => match sync_id { + id @ SyncRequestId::SingleBlob { .. } => id, + SyncRequestId::SingleBlock { .. } => { crit!(self.log, "Block response to blobs by roots request"; "peer_id" => %peer_id); return; } - SyncId::RangeBlockAndBlobs { .. } => { + SyncRequestId::RangeBlockAndBlobs { .. } => { crit!(self.log, "Batch syncing does not request BBRoot requests"; "peer_id" => %peer_id); return; } }, - RequestId::Router => { + AppRequestId::Router => { crit!(self.log, "All BlobsByRoot requests belong to sync"; "peer_id" => %peer_id); return; } @@ -667,7 +667,7 @@ impl HandlerNetworkContext { pub fn send_processor_request(&mut self, peer_id: PeerId, request: Request) { self.inform_network(NetworkMessage::SendRequest { peer_id, - request_id: RequestId::Router, + request_id: AppRequestId::Router, request, }) } diff --git a/beacon_node/network/src/service.rs b/beacon_node/network/src/service.rs index e215f25387..e522285a9e 100644 --- a/beacon_node/network/src/service.rs +++ b/beacon_node/network/src/service.rs @@ -1,4 +1,3 @@ -use super::sync::manager::RequestId as SyncId; use crate::nat; use crate::network_beacon_processor::InvalidBlockStorage; use crate::persisted_dht::{clear_dht, load_dht, persist_dht}; @@ -23,6 +22,7 @@ use lighthouse_network::{ Context, PeerAction, PeerRequestId, PubsubMessage, ReportSource, Request, Response, Subnet, }; use lighthouse_network::{ + service::api_types::AppRequestId, types::{core_topics_to_subscribe, GossipEncoding, GossipTopic}, MessageId, NetworkEvent, NetworkGlobals, PeerId, }; @@ -51,13 +51,6 @@ const UNSUBSCRIBE_DELAY_EPOCHS: u64 = 2; /// able to run tens of thousands of validators on one BN. const VALIDATOR_SUBSCRIPTION_MESSAGE_QUEUE_SIZE: usize = 65_536; -/// Application level requests sent to the network. -#[derive(Debug, Clone, Copy)] -pub enum RequestId { - Sync(SyncId), - Router, -} - /// Types of messages that the network service can receive. #[derive(Debug, IntoStaticStr)] #[strum(serialize_all = "snake_case")] @@ -69,7 +62,7 @@ pub enum NetworkMessage { SendRequest { peer_id: PeerId, request: Request, - request_id: RequestId, + request_id: AppRequestId, }, /// Send a successful Response to the libp2p service. SendResponse { @@ -168,7 +161,7 @@ pub struct NetworkService { /// A reference to the underlying beacon chain. beacon_chain: Arc>, /// The underlying libp2p service that drives all the network interactions. - libp2p: Network, + libp2p: Network, /// An attestation and subnet manager service. attestation_service: AttestationService, /// A sync committeee subnet manager service. @@ -499,7 +492,7 @@ impl NetworkService { /// Handle an event received from the network. async fn on_libp2p_event( &mut self, - ev: NetworkEvent, + ev: NetworkEvent, shutdown_sender: &mut Sender, ) { match ev { diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index fe133e8e1c..dfb05da19b 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -9,7 +9,7 @@ //! sync as failed, log an error and attempt to retry once a new peer joins the node. use crate::network_beacon_processor::ChainSegmentProcessId; -use crate::sync::manager::{BatchProcessResult, Id}; +use crate::sync::manager::BatchProcessResult; use crate::sync::network_context::RangeRequestId; use crate::sync::network_context::SyncNetworkContext; use crate::sync::range_sync::{ @@ -17,6 +17,7 @@ use crate::sync::range_sync::{ }; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes}; +use lighthouse_network::service::api_types::Id; use lighthouse_network::types::{BackFillState, NetworkGlobals}; use lighthouse_network::{PeerAction, PeerId}; use rand::seq::SliceRandom; @@ -527,7 +528,7 @@ impl BackFillSync { // result callback. This is done, because an empty batch could end a chain and the logic // for removing chains and checking completion is in the callback. - let blocks = match batch.start_processing() { + let (blocks, _) = match batch.start_processing() { Err(e) => { return self .fail_sync(BackFillError::BatchInvalidState(batch_id, e.0)) @@ -614,13 +615,15 @@ impl BackFillSync { "batch_epoch" => batch_id, "peer" => %peer, "client" => %network.client_type(&peer)); match result { - BatchProcessResult::Success { was_non_empty } => { + BatchProcessResult::Success { + imported_blocks, .. + } => { if let Err(e) = batch.processing_completed(BatchProcessingResult::Success) { self.fail_sync(BackFillError::BatchInvalidState(batch_id, e.0))?; } // If the processed batch was not empty, we can validate previous unvalidated // blocks. - if *was_non_empty { + if *imported_blocks > 0 { self.advance_chain(network, batch_id); } @@ -676,7 +679,7 @@ impl BackFillSync { Ok(BatchOperationOutcome::Continue) => { // chain can continue. Check if it can be progressed - if *imported_blocks { + if *imported_blocks > 0 { // At least one block was successfully verified and imported, then we can be sure all // previous batches are valid and we only need to download the current failed // batch. @@ -918,24 +921,22 @@ impl BackFillSync { // Find a peer to request the batch let failed_peers = batch.failed_peers(); - let new_peer = { - let mut priorized_peers = self - .network_globals - .peers - .read() - .synced_peers() - .map(|peer| { - ( - failed_peers.contains(peer), - self.active_requests.get(peer).map(|v| v.len()).unwrap_or(0), - *peer, - ) - }) - .collect::>(); + let new_peer = self + .network_globals + .peers + .read() + .synced_peers() + .map(|peer| { + ( + failed_peers.contains(peer), + self.active_requests.get(peer).map(|v| v.len()).unwrap_or(0), + rand::random::(), + *peer, + ) + }) // Sort peers prioritizing unrelated peers with less active requests. - priorized_peers.sort_unstable(); - priorized_peers.first().map(|&(_, _, peer)| peer) - }; + .min() + .map(|(_, _, _, peer)| peer); if let Some(peer) = new_peer { self.participating_peers.insert(peer); diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index aef76fb0da..e94e9589c0 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -2,10 +2,10 @@ use crate::sync::block_lookups::single_block_lookup::{ LookupRequestError, SingleBlockLookup, SingleLookupRequestState, }; use crate::sync::block_lookups::{BlobRequestState, BlockRequestState, PeerId}; -use crate::sync::manager::Id; use crate::sync::network_context::{LookupRequestResult, SyncNetworkContext}; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::BeaconChainTypes; +use lighthouse_network::service::api_types::Id; use std::sync::Arc; use types::blob_sidecar::FixedBlobSidecarList; use types::SignedBeaconBlock; diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 66b2d808d9..0a44cf2fdf 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -28,12 +28,12 @@ use super::network_context::{RpcResponseResult, SyncNetworkContext}; use crate::metrics; use crate::sync::block_lookups::common::ResponseType; use crate::sync::block_lookups::parent_chain::find_oldest_fork_ancestor; -use crate::sync::manager::{Id, SingleLookupReqId}; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::data_availability_checker::AvailabilityCheckErrorCategory; use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError}; pub use common::RequestState; use fnv::FnvHashMap; +use lighthouse_network::service::api_types::SingleLookupReqId; use lighthouse_network::{PeerAction, PeerId}; use lru_cache::LRUTimeCache; pub use single_block_lookup::{BlobRequestState, BlockRequestState}; @@ -67,6 +67,12 @@ const LOOKUP_MAX_DURATION_STUCK_SECS: u64 = 15 * PARENT_DEPTH_TOLERANCE as u64; /// lookup at most after 4 seconds, the lookup should gain peers. const LOOKUP_MAX_DURATION_NO_PEERS_SECS: u64 = 10; +/// Lookups contain untrusted data, including blocks that have not yet been validated. In case of +/// bugs or malicious activity we want to bound how much memory these lookups can consume. Aprox the +/// max size of a lookup is ~ 10 MB (current max size of gossip and RPC blocks). 200 lookups can +/// take at most 2 GB. 200 lookups allow 3 parallel chains of depth 64 (current maximum). +const MAX_LOOKUPS: usize = 200; + pub enum BlockComponent { Block(DownloadResult>>), Blob(DownloadResult>>), @@ -107,6 +113,9 @@ pub struct BlockLookups { log: Logger, } +#[cfg(test)] +use lighthouse_network::service::api_types::Id; + #[cfg(test)] /// Tuple of `SingleLookupId`, requested block root, awaiting parent block root (if any), /// and list of peers that claim to have imported this set of block components. @@ -318,24 +327,17 @@ impl BlockLookups { } } + // Lookups contain untrusted data, bound the total count of lookups hold in memory to reduce + // the risk of OOM in case of bugs of malicious activity. + if self.single_block_lookups.len() > MAX_LOOKUPS { + warn!(self.log, "Dropping lookup reached max"; "block_root" => ?block_root); + return false; + } + // If we know that this lookup has unknown parent (is awaiting a parent lookup to resolve), // signal here to hold processing downloaded data. let mut lookup = SingleBlockLookup::new(block_root, peers, cx.next_id(), awaiting_parent); - let msg = if block_component.is_some() { - "Searching for components of a block with unknown parent" - } else { - "Searching for block components" - }; - debug!( - self.log, - "{}", msg; - "peer_ids" => ?peers, - "block_root" => ?block_root, - "id" => lookup.id, - ); - metrics::inc_counter(&metrics::SYNC_LOOKUP_CREATED); - // Add block components to the new request if let Some(block_component) = block_component { lookup.add_child_components(block_component); @@ -351,6 +353,16 @@ impl BlockLookups { } }; + debug!( + self.log, + "Created block lookup"; + "peer_ids" => ?peers, + "block_root" => ?block_root, + "awaiting_parent" => awaiting_parent.map(|root| root.to_string()).unwrap_or("none".to_owned()), + "id" => lookup.id, + ); + metrics::inc_counter(&metrics::SYNC_LOOKUP_CREATED); + let result = lookup.continue_requests(cx); if self.on_lookup_result(id, result, "new_current_lookup", cx) { self.update_metrics(); diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index e17991286a..0466636fb7 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,12 +1,12 @@ use super::common::ResponseType; use super::{BlockComponent, PeerId, SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS}; use crate::sync::block_lookups::common::RequestState; -use crate::sync::block_lookups::Id; use crate::sync::network_context::{ LookupRequestResult, ReqId, RpcRequestSendError, SendErrorProcessor, SyncNetworkContext, }; use beacon_chain::BeaconChainTypes; use derivative::Derivative; +use lighthouse_network::service::api_types::Id; use rand::seq::IteratorRandom; use std::collections::HashSet; use std::fmt::Debug; diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 470e10c55c..ef2822fe56 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -1,9 +1,6 @@ use crate::network_beacon_processor::NetworkBeaconProcessor; -use crate::service::RequestId; -use crate::sync::manager::{ - BlockProcessType, RequestId as SyncRequestId, SingleLookupReqId, SyncManager, -}; +use crate::sync::manager::{BlockProcessType, SyncManager}; use crate::sync::SyncMessage; use crate::NetworkMessage; use std::sync::Arc; @@ -24,6 +21,7 @@ use beacon_chain::{ }; use beacon_processor::WorkEvent; use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode}; +use lighthouse_network::service::api_types::{AppRequestId, Id, SingleLookupReqId, SyncRequestId}; use lighthouse_network::types::SyncState; use lighthouse_network::{NetworkGlobals, Request}; use slog::info; @@ -550,7 +548,7 @@ impl TestRig { while let Ok(request_id) = self.pop_received_network_event(|ev| match ev { NetworkMessage::SendRequest { peer_id, - request_id: RequestId::Sync(id), + request_id: AppRequestId::Sync(id), .. } if *peer_id == disconnected_peer_id => Some(*id), _ => None, @@ -631,7 +629,7 @@ impl TestRig { NetworkMessage::SendRequest { peer_id: _, request: Request::BlocksByRoot(request), - request_id: RequestId::Sync(SyncRequestId::SingleBlock { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlock { id }), } if request.block_roots().to_vec().contains(&for_block) => Some(*id), _ => None, }) @@ -651,7 +649,7 @@ impl TestRig { NetworkMessage::SendRequest { peer_id: _, request: Request::BlobsByRoot(request), - request_id: RequestId::Sync(SyncRequestId::SingleBlob { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlob { id }), } if request .blob_ids .to_vec() @@ -676,7 +674,7 @@ impl TestRig { NetworkMessage::SendRequest { peer_id: _, request: Request::BlocksByRoot(request), - request_id: RequestId::Sync(SyncRequestId::SingleBlock { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlock { id }), } if request.block_roots().to_vec().contains(&for_block) => Some(*id), _ => None, }) @@ -698,7 +696,7 @@ impl TestRig { NetworkMessage::SendRequest { peer_id: _, request: Request::BlobsByRoot(request), - request_id: RequestId::Sync(SyncRequestId::SingleBlob { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlob { id }), } if request .blob_ids .to_vec() diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 9540709294..dd4fa56d53 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -53,6 +53,7 @@ use beacon_chain::{ }; use futures::StreamExt; use lighthouse_network::rpc::RPCError; +use lighthouse_network::service::api_types::{Id, SingleLookupReqId, SyncRequestId}; use lighthouse_network::types::{NetworkGlobals, SyncState}; use lighthouse_network::SyncInfo; use lighthouse_network::{PeerAction, PeerId}; @@ -78,25 +79,6 @@ pub const SLOT_IMPORT_TOLERANCE: usize = 32; /// arbitrary number that covers a full slot, but allows recovery if sync get stuck for a few slots. const NOTIFIED_UNKNOWN_ROOT_EXPIRY_SECONDS: u64 = 30; -pub type Id = u32; - -#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] -pub struct SingleLookupReqId { - pub lookup_id: Id, - pub req_id: Id, -} - -/// Id of rpc requests sent by sync to the network. -#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] -pub enum RequestId { - /// Request searching for a block given a hash. - SingleBlock { id: SingleLookupReqId }, - /// Request searching for a set of blobs given a hash. - SingleBlob { id: SingleLookupReqId }, - /// Range request that is composed by both a block range request and a blob range request. - RangeBlockAndBlobs { id: Id }, -} - #[derive(Debug)] /// A message that can be sent to the sync manager thread. pub enum SyncMessage { @@ -105,7 +87,7 @@ pub enum SyncMessage { /// A block has been received from the RPC. RpcBlock { - request_id: RequestId, + request_id: SyncRequestId, peer_id: PeerId, beacon_block: Option>>, seen_timestamp: Duration, @@ -113,7 +95,7 @@ pub enum SyncMessage { /// A blob has been received from the RPC. RpcBlob { - request_id: RequestId, + request_id: SyncRequestId, peer_id: PeerId, blob_sidecar: Option>>, seen_timestamp: Duration, @@ -135,7 +117,7 @@ pub enum SyncMessage { /// An RPC Error has occurred on a request. RpcError { peer_id: PeerId, - request_id: RequestId, + request_id: SyncRequestId, error: RPCError, }, @@ -174,11 +156,12 @@ pub enum BlockProcessingResult { pub enum BatchProcessResult { /// The batch was completed successfully. It carries whether the sent batch contained blocks. Success { - was_non_empty: bool, + sent_blocks: usize, + imported_blocks: usize, }, /// The batch processing failed. It carries whether the processing imported any block. FaultyFailure { - imported_blocks: bool, + imported_blocks: usize, penalty: PeerAction, }, NonFaultyFailure, @@ -342,16 +325,16 @@ impl SyncManager { } /// Handles RPC errors related to requests that were emitted from the sync manager. - fn inject_error(&mut self, peer_id: PeerId, request_id: RequestId, error: RPCError) { + fn inject_error(&mut self, peer_id: PeerId, request_id: SyncRequestId, error: RPCError) { trace!(self.log, "Sync manager received a failed RPC"); match request_id { - RequestId::SingleBlock { id } => { + SyncRequestId::SingleBlock { id } => { self.on_single_block_response(id, peer_id, RpcEvent::RPCError(error)) } - RequestId::SingleBlob { id } => { + SyncRequestId::SingleBlob { id } => { self.on_single_blob_response(id, peer_id, RpcEvent::RPCError(error)) } - RequestId::RangeBlockAndBlobs { id } => { + SyncRequestId::RangeBlockAndBlobs { id } => { if let Some(sender_id) = self.network.range_request_failed(id) { match sender_id { RangeRequestId::RangeSync { chain_id, batch_id } => { @@ -588,6 +571,8 @@ impl SyncManager { // unless there is a bug. let mut prune_lookups_interval = tokio::time::interval(Duration::from_secs(15)); + let mut register_metrics_interval = tokio::time::interval(Duration::from_secs(5)); + // process any inbound messages loop { tokio::select! { @@ -600,6 +585,9 @@ impl SyncManager { _ = prune_lookups_interval.tick() => { self.block_lookups.prune_lookups(); } + _ = register_metrics_interval.tick() => { + self.network.register_metrics(); + } } } } @@ -835,13 +823,13 @@ impl SyncManager { fn rpc_block_received( &mut self, - request_id: RequestId, + request_id: SyncRequestId, peer_id: PeerId, block: Option>>, seen_timestamp: Duration, ) { match request_id { - RequestId::SingleBlock { id } => self.on_single_block_response( + SyncRequestId::SingleBlock { id } => self.on_single_block_response( id, peer_id, match block { @@ -849,10 +837,10 @@ impl SyncManager { None => RpcEvent::StreamTermination, }, ), - RequestId::SingleBlob { .. } => { + SyncRequestId::SingleBlob { .. } => { crit!(self.log, "Block received during blob request"; "peer_id" => %peer_id ); } - RequestId::RangeBlockAndBlobs { id } => { + SyncRequestId::RangeBlockAndBlobs { id } => { self.range_block_and_blobs_response(id, peer_id, block.into()) } } @@ -877,16 +865,16 @@ impl SyncManager { fn rpc_blob_received( &mut self, - request_id: RequestId, + request_id: SyncRequestId, peer_id: PeerId, blob: Option>>, seen_timestamp: Duration, ) { match request_id { - RequestId::SingleBlock { .. } => { + SyncRequestId::SingleBlock { .. } => { crit!(self.log, "Single blob received during block request"; "peer_id" => %peer_id ); } - RequestId::SingleBlob { id } => self.on_single_blob_response( + SyncRequestId::SingleBlob { id } => self.on_single_blob_response( id, peer_id, match blob { @@ -894,7 +882,7 @@ impl SyncManager { None => RpcEvent::StreamTermination, }, ), - RequestId::RangeBlockAndBlobs { id } => { + SyncRequestId::RangeBlockAndBlobs { id } => { self.range_block_and_blobs_response(id, peer_id, blob.into()) } } @@ -978,7 +966,7 @@ impl SyncManager { "sender_id" => ?resp.sender_id, "error" => e.clone() ); - let id = RequestId::RangeBlockAndBlobs { id }; + let id = SyncRequestId::RangeBlockAndBlobs { id }; self.network.report_peer( peer_id, PeerAction::MidToleranceError, diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 6f89b954b3..df8be9f6d5 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -4,18 +4,19 @@ use self::requests::{ActiveBlobsByRootRequest, ActiveBlocksByRootRequest}; pub use self::requests::{BlobsByRootSingleBlockRequest, BlocksByRootSingleRequest}; use super::block_sidecar_coupling::BlocksAndBlobsRequestInfo; -use super::manager::{Id, RequestId as SyncRequestId}; use super::range_sync::{BatchId, ByRangeRequestType, ChainId}; +use crate::metrics; use crate::network_beacon_processor::NetworkBeaconProcessor; -use crate::service::{NetworkMessage, RequestId}; +use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::block_lookups::SingleLookupId; -use crate::sync::manager::{BlockProcessType, SingleLookupReqId}; +use crate::sync::manager::BlockProcessType; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProcessStatus, EngineState}; use fnv::FnvHashMap; use lighthouse_network::rpc::methods::BlobsByRangeRequest; use lighthouse_network::rpc::{BlocksByRangeRequest, GoodbyeReason, RPCError}; +use lighthouse_network::service::api_types::{AppRequestId, Id, SingleLookupReqId, SyncRequestId}; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Request}; pub use requests::LookupVerifyError; use slog::{debug, error, trace, warn}; @@ -246,7 +247,7 @@ impl SyncNetworkContext { ); let request = Request::Status(status_message.clone()); - let request_id = RequestId::Router; + let request_id = AppRequestId::Router; let _ = self.send_network_msg(NetworkMessage::SendRequest { peer_id, request, @@ -274,7 +275,7 @@ impl SyncNetworkContext { .send(NetworkMessage::SendRequest { peer_id, request: Request::BlocksByRange(request.clone()), - request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), + request_id: AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; @@ -295,7 +296,7 @@ impl SyncNetworkContext { start_slot: *request.start_slot(), count: *request.count(), }), - request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), + request_id: AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; } @@ -348,27 +349,28 @@ impl SyncNetworkContext { request_id: Id, block_or_blob: BlockOrBlob, ) -> Option> { - match self.range_blocks_and_blobs_requests.entry(request_id) { - Entry::Occupied(mut entry) => { - let (_, info) = entry.get_mut(); - match block_or_blob { - BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block), - BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), - } - if info.is_finished() { - // If the request is finished, dequeue everything - let (sender_id, info) = entry.remove(); - let request_type = info.get_request_type(); - Some(BlocksAndBlobsByRangeResponse { - sender_id, - request_type, - responses: info.into_responses(), - }) - } else { - None - } - } - Entry::Vacant(_) => None, + let Entry::Occupied(mut entry) = self.range_blocks_and_blobs_requests.entry(request_id) + else { + metrics::inc_counter_vec(&metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, &["range_blocks"]); + return None; + }; + + let (_, info) = entry.get_mut(); + match block_or_blob { + BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block), + BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), + } + if info.is_finished() { + // If the request is finished, dequeue everything + let (sender_id, info) = entry.remove(); + let request_type = info.get_request_type(); + Some(BlocksAndBlobsByRangeResponse { + sender_id, + request_type, + responses: info.into_responses(), + }) + } else { + None } } @@ -424,7 +426,7 @@ impl SyncNetworkContext { .send(NetworkMessage::SendRequest { peer_id, request: Request::BlocksByRoot(request.into_request(&self.chain.spec)), - request_id: RequestId::Sync(SyncRequestId::SingleBlock { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlock { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; @@ -510,7 +512,7 @@ impl SyncNetworkContext { .send(NetworkMessage::SendRequest { peer_id, request: Request::BlobsByRoot(request.clone().into_request(&self.chain.spec)), - request_id: RequestId::Sync(SyncRequestId::SingleBlob { id }), + request_id: AppRequestId::Sync(SyncRequestId::SingleBlob { id }), }) .map_err(|_| RpcRequestSendError::NetworkSendError)?; @@ -631,6 +633,7 @@ impl SyncNetworkContext { block: RpcEvent>>, ) -> Option>>> { let Entry::Occupied(mut request) = self.blocks_by_root_requests.entry(request_id) else { + metrics::inc_counter_vec(&metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, &["blocks_by_root"]); return None; }; @@ -668,6 +671,7 @@ impl SyncNetworkContext { blob: RpcEvent>>, ) -> Option>> { let Entry::Occupied(mut request) = self.blobs_by_root_requests.entry(request_id) else { + metrics::inc_counter_vec(&metrics::SYNC_UNKNOWN_NETWORK_REQUESTS, &["blobs_by_root"]); return None; }; @@ -771,6 +775,24 @@ impl SyncNetworkContext { SendErrorProcessor::SendError }) } + + pub(crate) fn register_metrics(&self) { + metrics::set_gauge_vec( + &metrics::SYNC_ACTIVE_NETWORK_REQUESTS, + &["blocks_by_root"], + self.blocks_by_root_requests.len() as i64, + ); + metrics::set_gauge_vec( + &metrics::SYNC_ACTIVE_NETWORK_REQUESTS, + &["blobs_by_root"], + self.blobs_by_root_requests.len() as i64, + ); + metrics::set_gauge_vec( + &metrics::SYNC_ACTIVE_NETWORK_REQUESTS, + &["range_blocks"], + self.range_blocks_and_blobs_requests.len() as i64, + ); + } } fn to_fixed_blob_sidecar_list( diff --git a/beacon_node/network/src/sync/range_sync/batch.rs b/beacon_node/network/src/sync/range_sync/batch.rs index baba8c9a62..49e3ac3a81 100644 --- a/beacon_node/network/src/sync/range_sync/batch.rs +++ b/beacon_node/network/src/sync/range_sync/batch.rs @@ -1,10 +1,11 @@ -use crate::sync::manager::Id; use beacon_chain::block_verification_types::{AsBlock, RpcBlock}; use lighthouse_network::rpc::methods::BlocksByRangeRequest; +use lighthouse_network::service::api_types::Id; use lighthouse_network::PeerId; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::ops::Sub; +use std::time::{Duration, Instant}; use strum::Display; use types::{Epoch, EthSpec, Slot}; @@ -118,7 +119,7 @@ pub enum BatchState { /// The batch is being downloaded. Downloading(PeerId, Id), /// The batch has been completely downloaded and is ready for processing. - AwaitingProcessing(PeerId, Vec>), + AwaitingProcessing(PeerId, Vec>, Instant), /// The batch is being processed. Processing(Attempt), /// The batch was successfully processed and is waiting to be validated. @@ -210,13 +211,26 @@ impl BatchInfo { match &self.state { BatchState::AwaitingDownload | BatchState::Failed => None, BatchState::Downloading(peer_id, _) - | BatchState::AwaitingProcessing(peer_id, _) + | BatchState::AwaitingProcessing(peer_id, _, _) | BatchState::Processing(Attempt { peer_id, .. }) | BatchState::AwaitingValidation(Attempt { peer_id, .. }) => Some(peer_id), BatchState::Poisoned => unreachable!("Poisoned batch"), } } + /// Returns the count of stored pending blocks if in awaiting processing state + pub fn pending_blocks(&self) -> usize { + match &self.state { + BatchState::AwaitingProcessing(_, blocks, _) => blocks.len(), + BatchState::AwaitingDownload + | BatchState::Downloading { .. } + | BatchState::Processing { .. } + | BatchState::AwaitingValidation { .. } + | BatchState::Poisoned + | BatchState::Failed => 0, + } + } + /// Returns a BlocksByRange request associated with the batch. pub fn to_blocks_by_range_request(&self) -> (BlocksByRangeRequest, ByRangeRequestType) { ( @@ -293,7 +307,7 @@ impl BatchInfo { } let received = blocks.len(); - self.state = BatchState::AwaitingProcessing(peer, blocks); + self.state = BatchState::AwaitingProcessing(peer, blocks, Instant::now()); Ok(received) } BatchState::Poisoned => unreachable!("Poisoned batch"), @@ -365,11 +379,11 @@ impl BatchInfo { } } - pub fn start_processing(&mut self) -> Result>, WrongState> { + pub fn start_processing(&mut self) -> Result<(Vec>, Duration), WrongState> { match self.state.poison() { - BatchState::AwaitingProcessing(peer, blocks) => { + BatchState::AwaitingProcessing(peer, blocks, start_instant) => { self.state = BatchState::Processing(Attempt::new::(peer, &blocks)); - Ok(blocks) + Ok((blocks, start_instant.elapsed())) } BatchState::Poisoned => unreachable!("Poisoned batch"), other => { @@ -515,7 +529,7 @@ impl std::fmt::Debug for BatchState { }) => write!(f, "AwaitingValidation({})", peer_id), BatchState::AwaitingDownload => f.write_str("AwaitingDownload"), BatchState::Failed => f.write_str("Failed"), - BatchState::AwaitingProcessing(ref peer, ref blocks) => { + BatchState::AwaitingProcessing(ref peer, ref blocks, _) => { write!(f, "AwaitingProcessing({}, {} blocks)", peer, blocks.len()) } BatchState::Downloading(peer, request_id) => { diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index ea873bdca0..d63b2f95d8 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -1,17 +1,19 @@ use super::batch::{BatchInfo, BatchProcessingResult, BatchState}; +use super::RangeSyncType; +use crate::metrics; use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::network_context::RangeRequestId; -use crate::sync::{ - manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult, -}; +use crate::sync::{network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult}; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::BeaconChainTypes; use fnv::FnvHashMap; +use lighthouse_network::service::api_types::Id; use lighthouse_network::{PeerAction, PeerId}; -use rand::seq::SliceRandom; +use rand::{seq::SliceRandom, Rng}; use slog::{crit, debug, o, warn}; use std::collections::{btree_map::Entry, BTreeMap, HashSet}; use std::hash::{Hash, Hasher}; +use strum::IntoStaticStr; use types::{Epoch, EthSpec, Hash256, Slot}; /// Blocks are downloaded in batches from peers. This constant specifies how many epochs worth of @@ -54,6 +56,13 @@ pub struct KeepChain; pub type ChainId = u64; pub type BatchId = Epoch; +#[derive(Debug, Copy, Clone, IntoStaticStr)] +pub enum SyncingChainType { + Head, + Finalized, + Backfill, +} + /// A chain of blocks that need to be downloaded. Peers who claim to contain the target head /// root are grouped into the peer pool and queried for batches when downloading the /// chain. @@ -61,6 +70,9 @@ pub struct SyncingChain { /// A random id used to identify this chain. id: ChainId, + /// SyncingChain type + pub chain_type: SyncingChainType, + /// The start of the chain segment. Any epoch previous to this one has been validated. pub start_epoch: Epoch, @@ -127,6 +139,7 @@ impl SyncingChain { target_head_slot: Slot, target_head_root: Hash256, peer_id: PeerId, + chain_type: SyncingChainType, log: &slog::Logger, ) -> Self { let mut peers = FnvHashMap::default(); @@ -136,6 +149,7 @@ impl SyncingChain { SyncingChain { id, + chain_type, start_epoch, target_head_slot, target_head_root, @@ -172,6 +186,14 @@ impl SyncingChain { self.validated_batches * EPOCHS_PER_BATCH } + /// Returns the total count of pending blocks in all the batches of this chain + pub fn pending_blocks(&self) -> usize { + self.batches + .values() + .map(|batch| batch.pending_blocks()) + .sum() + } + /// Removes a peer from the chain. /// If the peer has active batches, those are considered failed and re-requested. pub fn remove_peer( @@ -306,7 +328,12 @@ impl SyncingChain { // result callback. This is done, because an empty batch could end a chain and the logic // for removing chains and checking completion is in the callback. - let blocks = batch.start_processing()?; + let (blocks, duration_in_awaiting_processing) = batch.start_processing()?; + metrics::observe_duration( + &metrics::SYNCING_CHAIN_BATCH_AWAITING_PROCESSING, + duration_in_awaiting_processing, + ); + let process_id = ChainSegmentProcessId::RangeBatchId(self.id, batch_id); self.current_processing_batch = Some(batch_id); @@ -470,10 +497,27 @@ impl SyncingChain { // We consider three cases. Batch was successfully processed, Batch failed processing due // to a faulty peer, or batch failed processing but the peer can't be deemed faulty. match result { - BatchProcessResult::Success { was_non_empty } => { + BatchProcessResult::Success { + sent_blocks, + imported_blocks, + } => { + if sent_blocks > imported_blocks { + let ignored_blocks = sent_blocks - imported_blocks; + metrics::inc_counter_vec_by( + &metrics::SYNCING_CHAINS_IGNORED_BLOCKS, + &[self.chain_type.into()], + ignored_blocks as u64, + ); + } + metrics::inc_counter_vec( + &metrics::SYNCING_CHAINS_PROCESSED_BATCHES, + &[self.chain_type.into()], + ); + batch.processing_completed(BatchProcessingResult::Success)?; - if *was_non_empty { + // was not empty = sent_blocks > 0 + if *sent_blocks > 0 { // If the processed batch was not empty, we can validate previous unvalidated // blocks. self.advance_chain(network, batch_id); @@ -516,7 +560,7 @@ impl SyncingChain { match batch.processing_completed(BatchProcessingResult::FaultyFailure)? { BatchOperationOutcome::Continue => { // Chain can continue. Check if it can be moved forward. - if *imported_blocks { + if *imported_blocks > 0 { // At least one block was successfully verified and imported, so we can be sure all // previous batches are valid and we only need to download the current failed // batch. @@ -874,16 +918,20 @@ impl SyncingChain { // Find a peer to request the batch let failed_peers = batch.failed_peers(); - let new_peer = { - let mut priorized_peers = self - .peers - .iter() - .map(|(peer, requests)| (failed_peers.contains(peer), requests.len(), *peer)) - .collect::>(); + let new_peer = self + .peers + .iter() + .map(|(peer, requests)| { + ( + failed_peers.contains(peer), + requests.len(), + rand::thread_rng().gen::(), + *peer, + ) + }) // Sort peers prioritizing unrelated peers with less active requests. - priorized_peers.sort_unstable(); - priorized_peers.first().map(|&(_, _, peer)| peer) - }; + .min() + .map(|(_, _, _, peer)| peer); if let Some(peer) = new_peer { self.send_batch(network, batch_id, peer) @@ -1139,3 +1187,12 @@ impl RemoveChain { ) } } + +impl From for SyncingChainType { + fn from(value: RangeSyncType) -> Self { + match value { + RangeSyncType::Head => Self::Head, + RangeSyncType::Finalized => Self::Finalized, + } + } +} diff --git a/beacon_node/network/src/sync/range_sync/chain_collection.rs b/beacon_node/network/src/sync/range_sync/chain_collection.rs index 364514a358..3621a6605a 100644 --- a/beacon_node/network/src/sync/range_sync/chain_collection.rs +++ b/beacon_node/network/src/sync/range_sync/chain_collection.rs @@ -64,8 +64,8 @@ impl ChainCollection { /// Updates the Syncing state of the collection after a chain is removed. fn on_chain_removed(&mut self, id: &ChainId, was_syncing: bool, sync_type: RangeSyncType) { - let _ = metrics::get_int_gauge(&metrics::SYNCING_CHAINS_COUNT, &[sync_type.as_str()]) - .map(|m| m.dec()); + metrics::inc_counter_vec(&metrics::SYNCING_CHAINS_REMOVED, &[sync_type.as_str()]); + self.update_metrics(); match self.state { RangeSyncState::Finalized(ref syncing_id) => { @@ -493,15 +493,28 @@ impl ChainCollection { target_head_slot, target_head_root, peer, + sync_type.into(), &self.log, ); debug_assert_eq!(new_chain.get_id(), id); debug!(self.log, "New chain added to sync"; "peer_id" => peer_rpr, "sync_type" => ?sync_type, &new_chain); entry.insert(new_chain); - let _ = - metrics::get_int_gauge(&metrics::SYNCING_CHAINS_COUNT, &[sync_type.as_str()]) - .map(|m| m.inc()); + metrics::inc_counter_vec(&metrics::SYNCING_CHAINS_ADDED, &[sync_type.as_str()]); + self.update_metrics(); } } } + + fn update_metrics(&self) { + metrics::set_gauge_vec( + &metrics::SYNCING_CHAINS_COUNT, + &[RangeSyncType::Finalized.as_str()], + self.finalized_chains.len() as i64, + ); + metrics::set_gauge_vec( + &metrics::SYNCING_CHAINS_COUNT, + &[RangeSyncType::Head.as_str()], + self.head_chains.len() as i64, + ); + } } diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index 45d2918331..4213771d48 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -43,13 +43,14 @@ use super::block_storage::BlockStorage; use super::chain::{BatchId, ChainId, RemoveChain, SyncingChain}; use super::chain_collection::ChainCollection; use super::sync_type::RangeSyncType; +use crate::metrics; use crate::status::ToStatusMessage; -use crate::sync::manager::Id; use crate::sync::network_context::SyncNetworkContext; use crate::sync::BatchProcessResult; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::{BeaconChain, BeaconChainTypes}; use lighthouse_network::rpc::GoodbyeReason; +use lighthouse_network::service::api_types::Id; use lighthouse_network::PeerId; use lighthouse_network::SyncInfo; use lru_cache::LRUTimeCache; @@ -346,6 +347,12 @@ where } } + metrics::inc_counter_vec_by( + &metrics::SYNCING_CHAINS_DROPPED_BLOCKS, + &[sync_type.as_str()], + chain.pending_blocks() as u64, + ); + network.status_peers(self.beacon_chain.as_ref(), chain.peers()); let status = self.beacon_chain.status_message(); @@ -380,7 +387,6 @@ where #[cfg(test)] mod tests { use crate::network_beacon_processor::NetworkBeaconProcessor; - use crate::service::RequestId; use crate::NetworkMessage; use super::*; @@ -391,7 +397,10 @@ mod tests { use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::EngineState; use beacon_processor::WorkEvent as BeaconWorkEvent; - use lighthouse_network::{rpc::StatusMessage, NetworkGlobals}; + use lighthouse_network::service::api_types::SyncRequestId; + use lighthouse_network::{ + rpc::StatusMessage, service::api_types::AppRequestId, NetworkGlobals, + }; use slog::{o, Drain}; use slot_clock::TestingSlotClock; use std::collections::HashSet; @@ -517,7 +526,7 @@ mod tests { &mut self, expected_peer: &PeerId, fork_name: ForkName, - ) -> (RequestId, Option) { + ) -> (AppRequestId, Option) { let block_req_id = if let Ok(NetworkMessage::SendRequest { peer_id, request: _, @@ -550,12 +559,12 @@ mod tests { fn complete_range_block_and_blobs_response( &mut self, - block_req: RequestId, - blob_req_opt: Option, + block_req: AppRequestId, + blob_req_opt: Option, ) -> (ChainId, BatchId, Id) { if blob_req_opt.is_some() { match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { + AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }) => { let _ = self .cx .range_block_and_blob_response(id, BlockOrBlob::Block(None)); @@ -571,7 +580,7 @@ mod tests { } } else { match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { + AppRequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }) => { let response = self .cx .range_block_and_blob_response(id, BlockOrBlob::Block(None)) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 40a343a7fe..c32c5e7ec6 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -372,17 +372,22 @@ pub fn cli_app() -> Command { .arg( Arg::new("self-limiter") .long("self-limiter") - .help( - "Enables the outbound rate limiter (requests made by this node). \ - Use the self-limiter-protocol flag to set per protocol configurations. \ - If the self rate limiter is enabled and a protocol is not \ - present in the configuration, the quotas used for the inbound rate limiter will be \ - used." - ) + .help("This flag is deprecated and has no effect.") + .hide(true) .action(ArgAction::SetTrue) .help_heading(FLAG_HEADER) .display_order(0) ) + .arg( + Arg::new("disable-self-limiter") + .long("disable-self-limiter") + .help( + "Disables the outbound rate limiter (requests sent by this node)." + ) + .action(ArgAction::SetTrue) + .help_heading(FLAG_HEADER) + .display_order(0) + ) .arg( Arg::new("self-limiter-protocols") .long("self-limiter-protocols") @@ -397,7 +402,7 @@ pub fn cli_app() -> Command { ) .action(ArgAction::Append) .value_delimiter(';') - .requires("self-limiter") + .conflicts_with("disable-self-limiter") .display_order(0) ) .arg( diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 35fad0718c..f0d02f6c51 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -402,7 +402,10 @@ pub fn get_config( client_config.blobs_db_path = Some(PathBuf::from(blobs_db_dir)); } - let (sprp, sprp_explicit) = get_slots_per_restore_point::(cli_args)?; + let (sprp, sprp_explicit) = get_slots_per_restore_point::(clap_utils::parse_optional( + cli_args, + "slots-per-restore-point", + )?)?; client_config.store.slots_per_restore_point = sprp; client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit; @@ -1413,16 +1416,15 @@ pub fn set_network_config( // Light client server config. config.enable_light_client_server = parse_flag(cli_args, "light-client-server"); - // The self limiter is disabled by default. If the `self-limiter` flag is provided - // without the `self-limiter-protocols` flag, the default params will be used. - if parse_flag(cli_args, "self-limiter") { - config.outbound_rate_limiter_config = - if let Some(protocols) = cli_args.get_one::("self-limiter-protocols") { - Some(protocols.parse()?) - } else { - Some(Default::default()) - }; - } + // The self limiter is enabled by default. If the `self-limiter-protocols` flag is not provided, + // the default params will be used. + config.outbound_rate_limiter_config = if parse_flag(cli_args, "disable-self-limiter") { + None + } else if let Some(protocols) = cli_args.get_one::("self-limiter-protocols") { + Some(protocols.parse()?) + } else { + Some(Default::default()) + }; // Proposer-only mode overrides a number of previous configuration parameters. // Specifically, we avoid subscribing to long-lived subnets and wish to maintain a minimal set @@ -1476,11 +1478,9 @@ pub fn get_data_dir(cli_args: &ArgMatches) -> PathBuf { /// /// Return `(sprp, set_explicitly)` where `set_explicitly` is `true` if the user provided the value. pub fn get_slots_per_restore_point( - cli_args: &ArgMatches, + slots_per_restore_point: Option, ) -> Result<(u64, bool), String> { - if let Some(slots_per_restore_point) = - clap_utils::parse_optional(cli_args, "slots-per-restore-point")? - { + if let Some(slots_per_restore_point) = slots_per_restore_point { Ok((slots_per_restore_point, true)) } else { let default = std::cmp::min( diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index 385da5b4fe..40b667a744 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -17,7 +17,7 @@ use slasher::{DatabaseBackendOverride, Slasher}; use slog::{info, warn}; use std::ops::{Deref, DerefMut}; use std::sync::Arc; -use types::EthSpec; +use types::{ChainSpec, Epoch, EthSpec, ForkName}; /// A type-alias to the tighten the definition of a production-intended `Client`. pub type ProductionClient = @@ -78,6 +78,16 @@ impl ProductionBeaconNode { TimeoutRwLock::disable_timeouts() } + if let Err(misaligned_forks) = validator_fork_epochs(&spec) { + warn!( + log, + "Fork boundaries are not well aligned / multiples of 256"; + "info" => "This may cause issues as fork boundaries do not align with the \ + start of sync committee period.", + "misaligned_forks" => ?misaligned_forks, + ); + } + let builder = ClientBuilder::new(context.eth_spec_instance.clone()) .runtime_context(context) .chain_spec(spec.clone()) @@ -183,6 +193,28 @@ impl ProductionBeaconNode { } } +fn validator_fork_epochs(spec: &ChainSpec) -> Result<(), Vec<(ForkName, Epoch)>> { + // @dapplion: "We try to schedule forks such that the fork epoch is a multiple of 256, to keep + // historical vectors in the same fork. Indirectly that makes light client periods align with + // fork boundaries." + let sync_committee_period = spec.epochs_per_sync_committee_period; // 256 + let is_fork_boundary_misaligned = |epoch: Epoch| epoch % sync_committee_period != 0; + + let forks_with_misaligned_epochs = ForkName::list_all_fork_epochs(spec) + .iter() + .filter_map(|(fork, fork_epoch_opt)| { + fork_epoch_opt + .and_then(|epoch| is_fork_boundary_misaligned(epoch).then_some((*fork, epoch))) + }) + .collect::>(); + + if forks_with_misaligned_epochs.is_empty() { + Ok(()) + } else { + Err(forks_with_misaligned_epochs) + } +} + impl Deref for ProductionBeaconNode { type Target = ProductionClient; @@ -206,3 +238,23 @@ impl lighthouse_network::discv5::Executor for Discv5Executor { self.0.spawn(future, "discv5") } } + +#[cfg(test)] +mod test { + use super::*; + use types::MainnetEthSpec; + + #[test] + fn test_validator_fork_epoch_alignments() { + let mut spec = MainnetEthSpec::default_spec(); + spec.altair_fork_epoch = Some(Epoch::new(0)); + spec.bellatrix_fork_epoch = Some(Epoch::new(256)); + spec.deneb_fork_epoch = Some(Epoch::new(257)); + spec.electra_fork_epoch = None; + let result = validator_fork_epochs(&spec); + assert_eq!( + result, + Err(vec![(ForkName::Deneb, spec.deneb_fork_epoch.unwrap())]) + ); + } +} diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index 0247bea554..0e0965670b 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -289,7 +289,7 @@ impl DBColumn { /// This function returns the number of bytes used by keys in a given column. pub fn key_size(self) -> usize { match self { - Self::OverflowLRUCache => 33, // See `OverflowKey` encode impl. + Self::OverflowLRUCache => 33, // DEPRECATED Self::BeaconMeta | Self::BeaconBlock | Self::BeaconState diff --git a/beacon_node/store/src/metrics.rs b/beacon_node/store/src/metrics.rs index 1e614036ea..f8dbbfec98 100644 --- a/beacon_node/store/src/metrics.rs +++ b/beacon_node/store/src/metrics.rs @@ -55,14 +55,6 @@ lazy_static! { "store_beacon_state_hot_get_total", "Total number of hot beacon states requested from the store (cache or DB)" ); - pub static ref BEACON_STATE_CACHE_HIT_COUNT: Result = try_create_int_counter( - "store_beacon_state_cache_hit_total", - "Number of hits to the store's state cache" - ); - pub static ref BEACON_STATE_CACHE_CLONE_TIME: Result = try_create_histogram( - "store_beacon_state_cache_clone_time", - "Time to load a beacon block from the block cache" - ); pub static ref BEACON_STATE_READ_TIMES: Result = try_create_histogram( "store_beacon_state_read_seconds", "Total time required to read a BeaconState from the database" @@ -106,30 +98,6 @@ lazy_static! { "store_beacon_blobs_cache_hit_total", "Number of hits to the store's blob cache" ); - pub static ref BEACON_BLOCK_READ_TIMES: Result = try_create_histogram( - "store_beacon_block_read_overhead_seconds", - "Overhead on reading a beacon block from the DB (e.g., decoding)" - ); - pub static ref BEACON_BLOCK_READ_COUNT: Result = try_create_int_counter( - "store_beacon_block_read_total", - "Total number of beacon block reads from the DB" - ); - pub static ref BEACON_BLOCK_READ_BYTES: Result = try_create_int_counter( - "store_beacon_block_read_bytes_total", - "Total number of beacon block bytes read from the DB" - ); - pub static ref BEACON_BLOCK_WRITE_TIMES: Result = try_create_histogram( - "store_beacon_block_write_overhead_seconds", - "Overhead on writing a beacon block to the DB (e.g., encoding)" - ); - pub static ref BEACON_BLOCK_WRITE_COUNT: Result = try_create_int_counter( - "store_beacon_block_write_total", - "Total number of beacon block writes the DB" - ); - pub static ref BEACON_BLOCK_WRITE_BYTES: Result = try_create_int_counter( - "store_beacon_block_write_bytes_total", - "Total number of beacon block bytes written to the DB" - ); } /// Updates the global metrics registry with store-related information. diff --git a/book/src/database-migrations.md b/book/src/database-migrations.md index 611c61cb9c..fc16641da0 100644 --- a/book/src/database-migrations.md +++ b/book/src/database-migrations.md @@ -16,21 +16,19 @@ validator client or the slasher**. | Lighthouse version | Release date | Schema version | Downgrade available? | |--------------------|--------------|----------------|----------------------| -| v5.2.0 | Jun 2024 | v19 | yes before Deneb | -| v5.1.0 | Mar 2024 | v19 | yes before Deneb | -| v5.0.0 | Feb 2024 | v19 | yes before Deneb | -| v4.6.0 | Dec 2023 | v19 | yes before Deneb | -| v4.6.0-rc.0 | Dec 2023 | v18 | yes before Deneb | -| v4.5.0 | Sep 2023 | v17 | yes | -| v4.4.0 | Aug 2023 | v17 | yes | -| v4.3.0 | Jul 2023 | v17 | yes | -| v4.2.0 | May 2023 | v17 | yes | -| v4.1.0 | Apr 2023 | v16 | no | -| v4.0.1 | Mar 2023 | v16 | no | +| v5.3.0 | Aug 2024 TBD | v22 TBD | no (TBD) | +| v5.2.0 | Jun 2024 | v19 | no | +| v5.1.0 | Mar 2024 | v19 | no | +| v5.0.0 | Feb 2024 | v19 | no | +| v4.6.0 | Dec 2023 | v19 | no | > **Note**: All point releases (e.g. v4.4.1) are schema-compatible with the prior minor release > (e.g. v4.4.0). +> **Note**: Even if no schema downgrade is available, it is still possible to move between versions +> that use the same schema. E.g. you can downgrade from v5.2.0 to v5.0.0 because both use schema +> v19. + > **Note**: Support for old schemas is gradually removed from newer versions of Lighthouse. We usually do this after a major version has been out for a while and everyone has upgraded. Deprecated schema versions for previous releases are archived under @@ -210,12 +208,15 @@ Here are the steps to prune historic states: | Lighthouse version | Release date | Schema version | Downgrade available? | |--------------------|--------------|----------------|-------------------------------------| -| v4.6.0 | Dec 2023 | v19 | yes before Deneb | -| v4.6.0-rc.0 | Dec 2023 | v18 | yes before Deneb | -| v4.5.0 | Sep 2023 | v17 | yes | -| v4.4.0 | Aug 2023 | v17 | yes | -| v4.3.0 | Jul 2023 | v17 | yes | -| v4.2.0 | May 2023 | v17 | yes | +| v5.2.0 | Jun 2024 | v19 | yes before Deneb using <= v5.2.1 | +| v5.1.0 | Mar 2024 | v19 | yes before Deneb using <= v5.2.1 | +| v5.0.0 | Feb 2024 | v19 | yes before Deneb using <= v5.2.1 | +| v4.6.0 | Dec 2023 | v19 | yes before Deneb using <= v5.2.1 | +| v4.6.0-rc.0 | Dec 2023 | v18 | yes before Deneb using <= v5.2.1 | +| v4.5.0 | Sep 2023 | v17 | yes using <= v5.2.1 | +| v4.4.0 | Aug 2023 | v17 | yes using <= v5.2.1 | +| v4.3.0 | Jul 2023 | v17 | yes using <= v5.2.1 | +| v4.2.0 | May 2023 | v17 | yes using <= v5.2.1 | | v4.1.0 | Apr 2023 | v16 | yes before Capella using <= v4.5.0 | | v4.0.1 | Mar 2023 | v16 | yes before Capella using <= v4.5.0 | | v3.5.0 | Feb 2023 | v15 | yes before Capella using <= v4.5.0 | diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 8bbbd3eb6b..5288b6a1de 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -241,7 +241,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] @@ -505,6 +505,8 @@ Flags: --disable-quic Disables the quic transport. The node will rely solely on the TCP transport for libp2p connections. + --disable-self-limiter + Disables the outbound rate limiter (requests sent by this node). --disable-upnp Disables UPnP support. Setting this will prevent Lighthouse from attempting to automatically establish external port mappings. @@ -575,12 +577,6 @@ Flags: When present, Lighthouse will forget the payload statuses of any already-imported blocks. This can assist in the recovery from a consensus failure caused by the execution layer. - --self-limiter - Enables the outbound rate limiter (requests made by this node). Use - the self-limiter-protocol flag to set per protocol configurations. If - the self rate limiter is enabled and a protocol is not present in the - configuration, the quotas used for the inbound rate limiter will be - used. --shutdown-after-sync Shutdown beacon node as soon as sync is completed. Backfill sync will not be performed before shutdown. diff --git a/book/src/help_general.md b/book/src/help_general.md index a8cd459614..84bc67a86e 100644 --- a/book/src/help_general.md +++ b/book/src/help_general.md @@ -23,7 +23,7 @@ Commands: is the recommended way to provide a network boot-node since it has a reduced attack surface compared to a full beacon node. database_manager - Manage a beacon node database [aliases: db] + Manage a beacon node database. [aliases: db] validator_client When connected to a beacon node, performs the duties of a staked validator (e.g., proposing blocks and attestations). [aliases: v, vc, @@ -70,7 +70,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/book/src/help_vc.md b/book/src/help_vc.md index e896c8fa57..ccd09b321f 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -86,7 +86,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/book/src/help_vm.md b/book/src/help_vm.md index 6f9cc405e7..99a45c1a76 100644 --- a/book/src/help_vm.md +++ b/book/src/help_vm.md @@ -62,7 +62,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/book/src/help_vm_create.md b/book/src/help_vm_create.md index 4ddb360e48..1803bb534c 100644 --- a/book/src/help_vm_create.md +++ b/book/src/help_vm_create.md @@ -74,7 +74,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/book/src/help_vm_import.md b/book/src/help_vm_import.md index 799a1db82b..e18aad7958 100644 --- a/book/src/help_vm_import.md +++ b/book/src/help_vm_import.md @@ -43,7 +43,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/book/src/help_vm_move.md b/book/src/help_vm_move.md index 9b92e21bc2..faef0a5783 100644 --- a/book/src/help_vm_move.md +++ b/book/src/help_vm_move.md @@ -63,7 +63,7 @@ Options: [possible values: DEFAULT, JSON] --logfile-max-number The maximum number of log files that will be stored. If set to 0, - background file logging is disabled. [default: 5] + background file logging is disabled. [default: 10] --logfile-max-size The maximum size (in MB) each log file can grow to before rotating. If set to 0, background file logging is disabled. [default: 200] diff --git a/boot_node/src/config.rs b/boot_node/src/config.rs index a8b0f7aa56..aaa9f08482 100644 --- a/boot_node/src/config.rs +++ b/boot_node/src/config.rs @@ -102,14 +102,17 @@ impl BootNodeConfig { .map(Duration::from_secs)?; if eth2_network_config.genesis_state_is_known() { - let genesis_state = eth2_network_config + let mut genesis_state = eth2_network_config .genesis_state::(genesis_state_url.as_deref(), genesis_state_url_timeout, &logger).await? .ok_or_else(|| { "The genesis state for this network is not known, this is an unsupported mode" .to_string() })?; - slog::info!(logger, "Genesis state found"; "root" => genesis_state.canonical_root().to_string()); + let genesis_state_root = genesis_state + .canonical_root() + .map_err(|e| format!("Error hashing genesis state: {e:?}"))?; + slog::info!(logger, "Genesis state found"; "root" => ?genesis_state_root); let enr_fork = spec.enr_fork_id::( types::Slot::from(0u64), genesis_state.genesis_validators_root(), diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index cd53aa7300..3649e247f0 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -351,6 +351,19 @@ impl BeaconNodeHttpClient { Ok(()) } + /// Perform a HTTP POST request with a custom timeout and consensus header. + async fn post_with_timeout_and_consensus_header( + &self, + url: U, + body: &T, + timeout: Duration, + fork_name: ForkName, + ) -> Result<(), Error> { + self.post_generic_with_consensus_version(url, body, Some(timeout), fork_name) + .await?; + Ok(()) + } + /// Perform a HTTP POST request with a custom timeout, returning a JSON response. async fn post_with_timeout_and_response( &self, @@ -381,25 +394,6 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } - /// Generic POST function supporting arbitrary responses and timeouts. - /// Does not include Content-Type application/json in the request header. - async fn post_generic_json_without_content_type_header( - &self, - url: U, - body: &T, - timeout: Option, - ) -> Result { - let mut builder = self.client.post(url); - if let Some(timeout) = timeout { - builder = builder.timeout(timeout); - } - - let serialized_body = serde_json::to_vec(body).map_err(Error::InvalidJson)?; - - let response = builder.body(serialized_body).send().await?; - ok_or_error(response).await - } - /// Generic POST function supporting arbitrary responses and timeouts. async fn post_generic_with_consensus_version( &self, @@ -420,6 +414,23 @@ impl BeaconNodeHttpClient { ok_or_error(response).await } + /// Generic POST function that includes octet-stream content type header. + async fn post_generic_with_ssz_header( + &self, + url: U, + body: &T, + ) -> Result { + let builder = self.client.post(url); + let mut headers = HeaderMap::new(); + + headers.insert( + "Content-Type", + HeaderValue::from_static("application/octet-stream"), + ); + let response = builder.headers(headers).json(body).send().await?; + ok_or_error(response).await + } + /// Generic POST function supporting arbitrary responses and timeouts. async fn post_generic_with_consensus_version_and_ssz_body, U: IntoUrl>( &self, @@ -548,6 +559,26 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } + /// TESTING ONLY: This request should fail with a 415 response code. + pub async fn post_beacon_states_validator_balances_with_ssz_header( + &self, + state_id: StateId, + ids: Vec, + ) -> Result { + let mut path = self.eth_path(V1)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("states") + .push(&state_id.to_string()) + .push("validator_balances"); + + let request = ValidatorBalancesRequestBody { ids }; + + self.post_generic_with_ssz_header(path, &request).await + } + /// `POST beacon/states/{state_id}/validator_balances` /// /// Returns `Ok(None)` on a 404 error. @@ -1196,10 +1227,10 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `GET beacon/blocks/{block_id}/attestations` + /// `GET v1/beacon/blocks/{block_id}/attestations` /// /// Returns `Ok(None)` on a 404 error. - pub async fn get_beacon_blocks_attestations( + pub async fn get_beacon_blocks_attestations_v1( &self, block_id: BlockId, ) -> Result>>>, Error> { @@ -1215,8 +1246,28 @@ impl BeaconNodeHttpClient { self.get_opt(path).await } - /// `POST beacon/pool/attestations` - pub async fn post_beacon_pool_attestations( + /// `GET v2/beacon/blocks/{block_id}/attestations` + /// + /// Returns `Ok(None)` on a 404 error. + pub async fn get_beacon_blocks_attestations_v2( + &self, + block_id: BlockId, + ) -> Result>>>, Error> + { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("blocks") + .push(&block_id.to_string()) + .push("attestations"); + + self.get_opt(path).await + } + + /// `POST v1/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v1( &self, attestations: &[Attestation], ) -> Result<(), Error> { @@ -1234,8 +1285,33 @@ impl BeaconNodeHttpClient { Ok(()) } - /// `GET beacon/pool/attestations?slot,committee_index` - pub async fn get_beacon_pool_attestations( + /// `POST v2/beacon/pool/attestations` + pub async fn post_beacon_pool_attestations_v2( + &self, + attestations: &[Attestation], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + self.post_with_timeout_and_consensus_header( + path, + &attestations, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + + /// `GET v1/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v1( &self, slot: Option, committee_index: Option, @@ -1261,8 +1337,35 @@ impl BeaconNodeHttpClient { self.get(path).await } - /// `POST beacon/pool/attester_slashings` - pub async fn post_beacon_pool_attester_slashings( + /// `GET v2/beacon/pool/attestations?slot,committee_index` + pub async fn get_beacon_pool_attestations_v2( + &self, + slot: Option, + committee_index: Option, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attestations"); + + if let Some(slot) = slot { + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()); + } + + if let Some(index) = committee_index { + path.query_pairs_mut() + .append_pair("committee_index", &index.to_string()); + } + + self.get(path).await + } + + /// `POST v1/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v1( &self, slashing: &AttesterSlashing, ) -> Result<(), Error> { @@ -1274,14 +1377,33 @@ impl BeaconNodeHttpClient { .push("pool") .push("attester_slashings"); - self.post_generic_json_without_content_type_header(path, slashing, None) + self.post_generic(path, slashing, None).await?; + + Ok(()) + } + + /// `POST v2/beacon/pool/attester_slashings` + pub async fn post_beacon_pool_attester_slashings_v2( + &self, + slashing: &AttesterSlashing, + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.post_generic_with_consensus_version(path, slashing, None, fork_name) .await?; Ok(()) } - /// `GET beacon/pool/attester_slashings` - pub async fn get_beacon_pool_attester_slashings( + /// `GET v1/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v1( &self, ) -> Result>>, Error> { let mut path = self.eth_path(V1)?; @@ -1295,6 +1417,21 @@ impl BeaconNodeHttpClient { self.get(path).await } + /// `GET v2/beacon/pool/attester_slashings` + pub async fn get_beacon_pool_attester_slashings_v2( + &self, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("beacon") + .push("pool") + .push("attester_slashings"); + + self.get(path).await + } + /// `POST beacon/pool/proposer_slashings` pub async fn post_beacon_pool_proposer_slashings( &self, @@ -2184,8 +2321,8 @@ impl BeaconNodeHttpClient { self.get_with_timeout(path, self.timeouts.attestation).await } - /// `GET validator/aggregate_attestation?slot,attestation_data_root` - pub async fn get_validator_aggregate_attestation( + /// `GET v1/validator/aggregate_attestation?slot,attestation_data_root` + pub async fn get_validator_aggregate_attestation_v1( &self, slot: Slot, attestation_data_root: Hash256, @@ -2208,6 +2345,32 @@ impl BeaconNodeHttpClient { .await } + /// `GET v2/validator/aggregate_attestation?slot,attestation_data_root,committee_index` + pub async fn get_validator_aggregate_attestation_v2( + &self, + slot: Slot, + attestation_data_root: Hash256, + committee_index: CommitteeIndex, + ) -> Result>>, Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_attestation"); + + path.query_pairs_mut() + .append_pair("slot", &slot.to_string()) + .append_pair( + "attestation_data_root", + &format!("{:?}", attestation_data_root), + ) + .append_pair("committee_index", &committee_index.to_string()); + + self.get_opt_with_timeout(path, self.timeouts.attestation) + .await + } + /// `GET validator/sync_committee_contribution` pub async fn get_validator_sync_committee_contribution( &self, @@ -2303,8 +2466,8 @@ impl BeaconNodeHttpClient { .await } - /// `POST validator/aggregate_and_proofs` - pub async fn post_validator_aggregate_and_proof( + /// `POST v1/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v1( &self, aggregates: &[SignedAggregateAndProof], ) -> Result<(), Error> { @@ -2321,6 +2484,30 @@ impl BeaconNodeHttpClient { Ok(()) } + /// `POST v2/validator/aggregate_and_proofs` + pub async fn post_validator_aggregate_and_proof_v2( + &self, + aggregates: &[SignedAggregateAndProof], + fork_name: ForkName, + ) -> Result<(), Error> { + let mut path = self.eth_path(V2)?; + + path.path_segments_mut() + .map_err(|()| Error::InvalidUrl(self.server.clone()))? + .push("validator") + .push("aggregate_and_proofs"); + + self.post_with_timeout_and_consensus_header( + path, + &aggregates, + self.timeouts.attestation, + fork_name, + ) + .await?; + + Ok(()) + } + /// `POST validator/beacon_committee_subscriptions` pub async fn post_validator_beacon_committee_subscriptions( &self, diff --git a/common/eth2/src/lighthouse_vc/http_client.rs b/common/eth2/src/lighthouse_vc/http_client.rs index 83aeea4bfc..67fe77a315 100644 --- a/common/eth2/src/lighthouse_vc/http_client.rs +++ b/common/eth2/src/lighthouse_vc/http_client.rs @@ -1,13 +1,10 @@ -use super::{types::*, PK_LEN, SECRET_PREFIX}; +use super::types::*; use crate::Error; use account_utils::ZeroizeString; -use bytes::Bytes; -use libsecp256k1::{Message, PublicKey, Signature}; use reqwest::{ header::{HeaderMap, HeaderValue}, IntoUrl, }; -use ring::digest::{digest, SHA256}; use sensitive_url::SensitiveUrl; use serde::{de::DeserializeOwned, Serialize}; use std::fmt::{self, Display}; @@ -24,8 +21,7 @@ use types::graffiti::GraffitiString; pub struct ValidatorClientHttpClient { client: reqwest::Client, server: SensitiveUrl, - secret: Option, - server_pubkey: Option, + api_token: Option, authorization_header: AuthorizationHeader, } @@ -46,45 +42,13 @@ impl Display for AuthorizationHeader { } } -/// Parse an API token and return a secp256k1 public key. -/// -/// If the token does not start with the Lighthouse token prefix then `Ok(None)` will be returned. -/// An error will be returned if the token looks like a Lighthouse token but doesn't correspond to a -/// valid public key. -pub fn parse_pubkey(secret: &str) -> Result, Error> { - let secret = if !secret.starts_with(SECRET_PREFIX) { - return Ok(None); - } else { - &secret[SECRET_PREFIX.len()..] - }; - - serde_utils::hex::decode(secret) - .map_err(|e| Error::InvalidSecret(format!("invalid hex: {:?}", e))) - .and_then(|bytes| { - if bytes.len() != PK_LEN { - return Err(Error::InvalidSecret(format!( - "expected {} bytes not {}", - PK_LEN, - bytes.len() - ))); - } - - let mut arr = [0; PK_LEN]; - arr.copy_from_slice(&bytes); - PublicKey::parse_compressed(&arr) - .map_err(|e| Error::InvalidSecret(format!("invalid secp256k1 pubkey: {:?}", e))) - }) - .map(Some) -} - impl ValidatorClientHttpClient { /// Create a new client pre-initialised with an API token. pub fn new(server: SensitiveUrl, secret: String) -> Result { Ok(Self { client: reqwest::Client::new(), server, - server_pubkey: parse_pubkey(&secret)?, - secret: Some(secret.into()), + api_token: Some(secret.into()), authorization_header: AuthorizationHeader::Bearer, }) } @@ -96,8 +60,7 @@ impl ValidatorClientHttpClient { Ok(Self { client: reqwest::Client::new(), server, - secret: None, - server_pubkey: None, + api_token: None, authorization_header: AuthorizationHeader::Omit, }) } @@ -110,15 +73,14 @@ impl ValidatorClientHttpClient { Ok(Self { client, server, - server_pubkey: parse_pubkey(&secret)?, - secret: Some(secret.into()), + api_token: Some(secret.into()), authorization_header: AuthorizationHeader::Bearer, }) } /// Get a reference to this client's API token, if any. pub fn api_token(&self) -> Option<&ZeroizeString> { - self.secret.as_ref() + self.api_token.as_ref() } /// Read an API token from the specified `path`, stripping any trailing whitespace. @@ -128,19 +90,11 @@ impl ValidatorClientHttpClient { } /// Add an authentication token to use when making requests. - /// - /// If the token is Lighthouse-like, a pubkey derivation will be attempted. In the case - /// of failure the token will still be stored, and the client can continue to be used to - /// communicate with non-Lighthouse nodes. pub fn add_auth_token(&mut self, token: ZeroizeString) -> Result<(), Error> { - let pubkey_res = parse_pubkey(token.as_str()); - - self.secret = Some(token); + self.api_token = Some(token); self.authorization_header = AuthorizationHeader::Bearer; - pubkey_res.map(|opt_pubkey| { - self.server_pubkey = opt_pubkey; - }) + Ok(()) } /// Set to `false` to disable sending the `Authorization` header on requests. @@ -160,49 +114,17 @@ impl ValidatorClientHttpClient { self.authorization_header = AuthorizationHeader::Basic; } - async fn signed_body(&self, response: Response) -> Result { - let server_pubkey = self.server_pubkey.as_ref().ok_or(Error::NoServerPubkey)?; - let sig = response - .headers() - .get("Signature") - .ok_or(Error::MissingSignatureHeader)? - .to_str() - .map_err(|_| Error::InvalidSignatureHeader)? - .to_string(); - - let body = response.bytes().await.map_err(Error::from)?; - - let message = - Message::parse_slice(digest(&SHA256, &body).as_ref()).expect("sha256 is 32 bytes"); - - serde_utils::hex::decode(&sig) - .ok() - .and_then(|bytes| { - let sig = Signature::parse_der(&bytes).ok()?; - Some(libsecp256k1::verify(&message, &sig, server_pubkey)) - }) - .filter(|is_valid| *is_valid) - .ok_or(Error::InvalidSignatureHeader)?; - - Ok(body) - } - - async fn signed_json(&self, response: Response) -> Result { - let body = self.signed_body(response).await?; - serde_json::from_slice(&body).map_err(Error::InvalidJson) - } - fn headers(&self) -> Result { let mut headers = HeaderMap::new(); if self.authorization_header == AuthorizationHeader::Basic || self.authorization_header == AuthorizationHeader::Bearer { - let secret = self.secret.as_ref().ok_or(Error::NoToken)?; + let auth_header_token = self.api_token().ok_or(Error::NoToken)?; let header_value = HeaderValue::from_str(&format!( "{} {}", self.authorization_header, - secret.as_str() + auth_header_token.as_str() )) .map_err(|e| { Error::InvalidSecret(format!("secret is invalid as a header value: {}", e)) @@ -240,7 +162,8 @@ impl ValidatorClientHttpClient { async fn get(&self, url: U) -> Result { let response = self.get_response(url).await?; - self.signed_json(response).await + let body = response.bytes().await.map_err(Error::from)?; + serde_json::from_slice(&body).map_err(Error::InvalidJson) } async fn delete(&self, url: U) -> Result<(), Error> { @@ -263,7 +186,14 @@ impl ValidatorClientHttpClient { /// Perform a HTTP GET request, returning `None` on a 404 error. async fn get_opt(&self, url: U) -> Result, Error> { match self.get_response(url).await { - Ok(resp) => self.signed_json(resp).await.map(Option::Some), + Ok(resp) => { + let body = resp.bytes().await.map(Option::Some)?; + if let Some(body) = body { + serde_json::from_slice(&body).map_err(Error::InvalidJson) + } else { + Ok(None) + } + } Err(err) => { if err.status() == Some(StatusCode::NOT_FOUND) { Ok(None) @@ -297,7 +227,8 @@ impl ValidatorClientHttpClient { body: &T, ) -> Result { let response = self.post_with_raw_response(url, body).await?; - self.signed_json(response).await + let body = response.bytes().await.map_err(Error::from)?; + serde_json::from_slice(&body).map_err(Error::InvalidJson) } async fn post_with_unsigned_response( @@ -319,8 +250,7 @@ impl ValidatorClientHttpClient { .send() .await .map_err(Error::from)?; - let response = ok_or_error(response).await?; - self.signed_body(response).await?; + ok_or_error(response).await?; Ok(()) } diff --git a/common/eth2/src/lighthouse_vc/mod.rs b/common/eth2/src/lighthouse_vc/mod.rs index 81b4fca283..038726c829 100644 --- a/common/eth2/src/lighthouse_vc/mod.rs +++ b/common/eth2/src/lighthouse_vc/mod.rs @@ -1,10 +1,3 @@ pub mod http_client; pub mod std_types; pub mod types; - -/// The number of bytes in the secp256k1 public key used as the authorization token for the VC API. -pub const PK_LEN: usize = 33; - -/// The prefix for the secp256k1 public key when it is used as the authorization token for the VC -/// API. -pub const SECRET_PREFIX: &str = "api-token-"; diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 70aa5aab3e..bbcbda3ae5 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -780,6 +780,8 @@ pub struct ValidatorAttestationDataQuery { pub struct ValidatorAggregateAttestationQuery { pub attestation_data_root: Hash256, pub slot: Slot, + #[serde(skip_serializing_if = "Option::is_none")] + pub committee_index: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] @@ -969,6 +971,11 @@ pub struct SseHead { pub execution_optimistic: bool, } +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] +pub struct BlockGossip { + pub slot: Slot, + pub block: Hash256, +} #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] pub struct SseChainReorg { pub slot: Slot, @@ -1098,6 +1105,7 @@ pub enum EventKind { ProposerSlashing(Box), AttesterSlashing(Box>), BlsToExecutionChange(Box), + BlockGossip(Box), } impl EventKind { @@ -1120,6 +1128,7 @@ impl EventKind { EventKind::ProposerSlashing(_) => "proposer_slashing", EventKind::AttesterSlashing(_) => "attester_slashing", EventKind::BlsToExecutionChange(_) => "bls_to_execution_change", + EventKind::BlockGossip(_) => "block_gossip", } } @@ -1215,6 +1224,9 @@ impl EventKind { ServerError::InvalidServerSentEvent(format!("Bls To Execution Change: {:?}", e)) })?, )), + "block_gossip" => Ok(EventKind::BlockGossip(serde_json::from_str(data).map_err( + |e| ServerError::InvalidServerSentEvent(format!("Block Gossip: {:?}", e)), + )?)), _ => Err(ServerError::InvalidServerSentEvent( "Could not parse event tag".to_string(), )), @@ -1249,6 +1261,7 @@ pub enum EventTopic { AttesterSlashing, ProposerSlashing, BlsToExecutionChange, + BlockGossip, } impl FromStr for EventTopic { @@ -1273,6 +1286,7 @@ impl FromStr for EventTopic { "attester_slashing" => Ok(EventTopic::AttesterSlashing), "proposer_slashing" => Ok(EventTopic::ProposerSlashing), "bls_to_execution_change" => Ok(EventTopic::BlsToExecutionChange), + "block_gossip" => Ok(EventTopic::BlockGossip), _ => Err("event topic cannot be parsed.".to_string()), } } @@ -1298,6 +1312,7 @@ impl fmt::Display for EventTopic { EventTopic::AttesterSlashing => write!(f, "attester_slashing"), EventTopic::ProposerSlashing => write!(f, "proposer_slashing"), EventTopic::BlsToExecutionChange => write!(f, "bls_to_execution_change"), + EventTopic::BlockGossip => write!(f, "block_gossip"), } } } diff --git a/common/warp_utils/src/json.rs b/common/warp_utils/src/json.rs index 203a6495a4..6ee5e77261 100644 --- a/common/warp_utils/src/json.rs +++ b/common/warp_utils/src/json.rs @@ -1,4 +1,5 @@ use bytes::Bytes; +use eth2::{CONTENT_TYPE_HEADER, SSZ_CONTENT_TYPE_HEADER}; use serde::de::DeserializeOwned; use std::error::Error as StdError; use warp::{Filter, Rejection}; @@ -16,7 +17,17 @@ impl Json { } pub fn json() -> impl Filter + Copy { - warp::body::bytes().and_then(|bytes: Bytes| async move { - Json::decode(bytes).map_err(|err| reject::custom_deserialize_error(format!("{:?}", err))) - }) + warp::header::optional::(CONTENT_TYPE_HEADER) + .and(warp::body::bytes()) + .and_then(|header: Option, bytes: Bytes| async move { + if let Some(header) = header { + if header == SSZ_CONTENT_TYPE_HEADER { + return Err(reject::unsupported_media_type( + "The request's content-type is not supported".to_string(), + )); + } + } + Json::decode(bytes) + .map_err(|err| reject::custom_deserialize_error(format!("{:?}", err))) + }) } diff --git a/common/warp_utils/src/reject.rs b/common/warp_utils/src/reject.rs index b6bb5ace3d..9b28c65212 100644 --- a/common/warp_utils/src/reject.rs +++ b/common/warp_utils/src/reject.rs @@ -2,7 +2,7 @@ use eth2::types::{ErrorMessage, Failure, IndexedErrorMessage}; use std::convert::Infallible; use std::error::Error; use std::fmt; -use warp::{http::StatusCode, reject::Reject}; +use warp::{http::StatusCode, reject::Reject, reply::Response, Reply}; #[derive(Debug)] pub struct ServerSentEventError(pub String); @@ -136,6 +136,15 @@ pub fn invalid_auth(msg: String) -> warp::reject::Rejection { warp::reject::custom(InvalidAuthorization(msg)) } +#[derive(Debug)] +pub struct UnsupportedMediaType(pub String); + +impl Reject for UnsupportedMediaType {} + +pub fn unsupported_media_type(msg: String) -> warp::reject::Rejection { + warp::reject::custom(UnsupportedMediaType(msg)) +} + #[derive(Debug)] pub struct IndexedBadRequestErrors { pub message: String, @@ -170,6 +179,9 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result().is_some() { + code = StatusCode::UNSUPPORTED_MEDIA_TYPE; + message = "UNSUPPORTED_MEDIA_TYPE".to_string(); } else if let Some(e) = err.find::() { message = format!("BAD_REQUEST: body deserialize error: {}", e.0); code = StatusCode::BAD_REQUEST; @@ -243,3 +255,21 @@ pub async fn handle_rejection(err: warp::Rejection) -> Result(res: Result) -> Response { + match res { + Ok(response) => response.into_response(), + Err(e) => match handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + warp::reply::json(&"unhandled error"), + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } +} diff --git a/common/warp_utils/src/task.rs b/common/warp_utils/src/task.rs index 001231f2c6..e2fa4ebc36 100644 --- a/common/warp_utils/src/task.rs +++ b/common/warp_utils/src/task.rs @@ -1,3 +1,4 @@ +use crate::reject::convert_rejection; use serde::Serialize; use warp::reply::{Reply, Response}; @@ -24,14 +25,16 @@ where } /// A convenience wrapper around `blocking_task` for use with `warp` JSON responses. -pub async fn blocking_json_task(func: F) -> Result +pub async fn blocking_json_task(func: F) -> Response where F: FnOnce() -> Result + Send + 'static, T: Serialize + Send + 'static, { - blocking_response_task(|| { + let result = blocking_response_task(|| { let response = func()?; Ok(warp::reply::json(&response)) }) - .await + .await; + + convert_rejection(result).await } diff --git a/consensus/proto_array/src/lib.rs b/consensus/proto_array/src/lib.rs index 780563954c..b05a55e686 100644 --- a/consensus/proto_array/src/lib.rs +++ b/consensus/proto_array/src/lib.rs @@ -16,5 +16,5 @@ pub use error::Error; pub mod core { pub use super::proto_array::{ProposerBoost, ProtoArray, ProtoNode}; pub use super::proto_array_fork_choice::VoteTracker; - pub use super::ssz_container::{SszContainer, SszContainerV16, SszContainerV17}; + pub use super::ssz_container::{SszContainer, SszContainerV17}; } diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index d50153bfe8..efe154a27e 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -70,7 +70,7 @@ impl InvalidationOperation { pub type ProtoNode = ProtoNodeV17; #[superstruct( - variants(V16, V17), + variants(V17), variant_attributes(derive(Clone, PartialEq, Debug, Encode, Decode, Serialize, Deserialize)), no_enum )] @@ -92,12 +92,6 @@ pub struct ProtoNode { pub root: Hash256, #[ssz(with = "four_byte_option_usize")] pub parent: Option, - #[superstruct(only(V16))] - #[ssz(with = "four_byte_option_checkpoint")] - pub justified_checkpoint: Option, - #[superstruct(only(V16))] - #[ssz(with = "four_byte_option_checkpoint")] - pub finalized_checkpoint: Option, #[superstruct(only(V17))] pub justified_checkpoint: Checkpoint, #[superstruct(only(V17))] @@ -116,57 +110,6 @@ pub struct ProtoNode { pub unrealized_finalized_checkpoint: Option, } -impl TryInto for ProtoNodeV16 { - type Error = Error; - - fn try_into(self) -> Result { - let result = ProtoNode { - slot: self.slot, - state_root: self.state_root, - target_root: self.target_root, - current_epoch_shuffling_id: self.current_epoch_shuffling_id, - next_epoch_shuffling_id: self.next_epoch_shuffling_id, - root: self.root, - parent: self.parent, - justified_checkpoint: self - .justified_checkpoint - .ok_or(Error::MissingJustifiedCheckpoint)?, - finalized_checkpoint: self - .finalized_checkpoint - .ok_or(Error::MissingFinalizedCheckpoint)?, - weight: self.weight, - best_child: self.best_child, - best_descendant: self.best_descendant, - execution_status: self.execution_status, - unrealized_justified_checkpoint: self.unrealized_justified_checkpoint, - unrealized_finalized_checkpoint: self.unrealized_finalized_checkpoint, - }; - Ok(result) - } -} - -impl From for ProtoNodeV16 { - fn from(from: ProtoNode) -> ProtoNodeV16 { - ProtoNodeV16 { - slot: from.slot, - state_root: from.state_root, - target_root: from.target_root, - current_epoch_shuffling_id: from.current_epoch_shuffling_id, - next_epoch_shuffling_id: from.next_epoch_shuffling_id, - root: from.root, - parent: from.parent, - justified_checkpoint: Some(from.justified_checkpoint), - finalized_checkpoint: Some(from.finalized_checkpoint), - weight: from.weight, - best_child: from.best_child, - best_descendant: from.best_descendant, - execution_status: from.execution_status, - unrealized_justified_checkpoint: from.unrealized_justified_checkpoint, - unrealized_finalized_checkpoint: from.unrealized_finalized_checkpoint, - } - } -} - #[derive(PartialEq, Debug, Encode, Decode, Serialize, Deserialize, Copy, Clone)] pub struct ProposerBoost { pub root: Hash256, diff --git a/consensus/proto_array/src/ssz_container.rs b/consensus/proto_array/src/ssz_container.rs index a6d585758b..8abb60d8e6 100644 --- a/consensus/proto_array/src/ssz_container.rs +++ b/consensus/proto_array/src/ssz_container.rs @@ -1,6 +1,6 @@ use crate::proto_array::ProposerBoost; use crate::{ - proto_array::{ProtoArray, ProtoNodeV16, ProtoNodeV17}, + proto_array::{ProtoArray, ProtoNodeV17}, proto_array_fork_choice::{ElasticList, ProtoArrayForkChoice, VoteTracker}, Error, JustifiedBalances, }; @@ -16,62 +16,19 @@ four_byte_option_impl!(four_byte_option_checkpoint, Checkpoint); pub type SszContainer = SszContainerV17; -#[superstruct( - variants(V16, V17), - variant_attributes(derive(Encode, Decode)), - no_enum -)] +#[superstruct(variants(V17), variant_attributes(derive(Encode, Decode)), no_enum)] pub struct SszContainer { pub votes: Vec, pub balances: Vec, pub prune_threshold: usize, pub justified_checkpoint: Checkpoint, pub finalized_checkpoint: Checkpoint, - #[superstruct(only(V16))] - pub nodes: Vec, #[superstruct(only(V17))] pub nodes: Vec, pub indices: Vec<(Hash256, usize)>, pub previous_proposer_boost: ProposerBoost, } -impl TryInto for SszContainerV16 { - type Error = Error; - - fn try_into(self) -> Result { - let nodes: Result, Error> = - self.nodes.into_iter().map(TryInto::try_into).collect(); - - Ok(SszContainer { - votes: self.votes, - balances: self.balances, - prune_threshold: self.prune_threshold, - justified_checkpoint: self.justified_checkpoint, - finalized_checkpoint: self.finalized_checkpoint, - nodes: nodes?, - indices: self.indices, - previous_proposer_boost: self.previous_proposer_boost, - }) - } -} - -impl From for SszContainerV16 { - fn from(from: SszContainer) -> SszContainerV16 { - let nodes = from.nodes.into_iter().map(Into::into).collect(); - - SszContainerV16 { - votes: from.votes, - balances: from.balances, - prune_threshold: from.prune_threshold, - justified_checkpoint: from.justified_checkpoint, - finalized_checkpoint: from.finalized_checkpoint, - nodes, - indices: from.indices, - previous_proposer_boost: from.previous_proposer_boost, - } - } -} - impl From<&ProtoArrayForkChoice> for SszContainer { fn from(from: &ProtoArrayForkChoice) -> Self { let proto_array = &from.proto_array; diff --git a/consensus/types/src/attestation.rs b/consensus/types/src/attestation.rs index 88993267a9..7b53a98caa 100644 --- a/consensus/types/src/attestation.rs +++ b/consensus/types/src/attestation.rs @@ -1,6 +1,6 @@ use crate::slot_data::SlotData; -use crate::Checkpoint; use crate::{test_utils::TestRandom, Hash256, Slot}; +use crate::{Checkpoint, ForkVersionDeserialize}; use derivative::Derivative; use safe_arith::ArithError; use serde::{Deserialize, Serialize}; @@ -26,6 +26,12 @@ pub enum Error { InvalidCommitteeIndex, } +impl From for Error { + fn from(e: ssz_types::Error) -> Self { + Error::SszTypesError(e) + } +} + #[superstruct( variants(Base, Electra), variant_attributes( @@ -487,6 +493,46 @@ impl<'a, E: EthSpec> From> for AttestationRef<'a, E> } } +impl ForkVersionDeserialize for Attestation { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestation: AttestationElectra = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Electra(attestation)) + } else { + let attestation: AttestationBase = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(Attestation::Base(attestation)) + } + } +} + +impl ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Electra) + .collect::>()) + } else { + let attestations: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(attestations + .into_iter() + .map(Attestation::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/attester_slashing.rs b/consensus/types/src/attester_slashing.rs index c8e2fb4f82..f6aa654d44 100644 --- a/consensus/types/src/attester_slashing.rs +++ b/consensus/types/src/attester_slashing.rs @@ -171,6 +171,29 @@ impl TestRandom for AttesterSlashing { } } +impl crate::ForkVersionDeserialize for Vec> { + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::Value, + fork_name: crate::ForkName, + ) -> Result { + if fork_name.electra_enabled() { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Electra) + .collect::>()) + } else { + let slashings: Vec> = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + Ok(slashings + .into_iter() + .map(AttesterSlashing::Base) + .collect::>()) + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 9cf66184f8..363ba08f7d 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -226,6 +226,19 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, Ok(proof.into()) } + /// Produces the proof of inclusion for `self.blob_kzg_commitments`. + pub fn kzg_commitments_merkle_proof( + &self, + ) -> Result, Error> { + let body_leaves = self.body_merkle_leaves(); + let beacon_block_body_depth = body_leaves.len().next_power_of_two().ilog2() as usize; + let tree = MerkleTree::create(&body_leaves, beacon_block_body_depth); + let (_, proof) = tree + .generate_proof(BLOB_KZG_COMMITMENTS_INDEX, beacon_block_body_depth) + .map_err(Error::MerkleTreeError)?; + Ok(proof.into()) + } + /// Return `true` if this block body has a non-zero number of blobs. pub fn has_blobs(self) -> bool { self.blob_kzg_commitments() diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 0622039179..456aa98fad 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -214,6 +214,13 @@ impl From for Hash256 { } /// The state of the `BeaconChain` at some slot. +/// +/// Note: `BeaconState` does not implement `TreeHash` on the top-level type in order to +/// encourage use of the `canonical_root`/`update_tree_hash_cache` methods which flush pending +/// updates to the underlying persistent data structures. This is the safest option for now until +/// we add internal mutability to `milhouse::{List, Vector}`. See: +/// +/// https://github.com/sigp/milhouse/issues/43 #[superstruct( variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes( @@ -324,13 +331,10 @@ impl From for Hash256 { partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant"), map_ref_mut_into(BeaconStateRef) )] -#[derive( - Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash, arbitrary::Arbitrary, -)] +#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Encode, arbitrary::Arbitrary)] #[serde(untagged)] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] -#[tree_hash(enum_behaviour = "transparent")] #[ssz(enum_behaviour = "transparent")] pub struct BeaconState where @@ -652,10 +656,8 @@ impl BeaconState { } /// Returns the `tree_hash_root` of the state. - /// - /// Spec v0.12.1 - pub fn canonical_root(&self) -> Hash256 { - Hash256::from_slice(&self.tree_hash_root()[..]) + pub fn canonical_root(&mut self) -> Result { + self.update_tree_hash_cache() } pub fn historical_batch(&mut self) -> Result, Error> { @@ -2016,9 +2018,13 @@ impl BeaconState { /// Compute the tree hash root of the state using the tree hash cache. /// /// Initialize the tree hash cache if it isn't already initialized. - pub fn update_tree_hash_cache(&mut self) -> Result { + pub fn update_tree_hash_cache<'a>(&'a mut self) -> Result { self.apply_pending_mutations()?; - Ok(self.tree_hash_root()) + map_beacon_state_ref!(&'a _, self.to_ref(), |inner, cons| { + let root = inner.tree_hash_root(); + cons(inner); + Ok(root) + }) } /// Compute the tree hash root of the validators using the tree hash cache. diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 16d4f046ba..0f61f74efe 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -190,6 +190,11 @@ pub struct ChainSpec { pub min_per_epoch_churn_limit_electra: u64, pub max_per_epoch_activation_exit_churn_limit: u64, + /* + * DAS params + */ + pub number_of_columns: usize, + /* * Networking */ @@ -772,6 +777,8 @@ impl ChainSpec { }) .expect("calculation does not overflow"), + number_of_columns: 128, + /* * Network specific */ @@ -1074,6 +1081,8 @@ impl ChainSpec { }) .expect("calculation does not overflow"), + number_of_columns: 128, + /* * Network specific */ diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs new file mode 100644 index 0000000000..a0e3ca6cce --- /dev/null +++ b/consensus/types/src/data_column_sidecar.rs @@ -0,0 +1,394 @@ +use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX}; +use crate::test_utils::TestRandom; +use crate::{ + BeaconBlockHeader, ChainSpec, EthSpec, Hash256, KzgProofs, SignedBeaconBlock, + SignedBeaconBlockHeader, Slot, +}; +use crate::{BeaconStateError, BlobsList}; +use bls::Signature; +use derivative::Derivative; +use kzg::Kzg; +use kzg::{Blob as KzgBlob, Cell as KzgCell, Error as KzgError}; +use kzg::{KzgCommitment, KzgProof}; +use merkle_proof::verify_merkle_proof; +use rayon::prelude::*; +use safe_arith::ArithError; +use serde::{Deserialize, Serialize}; +use ssz::Encode; +use ssz_derive::{Decode, Encode}; +use ssz_types::typenum::Unsigned; +use ssz_types::Error as SszError; +use ssz_types::{FixedVector, VariableList}; +use std::hash::Hash; +use std::sync::Arc; +use test_random_derive::TestRandom; +use tree_hash::TreeHash; +use tree_hash_derive::TreeHash; + +pub type ColumnIndex = u64; +pub type Cell = FixedVector::BytesPerCell>; +pub type DataColumn = VariableList, ::MaxBlobCommitmentsPerBlock>; + +/// Container of the data that identifies an individual data column. +#[derive( + Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash, +)] +pub struct DataColumnIdentifier { + pub block_root: Hash256, + pub index: ColumnIndex, +} + +pub type DataColumnSidecarList = Vec>>; + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[serde(bound = "E: EthSpec")] +#[arbitrary(bound = "E: EthSpec")] +#[derivative(PartialEq, Eq, Hash(bound = "E: EthSpec"))] +pub struct DataColumnSidecar { + #[serde(with = "serde_utils::quoted_u64")] + pub index: ColumnIndex, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub column: DataColumn, + /// All of the KZG commitments and proofs associated with the block, used for verifying sample cells. + pub kzg_commitments: KzgCommitments, + pub kzg_proofs: KzgProofs, + pub signed_block_header: SignedBeaconBlockHeader, + /// An inclusion proof, proving the inclusion of `blob_kzg_commitments` in `BeaconBlockBody`. + pub kzg_commitments_inclusion_proof: FixedVector, +} + +impl DataColumnSidecar { + pub fn slot(&self) -> Slot { + self.signed_block_header.message.slot + } + + pub fn block_root(&self) -> Hash256 { + self.signed_block_header.message.tree_hash_root() + } + + pub fn block_parent_root(&self) -> Hash256 { + self.signed_block_header.message.parent_root + } + + pub fn block_proposer_index(&self) -> u64 { + self.signed_block_header.message.proposer_index + } + + /// Verifies the kzg commitment inclusion merkle proof. + pub fn verify_inclusion_proof(&self) -> bool { + let blob_kzg_commitments_root = self.kzg_commitments.tree_hash_root(); + + verify_merkle_proof( + blob_kzg_commitments_root, + &self.kzg_commitments_inclusion_proof, + E::kzg_commitments_inclusion_proof_depth(), + BLOB_KZG_COMMITMENTS_INDEX, + self.signed_block_header.message.body_root, + ) + } + + pub fn build_sidecars( + blobs: &BlobsList, + block: &SignedBeaconBlock, + kzg: &Kzg, + spec: &ChainSpec, + ) -> Result, DataColumnSidecarError> { + let number_of_columns = spec.number_of_columns; + if blobs.is_empty() { + return Ok(vec![]); + } + let kzg_commitments = block + .message() + .body() + .blob_kzg_commitments() + .map_err(|_err| DataColumnSidecarError::PreDeneb)?; + let kzg_commitments_inclusion_proof = + block.message().body().kzg_commitments_merkle_proof()?; + let signed_block_header = block.signed_block_header(); + + let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + let mut column_kzg_proofs = + vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + + // NOTE: assumes blob sidecars are ordered by index + let blob_cells_and_proofs_vec = blobs + .into_par_iter() + .map(|blob| { + let blob = KzgBlob::from_bytes(blob).map_err(KzgError::from)?; + kzg.compute_cells_and_proofs(&blob) + }) + .collect::, KzgError>>()?; + + for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec { + // we iterate over each column, and we construct the column from "top to bottom", + // pushing on the cell and the corresponding proof at each column index. we do this for + // each blob (i.e. the outer loop). + for col in 0..number_of_columns { + let cell = + blob_cells + .get(col) + .ok_or(DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing blob cell at index {col}" + )))?; + let cell: Vec = cell.into_inner().into_iter().collect(); + let cell = Cell::::from(cell); + + let proof = blob_cell_proofs.get(col).ok_or( + DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing blob cell KZG proof at index {col}" + )), + )?; + + let column = + columns + .get_mut(col) + .ok_or(DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing data column at index {col}" + )))?; + let column_proofs = column_kzg_proofs.get_mut(col).ok_or( + DataColumnSidecarError::InconsistentArrayLength(format!( + "Missing data column proofs at index {col}" + )), + )?; + + column.push(cell); + column_proofs.push(*proof); + } + } + + let sidecars: Vec>> = columns + .into_iter() + .zip(column_kzg_proofs) + .enumerate() + .map(|(index, (col, proofs))| { + Arc::new(DataColumnSidecar { + index: index as u64, + column: DataColumn::::from(col), + kzg_commitments: kzg_commitments.clone(), + kzg_proofs: KzgProofs::::from(proofs), + signed_block_header: signed_block_header.clone(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }) + }) + .collect(); + + Ok(sidecars) + } + + pub fn reconstruct( + kzg: &Kzg, + data_columns: &[Arc], + spec: &ChainSpec, + ) -> Result>, KzgError> { + let number_of_columns = spec.number_of_columns; + let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + let mut column_kzg_proofs = + vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + + let first_data_column = data_columns + .first() + .ok_or(KzgError::InconsistentArrayLength( + "data_columns should have at least one element".to_string(), + ))?; + let num_of_blobs = first_data_column.kzg_commitments.len(); + + let blob_cells_and_proofs_vec = (0..num_of_blobs) + .into_par_iter() + .map(|row_index| { + let mut cells: Vec = vec![]; + let mut cell_ids: Vec = vec![]; + for data_column in data_columns { + let cell = data_column.column.get(row_index).ok_or( + KzgError::InconsistentArrayLength(format!( + "Missing data column at index {row_index}" + )), + )?; + + cells.push(ssz_cell_to_crypto_cell::(cell)?); + cell_ids.push(data_column.index); + } + // recover_all_cells does not expect sorted + let all_cells = kzg.recover_all_cells(&cell_ids, &cells)?; + let blob = kzg.cells_to_blob(&all_cells)?; + + // Note: This function computes all cells and proofs. According to Justin this is okay, + // computing a partial set may be more expensive and requires code paths that don't exist. + // Computing the blobs cells is technically unnecessary but very cheap. It's done here again + // for simplicity. + kzg.compute_cells_and_proofs(&blob) + }) + .collect::, KzgError>>()?; + + for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec { + // we iterate over each column, and we construct the column from "top to bottom", + // pushing on the cell and the corresponding proof at each column index. we do this for + // each blob (i.e. the outer loop). + for col in 0..number_of_columns { + let cell = blob_cells + .get(col) + .ok_or(KzgError::InconsistentArrayLength(format!( + "Missing blob cell at index {col}" + )))?; + let cell: Vec = cell.into_inner().into_iter().collect(); + let cell = Cell::::from(cell); + + let proof = blob_cell_proofs + .get(col) + .ok_or(KzgError::InconsistentArrayLength(format!( + "Missing blob cell KZG proof at index {col}" + )))?; + + let column = columns + .get_mut(col) + .ok_or(KzgError::InconsistentArrayLength(format!( + "Missing data column at index {col}" + )))?; + let column_proofs = + column_kzg_proofs + .get_mut(col) + .ok_or(KzgError::InconsistentArrayLength(format!( + "Missing data column proofs at index {col}" + )))?; + + column.push(cell); + column_proofs.push(*proof); + } + } + + // Clone sidecar elements from existing data column, no need to re-compute + let kzg_commitments = &first_data_column.kzg_commitments; + let signed_block_header = &first_data_column.signed_block_header; + let kzg_commitments_inclusion_proof = &first_data_column.kzg_commitments_inclusion_proof; + + let sidecars: Vec>> = columns + .into_iter() + .zip(column_kzg_proofs) + .enumerate() + .map(|(index, (col, proofs))| { + Arc::new(DataColumnSidecar { + index: index as u64, + column: DataColumn::::from(col), + kzg_commitments: kzg_commitments.clone(), + kzg_proofs: KzgProofs::::from(proofs), + signed_block_header: signed_block_header.clone(), + kzg_commitments_inclusion_proof: kzg_commitments_inclusion_proof.clone(), + }) + }) + .collect(); + Ok(sidecars) + } + + pub fn min_size() -> usize { + // min size is one cell + Self { + index: 0, + column: VariableList::new(vec![Cell::::default()]).unwrap(), + kzg_commitments: VariableList::new(vec![KzgCommitment::empty_for_testing()]).unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty()]).unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader::empty(), + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: Default::default(), + } + .as_ssz_bytes() + .len() + } + + pub fn max_size() -> usize { + Self { + index: 0, + column: VariableList::new(vec![Cell::::default(); E::MaxBlobsPerBlock::to_usize()]) + .unwrap(), + kzg_commitments: VariableList::new(vec![ + KzgCommitment::empty_for_testing(); + E::MaxBlobsPerBlock::to_usize() + ]) + .unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); E::MaxBlobsPerBlock::to_usize()]) + .unwrap(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader::empty(), + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: Default::default(), + } + .as_ssz_bytes() + .len() + } + + pub fn empty() -> Self { + Self { + index: 0, + column: DataColumn::::default(), + kzg_commitments: VariableList::default(), + kzg_proofs: VariableList::default(), + signed_block_header: SignedBeaconBlockHeader { + message: BeaconBlockHeader::empty(), + signature: Signature::empty(), + }, + kzg_commitments_inclusion_proof: Default::default(), + } + } + + pub fn id(&self) -> DataColumnIdentifier { + DataColumnIdentifier { + block_root: self.block_root(), + index: self.index, + } + } +} + +#[derive(Debug)] +pub enum DataColumnSidecarError { + ArithError(ArithError), + BeaconStateError(BeaconStateError), + DataColumnIndexOutOfBounds, + KzgCommitmentInclusionProofOutOfBounds, + KzgError(KzgError), + MissingBlobSidecars, + PreDeneb, + SszError(SszError), + InconsistentArrayLength(String), +} + +impl From for DataColumnSidecarError { + fn from(e: ArithError) -> Self { + Self::ArithError(e) + } +} + +impl From for DataColumnSidecarError { + fn from(e: BeaconStateError) -> Self { + Self::BeaconStateError(e) + } +} + +impl From for DataColumnSidecarError { + fn from(e: KzgError) -> Self { + Self::KzgError(e) + } +} + +impl From for DataColumnSidecarError { + fn from(e: SszError) -> Self { + Self::SszError(e) + } +} + +/// Converts a cell ssz List object to an array to be used with the kzg +/// crypto library. +fn ssz_cell_to_crypto_cell(cell: &Cell) -> Result { + KzgCell::from_bytes(cell.as_ref()).map_err(Into::into) +} diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index c6e816a487..15084cb14c 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -114,6 +114,12 @@ pub trait EthSpec: type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; type KzgCommitmentInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /* + * New in PeerDAS + */ + type FieldElementsPerCell: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type FieldElementsPerExtBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type KzgCommitmentsInclusionProofDepth: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* * Derived values (set these CAREFULLY) */ @@ -137,6 +143,11 @@ pub trait EthSpec: /// Must be set to `BytesPerFieldElement * FieldElementsPerBlob`. type BytesPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /// The total length of a data column in bytes. + /// + /// Must be set to `BytesPerFieldElement * FieldElementsPerCell`. + type BytesPerCell: Unsigned + Clone + Sync + Send + Debug + PartialEq; + /* * New in Electra */ @@ -284,6 +295,16 @@ pub trait EthSpec: Self::FieldElementsPerBlob::to_usize() } + /// Returns the `FIELD_ELEMENTS_PER_EXT_BLOB` constant for this specification. + fn field_elements_per_ext_blob() -> usize { + Self::FieldElementsPerExtBlob::to_usize() + } + + /// Returns the `FIELD_ELEMENTS_PER_CELL` constant for this specification. + fn field_elements_per_cell() -> usize { + Self::FieldElementsPerCell::to_usize() + } + /// Returns the `BYTES_PER_BLOB` constant for this specification. fn bytes_per_blob() -> usize { Self::BytesPerBlob::to_usize() @@ -349,6 +370,10 @@ pub trait EthSpec: fn max_withdrawal_requests_per_payload() -> usize { Self::MaxWithdrawalRequestsPerPayload::to_usize() } + + fn kzg_commitments_inclusion_proof_depth() -> usize { + Self::KzgCommitmentsInclusionProofDepth::to_usize() + } } /// Macro to inherit some type values from another EthSpec. @@ -394,8 +419,12 @@ impl EthSpec for MainnetEthSpec { type MaxBlobCommitmentsPerBlock = U4096; type BytesPerFieldElement = U32; type FieldElementsPerBlob = U4096; + type FieldElementsPerCell = U64; + type FieldElementsPerExtBlob = U8192; type BytesPerBlob = U131072; + type BytesPerCell = U2048; type KzgCommitmentInclusionProofDepth = U17; + type KzgCommitmentsInclusionProofDepth = U4; // inclusion of the whole list of commitments type SyncSubcommitteeSize = U128; // 512 committee size / 4 sync committee subnet count type MaxPendingAttestations = U4096; // 128 max attestations * 32 slots per epoch type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch @@ -444,6 +473,10 @@ impl EthSpec for MinimalEthSpec { type PendingConsolidationsLimit = U64; type MaxDepositRequestsPerPayload = U4; type MaxWithdrawalRequestsPerPayload = U2; + type FieldElementsPerCell = U64; + type FieldElementsPerExtBlob = U8192; + type BytesPerCell = U2048; + type KzgCommitmentsInclusionProofDepth = U4; params_from_eth_spec!(MainnetEthSpec { JustificationBitsLength, @@ -532,6 +565,10 @@ impl EthSpec for GnosisEthSpec { type MaxAttesterSlashingsElectra = U1; type MaxAttestationsElectra = U8; type MaxWithdrawalRequestsPerPayload = U16; + type FieldElementsPerCell = U64; + type FieldElementsPerExtBlob = U8192; + type BytesPerCell = U2048; + type KzgCommitmentsInclusionProofDepth = U4; fn default_spec() -> ChainSpec { ChainSpec::gnosis() diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index 96f206d454..55a393e018 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -31,6 +31,16 @@ impl ForkName { ] } + pub fn list_all_fork_epochs(spec: &ChainSpec) -> Vec<(ForkName, Option)> { + vec![ + (ForkName::Altair, spec.altair_fork_epoch), + (ForkName::Bellatrix, spec.bellatrix_fork_epoch), + (ForkName::Capella, spec.capella_fork_epoch), + (ForkName::Deneb, spec.deneb_fork_epoch), + (ForkName::Electra, spec.electra_fork_epoch), + ] + } + pub fn latest() -> ForkName { // This unwrap is safe as long as we have 1+ forks. It is tested below. *ForkName::list_all().last().unwrap() diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 8e72d5647a..c8c37ad708 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -104,6 +104,7 @@ pub mod slot_data; pub mod sqlite; pub mod blob_sidecar; +pub mod data_column_sidecar; pub mod light_client_header; pub mod non_zero_usize; pub mod runtime_var_list; diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index 76ad568988..210fa0eeeb 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -168,7 +168,7 @@ impl LightClientUpdate { let signature_period = block.epoch().sync_committee_period(chain_spec)?; // Compute and validate attested header. let mut attested_header = attested_state.latest_block_header().clone(); - attested_header.state_root = attested_state.tree_hash_root(); + attested_header.state_root = attested_state.update_tree_hash_cache()?; let attested_period = attested_header .slot .epoch(E::slots_per_epoch()) diff --git a/crypto/kzg/src/lib.rs b/crypto/kzg/src/lib.rs index 658dc9fe06..181642df39 100644 --- a/crypto/kzg/src/lib.rs +++ b/crypto/kzg/src/lib.rs @@ -19,6 +19,8 @@ pub enum Error { Kzg(c_kzg::Error), /// The kzg verification failed KzgVerificationFailed, + /// Misc indexing error + InconsistentArrayLength(String), } impl From for Error { @@ -27,6 +29,28 @@ impl From for Error { } } +pub const CELLS_PER_EXT_BLOB: usize = 128; + +// TODO(das): use proper crypto once ckzg merges das branch +#[allow(dead_code)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cell { + bytes: [u8; 2048usize], +} + +impl Cell { + pub fn from_bytes(b: &[u8]) -> Result { + Ok(Self { + bytes: b + .try_into() + .map_err(|_| Error::Kzg(c_kzg::Error::MismatchLength("".to_owned())))?, + }) + } + pub fn into_inner(self) -> [u8; 2048usize] { + self.bytes + } +} + /// A wrapper over a kzg library that holds the trusted setup parameters. #[derive(Debug)] pub struct Kzg { @@ -141,6 +165,55 @@ impl Kzg { ) .map_err(Into::into) } + + /// Computes the cells and associated proofs for a given `blob` at index `index`. + #[allow(clippy::type_complexity)] + pub fn compute_cells_and_proofs( + &self, + _blob: &Blob, + ) -> Result< + ( + Box<[Cell; CELLS_PER_EXT_BLOB]>, + Box<[KzgProof; CELLS_PER_EXT_BLOB]>, + ), + Error, + > { + // TODO(das): use proper crypto once ckzg merges das branch + let cells = Box::new(core::array::from_fn(|_| Cell { bytes: [0u8; 2048] })); + let proofs = Box::new([KzgProof([0u8; BYTES_PER_PROOF]); CELLS_PER_EXT_BLOB]); + Ok((cells, proofs)) + } + + /// Verifies a batch of cell-proof-commitment triplets. + /// + /// Here, `coordinates` correspond to the (row, col) coordinate of the cell in the extended + /// blob "matrix". In the 1D extension, row corresponds to the blob index, and col corresponds + /// to the data column index. + pub fn verify_cell_proof_batch( + &self, + _cells: &[Cell], + _kzg_proofs: &[Bytes48], + _coordinates: &[(u64, u64)], + _kzg_commitments: &[Bytes48], + ) -> Result<(), Error> { + // TODO(das): use proper crypto once ckzg merges das branch + Ok(()) + } + + pub fn cells_to_blob(&self, _cells: &[Cell; CELLS_PER_EXT_BLOB]) -> Result { + // TODO(das): use proper crypto once ckzg merges das branch + Ok(Blob::new([0u8; 131072usize])) + } + + pub fn recover_all_cells( + &self, + _cell_ids: &[u64], + _cells: &[Cell], + ) -> Result, Error> { + // TODO(das): use proper crypto once ckzg merges das branch + let cells = Box::new(core::array::from_fn(|_| Cell { bytes: [0u8; 2048] })); + Ok(cells) + } } impl TryFrom for Kzg { diff --git a/database_manager/Cargo.toml b/database_manager/Cargo.toml index 250188e2db..96176f3fba 100644 --- a/database_manager/Cargo.toml +++ b/database_manager/Cargo.toml @@ -14,3 +14,4 @@ store = { workspace = true } types = { workspace = true } slog = { workspace = true } strum = { workspace = true } +serde = { workspace = true } diff --git a/database_manager/src/cli.rs b/database_manager/src/cli.rs new file mode 100644 index 0000000000..5521b97805 --- /dev/null +++ b/database_manager/src/cli.rs @@ -0,0 +1,229 @@ +pub use clap::{Arg, ArgAction, Args, Command, FromArgMatches, Parser}; +use clap_utils::get_color_style; +use clap_utils::FLAG_HEADER; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +use crate::InspectTarget; + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap( + name = "database_manager", + visible_alias = "db", + about = "Manage a beacon node database.", + styles = get_color_style(), + next_line_help = true, + term_width = 80, + disable_help_flag = true, + disable_help_subcommand = true, + display_order = 0, +)] +pub struct DatabaseManager { + #[clap( + long, + value_name = "SLOT_COUNT", + help = "Specifies how often a freezer DB restore point should be stored. \ + Cannot be changed after initialization. \ + [default: 2048 (mainnet) or 64 (minimal)]", + display_order = 0 + )] + pub slots_per_restore_point: Option, + + #[clap( + long, + value_name = "DIR", + help = "Data directory for the freezer database.", + display_order = 0 + )] + pub freezer_dir: Option, + + #[clap( + long, + value_name = "EPOCHS", + default_value_t = 0, + help = "The margin for blob pruning in epochs. The oldest blobs are pruned \ + up until data_availability_boundary - blob_prune_margin_epochs.", + display_order = 0 + )] + pub blob_prune_margin_epochs: u64, + + #[clap( + long, + value_name = "DIR", + help = "Data directory for the blobs database.", + display_order = 0 + )] + pub blobs_dir: Option, + + #[clap( + long, + global = true, + help = "Prints help information", + action = clap::ArgAction::HelpLong, + display_order = 0, + help_heading = FLAG_HEADER + )] + help: Option, + + #[clap(subcommand)] + pub subcommand: DatabaseManagerSubcommand, +} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap(rename_all = "kebab-case")] +pub enum DatabaseManagerSubcommand { + Migrate(Migrate), + Inspect(Inspect), + Version(Version), + PrunePayloads(PrunePayloads), + PruneBlobs(PruneBlobs), + PruneStates(PruneStates), + Compact(Compact), +} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap(about = "Migrate the database to a specific schema version.")] +pub struct Migrate { + #[clap( + long, + value_name = "VERSION", + help = "Schema version to migrate to", + display_order = 0 + )] + pub to: u64, +} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap(about = "Inspect raw database values.")] +pub struct Inspect { + #[clap( + long, + value_name = "TAG", + help = "3-byte column ID (see `DBColumn`)", + display_order = 0 + )] + pub column: String, + + #[clap( + long, + value_enum, + value_name = "TARGET", + default_value_t = InspectTarget::ValueSizes, + help = "Select the type of output to show", + display_order = 0, + )] + pub output: InspectTarget, + + #[clap( + long, + value_name = "N", + help = "Skip over the first N keys", + display_order = 0 + )] + pub skip: Option, + + #[clap( + long, + value_name = "N", + help = "Output at most N keys", + display_order = 0 + )] + pub limit: Option, + + #[clap( + long, + conflicts_with = "blobs_db", + help = "Inspect the freezer DB rather than the hot DB", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub freezer: bool, + + #[clap( + long, + conflicts_with = "freezer", + help = "Inspect the blobs DB rather than the hot DB", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub blobs_db: bool, + + #[clap( + long, + value_name = "DIR", + help = "Base directory for the output files. Defaults to the current directory", + display_order = 0 + )] + pub output_dir: Option, +} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap(about = "Display database schema version.", visible_aliases = &["v"])] +pub struct Version {} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap( + about = "Prune finalized execution payloads.", + alias = "prune_payloads" +)] +pub struct PrunePayloads {} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap( + about = "Prune blobs older than data availability boundary.", + alias = "prune_blobs" +)] +pub struct PruneBlobs {} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap( + about = "Prune all beacon states from the freezer database.", + alias = "prune_states" +)] +pub struct PruneStates { + #[clap( + long, + help = "Commit to pruning states irreversably. Without this flag the command will \ + just check that the database is capable of being pruned.", + help_heading = FLAG_HEADER, + )] + pub confirm: bool, +} + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +#[clap(about = "Compact database manually.")] +pub struct Compact { + #[clap( + long, + value_name = "TAG", + help = "3-byte column ID (see `DBColumn`)", + display_order = 0 + )] + pub column: String, + + #[clap( + long, + conflicts_with = "blobs_db", + help = "Inspect the freezer DB rather than the hot DB", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub freezer: bool, + + #[clap( + long, + conflicts_with = "freezer", + help = "Inspect the blobs DB rather than the hot DB", + display_order = 0, + help_heading = FLAG_HEADER + )] + pub blobs_db: bool, + + #[clap( + long, + value_name = "DIR", + help = "Base directory for the output files. Defaults to the current directory", + display_order = 0 + )] + pub output_dir: Option, +} diff --git a/database_manager/src/lib.rs b/database_manager/src/lib.rs index fafff0f0f9..c5344f1f92 100644 --- a/database_manager/src/lib.rs +++ b/database_manager/src/lib.rs @@ -1,11 +1,17 @@ +pub mod cli; +use crate::cli::DatabaseManager; +use crate::cli::Migrate; +use crate::cli::PruneStates; use beacon_chain::{ builder::Witness, eth1_chain::CachingEth1Backend, schema_change::migrate_schema, slot_clock::SystemTimeSlotClock, }; use beacon_node::{get_data_dir, get_slots_per_restore_point, ClientConfig}; -use clap::{Arg, ArgAction, ArgMatches, Command}; -use clap_utils::{get_color_style, FLAG_HEADER}; +use clap::ArgMatches; +use clap::ValueEnum; +use cli::{Compact, Inspect}; use environment::{Environment, RuntimeContext}; +use serde::{Deserialize, Serialize}; use slog::{info, warn, Logger}; use std::fs; use std::io::Write; @@ -16,250 +22,30 @@ use store::{ metadata::{SchemaVersion, CURRENT_SCHEMA_VERSION}, DBColumn, HotColdDB, KeyValueStore, LevelDB, }; -use strum::{EnumString, EnumVariantNames, VariantNames}; +use strum::{EnumString, EnumVariantNames}; use types::{BeaconState, EthSpec, Slot}; -pub const CMD: &str = "database_manager"; - -pub fn version_cli_app() -> Command { - Command::new("version") - .visible_aliases(["v"]) - .styles(get_color_style()) - .about("Display database schema version") -} - -pub fn migrate_cli_app() -> Command { - Command::new("migrate") - .styles(get_color_style()) - .about("Migrate the database to a specific schema version") - .arg( - Arg::new("to") - .long("to") - .value_name("VERSION") - .help("Schema version to migrate to") - .action(ArgAction::Set) - .required(true), - ) -} - -pub fn inspect_cli_app() -> Command { - Command::new("inspect") - .styles(get_color_style()) - .about("Inspect raw database values") - .arg( - Arg::new("column") - .long("column") - .value_name("TAG") - .help("3-byte column ID (see `DBColumn`)") - .action(ArgAction::Set) - .required(true) - .display_order(0), - ) - .arg( - Arg::new("output") - .long("output") - .value_name("TARGET") - .help("Select the type of output to show") - .default_value("sizes") - .value_parser(InspectTarget::VARIANTS.to_vec()) - .display_order(0), - ) - .arg( - Arg::new("skip") - .long("skip") - .value_name("N") - .help("Skip over the first N keys") - .display_order(0), - ) - .arg( - Arg::new("limit") - .long("limit") - .value_name("N") - .help("Output at most N keys") - .display_order(0), - ) - .arg( - Arg::new("freezer") - .long("freezer") - .help("Inspect the freezer DB rather than the hot DB") - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .conflicts_with("blobs-db") - .display_order(0), - ) - .arg( - Arg::new("blobs-db") - .long("blobs-db") - .help("Inspect the blobs DB rather than the hot DB") - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .conflicts_with("freezer") - .display_order(0), - ) - .arg( - Arg::new("output-dir") - .long("output-dir") - .value_name("DIR") - .help("Base directory for the output files. Defaults to the current directory") - .action(ArgAction::Set) - .display_order(0), - ) -} - -pub fn compact_cli_app() -> Command { - Command::new("compact") - .styles(get_color_style()) - .about("Compact database manually") - .arg( - Arg::new("column") - .long("column") - .value_name("TAG") - .help("3-byte column ID (see `DBColumn`)") - .action(ArgAction::Set) - .required(true) - .display_order(0), - ) - .arg( - Arg::new("freezer") - .long("freezer") - .help("Inspect the freezer DB rather than the hot DB") - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .conflicts_with("blobs-db") - .display_order(0), - ) - .arg( - Arg::new("blobs-db") - .long("blobs-db") - .help("Inspect the blobs DB rather than the hot DB") - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .conflicts_with("freezer") - .display_order(0), - ) -} - -pub fn prune_payloads_app() -> Command { - Command::new("prune-payloads") - .alias("prune_payloads") - .styles(get_color_style()) - .about("Prune finalized execution payloads") -} - -pub fn prune_blobs_app() -> Command { - Command::new("prune-blobs") - .alias("prune_blobs") - .styles(get_color_style()) - .about("Prune blobs older than data availability boundary") -} - -pub fn prune_states_app() -> Command { - Command::new("prune-states") - .alias("prune_states") - .arg( - Arg::new("confirm") - .long("confirm") - .help( - "Commit to pruning states irreversably. Without this flag the command will \ - just check that the database is capable of being pruned.", - ) - .action(ArgAction::SetTrue) - .help_heading(FLAG_HEADER) - .display_order(0), - ) - .styles(get_color_style()) - .about("Prune all beacon states from the freezer database") -} - -pub fn cli_app() -> Command { - Command::new(CMD) - .display_order(0) - .visible_aliases(["db"]) - .styles(get_color_style()) - .about("Manage a beacon node database") - .arg( - Arg::new("help") - .long("help") - .short('h') - .help("Prints help information") - .action(ArgAction::HelpLong) - .display_order(0) - .help_heading(FLAG_HEADER), - ) - .arg( - Arg::new("slots-per-restore-point") - .long("slots-per-restore-point") - .value_name("SLOT_COUNT") - .help( - "Specifies how often a freezer DB restore point should be stored. \ - Cannot be changed after initialization. \ - [default: 2048 (mainnet) or 64 (minimal)]", - ) - .action(ArgAction::Set) - .display_order(0), - ) - .arg( - Arg::new("freezer-dir") - .long("freezer-dir") - .value_name("DIR") - .help("Data directory for the freezer database.") - .action(ArgAction::Set) - .display_order(0), - ) - .arg( - Arg::new("blob-prune-margin-epochs") - .long("blob-prune-margin-epochs") - .value_name("EPOCHS") - .help( - "The margin for blob pruning in epochs. The oldest blobs are pruned \ - up until data_availability_boundary - blob_prune_margin_epochs.", - ) - .action(ArgAction::Set) - .default_value("0") - .display_order(0), - ) - .arg( - Arg::new("blobs-dir") - .long("blobs-dir") - .value_name("DIR") - .help("Data directory for the blobs database.") - .action(ArgAction::Set) - .display_order(0), - ) - .subcommand(migrate_cli_app()) - .subcommand(version_cli_app()) - .subcommand(inspect_cli_app()) - .subcommand(compact_cli_app()) - .subcommand(prune_payloads_app()) - .subcommand(prune_blobs_app()) - .subcommand(prune_states_app()) -} - fn parse_client_config( cli_args: &ArgMatches, + database_manager_config: &DatabaseManager, _env: &Environment, ) -> Result { let mut client_config = ClientConfig::default(); client_config.set_data_dir(get_data_dir(cli_args)); + client_config + .freezer_db_path + .clone_from(&database_manager_config.freezer_dir); + client_config + .blobs_db_path + .clone_from(&database_manager_config.blobs_dir); - if let Some(freezer_dir) = clap_utils::parse_optional(cli_args, "freezer-dir")? { - client_config.freezer_db_path = Some(freezer_dir); - } + let (sprp, sprp_explicit) = + get_slots_per_restore_point::(database_manager_config.slots_per_restore_point)?; - if let Some(blobs_db_dir) = clap_utils::parse_optional(cli_args, "blobs-dir")? { - client_config.blobs_db_path = Some(blobs_db_dir); - } - - let (sprp, sprp_explicit) = get_slots_per_restore_point::(cli_args)?; client_config.store.slots_per_restore_point = sprp; client_config.store.slots_per_restore_point_set_explicitly = sprp_explicit; - - if let Some(blob_prune_margin_epochs) = - clap_utils::parse_optional(cli_args, "blob-prune-margin-epochs")? - { - client_config.store.blob_prune_margin_epochs = blob_prune_margin_epochs; - } + client_config.store.blob_prune_margin_epochs = database_manager_config.blob_prune_margin_epochs; Ok(client_config) } @@ -301,15 +87,21 @@ pub fn display_db_version( Ok(()) } -#[derive(Debug, PartialEq, Eq, EnumString, EnumVariantNames)] +#[derive( + Debug, PartialEq, Eq, Clone, EnumString, Deserialize, Serialize, EnumVariantNames, ValueEnum, +)] pub enum InspectTarget { #[strum(serialize = "sizes")] + #[clap(name = "sizes")] ValueSizes, #[strum(serialize = "total")] + #[clap(name = "total")] ValueTotal, #[strum(serialize = "values")] + #[clap(name = "values")] Values, #[strum(serialize = "gaps")] + #[clap(name = "gaps")] Gaps, } @@ -324,16 +116,18 @@ pub struct InspectConfig { output_dir: PathBuf, } -fn parse_inspect_config(cli_args: &ArgMatches) -> Result { - let column = clap_utils::parse_required(cli_args, "column")?; - let target = clap_utils::parse_required(cli_args, "output")?; - let skip = clap_utils::parse_optional(cli_args, "skip")?; - let limit = clap_utils::parse_optional(cli_args, "limit")?; - let freezer = cli_args.get_flag("freezer"); - let blobs_db = cli_args.get_flag("blobs-db"); +fn parse_inspect_config(inspect_config: &Inspect) -> Result { + let column: DBColumn = inspect_config + .column + .parse() + .map_err(|e| format!("Unable to parse column flag: {e:?}"))?; + let target: InspectTarget = inspect_config.output.clone(); + let skip = inspect_config.skip; + let limit = inspect_config.limit; + let freezer = inspect_config.freezer; + let blobs_db = inspect_config.blobs_db; - let output_dir: PathBuf = - clap_utils::parse_optional(cli_args, "output-dir")?.unwrap_or_else(PathBuf::new); + let output_dir: PathBuf = inspect_config.output_dir.clone().unwrap_or_default(); Ok(InspectConfig { column, target, @@ -450,10 +244,13 @@ pub struct CompactConfig { blobs_db: bool, } -fn parse_compact_config(cli_args: &ArgMatches) -> Result { - let column = clap_utils::parse_required(cli_args, "column")?; - let freezer = cli_args.get_flag("freezer"); - let blobs_db = cli_args.get_flag("blobs-db"); +fn parse_compact_config(compact_config: &Compact) -> Result { + let column: DBColumn = compact_config + .column + .parse() + .expect("column is a required field"); + let freezer = compact_config.freezer; + let blobs_db = compact_config.blobs_db; Ok(CompactConfig { column, freezer, @@ -492,8 +289,8 @@ pub struct MigrateConfig { to: SchemaVersion, } -fn parse_migrate_config(cli_args: &ArgMatches) -> Result { - let to = SchemaVersion(clap_utils::parse_required(cli_args, "to")?); +fn parse_migrate_config(migrate_config: &Migrate) -> Result { + let to = SchemaVersion(migrate_config.to); Ok(MigrateConfig { to }) } @@ -595,9 +392,10 @@ pub fn prune_blobs( pub struct PruneStatesConfig { confirm: bool, } - -fn parse_prune_states_config(cli_args: &ArgMatches) -> Result { - let confirm = cli_args.get_flag("confirm"); +fn parse_prune_states_config( + prune_states_config: &PruneStates, +) -> Result { + let confirm = prune_states_config.confirm; Ok(PruneStatesConfig { confirm }) } @@ -676,33 +474,35 @@ pub fn prune_states( } /// Run the database manager, returning an error string if the operation did not succeed. -pub fn run(cli_args: &ArgMatches, env: Environment) -> Result<(), String> { - let client_config = parse_client_config(cli_args, &env)?; +pub fn run( + cli_args: &ArgMatches, + db_manager_config: &DatabaseManager, + env: Environment, +) -> Result<(), String> { + let client_config = parse_client_config(cli_args, db_manager_config, &env)?; let context = env.core_context(); let log = context.log().clone(); let format_err = |e| format!("Fatal error: {:?}", e); - match cli_args.subcommand() { - Some(("version", _)) => { - display_db_version(client_config, &context, log).map_err(format_err) - } - Some(("migrate", cli_args)) => { - let migrate_config = parse_migrate_config(cli_args)?; + match &db_manager_config.subcommand { + cli::DatabaseManagerSubcommand::Migrate(migrate_config) => { + let migrate_config = parse_migrate_config(migrate_config)?; migrate_db(migrate_config, client_config, &context, log).map_err(format_err) } - Some(("inspect", cli_args)) => { - let inspect_config = parse_inspect_config(cli_args)?; + cli::DatabaseManagerSubcommand::Inspect(inspect_config) => { + let inspect_config = parse_inspect_config(inspect_config)?; inspect_db::(inspect_config, client_config) } - Some(("compact", cli_args)) => { - let compact_config = parse_compact_config(cli_args)?; - compact_db::(compact_config, client_config, log).map_err(format_err) + cli::DatabaseManagerSubcommand::Version(_) => { + display_db_version(client_config, &context, log).map_err(format_err) } - Some(("prune-payloads", _)) => { + cli::DatabaseManagerSubcommand::PrunePayloads(_) => { prune_payloads(client_config, &context, log).map_err(format_err) } - Some(("prune-blobs", _)) => prune_blobs(client_config, &context, log).map_err(format_err), - Some(("prune-states", cli_args)) => { + cli::DatabaseManagerSubcommand::PruneBlobs(_) => { + prune_blobs(client_config, &context, log).map_err(format_err) + } + cli::DatabaseManagerSubcommand::PruneStates(prune_states_config) => { let executor = env.core_context().executor; let network_config = context .eth2_network_config @@ -722,10 +522,13 @@ pub fn run(cli_args: &ArgMatches, env: Environment) -> Result<(), .map_err(|e| format!("Error getting genesis state: {e}"))? .ok_or("Genesis state missing")?; - let prune_config = parse_prune_states_config(cli_args)?; + let prune_config = parse_prune_states_config(prune_states_config)?; prune_states(client_config, prune_config, genesis_state, &context, log) } - _ => Err("Unknown subcommand, for help `lighthouse database_manager --help`".into()), + cli::DatabaseManagerSubcommand::Compact(compact_config) => { + let compact_config = parse_compact_config(compact_config)?; + compact_db::(compact_config, client_config, log).map_err(format_err) + } } } diff --git a/lcli/Dockerfile b/lcli/Dockerfile index 4f5c3f2972..2cc9ce605b 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -5,7 +5,7 @@ FROM rust:1.75.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES -ENV FEATURES $FEATURES +ENV FEATURES=$FEATURES RUN cd lighthouse && make install-lcli FROM ubuntu:22.04 diff --git a/lighthouse/src/cli.rs b/lighthouse/src/cli.rs new file mode 100644 index 0000000000..90d3e811eb --- /dev/null +++ b/lighthouse/src/cli.rs @@ -0,0 +1,9 @@ +use clap::Parser; +use database_manager::cli::DatabaseManager; +use serde::{Deserialize, Serialize}; + +#[derive(Parser, Clone, Deserialize, Serialize, Debug)] +pub enum LighthouseSubcommands { + #[clap(name = "database_manager")] + DatabaseManager(DatabaseManager), +} diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index a7521d5f8c..481e17dbc8 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -1,10 +1,14 @@ +mod cli; mod metrics; use beacon_node::ProductionBeaconNode; +use clap::FromArgMatches; +use clap::Subcommand; use clap::{Arg, ArgAction, ArgMatches, Command}; use clap_utils::{ flags::DISABLE_MALLOC_TUNING_FLAG, get_color_style, get_eth2_network_config, FLAG_HEADER, }; +use cli::LighthouseSubcommands; use directory::{parse_path_or_default, DEFAULT_BEACON_NODE_DIR, DEFAULT_VALIDATOR_DIR}; use environment::{EnvironmentBuilder, LoggerConfig}; use eth2_network_config::{Eth2NetworkConfig, DEFAULT_HARDCODED_NETWORK, HARDCODED_NET_NAMES}; @@ -87,7 +91,7 @@ fn main() { } // Parse the CLI parameters. - let matches = Command::new("Lighthouse") + let cli = Command::new("Lighthouse") .version(SHORT_VERSION.as_str()) .author("Sigma Prime ") .styles(get_color_style()) @@ -165,7 +169,7 @@ fn main() { "The maximum number of log files that will be stored. If set to 0, \ background file logging is disabled.") .action(ArgAction::Set) - .default_value("5") + .default_value("10") .global(true) .display_order(0) ) @@ -409,9 +413,11 @@ fn main() { .subcommand(boot_node::cli_app()) .subcommand(validator_client::cli_app()) .subcommand(account_manager::cli_app()) - .subcommand(database_manager::cli_app()) - .subcommand(validator_manager::cli_app()) - .get_matches(); + .subcommand(validator_manager::cli_app()); + + let cli = LighthouseSubcommands::augment_subcommands(cli); + + let matches = cli.get_matches(); // Configure the allocator early in the process, before it has the chance to use the default values for // anything important. @@ -676,14 +682,13 @@ fn run( return Ok(()); } - if let Some(sub_matches) = matches.subcommand_matches(database_manager::CMD) { + if let Ok(LighthouseSubcommands::DatabaseManager(db_manager_config)) = + LighthouseSubcommands::from_arg_matches(matches) + { info!(log, "Running database manager for {} network", network_name); - // Pass the entire `environment` to the database manager so it can run blocking operations. - database_manager::run(sub_matches, environment)?; - - // Exit as soon as database manager returns control. + database_manager::run(matches, &db_manager_config, environment)?; return Ok(()); - } + }; info!(log, "Lighthouse started"; "version" => VERSION); info!( diff --git a/slasher/src/database/redb_impl.rs b/slasher/src/database/redb_impl.rs index da7b4e38ed..6c5b62a44f 100644 --- a/slasher/src/database/redb_impl.rs +++ b/slasher/src/database/redb_impl.rs @@ -91,7 +91,7 @@ impl Environment { } pub fn filenames(&self, config: &Config) -> Vec { - vec![config.database_path.join(BASE_DB)] + vec![config.database_path.join(REDB_DATA_FILENAME)] } pub fn begin_rw_txn(&self) -> Result { diff --git a/slasher/tests/backend.rs b/slasher/tests/backend.rs index c24b861b18..fd1a6ae14f 100644 --- a/slasher/tests/backend.rs +++ b/slasher/tests/backend.rs @@ -1,4 +1,4 @@ -#![cfg(any(feature = "lmdb", feature = "redb"))] +#![cfg(feature = "lmdb")] use slasher::{config::MDBX_DATA_FILENAME, Config, DatabaseBackend, DatabaseBackendOverride}; use std::fs::File; @@ -41,7 +41,7 @@ fn no_override_with_existing_mdbx_db() { } #[test] -#[cfg(all(not(feature = "mdbx"), feature = "lmdb", not(feature = "redb")))] +#[cfg(all(not(feature = "mdbx"), feature = "lmdb"))] fn failed_override_with_existing_mdbx_db() { let tempdir = tempdir().unwrap(); let mut config = Config::new(tempdir.path().into()); @@ -55,19 +55,3 @@ fn failed_override_with_existing_mdbx_db() { ); assert_eq!(config.backend, DatabaseBackend::Lmdb); } - -#[test] -#[cfg(feature = "redb")] -fn failed_override_with_existing_mdbx_db() { - let tempdir = tempdir().unwrap(); - let mut config = Config::new(tempdir.path().into()); - - let filename = config.database_path.join(MDBX_DATA_FILENAME); - File::create(&filename).unwrap(); - - assert_eq!( - config.override_backend(), - DatabaseBackendOverride::Failure(filename) - ); - assert_eq!(config.backend, DatabaseBackend::Redb); -} diff --git a/testing/ef_tests/src/cases/common.rs b/testing/ef_tests/src/cases/common.rs index 6763edbe22..8b25391980 100644 --- a/testing/ef_tests/src/cases/common.rs +++ b/testing/ef_tests/src/cases/common.rs @@ -2,7 +2,6 @@ use serde::Deserialize; use ssz::Encode; use ssz_derive::{Decode, Encode}; use std::fmt::Debug; -use tree_hash::TreeHash; use types::ForkName; /// Macro to wrap U128 and U256 so they deserialize correctly. @@ -49,12 +48,12 @@ uint_wrapper!(TestU256, ethereum_types::U256); /// Trait for types that can be used in SSZ static tests. pub trait SszStaticType: - serde::de::DeserializeOwned + Encode + TreeHash + Clone + PartialEq + Debug + Sync + serde::de::DeserializeOwned + Encode + Clone + PartialEq + Debug + Sync { } impl SszStaticType for T where - T: serde::de::DeserializeOwned + Encode + TreeHash + Clone + PartialEq + Debug + Sync + T: serde::de::DeserializeOwned + Encode + Clone + PartialEq + Debug + Sync { } diff --git a/testing/ef_tests/src/cases/ssz_generic.rs b/testing/ef_tests/src/cases/ssz_generic.rs index 8de3e217f0..7933fc65c7 100644 --- a/testing/ef_tests/src/cases/ssz_generic.rs +++ b/testing/ef_tests/src/cases/ssz_generic.rs @@ -6,6 +6,7 @@ use crate::cases::ssz_static::{check_serialization, check_tree_hash}; use crate::decode::{log_file_access, snappy_decode_file, yaml_decode_file}; use serde::{de::Error as SerdeError, Deserialize, Deserializer}; use ssz_derive::{Decode, Encode}; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::typenum::*; use types::{BitList, BitVector, FixedVector, ForkName, VariableList, Vector}; @@ -206,7 +207,7 @@ impl Case for SszGeneric { } } -fn ssz_generic_test(path: &Path) -> Result<(), Error> { +fn ssz_generic_test(path: &Path) -> Result<(), Error> { let meta_path = path.join("meta.yaml"); let meta: Option = if meta_path.is_file() { Some(yaml_decode_file(&meta_path)?) diff --git a/testing/ef_tests/src/cases/ssz_static.rs b/testing/ef_tests/src/cases/ssz_static.rs index 5f0ac3525c..e17aa469bf 100644 --- a/testing/ef_tests/src/cases/ssz_static.rs +++ b/testing/ef_tests/src/cases/ssz_static.rs @@ -101,7 +101,7 @@ pub fn check_tree_hash(expected_str: &str, actual_root: &[u8]) -> Result<(), Err compare_result::(&Ok(tree_hash_root), &Some(expected_root)) } -impl Case for SszStatic { +impl Case for SszStatic { fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { check_serialization(&self.value, &self.serialized, T::from_ssz_bytes)?; check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?; @@ -115,7 +115,6 @@ impl Case for SszStaticTHC> { check_serialization(&self.value, &self.serialized, |bytes| { BeaconState::from_ssz_bytes(bytes, spec) })?; - check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?; let mut state = self.value.clone(); let cached_tree_hash_root = state.update_tree_hash_cache().unwrap(); diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index c1e7386bc4..69f6937ad8 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -262,7 +262,7 @@ pub struct SszStaticWithSpecHandler(PhantomData<(T, E)>); impl Handler for SszStaticHandler where - T: cases::SszStaticType + ssz::Decode + TypeName, + T: cases::SszStaticType + tree_hash::TreeHash + ssz::Decode + TypeName, E: TypeName, { type Case = cases::SszStatic; diff --git a/validator_client/src/attestation_service.rs b/validator_client/src/attestation_service.rs index dfd140059d..5363f36f66 100644 --- a/validator_client/src/attestation_service.rs +++ b/validator_client/src/attestation_service.rs @@ -286,17 +286,21 @@ impl AttestationService { // Then download, sign and publish a `SignedAggregateAndProof` for each // validator that is elected to aggregate for this `slot` and // `committee_index`. - self.produce_and_publish_aggregates(&attestation_data, &validator_duties) - .await - .map_err(move |e| { - crit!( - log, - "Error during attestation routine"; - "error" => format!("{:?}", e), - "committee_index" => committee_index, - "slot" => slot.as_u64(), - ) - })?; + self.produce_and_publish_aggregates( + &attestation_data, + committee_index, + &validator_duties, + ) + .await + .map_err(move |e| { + crit!( + log, + "Error during attestation routine"; + "error" => format!("{:?}", e), + "committee_index" => committee_index, + "slot" => slot.as_u64(), + ) + })?; } Ok(()) @@ -440,6 +444,11 @@ impl AttestationService { warn!(log, "No attestations were published"); return Ok(None); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); // Post the attestations to the BN. match self @@ -449,9 +458,15 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::ATTESTATIONS_HTTP_POST], ); - beacon_node - .post_beacon_pool_attestations(attestations) - .await + if fork_name.electra_enabled() { + beacon_node + .post_beacon_pool_attestations_v2(attestations, fork_name) + .await + } else { + beacon_node + .post_beacon_pool_attestations_v1(attestations) + .await + } }) .await { @@ -494,6 +509,7 @@ impl AttestationService { async fn produce_and_publish_aggregates( &self, attestation_data: &AttestationData, + committee_index: CommitteeIndex, validator_duties: &[DutyAndProof], ) -> Result<(), String> { let log = self.context.log(); @@ -506,6 +522,12 @@ impl AttestationService { return Ok(()); } + let fork_name = self + .context + .eth2_config + .spec + .fork_name_at_slot::(attestation_data.slot); + let aggregated_attestation = &self .beacon_nodes .first_success(|beacon_node| async move { @@ -513,15 +535,32 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_GET], ); - beacon_node - .get_validator_aggregate_attestation( - attestation_data.slot, - attestation_data.tree_hash_root(), - ) - .await - .map_err(|e| format!("Failed to produce an aggregate attestation: {:?}", e))? - .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data)) - .map(|result| result.data) + if fork_name.electra_enabled() { + beacon_node + .get_validator_aggregate_attestation_v2( + attestation_data.slot, + attestation_data.tree_hash_root(), + committee_index, + ) + .await + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data)) + .map(|result| result.data) + } else { + beacon_node + .get_validator_aggregate_attestation_v1( + attestation_data.slot, + attestation_data.tree_hash_root(), + ) + .await + .map_err(|e| { + format!("Failed to produce an aggregate attestation: {:?}", e) + })? + .ok_or_else(|| format!("No aggregate available for {:?}", attestation_data)) + .map(|result| result.data) + } }) .await .map_err(|e| e.to_string())?; @@ -585,9 +624,20 @@ impl AttestationService { &metrics::ATTESTATION_SERVICE_TIMES, &[metrics::AGGREGATES_HTTP_POST], ); - beacon_node - .post_validator_aggregate_and_proof(signed_aggregate_and_proofs_slice) - .await + if fork_name.electra_enabled() { + beacon_node + .post_validator_aggregate_and_proof_v2( + signed_aggregate_and_proofs_slice, + fork_name, + ) + .await + } else { + beacon_node + .post_validator_aggregate_and_proof_v1( + signed_aggregate_and_proofs_slice, + ) + .await + } }) .await { diff --git a/validator_client/src/http_api/api_secret.rs b/validator_client/src/http_api/api_secret.rs index e688792ddc..32035caf47 100644 --- a/validator_client/src/http_api/api_secret.rs +++ b/validator_client/src/http_api/api_secret.rs @@ -1,85 +1,53 @@ -use eth2::lighthouse_vc::{PK_LEN, SECRET_PREFIX as PK_PREFIX}; use filesystem::create_with_600_perms; -use libsecp256k1::{Message, PublicKey, SecretKey}; -use rand::thread_rng; -use ring::digest::{digest, SHA256}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; use std::fs; use std::path::{Path, PathBuf}; use warp::Filter; -/// The name of the file which stores the secret key. -/// -/// It is purposefully opaque to prevent users confusing it with the "secret" that they need to -/// share with API consumers (which is actually the public key). -pub const SK_FILENAME: &str = ".secp-sk"; - -/// Length of the raw secret key, in bytes. -pub const SK_LEN: usize = 32; - -/// The name of the file which stores the public key. -/// -/// For users, this public key is a "secret" that can be shared with API consumers to provide them -/// access to the API. We avoid calling it a "public" key to users, since they should not post this -/// value in a public forum. +/// The name of the file which stores the API token. pub const PK_FILENAME: &str = "api-token.txt"; -/// Contains a `secp256k1` keypair that is saved-to/loaded-from disk on instantiation. The keypair -/// is used for authorization/authentication for requests/responses on the HTTP API. +pub const PK_LEN: usize = 33; + +/// Contains a randomly generated string which is used for authorization of requests to the HTTP API. /// /// Provides convenience functions to ultimately provide: /// -/// - A signature across outgoing HTTP responses, applied to the `Signature` header. /// - Verification of proof-of-knowledge of the public key in `self` for incoming HTTP requests, /// via the `Authorization` header. /// /// The aforementioned scheme was first defined here: /// /// https://github.com/sigp/lighthouse/issues/1269#issuecomment-649879855 +/// +/// This scheme has since been tweaked to remove VC response signing and secp256k1 key generation. +/// https://github.com/sigp/lighthouse/issues/5423 pub struct ApiSecret { - pk: PublicKey, - sk: SecretKey, + pk: String, pk_path: PathBuf, } impl ApiSecret { - /// If both the secret and public keys are already on-disk, parse them and ensure they're both - /// from the same keypair. + /// If the public key is already on-disk, use it. /// - /// The provided `dir` is a directory containing two files, `SK_FILENAME` and `PK_FILENAME`. + /// The provided `dir` is a directory containing `PK_FILENAME`. /// - /// If either the secret or public key files are missing on disk, create a new keypair and + /// If the public key file is missing on disk, create a new key and /// write it to disk (over-writing any existing files). pub fn create_or_open>(dir: P) -> Result { - let sk_path = dir.as_ref().join(SK_FILENAME); let pk_path = dir.as_ref().join(PK_FILENAME); - if !(sk_path.exists() && pk_path.exists()) { - let sk = SecretKey::random(&mut thread_rng()); - let pk = PublicKey::from_secret_key(&sk); - - // Create and write the secret key to file with appropriate permissions - create_with_600_perms( - &sk_path, - serde_utils::hex::encode(sk.serialize()).as_bytes(), - ) - .map_err(|e| { - format!( - "Unable to create file with permissions for {:?}: {:?}", - sk_path, e - ) - })?; + if !pk_path.exists() { + let length = PK_LEN; + let pk: String = thread_rng() + .sample_iter(&Alphanumeric) + .take(length) + .map(char::from) + .collect(); // Create and write the public key to file with appropriate permissions - create_with_600_perms( - &pk_path, - format!( - "{}{}", - PK_PREFIX, - serde_utils::hex::encode(&pk.serialize_compressed()[..]) - ) - .as_bytes(), - ) - .map_err(|e| { + create_with_600_perms(&pk_path, pk.to_string().as_bytes()).map_err(|e| { format!( "Unable to create file with permissions for {:?}: {:?}", pk_path, e @@ -87,78 +55,18 @@ impl ApiSecret { })?; } - let sk = fs::read(&sk_path) - .map_err(|e| format!("cannot read {}: {}", SK_FILENAME, e)) - .and_then(|bytes| { - serde_utils::hex::decode(&String::from_utf8_lossy(&bytes)) - .map_err(|_| format!("{} should be 0x-prefixed hex", PK_FILENAME)) - }) - .and_then(|bytes| { - if bytes.len() == SK_LEN { - let mut array = [0; SK_LEN]; - array.copy_from_slice(&bytes); - SecretKey::parse(&array).map_err(|e| format!("invalid {}: {}", SK_FILENAME, e)) - } else { - Err(format!( - "{} expected {} bytes not {}", - SK_FILENAME, - SK_LEN, - bytes.len() - )) - } - })?; - let pk = fs::read(&pk_path) - .map_err(|e| format!("cannot read {}: {}", PK_FILENAME, e)) - .and_then(|bytes| { - let hex = - String::from_utf8(bytes).map_err(|_| format!("{} is not utf8", SK_FILENAME))?; - if let Some(stripped) = hex.strip_prefix(PK_PREFIX) { - serde_utils::hex::decode(stripped) - .map_err(|_| format!("{} should be 0x-prefixed hex", SK_FILENAME)) - } else { - Err(format!("unable to parse {}", SK_FILENAME)) - } - }) - .and_then(|bytes| { - if bytes.len() == PK_LEN { - let mut array = [0; PK_LEN]; - array.copy_from_slice(&bytes); - PublicKey::parse_compressed(&array) - .map_err(|e| format!("invalid {}: {}", PK_FILENAME, e)) - } else { - Err(format!( - "{} expected {} bytes not {}", - PK_FILENAME, - PK_LEN, - bytes.len() - )) - } - })?; + .map_err(|e| format!("cannot read {}: {}", PK_FILENAME, e))? + .iter() + .map(|&c| char::from(c)) + .collect(); - // Ensure that the keys loaded from disk are indeed a pair. - if PublicKey::from_secret_key(&sk) != pk { - fs::remove_file(&sk_path) - .map_err(|e| format!("unable to remove {}: {}", SK_FILENAME, e))?; - fs::remove_file(&pk_path) - .map_err(|e| format!("unable to remove {}: {}", PK_FILENAME, e))?; - return Err(format!( - "{:?} does not match {:?} and the files have been deleted. Please try again.", - sk_path, pk_path - )); - } - - Ok(Self { pk, sk, pk_path }) - } - - /// Returns the public key of `self` as a 0x-prefixed hex string. - fn pubkey_string(&self) -> String { - serde_utils::hex::encode(&self.pk.serialize_compressed()[..]) + Ok(Self { pk, pk_path }) } /// Returns the API token. pub fn api_token(&self) -> String { - format!("{}{}", PK_PREFIX, self.pubkey_string()) + self.pk.clone() } /// Returns the path for the API token file @@ -196,16 +104,4 @@ impl ApiSecret { .untuple_one() .boxed() } - - /// Returns a closure which produces a signature over some bytes using the secret key in - /// `self`. The signature is a 32-byte hash formatted as a 0x-prefixed string. - pub fn signer(&self) -> impl Fn(&[u8]) -> String + Clone { - let sk = self.sk; - move |input: &[u8]| -> String { - let message = - Message::parse_slice(digest(&SHA256, input).as_ref()).expect("sha256 is 32 bytes"); - let (signature, _) = libsecp256k1::sign(&message, &sk); - serde_utils::hex::encode(signature.serialize_der().as_ref()) - } - } } diff --git a/validator_client/src/http_api/mod.rs b/validator_client/src/http_api/mod.rs index 4ffe6d25f3..2625baa5c9 100644 --- a/validator_client/src/http_api/mod.rs +++ b/validator_client/src/http_api/mod.rs @@ -47,15 +47,8 @@ use task_executor::TaskExecutor; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ChainSpec, ConfigAndPreset, EthSpec}; use validator_dir::Builder as ValidatorDirBuilder; -use warp::{ - http::{ - header::{HeaderValue, CONTENT_TYPE}, - response::Response, - StatusCode, - }, - sse::Event, - Filter, -}; +use warp::{sse::Event, Filter}; +use warp_utils::task::blocking_json_task; #[derive(Debug)] pub enum Error { @@ -179,9 +172,6 @@ pub fn serve( } }; - let signer = ctx.api_secret.signer(); - let signer = warp::any().map(move || signer.clone()); - let inner_block_service = ctx.block_service.clone(); let block_service_filter = warp::any() .map(move || inner_block_service.clone()) @@ -284,9 +274,8 @@ pub fn serve( let get_node_version = warp::path("lighthouse") .and(warp::path("version")) .and(warp::path::end()) - .and(signer.clone()) - .and_then(|signer| { - blocking_signed_json_task(signer, move || { + .then(|| { + blocking_json_task(move || { Ok(api_types::GenericResponse::from(api_types::VersionData { version: version_with_platform(), })) @@ -297,9 +286,8 @@ pub fn serve( let get_lighthouse_health = warp::path("lighthouse") .and(warp::path("health")) .and(warp::path::end()) - .and(signer.clone()) - .and_then(|signer| { - blocking_signed_json_task(signer, move || { + .then(|| { + blocking_json_task(move || { eth2::lighthouse::Health::observe() .map(api_types::GenericResponse::from) .map_err(warp_utils::reject::custom_bad_request) @@ -311,9 +299,8 @@ pub fn serve( .and(warp::path("spec")) .and(warp::path::end()) .and(spec_filter.clone()) - .and(signer.clone()) - .and_then(|spec: Arc<_>, signer| { - blocking_signed_json_task(signer, move || { + .then(|spec: Arc<_>| { + blocking_json_task(move || { let config = ConfigAndPreset::from_chain_spec::(&spec, None); Ok(api_types::GenericResponse::from(config)) }) @@ -324,9 +311,8 @@ pub fn serve( .and(warp::path("validators")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then(|validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then(|validator_store: Arc>| { + blocking_json_task(move || { let validators = validator_store .initialized_validators() .read() @@ -349,10 +335,9 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( - |validator_pubkey: PublicKey, validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then( + |validator_pubkey: PublicKey, validator_store: Arc>| { + blocking_json_task(move || { let validator = validator_store .initialized_validators() .read() @@ -384,9 +369,8 @@ pub fn serve( .and(system_info_filter) .and(app_start_filter) .and(validator_dir_filter.clone()) - .and(signer.clone()) - .and_then(|sysinfo, app_start: std::time::Instant, val_dir, signer| { - blocking_signed_json_task(signer, move || { + .then(|sysinfo, app_start: std::time::Instant, val_dir| { + blocking_json_task(move || { let app_uptime = app_start.elapsed().as_secs(); Ok(api_types::GenericResponse::from(observe_system_health_vc( sysinfo, val_dir, app_uptime, @@ -401,15 +385,13 @@ pub fn serve( .and(validator_store_filter.clone()) .and(graffiti_file_filter.clone()) .and(graffiti_flag_filter) - .and(signer.clone()) .and(log_filter.clone()) - .and_then( + .then( |validator_store: Arc>, graffiti_file: Option, graffiti_flag: Option, - signer, log| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { let mut result = HashMap::new(); for (key, graffiti_definition) in validator_store .initialized_validators() @@ -435,9 +417,8 @@ pub fn serve( .and(warp::path("ui")) .and(warp::path("fallback_health")) .and(warp::path::end()) - .and(signer.clone()) .and(block_service_filter.clone()) - .and_then(|signer, block_filter: BlockService| async move { + .then(|block_filter: BlockService| async move { let mut result: HashMap<(usize, String), Result> = HashMap::new(); for node in &*block_filter.beacon_nodes.candidates.read().await { @@ -455,8 +436,7 @@ pub fn serve( } } - blocking_signed_json_task(signer, move || Ok(api_types::GenericResponse::from(result))) - .await + blocking_json_task(move || Ok(api_types::GenericResponse::from(result))).await }); // POST lighthouse/validators/ @@ -468,17 +448,15 @@ pub fn serve( .and(secrets_dir_filter.clone()) .and(validator_store_filter.clone()) .and(spec_filter.clone()) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( move |body: Vec, validator_dir: PathBuf, secrets_dir: PathBuf, validator_store: Arc>, spec: Arc, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir); if let Some(handle) = task_executor.handle() { let (validators, mnemonic) = @@ -515,17 +493,15 @@ pub fn serve( .and(secrets_dir_filter.clone()) .and(validator_store_filter.clone()) .and(spec_filter) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( move |body: api_types::CreateValidatorsMnemonicRequest, validator_dir: PathBuf, secrets_dir: PathBuf, validator_store: Arc>, spec: Arc, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir); if let Some(handle) = task_executor.handle() { let mnemonic = @@ -564,16 +540,14 @@ pub fn serve( .and(validator_dir_filter.clone()) .and(secrets_dir_filter.clone()) .and(validator_store_filter.clone()) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( move |body: api_types::KeystoreValidatorsPostRequest, validator_dir: PathBuf, secrets_dir: PathBuf, validator_store: Arc>, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { // Check to ensure the password is correct. let keypair = body .keystore @@ -654,14 +628,12 @@ pub fn serve( .and(warp::path::end()) .and(warp::body::json()) .and(validator_store_filter.clone()) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( |body: Vec, validator_store: Arc>, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { if let Some(handle) = task_executor.handle() { let web3signers: Vec = body .into_iter() @@ -709,16 +681,14 @@ pub fn serve( .and(warp::body::json()) .and(validator_store_filter.clone()) .and(graffiti_file_filter.clone()) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( |validator_pubkey: PublicKey, body: api_types::ValidatorPatchRequest, validator_store: Arc>, graffiti_file: Option, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { if body.graffiti.is_some() && graffiti_file.is_some() { return Err(warp_utils::reject::custom_bad_request( "Unable to update graffiti as the \"--graffiti-file\" flag is set" @@ -827,10 +797,9 @@ pub fn serve( // GET /lighthouse/auth let get_auth = warp::path("lighthouse").and(warp::path("auth").and(warp::path::end())); let get_auth = get_auth - .and(signer.clone()) .and(api_token_path_filter) - .and_then(|signer, token_path: PathBuf| { - blocking_signed_json_task(signer, move || { + .then(move |token_path: PathBuf| { + blocking_json_task(move || { Ok(AuthResponse { token_path: token_path.display().to_string(), }) @@ -842,23 +811,20 @@ pub fn serve( .and(warp::path("keystores")) .and(warp::path::end()) .and(warp::body::json()) - .and(signer.clone()) .and(validator_store_filter.clone()) .and(task_executor_filter.clone()) .and(log_filter.clone()) - .and_then( - move |request, signer, validator_store, task_executor, log| { - blocking_signed_json_task(signer, move || { - if allow_keystore_export { - keystores::export(request, validator_store, task_executor, log) - } else { - Err(warp_utils::reject::custom_bad_request( - "keystore export is disabled".to_string(), - )) - } - }) - }, - ); + .then(move |request, validator_store, task_executor, log| { + blocking_json_task(move || { + if allow_keystore_export { + keystores::export(request, validator_store, task_executor, log) + } else { + Err(warp_utils::reject::custom_bad_request( + "keystore export is disabled".to_string(), + )) + } + }) + }); // Standard key-manager endpoints. let eth_v1 = warp::path("eth").and(warp::path("v1")); @@ -872,10 +838,9 @@ pub fn serve( .and(warp::path("feerecipient")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( - |validator_pubkey: PublicKey, validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then( + |validator_pubkey: PublicKey, validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -912,13 +877,11 @@ pub fn serve( .and(warp::body::json()) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( + .then( |validator_pubkey: PublicKey, request: api_types::UpdateFeeRecipientRequest, - validator_store: Arc>, - signer| { - blocking_signed_json_task(signer, move || { + validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -952,10 +915,9 @@ pub fn serve( .and(warp::path("feerecipient")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( - |validator_pubkey: PublicKey, validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then( + |validator_pubkey: PublicKey, validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -989,10 +951,9 @@ pub fn serve( .and(warp::path("gas_limit")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( - |validator_pubkey: PublicKey, validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then( + |validator_pubkey: PublicKey, validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -1021,13 +982,11 @@ pub fn serve( .and(warp::body::json()) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( + .then( |validator_pubkey: PublicKey, request: api_types::UpdateGasLimitRequest, - validator_store: Arc>, - signer| { - blocking_signed_json_task(signer, move || { + validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -1061,10 +1020,9 @@ pub fn serve( .and(warp::path("gas_limit")) .and(warp::path::end()) .and(validator_store_filter.clone()) - .and(signer.clone()) - .and_then( - |validator_pubkey: PublicKey, validator_store: Arc>, signer| { - blocking_signed_json_task(signer, move || { + .then( + |validator_pubkey: PublicKey, validator_store: Arc>| { + blocking_json_task(move || { if validator_store .initialized_validators() .read() @@ -1101,17 +1059,15 @@ pub fn serve( .and(validator_store_filter.clone()) .and(slot_clock_filter) .and(log_filter.clone()) - .and(signer.clone()) .and(task_executor_filter.clone()) - .and_then( + .then( |pubkey: PublicKey, query: api_types::VoluntaryExitQuery, validator_store: Arc>, slot_clock: T, log, - signer, task_executor: TaskExecutor| { - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { if let Some(handle) = task_executor.handle() { let signed_voluntary_exit = handle.block_on(create_signed_voluntary_exit( @@ -1139,13 +1095,11 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .and(graffiti_flag_filter) - .and(signer.clone()) - .and_then( + .then( |pubkey: PublicKey, validator_store: Arc>, - graffiti_flag: Option, - signer| { - blocking_signed_json_task(signer, move || { + graffiti_flag: Option| { + blocking_json_task(move || { let graffiti = get_graffiti(pubkey.clone(), validator_store, graffiti_flag)?; Ok(GenericResponse::from(GetGraffitiResponse { pubkey: pubkey.into(), @@ -1164,14 +1118,12 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .and(graffiti_file_filter.clone()) - .and(signer.clone()) - .and_then( + .then( |pubkey: PublicKey, query: SetGraffitiRequest, validator_store: Arc>, - graffiti_file: Option, - signer| { - blocking_signed_json_task(signer, move || { + graffiti_file: Option| { + blocking_json_task(move || { if graffiti_file.is_some() { return Err(warp_utils::reject::invalid_auth( "Unable to update graffiti as the \"--graffiti-file\" flag is set" @@ -1192,13 +1144,11 @@ pub fn serve( .and(warp::path::end()) .and(validator_store_filter.clone()) .and(graffiti_file_filter.clone()) - .and(signer.clone()) - .and_then( + .then( |pubkey: PublicKey, validator_store: Arc>, - graffiti_file: Option, - signer| { - blocking_signed_json_task(signer, move || { + graffiti_file: Option| { + blocking_json_task(move || { if graffiti_file.is_some() { return Err(warp_utils::reject::invalid_auth( "Unable to delete graffiti as the \"--graffiti-file\" flag is set" @@ -1212,32 +1162,24 @@ pub fn serve( .map(|reply| warp::reply::with_status(reply, warp::http::StatusCode::NO_CONTENT)); // GET /eth/v1/keystores - let get_std_keystores = std_keystores - .and(signer.clone()) - .and(validator_store_filter.clone()) - .and_then(|signer, validator_store: Arc>| { - blocking_signed_json_task(signer, move || Ok(keystores::list(validator_store))) - }); + let get_std_keystores = std_keystores.and(validator_store_filter.clone()).then( + |validator_store: Arc>| { + blocking_json_task(move || Ok(keystores::list(validator_store))) + }, + ); // POST /eth/v1/keystores let post_std_keystores = std_keystores .and(warp::body::json()) - .and(signer.clone()) .and(validator_dir_filter) .and(secrets_dir_filter) .and(validator_store_filter.clone()) .and(task_executor_filter.clone()) .and(log_filter.clone()) - .and_then( - move |request, - signer, - validator_dir, - secrets_dir, - validator_store, - task_executor, - log| { + .then( + move |request, validator_dir, secrets_dir, validator_store, task_executor, log| { let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir); - blocking_signed_json_task(signer, move || { + blocking_json_task(move || { keystores::import( request, validator_dir, @@ -1253,33 +1195,30 @@ pub fn serve( // DELETE /eth/v1/keystores let delete_std_keystores = std_keystores .and(warp::body::json()) - .and(signer.clone()) .and(validator_store_filter.clone()) .and(task_executor_filter.clone()) .and(log_filter.clone()) - .and_then(|request, signer, validator_store, task_executor, log| { - blocking_signed_json_task(signer, move || { + .then(|request, validator_store, task_executor, log| { + blocking_json_task(move || { keystores::delete(request, validator_store, task_executor, log) }) }); // GET /eth/v1/remotekeys - let get_std_remotekeys = std_remotekeys - .and(signer.clone()) - .and(validator_store_filter.clone()) - .and_then(|signer, validator_store: Arc>| { - blocking_signed_json_task(signer, move || Ok(remotekeys::list(validator_store))) - }); + let get_std_remotekeys = std_remotekeys.and(validator_store_filter.clone()).then( + |validator_store: Arc>| { + blocking_json_task(move || Ok(remotekeys::list(validator_store))) + }, + ); // POST /eth/v1/remotekeys let post_std_remotekeys = std_remotekeys .and(warp::body::json()) - .and(signer.clone()) .and(validator_store_filter.clone()) .and(task_executor_filter.clone()) .and(log_filter.clone()) - .and_then(|request, signer, validator_store, task_executor, log| { - blocking_signed_json_task(signer, move || { + .then(|request, validator_store, task_executor, log| { + blocking_json_task(move || { remotekeys::import(request, validator_store, task_executor, log) }) }); @@ -1287,12 +1226,11 @@ pub fn serve( // DELETE /eth/v1/remotekeys let delete_std_remotekeys = std_remotekeys .and(warp::body::json()) - .and(signer) .and(validator_store_filter) .and(task_executor_filter) .and(log_filter.clone()) - .and_then(|request, signer, validator_store, task_executor, log| { - blocking_signed_json_task(signer, move || { + .then(|request, validator_store, task_executor, log| { + blocking_json_task(move || { remotekeys::delete(request, validator_store, task_executor, log) }) }); @@ -1413,42 +1351,3 @@ pub fn serve( Ok((listening_socket, server)) } - -/// Executes `func` in blocking tokio task (i.e., where long-running tasks are permitted). -/// JSON-encodes the return value of `func`, using the `signer` function to produce a signature of -/// those bytes. -pub async fn blocking_signed_json_task( - signer: S, - func: F, -) -> Result -where - S: Fn(&[u8]) -> String, - F: FnOnce() -> Result + Send + 'static, - T: Serialize + Send + 'static, -{ - warp_utils::task::blocking_task(func) - .await - .map(|func_output| { - let mut response = match serde_json::to_vec(&func_output) { - Ok(body) => { - let mut res = Response::new(body); - res.headers_mut() - .insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); - res - } - Err(_) => Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(vec![]) - .expect("can produce simple response from static values"), - }; - - let body: &Vec = response.body(); - let signature = signer(body); - let header_value = - HeaderValue::from_str(&signature).expect("hash can be encoded as header"); - - response.headers_mut().append("Signature", header_value); - - response - }) -}