gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-26 01:03:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clarify network limits (#7175)

Browse Source 
Resolves #6811


  Rename `GOSSIP_MAX_SIZE` to `MAX_PAYLOAD_SIZE` and remove `MAX_CHUNK_SIZE` in accordance with the spec.

The spec also "clarifies"  the message size limits at different levels. The rpc limits are equivalent to what we had before imo.
The gossip limits have additional checks.

I have gotten rid of the `is_bellatrix_enabled`  checks that used a lower limit (1mb) pre-merge. Since all networks we run start from the merge, I don't think this will break any setups.
This commit is contained in:
Pawan Dhananjay
2025-04-08 19:50:45 -07:00
committed by GitHub
parent d24a4ffe30
commit 076f3f0984
17 changed files with 114 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
beacon_node/lighthouse_network/src/config.rs
View File

@@ -14,7 +14,7 @@ use std::num::NonZeroU16;
use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
use types::{ForkContext, ForkName};
use types::ForkContext;
pub const DEFAULT_IPV4_ADDRESS: Ipv4Addr = Ipv4Addr::UNSPECIFIED;
pub const DEFAULT_TCP_PORT: u16 = 9000u16;
@@ -22,18 +22,9 @@ pub const DEFAULT_DISC_PORT: u16 = 9000u16;
pub const DEFAULT_QUIC_PORT: u16 = 9001u16;
pub const DEFAULT_IDONTWANT_MESSAGE_SIZE_THRESHOLD: usize = 1000usize;
/// The maximum size of gossip messages.
pub fn gossip_max_size(is_merge_enabled: bool, gossip_max_size: usize) -> usize {
if is_merge_enabled {
gossip_max_size
} else {
gossip_max_size / 10
}
}
pub struct GossipsubConfigParams {
pub message_domain_valid_snappy: [u8; 4],
pub gossip_max_size: usize,
pub gossipsub_max_transmit_size: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
@@ -480,7 +471,6 @@ pub fn gossipsub_config(
}
}
let message_domain_valid_snappy = gossipsub_config_params.message_domain_valid_snappy;
let is_bellatrix_enabled = fork_context.fork_exists(ForkName::Bellatrix);
let gossip_message_id = move |message: &gossipsub::Message| {
gossipsub::MessageId::from(
&Sha256::digest(
@@ -499,10 +489,7 @@ pub fn gossipsub_config(
let duplicate_cache_time = Duration::from_secs(slots_per_epoch * seconds_per_slot * 2);
gossipsub::ConfigBuilder::default()
.max_transmit_size(gossip_max_size(
is_bellatrix_enabled,
gossipsub_config_params.gossip_max_size,
))
.max_transmit_size(gossipsub_config_params.gossipsub_max_transmit_size)
.heartbeat_interval(load.heartbeat_interval)
.mesh_n(load.mesh_n)
.mesh_n_low(load.mesh_n_low)

1
beacon_node/lighthouse_network/src/lib.rs
View File

@@ -12,7 +12,6 @@ pub mod peer_manager;
pub mod rpc;
pub mod types;
pub use config::gossip_max_size;
use libp2p::swarm::DialError;
pub use listen_addr::*;

34
beacon_node/lighthouse_network/src/rpc/codec.rs
View File

@@ -1002,10 +1002,7 @@ mod tests {
}
/// Bellatrix block with length < max_rpc_size.
fn bellatrix_block_small(
fork_context: &ForkContext,
spec: &ChainSpec,
) -> SignedBeaconBlock<Spec> {
fn bellatrix_block_small(spec: &ChainSpec) -> SignedBeaconBlock<Spec> {
let mut block: BeaconBlockBellatrix<_, FullPayload<Spec>> =
BeaconBlockBellatrix::empty(&Spec::default_spec());
let tx = VariableList::from(vec![0; 1024]);
@@ -1014,17 +1011,14 @@ mod tests {
block.body.execution_payload.execution_payload.transactions = txs;
let block = BeaconBlock::Bellatrix(block);
assert!(block.ssz_bytes_len() <= max_rpc_size(fork_context, spec.max_chunk_size as usize));
assert!(block.ssz_bytes_len() <= spec.max_payload_size as usize);
SignedBeaconBlock::from_block(block, Signature::empty())
}
/// Bellatrix block with length > MAX_RPC_SIZE.
/// The max limit for a Bellatrix block is in the order of ~16GiB which wouldn't fit in memory.
/// Hence, we generate a Bellatrix block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer.
fn bellatrix_block_large(
fork_context: &ForkContext,
spec: &ChainSpec,
) -> SignedBeaconBlock<Spec> {
fn bellatrix_block_large(spec: &ChainSpec) -> SignedBeaconBlock<Spec> {
let mut block: BeaconBlockBellatrix<_, FullPayload<Spec>> =
BeaconBlockBellatrix::empty(&Spec::default_spec());
let tx = VariableList::from(vec![0; 1024]);
@@ -1033,7 +1027,7 @@ mod tests {
block.body.execution_payload.execution_payload.transactions = txs;
let block = BeaconBlock::Bellatrix(block);
assert!(block.ssz_bytes_len() > max_rpc_size(fork_context, spec.max_chunk_size as usize));
assert!(block.ssz_bytes_len() > spec.max_payload_size as usize);
SignedBeaconBlock::from_block(block, Signature::empty())
}
@@ -1138,7 +1132,7 @@ mod tests {
) -> Result<BytesMut, RPCError> {
let snappy_protocol_id = ProtocolId::new(protocol, Encoding::SSZSnappy);
let fork_context = Arc::new(fork_context(fork_name));
let max_packet_size = max_rpc_size(&fork_context, spec.max_chunk_size as usize);
let max_packet_size = spec.max_payload_size as usize;
let mut buf = BytesMut::new();
let mut snappy_inbound_codec =
@@ -1185,7 +1179,7 @@ mod tests {
) -> Result<Option<RpcSuccessResponse<Spec>>, RPCError> {
let snappy_protocol_id = ProtocolId::new(protocol, Encoding::SSZSnappy);
let fork_context = Arc::new(fork_context(fork_name));
let max_packet_size = max_rpc_size(&fork_context, spec.max_chunk_size as usize);
let max_packet_size = spec.max_payload_size as usize;
let mut snappy_outbound_codec =
SSZSnappyOutboundCodec::<Spec>::new(snappy_protocol_id, max_packet_size, fork_context);
// decode message just as snappy message
@@ -1206,7 +1200,7 @@ mod tests {
/// Verifies that requests we send are encoded in a way that we would correctly decode too.
fn encode_then_decode_request(req: RequestType<Spec>, fork_name: ForkName, spec: &ChainSpec) {
let fork_context = Arc::new(fork_context(fork_name));
let max_packet_size = max_rpc_size(&fork_context, spec.max_chunk_size as usize);
let max_packet_size = spec.max_payload_size as usize;
let protocol = ProtocolId::new(req.versioned_protocol(), Encoding::SSZSnappy);
// Encode a request we send
let mut buf = BytesMut::new();
@@ -1583,10 +1577,8 @@ mod tests {
))))
);
let bellatrix_block_small =
bellatrix_block_small(&fork_context(ForkName::Bellatrix), &chain_spec);
let bellatrix_block_large =
bellatrix_block_large(&fork_context(ForkName::Bellatrix), &chain_spec);
let bellatrix_block_small = bellatrix_block_small(&chain_spec);
let bellatrix_block_large = bellatrix_block_large(&chain_spec);
assert_eq!(
encode_then_decode_response(
@@ -2086,7 +2078,7 @@ mod tests {
// Insert length-prefix
uvi_codec
.encode(chain_spec.max_chunk_size as usize + 1, &mut dst)
.encode(chain_spec.max_payload_size as usize + 1, &mut dst)
.unwrap();
// Insert snappy stream identifier
@@ -2124,7 +2116,7 @@ mod tests {
let mut snappy_outbound_codec = SSZSnappyOutboundCodec::<Spec>::new(
snappy_protocol_id,
max_rpc_size(&fork_context, chain_spec.max_chunk_size as usize),
chain_spec.max_payload_size as usize,
fork_context,
);
@@ -2160,7 +2152,7 @@ mod tests {
let mut snappy_outbound_codec = SSZSnappyOutboundCodec::<Spec>::new(
snappy_protocol_id,
max_rpc_size(&fork_context, chain_spec.max_chunk_size as usize),
chain_spec.max_payload_size as usize,
fork_context,
);
@@ -2189,7 +2181,7 @@ mod tests {
let chain_spec = Spec::default_spec();
let max_rpc_size = max_rpc_size(&fork_context, chain_spec.max_chunk_size as usize);
let max_rpc_size = chain_spec.max_payload_size as usize;
let limit = protocol_id.rpc_response_limits::<Spec>(&fork_context);
let mut max = encode_len(limit.max + 1);
let mut codec = SSZSnappyOutboundCodec::<Spec>::new(

8
beacon_node/lighthouse_network/src/rpc/mod.rs
View File

@@ -33,7 +33,7 @@ pub use methods::{
BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason, LightClientBootstrapRequest,
ResponseTermination, RpcErrorResponse, StatusMessage,
};
pub use protocol::{max_rpc_size, Protocol, RPCError};
pub use protocol::{Protocol, RPCError};
use self::config::{InboundRateLimiterConfig, OutboundRateLimiterConfig};
use self::protocol::RPCProtocol;
@@ -143,7 +143,7 @@ pub struct RPCMessage<Id, E: EthSpec> {
type BehaviourAction<Id, E> = ToSwarm<RPCMessage<Id, E>, RPCSend<Id, E>>;
pub struct NetworkParams {
pub max_chunk_size: usize,
pub max_payload_size: usize,
pub ttfb_timeout: Duration,
pub resp_timeout: Duration,
}
@@ -284,7 +284,7 @@ where
let protocol = SubstreamProtocol::new(
RPCProtocol {
fork_context: self.fork_context.clone(),
max_rpc_size: max_rpc_size(&self.fork_context, self.network_params.max_chunk_size),
max_rpc_size: self.fork_context.spec.max_payload_size as usize,
enable_light_client_server: self.enable_light_client_server,
phantom: PhantomData,
ttfb_timeout: self.network_params.ttfb_timeout,
@@ -315,7 +315,7 @@ where
let protocol = SubstreamProtocol::new(
RPCProtocol {
fork_context: self.fork_context.clone(),
max_rpc_size: max_rpc_size(&self.fork_context, self.network_params.max_chunk_size),
max_rpc_size: self.fork_context.spec.max_payload_size as usize,
enable_light_client_server: self.enable_light_client_server,
phantom: PhantomData,
ttfb_timeout: self.network_params.ttfb_timeout,

11
beacon_node/lighthouse_network/src/rpc/protocol.rs
View File

@@ -57,7 +57,7 @@ pub static SIGNED_BEACON_BLOCK_ALTAIR_MAX: LazyLock<usize> = LazyLock::new(|| {
/// The `BeaconBlockBellatrix` block has an `ExecutionPayload` field which has a max size ~16 GiB for future proofing.
/// We calculate the value from its fields instead of constructing the block and checking the length.
/// Note: This is only the theoretical upper bound. We further bound the max size we receive over the network
/// with `max_chunk_size`.
/// with `max_payload_size`.
pub static SIGNED_BEACON_BLOCK_BELLATRIX_MAX: LazyLock<usize> =
LazyLock::new(|| // Size of a full altair block
*SIGNED_BEACON_BLOCK_ALTAIR_MAX
@@ -122,15 +122,6 @@ const PROTOCOL_PREFIX: &str = "/eth2/beacon_chain/req";
/// established before the stream is terminated.
const REQUEST_TIMEOUT: u64 = 15;
/// Returns the maximum bytes that can be sent across the RPC.
pub fn max_rpc_size(fork_context: &ForkContext, max_chunk_size: usize) -> usize {
if fork_context.current_fork().bellatrix_enabled() {
max_chunk_size
} else {
max_chunk_size / 10
}
}
/// Returns the rpc limits for beacon_block_by_range and beacon_block_by_root responses.
///
/// Note: This function should take care to return the min/max limits accounting for all

8
beacon_node/lighthouse_network/src/service/mod.rs
View File

@@ -224,7 +224,7 @@ impl<E: EthSpec> Network<E> {
let gossipsub_config_params = GossipsubConfigParams {
message_domain_valid_snappy: ctx.chain_spec.message_domain_valid_snappy,
gossip_max_size: ctx.chain_spec.gossip_max_size as usize,
gossipsub_max_transmit_size: ctx.chain_spec.max_message_size(),
};
let gs_config = gossipsub_config(
config.network_load,
@@ -310,7 +310,9 @@ impl<E: EthSpec> Network<E> {
)
});
let snappy_transform = SnappyTransform::new(gs_config.max_transmit_size());
let spec = &ctx.chain_spec;
let snappy_transform =
SnappyTransform::new(spec.max_payload_size as usize, spec.max_compressed_len());
let mut gossipsub = Gossipsub::new_with_subscription_filter_and_transform(
MessageAuthenticity::Anonymous,
gs_config.clone(),
@@ -349,7 +351,7 @@ impl<E: EthSpec> Network<E> {
};
let network_params = NetworkParams {
max_chunk_size: ctx.chain_spec.max_chunk_size as usize,
max_payload_size: ctx.chain_spec.max_payload_size as usize,
ttfb_timeout: ctx.chain_spec.ttfb_timeout(),
resp_timeout: ctx.chain_spec.resp_timeout(),
};

28
beacon_node/lighthouse_network/src/types/pubsub.rs
View File

@@ -52,13 +52,16 @@ pub enum PubsubMessage<E: EthSpec> {
// Implements the `DataTransform` trait of gossipsub to employ snappy compression
pub struct SnappyTransform {
/// Sets the maximum size we allow gossipsub messages to decompress to.
max_size_per_message: usize,
max_uncompressed_len: usize,
/// Sets the maximum size we allow for compressed gossipsub message data.
max_compressed_len: usize,
}
impl SnappyTransform {
pub fn new(max_size_per_message: usize) -> Self {
pub fn new(max_uncompressed_len: usize, max_compressed_len: usize) -> Self {
SnappyTransform {
max_size_per_message,
max_uncompressed_len,
max_compressed_len,
}
}
}
@@ -69,12 +72,19 @@ impl gossipsub::DataTransform for SnappyTransform {
&self,
raw_message: gossipsub::RawMessage,
) -> Result<gossipsub::Message, std::io::Error> {
// check the length of the raw bytes
let len = decompress_len(&raw_message.data)?;
if len > self.max_size_per_message {
// first check the size of the compressed payload
if raw_message.data.len() > self.max_compressed_len {
return Err(Error::new(
ErrorKind::InvalidData,
"ssz_snappy decoded data > GOSSIP_MAX_SIZE",
"ssz_snappy encoded data > max_compressed_len",
));
}
// check the length of the uncompressed bytes
let len = decompress_len(&raw_message.data)?;
if len > self.max_uncompressed_len {
return Err(Error::new(
ErrorKind::InvalidData,
"ssz_snappy decoded data > MAX_PAYLOAD_SIZE",
));
}
@@ -98,10 +108,10 @@ impl gossipsub::DataTransform for SnappyTransform {
) -> Result<Vec<u8>, std::io::Error> {
// Currently we are not employing topic-based compression. Everything is expected to be
// snappy compressed.
if data.len() > self.max_size_per_message {
if data.len() > self.max_uncompressed_len {
return Err(Error::new(
ErrorKind::InvalidData,
"ssz_snappy Encoded data > GOSSIP_MAX_SIZE",
"ssz_snappy Encoded data > MAX_PAYLOAD_SIZE",
));
}
let mut encoder = Encoder::new();

18
beacon_node/lighthouse_network/tests/rpc_tests.rs
View File

@@ -5,7 +5,7 @@ mod common;
use common::Protocol;
use lighthouse_network::rpc::{methods::*, RequestType};
use lighthouse_network::service::api_types::AppRequestId;
use lighthouse_network::{rpc::max_rpc_size, NetworkEvent, ReportSource, Response};
use lighthouse_network::{NetworkEvent, ReportSource, Response};
use slog::{debug, warn, Level};
use ssz::Encode;
use ssz_types::VariableList;
@@ -15,14 +15,14 @@ use tokio::runtime::Runtime;
use tokio::time::sleep;
use types::{
BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BlobSidecar, ChainSpec,
EmptyBlock, Epoch, EthSpec, FixedBytesExtended, ForkContext, ForkName, Hash256, MinimalEthSpec,
EmptyBlock, Epoch, EthSpec, FixedBytesExtended, ForkName, Hash256, MinimalEthSpec,
RuntimeVariableList, Signature, SignedBeaconBlock, Slot,
};
type E = MinimalEthSpec;
/// Bellatrix block with length < max_rpc_size.
fn bellatrix_block_small(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock<E> {
fn bellatrix_block_small(spec: &ChainSpec) -> BeaconBlock<E> {
let mut block = BeaconBlockBellatrix::<E>::empty(spec);
let tx = VariableList::from(vec![0; 1024]);
let txs = VariableList::from(std::iter::repeat_n(tx, 5000).collect::<Vec<_>>());
@@ -30,14 +30,14 @@ fn bellatrix_block_small(fork_context: &ForkContext, spec: &ChainSpec) -> Beacon
block.body.execution_payload.execution_payload.transactions = txs;
let block = BeaconBlock::Bellatrix(block);
assert!(block.ssz_bytes_len() <= max_rpc_size(fork_context, spec.max_chunk_size as usize));
assert!(block.ssz_bytes_len() <= spec.max_payload_size as usize);
block
}
/// Bellatrix block with length > MAX_RPC_SIZE.
/// The max limit for a bellatrix block is in the order of ~16GiB which wouldn't fit in memory.
/// Hence, we generate a bellatrix block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer.
fn bellatrix_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock<E> {
fn bellatrix_block_large(spec: &ChainSpec) -> BeaconBlock<E> {
let mut block = BeaconBlockBellatrix::<E>::empty(spec);
let tx = VariableList::from(vec![0; 1024]);
let txs = VariableList::from(std::iter::repeat_n(tx, 100000).collect::<Vec<_>>());
@@ -45,7 +45,7 @@ fn bellatrix_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> Beacon
block.body.execution_payload.execution_payload.transactions = txs;
let block = BeaconBlock::Bellatrix(block);
assert!(block.ssz_bytes_len() > max_rpc_size(fork_context, spec.max_chunk_size as usize));
assert!(block.ssz_bytes_len() > spec.max_payload_size as usize);
block
}
@@ -193,7 +193,7 @@ fn test_tcp_blocks_by_range_chunked_rpc() {
let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty());
let rpc_response_altair = Response::BlocksByRange(Some(Arc::new(signed_full_block)));
let full_block = bellatrix_block_small(&common::fork_context(ForkName::Bellatrix), &spec);
let full_block = bellatrix_block_small(&spec);
let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty());
let rpc_response_bellatrix_small =
Response::BlocksByRange(Some(Arc::new(signed_full_block)));
@@ -455,7 +455,7 @@ fn test_tcp_blocks_by_range_over_limit() {
}));
// BlocksByRange Response
let full_block = bellatrix_block_large(&common::fork_context(ForkName::Bellatrix), &spec);
let full_block = bellatrix_block_large(&spec);
let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty());
let rpc_response_bellatrix_large =
Response::BlocksByRange(Some(Arc::new(signed_full_block)));
@@ -834,7 +834,7 @@ fn test_tcp_blocks_by_root_chunked_rpc() {
let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty());
let rpc_response_altair = Response::BlocksByRoot(Some(Arc::new(signed_full_block)));
let full_block = bellatrix_block_small(&common::fork_context(ForkName::Bellatrix), &spec);
let full_block = bellatrix_block_small(&spec);
let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty());
let rpc_response_bellatrix_small =
Response::BlocksByRoot(Some(Arc::new(signed_full_block)));