Runtime rpc request sizes (#4841)

* add runtime variable list type

* add configs to ChainSpec

* git rid of max request blocks type

* fix tests and lints

* remove todos

* git rid of old const usage

* fix decode impl

* add new config to `Config` api struct

* add docs fix compilt

* move methods for per-fork-spec to chainspec

* get values off chain spec

* fix compile

* remove min by root size

* add tests for runtime var list

---------

Co-authored-by: Jimmy Chen <jchen.tc@gmail.com>

consensus/types/src/chain_spec.rs

@@ -4,6 +4,7 @@ use int_to_bytes::int_to_bytes4;
 use serde::Deserialize;
 use serde::{Deserializer, Serialize, Serializer};
 use serde_utils::quoted_u64::MaybeQuoted;
+use ssz::Encode;
 use std::fs::File;
 use std::path::Path;
 use std::time::Duration;
@@ -172,22 +173,40 @@ pub struct ChainSpec {
      */
     pub boot_nodes: Vec<String>,
     pub network_id: u8,
-    pub attestation_propagation_slot_range: u64,
-    pub maximum_gossip_clock_disparity_millis: u64,
     pub target_aggregators_per_committee: u64,
-    pub attestation_subnet_count: u64,
-    pub subnets_per_node: u8,
-    pub epochs_per_subnet_subscription: u64,
     pub gossip_max_size: u64,
+    pub max_request_blocks: u64,
+    pub epochs_per_subnet_subscription: u64,
     pub min_epochs_for_block_requests: u64,
     pub max_chunk_size: u64,
     pub ttfb_timeout: u64,
     pub resp_timeout: u64,
+    pub attestation_propagation_slot_range: u64,
+    pub maximum_gossip_clock_disparity_millis: u64,
     pub message_domain_invalid_snappy: [u8; 4],
     pub message_domain_valid_snappy: [u8; 4],
+    pub subnets_per_node: u8,
+    pub attestation_subnet_count: u64,
     pub attestation_subnet_extra_bits: u8,
     pub attestation_subnet_prefix_bits: u8,
 
+    /*
+     * Networking Deneb
+     */
+    pub max_request_blocks_deneb: u64,
+    pub max_request_blob_sidecars: u64,
+    pub min_epochs_for_blob_sidecars_requests: u64,
+    pub blob_sidecar_subnet_count: u64,
+
+    /*
+     * Networking Derived
+     *
+     * When adding fields here, make sure any values are derived again during `apply_to_chain_spec`.
+     */
+    pub max_blocks_by_root_request: usize,
+    pub max_blocks_by_root_request_deneb: usize,
+    pub max_blobs_by_root_request: usize,
+
     /*
      * Application params
      */
@@ -487,6 +506,25 @@ impl ChainSpec {
         Duration::from_secs(self.resp_timeout)
     }
 
+    pub fn max_blocks_by_root_request(&self, fork_name: ForkName) -> usize {
+        match fork_name {
+            ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {
+                self.max_blocks_by_root_request
+            }
+            ForkName::Deneb => self.max_blocks_by_root_request_deneb,
+        }
+    }
+
+    pub fn max_request_blocks(&self, fork_name: ForkName) -> usize {
+        let max_request_blocks = match fork_name {
+            ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {
+                self.max_request_blocks
+            }
+            ForkName::Deneb => self.max_request_blocks_deneb,
+        };
+        max_request_blocks as usize
+    }
+
     /// Returns a `ChainSpec` compatible with the Ethereum Foundation specification.
     pub fn mainnet() -> Self {
         Self {
@@ -648,12 +686,12 @@ impl ChainSpec {
              */
             boot_nodes: vec![],
             network_id: 1, // mainnet network id
-            attestation_propagation_slot_range: 32,
+            attestation_propagation_slot_range: default_attestation_propagation_slot_range(),
             attestation_subnet_count: 64,
             subnets_per_node: 2,
-            maximum_gossip_clock_disparity_millis: 500,
+            maximum_gossip_clock_disparity_millis: default_maximum_gossip_clock_disparity_millis(),
             target_aggregators_per_committee: 16,
-            epochs_per_subnet_subscription: 256,
+            epochs_per_subnet_subscription: default_epochs_per_subnet_subscription(),
             gossip_max_size: default_gossip_max_size(),
             min_epochs_for_block_requests: default_min_epochs_for_block_requests(),
             max_chunk_size: default_max_chunk_size(),
@@ -663,6 +701,23 @@ impl ChainSpec {
             message_domain_valid_snappy: default_message_domain_valid_snappy(),
             attestation_subnet_extra_bits: default_attestation_subnet_extra_bits(),
             attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(),
+            max_request_blocks: default_max_request_blocks(),
+
+            /*
+             * Networking Deneb Specific
+             */
+            max_request_blocks_deneb: default_max_request_blocks_deneb(),
+            max_request_blob_sidecars: default_max_request_blob_sidecars(),
+            min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(),
+            blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(),
+
+            /*
+             * Derived Deneb Specific
+             */
+            max_blocks_by_root_request: default_max_blocks_by_root_request(),
+            max_blocks_by_root_request_deneb: default_max_blocks_by_root_request_deneb(),
+            max_blobs_by_root_request: default_max_blobs_by_root_request(),
+
             /*
              * Application specific
              */
@@ -892,12 +947,12 @@ impl ChainSpec {
              */
             boot_nodes: vec![],
             network_id: 100, // Gnosis Chain network id
-            attestation_propagation_slot_range: 32,
+            attestation_propagation_slot_range: default_attestation_propagation_slot_range(),
             attestation_subnet_count: 64,
             subnets_per_node: 4, // Make this larger than usual to avoid network damage
-            maximum_gossip_clock_disparity_millis: 500,
+            maximum_gossip_clock_disparity_millis: default_maximum_gossip_clock_disparity_millis(),
             target_aggregators_per_committee: 16,
-            epochs_per_subnet_subscription: 256,
+            epochs_per_subnet_subscription: default_epochs_per_subnet_subscription(),
             gossip_max_size: default_gossip_max_size(),
             min_epochs_for_block_requests: default_min_epochs_for_block_requests(),
             max_chunk_size: default_max_chunk_size(),
@@ -907,6 +962,22 @@ impl ChainSpec {
             message_domain_valid_snappy: default_message_domain_valid_snappy(),
             attestation_subnet_extra_bits: default_attestation_subnet_extra_bits(),
             attestation_subnet_prefix_bits: default_attestation_subnet_prefix_bits(),
+            max_request_blocks: default_max_request_blocks(),
+
+            /*
+             * Networking Deneb Specific
+             */
+            max_request_blocks_deneb: default_max_request_blocks_deneb(),
+            max_request_blob_sidecars: default_max_request_blob_sidecars(),
+            min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(),
+            blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(),
+
+            /*
+             * Derived Deneb Specific
+             */
+            max_blocks_by_root_request: default_max_blocks_by_root_request(),
+            max_blocks_by_root_request_deneb: default_max_blocks_by_root_request_deneb(),
+            max_blobs_by_root_request: default_max_blobs_by_root_request(),
 
             /*
              * Application specific
@@ -1032,6 +1103,12 @@ pub struct Config {
     #[serde(default = "default_gossip_max_size")]
     #[serde(with = "serde_utils::quoted_u64")]
     gossip_max_size: u64,
+    #[serde(default = "default_max_request_blocks")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    max_request_blocks: u64,
+    #[serde(default = "default_epochs_per_subnet_subscription")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    epochs_per_subnet_subscription: u64,
     #[serde(default = "default_min_epochs_for_block_requests")]
     #[serde(with = "serde_utils::quoted_u64")]
     min_epochs_for_block_requests: u64,
@@ -1044,6 +1121,12 @@ pub struct Config {
     #[serde(default = "default_resp_timeout")]
     #[serde(with = "serde_utils::quoted_u64")]
     resp_timeout: u64,
+    #[serde(default = "default_attestation_propagation_slot_range")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    attestation_propagation_slot_range: u64,
+    #[serde(default = "default_maximum_gossip_clock_disparity_millis")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    maximum_gossip_clock_disparity_millis: u64,
     #[serde(default = "default_message_domain_invalid_snappy")]
     #[serde(with = "serde_utils::bytes_4_hex")]
     message_domain_invalid_snappy: [u8; 4],
@@ -1056,6 +1139,18 @@ pub struct Config {
     #[serde(default = "default_attestation_subnet_prefix_bits")]
     #[serde(with = "serde_utils::quoted_u8")]
     attestation_subnet_prefix_bits: u8,
+    #[serde(default = "default_max_request_blocks_deneb")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    max_request_blocks_deneb: u64,
+    #[serde(default = "default_max_request_blob_sidecars")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    max_request_blob_sidecars: u64,
+    #[serde(default = "default_min_epochs_for_blob_sidecars_requests")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    min_epochs_for_blob_sidecars_requests: u64,
+    #[serde(default = "default_blob_sidecar_subnet_count")]
+    #[serde(with = "serde_utils::quoted_u64")]
+    blob_sidecar_subnet_count: u64,
 }
 
 fn default_bellatrix_fork_version() -> [u8; 4] {
@@ -1141,6 +1236,70 @@ const fn default_attestation_subnet_prefix_bits() -> u8 {
     6
 }
 
+const fn default_max_request_blocks() -> u64 {
+    1024
+}
+
+const fn default_max_request_blocks_deneb() -> u64 {
+    128
+}
+
+const fn default_max_request_blob_sidecars() -> u64 {
+    768
+}
+
+const fn default_min_epochs_for_blob_sidecars_requests() -> u64 {
+    4096
+}
+
+const fn default_blob_sidecar_subnet_count() -> u64 {
+    6
+}
+
+const fn default_epochs_per_subnet_subscription() -> u64 {
+    256
+}
+
+const fn default_attestation_propagation_slot_range() -> u64 {
+    32
+}
+
+const fn default_maximum_gossip_clock_disparity_millis() -> u64 {
+    500
+}
+
+fn max_blocks_by_root_request_common(max_request_blocks: u64) -> usize {
+    let max_request_blocks = max_request_blocks as usize;
+    RuntimeVariableList::<Hash256>::from_vec(
+        vec![Hash256::zero(); max_request_blocks],
+        max_request_blocks,
+    )
+    .as_ssz_bytes()
+    .len()
+}
+
+fn max_blobs_by_root_request_common(max_request_blob_sidecars: u64) -> usize {
+    let max_request_blob_sidecars = max_request_blob_sidecars as usize;
+    RuntimeVariableList::<Hash256>::from_vec(
+        vec![Hash256::zero(); max_request_blob_sidecars],
+        max_request_blob_sidecars,
+    )
+    .as_ssz_bytes()
+    .len()
+}
+
+fn default_max_blocks_by_root_request() -> usize {
+    max_blocks_by_root_request_common(default_max_request_blocks())
+}
+
+fn default_max_blocks_by_root_request_deneb() -> usize {
+    max_blocks_by_root_request_common(default_max_request_blocks_deneb())
+}
+
+fn default_max_blobs_by_root_request() -> usize {
+    max_blobs_by_root_request_common(default_max_request_blob_sidecars())
+}
+
 impl Default for Config {
     fn default() -> Self {
         let chain_spec = MainnetEthSpec::default_spec();
@@ -1243,14 +1402,22 @@ impl Config {
             deposit_contract_address: spec.deposit_contract_address,
 
             gossip_max_size: spec.gossip_max_size,
+            max_request_blocks: spec.max_request_blocks,
+            epochs_per_subnet_subscription: spec.epochs_per_subnet_subscription,
             min_epochs_for_block_requests: spec.min_epochs_for_block_requests,
             max_chunk_size: spec.max_chunk_size,
             ttfb_timeout: spec.ttfb_timeout,
             resp_timeout: spec.resp_timeout,
+            attestation_propagation_slot_range: spec.attestation_propagation_slot_range,
+            maximum_gossip_clock_disparity_millis: spec.maximum_gossip_clock_disparity_millis,
             message_domain_invalid_snappy: spec.message_domain_invalid_snappy,
             message_domain_valid_snappy: spec.message_domain_valid_snappy,
             attestation_subnet_extra_bits: spec.attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits: spec.attestation_subnet_prefix_bits,
+            max_request_blocks_deneb: spec.max_request_blocks_deneb,
+            max_request_blob_sidecars: spec.max_request_blob_sidecars,
+            min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests,
+            blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count,
         }
     }
 
@@ -1307,6 +1474,14 @@ impl Config {
             message_domain_valid_snappy,
             attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits,
+            max_request_blocks,
+            epochs_per_subnet_subscription,
+            attestation_propagation_slot_range,
+            maximum_gossip_clock_disparity_millis,
+            max_request_blocks_deneb,
+            max_request_blob_sidecars,
+            min_epochs_for_blob_sidecars_requests,
+            blob_sidecar_subnet_count,
         } = self;
 
         if preset_base != T::spec_name().to_string().as_str() {
@@ -1356,6 +1531,22 @@ impl Config {
             message_domain_valid_snappy,
             attestation_subnet_extra_bits,
             attestation_subnet_prefix_bits,
+            max_request_blocks,
+            epochs_per_subnet_subscription,
+            attestation_propagation_slot_range,
+            maximum_gossip_clock_disparity_millis,
+            max_request_blocks_deneb,
+            max_request_blob_sidecars,
+            min_epochs_for_blob_sidecars_requests,
+            blob_sidecar_subnet_count,
+
+            // We need to re-derive any values that might have changed in the config.
+            max_blocks_by_root_request: max_blocks_by_root_request_common(max_request_blocks),
+            max_blocks_by_root_request_deneb: max_blocks_by_root_request_common(
+                max_request_blocks_deneb,
+            ),
+            max_blobs_by_root_request: max_blobs_by_root_request_common(max_request_blob_sidecars),
+
             ..chain_spec.clone()
         })
     }

consensus/types/src/consts.rs

@@ -22,11 +22,3 @@ pub mod altair {
 pub mod merge {
     pub const INTERVALS_PER_SLOT: u64 = 3;
 }
-pub mod deneb {
-    use crate::Epoch;
-
-    pub const VERSIONED_HASH_VERSION_KZG: u8 = 1;
-    pub const BLOB_SIDECAR_SUBNET_COUNT: u64 = 6;
-    pub const MAX_BLOBS_PER_BLOCK: u64 = BLOB_SIDECAR_SUBNET_COUNT;
-    pub const MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: Epoch = Epoch::new(4096);
-}

consensus/types/src/fork_context.rs

@@ -9,6 +9,7 @@ pub struct ForkContext {
     current_fork: RwLock<ForkName>,
     fork_to_digest: HashMap<ForkName, [u8; 4]>,
     digest_to_fork: HashMap<[u8; 4], ForkName>,
+    pub spec: ChainSpec,
 }
 
 impl ForkContext {
@@ -73,6 +74,7 @@ impl ForkContext {
             current_fork: RwLock::new(spec.fork_name_at_slot::<T>(current_slot)),
             fork_to_digest,
             digest_to_fork,
+            spec: spec.clone(),
         }
     }
 

consensus/types/src/lib.rs

@@ -101,6 +101,7 @@ pub mod sqlite;
 pub mod blob_sidecar;
 pub mod light_client_header;
 pub mod non_zero_usize;
+pub mod runtime_var_list;
 
 use ethereum_types::{H160, H256};
 
@@ -168,6 +169,7 @@ pub use crate::preset::{AltairPreset, BasePreset, BellatrixPreset, CapellaPreset
 pub use crate::proposer_preparation_data::ProposerPreparationData;
 pub use crate::proposer_slashing::ProposerSlashing;
 pub use crate::relative_epoch::{Error as RelativeEpochError, RelativeEpoch};
+pub use crate::runtime_var_list::RuntimeVariableList;
 pub use crate::selection_proof::SelectionProof;
 pub use crate::shuffling_id::AttestationShufflingId;
 pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof;
@@ -216,7 +218,7 @@ pub use bls::{
     Signature, SignatureBytes,
 };
 
-pub use kzg::{KzgCommitment, KzgProof};
+pub use kzg::{KzgCommitment, KzgProof, VERSIONED_HASH_VERSION_KZG};
 
 pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList};
 pub use superstruct::superstruct;

consensus/types/src/runtime_var_list.rs

@@ -0,0 +1,137 @@
+use ssz::{Decode, Encode};
+use ssz_derive::Encode;
+
+#[derive(Debug, Clone, PartialEq, Encode)]
+#[ssz(struct_behaviour = "transparent")]
+pub struct RuntimeVariableList<T: Encode> {
+    vec: Vec<T>,
+    #[ssz(skip_serializing, skip_deserializing)]
+    max_len: usize,
+}
+
+impl<T: Encode + Decode + Clone> RuntimeVariableList<T> {
+    pub fn new(vec: Vec<T>, max_len: usize) -> Result<Self, ssz_types::Error> {
+        if vec.len() <= max_len {
+            Ok(Self { vec, max_len })
+        } else {
+            Err(ssz_types::Error::OutOfBounds {
+                i: vec.len(),
+                len: max_len,
+            })
+        }
+    }
+
+    pub fn from_vec(mut vec: Vec<T>, max_len: usize) -> Self {
+        vec.truncate(max_len);
+
+        Self { vec, max_len }
+    }
+
+    pub fn to_vec(&self) -> Vec<T> {
+        self.vec.clone()
+    }
+
+    pub fn as_slice(&self) -> &[T] {
+        self.vec.as_slice()
+    }
+
+    pub fn len(&self) -> usize {
+        self.vec.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.vec.is_empty()
+    }
+
+    pub fn from_ssz_bytes(bytes: &[u8], max_len: usize) -> Result<Self, ssz::DecodeError> {
+        let vec = if bytes.is_empty() {
+            vec![]
+        } else if <T as Decode>::is_ssz_fixed_len() {
+            let num_items = bytes
+                .len()
+                .checked_div(<T as Decode>::ssz_fixed_len())
+                .ok_or(ssz::DecodeError::ZeroLengthItem)?;
+
+            if num_items > max_len {
+                return Err(ssz::DecodeError::BytesInvalid(format!(
+                    "VariableList of {} items exceeds maximum of {}",
+                    num_items, max_len
+                )));
+            }
+
+            bytes
+                .chunks(<T as Decode>::ssz_fixed_len())
+                .try_fold(Vec::with_capacity(num_items), |mut vec, chunk| {
+                    vec.push(<T as Decode>::from_ssz_bytes(chunk)?);
+                    Ok(vec)
+                })
+                .map(Into::into)?
+        } else {
+            ssz::decode_list_of_variable_length_items(bytes, Some(max_len))?
+        };
+        Ok(Self { vec, max_len })
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use ssz_types::{typenum::U4, VariableList};
+
+    use super::*;
+
+    #[test]
+    fn new() {
+        let vec = vec![42; 5];
+        let runtime_var_list: Result<RuntimeVariableList<u64>, _> =
+            RuntimeVariableList::new(vec, 4);
+        assert!(runtime_var_list.is_err());
+
+        let vec = vec![42; 3];
+        let runtime_var_list: Result<RuntimeVariableList<u64>, _> =
+            RuntimeVariableList::new(vec, 4);
+        assert!(runtime_var_list.is_ok());
+
+        let vec = vec![42; 4];
+        let runtime_var_list: Result<RuntimeVariableList<u64>, _> =
+            RuntimeVariableList::new(vec, 4);
+        assert!(runtime_var_list.is_ok());
+    }
+
+    #[test]
+    fn length() {
+        let vec = vec![42; 3];
+        let runtime_var_list: RuntimeVariableList<u64> =
+            RuntimeVariableList::new(vec.clone(), 4).unwrap();
+        let var_list: VariableList<u64, U4> = VariableList::from(vec.clone());
+        assert_eq!(&runtime_var_list.as_slice()[0..3], &vec[..]);
+        assert_eq!(runtime_var_list.as_slice(), &vec![42, 42, 42][..]);
+        assert_eq!(runtime_var_list.len(), var_list.len());
+
+        let vec = vec![];
+        let runtime_var_list: RuntimeVariableList<u64> = RuntimeVariableList::new(vec, 4).unwrap();
+        assert_eq!(runtime_var_list.as_slice(), &[] as &[u64]);
+        assert!(runtime_var_list.is_empty());
+    }
+
+    #[test]
+    fn encode() {
+        let runtime_var_list: RuntimeVariableList<u16> =
+            RuntimeVariableList::new(vec![0; 2], 2).unwrap();
+
+        assert_eq!(runtime_var_list.as_ssz_bytes(), vec![0, 0, 0, 0]);
+        assert_eq!(<RuntimeVariableList<u16> as Encode>::ssz_fixed_len(), 4);
+    }
+
+    #[test]
+    fn round_trip() {
+        let item = RuntimeVariableList::<u16>::new(vec![42; 8], 8).unwrap();
+        let encoded = &item.as_ssz_bytes();
+        assert_eq!(item.ssz_bytes_len(), encoded.len());
+        assert_eq!(RuntimeVariableList::from_ssz_bytes(encoded, 8), Ok(item));
+
+        let item = RuntimeVariableList::<u16>::new(vec![0; 8], 8).unwrap();
+        let encoded = &item.as_ssz_bytes();
+        assert_eq!(item.ssz_bytes_len(), encoded.len());
+        assert_eq!(RuntimeVariableList::from_ssz_bytes(encoded, 8), Ok(item));
+    }
+}