feat: adds CLI flags to delay publishing for edge case testing on PeerDAS devnets (#6947)

Closes #6919

beacon_node/beacon_chain/src/chain_config.rs

@@ -94,6 +94,10 @@ pub struct ChainConfig {
     /// The delay in milliseconds applied by the node between sending each blob or data column batch.
     /// This doesn't apply if the node is the block proposer.
     pub blob_publication_batch_interval: Duration,
+    /// Artificial delay for block publishing. For PeerDAS testing only.
+    pub block_publishing_delay: Option<Duration>,
+    /// Artificial delay for data column publishing. For PeerDAS testing only.
+    pub data_column_publishing_delay: Option<Duration>,
 }
 
 impl Default for ChainConfig {
@@ -129,6 +133,8 @@ impl Default for ChainConfig {
             enable_sampling: false,
             blob_publication_batches: 4,
             blob_publication_batch_interval: Duration::from_millis(300),
+            block_publishing_delay: None,
+            data_column_publishing_delay: None,
         }
     }
 }

beacon_node/http_api/src/publish_blocks.rs

@@ -86,6 +86,8 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
     network_globals: Arc<NetworkGlobals<T::EthSpec>>,
 ) -> Result<Response, Rejection> {
     let seen_timestamp = timestamp_now();
+    let block_publishing_delay_for_testing = chain.config.block_publishing_delay;
+    let data_column_publishing_delay_for_testing = chain.config.data_column_publishing_delay;
 
     let (unverified_block, unverified_blobs, is_locally_built_block) = match provenanced_block {
         ProvenancedBlock::Local(block, blobs, _) => (block, blobs, true),
@@ -147,6 +149,14 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
 
     let should_publish_block = gossip_verified_block_result.is_ok();
     if BroadcastValidation::Gossip == validation_level && should_publish_block {
+        if let Some(block_publishing_delay) = block_publishing_delay_for_testing {
+            debug!(
+                log,
+                "Publishing block with artificial delay";
+                "block_publishing_delay" => ?block_publishing_delay
+            );
+            tokio::time::sleep(block_publishing_delay).await;
+        }
         publish_block_p2p(
             block.clone(),
             sender_clone.clone(),
@@ -207,6 +217,23 @@ pub async fn publish_block<T: BeaconChainTypes, B: IntoGossipVerifiedBlock<T>>(
     }
 
     if gossip_verified_columns.iter().map(Option::is_some).count() > 0 {
+        if let Some(data_column_publishing_delay) = data_column_publishing_delay_for_testing {
+            // Subtract block publishing delay if it is also used.
+            // Note: if `data_column_publishing_delay` is less than `block_publishing_delay`, it
+            // will still be delayed by `block_publishing_delay`. This could be solved with spawning
+            // async tasks but the limitation is minor and I believe it's probably not worth
+            // affecting the mainnet code path.
+            let block_publishing_delay = block_publishing_delay_for_testing.unwrap_or_default();
+            let delay = data_column_publishing_delay.saturating_sub(block_publishing_delay);
+            if !delay.is_zero() {
+                debug!(
+                    log,
+                    "Publishing data columns with artificial delay";
+                    "data_column_publishing_delay" => ?data_column_publishing_delay
+                );
+                tokio::time::sleep(delay).await;
+            }
+        }
         publish_column_sidecars(network_tx, &gossip_verified_columns, &chain).map_err(|_| {
             warp_utils::reject::custom_server_error("unable to publish data column sidecars".into())
         })?;

beacon_node/src/cli.rs

@@ -1599,5 +1599,30 @@ pub fn cli_app() -> Command {
                 .action(ArgAction::Set)
                 .display_order(0)
         )
+        .arg(
+            Arg::new("delay-block-publishing")
+                .long("delay-block-publishing")
+                .value_name("SECONDS")
+                .action(ArgAction::Set)
+                .help_heading(FLAG_HEADER)
+                .help("TESTING ONLY: Artificially delay block publishing by the specified number of seconds. \
+                        This only works for if `BroadcastValidation::Gossip` is used (default). \
+                        DO NOT USE IN PRODUCTION.")
+                .hide(true)
+                .display_order(0)
+        )
+        .arg(
+            Arg::new("delay-data-column-publishing")
+                .long("delay-data-column-publishing")
+                .value_name("SECONDS") 
+                .action(ArgAction::Set)
+                .help_heading(FLAG_HEADER)
+                .help("TESTING ONLY: Artificially delay data column publishing by the specified number of seconds. \
+                       Limitation: If `delay-block-publishing` is also used, data columns will be delayed for a \
+                       minimum of `delay-block-publishing` seconds.
+                       DO NOT USE IN PRODUCTION.")
+                .hide(true)
+                .display_order(0)
+        )
         .group(ArgGroup::new("enable_http").args(["http", "gui", "staking"]).multiple(true))
 }

beacon_node/src/config.rs

@@ -895,6 +895,14 @@ pub fn get_config<E: EthSpec>(
         .max_gossip_aggregate_batch_size =
         clap_utils::parse_required(cli_args, "beacon-processor-aggregate-batch-size")?;
 
+    if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-block-publishing")? {
+        client_config.chain.block_publishing_delay = Some(Duration::from_secs_f64(delay));
+    }
+
+    if let Some(delay) = clap_utils::parse_optional(cli_args, "delay-data-column-publishing")? {
+        client_config.chain.data_column_publishing_delay = Some(Duration::from_secs_f64(delay));
+    }
+
     Ok(client_config)
 }
 

lighthouse/tests/beacon_node.rs

@@ -2715,3 +2715,29 @@ fn beacon_node_backend_override() {
             assert_eq!(config.store.backend, BeaconNodeBackend::LevelDb);
         });
 }
+
+#[test]
+fn block_publishing_delay_for_testing() {
+    CommandLineTest::new()
+        .flag("delay-block-publishing", Some("2.5"))
+        .run_with_zero_port()
+        .with_config(|config| {
+            assert_eq!(
+                config.chain.block_publishing_delay,
+                Some(Duration::from_secs_f64(2.5f64))
+            );
+        });
+}
+
+#[test]
+fn data_column_publishing_delay_for_testing() {
+    CommandLineTest::new()
+        .flag("delay-data-column-publishing", Some("3.5"))
+        .run_with_zero_port()
+        .with_config(|config| {
+            assert_eq!(
+                config.chain.data_column_publishing_delay,
+                Some(Duration::from_secs_f64(3.5f64))
+            );
+        });
+}