Implement Metadatav3 (#6303)

* Add a V3 variant for metadata

* Add v3 for requests; persistence logic

* Set custody_subnets on setting metadata

* Fix tests

* Address some comments

* fmt

* Address more comments

* Fix tests

* Update metadata rpc limits

* Update method doc.

beacon_node/lighthouse_network/src/service/mod.rs

@@ -165,7 +165,17 @@ impl<E: EthSpec> Network<E> {
                 ctx.chain_spec,
             )?;
             // Construct the metadata
-            let meta_data = utils::load_or_build_metadata(&config.network_dir, &log);
+            let custody_subnet_count = if ctx.chain_spec.is_peer_das_scheduled() {
+                if config.subscribe_all_data_column_subnets {
+                    Some(ctx.chain_spec.data_column_sidecar_subnet_count)
+                } else {
+                    Some(ctx.chain_spec.custody_requirement)
+                }
+            } else {
+                None
+            };
+            let meta_data =
+                utils::load_or_build_metadata(&config.network_dir, custody_subnet_count, &log);
             let globals = NetworkGlobals::new(
                 enr,
                 meta_data,
@@ -1130,8 +1140,14 @@ impl<E: EthSpec> Network<E> {
 
     /// Sends a METADATA request to a peer.
     fn send_meta_data_request(&mut self, peer_id: PeerId) {
-        // We always prefer sending V2 requests
-        let event = OutboundRequest::MetaData(MetadataRequest::new_v2());
+        let event = if self.fork_context.spec.is_peer_das_scheduled() {
+            // Nodes with higher custody will probably start advertising it
+            // before peerdas is activated
+            OutboundRequest::MetaData(MetadataRequest::new_v3())
+        } else {
+            // We always prefer sending V2 requests otherwise
+            OutboundRequest::MetaData(MetadataRequest::new_v2())
+        };
         self.eth2_rpc_mut()
             .send_request(peer_id, RequestId::Internal, event);
     }
@@ -1139,15 +1155,12 @@ impl<E: EthSpec> Network<E> {
     /// Sends a METADATA response to a peer.
     fn send_meta_data_response(
         &mut self,
-        req: MetadataRequest<E>,
+        _req: MetadataRequest<E>,
         id: PeerRequestId,
         peer_id: PeerId,
     ) {
         let metadata = self.network_globals.local_metadata.read().clone();
-        let metadata = match req {
-            MetadataRequest::V1(_) => metadata.metadata_v1(),
-            MetadataRequest::V2(_) => metadata,
-        };
+        // The encoder is responsible for sending the negotiated version of the metadata
         let event = RPCCodedResponse::Success(RPCResponse::MetaData(metadata));
         self.eth2_rpc_mut().send_response(peer_id, id, event);
     }

beacon_node/lighthouse_network/src/service/utils.rs

@@ -1,4 +1,5 @@
 use crate::multiaddr::Protocol;
+use crate::rpc::methods::MetaDataV3;
 use crate::rpc::{MetaData, MetaDataV1, MetaDataV2};
 use crate::types::{
     error, EnrAttestationBitfield, EnrSyncCommitteeBitfield, GossipEncoding, GossipKind,
@@ -12,7 +13,6 @@ use libp2p::{core, noise, yamux, PeerId, Transport};
 use prometheus_client::registry::Registry;
 use slog::{debug, warn};
 use ssz::Decode;
-use ssz::Encode;
 use std::collections::HashSet;
 use std::fs::File;
 use std::io::prelude::*;
@@ -169,6 +169,7 @@ pub fn strip_peer_id(addr: &mut Multiaddr) {
 /// Load metadata from persisted file. Return default metadata if loading fails.
 pub fn load_or_build_metadata<E: EthSpec>(
     network_dir: &std::path::Path,
+    custody_subnet_count: Option<u64>,
     log: &slog::Logger,
 ) -> MetaData<E> {
     // We load a V2 metadata version by default (regardless of current fork)
@@ -219,7 +220,16 @@ pub fn load_or_build_metadata<E: EthSpec>(
     };
 
     // Wrap the MetaData
-    let meta_data = MetaData::V2(meta_data);
+    let meta_data = if let Some(custody_count) = custody_subnet_count {
+        MetaData::V3(MetaDataV3 {
+            attnets: meta_data.attnets,
+            seq_number: meta_data.seq_number,
+            syncnets: meta_data.syncnets,
+            custody_subnet_count: custody_count,
+        })
+    } else {
+        MetaData::V2(meta_data)
+    };
 
     debug!(log, "Metadata sequence number"; "seq_num" => meta_data.seq_number());
     save_metadata_to_disk(network_dir, meta_data.clone(), log);
@@ -276,10 +286,11 @@ pub(crate) fn save_metadata_to_disk<E: EthSpec>(
     log: &slog::Logger,
 ) {
     let _ = std::fs::create_dir_all(dir);
-    let metadata_bytes = match metadata {
-        MetaData::V1(md) => md.as_ssz_bytes(),
-        MetaData::V2(md) => md.as_ssz_bytes(),
-    };
+    // We always store the metadata v2 to disk because
+    // custody_subnet_count parameter doesn't need to be persisted across runs.
+    // custody_subnet_count is what the user sets it for the current run.
+    // This is to prevent ugly branching logic when reading the metadata from disk.
+    let metadata_bytes = metadata.metadata_v2().as_ssz_bytes();
     match File::create(dir.join(METADATA_FILENAME)).and_then(|mut f| f.write_all(&metadata_bytes)) {
         Ok(_) => {
             debug!(log, "Metadata written to disk");