Merge remote-tracking branch 'origin/stable' into back-merge-6.0.1

beacon_node/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "beacon_node"
-version = "6.0.0"
+version = "6.0.1"
 authors = [
     "Paul Hauner <paul@paulhauner.com>",
     "Age Manning <Age@AgeManning.com",

beacon_node/beacon_chain/src/builder.rs

@@ -1037,7 +1037,9 @@ where
         );
 
         // Check for states to reconstruct (in the background).
-        if beacon_chain.config.reconstruct_historic_states {
+        if beacon_chain.config.reconstruct_historic_states
+            && beacon_chain.store.get_oldest_block_slot() == 0
+        {
             beacon_chain.store_migrator.process_reconstruction();
         }
 

beacon_node/beacon_chain/src/migrate.rs

@@ -26,8 +26,10 @@ const MIN_COMPACTION_PERIOD_SECONDS: u64 = 7200;
 const COMPACTION_FINALITY_DISTANCE: u64 = 1024;
 /// Maximum number of blocks applied in each reconstruction burst.
 ///
-/// This limits the amount of time that the finalization migration is paused for.
-const BLOCKS_PER_RECONSTRUCTION: usize = 8192 * 4;
+/// This limits the amount of time that the finalization migration is paused for. We set this
+/// conservatively because pausing the finalization migration for too long can cause hot state
+/// cache misses and excessive disk use.
+const BLOCKS_PER_RECONSTRUCTION: usize = 1024;
 
 /// Default number of epochs to wait between finalization migrations.
 pub const DEFAULT_EPOCHS_PER_MIGRATION: u64 = 1;

beacon_node/beacon_chain/src/schema_change/migration_schema_v22.rs

@@ -152,7 +152,7 @@ pub fn delete_old_schema_freezer_data<T: BeaconChainTypes>(
     db.cold_db.do_atomically(cold_ops)?;
 
     // In order to reclaim space, we need to compact the freezer DB as well.
-    db.cold_db.compact()?;
+    db.compact_freezer()?;
 
     Ok(())
 }

beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs

@@ -141,10 +141,6 @@ impl<E: EthSpec> NetworkBehaviour for PeerManager<E> {
                 debug!(self.log, "Failed to dial peer"; "peer_id"=> ?peer_id, "error" => %ClearDialError(error));
                 self.on_dial_failure(peer_id);
             }
-            FromSwarm::ExternalAddrConfirmed(_) => {
-                // We have an external address confirmed, means we are able to do NAT traversal.
-                metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p"], 1);
-            }
             _ => {
                 // NOTE: FromSwarm is a non exhaustive enum so updates should be based on release
                 // notes more than compiler feedback

beacon_node/network/src/subnet_service/mod.rs

@@ -86,7 +86,7 @@ pub struct SubnetService<T: BeaconChainTypes> {
     subscriptions: HashSetDelay<Subnet>,
 
     /// Subscriptions that need to be executed in the future.
-    scheduled_subscriptions: HashSetDelay<Subnet>,
+    scheduled_subscriptions: HashSetDelay<ExactSubnet>,
 
     /// A list of permanent subnets that this node is subscribed to.
     // TODO: Shift this to a dynamic bitfield
@@ -484,8 +484,10 @@ impl<T: BeaconChainTypes> SubnetService<T> {
             self.subscribe_to_subnet_immediately(subnet, slot + 1)?;
         } else {
             // This is a future slot, schedule subscribing.
+            // We need to include the slot to make the key unique to prevent overwriting the entry
+            // for the same subnet.
             self.scheduled_subscriptions
-                .insert_at(subnet, time_to_subscription_start);
+                .insert_at(ExactSubnet { subnet, slot }, time_to_subscription_start);
         }
 
         Ok(())
@@ -626,7 +628,8 @@ impl<T: BeaconChainTypes> Stream for SubnetService<T> {
         // Process scheduled subscriptions that might be ready, since those can extend a soon to
         // expire subscription.
         match self.scheduled_subscriptions.poll_next_unpin(cx) {
-            Poll::Ready(Some(Ok(subnet))) => {
+            Poll::Ready(Some(Ok(exact_subnet))) => {
+                let ExactSubnet { subnet, .. } = exact_subnet;
                 let current_slot = self.beacon_chain.slot_clock.now().unwrap_or_default();
                 if let Err(e) = self.subscribe_to_subnet_immediately(subnet, current_slot + 1) {
                     debug!(self.log, "Failed to subscribe to short lived subnet"; "subnet" => ?subnet, "err" => e);

beacon_node/network/src/subnet_service/tests/mod.rs

@@ -500,12 +500,15 @@ mod test {
         // subscription config
         let committee_count = 1;
 
-        // Makes 2 validator subscriptions to the same subnet but at different slots.
-        // There should be just 1 unsubscription event for the later slot subscription (subscription_slot2).
+        // Makes 3 validator subscriptions to the same subnet but at different slots.
+        // There should be just 1 unsubscription event for each of the later slots subscriptions
+        // (subscription_slot2 and subscription_slot3).
         let subscription_slot1 = 0;
         let subscription_slot2 = MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD + 4;
+        let subscription_slot3 = subscription_slot2 * 2;
         let com1 = MIN_PEER_DISCOVERY_SLOT_LOOK_AHEAD + 4;
         let com2 = 0;
+        let com3 = CHAIN.chain.spec.attestation_subnet_count - com1;
 
         // create the attestation service and subscriptions
         let mut subnet_service = get_subnet_service();
@@ -532,6 +535,13 @@ mod test {
             true,
         );
 
+        let sub3 = get_subscription(
+            com3,
+            current_slot + Slot::new(subscription_slot3),
+            committee_count,
+            true,
+        );
+
         let subnet_id1 = SubnetId::compute_subnet::<MainnetEthSpec>(
             current_slot + Slot::new(subscription_slot1),
             com1,
@@ -548,12 +558,23 @@ mod test {
         )
         .unwrap();
 
+        let subnet_id3 = SubnetId::compute_subnet::<MainnetEthSpec>(
+            current_slot + Slot::new(subscription_slot3),
+            com3,
+            committee_count,
+            &subnet_service.beacon_chain.spec,
+        )
+        .unwrap();
+
         // Assert that subscriptions are different but their subnet is the same
         assert_ne!(sub1, sub2);
+        assert_ne!(sub1, sub3);
+        assert_ne!(sub2, sub3);
         assert_eq!(subnet_id1, subnet_id2);
+        assert_eq!(subnet_id1, subnet_id3);
 
         // submit the subscriptions
-        subnet_service.validator_subscriptions(vec![sub1, sub2].into_iter());
+        subnet_service.validator_subscriptions(vec![sub1, sub2, sub3].into_iter());
 
         // Unsubscription event should happen at the end of the slot.
         // We wait for 2 slots, to avoid timeout issues
@@ -590,10 +611,36 @@ mod test {
         // If the permanent and short lived subnets are different, we should get an unsubscription event.
         if !subnet_service.is_subscribed(&Subnet::Attestation(subnet_id1)) {
             assert_eq!(
-                [expected_subscription, expected_unsubscription],
+                [
+                    expected_subscription.clone(),
+                    expected_unsubscription.clone(),
+                ],
                 second_subscribe_event[..]
             );
         }
+
+        let subscription_slot = current_slot + subscription_slot3 - 1;
+
+        let wait_slots = subnet_service
+            .beacon_chain
+            .slot_clock
+            .duration_to_slot(subscription_slot)
+            .unwrap()
+            .as_millis() as u64
+            / SLOT_DURATION_MILLIS;
+
+        let no_events = dbg!(get_events(&mut subnet_service, None, wait_slots as u32).await);
+
+        assert_eq!(no_events, []);
+
+        let third_subscribe_event = get_events(&mut subnet_service, None, 2).await;
+
+        if !subnet_service.is_subscribed(&Subnet::Attestation(subnet_id1)) {
+            assert_eq!(
+                [expected_subscription, expected_unsubscription],
+                third_subscribe_event[..]
+            );
+        }
     }
 
     #[tokio::test]

beacon_node/network/src/sync/block_lookups/single_block_lookup.rs

@@ -171,7 +171,10 @@ impl<T: BeaconChainTypes> SingleBlockLookup<T> {
         self.awaiting_parent.is_some()
             || self.block_request_state.state.is_awaiting_event()
             || match &self.component_requests {
-                ComponentRequests::WaitingForBlock => true,
+                // If components are waiting for the block request to complete, here we should
+                // check if the`block_request_state.state.is_awaiting_event(). However we already
+                // checked that above, so `WaitingForBlock => false` is equivalent.
+                ComponentRequests::WaitingForBlock => false,
                 ComponentRequests::ActiveBlobRequest(request, _) => {
                     request.state.is_awaiting_event()
                 }

beacon_node/store/src/hot_cold_store.rs

@@ -2484,6 +2484,45 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         Ok(())
     }
 
+    /// Run a compaction pass on the freezer DB to free up space used by deleted states.
+    pub fn compact_freezer(&self) -> Result<(), Error> {
+        let current_schema_columns = vec![
+            DBColumn::BeaconColdStateSummary,
+            DBColumn::BeaconStateSnapshot,
+            DBColumn::BeaconStateDiff,
+            DBColumn::BeaconStateRoots,
+        ];
+
+        // We can remove this once schema V21 has been gone for a while.
+        let previous_schema_columns = vec![
+            DBColumn::BeaconState,
+            DBColumn::BeaconStateSummary,
+            DBColumn::BeaconBlockRootsChunked,
+            DBColumn::BeaconStateRootsChunked,
+            DBColumn::BeaconRestorePoint,
+            DBColumn::BeaconHistoricalRoots,
+            DBColumn::BeaconRandaoMixes,
+            DBColumn::BeaconHistoricalSummaries,
+        ];
+        let mut columns = current_schema_columns;
+        columns.extend(previous_schema_columns);
+
+        for column in columns {
+            info!(
+                self.log,
+                "Starting compaction";
+                "column" => ?column
+            );
+            self.cold_db.compact_column(column)?;
+            info!(
+                self.log,
+                "Finishing compaction";
+                "column" => ?column
+            );
+        }
+        Ok(())
+    }
+
     /// Return `true` if compaction on finalization/pruning is enabled.
     pub fn compact_on_prune(&self) -> bool {
         self.config.compact_on_prune
@@ -2875,6 +2914,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         //
         // We can remove this once schema V21 has been gone for a while.
         let previous_schema_columns = vec![
+            DBColumn::BeaconState,
             DBColumn::BeaconStateSummary,
             DBColumn::BeaconBlockRootsChunked,
             DBColumn::BeaconStateRootsChunked,
@@ -2916,7 +2956,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold>
         self.cold_db.do_atomically(cold_ops)?;
 
         // In order to reclaim space, we need to compact the freezer DB as well.
-        self.cold_db.compact()?;
+        self.compact_freezer()?;
 
         Ok(())
     }