Ensure custody backfill sync couples all responses before importing (#8339)

Custody backfill sync has a bug when we request columns from more than one peer per batch. The fix here ensures we wait for all requests to be completed before performing verification and importing the responses.

I've also added an endpoint `lighthouse/custody/backfill` that resets a nodes earliest available data column to the current epoch so that custody backfill can be triggered. This endpoint is needed to rescue any nodes that may have missing columns due to the custody backfill sync bug without requiring a full re-sync.


  


Co-Authored-By: Eitan Seri- Levi <eserilev@gmail.com>

Co-Authored-By: Eitan Seri-Levi <eserilev@ucsc.edu>

Co-Authored-By: Jimmy Chen <jchen.tc@gmail.com>

Co-Authored-By: Michael Sproul <michaelsproul@users.noreply.github.com>

beacon_node/beacon_chain/src/custody_context.rs

@@ -120,9 +120,7 @@ impl ValidatorRegistrations {
             let effective_epoch =
                 (current_slot + effective_delay_slots).epoch(E::slots_per_epoch()) + 1;
             self.epoch_validator_custody_requirements
-                .entry(effective_epoch)
-                .and_modify(|old_custody| *old_custody = validator_custody_requirement)
-                .or_insert(validator_custody_requirement);
+                .insert(effective_epoch, validator_custody_requirement);
             Some((effective_epoch, validator_custody_requirement))
         } else {
             None
@@ -154,11 +152,25 @@ impl ValidatorRegistrations {
                 });
 
             self.epoch_validator_custody_requirements
-                .entry(effective_epoch)
-                .and_modify(|old_custody| *old_custody = latest_validator_custody)
-                .or_insert(latest_validator_custody);
+                .insert(effective_epoch, latest_validator_custody);
         }
     }
+
+    /// Updates the `epoch -> cgc` map by pruning records before `effective_epoch`
+    /// while setting the `cgc` at `effective_epoch` to the latest validator custody requirement.
+    ///
+    /// This is used to restart custody backfill sync at `effective_epoch`
+    pub fn reset_validator_custody_requirements(&mut self, effective_epoch: Epoch) {
+        if let Some(latest_validator_custody_requirements) =
+            self.latest_validator_custody_requirement()
+        {
+            self.epoch_validator_custody_requirements
+                .retain(|&epoch, _| epoch >= effective_epoch);
+
+            self.epoch_validator_custody_requirements
+                .insert(effective_epoch, latest_validator_custody_requirements);
+        };
+    }
 }
 
 /// Given the `validator_custody_units`, return the custody requirement based on
@@ -535,6 +547,14 @@ impl<E: EthSpec> CustodyContext<E> {
             .write()
             .backfill_validator_custody_requirements(effective_epoch, expected_cgc);
     }
+
+    /// The node is attempting to restart custody backfill. Update the internal records so that
+    /// custody backfill can start backfilling at `effective_epoch`.
+    pub fn reset_validator_custody_requirements(&self, effective_epoch: Epoch) {
+        self.validator_registrations
+            .write()
+            .reset_validator_custody_requirements(effective_epoch);
+    }
 }
 
 /// Indicates that the custody group count (CGC) has increased.
@@ -1491,4 +1511,53 @@ mod tests {
             );
         }
     }
+
+    #[test]
+    fn reset_validator_custody_requirements() {
+        let spec = E::default_spec();
+        let minimum_cgc = 4u64;
+        let initial_cgc = 8u64;
+        let mid_cgc = 16u64;
+        let final_cgc = 32u64;
+
+        // Setup: Node restart after multiple validator registrations causing CGC increases
+        let head_epoch = Epoch::new(20);
+        let epoch_and_cgc_tuples = vec![
+            (Epoch::new(0), initial_cgc),
+            (Epoch::new(10), mid_cgc),
+            (head_epoch, final_cgc),
+        ];
+        let custody_context = setup_custody_context(&spec, head_epoch, epoch_and_cgc_tuples);
+
+        // Backfill from epoch 20 to 9
+        complete_backfill_for_epochs(&custody_context, Epoch::new(20), Epoch::new(9), final_cgc);
+
+        // Reset validator custody requirements to the latest cgc requirements at `head_epoch` up to the boundary epoch
+        custody_context.reset_validator_custody_requirements(head_epoch);
+
+        // Verify epochs 0 - 19 return the minimum cgc requirement because of the validator custody requirement reset
+        for epoch in 0..=19 {
+            assert_eq!(
+                custody_context.custody_group_count_at_epoch(Epoch::new(epoch), &spec),
+                minimum_cgc,
+            );
+        }
+
+        // Verify epoch 20 returns a CGC of 32
+        assert_eq!(
+            custody_context.custody_group_count_at_epoch(head_epoch, &spec),
+            final_cgc
+        );
+
+        // Rerun Backfill to epoch 20
+        complete_backfill_for_epochs(&custody_context, Epoch::new(20), Epoch::new(0), final_cgc);
+
+        // Verify epochs 0 - 20 return the final cgc requirements
+        for epoch in 0..=20 {
+            assert_eq!(
+                custody_context.custody_group_count_at_epoch(Epoch::new(epoch), &spec),
+                final_cgc,
+            );
+        }
+    }
 }

beacon_node/beacon_chain/src/historical_data_columns.rs

@@ -89,11 +89,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         .get_data_column(&block_root, &data_column.index)?
                         .is_some()
                     {
-                        debug!(
-                            block_root = ?block_root,
-                            column_index = data_column.index,
-                            "Skipping data column import as identical data column exists"
-                        );
                         continue;
                     }
                     if block_root != data_column.block_root() {