diff --git a/beacon_node/network/src/sync/custody_backfill_sync/mod.rs b/beacon_node/network/src/sync/custody_backfill_sync/mod.rs
index 7daf2a47d3..c85610613c 100644
--- a/beacon_node/network/src/sync/custody_backfill_sync/mod.rs
+++ b/beacon_node/network/src/sync/custody_backfill_sync/mod.rs
@@ -6,7 +6,7 @@ use std::{
 
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use lighthouse_network::{
-    NetworkGlobals, PeerId,
+    NetworkGlobals, PeerAction, PeerId,
     service::api_types::{CustodyBackFillBatchRequestId, CustodyBackfillBatchId},
     types::CustodyBackFillState,
 };
@@ -601,13 +601,21 @@ impl<T: BeaconChainTypes> CustodyBackFillSync<T> {
                         exceeded_retries: _,
                     } = coupling_error
                 {
-                    let unique_faulty_peers: HashSet<PeerId> =
-                        faulty_peers.iter().map(|(_, peer)| *peer).collect();
-                    for faulty_peer in unique_faulty_peers {
+                    let mut failed_peers = HashSet::new();
+                    for (column_index, faulty_peer) in faulty_peers {
                         debug!(
                             ?error,
+                            ?column_index,
                             ?faulty_peer,
-                            "Custody backfill sync: peer failed to serve columns (not penalizing)"
+                            "Custody backfill sync: peer failed to serve column"
+                        );
+                        failed_peers.insert(faulty_peer);
+                    }
+                    for peer in failed_peers {
+                        network.report_peer(
+                            peer,
+                            PeerAction::LowToleranceError,
+                            "Peer failed to serve column",
                         );
                     }
                 }