From 59e45fe349f091760737cfee14b49e35186a62b9 Mon Sep 17 00:00:00 2001
From: Paul Hauner <paul@paulhauner.com>
Date: Tue, 21 Mar 2023 05:15:00 +0000
Subject: [PATCH] Reduce verbosity of reprocess queue logs (#4101)

## Issue Addressed

NA

## Proposed Changes

Replaces #4058 to attempt to reduce `ERRO Failed to send scheduled attestation` spam and provide more information for diagnosis. With this PR we achieve:

- When dequeuing attestations after a block is received, send only one log which reports `n` failures (rather than `n` logs reporting `n` failures).
- Make a distinction in logs between two separate attestation dequeuing events.
- Add more information to both log events to help assist with troubleshooting.

## Additional Info

NA
---
 .../work_reprocessing_queue.rs                | 26 +++++++++++++++----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs
index 8c568a7eef..21fc2b6416 100644
--- a/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs
+++ b/beacon_node/network/src/beacon_processor/work_reprocessing_queue.rs
@@ -573,6 +573,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
             }) => {
                 // Unqueue the attestations we have for this root, if any.
                 if let Some(queued_ids) = self.awaiting_attestations_per_root.remove(&block_root) {
+                    let mut sent_count = 0;
+                    let mut failed_to_send_count = 0;
+
                     for id in queued_ids {
                         metrics::inc_counter(
                             &metrics::BEACON_PROCESSOR_REPROCESSING_QUEUE_MATCHED_ATTESTATIONS,
@@ -597,10 +600,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
 
                             // Send the work.
                             if self.ready_work_tx.try_send(work).is_err() {
-                                error!(
-                                    log,
-                                    "Failed to send scheduled attestation";
-                                );
+                                failed_to_send_count += 1;
+                            } else {
+                                sent_count += 1;
                             }
                         } else {
                             // There is a mismatch between the attestation ids registered for this
@@ -613,6 +615,18 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                             );
                         }
                     }
+
+                    if failed_to_send_count > 0 {
+                        error!(
+                            log,
+                            "Ignored scheduled attestation(s) for block";
+                            "hint" => "system may be overloaded",
+                            "parent_root" => ?parent_root,
+                            "block_root" => ?block_root,
+                            "failed_count" => failed_to_send_count,
+                            "sent_count" => sent_count,
+                        );
+                    }
                 }
                 // Unqueue the light client optimistic updates we have for this root, if any.
                 if let Some(queued_lc_id) = self
@@ -727,7 +741,9 @@ impl<T: BeaconChainTypes> ReprocessQueue<T> {
                     if self.ready_work_tx.try_send(work).is_err() {
                         error!(
                             log,
-                            "Failed to send scheduled attestation";
+                            "Ignored scheduled attestation";
+                            "hint" => "system may be overloaded",
+                            "beacon_block_root" => ?root
                         );
                     }