gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-04-19 05:48:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Register processor queue length as histogram (#6012)

Browse Source 
* Register processor queue length as histogram

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into processor-queue-histogram
This commit is contained in:
Lion - dapplion
2024-09-09 14:18:30 +02:00
committed by GitHub
parent 815567a91a
commit 51091a40fa
7 changed files with 319 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
beacon_node/network/src/network_beacon_processor/tests.rs
View File

@@ -467,10 +467,11 @@ impl TestRig {
///
/// Given the described logic, `expected` must not contain `WORKER_FREED` or `NOTHING_TO_DO`
/// events.
pub async fn assert_event_journal_contains_ordered(&mut self, expected: &[&str]) {
assert!(expected
pub async fn assert_event_journal_contains_ordered(&mut self, expected: &[WorkType]) {
let expected = expected
.iter()
.all(|ev| ev != &WORKER_FREED && ev != &NOTHING_TO_DO));
.map(|ev| ev.into())
.collect::<Vec<&'static str>>();
let mut events = Vec::with_capacity(expected.len());
let mut worker_freed_remaining = expected.len();
@@ -517,6 +518,18 @@ impl TestRig {
.await
}
pub async fn assert_event_journal_completes(&mut self, expected: &[WorkType]) {
self.assert_event_journal(
&expected
.iter()
.map(|ev| Into::<&'static str>::into(ev))
.chain(std::iter::once(WORKER_FREED))
.chain(std::iter::once(NOTHING_TO_DO))
.collect::<Vec<_>>(),
)
.await
}
/// Assert that the `BeaconProcessor` event journal is as `expected`.
///
/// ## Note
@@ -587,13 +600,13 @@ async fn import_gossip_block_acceptably_early() {
rig.enqueue_gossip_block();
rig.assert_event_journal(&[GOSSIP_BLOCK, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipBlock])
.await;
let num_blobs = rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0);
for i in 0..num_blobs {
rig.enqueue_gossip_blob(i);
rig.assert_event_journal(&[GOSSIP_BLOBS_SIDECAR, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipBlobSidecar])
.await;
}
@@ -611,7 +624,7 @@ async fn import_gossip_block_acceptably_early() {
"block not yet imported"
);
rig.assert_event_journal(&[DELAYED_IMPORT_BLOCK, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::DelayedImportBlock])
.await;
assert_eq!(
@@ -644,7 +657,7 @@ async fn import_gossip_block_unacceptably_early() {
rig.enqueue_gossip_block();
rig.assert_event_journal(&[GOSSIP_BLOCK, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipBlock])
.await;
// Waiting for 5 seconds is a bit arbitrary, however it *should* be long enough to ensure the
@@ -670,7 +683,7 @@ async fn import_gossip_block_at_current_slot() {
rig.enqueue_gossip_block();
rig.assert_event_journal(&[GOSSIP_BLOCK, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipBlock])
.await;
let num_blobs = rig
@@ -682,7 +695,7 @@ async fn import_gossip_block_at_current_slot() {
for i in 0..num_blobs {
rig.enqueue_gossip_blob(i);
rig.assert_event_journal(&[GOSSIP_BLOBS_SIDECAR, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipBlobSidecar])
.await;
}
@@ -702,7 +715,7 @@ async fn import_gossip_attestation() {
rig.enqueue_unaggregated_attestation();
rig.assert_event_journal(&[GOSSIP_ATTESTATION, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAttestation])
.await;
assert_eq!(
@@ -728,7 +741,7 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod
rig.enqueue_next_block_unaggregated_attestation();
rig.assert_event_journal(&[GOSSIP_ATTESTATION, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAttestation])
.await;
assert_eq!(
@@ -747,23 +760,23 @@ async fn attestation_to_unknown_block_processed(import_method: BlockImportMethod
match import_method {
BlockImportMethod::Gossip => {
rig.enqueue_gossip_block();
events.push(GOSSIP_BLOCK);
events.push(WorkType::GossipBlock);
for i in 0..num_blobs {
rig.enqueue_gossip_blob(i);
events.push(GOSSIP_BLOBS_SIDECAR);
events.push(WorkType::GossipBlobSidecar);
}
}
BlockImportMethod::Rpc => {
rig.enqueue_rpc_block();
events.push(RPC_BLOCK);
events.push(WorkType::RpcBlock);
if num_blobs > 0 {
rig.enqueue_single_lookup_rpc_blobs();
events.push(RPC_BLOBS);
events.push(WorkType::RpcBlobs);
}
}
};
events.push(UNKNOWN_BLOCK_ATTESTATION);
events.push(WorkType::UnknownBlockAttestation);
rig.assert_event_journal_contains_ordered(&events).await;
@@ -809,7 +822,7 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod
rig.enqueue_next_block_aggregated_attestation();
rig.assert_event_journal(&[GOSSIP_AGGREGATE, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAggregate])
.await;
assert_eq!(
@@ -828,23 +841,23 @@ async fn aggregate_attestation_to_unknown_block(import_method: BlockImportMethod
match import_method {
BlockImportMethod::Gossip => {
rig.enqueue_gossip_block();
events.push(GOSSIP_BLOCK);
events.push(WorkType::GossipBlock);
for i in 0..num_blobs {
rig.enqueue_gossip_blob(i);
events.push(GOSSIP_BLOBS_SIDECAR);
events.push(WorkType::GossipBlobSidecar);
}
}
BlockImportMethod::Rpc => {
rig.enqueue_rpc_block();
events.push(RPC_BLOCK);
events.push(WorkType::RpcBlock);
if num_blobs > 0 {
rig.enqueue_single_lookup_rpc_blobs();
events.push(RPC_BLOBS);
events.push(WorkType::RpcBlobs);
}
}
};
events.push(UNKNOWN_BLOCK_AGGREGATE);
events.push(WorkType::UnknownBlockAggregate);
rig.assert_event_journal_contains_ordered(&events).await;
@@ -887,7 +900,7 @@ async fn requeue_unknown_block_gossip_attestation_without_import() {
rig.enqueue_next_block_unaggregated_attestation();
rig.assert_event_journal(&[GOSSIP_ATTESTATION, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAttestation])
.await;
assert_eq!(
@@ -899,7 +912,11 @@ async fn requeue_unknown_block_gossip_attestation_without_import() {
// Ensure that the attestation is received back but not imported.
rig.assert_event_journal_with_timeout(
&[UNKNOWN_BLOCK_ATTESTATION, WORKER_FREED, NOTHING_TO_DO],
&[
WorkType::UnknownBlockAttestation.into(),
WORKER_FREED,
NOTHING_TO_DO,
],
Duration::from_secs(1) + QUEUED_ATTESTATION_DELAY,
)
.await;
@@ -923,7 +940,7 @@ async fn requeue_unknown_block_gossip_aggregated_attestation_without_import() {
rig.enqueue_next_block_aggregated_attestation();
rig.assert_event_journal(&[GOSSIP_AGGREGATE, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAggregate])
.await;
assert_eq!(
@@ -935,7 +952,11 @@ async fn requeue_unknown_block_gossip_aggregated_attestation_without_import() {
// Ensure that the attestation is received back but not imported.
rig.assert_event_journal_with_timeout(
&[UNKNOWN_BLOCK_AGGREGATE, WORKER_FREED, NOTHING_TO_DO],
&[
WorkType::UnknownBlockAggregate.into(),
WORKER_FREED,
NOTHING_TO_DO,
],
Duration::from_secs(1) + QUEUED_ATTESTATION_DELAY,
)
.await;
@@ -961,7 +982,7 @@ async fn import_misc_gossip_ops() {
rig.enqueue_gossip_attester_slashing();
rig.assert_event_journal(&[GOSSIP_ATTESTER_SLASHING, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipAttesterSlashing])
.await;
assert_eq!(
@@ -978,7 +999,7 @@ async fn import_misc_gossip_ops() {
rig.enqueue_gossip_proposer_slashing();
rig.assert_event_journal(&[GOSSIP_PROPOSER_SLASHING, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipProposerSlashing])
.await;
assert_eq!(
@@ -995,7 +1016,7 @@ async fn import_misc_gossip_ops() {
rig.enqueue_gossip_voluntary_exit();
rig.assert_event_journal(&[GOSSIP_VOLUNTARY_EXIT, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::GossipVoluntaryExit])
.await;
assert_eq!(
@@ -1014,12 +1035,12 @@ async fn test_rpc_block_reprocessing() {
// Insert the next block into the duplicate cache manually
let handle = rig.duplicate_cache.check_and_insert(next_block_root);
rig.enqueue_single_lookup_rpc_block();
rig.assert_event_journal(&[RPC_BLOCK, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::RpcBlock])
.await;
rig.enqueue_single_lookup_rpc_blobs();
if rig.next_blobs.as_ref().map(|b| b.len()).unwrap_or(0) > 0 {
rig.assert_event_journal(&[RPC_BLOBS, WORKER_FREED, NOTHING_TO_DO])
rig.assert_event_journal_completes(&[WorkType::RpcBlobs])
.await;
}
@@ -1033,7 +1054,7 @@ async fn test_rpc_block_reprocessing() {
// the specified delay.
tokio::time::sleep(QUEUED_RPC_BLOCK_DELAY).await;
rig.assert_event_journal(&[RPC_BLOCK]).await;
rig.assert_event_journal(&[WorkType::RpcBlock.into()]).await;
// Add an extra delay for block processing
tokio::time::sleep(Duration::from_millis(10)).await;
// head should update to next block now since the duplicate
@@ -1055,7 +1076,11 @@ async fn test_backfill_sync_processing() {
rig.assert_no_events_for(Duration::from_millis(100)).await;
// A new batch should be processed within a slot.
rig.assert_event_journal_with_timeout(
&[CHAIN_SEGMENT_BACKFILL, WORKER_FREED, NOTHING_TO_DO],
&[
WorkType::ChainSegmentBackfill.into(),
WORKER_FREED,
NOTHING_TO_DO,
],
rig.chain.slot_clock.slot_duration(),
)
.await;
@@ -1075,9 +1100,9 @@ async fn test_backfill_sync_processing_rate_limiting_disabled() {
// ensure all batches are processed
rig.assert_event_journal_with_timeout(
&[
CHAIN_SEGMENT_BACKFILL,
CHAIN_SEGMENT_BACKFILL,
CHAIN_SEGMENT_BACKFILL,
WorkType::ChainSegmentBackfill.into(),
WorkType::ChainSegmentBackfill.into(),
WorkType::ChainSegmentBackfill.into(),
],
Duration::from_millis(100),
)

2
beacon_node/network/src/router.rs
View File

@@ -710,7 +710,7 @@ impl<T: BeaconChainTypes> Router<T> {
if let Err(e) = result {
let work_type = match &e {
mpsc::error::TrySendError::Closed(work) | mpsc::error::TrySendError::Full(work) => {
work.work_type()
work.work_type_str()
}
};

14
beacon_node/network/src/sync/block_lookups/tests.rs
View File

@@ -1032,17 +1032,17 @@ impl TestRig {
match response_type {
ResponseType::Block => self
.pop_received_processor_event(|ev| {
(ev.work_type() == beacon_processor::RPC_BLOCK).then_some(())
(ev.work_type() == beacon_processor::WorkType::RpcBlock).then_some(())
})
.unwrap_or_else(|e| panic!("Expected block work event: {e}")),
ResponseType::Blob => self
.pop_received_processor_event(|ev| {
(ev.work_type() == beacon_processor::RPC_BLOBS).then_some(())
(ev.work_type() == beacon_processor::WorkType::RpcBlobs).then_some(())
})
.unwrap_or_else(|e| panic!("Expected blobs work event: {e}")),
ResponseType::CustodyColumn => self
.pop_received_processor_event(|ev| {
(ev.work_type() == beacon_processor::RPC_CUSTODY_COLUMN).then_some(())
(ev.work_type() == beacon_processor::WorkType::RpcCustodyColumn).then_some(())
})
.unwrap_or_else(|e| panic!("Expected column work event: {e}")),
}
@@ -1050,7 +1050,7 @@ impl TestRig {
fn expect_rpc_custody_column_work_event(&mut self) {
self.pop_received_processor_event(|ev| {
if ev.work_type() == beacon_processor::RPC_CUSTODY_COLUMN {
if ev.work_type() == beacon_processor::WorkType::RpcCustodyColumn {
Some(())
} else {
None
@@ -1061,7 +1061,7 @@ impl TestRig {
fn expect_rpc_sample_verify_work_event(&mut self) {
self.pop_received_processor_event(|ev| {
if ev.work_type() == beacon_processor::RPC_VERIFY_DATA_COLUMNS {
if ev.work_type() == beacon_processor::WorkType::RpcVerifyDataColumn {
Some(())
} else {
None
@@ -1072,7 +1072,7 @@ impl TestRig {
fn expect_sampling_result_work(&mut self) {
self.pop_received_processor_event(|ev| {
if ev.work_type() == beacon_processor::SAMPLING_RESULT {
if ev.work_type() == beacon_processor::WorkType::SamplingResult {
Some(())
} else {
None
@@ -1103,7 +1103,7 @@ impl TestRig {
match self.beacon_processor_rx.try_recv() {
Ok(work) => {
// Parent chain sends blocks one by one
assert_eq!(work.work_type(), beacon_processor::RPC_BLOCK);
assert_eq!(work.work_type(), beacon_processor::WorkType::RpcBlock);
}
other => panic!(
"Expected rpc_block from chain segment process, found {:?}",

7
beacon_node/network/src/sync/range_sync/range.rs
View File

@@ -652,7 +652,10 @@ mod tests {
fn expect_empty_processor(&mut self) {
match self.beacon_processor_rx.try_recv() {
Ok(work) => {
panic!("Expected empty processor. Instead got {}", work.work_type());
panic!(
"Expected empty processor. Instead got {}",
work.work_type_str()
);
}
Err(e) => match e {
mpsc::error::TryRecvError::Empty => {}
@@ -665,7 +668,7 @@ mod tests {
fn expect_chain_segment(&mut self) {
match self.beacon_processor_rx.try_recv() {
Ok(work) => {
assert_eq!(work.work_type(), beacon_processor::CHAIN_SEGMENT);
assert_eq!(work.work_type(), beacon_processor::WorkType::ChainSegment);
}
other => panic!("Expected chain segment process, found {:?}", other),
}