gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-05-30 12:47:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Mallory - Single commit

Browse Source
This commit is contained in:
Age Manning
2025-03-25 16:53:10 +11:00
parent b8178515cd
commit e2acce9468
118 changed files with 4753 additions and 3938 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
common/eth2/src/lib.rs
View File

@@ -1336,7 +1336,7 @@ impl BeaconNodeHttpClient {
}
/// Path for `v1/beacon/blob_sidecars/{block_id}`
pub fn get_blobs_path(&self, block_id: BlockId) -> Result<Url, Error> {
pub fn get_blob_sidecars_path(&self, block_id: BlockId) -> Result<Url, Error> {
let mut path = self.eth_path(V1)?;
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
@@ -1346,6 +1346,17 @@ impl BeaconNodeHttpClient {
Ok(path)
}
/// Path for `v1/beacon/blobs/{blob_id}`
pub fn get_blobs_path(&self, block_id: BlockId) -> Result<Url, Error> {
let mut path = self.eth_path(V1)?;
path.path_segments_mut()
.map_err(|()| Error::InvalidUrl(self.server.clone()))?
.push("beacon")
.push("blobs")
.push(&block_id.to_string());
Ok(path)
}
/// Path for `v1/beacon/blinded_blocks/{block_id}`
pub fn get_beacon_blinded_blocks_path(&self, block_id: BlockId) -> Result<Url, Error> {
let mut path = self.eth_path(V1)?;
@@ -1374,13 +1385,13 @@ impl BeaconNodeHttpClient {
/// `GET v1/beacon/blob_sidecars/{block_id}`
///
/// Returns `Ok(None)` on a 404 error.
pub async fn get_blobs<E: EthSpec>(
pub async fn get_blob_sidecars<E: EthSpec>(
&self,
block_id: BlockId,
indices: Option<&[u64]>,
spec: &ChainSpec,
) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<BlobSidecarList<E>>>, Error> {
let mut path = self.get_blobs_path(block_id)?;
let mut path = self.get_blob_sidecars_path(block_id)?;
if let Some(indices) = indices {
let indices_string = indices
.iter()
@@ -1400,6 +1411,31 @@ impl BeaconNodeHttpClient {
.map(|opt| opt.map(BeaconResponse::ForkVersioned))
}
/// `GET v1/beacon/blobs/{block_id}`
///
/// Returns `Ok(None)` on a 404 error.
pub async fn get_blobs<E: EthSpec>(
&self,
block_id: BlockId,
versioned_hashes: Option<&[Hash256]>,
) -> Result<Option<ExecutionOptimisticFinalizedBeaconResponse<Vec<BlobWrapper<E>>>>, Error>
{
let mut path = self.get_blobs_path(block_id)?;
if let Some(hashes) = versioned_hashes {
let hashes_string = hashes
.iter()
.map(|hash| hash.to_string())
.collect::<Vec<_>>()
.join(",");
path.query_pairs_mut()
.append_pair("versioned_hashes", &hashes_string);
}
self.get_opt(path)
.await
.map(|opt| opt.map(BeaconResponse::Unversioned))
}
/// `GET v1/beacon/blinded_blocks/{block_id}`
///
/// Returns `Ok(None)` on a 404 error.

15
common/eth2/src/types.rs
View File

@@ -716,6 +716,13 @@ pub struct BlobIndicesQuery {
pub indices: Option<Vec<u64>>,
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BlobsVersionedHashesQuery {
#[serde(default, deserialize_with = "option_query_vec")]
pub versioned_hashes: Option<Vec<Hash256>>,
}
#[derive(Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DataColumnIndicesQuery {
@@ -2317,6 +2324,14 @@ pub struct StandardAttestationRewards {
pub total_rewards: Vec<TotalAttestationRewards>,
}
#[derive(Debug, Clone, Serialize, Deserialize, Encode, Decode)]
#[serde(bound = "E: EthSpec")]
#[serde(transparent)]
pub struct BlobWrapper<E: EthSpec> {
#[serde(with = "ssz_types::serde_utils::hex_fixed_vec")]
pub blob: Blob<E>,
}
#[cfg(test)]
mod test {
use std::fmt::Debug;

33
common/eth2_network_config/built_in_network_configs/holesky/config.yaml
View File

@@ -38,7 +38,7 @@ ELECTRA_FORK_VERSION: 0x06017000
ELECTRA_FORK_EPOCH: 115968
# Fulu
FULU_FORK_VERSION: 0x07017000
FULU_FORK_EPOCH: 18446744073709551615
FULU_FORK_EPOCH: 165120
# Gloas
GLOAS_FORK_VERSION: 0x08017000
GLOAS_FORK_EPOCH: 18446744073709551615
@@ -47,6 +47,8 @@ GLOAS_FORK_EPOCH: 18446744073709551615
# ---------------------------------------------------------------
# 12 seconds
SECONDS_PER_SLOT: 12
# 1200 milliseconds
SLOT_DURATION_MS: 12000
# 14 (estimate from Eth1 mainnet)
SECONDS_PER_ETH1_BLOCK: 14
# 2**8 (= 256) epochs ~27 hours
@@ -55,6 +57,18 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
SHARD_COMMITTEE_PERIOD: 256
# 2**11 (= 2,048) Eth1 blocks ~8 hours
ETH1_FOLLOW_DISTANCE: 2048
# 1667 basis points, ~17% of SLOT_DURATION_MS
PROPOSER_REORG_CUTOFF_BPS: 1667
# 3333 basis points, ~33% of SLOT_DURATION_MS
ATTESTATION_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
AGGREGATE_DUE_BPS: 6667
# Altair
# 3333 basis points, ~33% of SLOT_DURATION_MS
SYNC_MESSAGE_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
CONTRIBUTION_DUE_BPS: 6667
# Validator cycle
# ---------------------------------------------------------------
@@ -141,13 +155,30 @@ MAX_BLOBS_PER_BLOCK_ELECTRA: 9
MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152
# Fulu
# 2**7 (= 128) groups
NUMBER_OF_CUSTODY_GROUPS: 128
# 2**7 (= 128) subnets
DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
# MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS (= 128 * 128) sidecars
MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
# 2**3 (= 8) samples
SAMPLES_PER_SLOT: 8
# 2**2 (= 4) sidecars
CUSTODY_REQUIREMENT: 4
# 2**3 (= 8) sidecars
VALIDATOR_CUSTODY_REQUIREMENT: 8
# 2**5 * 10**9 (= 32,000,000,000) Gwei
BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
# 2**12 (= 4,096) epochs
MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
# Blob Scheduling
# ---------------------------------------------------------------
BLOB_SCHEDULE:
- EPOCH: 166400
MAX_BLOBS_PER_BLOCK: 15
- EPOCH: 167936
MAX_BLOBS_PER_BLOCK: 21
# Gloas

34
common/eth2_network_config/built_in_network_configs/hoodi/config.yaml
View File

@@ -42,7 +42,7 @@ ELECTRA_FORK_EPOCH: 2048
# Fulu
FULU_FORK_VERSION: 0x70000910
FULU_FORK_EPOCH: 18446744073709551615
FULU_FORK_EPOCH: 50688
# Gloas
GLOAS_FORK_VERSION: 0x80000910
@@ -53,6 +53,8 @@ GLOAS_FORK_EPOCH: 18446744073709551615
# ---------------------------------------------------------------
# 12 seconds
SECONDS_PER_SLOT: 12
# 12000 milliseconds
SLOT_DURATION_MS: 12000
# 14 (estimate from Eth1 mainnet)
SECONDS_PER_ETH1_BLOCK: 12
# 2**8 (= 256) epochs ~27 hours
@@ -61,6 +63,18 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
SHARD_COMMITTEE_PERIOD: 256
# 2**11 (= 2,048) Eth1 blocks ~8 hours
ETH1_FOLLOW_DISTANCE: 2048
# 1667 basis points, ~17% of SLOT_DURATION_MS
PROPOSER_REORG_CUTOFF_BPS: 1667
# 3333 basis points, ~33% of SLOT_DURATION_MS
ATTESTATION_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
AGGREGATE_DUE_BPS: 6667
# Altair
# 3333 basis points, ~33% of SLOT_DURATION_MS
SYNC_MESSAGE_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
CONTRIBUTION_DUE_BPS: 6667
# Validator cycle
# ---------------------------------------------------------------
@@ -154,15 +168,33 @@ WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256
WHISK_PROPOSER_SELECTION_GAP: 2
# Fulu
# 2**7 (= 128) groups
NUMBER_OF_CUSTODY_GROUPS: 128
# 2**7 (= 128) subnets
DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
# MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS (= 128 * 128) sidecars
MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
# 2**3 (= 8) samples
SAMPLES_PER_SLOT: 8
# 2**2 (= 4) sidecars
CUSTODY_REQUIREMENT: 4
# 2**3 (= 8) sidecars
VALIDATOR_CUSTODY_REQUIREMENT: 8
# 2**5 * 10**9 (= 32,000,000,000) Gwei
BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
# 2**12 (= 4,096) epochs
MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
# Blob Scheduling
# ---------------------------------------------------------------
BLOB_SCHEDULE:
- EPOCH: 52480
MAX_BLOBS_PER_BLOCK: 15
- EPOCH: 54016
MAX_BLOBS_PER_BLOCK: 21
# Gloas
# EIP7732

34
common/eth2_network_config/built_in_network_configs/sepolia/config.yaml
View File

@@ -42,7 +42,7 @@ ELECTRA_FORK_EPOCH: 222464
# Fulu
FULU_FORK_VERSION: 0x90000075
FULU_FORK_EPOCH: 18446744073709551615
FULU_FORK_EPOCH: 272640
# Gloas
GLOAS_FORK_VERSION: 0x90000076
@@ -52,6 +52,8 @@ GLOAS_FORK_EPOCH: 18446744073709551615
# ---------------------------------------------------------------
# 12 seconds
SECONDS_PER_SLOT: 12
# 12000 milliseconds
SLOT_DURATION_MS: 12000
# 14 (estimate from Eth1 mainnet)
SECONDS_PER_ETH1_BLOCK: 14
# 2**8 (= 256) epochs ~27 hours
@@ -60,6 +62,18 @@ MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256
SHARD_COMMITTEE_PERIOD: 256
# 2**11 (= 2,048) Eth1 blocks ~8 hours
ETH1_FOLLOW_DISTANCE: 2048
# 1667 basis points, ~17% of SLOT_DURATION_MS
PROPOSER_REORG_CUTOFF_BPS: 1667
# 3333 basis points, ~33% of SLOT_DURATION_MS
ATTESTATION_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
AGGREGATE_DUE_BPS: 6667
# Altair
# 3333 basis points, ~33% of SLOT_DURATION_MS
SYNC_MESSAGE_DUE_BPS: 3333
# 6667 basis points, ~67% of SLOT_DURATION_MS
CONTRIBUTION_DUE_BPS: 6667
# Validator cycle
@@ -147,13 +161,31 @@ MAX_BLOBS_PER_BLOCK_ELECTRA: 9
MAX_REQUEST_BLOB_SIDECARS_ELECTRA: 1152
# Fulu
# 2**7 (= 128) groups
NUMBER_OF_CUSTODY_GROUPS: 128
# 2**7 (= 128) subnets
DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
# MAX_REQUEST_BLOCKS_DENEB * NUMBER_OF_COLUMNS (= 128 * 128) sidecars
MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
# 2**3 (= 8) samples
SAMPLES_PER_SLOT: 8
# 2**2 (= 4) sidecars
CUSTODY_REQUIREMENT: 4
# 2**3 (= 8) sidecars
VALIDATOR_CUSTODY_REQUIREMENT: 8
# 2**5 * 10**9 (= 32,000,000,000) Gwei
BALANCE_PER_ADDITIONAL_CUSTODY_GROUP: 32000000000
# 2**12 (= 4,096) epochs
MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS: 4096
# Blob Scheduling
# ---------------------------------------------------------------
BLOB_SCHEDULE:
- EPOCH: 274176
MAX_BLOBS_PER_BLOCK: 15
- EPOCH: 275712
MAX_BLOBS_PER_BLOCK: 21
# Gloas

2
common/eth2_network_config/src/lib.rs
View File

@@ -244,7 +244,7 @@ impl Eth2NetworkConfig {
}
}
fn get_genesis_state_from_bytes<E: EthSpec>(&self) -> Result<BeaconState<E>, String> {
pub fn get_genesis_state_from_bytes<E: EthSpec>(&self) -> Result<BeaconState<E>, String> {
let spec = self.chain_spec::<E>()?;
self.genesis_state_bytes
.as_ref()

6
common/lighthouse_version/src/lib.rs
View File

@@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!(
// NOTE: using --match instead of --exclude for compatibility with old Git
"--match=thiswillnevermatchlol"
],
prefix = "Lighthouse/v7.1.0-",
fallback = "Lighthouse/v7.1.0"
prefix = "Lighthouse/v8.0.0-rc.1-",
fallback = "Lighthouse/v8.0.0-rc.1"
);
/// Returns the first eight characters of the latest commit hash for this build.
@@ -54,7 +54,7 @@ pub fn version_with_platform() -> String {
///
/// `1.5.1`
pub fn version() -> &'static str {
"7.1.0"
"8.0.0-rc.1"
}
/// Returns the name of the current client running.

114
common/logging/src/tracing_logging_layer.rs
View File

@@ -1,4 +1,5 @@
use crate::utils::is_ascii_control;
use std::collections::HashSet;
use chrono::prelude::*;
use serde_json::{Map, Value};
@@ -261,6 +262,12 @@ fn build_log_json(
let module_field = format!("{}:{}", module_path, line_number);
log_map.insert("module".to_string(), Value::String(module_field));
// Avoid adding duplicate fields; prefer event fields when duplicates exist.
for (key, val) in span_fields {
let parsed_span_val = parse_field(val);
log_map.insert(key.clone(), parsed_span_val);
}
for (key, val) in visitor.fields.clone().into_iter() {
let cleaned_value = if val.starts_with('\"') && val.ends_with('\"') && val.len() >= 2 {
&val[1..val.len() - 1]
@@ -272,11 +279,6 @@ fn build_log_json(
log_map.insert(key, parsed_val);
}
for (key, val) in span_fields {
let parsed_span_val = parse_field(val);
log_map.insert(key.clone(), parsed_span_val);
}
let json_obj = Value::Object(log_map);
let output = format!("{}\n", json_obj);
@@ -299,23 +301,6 @@ fn build_log_text(
let bold_start = "\x1b[1m";
let bold_end = "\x1b[0m";
let mut formatted_spans = String::new();
for (i, (field_name, field_value)) in span_fields.iter().rev().enumerate() {
if use_color {
formatted_spans.push_str(&format!(
"{}{}{}: {}",
bold_start, field_name, bold_end, field_value
));
} else {
formatted_spans.push_str(&format!("{}: {}", field_name, field_value));
}
// Check if this is not the last span.
if i != span_fields.len() - 1 {
formatted_spans.push_str(", ");
}
}
let pad = if plain_level_str.len() < ALIGNED_LEVEL_WIDTH {
" "
} else {
@@ -351,24 +336,26 @@ fn build_log_text(
message_content.clone()
};
let mut formatted_fields = String::new();
for (i, (field_name, field_value)) in visitor.fields.iter().enumerate() {
if i > 0 {
formatted_fields.push_str(", ");
}
if use_color {
formatted_fields.push_str(&format!(
"{}{}{}: {}",
bold_start, field_name, bold_end, field_value
));
} else {
formatted_fields.push_str(&format!("{}: {}", field_name, field_value));
}
// Check if this is the last field and that we are also adding spans.
if i == visitor.fields.len() - 1 && !span_fields.is_empty() {
formatted_fields.push(',');
}
}
// Avoid adding duplicate fields; prefer event fields when duplicates exist.
let mut added_field_names = HashSet::new();
let formatted_fields = visitor
.fields
.iter()
.chain(span_fields.iter())
.filter_map(|(field_name, field_value)| {
if added_field_names.insert(field_name) {
let formatted_field = if use_color {
format!("{}{}{}: {}", bold_start, field_name, bold_end, field_value)
} else {
format!("{}: {}", field_name, field_value)
};
Some(formatted_field)
} else {
None
}
})
.collect::<Vec<_>>()
.join(", ");
let full_message = if !formatted_fields.is_empty() {
format!("{} {}", padded_message, formatted_fields)
@@ -378,14 +365,11 @@ fn build_log_text(
let message = if !location.is_empty() {
format!(
"{} {} {} {} {}\n",
timestamp, level_str, location, full_message, formatted_spans
"{} {} {} {}\n",
timestamp, level_str, location, full_message
)
} else {
format!(
"{} {} {} {}\n",
timestamp, level_str, full_message, formatted_spans
)
format!("{} {} {}\n", timestamp, level_str, full_message)
};
if let Err(e) = writer.write_all(message.as_bytes()) {
@@ -436,7 +420,7 @@ mod tests {
fn test_build_log_text_single_log_field() {
let log_fields = vec![("field_name".to_string(), "field_value".to_string())];
let span_fields = vec![];
let expected = "Jan 1 08:00:00.000 INFO test message field_name: field_value \n";
let expected = "Jan 1 08:00:00.000 INFO test message field_name: field_value\n";
test_build_log_text(log_fields, span_fields, expected);
}
@@ -447,7 +431,7 @@ mod tests {
("field_name2".to_string(), "field_value2".to_string()),
];
let span_fields = vec![];
let expected = "Jan 1 08:00:00.000 INFO test message field_name1: field_value1, field_name2: field_value2 \n";
let expected = "Jan 1 08:00:00.000 INFO test message field_name1: field_value1, field_name2: field_value2\n";
test_build_log_text(log_fields, span_fields, expected);
}
@@ -469,7 +453,7 @@ mod tests {
"span_field_name".to_string(),
"span_field_value".to_string(),
)];
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name: span_field_value\n";
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name: span_field_value\n";
test_build_log_text(log_fields, span_fields, expected);
}
@@ -486,7 +470,7 @@ mod tests {
"span_field_value2".to_string(),
),
];
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name2: span_field_value2, span_field_name1: span_field_value1\n";
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name1: span_field_value1, span_field_name2: span_field_value2\n";
test_build_log_text(log_fields, span_fields, expected);
}
@@ -503,7 +487,35 @@ mod tests {
"span_field_value1-2".to_string(),
),
];
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name1-2: span_field_value1-2, span_field_name1-1: span_field_value1-1\n";
let expected = "Jan 1 08:00:00.000 INFO test message span_field_name1-1: span_field_value1-1, span_field_name1-2: span_field_value1-2\n";
test_build_log_text(log_fields, span_fields, expected);
}
#[test]
fn test_build_log_text_no_duplicate_log_span_fields() {
let log_fields = vec![
("field_name_1".to_string(), "field_value_1".to_string()),
("field_name_2".to_string(), "field_value_2".to_string()),
];
let span_fields = vec![
("field_name_1".to_string(), "field_value_1".to_string()),
("field_name_3".to_string(), "field_value_3".to_string()),
];
let expected = "Jan 1 08:00:00.000 INFO test message field_name_1: field_value_1, field_name_2: field_value_2, field_name_3: field_value_3\n";
test_build_log_text(log_fields, span_fields, expected);
}
#[test]
fn test_build_log_text_duplicate_fields_prefer_log_fields() {
let log_fields = vec![
("field_name_1".to_string(), "field_value_1_log".to_string()),
("field_name_2".to_string(), "field_value_2".to_string()),
];
let span_fields = vec![
("field_name_1".to_string(), "field_value_1_span".to_string()),
("field_name_3".to_string(), "field_value_3".to_string()),
];
let expected = "Jan 1 08:00:00.000 INFO test message field_name_1: field_value_1_log, field_name_2: field_value_2, field_name_3: field_value_3\n";
test_build_log_text(log_fields, span_fields, expected);
}

3
common/network_utils/src/discovery_metrics.rs
View File

@@ -35,8 +35,7 @@ pub static DISCOVERY_SESSIONS: LazyLock<Result<IntGauge>> = LazyLock::new(|| {
});
pub fn scrape_discovery_metrics() {
let metrics =
discv5::metrics::Metrics::from(discv5::Discv5::<discv5::DefaultProtocolId>::raw_metrics());
let metrics = discv5::metrics::Metrics::from(discv5::Discv5::raw_metrics());
set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second);
set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64);
set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64);

2
common/task_executor/Cargo.toml
View File

@@ -8,6 +8,8 @@ edition = { workspace = true }
async-channel = { workspace = true }
futures = { workspace = true }
metrics = { workspace = true }
num_cpus = { workspace = true }
rayon = { workspace = true }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
tracing = { workspace = true }

50
common/task_executor/src/lib.rs
View File

@@ -1,12 +1,15 @@
mod metrics;
mod rayon_pool_provider;
pub mod test_utils;
use futures::channel::mpsc::Sender;
use futures::prelude::*;
use std::sync::Weak;
use std::sync::{Arc, Weak};
use tokio::runtime::{Handle, Runtime};
use tracing::debug;
use crate::rayon_pool_provider::RayonPoolProvider;
pub use crate::rayon_pool_provider::RayonPoolType;
pub use tokio::task::JoinHandle;
/// Provides a reason when Lighthouse is shut down.
@@ -84,6 +87,8 @@ pub struct TaskExecutor {
// FIXME(sproul): delete?
#[allow(dead_code)]
service_name: String,
rayon_pool_provider: Arc<RayonPoolProvider>,
}
impl TaskExecutor {
@@ -105,6 +110,7 @@ impl TaskExecutor {
exit,
signal_tx,
service_name,
rayon_pool_provider: Arc::new(RayonPoolProvider::default()),
}
}
@@ -115,6 +121,7 @@ impl TaskExecutor {
exit: self.exit.clone(),
signal_tx: self.signal_tx.clone(),
service_name,
rayon_pool_provider: self.rayon_pool_provider.clone(),
}
}
@@ -226,6 +233,47 @@ impl TaskExecutor {
}
}
/// Spawns a blocking task on a dedicated tokio thread pool and installs a rayon context within it.
pub fn spawn_blocking_with_rayon<F>(
self,
task: F,
rayon_pool_type: RayonPoolType,
name: &'static str,
) where
F: FnOnce() + Send + 'static,
{
let thread_pool = self.rayon_pool_provider.get_thread_pool(rayon_pool_type);
self.spawn_blocking(
move || {
thread_pool.install(|| {
task();
});
},
name,
)
}
/// Spawns a blocking computation on a rayon thread pool and awaits the result.
pub async fn spawn_blocking_with_rayon_async<F, R>(
&self,
rayon_pool_type: RayonPoolType,
task: F,
) -> Result<R, tokio::sync::oneshot::error::RecvError>
where
F: FnOnce() -> R + Send + 'static,
R: Send + 'static,
{
let thread_pool = self.rayon_pool_provider.get_thread_pool(rayon_pool_type);
let (tx, rx) = tokio::sync::oneshot::channel();
thread_pool.spawn(move || {
let result = task();
let _ = tx.send(result);
});
rx.await
}
/// Spawn a future on the tokio runtime wrapped in an `async-channel::Receiver` returning an optional
/// join handle to the future.
/// The task is cancelled when the corresponding async-channel is dropped.

58
common/task_executor/src/rayon_pool_provider.rs Normal file
View File

@@ -0,0 +1,58 @@
use rayon::{ThreadPool, ThreadPoolBuilder};
use std::sync::Arc;
const DEFAULT_LOW_PRIORITY_CPU_PERCENTAGE: usize = 25;
const DEFAULT_HIGH_PRIORITY_CPU_PERCENTAGE: usize = 80;
const MINIMUM_THREAD_COUNT: usize = 1;
pub enum RayonPoolType {
HighPriority,
LowPriority,
}
pub struct RayonPoolProvider {
/// Smaller rayon thread pool for lower-priority, compute-intensive tasks.
/// By default ~25% of CPUs or a minimum of 1 thread.
low_priority_thread_pool: Arc<ThreadPool>,
/// Larger rayon thread pool for high-priority, compute-intensive tasks.
/// By default ~80% of CPUs or a minimum of 1 thread. Citical/highest
/// priority tasks should use the global pool instead.
high_priority_thread_pool: Arc<ThreadPool>,
}
impl Default for RayonPoolProvider {
fn default() -> Self {
let low_prio_threads =
(num_cpus::get() * DEFAULT_LOW_PRIORITY_CPU_PERCENTAGE / 100).max(MINIMUM_THREAD_COUNT);
let low_priority_thread_pool = Arc::new(
ThreadPoolBuilder::new()
.num_threads(low_prio_threads)
.build()
.expect("failed to build low-priority rayon pool"),
);
let high_prio_threads = (num_cpus::get() * DEFAULT_HIGH_PRIORITY_CPU_PERCENTAGE / 100)
.max(MINIMUM_THREAD_COUNT);
let high_priority_thread_pool = Arc::new(
ThreadPoolBuilder::new()
.num_threads(high_prio_threads)
.build()
.expect("failed to build high-priority rayon pool"),
);
Self {
low_priority_thread_pool,
high_priority_thread_pool,
}
}
}
impl RayonPoolProvider {
/// Get a scoped thread pool by priority level.
/// For critical/highest priority tasks, use the global pool instead.
pub fn get_thread_pool(&self, rayon_pool_type: RayonPoolType) -> Arc<ThreadPool> {
match rayon_pool_type {
RayonPoolType::HighPriority => self.high_priority_thread_pool.clone(),
RayonPoolType::LowPriority => self.low_priority_thread_pool.clone(),
}
}
}