Add is_parent_strong proposer re-org check (#5417)

* initial fork choice additions

* add helper fns

* add is_parent_strong

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into add_is_parent_strong_check

* disabling proposer reorg should set parent_threshold to u64 max

* add new flag, is_parent_strong check in override fcu params

* cherry-pick changes

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into add_is_parent_strong_check

* cleanup

* fmt

* Minor review tweaks
This commit is contained in:
Eitan Seri-Levi
2024-04-04 22:38:06 +03:00
committed by GitHub
parent 053525e281
commit ee69e14db9
16 changed files with 181 additions and 53 deletions

View File

@@ -4262,7 +4262,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
head_slot: Slot,
canonical_head: Hash256,
) -> Option<BlockProductionPreState<T::EthSpec>> {
let re_org_threshold = self.config.re_org_threshold?;
let re_org_head_threshold = self.config.re_org_head_threshold?;
let re_org_parent_threshold = self.config.re_org_parent_threshold?;
if self.spec.proposer_score_boost.is_none() {
warn!(
@@ -4319,7 +4320,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.get_proposer_head(
slot,
canonical_head,
re_org_threshold,
re_org_head_threshold,
re_org_parent_threshold,
&self.config.re_org_disallowed_offsets,
self.config.re_org_max_epochs_since_finalization,
)
@@ -4373,7 +4375,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
"weak_head" => ?canonical_head,
"parent" => ?re_org_parent_block,
"head_weight" => proposer_head.head_node.weight,
"threshold_weight" => proposer_head.re_org_weight_threshold
"threshold_weight" => proposer_head.re_org_head_weight_threshold
);
Some(pre_state)
@@ -4593,9 +4595,14 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
let _timer = metrics::start_timer(&metrics::FORK_CHOICE_OVERRIDE_FCU_TIMES);
// Never override if proposer re-orgs are disabled.
let re_org_threshold = self
let re_org_head_threshold = self
.config
.re_org_threshold
.re_org_head_threshold
.ok_or(DoNotReOrg::ReOrgsDisabled)?;
let re_org_parent_threshold = self
.config
.re_org_parent_threshold
.ok_or(DoNotReOrg::ReOrgsDisabled)?;
let head_block_root = canonical_forkchoice_params.head_root;
@@ -4606,7 +4613,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
.fork_choice_read_lock()
.get_preliminary_proposer_head(
head_block_root,
re_org_threshold,
re_org_head_threshold,
re_org_parent_threshold,
&self.config.re_org_disallowed_offsets,
self.config.re_org_max_epochs_since_finalization,
)
@@ -4674,16 +4682,27 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
}
// If the current slot is already equal to the proposal slot (or we are in the tail end of
// the prior slot), then check the actual weight of the head against the re-org threshold.
let head_weak = if fork_choice_slot == re_org_block_slot {
info.head_node.weight < info.re_org_weight_threshold
// the prior slot), then check the actual weight of the head against the head re-org threshold
// and the actual weight of the parent against the parent re-org threshold.
let (head_weak, parent_strong) = if fork_choice_slot == re_org_block_slot {
(
info.head_node.weight < info.re_org_head_weight_threshold,
info.parent_node.weight > info.re_org_parent_weight_threshold,
)
} else {
true
(true, true)
};
if !head_weak {
return Err(DoNotReOrg::HeadNotWeak {
head_weight: info.head_node.weight,
re_org_weight_threshold: info.re_org_weight_threshold,
re_org_head_weight_threshold: info.re_org_head_weight_threshold,
}
.into());
}
if !parent_strong {
return Err(DoNotReOrg::ParentNotStrong {
parent_weight: info.parent_node.weight,
re_org_parent_weight_threshold: info.re_org_parent_weight_threshold,
}
.into());
}

View File

@@ -172,8 +172,8 @@ where
}
/// Sets the proposer re-org threshold.
pub fn proposer_re_org_threshold(mut self, threshold: Option<ReOrgThreshold>) -> Self {
self.chain_config.re_org_threshold = threshold;
pub fn proposer_re_org_head_threshold(mut self, threshold: Option<ReOrgThreshold>) -> Self {
self.chain_config.re_org_head_threshold = threshold;
self
}

View File

@@ -3,7 +3,8 @@ use serde::{Deserialize, Serialize};
use std::time::Duration;
use types::{Checkpoint, Epoch};
pub const DEFAULT_RE_ORG_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20);
pub const DEFAULT_RE_ORG_HEAD_THRESHOLD: ReOrgThreshold = ReOrgThreshold(20);
pub const DEFAULT_RE_ORG_PARENT_THRESHOLD: ReOrgThreshold = ReOrgThreshold(160);
pub const DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION: Epoch = Epoch::new(2);
/// Default to 1/12th of the slot, which is 1 second on mainnet.
pub const DEFAULT_RE_ORG_CUTOFF_DENOMINATOR: u32 = 12;
@@ -31,8 +32,10 @@ pub struct ChainConfig {
pub enable_lock_timeouts: bool,
/// The max size of a message that can be sent over the network.
pub max_network_size: usize,
/// Maximum percentage of committee weight at which to attempt re-orging the canonical head.
pub re_org_threshold: Option<ReOrgThreshold>,
/// Maximum percentage of the head committee weight at which to attempt re-orging the canonical head.
pub re_org_head_threshold: Option<ReOrgThreshold>,
/// Minimum percentage of the parent committee weight at which to attempt re-orging the canonical head.
pub re_org_parent_threshold: Option<ReOrgThreshold>,
/// Maximum number of epochs since finalization for attempting a proposer re-org.
pub re_org_max_epochs_since_finalization: Epoch,
/// Maximum delay after the start of the slot at which to propose a reorging block.
@@ -95,7 +98,8 @@ impl Default for ChainConfig {
reconstruct_historic_states: false,
enable_lock_timeouts: true,
max_network_size: 10 * 1_048_576, // 10M
re_org_threshold: Some(DEFAULT_RE_ORG_THRESHOLD),
re_org_head_threshold: Some(DEFAULT_RE_ORG_HEAD_THRESHOLD),
re_org_parent_threshold: Some(DEFAULT_RE_ORG_PARENT_THRESHOLD),
re_org_max_epochs_since_finalization: DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
re_org_cutoff_millis: None,
re_org_disallowed_offsets: DisallowedReOrgOffsets::default(),

View File

@@ -419,7 +419,7 @@ pub async fn proposer_boost_re_org_test(
None,
Some(Box::new(move |builder| {
builder
.proposer_re_org_threshold(Some(ReOrgThreshold(re_org_threshold)))
.proposer_re_org_head_threshold(Some(ReOrgThreshold(re_org_threshold)))
.proposer_re_org_max_epochs_since_finalization(Epoch::new(
max_epochs_since_finalization,
))

View File

@@ -1040,10 +1040,18 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
Arg::with_name("proposer-reorg-threshold")
.long("proposer-reorg-threshold")
.value_name("PERCENT")
.help("Percentage of vote weight below which to attempt a proposer reorg. \
.help("Percentage of head vote weight below which to attempt a proposer reorg. \
Default: 20%")
.conflicts_with("disable-proposer-reorgs")
)
.arg(
Arg::with_name("proposer-reorg-parent-threshold")
.long("proposer-reorg-parent-threshold")
.value_name("PERCENT")
.help("Percentage of parent vote weight above which to attempt a proposer reorg. \
Default: 160%")
.conflicts_with("disable-proposer-reorgs")
)
.arg(
Arg::with_name("proposer-reorg-epochs-since-finalization")
.long("proposer-reorg-epochs-since-finalization")

View File

@@ -1,6 +1,7 @@
use beacon_chain::chain_config::{
DisallowedReOrgOffsets, ReOrgThreshold, DEFAULT_PREPARE_PAYLOAD_LOOKAHEAD_FACTOR,
DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD,
DEFAULT_RE_ORG_HEAD_THRESHOLD, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
DEFAULT_RE_ORG_PARENT_THRESHOLD,
};
use beacon_chain::TrustedSetup;
use clap::ArgMatches;
@@ -749,12 +750,13 @@ pub fn get_config<E: EthSpec>(
}
if cli_args.is_present("disable-proposer-reorgs") {
client_config.chain.re_org_threshold = None;
client_config.chain.re_org_head_threshold = None;
client_config.chain.re_org_parent_threshold = None;
} else {
client_config.chain.re_org_threshold = Some(
client_config.chain.re_org_head_threshold = Some(
clap_utils::parse_optional(cli_args, "proposer-reorg-threshold")?
.map(ReOrgThreshold)
.unwrap_or(DEFAULT_RE_ORG_THRESHOLD),
.unwrap_or(DEFAULT_RE_ORG_HEAD_THRESHOLD),
);
client_config.chain.re_org_max_epochs_since_finalization =
clap_utils::parse_optional(cli_args, "proposer-reorg-epochs-since-finalization")?
@@ -762,6 +764,12 @@ pub fn get_config<E: EthSpec>(
client_config.chain.re_org_cutoff_millis =
clap_utils::parse_optional(cli_args, "proposer-reorg-cutoff")?;
client_config.chain.re_org_parent_threshold = Some(
clap_utils::parse_optional(cli_args, "proposer-reorg-parent-threshold")?
.map(ReOrgThreshold)
.unwrap_or(DEFAULT_RE_ORG_PARENT_THRESHOLD),
);
if let Some(disallowed_offsets_str) =
clap_utils::parse_optional::<String>(cli_args, "proposer-reorg-disallowed-offsets")?
{

View File

@@ -383,8 +383,11 @@ OPTIONS:
--proposer-reorg-epochs-since-finalization <EPOCHS>
Maximum number of epochs since finalization at which proposer reorgs are allowed. Default: 2
--proposer-reorg-parent-threshold <PERCENT>
Percentage of parent vote weight above which to attempt a proposer reorg. Default: 160%
--proposer-reorg-threshold <PERCENT>
Percentage of vote weight below which to attempt a proposer reorg. Default: 20%
Percentage of head vote weight below which to attempt a proposer reorg. Default: 20%
--prune-blobs <BOOLEAN>
Prune blobs from Lighthouse's database when they are older than the data data availability boundary relative

View File

@@ -81,6 +81,12 @@ CHURN_LIMIT_QUOTIENT: 4096
# ---------------------------------------------------------------
# 40%
PROPOSER_SCORE_BOOST: 40
# 20%
REORG_HEAD_WEIGHT_THRESHOLD: 20
# 160%
REORG_PARENT_WEIGHT_THRESHOLD: 160
# `2` epochs
REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
# Deposit contract
# ---------------------------------------------------------------

View File

@@ -71,6 +71,12 @@ MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8
# ---------------------------------------------------------------
# 40%
PROPOSER_SCORE_BOOST: 40
# 20%
REORG_HEAD_WEIGHT_THRESHOLD: 20
# 160%
REORG_PARENT_WEIGHT_THRESHOLD: 160
# `2` epochs
REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
# Deposit contract
# ---------------------------------------------------------------

View File

@@ -78,6 +78,12 @@ MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8
# ---------------------------------------------------------------
# 40%
PROPOSER_SCORE_BOOST: 40
# 20%
REORG_HEAD_WEIGHT_THRESHOLD: 20
# 160%
REORG_PARENT_WEIGHT_THRESHOLD: 160
# `2` epochs
REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2
# Deposit contract
# ---------------------------------------------------------------

View File

@@ -69,6 +69,10 @@ MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8
# ---------------------------------------------------------------
# 40%
PROPOSER_SCORE_BOOST: 40
# 20%
REORG_HEAD_WEIGHT_THRESHOLD: 20
# 160%
REORG_PARENT_WEIGHT_THRESHOLD: 160
# Deposit contract
# ---------------------------------------------------------------

View File

@@ -517,7 +517,8 @@ where
&self,
current_slot: Slot,
canonical_head: Hash256,
re_org_threshold: ReOrgThreshold,
re_org_head_threshold: ReOrgThreshold,
re_org_parent_threshold: ReOrgThreshold,
disallowed_offsets: &DisallowedReOrgOffsets,
max_epochs_since_finalization: Epoch,
) -> Result<ProposerHeadInfo, ProposerHeadError<Error<proto_array::Error>>> {
@@ -549,7 +550,8 @@ where
current_slot,
canonical_head,
self.fc_store.justified_balances(),
re_org_threshold,
re_org_head_threshold,
re_org_parent_threshold,
disallowed_offsets,
max_epochs_since_finalization,
)
@@ -559,7 +561,8 @@ where
pub fn get_preliminary_proposer_head(
&self,
canonical_head: Hash256,
re_org_threshold: ReOrgThreshold,
re_org_head_threshold: ReOrgThreshold,
re_org_parent_threshold: ReOrgThreshold,
disallowed_offsets: &DisallowedReOrgOffsets,
max_epochs_since_finalization: Epoch,
) -> Result<ProposerHeadInfo, ProposerHeadError<Error<proto_array::Error>>> {
@@ -569,7 +572,8 @@ where
current_slot,
canonical_head,
self.fc_store.justified_balances(),
re_org_threshold,
re_org_head_threshold,
re_org_parent_threshold,
disallowed_offsets,
max_epochs_since_finalization,
)
@@ -706,6 +710,7 @@ where
// Add proposer score boost if the block is timely.
let is_before_attesting_interval =
block_delay < Duration::from_secs(spec.seconds_per_slot / INTERVALS_PER_SLOT);
let is_first_block = self.fc_store.proposer_boost_root().is_zero();
if current_slot == block.slot() && is_before_attesting_interval && is_first_block {
self.fc_store.set_proposer_boost_root(block_root);

View File

@@ -195,8 +195,10 @@ pub struct ProposerHeadInfo {
/// Information about the parent of the current head, which should be selected as the parent
/// for a new proposal *if* a re-org is decided on.
pub parent_node: ProtoNode,
/// The computed fraction of the active committee balance below which we can re-org.
pub re_org_weight_threshold: u64,
/// The computed fraction of the active head committee balance below which we can re-org.
pub re_org_head_weight_threshold: u64,
/// The computed fraction of the active parent committee balance above which we can re-org.
pub re_org_parent_weight_threshold: u64,
/// The current slot from fork choice's point of view, may lead the wall-clock slot by upto
/// 500ms.
pub current_slot: Slot,
@@ -259,7 +261,11 @@ pub enum DoNotReOrg {
},
HeadNotWeak {
head_weight: u64,
re_org_weight_threshold: u64,
re_org_head_weight_threshold: u64,
},
ParentNotStrong {
parent_weight: u64,
re_org_parent_weight_threshold: u64,
},
HeadNotLate,
NotProposing,
@@ -288,9 +294,21 @@ impl std::fmt::Display for DoNotReOrg {
),
Self::HeadNotWeak {
head_weight,
re_org_weight_threshold,
re_org_head_weight_threshold,
} => {
write!(f, "head not weak ({head_weight}/{re_org_weight_threshold})")
write!(
f,
"head not weak ({head_weight}/{re_org_head_weight_threshold})"
)
}
Self::ParentNotStrong {
parent_weight,
re_org_parent_weight_threshold,
} => {
write!(
f,
"parent not strong ({parent_weight}/{re_org_parent_weight_threshold})"
)
}
Self::HeadNotLate => {
write!(f, "head arrived on time")
@@ -486,12 +504,14 @@ impl ProtoArrayForkChoice {
/// Get the block to propose on during `current_slot`.
///
/// This function returns a *definitive* result which should be acted on.
#[allow(clippy::too_many_arguments)]
pub fn get_proposer_head<E: EthSpec>(
&self,
current_slot: Slot,
canonical_head: Hash256,
justified_balances: &JustifiedBalances,
re_org_threshold: ReOrgThreshold,
re_org_head_threshold: ReOrgThreshold,
re_org_parent_threshold: ReOrgThreshold,
disallowed_offsets: &DisallowedReOrgOffsets,
max_epochs_since_finalization: Epoch,
) -> Result<ProposerHeadInfo, ProposerHeadError<Error>> {
@@ -499,7 +519,8 @@ impl ProtoArrayForkChoice {
current_slot,
canonical_head,
justified_balances,
re_org_threshold,
re_org_head_threshold,
re_org_parent_threshold,
disallowed_offsets,
max_epochs_since_finalization,
)?;
@@ -510,14 +531,26 @@ impl ProtoArrayForkChoice {
return Err(DoNotReOrg::HeadDistance.into());
}
// Only re-org if the head's weight is less than the configured committee fraction.
// Only re-org if the head's weight is less than the heads configured committee fraction.
let head_weight = info.head_node.weight;
let re_org_weight_threshold = info.re_org_weight_threshold;
let weak_head = head_weight < re_org_weight_threshold;
let re_org_head_weight_threshold = info.re_org_head_weight_threshold;
let weak_head = head_weight < re_org_head_weight_threshold;
if !weak_head {
return Err(DoNotReOrg::HeadNotWeak {
head_weight,
re_org_weight_threshold,
re_org_head_weight_threshold,
}
.into());
}
// Only re-org if the parent's weight is greater than the parents configured committee fraction.
let parent_weight = info.parent_node.weight;
let re_org_parent_weight_threshold = info.re_org_parent_weight_threshold;
let parent_strong = parent_weight > re_org_parent_weight_threshold;
if !parent_strong {
return Err(DoNotReOrg::ParentNotStrong {
parent_weight,
re_org_parent_weight_threshold,
}
.into());
}
@@ -529,12 +562,14 @@ impl ProtoArrayForkChoice {
/// Get information about the block to propose on during `current_slot`.
///
/// This function returns a *partial* result which must be processed further.
#[allow(clippy::too_many_arguments)]
pub fn get_proposer_head_info<E: EthSpec>(
&self,
current_slot: Slot,
canonical_head: Hash256,
justified_balances: &JustifiedBalances,
re_org_threshold: ReOrgThreshold,
re_org_head_threshold: ReOrgThreshold,
re_org_parent_threshold: ReOrgThreshold,
disallowed_offsets: &DisallowedReOrgOffsets,
max_epochs_since_finalization: Epoch,
) -> Result<ProposerHeadInfo, ProposerHeadError<Error>> {
@@ -595,15 +630,20 @@ impl ProtoArrayForkChoice {
return Err(DoNotReOrg::JustificationAndFinalizationNotCompetitive.into());
}
// Compute re-org weight threshold.
let re_org_weight_threshold =
calculate_committee_fraction::<E>(justified_balances, re_org_threshold.0)
// Compute re-org weight thresholds for head and parent.
let re_org_head_weight_threshold =
calculate_committee_fraction::<E>(justified_balances, re_org_head_threshold.0)
.ok_or(Error::ReOrgThresholdOverflow)?;
let re_org_parent_weight_threshold =
calculate_committee_fraction::<E>(justified_balances, re_org_parent_threshold.0)
.ok_or(Error::ReOrgThresholdOverflow)?;
Ok(ProposerHeadInfo {
head_node,
parent_node,
re_org_weight_threshold,
re_org_head_weight_threshold,
re_org_parent_weight_threshold,
current_slot,
})
}

View File

@@ -114,6 +114,8 @@ pub struct ChainSpec {
*/
pub safe_slots_to_update_justified: u64,
pub proposer_score_boost: Option<u64>,
pub reorg_head_weight_threshold: Option<u64>,
pub reorg_parent_weight_threshold: Option<u64>,
/*
* Eth1
@@ -642,6 +644,8 @@ impl ChainSpec {
*/
safe_slots_to_update_justified: 8,
proposer_score_boost: Some(40),
reorg_head_weight_threshold: Some(20),
reorg_parent_weight_threshold: Some(160),
/*
* Eth1
@@ -912,6 +916,8 @@ impl ChainSpec {
*/
safe_slots_to_update_justified: 8,
proposer_score_boost: Some(40),
reorg_head_weight_threshold: Some(20),
reorg_parent_weight_threshold: Some(160),
/*
* Eth1

View File

@@ -2,8 +2,8 @@ use beacon_node::ClientConfig as Config;
use crate::exec::{CommandLineTestExec, CompletedTest};
use beacon_node::beacon_chain::chain_config::{
DisallowedReOrgOffsets, DEFAULT_RE_ORG_CUTOFF_DENOMINATOR,
DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD,
DisallowedReOrgOffsets, DEFAULT_RE_ORG_CUTOFF_DENOMINATOR, DEFAULT_RE_ORG_HEAD_THRESHOLD,
DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
};
use beacon_processor::BeaconProcessorConfig;
use eth1::Eth1Endpoint;
@@ -2200,8 +2200,8 @@ fn enable_proposer_re_orgs_default() {
.run_with_zero_port()
.with_config(|config| {
assert_eq!(
config.chain.re_org_threshold,
Some(DEFAULT_RE_ORG_THRESHOLD)
config.chain.re_org_head_threshold,
Some(DEFAULT_RE_ORG_HEAD_THRESHOLD)
);
assert_eq!(
config.chain.re_org_max_epochs_since_finalization,
@@ -2219,15 +2219,26 @@ fn disable_proposer_re_orgs() {
CommandLineTest::new()
.flag("disable-proposer-reorgs", None)
.run_with_zero_port()
.with_config(|config| assert_eq!(config.chain.re_org_threshold, None));
.with_config(|config| {
assert_eq!(config.chain.re_org_head_threshold, None);
assert_eq!(config.chain.re_org_parent_threshold, None)
});
}
#[test]
fn proposer_re_org_threshold() {
fn proposer_re_org_parent_threshold() {
CommandLineTest::new()
.flag("proposer-reorg-parent-threshold", Some("90"))
.run_with_zero_port()
.with_config(|config| assert_eq!(config.chain.re_org_parent_threshold.unwrap().0, 90));
}
#[test]
fn proposer_re_org_head_threshold() {
CommandLineTest::new()
.flag("proposer-reorg-threshold", Some("90"))
.run_with_zero_port()
.with_config(|config| assert_eq!(config.chain.re_org_threshold.unwrap().0, 90));
.with_config(|config| assert_eq!(config.chain.re_org_head_threshold.unwrap().0, 90));
}
#[test]

View File

@@ -4,7 +4,8 @@ use ::fork_choice::{PayloadVerificationStatus, ProposerHeadError};
use beacon_chain::beacon_proposer_cache::compute_proposer_duties_from_head;
use beacon_chain::blob_verification::GossipBlobError;
use beacon_chain::chain_config::{
DisallowedReOrgOffsets, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD,
DisallowedReOrgOffsets, DEFAULT_RE_ORG_HEAD_THRESHOLD,
DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_PARENT_THRESHOLD,
};
use beacon_chain::slot_clock::SlotClock;
use beacon_chain::{
@@ -746,7 +747,8 @@ impl<E: EthSpec> Tester<E> {
let proposer_head_result = fc.get_proposer_head(
slot,
canonical_head,
DEFAULT_RE_ORG_THRESHOLD,
DEFAULT_RE_ORG_HEAD_THRESHOLD,
DEFAULT_RE_ORG_PARENT_THRESHOLD,
&DisallowedReOrgOffsets::default(),
DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION,
);