Persist light client updates (#5545)

* persist light client updates

* update beacon chain to serve light client updates

* resolve todos

* cache best update

* extend cache parts

* is better light client update

* resolve merge conflict

* initial api changes

* add lc update db column

* fmt

* added tests

* add sim

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* fix some weird issues with the simulator

* tests

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* test changes

* merge conflict

* testing

* started work on ef tests and some code clean up

* update tests

* linting

* noop pre altair, were still failing on electra though

* allow for zeroed light client header

* Merge branch 'unstable' of https://github.com/sigp/lighthouse into persist-light-client-updates

* merge unstable

* remove unwraps

* remove unwraps

* Update light_client_update.rs

* merge unstable

* move functionality to helper methods

* refactor is best update fn

* refactor is best update fn

* improve organization of light client server cache logic

* fork diget calc, and only spawn as many blcoks as we need for the lc update test

* fetch lc update from the cache if it exists

* fmt

* Fix beacon_chain tests

* Add debug code to update ranking_order ef test

* Fix compare code

* merge conflicts

* fix test

* Merge branch 'persist-light-client-updates' of https://github.com/eserilev/lighthouse into persist-light-client-updates

* Use blinded blocks for light client proofs

* fix ef test

* merge conflicts

* fix lc update check

* Lint

* resolve merge conflict

* Merge branch 'persist-light-client-updates' of https://github.com/eserilev/lighthouse into persist-light-client-updates

* revert basic sim

* small fix

* revert sim

* Review PR

* resolve merge conflicts

* Merge branch 'unstable' into persist-light-client-updates

testing/ef_tests/src/cases.rs

@@ -24,6 +24,7 @@ mod kzg_compute_kzg_proof;
 mod kzg_verify_blob_kzg_proof;
 mod kzg_verify_blob_kzg_proof_batch;
 mod kzg_verify_kzg_proof;
+mod light_client_verify_is_better_update;
 mod merkle_proof_validity;
 mod operations;
 mod rewards;
@@ -54,6 +55,7 @@ pub use kzg_compute_kzg_proof::*;
 pub use kzg_verify_blob_kzg_proof::*;
 pub use kzg_verify_blob_kzg_proof_batch::*;
 pub use kzg_verify_kzg_proof::*;
+pub use light_client_verify_is_better_update::*;
 pub use merkle_proof_validity::*;
 pub use operations::*;
 pub use rewards::RewardsTest;

testing/ef_tests/src/cases/light_client_verify_is_better_update.rs

@@ -0,0 +1,110 @@
+use super::*;
+use decode::ssz_decode_light_client_update;
+use serde::Deserialize;
+use types::{LightClientUpdate, Slot};
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct LightClientVerifyIsBetterUpdate<E: EthSpec> {
+    light_client_updates: Vec<LightClientUpdate<E>>,
+}
+
+#[derive(Debug, Clone, Default, Deserialize)]
+pub struct Metadata {
+    updates_count: u64,
+}
+
+impl<E: EthSpec> LoadCase for LightClientVerifyIsBetterUpdate<E> {
+    fn load_from_dir(path: &Path, fork_name: ForkName) -> Result<Self, Error> {
+        let mut light_client_updates = vec![];
+        let metadata: Metadata = decode::yaml_decode_file(path.join("meta.yaml").as_path())?;
+        for index in 0..metadata.updates_count {
+            let light_client_update = ssz_decode_light_client_update(
+                &path.join(format!("updates_{}.ssz_snappy", index)),
+                &fork_name,
+            )?;
+            light_client_updates.push(light_client_update);
+        }
+
+        Ok(Self {
+            light_client_updates,
+        })
+    }
+}
+
+impl<E: EthSpec> Case for LightClientVerifyIsBetterUpdate<E> {
+    // Light client updates in `self.light_client_updates` are ordered in descending precedence
+    // where the update at index = 0 is considered the best update. This test iterates through
+    // all light client updates in a nested loop to make all possible comparisons. If a light client update
+    // at index `i`` is considered 'better' than a light client update at index `j`` when `i > j`, this test fails.
+    fn result(&self, _case_index: usize, fork_name: ForkName) -> Result<(), Error> {
+        let spec = fork_name.make_genesis_spec(E::default_spec());
+        for (i, ith_light_client_update) in self.light_client_updates.iter().enumerate() {
+            for (j, jth_light_client_update) in self.light_client_updates.iter().enumerate() {
+                eprintln!("{i} {j}");
+                if i == j {
+                    continue;
+                }
+
+                let is_better_update = ith_light_client_update
+                    .is_better_light_client_update(jth_light_client_update, &spec)
+                    .unwrap();
+
+                let ith_summary =
+                    LightClientUpdateSummary::from_update(ith_light_client_update, &spec);
+                let jth_summary =
+                    LightClientUpdateSummary::from_update(jth_light_client_update, &spec);
+
+                let (best_index, other_index, best_update, other_update, failed) = if i < j {
+                    // i is better, so is_better_update must return false
+                    (i, j, ith_summary, jth_summary, is_better_update)
+                } else {
+                    // j is better, so is_better must return true
+                    (j, i, jth_summary, ith_summary, !is_better_update)
+                };
+
+                if failed {
+                    eprintln!("is_better_update: {is_better_update}");
+                    eprintln!("index {best_index} update {best_update:?}");
+                    eprintln!("index {other_index} update {other_update:?}");
+                    eprintln!(
+                        "update at index {best_index} must be considered better than update at index {other_index}"
+                    );
+                    return Err(Error::FailedComparison(format!(
+                        "update at index {best_index} must be considered better than update at index {other_index}"
+                    )));
+                }
+            }
+        }
+
+        Ok(())
+    }
+}
+
+#[derive(Debug)]
+#[allow(dead_code)]
+struct LightClientUpdateSummary {
+    participants: usize,
+    supermajority: bool,
+    relevant_sync_committee: bool,
+    has_finality: bool,
+    has_sync_committee_finality: bool,
+    header_slot: Slot,
+    signature_slot: Slot,
+}
+
+impl LightClientUpdateSummary {
+    fn from_update<E: EthSpec>(update: &LightClientUpdate<E>, spec: &ChainSpec) -> Self {
+        let max_participants = update.sync_aggregate().sync_committee_bits.len();
+        let participants = update.sync_aggregate().sync_committee_bits.num_set_bits();
+        Self {
+            participants,
+            supermajority: participants * 3 > max_participants * 2,
+            relevant_sync_committee: update.is_sync_committee_update(spec).unwrap(),
+            has_finality: !update.is_finality_branch_empty(),
+            has_sync_committee_finality: update.has_sync_committee_finality(spec).unwrap(),
+            header_slot: update.attested_header_slot(),
+            signature_slot: *update.signature_slot(),
+        }
+    }
+}

testing/ef_tests/src/decode.rs

@@ -5,7 +5,7 @@ use std::fs::{self};
 use std::io::Write;
 use std::path::Path;
 use std::path::PathBuf;
-use types::BeaconState;
+use types::{BeaconState, LightClientUpdate};
 
 /// See `log_file_access` for details.
 const ACCESSED_FILE_LOG_FILENAME: &str = ".accessed_file_log.txt";
@@ -95,3 +95,13 @@ pub fn ssz_decode_state<E: EthSpec>(
     log_file_access(path);
     ssz_decode_file_with(path, |bytes| BeaconState::from_ssz_bytes(bytes, spec))
 }
+
+pub fn ssz_decode_light_client_update<E: EthSpec>(
+    path: &Path,
+    fork_name: &ForkName,
+) -> Result<LightClientUpdate<E>, Error> {
+    log_file_access(path);
+    ssz_decode_file_with(path, |bytes| {
+        LightClientUpdate::from_ssz_bytes(bytes, fork_name)
+    })
+}

testing/ef_tests/src/error.rs

@@ -14,6 +14,8 @@ pub enum Error {
     SkippedKnownFailure,
     /// The test failed due to some internal error preventing the test from running.
     InternalError(String),
+    /// The test failed while making some comparison.
+    FailedComparison(String),
 }
 
 impl Error {
@@ -26,6 +28,7 @@ impl Error {
             Error::SkippedBls => "SkippedBls",
             Error::SkippedKnownFailure => "SkippedKnownFailure",
             Error::InternalError(_) => "InternalError",
+            Error::FailedComparison(_) => "FailedComparison",
         }
     }
 

testing/ef_tests/src/handler.rs

@@ -837,6 +837,32 @@ impl<E: EthSpec + TypeName> Handler for KzgInclusionMerkleProofValidityHandler<E
     }
 }
 
+#[derive(Derivative)]
+#[derivative(Default(bound = ""))]
+pub struct LightClientUpdateHandler<E>(PhantomData<E>);
+
+impl<E: EthSpec + TypeName> Handler for LightClientUpdateHandler<E> {
+    type Case = cases::LightClientVerifyIsBetterUpdate<E>;
+
+    fn config_name() -> &'static str {
+        E::name()
+    }
+
+    fn runner_name() -> &'static str {
+        "light_client"
+    }
+
+    fn handler_name(&self) -> String {
+        "update_ranking".into()
+    }
+
+    fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool {
+        // Enabled in Altair
+        // TODO(electra) re-enable once https://github.com/sigp/lighthouse/issues/6002 is resolved
+        fork_name != ForkName::Base && fork_name != ForkName::Electra
+    }
+}
+
 #[derive(Derivative)]
 #[derivative(Default(bound = ""))]
 pub struct OperationsHandler<E, O>(PhantomData<(E, O)>);