From 788a4b718f689be4db611e403fb6a8c7426fe8db Mon Sep 17 00:00:00 2001
From: Maksim Shcherbo <max@happygopher.nl>
Date: Wed, 29 Mar 2023 02:56:39 +0000
Subject: [PATCH] Optimise `update_validators` by decrypting key cache only
 when necessary (#4126)

## Title

Optimise `update_validators` by decrypting key cache only when necessary

## Issue Addressed

Resolves [#3968: Slow performance of validator client PATCH API with hundreds of keys](https://github.com/sigp/lighthouse/issues/3968)

## Proposed Changes

1. Add a check to determine if there is at least one local definition before decrypting the key cache.
2. Assign an empty `KeyCache` when all definitions are of the `Web3Signer` type.
3. Perform cache-related operations (e.g., saving the modified key cache) only if there are local definitions.

## Additional Info

This PR addresses the excessive CPU usage and slow performance experienced when using the `PATCH lighthouse/validators/{pubkey}` request with a large number of keys. The issue was caused by the key cache using cryptography to decipher and cipher the cache entities every time the request was made. This operation called `scrypt`, which was very slow and required a lot of memory when there were many concurrent requests.

These changes have no impact on the overall functionality but can lead to significant performance improvements when working with remote signers. Importantly, the key cache is never used when there are only `Web3Signer` definitions, avoiding the expensive operation of decrypting the key cache in such cases.

Co-authored-by: Maksim Shcherbo <max.shcherbo@consensys.net>
---
 .../src/initialized_validators.rs             | 27 ++++++++++++++++---
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/validator_client/src/initialized_validators.rs b/validator_client/src/initialized_validators.rs
index 7fe2f5f8ec..468fc2b06b 100644
--- a/validator_client/src/initialized_validators.rs
+++ b/validator_client/src/initialized_validators.rs
@@ -989,7 +989,23 @@ impl InitializedValidators {
 
         let cache =
             KeyCache::open_or_create(&self.validators_dir).map_err(Error::UnableToOpenKeyCache)?;
-        let mut key_cache = self.decrypt_key_cache(cache, &mut key_stores).await?;
+
+        // Check if there is at least one local definition.
+        let has_local_definitions = self.definitions.as_slice().iter().any(|def| {
+            matches!(
+                def.signing_definition,
+                SigningDefinition::LocalKeystore { .. }
+            )
+        });
+
+        // Only decrypt cache when there is at least one local definition.
+        // Decrypting cache is a very expensive operation which is never used for web3signer.
+        let mut key_cache = if has_local_definitions {
+            self.decrypt_key_cache(cache, &mut key_stores).await?
+        } else {
+            // Assign an empty KeyCache if all definitions are of the Web3Signer type.
+            KeyCache::new()
+        };
 
         let mut disabled_uuids = HashSet::new();
         for def in self.definitions.as_slice() {
@@ -1115,13 +1131,16 @@ impl InitializedValidators {
                 );
             }
         }
-        for uuid in disabled_uuids {
-            key_cache.remove(&uuid);
+
+        if has_local_definitions {
+            for uuid in disabled_uuids {
+                key_cache.remove(&uuid);
+            }
         }
 
         let validators_dir = self.validators_dir.clone();
         let log = self.log.clone();
-        if key_cache.is_modified() {
+        if has_local_definitions && key_cache.is_modified() {
             tokio::task::spawn_blocking(move || {
                 match key_cache.save(validators_dir) {
                     Err(e) => warn!(