Integrate tracing (#6339)

Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev

validator_client/http_api/Cargo.toml

@@ -29,9 +29,9 @@ parking_lot = { workspace = true }
 rand = { workspace = true }
 sensitive_url = { workspace = true }
 serde = { workspace = true }
+serde_json = { workspace = true }
 signing_method = { workspace = true }
 slashing_protection = { workspace = true }
-slog =  { workspace = true }
 slot_clock =  { workspace = true }
 sysinfo =  { workspace = true }
 system_health =  { workspace = true }
@@ -39,6 +39,7 @@ task_executor =  { workspace = true }
 tempfile =  { workspace = true }
 tokio =  { workspace = true }
 tokio-stream =  { workspace = true }
+tracing =  { workspace = true }
 types =  { workspace = true }
 url =  { workspace = true }
 validator_dir =  { workspace = true }

validator_client/http_api/src/create_signed_voluntary_exit.rs

@@ -1,8 +1,8 @@
 use bls::{PublicKey, PublicKeyBytes};
 use eth2::types::GenericResponse;
-use slog::{info, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
+use tracing::info;
 use types::{Epoch, EthSpec, SignedVoluntaryExit, VoluntaryExit};
 use validator_store::ValidatorStore;
 
@@ -11,7 +11,6 @@ pub async fn create_signed_voluntary_exit<T: 'static + SlotClock + Clone, E: Eth
     maybe_epoch: Option<Epoch>,
     validator_store: Arc<ValidatorStore<T, E>>,
     slot_clock: T,
-    log: Logger,
 ) -> Result<GenericResponse<SignedVoluntaryExit>, warp::Rejection> {
     let epoch = match maybe_epoch {
         Some(epoch) => epoch,
@@ -45,10 +44,9 @@ pub async fn create_signed_voluntary_exit<T: 'static + SlotClock + Clone, E: Eth
     };
 
     info!(
-        log,
-        "Signing voluntary exit";
-        "validator" => pubkey_bytes.as_hex_string(),
-        "epoch" => epoch
+        validator = pubkey_bytes.as_hex_string(),
+        %epoch,
+        "Signing voluntary exit"
     );
 
     let signed_voluntary_exit = validator_store

validator_client/http_api/src/keystores.rs

@@ -11,12 +11,12 @@ use eth2::lighthouse_vc::{
 use eth2_keystore::Keystore;
 use initialized_validators::{Error, InitializedValidators};
 use signing_method::SigningMethod;
-use slog::{info, warn, Logger};
 use slot_clock::SlotClock;
 use std::path::PathBuf;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
 use tokio::runtime::Handle;
+use tracing::{info, warn};
 use types::{EthSpec, PublicKeyBytes};
 use validator_dir::{keystore_password_path, Builder as ValidatorDirBuilder};
 use validator_store::ValidatorStore;
@@ -64,7 +64,6 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
     secrets_dir: Option<PathBuf>,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
-    log: Logger,
 ) -> Result<ImportKeystoresResponse, Rejection> {
     // Check request validity. This is the only cases in which we should return a 4xx code.
     if request.keystores.len() != request.passwords.len() {
@@ -88,18 +87,14 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
                         .iter()
                         .any(|data| data.pubkey == pubkey_bytes)
                     {
-                        warn!(
-                            log,
-                            "Slashing protection data not provided";
-                            "public_key" => ?public_key,
-                        );
+                        warn!(?public_key, "Slashing protection data not provided");
                     }
                 }
             }
 
             validator_store.import_slashing_protection(slashing_protection)
         } else {
-            warn!(log, "No slashing protection data provided with keystores");
+            warn!("No slashing protection data provided with keystores");
             Ok(())
         };
 
@@ -133,10 +128,9 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
                 Ok(status) => Status::ok(status),
                 Err(e) => {
                     warn!(
-                        log,
-                        "Error importing keystore, skipped";
-                        "pubkey" => pubkey_str,
-                        "error" => ?e,
+                        pubkey = pubkey_str,
+                        error = ?e,
+                        "Error importing keystore, skipped"
                     );
                     Status::error(ImportKeystoreStatus::Error, e)
                 }
@@ -157,9 +151,8 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
 
     if successful_import > 0 {
         info!(
-            log,
-            "Imported keystores via standard HTTP API";
-            "count" => successful_import,
+            count = successful_import,
+            "Imported keystores via standard HTTP API"
         );
     }
 
@@ -243,9 +236,8 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
     request: DeleteKeystoresRequest,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
-    log: Logger,
 ) -> Result<DeleteKeystoresResponse, Rejection> {
-    let export_response = export(request, validator_store, task_executor, log.clone())?;
+    let export_response = export(request, validator_store, task_executor)?;
 
     // Check the status is Deleted to confirm deletion is successful, then only display the log
     let successful_deletion = export_response
@@ -256,9 +248,8 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
 
     if successful_deletion > 0 {
         info!(
-            log,
-            "Deleted keystore via standard HTTP API";
-            "count" => successful_deletion,
+            count = successful_deletion,
+            "Deleted keystore via standard HTTP API"
         );
     }
 
@@ -276,7 +267,6 @@ pub fn export<T: SlotClock + 'static, E: EthSpec>(
     request: DeleteKeystoresRequest,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
-    log: Logger,
 ) -> Result<ExportKeystoresResponse, Rejection> {
     // Remove from initialized validators.
     let initialized_validators_rwlock = validator_store.initialized_validators();
@@ -294,10 +284,9 @@ pub fn export<T: SlotClock + 'static, E: EthSpec>(
                 Ok(status) => status,
                 Err(error) => {
                     warn!(
-                        log,
-                        "Error deleting keystore";
-                        "pubkey" => ?pubkey_bytes,
-                        "error" => ?error,
+                        pubkey = ?pubkey_bytes,
+                        ?error,
+                        "Error deleting keystore"
                     );
                     SingleExportKeystoresResponse {
                         status: Status::error(DeleteKeystoreStatus::Error, error),

validator_client/http_api/src/lib.rs

@@ -34,10 +34,10 @@ use eth2::lighthouse_vc::{
 };
 use health_metrics::observe::Observe;
 use lighthouse_version::version_with_platform;
+use logging::crit;
 use logging::SSELoggingComponents;
 use parking_lot::RwLock;
 use serde::{Deserialize, Serialize};
-use slog::{crit, info, warn, Logger};
 use slot_clock::SlotClock;
 use std::collections::HashMap;
 use std::future::Future;
@@ -49,6 +49,7 @@ use sysinfo::{System, SystemExt};
 use system_health::observe_system_health_vc;
 use task_executor::TaskExecutor;
 use tokio_stream::{wrappers::BroadcastStream, StreamExt};
+use tracing::{info, warn};
 use types::{ChainSpec, ConfigAndPreset, EthSpec};
 use validator_dir::Builder as ValidatorDirBuilder;
 use validator_services::block_service::BlockService;
@@ -87,7 +88,6 @@ pub struct Context<T: SlotClock, E: EthSpec> {
     pub graffiti_flag: Option<Graffiti>,
     pub spec: Arc<ChainSpec>,
     pub config: Config,
-    pub log: Logger,
     pub sse_logging_components: Option<SSELoggingComponents>,
     pub slot_clock: T,
     pub _phantom: PhantomData<E>,
@@ -148,7 +148,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
     let config = &ctx.config;
     let allow_keystore_export = config.allow_keystore_export;
     let store_passwords_in_secrets_dir = config.store_passwords_in_secrets_dir;
-    let log = ctx.log.clone();
 
     // Configure CORS.
     let cors_builder = {
@@ -165,7 +164,7 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
 
     // Sanity check.
     if !config.enabled {
-        crit!(log, "Cannot start disabled metrics HTTP server");
+        crit!("Cannot start disabled metrics HTTP server");
         return Err(Error::Other(
             "A disabled metrics server should not be started".to_string(),
         ));
@@ -179,9 +178,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         Ok(abs_path) => api_token_path = abs_path,
         Err(e) => {
             warn!(
-                log,
-                "Error canonicalizing token path";
-                "error" => ?e,
+                error = ?e,
+                "Error canonicalizing token path"
             );
         }
     };
@@ -239,9 +237,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
     let inner_graffiti_flag = ctx.graffiti_flag;
     let graffiti_flag_filter = warp::any().map(move || inner_graffiti_flag);
 
-    let inner_ctx = ctx.clone();
-    let log_filter = warp::any().map(move || inner_ctx.log.clone());
-
     let inner_slot_clock = ctx.slot_clock.clone();
     let slot_clock_filter = warp::any().map(move || inner_slot_clock.clone());
 
@@ -399,12 +394,10 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(validator_store_filter.clone())
         .and(graffiti_file_filter.clone())
         .and(graffiti_flag_filter)
-        .and(log_filter.clone())
         .then(
             |validator_store: Arc<ValidatorStore<T, E>>,
              graffiti_file: Option<GraffitiFile>,
-             graffiti_flag: Option<Graffiti>,
-             log| {
+             graffiti_flag: Option<Graffiti>| {
                 blocking_json_task(move || {
                     let mut result = HashMap::new();
                     for (key, graffiti_definition) in validator_store
@@ -414,7 +407,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                     {
                         let graffiti = determine_graffiti(
                             key,
-                            &log,
                             graffiti_file.clone(),
                             graffiti_definition,
                             graffiti_flag,
@@ -834,11 +826,10 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
-        .and(log_filter.clone())
-        .then(move |request, validator_store, task_executor, log| {
+        .then(move |request, validator_store, task_executor| {
             blocking_json_task(move || {
                 if allow_keystore_export {
-                    keystores::export(request, validator_store, task_executor, log)
+                    keystores::export(request, validator_store, task_executor)
                 } else {
                     Err(warp_utils::reject::custom_bad_request(
                         "keystore export is disabled".to_string(),
@@ -1079,14 +1070,12 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::path::end())
         .and(validator_store_filter.clone())
         .and(slot_clock_filter)
-        .and(log_filter.clone())
         .and(task_executor_filter.clone())
         .then(
             |pubkey: PublicKey,
              query: api_types::VoluntaryExitQuery,
              validator_store: Arc<ValidatorStore<T, E>>,
              slot_clock: T,
-             log,
              task_executor: TaskExecutor| {
                 blocking_json_task(move || {
                     if let Some(handle) = task_executor.handle() {
@@ -1096,7 +1085,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                                 query.epoch,
                                 validator_store,
                                 slot_clock,
-                                log,
                             ))?;
                         Ok(signed_voluntary_exit)
                     } else {
@@ -1196,9 +1184,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(secrets_dir_filter)
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
-        .and(log_filter.clone())
         .then(
-            move |request, validator_dir, secrets_dir, validator_store, task_executor, log| {
+            move |request, validator_dir, secrets_dir, validator_store, task_executor| {
                 let secrets_dir = store_passwords_in_secrets_dir.then_some(secrets_dir);
                 blocking_json_task(move || {
                     keystores::import(
@@ -1207,7 +1194,6 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                         secrets_dir,
                         validator_store,
                         task_executor,
-                        log,
                     )
                 })
             },
@@ -1218,11 +1204,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
-        .and(log_filter.clone())
-        .then(|request, validator_store, task_executor, log| {
-            blocking_json_task(move || {
-                keystores::delete(request, validator_store, task_executor, log)
-            })
+        .then(|request, validator_store, task_executor| {
+            blocking_json_task(move || keystores::delete(request, validator_store, task_executor))
         });
 
     // GET /eth/v1/remotekeys
@@ -1237,11 +1220,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(validator_store_filter.clone())
         .and(task_executor_filter.clone())
-        .and(log_filter.clone())
-        .then(|request, validator_store, task_executor, log| {
-            blocking_json_task(move || {
-                remotekeys::import(request, validator_store, task_executor, log)
-            })
+        .then(|request, validator_store, task_executor| {
+            blocking_json_task(move || remotekeys::import(request, validator_store, task_executor))
         });
 
     // DELETE /eth/v1/remotekeys
@@ -1249,11 +1229,8 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
         .and(warp::body::json())
         .and(validator_store_filter)
         .and(task_executor_filter)
-        .and(log_filter.clone())
-        .then(|request, validator_store, task_executor, log| {
-            blocking_json_task(move || {
-                remotekeys::delete(request, validator_store, task_executor, log)
-            })
+        .then(|request, validator_store, task_executor| {
+            blocking_json_task(move || remotekeys::delete(request, validator_store, task_executor))
         });
 
     // Subscribe to get VC logs via Server side events
@@ -1271,7 +1248,9 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
                             match msg {
                                 Ok(data) => {
                                     // Serialize to json
-                                    match data.to_json_string() {
+                                    match serde_json::to_string(&data)
+                                        .map_err(|e| format!("{:?}", e))
+                                    {
                                         // Send the json as a Server Sent Event
                                         Ok(json) => Event::default().json_data(json).map_err(|e| {
                                             warp_utils::reject::server_sent_event_error(format!(
@@ -1364,10 +1343,9 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
     )?;
 
     info!(
-        log,
-        "HTTP API started";
-        "listen_address" => listening_socket.to_string(),
-        "api_token_file" => ?api_token_path,
+        listen_address = listening_socket.to_string(),
+        ?api_token_path,
+        "HTTP API started"
     );
 
     Ok((listening_socket, server))

validator_client/http_api/src/remotekeys.rs

@@ -8,11 +8,11 @@ use eth2::lighthouse_vc::std_types::{
     ListRemotekeysResponse, SingleListRemotekeysResponse, Status,
 };
 use initialized_validators::{Error, InitializedValidators};
-use slog::{info, warn, Logger};
 use slot_clock::SlotClock;
 use std::sync::Arc;
 use task_executor::TaskExecutor;
 use tokio::runtime::Handle;
+use tracing::{info, warn};
 use types::{EthSpec, PublicKeyBytes};
 use url::Url;
 use validator_store::ValidatorStore;
@@ -52,12 +52,10 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
     request: ImportRemotekeysRequest,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
-    log: Logger,
 ) -> Result<ImportRemotekeysResponse, Rejection> {
     info!(
-        log,
-        "Importing remotekeys via standard HTTP API";
-        "count" => request.remote_keys.len(),
+        count = request.remote_keys.len(),
+        "Importing remotekeys via standard HTTP API"
     );
     // Import each remotekey. Some remotekeys may fail to be imported, so we record a status for each.
     let mut statuses = Vec::with_capacity(request.remote_keys.len());
@@ -70,10 +68,9 @@ pub fn import<T: SlotClock + 'static, E: EthSpec>(
                 Ok(status) => Status::ok(status),
                 Err(e) => {
                     warn!(
-                        log,
-                        "Error importing keystore, skipped";
-                        "pubkey" => remotekey.pubkey.to_string(),
-                        "error" => ?e,
+                        pubkey = remotekey.pubkey.to_string(),
+                        error = ?e,
+                        "Error importing keystore, skipped"
                     );
                     Status::error(ImportRemotekeyStatus::Error, e)
                 }
@@ -148,12 +145,10 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
     request: DeleteRemotekeysRequest,
     validator_store: Arc<ValidatorStore<T, E>>,
     task_executor: TaskExecutor,
-    log: Logger,
 ) -> Result<DeleteRemotekeysResponse, Rejection> {
     info!(
-        log,
-        "Deleting remotekeys via standard HTTP API";
-        "count" => request.pubkeys.len(),
+        count = request.pubkeys.len(),
+        "Deleting remotekeys via standard HTTP API"
     );
     // Remove from initialized validators.
     let initialized_validators_rwlock = validator_store.initialized_validators();
@@ -171,10 +166,9 @@ pub fn delete<T: SlotClock + 'static, E: EthSpec>(
                 Ok(status) => Status::ok(status),
                 Err(error) => {
                     warn!(
-                        log,
-                        "Error deleting keystore";
-                        "pubkey" => ?pubkey_bytes,
-                        "error" => ?error,
+                        pubkey = ?pubkey_bytes,
+                        ?error,
+                        "Error deleting keystore"
                     );
                     Status::error(DeleteRemotekeyStatus::Error, error)
                 }

validator_client/http_api/src/test_utils.rs

@@ -14,7 +14,6 @@ use eth2::{
 use eth2_keystore::KeystoreBuilder;
 use initialized_validators::key_cache::{KeyCache, CACHE_FILENAME};
 use initialized_validators::{InitializedValidators, OnDecryptFailure};
-use logging::test_logger;
 use parking_lot::RwLock;
 use sensitive_url::SensitiveUrl;
 use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
@@ -70,8 +69,6 @@ impl ApiTester {
     }
 
     pub async fn new_with_http_config(http_config: HttpConfig) -> Self {
-        let log = test_logger();
-
         let validator_dir = tempdir().unwrap();
         let secrets_dir = tempdir().unwrap();
         let token_path = tempdir().unwrap().path().join(PK_FILENAME);
@@ -82,7 +79,6 @@ impl ApiTester {
             validator_defs,
             validator_dir.path().into(),
             Default::default(),
-            log.clone(),
         )
         .await
         .unwrap();
@@ -110,11 +106,10 @@ impl ApiTester {
             slashing_protection,
             Hash256::repeat_byte(42),
             spec.clone(),
-            Some(Arc::new(DoppelgangerService::new(log.clone()))),
+            Some(Arc::new(DoppelgangerService::default())),
             slot_clock.clone(),
             &config,
             test_runtime.task_executor.clone(),
-            log.clone(),
         ));
 
         validator_store
@@ -134,7 +129,6 @@ impl ApiTester {
             graffiti_flag: Some(Graffiti::default()),
             spec,
             config: http_config,
-            log,
             sse_logging_components: None,
             slot_clock,
             _phantom: PhantomData,

validator_client/http_api/src/tests.rs

@@ -18,7 +18,6 @@ use eth2::{
     Error as ApiError,
 };
 use eth2_keystore::KeystoreBuilder;
-use logging::test_logger;
 use parking_lot::RwLock;
 use sensitive_url::SensitiveUrl;
 use slashing_protection::{SlashingDatabase, SLASHING_PROTECTION_FILENAME};
@@ -61,8 +60,6 @@ impl ApiTester {
     }
 
     pub async fn new_with_config(config: ValidatorStoreConfig) -> Self {
-        let log = test_logger();
-
         let validator_dir = tempdir().unwrap();
         let secrets_dir = tempdir().unwrap();
         let token_path = tempdir().unwrap().path().join("api-token.txt");
@@ -73,7 +70,6 @@ impl ApiTester {
             validator_defs,
             validator_dir.path().into(),
             InitializedValidatorsConfig::default(),
-            log.clone(),
         )
         .await
         .unwrap();
@@ -100,11 +96,10 @@ impl ApiTester {
             slashing_protection,
             Hash256::repeat_byte(42),
             spec.clone(),
-            Some(Arc::new(DoppelgangerService::new(log.clone()))),
+            Some(Arc::new(DoppelgangerService::default())),
             slot_clock.clone(),
             &config,
             test_runtime.task_executor.clone(),
-            log.clone(),
         ));
 
         validator_store
@@ -133,7 +128,6 @@ impl ApiTester {
                 http_token_path: token_path,
             },
             sse_logging_components: None,
-            log,
             slot_clock: slot_clock.clone(),
             _phantom: PhantomData,
         });