Integrate tracing (#6339)

Tracing Integration
- [reference](5bbf1859e9/projects/project-ideas.md (L297))


  - [x] replace slog & log with tracing throughout the codebase
- [x] implement custom crit log
- [x] make relevant changes in the formatter
- [x] replace sloggers
- [x] re-write SSE logging components

cc: @macladson @eserilev

common/logging/src/sse_logging_components.rs

@@ -1,46 +1,108 @@
 //! This module provides an implementation of `slog::Drain` that optionally writes to a channel if
 //! there are subscribers to a HTTP SSE stream.
 
-use crate::async_record::AsyncRecord;
-use slog::{Drain, OwnedKVList, Record};
-use std::panic::AssertUnwindSafe;
+use serde_json::json;
+use serde_json::Value;
 use std::sync::Arc;
 use tokio::sync::broadcast::Sender;
+use tracing::field::{Field, Visit};
+use tracing::{Event, Subscriber};
+use tracing_subscriber::layer::{Context, Layer};
 
 /// Default log level for SSE Events.
 // NOTE: Made this a constant. Debug level seems to be pretty intense. Can make this
 // configurable later if needed.
-const LOG_LEVEL: slog::Level = slog::Level::Info;
+const LOG_LEVEL: tracing::Level = tracing::Level::INFO;
 
 /// The components required in the HTTP API task to receive logged events.
 #[derive(Clone)]
 pub struct SSELoggingComponents {
     /// The channel to receive events from.
-    pub sender: Arc<AssertUnwindSafe<Sender<AsyncRecord>>>,
+    pub sender: Arc<Sender<Arc<Value>>>,
 }
 
 impl SSELoggingComponents {
-    /// Create a new SSE drain.
     pub fn new(channel_size: usize) -> Self {
         let (sender, _receiver) = tokio::sync::broadcast::channel(channel_size);
 
-        let sender = Arc::new(AssertUnwindSafe(sender));
-        SSELoggingComponents { sender }
+        SSELoggingComponents {
+            sender: Arc::new(sender),
+        }
     }
 }
 
-impl Drain for SSELoggingComponents {
-    type Ok = ();
-    type Err = &'static str;
+impl<S: Subscriber> Layer<S> for SSELoggingComponents {
+    fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) {
+        if *event.metadata().level() > LOG_LEVEL {
+            return;
+        }
 
-    fn log(&self, record: &Record, logger_values: &OwnedKVList) -> Result<Self::Ok, Self::Err> {
-        if record.level().is_at_least(LOG_LEVEL) {
-            // Attempt to send the logs
-            match self.sender.send(AsyncRecord::from(record, logger_values)) {
-                Ok(_num_sent) => {} // Everything got sent
-                Err(_err) => {}     // There are no subscribers, do nothing
+        let mut visitor = TracingEventVisitor::new();
+        event.record(&mut visitor);
+        let mut log_entry = visitor.finish(event.metadata());
+
+        if let Some(error_type) = log_entry
+            .get("fields")
+            .and_then(|fields| fields.get("error_type"))
+            .and_then(|val| val.as_str())
+        {
+            if error_type.eq_ignore_ascii_case("crit") {
+                log_entry["level"] = json!("CRIT");
+
+                if let Some(Value::Object(ref mut map)) = log_entry.get_mut("fields") {
+                    map.remove("error_type");
+                }
             }
         }
-        Ok(())
+
+        let _ = self.sender.send(Arc::new(log_entry));
+    }
+}
+struct TracingEventVisitor {
+    fields: serde_json::Map<String, Value>,
+}
+
+impl TracingEventVisitor {
+    fn new() -> Self {
+        TracingEventVisitor {
+            fields: serde_json::Map::new(),
+        }
+    }
+
+    fn finish(self, metadata: &tracing::Metadata<'_>) -> Value {
+        let mut log_entry = serde_json::Map::new();
+        log_entry.insert(
+            "time".to_string(),
+            json!(chrono::Local::now()
+                .format("%b %d %H:%M:%S%.3f")
+                .to_string()),
+        );
+        log_entry.insert("level".to_string(), json!(metadata.level().to_string()));
+        log_entry.insert("target".to_string(), json!(metadata.target()));
+        log_entry.insert("fields".to_string(), Value::Object(self.fields));
+        Value::Object(log_entry)
+    }
+}
+
+impl Visit for TracingEventVisitor {
+    fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) {
+        self.fields
+            .insert(field.name().to_string(), json!(format!("{:?}", value)));
+    }
+
+    fn record_str(&mut self, field: &Field, value: &str) {
+        self.fields.insert(field.name().to_string(), json!(value));
+    }
+
+    fn record_i64(&mut self, field: &Field, value: i64) {
+        self.fields.insert(field.name().to_string(), json!(value));
+    }
+
+    fn record_u64(&mut self, field: &Field, value: u64) {
+        self.fields.insert(field.name().to_string(), json!(value));
+    }
+
+    fn record_bool(&mut self, field: &Field, value: bool) {
+        self.fields.insert(field.name().to_string(), json!(value));
     }
 }