Fix block processing blowup, upgrade metrics (#500)

* Renamed fork_choice::process_attestation_from_block

* Processing attestation in fork choice

* Retrieving state from store and checking signature

* Looser check on beacon state validity.

* Cleaned up get_attestation_state

* Expanded fork choice api to provide latest validator message.

* Checking if the an attestation contains a latest message

* Correct process_attestation error handling.

* Copy paste error in comment fixed.

* Tidy ancestor iterators

* Getting attestation slot via helper method

* Refactored attestation creation in test utils

* Revert "Refactored attestation creation in test utils"

This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306.

* Integration tests for free attestation processing

* Implicit conflicts resolved.

* formatting

* Do first pass on Grants code

* Add another attestation processing test

* Tidy attestation processing

* Remove old code fragment

* Add non-compiling half finished changes

* Simplify, fix bugs, add tests for chain iters

* Remove attestation processing from op pool

* Fix bug with fork choice, tidy

* Fix overly restrictive check in fork choice.

* Ensure committee cache is build during attn proc

* Ignore unknown blocks at fork choice

* Various minor fixes

* Make fork choice write lock in to read lock

* Remove unused method

* Tidy comments

* Fix attestation prod. target roots change

* Fix compile error in store iters

* Reject any attestation prior to finalization

* Begin metrics refactor

* Move beacon_chain to new metrics structure.

* Make metrics not panic if already defined

* Use global prometheus gather at rest api

* Unify common metric fns into a crate

* Add heavy metering to block processing

* Remove hypen from prometheus metric name

* Add more beacon chain metrics

* Add beacon chain persistence metric

* Prune op pool on finalization

* Add extra prom beacon chain metrics

* Prefix BeaconChain metrics with "beacon_"

* Add more store metrics

* Add basic metrics to libp2p

* Add metrics to HTTP server

* Remove old `http_server` crate

* Update metrics names to be more like standard

* Fix broken beacon chain metrics, add slot clock metrics

* Add lighthouse_metrics gather fn

* Remove http args

* Fix wrong state given to op pool prune

* Make prom metric names more consistent

* Add more metrics, tidy existing metrics

* Fix store block read metrics

* Tidy attestation metrics

* Fix minor PR comments

* Allow travis failures on beta (see desc)

There's a non-backward compatible change in `cargo fmt`. Stable and beta
do not agree.

* Tidy `lighthouse_metrics` docs

* Fix typo

beacon_node/rest_api/src/config.rs

@@ -18,7 +18,7 @@ impl Default for Config {
         Config {
             enabled: true, // rest_api enabled by default
             listen_address: Ipv4Addr::new(127, 0, 0, 1),
-            port: 1248,
+            port: 5052,
         }
     }
 }

beacon_node/rest_api/src/lib.rs

@@ -1,8 +1,10 @@
-extern crate futures;
-extern crate hyper;
+#[macro_use]
+extern crate lazy_static;
+
 mod beacon;
 mod config;
 mod helpers;
+mod metrics;
 mod node;
 mod url_query;
 
@@ -12,6 +14,8 @@ use hyper::rt::Future;
 use hyper::service::service_fn_ok;
 use hyper::{Body, Method, Response, Server, StatusCode};
 use slog::{info, o, warn};
+use std::ops::Deref;
+use std::path::PathBuf;
 use std::sync::Arc;
 use tokio::runtime::TaskExecutor;
 use url_query::UrlQuery;
@@ -67,6 +71,7 @@ pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
     config: &ApiConfig,
     executor: &TaskExecutor,
     beacon_chain: Arc<BeaconChain<T>>,
+    db_path: PathBuf,
     log: &slog::Logger,
 ) -> Result<exit_future::Signal, hyper::Error> {
     let log = log.new(o!("Service" => "Api"));
@@ -80,6 +85,8 @@ pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
         Ok(())
     });
 
+    let db_path = DBPath(db_path);
+
     // Get the address to bind to
     let bind_addr = (config.listen_address, config.port).into();
 
@@ -90,12 +97,17 @@ pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
     let service = move || {
         let log = server_log.clone();
         let beacon_chain = server_bc.clone();
+        let db_path = db_path.clone();
 
         // Create a simple handler for the router, inject our stateful objects into the request.
         service_fn_ok(move |mut req| {
+            metrics::inc_counter(&metrics::REQUEST_COUNT);
+            let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME);
+
             req.extensions_mut().insert::<slog::Logger>(log.clone());
             req.extensions_mut()
                 .insert::<Arc<BeaconChain<T>>>(beacon_chain.clone());
+            req.extensions_mut().insert::<DBPath>(db_path.clone());
 
             let path = req.uri().path().to_string();
 
@@ -103,14 +115,16 @@ pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
             let result = match (req.method(), path.as_ref()) {
                 (&Method::GET, "/beacon/state") => beacon::get_state::<T>(req),
                 (&Method::GET, "/beacon/state_root") => beacon::get_state_root::<T>(req),
+                (&Method::GET, "/metrics") => metrics::get_prometheus::<T>(req),
                 (&Method::GET, "/node/version") => node::get_version(req),
                 (&Method::GET, "/node/genesis_time") => node::get_genesis_time::<T>(req),
                 _ => Err(ApiError::MethodNotAllowed(path.clone())),
             };
 
-            match result {
+            let response = match result {
                 // Return the `hyper::Response`.
                 Ok(response) => {
+                    metrics::inc_counter(&metrics::SUCCESS_COUNT);
                     slog::debug!(log, "Request successful: {:?}", path);
                     response
                 }
@@ -119,7 +133,11 @@ pub fn start_server<T: BeaconChainTypes + Clone + 'static>(
                     slog::debug!(log, "Request failure: {:?}", path);
                     e.into()
                 }
-            }
+            };
+
+            metrics::stop_timer(timer);
+
+            response
         })
     };
 
@@ -152,3 +170,14 @@ fn success_response(body: Body) -> Response<Body> {
         .body(body)
         .expect("We should always be able to make response from the success body.")
 }
+
+#[derive(Clone)]
+pub struct DBPath(PathBuf);
+
+impl Deref for DBPath {
+    type Target = PathBuf;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

beacon_node/rest_api/src/metrics.rs

@@ -0,0 +1,69 @@
+use crate::{success_response, ApiError, ApiResult, DBPath};
+use beacon_chain::{BeaconChain, BeaconChainTypes};
+use hyper::{Body, Request};
+use prometheus::{Encoder, TextEncoder};
+use std::sync::Arc;
+
+pub use lighthouse_metrics::*;
+
+lazy_static! {
+    pub static ref REQUEST_RESPONSE_TIME: Result<Histogram> = try_create_histogram(
+        "http_server_request_duration_seconds",
+        "Time taken to build a response to a HTTP request"
+    );
+    pub static ref REQUEST_COUNT: Result<IntCounter> = try_create_int_counter(
+        "http_server_request_total",
+        "Total count of HTTP requests received"
+    );
+    pub static ref SUCCESS_COUNT: Result<IntCounter> = try_create_int_counter(
+        "http_server_success_total",
+        "Total count of HTTP 200 responses sent"
+    );
+}
+
+/// Returns the full set of Prometheus metrics for the Beacon Node application.
+///
+/// # Note
+///
+/// This is a HTTP handler method.
+pub fn get_prometheus<T: BeaconChainTypes + 'static>(req: Request<Body>) -> ApiResult {
+    let mut buffer = vec![];
+    let encoder = TextEncoder::new();
+
+    let beacon_chain = req
+        .extensions()
+        .get::<Arc<BeaconChain<T>>>()
+        .ok_or_else(|| ApiError::ServerError("Beacon chain extension missing".to_string()))?;
+    let db_path = req
+        .extensions()
+        .get::<DBPath>()
+        .ok_or_else(|| ApiError::ServerError("DBPath extension missing".to_string()))?;
+
+    // There are two categories of metrics:
+    //
+    // - Dynamically updated: things like histograms and event counters that are updated on the
+    // fly.
+    // - Statically updated: things which are only updated at the time of the scrape (used where we
+    // can avoid cluttering up code with metrics calls).
+    //
+    // The `lighthouse_metrics` crate has a `DEFAULT_REGISTRY` global singleton (via `lazy_static`)
+    // which keeps the state of all the metrics. Dynamically updated things will already be
+    // up-to-date in the registry (because they update themselves) however statically updated
+    // things need to be "scraped".
+    //
+    // We proceed by, first updating all the static metrics using `scrape_for_metrics(..)`. Then,
+    // using `lighthouse_metrics::gather(..)` to collect the global `DEFAULT_REGISTRY` metrics into
+    // a string that can be returned via HTTP.
+
+    slot_clock::scrape_for_metrics::<T::EthSpec, T::SlotClock>(&beacon_chain.slot_clock);
+    store::scrape_for_metrics(&db_path);
+    beacon_chain::scrape_for_metrics(&beacon_chain);
+
+    encoder
+        .encode(&lighthouse_metrics::gather(), &mut buffer)
+        .unwrap();
+
+    String::from_utf8(buffer)
+        .map(|string| success_response(Body::from(string)))
+        .map_err(|e| ApiError::ServerError(format!("Failed to encode prometheus info: {:?}", e)))
+}