Add proto_array fork choice (#804)

* Start implementing proto_array

* Add progress

* Add unfinished progress

* Add further progress

* Add progress

* Add tree filtering

* Add half-finished modifications

* Add refactored version

* Tidy, add incomplete LmdGhost impl

* Move impls in LmdGhost trait def

* Remove old reduced_tree fork choice

* Combine two functions in to `compute_deltas`

* Start testing

* Add more compute_deltas tests

* Add fork choice testing

* Add more fork choice testing

* Add more fork choice tests

* Add more testing to proto-array

* Remove old tests

* Modify tests

* Add more tests

* Add more testing

* Add comments and fixes

* Re-organise crate

* Tidy, finish pruning tests

* Add ssz encoding, other pub fns

* Rename lmd_ghost > proto_array_fork_choice

* Integrate proto_array into lighthouse

* Add first pass at fixing filter

* Clean out old comments

* Add more comments

* Attempt to fix prune error

* Adjust TODO

* Fix test compile errors

* Add extra justification change check

* Update cargo.lock

* Fix fork choice test compile errors

* Most remove ffg_update_required

* Fix bug with epoch of attestation votes

* Start adding new test format

* Make fork choice tests declarative

* Create test def concept

* Move test defs into crate

* Add binary, re-org crate

* Shuffle files

* Start adding ffg tests

* Add more fork choice tests

* Add fork choice JSON dumping

* Add more detail to best node error

* Ensure fin+just checkpoints from from same block

* Rename JustificationManager

* Move checkpoint manager into own file

* Tidy

* Add targetted logging for sneaky sync bug

* Fix justified balances bug

* Add cache metrics

* Add metrics for log levels

* Fix bug in checkpoint manager

* Fix compile error in fork choice tests

* Ignore duplicate blocks in fork choice

* Add block to fock choice before db

* Rename on_new_block fn

* Fix spec inconsistency in `CheckpointManager`

* Remove BlockRootTree

* Remove old reduced_tree code fragment

* Add API endpoint for fork choice

* Add more ffg tests

* Remove block_root_tree reminents

* Ensure effective balances are used

* Remove old debugging code, fix API fault

* Add check to ensure parent block is in fork choice

* Update readme dates

* Fix readme

* Tidy checkpoint manager

* Remove fork choice yaml files from repo

* Remove fork choice yaml from repo

* General tidy

* Address majority of Michael's comments

* Tidy bin/lib business

* Remove dangling file

* Undo changes for rpc/handler from master

* Revert "Undo changes for rpc/handler from master"

This reverts commit 876edff0e4.

Co-authored-by: Age Manning <Age@AgeManning.com>

eth2/utils/logging/Cargo.toml

@@ -7,3 +7,5 @@ edition = "2018"
 [dependencies]
 slog = "2.5.2"
 slog-term = "2.4.2"
+lighthouse_metrics = { path = "../lighthouse_metrics" }
+lazy_static = "1.4.0"

eth2/utils/logging/src/lib.rs

@@ -1,7 +1,24 @@
+#[macro_use]
+extern crate lazy_static;
+
+use lighthouse_metrics::{
+    inc_counter, try_create_int_counter, IntCounter, Result as MetricsResult,
+};
 use std::io::{Result, Write};
 
 pub const MAX_MESSAGE_WIDTH: usize = 40;
 
+lazy_static! {
+    pub static ref INFOS_TOTAL: MetricsResult<IntCounter> =
+        try_create_int_counter("info_total", "Count of infos logged");
+    pub static ref WARNS_TOTAL: MetricsResult<IntCounter> =
+        try_create_int_counter("warn_total", "Count of warns logged");
+    pub static ref ERRORS_TOTAL: MetricsResult<IntCounter> =
+        try_create_int_counter("error_total", "Count of errors logged");
+    pub static ref CRITS_TOTAL: MetricsResult<IntCounter> =
+        try_create_int_counter("crit_total", "Count of crits logged");
+}
+
 pub struct AlignedTermDecorator {
     wrapped: slog_term::TermDecorator,
     message_width: usize,
@@ -19,14 +36,22 @@ impl AlignedTermDecorator {
 impl slog_term::Decorator for AlignedTermDecorator {
     fn with_record<F>(
         &self,
-        _record: &slog::Record,
+        record: &slog::Record,
         _logger_values: &slog::OwnedKVList,
         f: F,
     ) -> Result<()>
     where
         F: FnOnce(&mut dyn slog_term::RecordDecorator) -> std::io::Result<()>,
     {
-        self.wrapped.with_record(_record, _logger_values, |deco| {
+        match record.level() {
+            slog::Level::Info => inc_counter(&INFOS_TOTAL),
+            slog::Level::Warning => inc_counter(&WARNS_TOTAL),
+            slog::Level::Error => inc_counter(&ERRORS_TOTAL),
+            slog::Level::Critical => inc_counter(&CRITS_TOTAL),
+            _ => (),
+        }
+
+        self.wrapped.with_record(record, _logger_values, |deco| {
             f(&mut AlignedRecordDecorator::new(deco, self.message_width))
         })
     }

eth2/utils/remote_beacon_node/Cargo.toml

@@ -17,3 +17,4 @@ hex = "0.3"
 eth2_ssz = { path = "../../../eth2/utils/ssz" }
 serde_json = "^1.0"
 eth2_config = { path = "../../../eth2/utils/eth2_config" }
+proto_array_fork_choice = { path = "../../../eth2/proto_array_fork_choice" }

eth2/utils/remote_beacon_node/src/lib.rs

@@ -5,6 +5,7 @@
 
 use eth2_config::Eth2Config;
 use futures::{future, Future, IntoFuture};
+use proto_array_fork_choice::core::ProtoArray;
 use reqwest::{
     r#async::{Client, ClientBuilder, Response},
     StatusCode,
@@ -101,6 +102,10 @@ impl<E: EthSpec> HttpClient<E> {
         Node(self.clone())
     }
 
+    pub fn advanced(&self) -> Advanced<E> {
+        Advanced(self.clone())
+    }
+
     fn url(&self, path: &str) -> Result<Url, Error> {
         self.url.join(path).map_err(|e| e.into())
     }
@@ -536,6 +541,27 @@ impl<E: EthSpec> Node<E> {
     }
 }
 
+/// Provides the functions on the `/advanced` endpoint of the node.
+#[derive(Clone)]
+pub struct Advanced<E>(HttpClient<E>);
+
+impl<E: EthSpec> Advanced<E> {
+    fn url(&self, path: &str) -> Result<Url, Error> {
+        self.0
+            .url("advanced/")
+            .and_then(move |url| url.join(path).map_err(Error::from))
+            .map_err(Into::into)
+    }
+
+    /// Gets the core `ProtoArray` struct from the node.
+    pub fn get_fork_choice(&self) -> impl Future<Item = ProtoArray, Error = Error> {
+        let client = self.0.clone();
+        self.url("fork_choice")
+            .into_future()
+            .and_then(move |url| client.json_get(url, vec![]))
+    }
+}
+
 #[derive(Deserialize)]
 #[serde(bound = "T: EthSpec")]
 pub struct BlockResponse<T: EthSpec> {