Fix block processing blowup, upgrade metrics (#500)

* Renamed fork_choice::process_attestation_from_block

* Processing attestation in fork choice

* Retrieving state from store and checking signature

* Looser check on beacon state validity.

* Cleaned up get_attestation_state

* Expanded fork choice api to provide latest validator message.

* Checking if the an attestation contains a latest message

* Correct process_attestation error handling.

* Copy paste error in comment fixed.

* Tidy ancestor iterators

* Getting attestation slot via helper method

* Refactored attestation creation in test utils

* Revert "Refactored attestation creation in test utils"

This reverts commit 4d277fe4239a7194758b18fb5c00dfe0b8231306.

* Integration tests for free attestation processing

* Implicit conflicts resolved.

* formatting

* Do first pass on Grants code

* Add another attestation processing test

* Tidy attestation processing

* Remove old code fragment

* Add non-compiling half finished changes

* Simplify, fix bugs, add tests for chain iters

* Remove attestation processing from op pool

* Fix bug with fork choice, tidy

* Fix overly restrictive check in fork choice.

* Ensure committee cache is build during attn proc

* Ignore unknown blocks at fork choice

* Various minor fixes

* Make fork choice write lock in to read lock

* Remove unused method

* Tidy comments

* Fix attestation prod. target roots change

* Fix compile error in store iters

* Reject any attestation prior to finalization

* Begin metrics refactor

* Move beacon_chain to new metrics structure.

* Make metrics not panic if already defined

* Use global prometheus gather at rest api

* Unify common metric fns into a crate

* Add heavy metering to block processing

* Remove hypen from prometheus metric name

* Add more beacon chain metrics

* Add beacon chain persistence metric

* Prune op pool on finalization

* Add extra prom beacon chain metrics

* Prefix BeaconChain metrics with "beacon_"

* Add more store metrics

* Add basic metrics to libp2p

* Add metrics to HTTP server

* Remove old `http_server` crate

* Update metrics names to be more like standard

* Fix broken beacon chain metrics, add slot clock metrics

* Add lighthouse_metrics gather fn

* Remove http args

* Fix wrong state given to op pool prune

* Make prom metric names more consistent

* Add more metrics, tidy existing metrics

* Fix store block read metrics

* Tidy attestation metrics

* Fix minor PR comments

* Allow travis failures on beta (see desc)

There's a non-backward compatible change in `cargo fmt`. Stable and beta
do not agree.

* Tidy `lighthouse_metrics` docs

* Fix typo

beacon_node/beacon_chain/src/fork_choice.rs

@@ -1,4 +1,4 @@
-use crate::{BeaconChain, BeaconChainTypes};
+use crate::{metrics, BeaconChain, BeaconChainTypes};
 use lmd_ghost::LmdGhost;
 use state_processing::common::get_attesting_indices;
 use std::sync::Arc;
@@ -46,6 +46,8 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
     }
 
     pub fn find_head(&self, chain: &BeaconChain<T>) -> Result<Hash256> {
+        let timer = metrics::start_timer(&metrics::FORK_CHOICE_FIND_HEAD_TIMES);
+
         let start_slot = |epoch: Epoch| epoch.start_slot(T::EthSpec::slots_per_epoch());
 
         // From the specification:
@@ -97,9 +99,14 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
                 .map(|v| v.effective_balance)
         };
 
-        self.backend
+        let result = self
+            .backend
             .find_head(start_block_slot, start_block_root, weight)
-            .map_err(Into::into)
+            .map_err(Into::into);
+
+        metrics::stop_timer(timer);
+
+        result
     }
 
     /// Process all attestations in the given `block`.
@@ -112,6 +119,7 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         block: &BeaconBlock<T::EthSpec>,
         block_root: Hash256,
     ) -> Result<()> {
+        let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_BLOCK_TIMES);
         // Note: we never count the block as a latest message, only attestations.
         //
         // I (Paul H) do not have an explicit reference to this, but I derive it from this
@@ -136,6 +144,8 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         // a block that has the majority of votes applied to it.
         self.backend.process_block(block, block_root)?;
 
+        metrics::stop_timer(timer);
+
         Ok(())
     }
 
@@ -148,6 +158,8 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
         attestation: &Attestation<T::EthSpec>,
         block: &BeaconBlock<T::EthSpec>,
     ) -> Result<()> {
+        let timer = metrics::start_timer(&metrics::FORK_CHOICE_PROCESS_ATTESTATION_TIMES);
+
         let block_hash = attestation.data.beacon_block_root;
 
         // Ignore any attestations to the zero hash.
@@ -175,6 +187,8 @@ impl<T: BeaconChainTypes> ForkChoice<T> {
             }
         }
 
+        metrics::stop_timer(timer);
+
         Ok(())
     }