Testnet stability (#451)

* Change reduced tree for adding weightless node

* Add more comments for reduced tree fork choice

* Small refactor on reduced tree for readability

* Move test_harness forking logic into itself

* Add new `AncestorIter` trait to store

* Add unfinished tests to fork choice

* Make `beacon_state.genesis_block_root` public

* Add failing lmd_ghost fork choice tests

* Extend fork_choice tests, create failing test

* Implement Debug for generic ReducedTree

* Add lazy_static to fork choice tests

* Add verify_integrity fn to reduced tree

* Fix bugs in reduced tree

* Ensure all reduced tree tests verify integrity

* Slightly alter reduce tree test params

* Add (failing) reduced tree test

* Fix bug in fork choice

Iter ancestors was not working well with skip slots

* Put maximum depth for common ancestor search

Ensures that we don't search back past the finalized root.

* Add basic finalization tests for reduced tree

* Change fork choice to use beacon_block_root

Previously it was using target_root, which was wrong

* Change reduced tree for adding weightless node

* Add more comments for reduced tree fork choice

* Small refactor on reduced tree for readability

* Move test_harness forking logic into itself

* Add new `AncestorIter` trait to store

* Add unfinished tests to fork choice

* Make `beacon_state.genesis_block_root` public

* Add failing lmd_ghost fork choice tests

* Extend fork_choice tests, create failing test

* Implement Debug for generic ReducedTree

* Add lazy_static to fork choice tests

* Add verify_integrity fn to reduced tree

* Fix bugs in reduced tree

* Ensure all reduced tree tests verify integrity

* Slightly alter reduce tree test params

* Add (failing) reduced tree test

* Fix bug in fork choice

Iter ancestors was not working well with skip slots

* Put maximum depth for common ancestor search

Ensures that we don't search back past the finalized root.

* Add basic finalization tests for reduced tree

* Add network dir CLI flag

* Simplify "NewSlot" log message

* Rename network-dir CLI flag

* Change fork choice to use beacon_block_root

Previously it was using target_root, which was wrong

* Update db dir size for metrics

* Change slog to use `FullFormat` logging

* Update some comments and log formatting

* Add prom gauge for best block root

* Only add known target blocks to fork choice

* Add finalized and justified root prom metrics

* Add CLI flag for setting log level

* Add logger to beacon chain

* Add debug-level CLI flag to validator

* Allow block processing if fork choice fails

* Create warn log when there's low libp2p peer count

* Minor change to logging

* Make ancestor iter return option

* Disable fork choice test when !debug_assertions

* Fix type, removed code fragment

* Tidy some borrow-checker evading

* Lower reduced tree random test iterations

beacon_node/http_server/src/metrics/local_metrics.rs

@@ -1,7 +1,7 @@
 use beacon_chain::{BeaconChain, BeaconChainTypes};
 use prometheus::{IntGauge, Opts, Registry};
 use slot_clock::SlotClock;
-use std::fs::File;
+use std::fs;
 use std::path::PathBuf;
 use types::{EthSpec, Slot};
 
@@ -13,6 +13,9 @@ pub struct LocalMetrics {
     present_slot: IntGauge,
     present_epoch: IntGauge,
     best_slot: IntGauge,
+    best_beacon_block_root: IntGauge,
+    justified_beacon_block_root: IntGauge,
+    finalized_beacon_block_root: IntGauge,
     validator_count: IntGauge,
     justified_epoch: IntGauge,
     finalized_epoch: IntGauge,
@@ -36,6 +39,24 @@ impl LocalMetrics {
                 let opts = Opts::new("best_slot", "slot_of_block_at_chain_head");
                 IntGauge::with_opts(opts)?
             },
+            best_beacon_block_root: {
+                let opts = Opts::new("best_beacon_block_root", "root_of_block_at_chain_head");
+                IntGauge::with_opts(opts)?
+            },
+            justified_beacon_block_root: {
+                let opts = Opts::new(
+                    "justified_beacon_block_root",
+                    "root_of_block_at_justified_head",
+                );
+                IntGauge::with_opts(opts)?
+            },
+            finalized_beacon_block_root: {
+                let opts = Opts::new(
+                    "finalized_beacon_block_root",
+                    "root_of_block_at_finalized_head",
+                );
+                IntGauge::with_opts(opts)?
+            },
             validator_count: {
                 let opts = Opts::new("validator_count", "number_of_validators");
                 IntGauge::with_opts(opts)?
@@ -64,6 +85,9 @@ impl LocalMetrics {
         registry.register(Box::new(self.present_slot.clone()))?;
         registry.register(Box::new(self.present_epoch.clone()))?;
         registry.register(Box::new(self.best_slot.clone()))?;
+        registry.register(Box::new(self.best_beacon_block_root.clone()))?;
+        registry.register(Box::new(self.justified_beacon_block_root.clone()))?;
+        registry.register(Box::new(self.finalized_beacon_block_root.clone()))?;
         registry.register(Box::new(self.validator_count.clone()))?;
         registry.register(Box::new(self.finalized_epoch.clone()))?;
         registry.register(Box::new(self.justified_epoch.clone()))?;
@@ -87,6 +111,22 @@ impl LocalMetrics {
             .set(present_slot.epoch(T::EthSpec::slots_per_epoch()).as_u64() as i64);
 
         self.best_slot.set(state.slot.as_u64() as i64);
+        self.best_beacon_block_root
+            .set(beacon_chain.head().beacon_block_root.to_low_u64_le() as i64);
+        self.justified_beacon_block_root.set(
+            beacon_chain
+                .head()
+                .beacon_state
+                .current_justified_root
+                .to_low_u64_le() as i64,
+        );
+        self.finalized_beacon_block_root.set(
+            beacon_chain
+                .head()
+                .beacon_state
+                .finalized_root
+                .to_low_u64_le() as i64,
+        );
         self.validator_count
             .set(state.validator_registry.len() as i64);
         self.justified_epoch
@@ -97,10 +137,17 @@ impl LocalMetrics {
             self.validator_balances_sum
                 .set(state.balances.iter().sum::<u64>() as i64);
         }
-        let db_size = File::open(db_path)
-            .and_then(|f| f.metadata())
-            .and_then(|m| Ok(m.len()))
-            .unwrap_or(0);
+        let db_size = if let Ok(iter) = fs::read_dir(db_path) {
+            iter.filter_map(Result::ok)
+                .map(size_of_dir_entry)
+                .fold(0_u64, |sum, val| sum + val)
+        } else {
+            0
+        };
         self.database_size.set(db_size as i64);
     }
 }
+
+fn size_of_dir_entry(dir: fs::DirEntry) -> u64 {
+    dir.metadata().map(|m| m.len()).unwrap_or(0)
+}