Testnet stability (#451)

* Change reduced tree for adding weightless node

* Add more comments for reduced tree fork choice

* Small refactor on reduced tree for readability

* Move test_harness forking logic into itself

* Add new `AncestorIter` trait to store

* Add unfinished tests to fork choice

* Make `beacon_state.genesis_block_root` public

* Add failing lmd_ghost fork choice tests

* Extend fork_choice tests, create failing test

* Implement Debug for generic ReducedTree

* Add lazy_static to fork choice tests

* Add verify_integrity fn to reduced tree

* Fix bugs in reduced tree

* Ensure all reduced tree tests verify integrity

* Slightly alter reduce tree test params

* Add (failing) reduced tree test

* Fix bug in fork choice

Iter ancestors was not working well with skip slots

* Put maximum depth for common ancestor search

Ensures that we don't search back past the finalized root.

* Add basic finalization tests for reduced tree

* Change fork choice to use beacon_block_root

Previously it was using target_root, which was wrong

* Change reduced tree for adding weightless node

* Add more comments for reduced tree fork choice

* Small refactor on reduced tree for readability

* Move test_harness forking logic into itself

* Add new `AncestorIter` trait to store

* Add unfinished tests to fork choice

* Make `beacon_state.genesis_block_root` public

* Add failing lmd_ghost fork choice tests

* Extend fork_choice tests, create failing test

* Implement Debug for generic ReducedTree

* Add lazy_static to fork choice tests

* Add verify_integrity fn to reduced tree

* Fix bugs in reduced tree

* Ensure all reduced tree tests verify integrity

* Slightly alter reduce tree test params

* Add (failing) reduced tree test

* Fix bug in fork choice

Iter ancestors was not working well with skip slots

* Put maximum depth for common ancestor search

Ensures that we don't search back past the finalized root.

* Add basic finalization tests for reduced tree

* Add network dir CLI flag

* Simplify "NewSlot" log message

* Rename network-dir CLI flag

* Change fork choice to use beacon_block_root

Previously it was using target_root, which was wrong

* Update db dir size for metrics

* Change slog to use `FullFormat` logging

* Update some comments and log formatting

* Add prom gauge for best block root

* Only add known target blocks to fork choice

* Add finalized and justified root prom metrics

* Add CLI flag for setting log level

* Add logger to beacon chain

* Add debug-level CLI flag to validator

* Allow block processing if fork choice fails

* Create warn log when there's low libp2p peer count

* Minor change to logging

* Make ancestor iter return option

* Disable fork choice test when !debug_assertions

* Fix type, removed code fragment

* Tidy some borrow-checker evading

* Lower reduced tree random test iterations

beacon_node/client/src/beacon_chain_types.rs

@@ -49,7 +49,9 @@ where
     T: BeaconChainTypes<Store = U, EthSpec = V>,
     T::LmdGhost: LmdGhost<U, V>,
 {
-    if let Ok(Some(beacon_chain)) = BeaconChain::from_store(store.clone(), spec.clone()) {
+    if let Ok(Some(beacon_chain)) =
+        BeaconChain::from_store(store.clone(), spec.clone(), log.clone())
+    {
         info!(
             log,
             "Loaded BeaconChain from store";
@@ -78,7 +80,7 @@ where
 
         // Genesis chain
         //TODO: Handle error correctly
-        BeaconChain::from_genesis(store, slot_clock, genesis_state, genesis_block, spec)
+        BeaconChain::from_genesis(store, slot_clock, genesis_state, genesis_block, spec, log)
             .expect("Terminate if beacon chain generation fails")
     }
 }

beacon_node/client/src/lib.rs

@@ -190,29 +190,38 @@ impl<T: BeaconChainTypes> Drop for Client<T> {
 }
 
 fn do_state_catchup<T: BeaconChainTypes>(chain: &Arc<BeaconChain<T>>, log: &slog::Logger) {
-    if let Some(genesis_height) = chain.slots_since_genesis() {
-        let result = chain.catchup_state();
+    // Only attempt to `catchup_state` if we can read the slot clock.
+    if let Some(current_slot) = chain.read_slot_clock() {
+        let state_catchup_result = chain.catchup_state();
+
+        let best_slot = chain.head().beacon_block.slot;
+        let latest_block_root = chain.head().beacon_block_root;
 
         let common = o!(
-            "best_slot" => chain.head().beacon_block.slot,
-            "latest_block_root" => format!("{}", chain.head().beacon_block_root),
-            "wall_clock_slot" => chain.read_slot_clock().unwrap(),
-            "state_slot" => chain.head().beacon_state.slot,
-            "slots_since_genesis" => genesis_height,
+            "skip_slots" => current_slot.saturating_sub(best_slot),
+            "best_block_root" => format!("{}", latest_block_root),
+            "best_block_slot" => best_slot,
+            "slot" => current_slot,
         );
 
-        match result {
-            Ok(_) => info!(
+        if let Err(e) = state_catchup_result {
+            error!(
                 log,
-                "NewSlot";
-                common
-            ),
-            Err(e) => error!(
-                log,
-                "StateCatchupFailed";
+                "State catchup failed";
                 "error" => format!("{:?}", e),
                 common
-            ),
-        };
-    }
+            )
+        } else {
+            info!(
+                log,
+                "Slot start";
+                common
+            )
+        }
+    } else {
+        error!(
+            log,
+            "Beacon chain running whilst slot clock is unavailable."
+        );
+    };
 }

beacon_node/client/src/notifier.rs

@@ -2,7 +2,7 @@ use crate::Client;
 use beacon_chain::BeaconChainTypes;
 use exit_future::Exit;
 use futures::{Future, Stream};
-use slog::{debug, o};
+use slog::{debug, o, warn};
 use std::time::{Duration, Instant};
 use tokio::runtime::TaskExecutor;
 use tokio::timer::Interval;
@@ -10,6 +10,9 @@ use tokio::timer::Interval;
 /// The interval between heartbeat events.
 pub const HEARTBEAT_INTERVAL_SECONDS: u64 = 15;
 
+/// Create a warning log whenever the peer count is at or below this value.
+pub const WARN_PEER_COUNT: usize = 1;
+
 /// Spawns a thread that can be used to run code periodically, on `HEARTBEAT_INTERVAL_SECONDS`
 /// durations.
 ///
@@ -30,9 +33,16 @@ pub fn run<T: BeaconChainTypes + Send + Sync + 'static>(
     let libp2p = client.network.libp2p_service();
 
     let heartbeat = move |_| {
-        // Notify the number of connected nodes
-        // Panic if libp2p is poisoned
-        debug!(log, ""; "Connected Peers" => libp2p.lock().swarm.connected_peers());
+        // Number of libp2p (not discv5) peers connected.
+        //
+        // Panics if libp2p is poisoned.
+        let connected_peer_count = libp2p.lock().swarm.connected_peers();
+
+        debug!(log, "libp2p"; "peer_count" => connected_peer_count);
+
+        if connected_peer_count <= WARN_PEER_COUNT {
+            warn!(log, "Low libp2p peer count"; "peer_count" => connected_peer_count);
+        }
 
         Ok(())
     };