Alternative (to BeaconChainHarness) BeaconChain testing API (#1380)

The PR:

* Adds the ability to generate a crucial test scenario that isn't possible with `BeaconChainHarness` (i.e. two blocks occupying the same slot; previously forks necessitated skipping slots):

![image](https://user-images.githubusercontent.com/165678/88195404-4bce3580-cc40-11ea-8c08-b48d2e1d5959.png)

* New testing API: Instead of repeatedly calling add_block(), you generate a sorted `Vec<Slot>` and leave it up to the framework to generate blocks at those slots.
* Jumping backwards to an earlier epoch is a hard error, so that tests necessarily generate blocks in a epoch-by-epoch manner.
* Configures the test logger so that output is printed on the console in case a test fails.  The logger also plays well with `--nocapture`, contrary to the existing testing framework
* Rewrites existing fork pruning tests to use the new API
* Adds a tests that triggers finalization at a non epoch boundary slot
* Renamed `BeaconChainYoke` to `BeaconChainTestingRig` because the former has been too confusing
* Fixed multiple tests (e.g. `block_production_different_shuffling_long`, `delete_blocks_and_states`, `shuffling_compatible_simple_fork`) that relied on a weird (and accidental) feature of the old `BeaconChainHarness` that attestations aren't produced for epochs earlier than the current one, thus masking potential bugs in test cases.

Co-authored-by: Michael Sproul <michael@sigmaprime.io>

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -1978,7 +1978,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             self.head_tracker.clone(),
             old_finalized_checkpoint,
             new_finalized_checkpoint,
-        );
+        )?;
 
         let _ = self.event_handler.register(EventKind::BeaconFinalization {
             epoch: new_finalized_checkpoint.epoch,
@@ -2070,10 +2070,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             .beacon_block_root;
         let mut visited: HashSet<Hash256> = HashSet::new();
         let mut finalized_blocks: HashSet<Hash256> = HashSet::new();
+        let mut justified_blocks: HashSet<Hash256> = HashSet::new();
 
         let genesis_block_hash = Hash256::zero();
         writeln!(output, "digraph beacon {{").unwrap();
-        writeln!(output, "\t_{:?}[label=\"genesis\"];", genesis_block_hash).unwrap();
+        writeln!(output, "\t_{:?}[label=\"zero\"];", genesis_block_hash).unwrap();
 
         // Canonical head needs to be processed first as otherwise finalized blocks aren't detected
         // properly.
@@ -2104,6 +2105,8 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         .unwrap()
                         .unwrap();
                     finalized_blocks.insert(state.finalized_checkpoint.root);
+                    justified_blocks.insert(state.current_justified_checkpoint.root);
+                    justified_blocks.insert(state.previous_justified_checkpoint.root);
                 }
 
                 if block_hash == canonical_head_hash {
@@ -2124,6 +2127,15 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                         signed_beacon_block.slot()
                     )
                     .unwrap();
+                } else if justified_blocks.contains(&block_hash) {
+                    writeln!(
+                        output,
+                        "\t_{:?}[label=\"{} ({})\" shape=cds];",
+                        block_hash,
+                        block_hash,
+                        signed_beacon_block.slot()
+                    )
+                    .unwrap();
                 } else {
                     writeln!(
                         output,
@@ -2153,6 +2165,11 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let mut file = std::fs::File::create(file_name).unwrap();
         self.dump_as_dot(&mut file);
     }
+
+    // Should be used in tests only
+    pub fn set_graffiti(&mut self, graffiti: Graffiti) {
+        self.graffiti = graffiti;
+    }
 }
 
 impl<T: BeaconChainTypes> Drop for BeaconChain<T> {

beacon_node/beacon_chain/src/lib.rs

@@ -2,6 +2,10 @@
 #[macro_use]
 extern crate lazy_static;
 
+#[macro_use]
+extern crate slog;
+extern crate slog_term;
+
 pub mod attestation_verification;
 mod beacon_chain;
 mod beacon_fork_choice_store;

beacon_node/beacon_chain/src/migrate.rs

@@ -1,13 +1,13 @@
 use crate::errors::BeaconChainError;
 use crate::head_tracker::HeadTracker;
 use parking_lot::Mutex;
-use slog::{debug, error, warn, Logger};
+use slog::{debug, warn, Logger};
 use std::collections::{HashMap, HashSet};
 use std::mem;
 use std::sync::mpsc;
 use std::sync::Arc;
 use std::thread;
-use store::hot_cold_store::{process_finalization, HotColdDBError};
+use store::hot_cold_store::{migrate_database, HotColdDBError};
 use store::iter::RootsIterator;
 use store::{Error, ItemStore, StoreOp};
 pub use store::{HotColdDB, MemoryStore};
@@ -43,7 +43,8 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
         _head_tracker: Arc<HeadTracker>,
         _old_finalized_checkpoint: Checkpoint,
         _new_finalized_checkpoint: Checkpoint,
-    ) {
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
     }
 
     /// Traverses live heads and prunes blocks and states of chains that we know can't be built
@@ -237,6 +238,7 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
                     .map(|(slot, state_hash)| StoreOp::DeleteState(state_hash, slot)),
             )
             .collect();
+
         store.do_atomically(batch)?;
         for head_hash in abandoned_heads.into_iter() {
             head_tracker.remove_head(head_hash);
@@ -252,6 +254,17 @@ pub trait Migrate<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>>:
 pub struct NullMigrator;
 
 impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold> for NullMigrator {
+    fn process_finalization(
+        &self,
+        _finalized_state_root: BeaconStateHash,
+        _new_finalized_state: BeaconState<E>,
+        _head_tracker: Arc<HeadTracker>,
+        _old_finalized_checkpoint: Checkpoint,
+        _new_finalized_checkpoint: Checkpoint,
+    ) -> Result<(), BeaconChainError> {
+        Ok(())
+    }
+
     fn new(_: Arc<HotColdDB<E, Hot, Cold>>, _: Logger) -> Self {
         NullMigrator
     }
@@ -279,8 +292,8 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         head_tracker: Arc<HeadTracker>,
         old_finalized_checkpoint: Checkpoint,
         new_finalized_checkpoint: Checkpoint,
-    ) {
-        if let Err(e) = Self::prune_abandoned_forks(
+    ) -> Result<(), BeaconChainError> {
+        Self::prune_abandoned_forks(
             self.db.clone(),
             head_tracker,
             finalized_state_root,
@@ -288,16 +301,23 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
             old_finalized_checkpoint,
             new_finalized_checkpoint,
             &self.log,
-        ) {
-            error!(&self.log, "Pruning error"; "error" => format!("{:?}", e));
-        }
+        )?;
 
-        if let Err(e) = process_finalization(
+        match migrate_database(
             self.db.clone(),
             finalized_state_root.into(),
             &new_finalized_state,
         ) {
-            error!(&self.log, "Migration error"; "error" => format!("{:?}", e));
+            Ok(()) => Ok(()),
+            Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
+                debug!(
+                    self.log,
+                    "Database migration postponed, unaligned finalized block";
+                    "slot" => slot.as_u64()
+                );
+                Ok(())
+            }
+            Err(e) => Err(e.into()),
         }
     }
 }
@@ -332,7 +352,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
         head_tracker: Arc<HeadTracker>,
         old_finalized_checkpoint: Checkpoint,
         new_finalized_checkpoint: Checkpoint,
-    ) {
+    ) -> Result<(), BeaconChainError> {
         let (ref mut tx, ref mut thread) = *self.tx_thread.lock();
 
         if let Err(tx_err) = tx.send((
@@ -360,6 +380,8 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> Migrate<E, Hot, Cold>
             // Retry at most once, we could recurse but that would risk overflowing the stack.
             let _ = tx.send(tx_err.0);
         }
+
+        Ok(())
     }
 }
 
@@ -394,7 +416,7 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> BackgroundMigrator<E, Ho
                     Err(e) => warn!(log, "Block pruning failed: {:?}", e),
                 }
 
-                match process_finalization(db.clone(), state_root.into(), &state) {
+                match migrate_database(db.clone(), state_root.into(), &state) {
                     Ok(()) => {}
                     Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => {
                         debug!(