Add lighthouse db command (#3129)

## Proposed Changes

Add a `lighthouse db` command with three initial subcommands:

- `lighthouse db version`: print the database schema version.
- `lighthouse db migrate --to N`: manually upgrade (or downgrade!) the database to a different version.
- `lighthouse db inspect --column C`: log the key and size in bytes of every value in a given `DBColumn`.

This PR lays the groundwork for other changes, namely:

- Mark's fast-deposit sync (https://github.com/sigp/lighthouse/pull/2915), for which I think we should implement a database downgrade (from v9 to v8).
- My `tree-states` work, which already implements a downgrade (v10 to v8).
- Standalone purge commands like `lighthouse db purge-dht` per https://github.com/sigp/lighthouse/issues/2824.

## Additional Info

I updated the `strum` crate to 0.24.0, which necessitated some changes in the network code to remove calls to deprecated methods.

Thanks to @winksaville for the motivation, and implementation work that I used as a source of inspiration (https://github.com/sigp/lighthouse/pull/2685).

beacon_node/store/src/hot_cold_store.rs

@@ -191,9 +191,8 @@ impl<E: EthSpec> HotColdDB<E, LevelDB<E>, LevelDB<E>> {
             info!(
                 db.log,
                 "Hot-Cold DB initialized";
-                "version" => CURRENT_SCHEMA_VERSION.as_u64(),
                 "split_slot" => split.slot,
-                "split_state" => format!("{:?}", split.state_root)
+                "split_state" => ?split.state_root
             );
         }
 

beacon_node/store/src/leveldb_store.rs

@@ -1,4 +1,5 @@
 use super::*;
+use crate::hot_cold_store::HotColdDBError;
 use crate::metrics;
 use db_key::Key;
 use leveldb::compaction::Compaction;
@@ -6,7 +7,7 @@ use leveldb::database::batch::{Batch, Writebatch};
 use leveldb::database::kv::KV;
 use leveldb::database::Database;
 use leveldb::error::Error as LevelDBError;
-use leveldb::iterator::{Iterable, KeyIterator};
+use leveldb::iterator::{Iterable, KeyIterator, LevelDBIterator};
 use leveldb::options::{Options, ReadOptions, WriteOptions};
 use parking_lot::{Mutex, MutexGuard};
 use std::marker::PhantomData;
@@ -174,6 +175,28 @@ impl<E: EthSpec> KeyValueStore<E> for LevelDB<E> {
         }
         Ok(())
     }
+
+    /// Iterate through all keys and values in a particular column.
+    fn iter_column(&self, column: DBColumn) -> ColumnIter {
+        let start_key =
+            BytesKey::from_vec(get_key_for_col(column.into(), Hash256::zero().as_bytes()));
+
+        let iter = self.db.iter(self.read_options());
+        iter.seek(&start_key);
+
+        Box::new(
+            iter.take_while(move |(key, _)| key.matches_column(column))
+                .map(move |(bytes_key, value)| {
+                    let key =
+                        bytes_key
+                            .remove_column(column)
+                            .ok_or(HotColdDBError::IterationError {
+                                unexpected_key: bytes_key,
+                            })?;
+                    Ok((key, value))
+                }),
+        )
+    }
 }
 
 impl<E: EthSpec> ItemStore<E> for LevelDB<E> {}

beacon_node/store/src/lib.rs

@@ -39,8 +39,11 @@ pub use impls::beacon_state::StorageContainer as BeaconStateStorageContainer;
 pub use metadata::AnchorInfo;
 pub use metrics::scrape_for_metrics;
 use parking_lot::MutexGuard;
+use strum::{EnumString, IntoStaticStr};
 pub use types::*;
 
+pub type ColumnIter<'a> = Box<dyn Iterator<Item = Result<(Hash256, Vec<u8>), Error>> + 'a>;
+
 pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
     /// Retrieve some bytes in `column` with `key`.
     fn get_bytes(&self, column: &str, key: &[u8]) -> Result<Option<Vec<u8>>, Error>;
@@ -73,6 +76,12 @@ pub trait KeyValueStore<E: EthSpec>: Sync + Send + Sized + 'static {
 
     /// Compact the database, freeing space used by deleted items.
     fn compact(&self) -> Result<(), Error>;
+
+    /// Iterate through all values in a particular column.
+    fn iter_column(&self, _column: DBColumn) -> ColumnIter {
+        // Default impl for non LevelDB databases
+        Box::new(std::iter::empty())
+    }
 }
 
 pub fn get_key_for_col(column: &str, key: &[u8]) -> Vec<u8> {
@@ -146,56 +155,49 @@ pub enum StoreOp<'a, E: EthSpec> {
 }
 
 /// A unique column identifier.
-#[derive(Debug, Clone, Copy, PartialEq)]
+#[derive(Debug, Clone, Copy, PartialEq, IntoStaticStr, EnumString)]
 pub enum DBColumn {
     /// For data related to the database itself.
+    #[strum(serialize = "bma")]
     BeaconMeta,
+    #[strum(serialize = "blk")]
     BeaconBlock,
+    /// For full `BeaconState`s in the hot database (finalized or fork-boundary states).
+    #[strum(serialize = "ste")]
     BeaconState,
-    /// For persisting in-memory state to the database.
-    BeaconChain,
-    OpPool,
-    Eth1Cache,
-    ForkChoice,
-    PubkeyCache,
-    /// For the table mapping restore point numbers to state roots.
-    BeaconRestorePoint,
     /// For the mapping from state roots to their slots or summaries.
+    #[strum(serialize = "bss")]
     BeaconStateSummary,
     /// For the list of temporary states stored during block import,
     /// and then made non-temporary by the deletion of their state root from this column.
+    #[strum(serialize = "bst")]
     BeaconStateTemporary,
+    /// For persisting in-memory state to the database.
+    #[strum(serialize = "bch")]
+    BeaconChain,
+    #[strum(serialize = "opo")]
+    OpPool,
+    #[strum(serialize = "etc")]
+    Eth1Cache,
+    #[strum(serialize = "frk")]
+    ForkChoice,
+    #[strum(serialize = "pkc")]
+    PubkeyCache,
+    /// For the table mapping restore point numbers to state roots.
+    #[strum(serialize = "brp")]
+    BeaconRestorePoint,
+    #[strum(serialize = "bbr")]
     BeaconBlockRoots,
+    #[strum(serialize = "bsr")]
     BeaconStateRoots,
+    #[strum(serialize = "bhr")]
     BeaconHistoricalRoots,
+    #[strum(serialize = "brm")]
     BeaconRandaoMixes,
+    #[strum(serialize = "dht")]
     DhtEnrs,
 }
 
-impl Into<&'static str> for DBColumn {
-    /// Returns a `&str` prefix to be added to keys before they hit the key-value database.
-    fn into(self) -> &'static str {
-        match self {
-            DBColumn::BeaconMeta => "bma",
-            DBColumn::BeaconBlock => "blk",
-            DBColumn::BeaconState => "ste",
-            DBColumn::BeaconChain => "bch",
-            DBColumn::OpPool => "opo",
-            DBColumn::Eth1Cache => "etc",
-            DBColumn::ForkChoice => "frk",
-            DBColumn::PubkeyCache => "pkc",
-            DBColumn::BeaconRestorePoint => "brp",
-            DBColumn::BeaconStateSummary => "bss",
-            DBColumn::BeaconStateTemporary => "bst",
-            DBColumn::BeaconBlockRoots => "bbr",
-            DBColumn::BeaconStateRoots => "bsr",
-            DBColumn::BeaconHistoricalRoots => "bhr",
-            DBColumn::BeaconRandaoMixes => "brm",
-            DBColumn::DhtEnrs => "dht",
-        }
-    }
-}
-
 impl DBColumn {
     pub fn as_str(self) -> &'static str {
         self.into()