From 5e7803f00b943ed9057cf2519878a37d73629da0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 13 Dec 2019 13:30:58 +1100 Subject: [PATCH] Clean up database metrics, add freezer DB size (#715) * Clean up database metrics, add freezer DB size * Address review comments --- beacon_node/beacon_chain/src/beacon_chain.rs | 8 ++-- beacon_node/client/src/builder.rs | 5 ++- beacon_node/rest_api/src/lib.rs | 3 ++ beacon_node/rest_api/src/metrics.rs | 3 +- beacon_node/rest_api/src/router.rs | 10 +++-- beacon_node/store/src/impls/beacon_state.rs | 13 +++++-- beacon_node/store/src/metrics.rs | 40 ++++++++++++++------ 7 files changed, 57 insertions(+), 25 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 8b0e879aa7..5250c76c04 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1209,14 +1209,14 @@ impl BeaconChain { }); } - if self.store.exists::>(&block_root)? { - return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown); - } - // Records the time taken to load the block and state from the database during block // processing. let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ); + if self.store.exists::>(&block_root)? { + return Ok(BlockProcessingOutcome::BlockIsAlreadyKnown); + } + // Load the blocks parent block from the database, returning invalid if that block is not // found. let parent_block: BeaconBlock = diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 9f93b7bd57..4a219ef0b1 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -314,7 +314,10 @@ where network_info, client_config .create_db_path() - .expect("unable to read datadir"), + .map_err(|_| "unable to read data dir")?, + client_config + .create_freezer_db_path() + .map_err(|_| "unable to read freezer DB dir")?, eth2_config.clone(), context.log, ) diff --git a/beacon_node/rest_api/src/lib.rs b/beacon_node/rest_api/src/lib.rs index 1c14649acf..d72a439986 100644 --- a/beacon_node/rest_api/src/lib.rs +++ b/beacon_node/rest_api/src/lib.rs @@ -56,6 +56,7 @@ pub fn start_server( beacon_chain: Arc>, network_info: NetworkInfo, db_path: PathBuf, + freezer_db_path: PathBuf, eth2_config: Eth2Config, log: slog::Logger, ) -> Result<(exit_future::Signal, SocketAddr), hyper::Error> { @@ -70,6 +71,7 @@ pub fn start_server( let network_service = network_info.network_service.clone(); let network_channel = Arc::new(RwLock::new(network_info.network_chan.clone())); let db_path = db_path.clone(); + let freezer_db_path = freezer_db_path.clone(); service_fn(move |req: Request| { router::route( @@ -80,6 +82,7 @@ pub fn start_server( eth2_config.clone(), log.clone(), db_path.clone(), + freezer_db_path.clone(), ) }) }); diff --git a/beacon_node/rest_api/src/metrics.rs b/beacon_node/rest_api/src/metrics.rs index 3b4466da12..3155e58511 100644 --- a/beacon_node/rest_api/src/metrics.rs +++ b/beacon_node/rest_api/src/metrics.rs @@ -32,6 +32,7 @@ pub fn get_prometheus( req: Request, beacon_chain: Arc>, db_path: PathBuf, + freezer_db_path: PathBuf, ) -> ApiResult { let mut buffer = vec![]; let encoder = TextEncoder::new(); @@ -53,7 +54,7 @@ pub fn get_prometheus( // a string that can be returned via HTTP. slot_clock::scrape_for_metrics::(&beacon_chain.slot_clock); - store::scrape_for_metrics(&db_path); + store::scrape_for_metrics(&db_path, &freezer_db_path); beacon_chain::scrape_for_metrics(&beacon_chain); encoder diff --git a/beacon_node/rest_api/src/router.rs b/beacon_node/rest_api/src/router.rs index 64b942156e..575c936e88 100644 --- a/beacon_node/rest_api/src/router.rs +++ b/beacon_node/rest_api/src/router.rs @@ -27,6 +27,7 @@ pub fn route( eth2_config: Arc, local_log: slog::Logger, db_path: PathBuf, + freezer_db_path: PathBuf, ) -> impl Future, Error = Error> { metrics::inc_counter(&metrics::REQUEST_COUNT); let timer = metrics::start_timer(&metrics::REQUEST_RESPONSE_TIME); @@ -143,9 +144,12 @@ pub fn route( into_boxfut(spec::get_eth2_config::(req, eth2_config)) } - (&Method::GET, "/metrics") => { - into_boxfut(metrics::get_prometheus::(req, beacon_chain, db_path)) - } + (&Method::GET, "/metrics") => into_boxfut(metrics::get_prometheus::( + req, + beacon_chain, + db_path, + freezer_db_path, + )), _ => Box::new(futures::future::err(ApiError::NotFound( "Request path and/or method not found.".to_owned(), diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs index 022bb67b02..553227747b 100644 --- a/beacon_node/store/src/impls/beacon_state.rs +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -9,12 +9,15 @@ pub fn store_full_state, E: EthSpec>( state_root: &Hash256, state: &BeaconState, ) -> Result<(), Error> { - let timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES); + let total_timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_TIMES); + let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_WRITE_OVERHEAD_TIMES); let bytes = StorageContainer::new(state).as_ssz_bytes(); + metrics::stop_timer(overhead_timer); + let result = store.put_bytes(DBColumn::BeaconState.into(), state_root.as_bytes(), &bytes); - metrics::stop_timer(timer); + metrics::stop_timer(total_timer); metrics::inc_counter(&metrics::BEACON_STATE_WRITE_COUNT); metrics::inc_counter_by(&metrics::BEACON_STATE_WRITE_BYTES, bytes.len() as i64); @@ -25,13 +28,15 @@ pub fn get_full_state, E: EthSpec>( store: &S, state_root: &Hash256, ) -> Result>, Error> { - let timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES); + let total_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_TIMES); match store.get_bytes(DBColumn::BeaconState.into(), state_root.as_bytes())? { Some(bytes) => { + let overhead_timer = metrics::start_timer(&metrics::BEACON_STATE_READ_OVERHEAD_TIMES); let container = StorageContainer::from_ssz_bytes(&bytes)?; - metrics::stop_timer(timer); + metrics::stop_timer(overhead_timer); + metrics::stop_timer(total_timer); metrics::inc_counter(&metrics::BEACON_STATE_READ_COUNT); metrics::inc_counter_by(&metrics::BEACON_STATE_READ_BYTES, bytes.len() as i64); diff --git a/beacon_node/store/src/metrics.rs b/beacon_node/store/src/metrics.rs index 022d814464..e401193594 100644 --- a/beacon_node/store/src/metrics.rs +++ b/beacon_node/store/src/metrics.rs @@ -1,29 +1,31 @@ pub use lighthouse_metrics::{set_gauge, try_create_int_gauge, *}; use std::fs; -use std::path::PathBuf; +use std::path::Path; lazy_static! { /* * General */ pub static ref DISK_DB_SIZE: Result = - try_create_int_gauge("store_disk_db_size", "Size of the on-disk database (bytes)"); + try_create_int_gauge("store_disk_db_size", "Size of the hot on-disk database (bytes)"); + pub static ref FREEZER_DB_SIZE: Result = + try_create_int_gauge("store_freezer_db_size", "Size of the on-disk freezer database (bytes)"); pub static ref DISK_DB_WRITE_BYTES: Result = try_create_int_counter( "store_disk_db_write_bytes_total", - "Number of bytes attempted to be written to the on-disk DB" + "Number of bytes attempted to be written to the hot on-disk DB" ); pub static ref DISK_DB_READ_BYTES: Result = try_create_int_counter( "store_disk_db_read_bytes_total", - "Number of bytes read from the on-disk DB" + "Number of bytes read from the hot on-disk DB" ); pub static ref DISK_DB_READ_COUNT: Result = try_create_int_counter( "store_disk_db_read_count_total", - "Total number of reads to the on-disk DB" + "Total number of reads to the hot on-disk DB" ); pub static ref DISK_DB_WRITE_COUNT: Result = try_create_int_counter( "store_disk_db_write_count_total", - "Total number of writes to the on-disk DB" + "Total number of writes to the hot on-disk DB" ); pub static ref DISK_DB_READ_TIMES: Result = try_create_histogram( "store_disk_db_read_seconds", @@ -35,16 +37,20 @@ lazy_static! { ); pub static ref DISK_DB_EXISTS_COUNT: Result = try_create_int_counter( "store_disk_db_exists_count_total", - "Total number of checks if a key is in the on-disk DB" + "Total number of checks if a key is in the hot on-disk DB" ); pub static ref DISK_DB_DELETE_COUNT: Result = try_create_int_counter( "store_disk_db_delete_count_total", - "Total number of deletions from the on-disk DB" + "Total number of deletions from the hot on-disk DB" ); /* * Beacon State */ pub static ref BEACON_STATE_READ_TIMES: Result = try_create_histogram( + "store_beacon_state_read_seconds", + "Total time required to read a BeaconState from the database" + ); + pub static ref BEACON_STATE_READ_OVERHEAD_TIMES: Result = try_create_histogram( "store_beacon_state_read_overhead_seconds", "Overhead on reading a beacon state from the DB (e.g., decoding)" ); @@ -57,6 +63,10 @@ lazy_static! { "Total number of beacon state bytes read from the DB" ); pub static ref BEACON_STATE_WRITE_TIMES: Result = try_create_histogram( + "store_beacon_state_write_seconds", + "Total time required to write a BeaconState to the database" + ); + pub static ref BEACON_STATE_WRITE_OVERHEAD_TIMES: Result = try_create_histogram( "store_beacon_state_write_overhead_seconds", "Overhead on writing a beacon state to the DB (e.g., encoding)" ); @@ -98,15 +108,21 @@ lazy_static! { } /// Updates the global metrics registry with store-related information. -pub fn scrape_for_metrics(db_path: &PathBuf) { - let db_size = if let Ok(iter) = fs::read_dir(db_path) { +pub fn scrape_for_metrics(db_path: &Path, freezer_db_path: &Path) { + let db_size = size_of_dir(db_path); + set_gauge(&DISK_DB_SIZE, db_size as i64); + let freezer_db_size = size_of_dir(freezer_db_path); + set_gauge(&FREEZER_DB_SIZE, freezer_db_size as i64); +} + +fn size_of_dir(path: &Path) -> u64 { + if let Ok(iter) = fs::read_dir(path) { iter.filter_map(std::result::Result::ok) .map(size_of_dir_entry) .sum() } else { 0 - }; - set_gauge(&DISK_DB_SIZE, db_size as i64); + } } fn size_of_dir_entry(dir: fs::DirEntry) -> u64 {