gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-10 12:11:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use OS file locks in validator client (#1958)

Browse Source 
## Issue Addressed

Closes #1823

## Proposed Changes

* Use OS-level file locking for validator keystores, eliminating problems with lockfiles lingering after ungraceful shutdowns (`SIGKILL`, power outage). I'm using the `fs2` crate because it's cross-platform (unlike `file-lock`), and it seems to have the most downloads on crates.io.
* Deprecate + disable `--delete-lockfiles` CLI param, it's no longer necessary
* Delete the `validator_dir::Manager`, as it was mostly dead code and was only used in the `validator list` command, which has been rewritten to read the validator definitions YAML instead.

## Additional Info

Tested on:

- [x] Linux
- [x] macOS
- [x] Docker Linux
- [x] Docker macOS
- [ ] Windows
This commit is contained in:
Michael Sproul
2020-11-26 11:25:46 +00:00
parent fc07cc3fdf
commit 3486d6a809
21 changed files with 282 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/validator_dir/Cargo.toml
View File

@@ -19,6 +19,8 @@ rayon = "1.4.1"
tree_hash = "0.1.1"
slog = { version = "2.5.2", features = ["max_level_trace", "release_max_level_trace"] }
hex = "0.4.2"
derivative = "2.1.1"
lockfile = { path = "../lockfile" }
[dev-dependencies]
tempfile = "3.1.0"

3
common/validator_dir/src/lib.rs
View File

@@ -2,14 +2,12 @@
//!
//! - `ValidatorDir`: manages a directory containing validator keypairs, deposit info and other
//! things.
//! - `Manager`: manages a directory that contains multiple `ValidatorDir`.
//!
//! This crate is intended to be used by the account manager to create validators and the validator
//! client to load those validators.
mod builder;
pub mod insecure_keys;
mod manager;
mod validator_dir;
pub use crate::validator_dir::{
@@ -20,4 +18,3 @@ pub use builder::{
Builder, Error as BuilderError, ETH1_DEPOSIT_DATA_FILE, VOTING_KEYSTORE_FILE,
WITHDRAWAL_KEYSTORE_FILE,
};
pub use manager::{Error as ManagerError, Manager};

175
common/validator_dir/src/manager.rs
View File

@@ -1,175 +0,0 @@
use crate::{Error as ValidatorDirError, ValidatorDir};
use bls::Keypair;
use rayon::prelude::*;
use slog::{info, warn, Logger};
use std::collections::BTreeMap;
use std::fs::read_dir;
use std::io;
use std::iter::FromIterator;
use std::path::{Path, PathBuf};
#[derive(Debug)]
pub enum Error {
DirectoryDoesNotExist(PathBuf),
UnableToReadBaseDir(io::Error),
UnableToReadFile(io::Error),
ValidatorDirError(ValidatorDirError),
}
/// Manages a directory containing multiple `ValidatorDir` directories.
///
/// ## Example
///
/// ```ignore
/// validators
/// └── 0x91494d3ac4c078049f37aa46934ba8cdf5a9cca6e1b9a9e12403d69d8a2c43a25a7f576df2a5a3d7cb3f45e6aa5e2812
/// ├── eth1_deposit_data.rlp
/// ├── deposit-tx-hash.txt
/// ├── voting-keystore.json
/// └── withdrawal-keystore.json
/// ```
pub struct Manager {
dir: PathBuf,
}
impl Manager {
/// Open a directory containing multiple validators.
///
/// Pass the `validators` director as `dir` (see struct-level example).
pub fn open<P: AsRef<Path>>(dir: P) -> Result<Self, Error> {
let dir: PathBuf = dir.as_ref().into();
if dir.exists() {
Ok(Self { dir })
} else {
Err(Error::DirectoryDoesNotExist(dir))
}
}
/// Iterate the nodes in `self.dir`, filtering out things that are unlikely to be a validator
/// directory.
fn iter_dir(&self) -> Result<Vec<PathBuf>, Error> {
read_dir(&self.dir)
.map_err(Error::UnableToReadBaseDir)?
.map(|file_res| file_res.map(|f| f.path()))
// We use `map_or` with `true` here to ensure that we always fail if there is any
// error.
.filter(|path_res| path_res.as_ref().map_or(true, |p| p.is_dir()))
.map(|res| res.map_err(Error::UnableToReadFile))
.collect()
}
/// Open a `ValidatorDir` at the given `path`.
///
/// ## Note
///
/// It is not enforced that `path` is contained in `self.dir`.
pub fn open_validator<P: AsRef<Path>>(&self, path: P) -> Result<ValidatorDir, Error> {
ValidatorDir::open(path).map_err(Error::ValidatorDirError)
}
/// Opens all the validator directories in `self`.
///
/// ## Errors
///
/// Returns an error if any of the directories is unable to be opened, perhaps due to a
/// file-system error or directory with an active lockfile.
pub fn open_all_validators(&self) -> Result<Vec<ValidatorDir>, Error> {
self.iter_dir()?
.into_iter()
.map(|path| ValidatorDir::open(path).map_err(Error::ValidatorDirError))
.collect()
}
/// Opens all the validator directories in `self` and decrypts the validator keypairs,
/// regardless if a lockfile exists or not.
///
/// If `log.is_some()`, an `info` log will be generated for each decrypted validator.
/// Additionally, a warning log will be created if a lockfile existed already.
///
/// ## Errors
///
/// Returns an error if any of the directories is unable to be opened.
pub fn force_decrypt_all_validators(
&self,
secrets_dir: PathBuf,
log_opt: Option<&Logger>,
) -> Result<Vec<(Keypair, ValidatorDir)>, Error> {
self.iter_dir()?
.into_par_iter()
.map(|path| {
ValidatorDir::force_open(path)
.and_then(|(v, existed)| {
v.voting_keypair(&secrets_dir).map(|kp| (kp, v, existed))
})
.map(|(kp, v, lockfile_existed)| {
if let Some(log) = log_opt {
info!(
log,
"Decrypted validator keystore";
"voting_pubkey" => kp.pk.to_hex_string()
);
if lockfile_existed {
warn!(
log,
"Lockfile already existed";
"msg" => "ensure no other validator client is running on this host",
"voting_pubkey" => kp.pk.to_hex_string()
);
}
}
(kp, v)
})
.map_err(Error::ValidatorDirError)
})
.collect()
}
/// Opens all the validator directories in `self` and decrypts the validator keypairs.
///
/// If `log.is_some()`, an `info` log will be generated for each decrypted validator.
///
/// ## Errors
///
/// Returns an error if any of the directories is unable to be opened.
pub fn decrypt_all_validators(
&self,
secrets_dir: PathBuf,
log_opt: Option<&Logger>,
) -> Result<Vec<(Keypair, ValidatorDir)>, Error> {
self.iter_dir()?
.into_par_iter()
.map(|path| {
ValidatorDir::open(path)
.and_then(|v| v.voting_keypair(&secrets_dir).map(|kp| (kp, v)))
.map(|(kp, v)| {
if let Some(log) = log_opt {
info!(
log,
"Decrypted validator keystore";
"voting_pubkey" => kp.pk.to_hex_string()
)
}
(kp, v)
})
.map_err(Error::ValidatorDirError)
})
.collect()
}
/// Returns a map of directory name to full directory path. E.g., `myval -> /home/vals/myval`.
/// Filters out nodes in `self.dir` that are unlikely to be a validator directory.
///
/// ## Errors
///
/// Returns an error if a directory is unable to be read.
pub fn directory_names(&self) -> Result<BTreeMap<String, PathBuf>, Error> {
Ok(BTreeMap::from_iter(
self.iter_dir()?.into_iter().filter_map(|path| {
path.file_name()
.and_then(|os_string| os_string.to_str().map(|s| s.to_string()))
.map(|filename| (filename, path))
}),
))
}
}

80
common/validator_dir/src/validator_dir.rs
View File

@@ -3,24 +3,22 @@ use crate::builder::{
WITHDRAWAL_KEYSTORE_FILE,
};
use deposit_contract::decode_eth1_tx_data;
use derivative::Derivative;
use eth2_keystore::{Error as KeystoreError, Keystore, PlainText};
use std::fs::{read, remove_file, write, OpenOptions};
use lockfile::{Lockfile, LockfileError};
use std::fs::{read, write, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use tree_hash::TreeHash;
use types::{DepositData, Hash256, Keypair};
/// The file used for indicating if a directory is in-use by another process.
const LOCK_FILE: &str = ".lock";
/// The file used to save the Eth1 transaction hash from a deposit.
pub const ETH1_DEPOSIT_TX_HASH_FILE: &str = "eth1-deposit-tx-hash.txt";
#[derive(Debug)]
pub enum Error {
DirectoryDoesNotExist(PathBuf),
DirectoryLocked(PathBuf),
UnableToCreateLockfile(io::Error),
LockfileError(LockfileError),
UnableToOpenKeystore(io::Error),
UnableToReadKeystore(KeystoreError),
UnableToOpenPassword(io::Error),
@@ -58,19 +56,22 @@ pub struct Eth1DepositData {
/// Provides a wrapper around a directory containing validator information.
///
/// Creates/deletes a lockfile in `self.dir` to attempt to prevent concurrent access from multiple
/// Holds a lockfile in `self.dir` to attempt to prevent concurrent access from multiple
/// processes.
#[derive(Debug, PartialEq)]
#[derive(Debug, Derivative)]
#[derivative(PartialEq)]
pub struct ValidatorDir {
dir: PathBuf,
#[derivative(PartialEq = "ignore")]
lockfile: Lockfile,
}
impl ValidatorDir {
/// Open `dir`, creating a lockfile to prevent concurrent access.
/// Open `dir`, obtaining a lockfile to prevent concurrent access.
///
/// ## Errors
///
/// If there is a filesystem error or if a lockfile already exists.
/// If there is a filesystem error or if the lockfile is locked by another process.
pub fn open<P: AsRef<Path>>(dir: P) -> Result<Self, Error> {
let dir: &Path = dir.as_ref();
let dir: PathBuf = dir.into();
@@ -79,49 +80,12 @@ impl ValidatorDir {
return Err(Error::DirectoryDoesNotExist(dir));
}
let lockfile = dir.join(LOCK_FILE);
if lockfile.exists() {
return Err(Error::DirectoryLocked(dir));
} else {
OpenOptions::new()
.write(true)
.create_new(true)
.open(lockfile)
.map_err(Error::UnableToCreateLockfile)?;
}
// Lock the keystore file that *might* be in this directory.
// This is not ideal, see: https://github.com/sigp/lighthouse/issues/1978
let lockfile_path = dir.join(format!("{}.lock", VOTING_KEYSTORE_FILE));
let lockfile = Lockfile::new(lockfile_path).map_err(Error::LockfileError)?;
Ok(Self { dir })
}
/// Open `dir`, regardless or not if a lockfile exists.
///
/// Returns `(validator_dir, lockfile_existed)`, where `lockfile_existed == true` if a lockfile
/// was already present before opening. Creates a lockfile if one did not already exist.
///
/// ## Errors
///
/// If there is a filesystem error.
pub fn force_open<P: AsRef<Path>>(dir: P) -> Result<(Self, bool), Error> {
let dir: &Path = dir.as_ref();
let dir: PathBuf = dir.into();
if !dir.exists() {
return Err(Error::DirectoryDoesNotExist(dir));
}
let lockfile = dir.join(LOCK_FILE);
let lockfile_exists = lockfile.exists();
if !lockfile_exists {
OpenOptions::new()
.write(true)
.create_new(true)
.open(lockfile)
.map_err(Error::UnableToCreateLockfile)?;
}
Ok((Self { dir }, lockfile_exists))
Ok(Self { dir, lockfile })
}
/// Returns the `dir` provided to `Self::open`.
@@ -238,18 +202,6 @@ impl ValidatorDir {
}
}
impl Drop for ValidatorDir {
fn drop(&mut self) {
let lockfile = self.dir.clone().join(LOCK_FILE);
if let Err(e) = remove_file(&lockfile) {
eprintln!(
"Unable to remove validator lockfile {:?}: {:?}",
lockfile, e
);
}
}
}
/// Attempts to load and decrypt a Keypair given path to the keystore.
pub fn unlock_keypair<P: AsRef<Path>>(
keystore_path: &PathBuf,