gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-06 18:21:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use OS file locks in validator client (#1958)

Browse Source 
## Issue Addressed

Closes #1823

## Proposed Changes

* Use OS-level file locking for validator keystores, eliminating problems with lockfiles lingering after ungraceful shutdowns (`SIGKILL`, power outage). I'm using the `fs2` crate because it's cross-platform (unlike `file-lock`), and it seems to have the most downloads on crates.io.
* Deprecate + disable `--delete-lockfiles` CLI param, it's no longer necessary
* Delete the `validator_dir::Manager`, as it was mostly dead code and was only used in the `validator list` command, which has been rewritten to read the validator definitions YAML instead.

## Additional Info

Tested on:

- [x] Linux
- [x] macOS
- [x] Docker Linux
- [x] Docker macOS
- [ ] Windows
This commit is contained in:
Michael Sproul
2020-11-26 11:25:46 +00:00
parent fc07cc3fdf
commit 3486d6a809
21 changed files with 282 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
validator_client/src/initialized_validators.rs
View File

@@ -14,16 +14,16 @@ use account_utils::{
ZeroizeString,
};
use eth2_keystore::Keystore;
use lockfile::{Lockfile, LockfileError};
use slog::{debug, error, info, warn, Logger};
use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::fs::File;
use std::io;
use std::path::PathBuf;
use types::{Keypair, PublicKey};
use crate::key_cache;
use crate::key_cache::KeyCache;
use std::ops::{Deref, DerefMut};
// Use TTY instead of stdin to capture passwords from users.
const USE_STDIN: bool = false;
@@ -32,9 +32,7 @@ const USE_STDIN: bool = false;
pub enum Error {
/// Refused to open a validator with an existing lockfile since that validator may be in-use by
/// another process.
LockfileExists(PathBuf),
/// There was a filesystem error when creating the lockfile.
UnableToCreateLockfile(io::Error),
LockfileError(LockfileError),
/// The voting public key in the definition did not match the one in the keystore.
VotingPublicKeyMismatch {
definition: Box<PublicKey>,
@@ -61,12 +59,16 @@ pub enum Error {
UnableToReadPasswordFromUser(String),
/// There was an error running a tokio async task.
TokioJoin(tokio::task::JoinError),
/// There was a filesystem error when deleting a lockfile.
UnableToDeleteLockfile(io::Error),
/// Cannot initialize the same validator twice.
DuplicatePublicKey,
}
impl From<LockfileError> for Error {
fn from(error: LockfileError) -> Self {
Self::LockfileError(error)
}
}
/// A method used by a validator to sign messages.
///
/// Presently there is only a single variant, however we expect more variants to arise (e.g.,
@@ -75,7 +77,7 @@ pub enum SigningMethod {
/// A validator that is defined by an EIP-2335 keystore on the local filesystem.
LocalKeystore {
voting_keystore_path: PathBuf,
voting_keystore_lockfile_path: PathBuf,
voting_keystore_lockfile: Lockfile,
voting_keystore: Keystore,
voting_keypair: Keypair,
},
@@ -86,6 +88,18 @@ pub struct InitializedValidator {
signing_method: SigningMethod,
}
impl InitializedValidator {
/// Return a reference to this validator's lockfile if it has one.
pub fn keystore_lockfile(&self) -> Option<&Lockfile> {
match self.signing_method {
SigningMethod::LocalKeystore {
ref voting_keystore_lockfile,
..
} => Some(voting_keystore_lockfile),
}
}
}
fn open_keystore(path: &PathBuf) -> Result<Keystore, Error> {
let keystore_file = File::open(path).map_err(Error::UnableToOpenVotingKeystore)?;
Keystore::from_json_reader(keystore_file).map_err(Error::UnableToParseVotingKeystore)
@@ -102,43 +116,6 @@ fn get_lockfile_path(file_path: &PathBuf) -> Option<PathBuf> {
})
}
fn create_lock_file(
file_path: &PathBuf,
delete_lockfiles: bool,
log: &Logger,
) -> Result<(), Error> {
if file_path.exists() {
if delete_lockfiles {
warn!(
log,
"Deleting validator lockfile";
"file" => format!("{:?}", file_path)
);
fs::remove_file(file_path).map_err(Error::UnableToDeleteLockfile)?;
} else {
return Err(Error::LockfileExists(file_path.clone()));
}
}
// Create a new lockfile.
OpenOptions::new()
.write(true)
.create_new(true)
.open(file_path)
.map_err(Error::UnableToCreateLockfile)?;
Ok(())
}
fn remove_lock(lock_path: &PathBuf) {
if lock_path.exists() {
if let Err(e) = fs::remove_file(&lock_path) {
eprintln!("Failed to remove {:?}: {:?}", lock_path, e)
}
} else {
eprintln!("Lockfile missing: {:?}", lock_path)
}
}
impl InitializedValidator {
/// Instantiate `self` from a `ValidatorDefinition`.
///
@@ -150,8 +127,6 @@ impl InitializedValidator {
/// If the validator is unable to be initialized for whatever reason.
async fn from_definition(
def: ValidatorDefinition,
delete_lockfiles: bool,
log: &Logger,
key_cache: &mut KeyCache,
key_stores: &mut HashMap<PathBuf, Keystore>,
) -> Result<Self, Error> {
@@ -182,7 +157,7 @@ impl InitializedValidator {
// to keep if off the core executor. This also has the fortunate effect of
// interrupting the potentially long-running task during shut down.
let (password, keypair) = tokio::task::spawn_blocking(move || {
Ok(
Result::<_, Error>::Ok(
match (voting_keystore_password_path, voting_keystore_password) {
// If the password is supplied, use it and ignore the path
// (if supplied).
@@ -226,15 +201,15 @@ impl InitializedValidator {
}
// Append a `.lock` suffix to the voting keystore.
let voting_keystore_lockfile_path = get_lockfile_path(&voting_keystore_path)
let lockfile_path = get_lockfile_path(&voting_keystore_path)
.ok_or_else(|| Error::BadVotingKeystorePath(voting_keystore_path.clone()))?;
create_lock_file(&voting_keystore_lockfile_path, delete_lockfiles, &log)?;
let voting_keystore_lockfile = Lockfile::new(lockfile_path)?;
Ok(Self {
signing_method: SigningMethod::LocalKeystore {
voting_keystore_path,
voting_keystore_lockfile_path,
voting_keystore_lockfile,
voting_keystore: voting_keystore.clone(),
voting_keypair,
},
@@ -258,20 +233,6 @@ impl InitializedValidator {
}
}
/// Custom drop implementation to allow for `LocalKeystore` to remove lockfiles.
impl Drop for InitializedValidator {
fn drop(&mut self) {
match &self.signing_method {
SigningMethod::LocalKeystore {
voting_keystore_lockfile_path,
..
} => {
remove_lock(voting_keystore_lockfile_path);
}
}
}
}
/// Try to unlock `keystore` at `keystore_path` by prompting the user via `stdin`.
fn unlock_keystore_via_stdin_password(
keystore: &Keystore,
@@ -316,8 +277,6 @@ fn unlock_keystore_via_stdin_password(
///
/// Forms the fundamental list of validators that are managed by this validator client instance.
pub struct InitializedValidators {
/// If `true`, delete any validator keystore lockfiles that would prevent starting.
delete_lockfiles: bool,
/// A list of validator definitions which can be stored on-disk.
definitions: ValidatorDefinitions,
/// The directory that the `self.definitions` will be saved into.
@@ -328,47 +287,14 @@ pub struct InitializedValidators {
log: Logger,
}
pub struct LockedData<T> {
data: T,
lock_path: PathBuf,
}
impl<T> LockedData<T> {
fn new(data: T, lock_path: PathBuf) -> Self {
Self { data, lock_path }
}
}
impl<T> Deref for LockedData<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.data
}
}
impl<T> DerefMut for LockedData<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.data
}
}
impl<T> Drop for LockedData<T> {
fn drop(&mut self) {
remove_lock(&self.lock_path);
}
}
impl InitializedValidators {
/// Instantiates `Self`, initializing all validators in `definitions`.
pub async fn from_definitions(
definitions: ValidatorDefinitions,
validators_dir: PathBuf,
delete_lockfiles: bool,
log: Logger,
) -> Result<Self, Error> {
let mut this = Self {
delete_lockfiles,
validators_dir,
definitions,
validators: HashMap::default(),
@@ -566,23 +492,11 @@ impl InitializedValidators {
let key_cache_path = KeyCache::cache_file_path(&self.validators_dir);
let cache_lockfile_path = get_lockfile_path(&key_cache_path)
.ok_or_else(|| Error::BadKeyCachePath(key_cache_path))?;
create_lock_file(&cache_lockfile_path, self.delete_lockfiles, &self.log)?;
let _cache_lockfile = Lockfile::new(cache_lockfile_path)?;
let mut key_cache = LockedData::new(
{
let cache = KeyCache::open_or_create(&self.validators_dir).map_err(|e| {
remove_lock(&cache_lockfile_path);
Error::UnableToOpenKeyCache(e)
})?;
self.decrypt_key_cache(cache, &mut key_stores)
.await
.map_err(|e| {
remove_lock(&cache_lockfile_path);
e
})?
},
cache_lockfile_path,
);
let cache =
KeyCache::open_or_create(&self.validators_dir).map_err(Error::UnableToOpenKeyCache)?;
let mut key_cache = self.decrypt_key_cache(cache, &mut key_stores).await?;
let mut disabled_uuids = HashSet::new();
for def in self.definitions.as_slice() {
@@ -602,14 +516,18 @@ impl InitializedValidators {
match InitializedValidator::from_definition(
def.clone(),
self.delete_lockfiles,
&self.log,
&mut key_cache,
&mut key_stores,
)
.await
{
Ok(init) => {
let existing_lockfile_path = init
.keystore_lockfile()
.as_ref()
.filter(|l| l.file_existed())
.map(|l| l.path().to_owned());
self.validators
.insert(init.voting_public_key().clone(), init);
info!(
@@ -617,6 +535,17 @@ impl InitializedValidators {
"Enabled validator";
"voting_pubkey" => format!("{:?}", def.voting_public_key)
);
if let Some(lockfile_path) = existing_lockfile_path {
warn!(
self.log,
"Ignored stale lockfile";
"path" => lockfile_path.display(),
"cause" => "Ungraceful shutdown (harmless) OR \
non-Lighthouse client using this keystore \
(risky)"
);
}
}
Err(e) => {
error!(