gnosis/lighthouse
1
0
Fork 0
mirror of https://github.com/sigp/lighthouse.git synced 2026-03-09 03:31:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use OS file locks in validator client (#1958)

Browse Source 
## Issue Addressed

Closes #1823

## Proposed Changes

* Use OS-level file locking for validator keystores, eliminating problems with lockfiles lingering after ungraceful shutdowns (`SIGKILL`, power outage). I'm using the `fs2` crate because it's cross-platform (unlike `file-lock`), and it seems to have the most downloads on crates.io.
* Deprecate + disable `--delete-lockfiles` CLI param, it's no longer necessary
* Delete the `validator_dir::Manager`, as it was mostly dead code and was only used in the `validator list` command, which has been rewritten to read the validator definitions YAML instead.

## Additional Info

Tested on:

- [x] Linux
- [x] macOS
- [x] Docker Linux
- [x] Docker macOS
- [ ] Windows
This commit is contained in:
Michael Sproul
2020-11-26 11:25:46 +00:00
parent fc07cc3fdf
commit 3486d6a809
21 changed files with 282 additions and 411 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
validator_client/src/cli.rs
View File

@@ -57,12 +57,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> {
Arg::with_name("delete-lockfiles")
.long("delete-lockfiles")
.help(
"If present, ignore and delete any keystore lockfiles encountered during start up. \
This is useful if the validator client did not exit gracefully on the last run. \
WARNING: lockfiles help prevent users from accidentally running the same validator \
using two different validator clients, an action that likely leads to slashing. \
Ensure you are certain that there are no other validator client instances running \
that might also be using the same keystores."
"DEPRECATED. This flag does nothing and will be removed in a future release."
)
)
.arg(

12
validator_client/src/config.rs
View File

@@ -29,8 +29,6 @@ pub struct Config {
/// If true, the validator client will still poll for duties and produce blocks even if the
/// beacon node is not synced at startup.
pub allow_unsynced_beacon_node: bool,
/// If true, delete any validator keystore lockfiles that would prevent starting.
pub delete_lockfiles: bool,
/// If true, don't scan the validators dir for new keystores.
pub disable_auto_discover: bool,
/// If true, re-register existing validators in definitions.yml for slashing protection.
@@ -59,7 +57,6 @@ impl Default for Config {
secrets_dir,
beacon_node: DEFAULT_BEACON_NODE.to_string(),
allow_unsynced_beacon_node: false,
delete_lockfiles: false,
disable_auto_discover: false,
init_slashing_protection: false,
graffiti: None,
@@ -123,8 +120,15 @@ impl Config {
config.beacon_node = server;
}
if cli_args.is_present("delete-lockfiles") {
warn!(
log,
"The --delete-lockfiles flag is deprecated";
"msg" => "it is no longer necessary, and no longer has any effect",
);
}
config.allow_unsynced_beacon_node = cli_args.is_present("allow-unsynced");
config.delete_lockfiles = cli_args.is_present("delete-lockfiles");
config.disable_auto_discover = cli_args.is_present("disable-auto-discover");
config.init_slashing_protection = cli_args.is_present("init-slashing-protection");

6
validator_client/src/http_api/create_validator.rs
View File

@@ -125,9 +125,13 @@ pub fn create_validators<P: AsRef<Path>, T: 'static + SlotClock, E: EthSpec>(
)));
}
// Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
let voting_keystore_path = validator_dir.voting_keystore_path();
drop(validator_dir);
tokio::runtime::Handle::current()
.block_on(validator_store.add_validator_keystore(
validator_dir.voting_keystore_path(),
voting_keystore_path,
voting_password_string,
request.enable,
))

5
validator_client/src/http_api/mod.rs
View File

@@ -352,11 +352,14 @@ pub fn serve<T: 'static + SlotClock + Clone, E: EthSpec>(
))
})?;
// Drop validator dir so that `add_validator_keystore` can re-lock the keystore.
let voting_keystore_path = validator_dir.voting_keystore_path();
drop(validator_dir);
let voting_password = body.password.clone();
let validator_def = tokio::runtime::Handle::current()
.block_on(validator_store.add_validator_keystore(
validator_dir.voting_keystore_path(),
voting_keystore_path,
voting_password,
body.enable,
))

1
validator_client/src/http_api/tests.rs
View File

@@ -49,7 +49,6 @@ impl ApiTester {
let initialized_validators = InitializedValidators::from_definitions(
validator_defs,
validator_dir.path().into(),
false,
log.clone(),
)
.await

165
validator_client/src/initialized_validators.rs
View File

@@ -14,16 +14,16 @@ use account_utils::{
ZeroizeString,
};
use eth2_keystore::Keystore;
use lockfile::{Lockfile, LockfileError};
use slog::{debug, error, info, warn, Logger};
use std::collections::{HashMap, HashSet};
use std::fs::{self, File, OpenOptions};
use std::fs::File;
use std::io;
use std::path::PathBuf;
use types::{Keypair, PublicKey};
use crate::key_cache;
use crate::key_cache::KeyCache;
use std::ops::{Deref, DerefMut};
// Use TTY instead of stdin to capture passwords from users.
const USE_STDIN: bool = false;
@@ -32,9 +32,7 @@ const USE_STDIN: bool = false;
pub enum Error {
/// Refused to open a validator with an existing lockfile since that validator may be in-use by
/// another process.
LockfileExists(PathBuf),
/// There was a filesystem error when creating the lockfile.
UnableToCreateLockfile(io::Error),
LockfileError(LockfileError),
/// The voting public key in the definition did not match the one in the keystore.
VotingPublicKeyMismatch {
definition: Box<PublicKey>,
@@ -61,12 +59,16 @@ pub enum Error {
UnableToReadPasswordFromUser(String),
/// There was an error running a tokio async task.
TokioJoin(tokio::task::JoinError),
/// There was a filesystem error when deleting a lockfile.
UnableToDeleteLockfile(io::Error),
/// Cannot initialize the same validator twice.
DuplicatePublicKey,
}
impl From<LockfileError> for Error {
fn from(error: LockfileError) -> Self {
Self::LockfileError(error)
}
}
/// A method used by a validator to sign messages.
///
/// Presently there is only a single variant, however we expect more variants to arise (e.g.,
@@ -75,7 +77,7 @@ pub enum SigningMethod {
/// A validator that is defined by an EIP-2335 keystore on the local filesystem.
LocalKeystore {
voting_keystore_path: PathBuf,
voting_keystore_lockfile_path: PathBuf,
voting_keystore_lockfile: Lockfile,
voting_keystore: Keystore,
voting_keypair: Keypair,
},
@@ -86,6 +88,18 @@ pub struct InitializedValidator {
signing_method: SigningMethod,
}
impl InitializedValidator {
/// Return a reference to this validator's lockfile if it has one.
pub fn keystore_lockfile(&self) -> Option<&Lockfile> {
match self.signing_method {
SigningMethod::LocalKeystore {
ref voting_keystore_lockfile,
..
} => Some(voting_keystore_lockfile),
}
}
}
fn open_keystore(path: &PathBuf) -> Result<Keystore, Error> {
let keystore_file = File::open(path).map_err(Error::UnableToOpenVotingKeystore)?;
Keystore::from_json_reader(keystore_file).map_err(Error::UnableToParseVotingKeystore)
@@ -102,43 +116,6 @@ fn get_lockfile_path(file_path: &PathBuf) -> Option<PathBuf> {
})
}
fn create_lock_file(
file_path: &PathBuf,
delete_lockfiles: bool,
log: &Logger,
) -> Result<(), Error> {
if file_path.exists() {
if delete_lockfiles {
warn!(
log,
"Deleting validator lockfile";
"file" => format!("{:?}", file_path)
);
fs::remove_file(file_path).map_err(Error::UnableToDeleteLockfile)?;
} else {
return Err(Error::LockfileExists(file_path.clone()));
}
}
// Create a new lockfile.
OpenOptions::new()
.write(true)
.create_new(true)
.open(file_path)
.map_err(Error::UnableToCreateLockfile)?;
Ok(())
}
fn remove_lock(lock_path: &PathBuf) {
if lock_path.exists() {
if let Err(e) = fs::remove_file(&lock_path) {
eprintln!("Failed to remove {:?}: {:?}", lock_path, e)
}
} else {
eprintln!("Lockfile missing: {:?}", lock_path)
}
}
impl InitializedValidator {
/// Instantiate `self` from a `ValidatorDefinition`.
///
@@ -150,8 +127,6 @@ impl InitializedValidator {
/// If the validator is unable to be initialized for whatever reason.
async fn from_definition(
def: ValidatorDefinition,
delete_lockfiles: bool,
log: &Logger,
key_cache: &mut KeyCache,
key_stores: &mut HashMap<PathBuf, Keystore>,
) -> Result<Self, Error> {
@@ -182,7 +157,7 @@ impl InitializedValidator {
// to keep if off the core executor. This also has the fortunate effect of
// interrupting the potentially long-running task during shut down.
let (password, keypair) = tokio::task::spawn_blocking(move || {
Ok(
Result::<_, Error>::Ok(
match (voting_keystore_password_path, voting_keystore_password) {
// If the password is supplied, use it and ignore the path
// (if supplied).
@@ -226,15 +201,15 @@ impl InitializedValidator {
}
// Append a `.lock` suffix to the voting keystore.
let voting_keystore_lockfile_path = get_lockfile_path(&voting_keystore_path)
let lockfile_path = get_lockfile_path(&voting_keystore_path)
.ok_or_else(|| Error::BadVotingKeystorePath(voting_keystore_path.clone()))?;
create_lock_file(&voting_keystore_lockfile_path, delete_lockfiles, &log)?;
let voting_keystore_lockfile = Lockfile::new(lockfile_path)?;
Ok(Self {
signing_method: SigningMethod::LocalKeystore {
voting_keystore_path,
voting_keystore_lockfile_path,
voting_keystore_lockfile,
voting_keystore: voting_keystore.clone(),
voting_keypair,
},
@@ -258,20 +233,6 @@ impl InitializedValidator {
}
}
/// Custom drop implementation to allow for `LocalKeystore` to remove lockfiles.
impl Drop for InitializedValidator {
fn drop(&mut self) {
match &self.signing_method {
SigningMethod::LocalKeystore {
voting_keystore_lockfile_path,
..
} => {
remove_lock(voting_keystore_lockfile_path);
}
}
}
}
/// Try to unlock `keystore` at `keystore_path` by prompting the user via `stdin`.
fn unlock_keystore_via_stdin_password(
keystore: &Keystore,
@@ -316,8 +277,6 @@ fn unlock_keystore_via_stdin_password(
///
/// Forms the fundamental list of validators that are managed by this validator client instance.
pub struct InitializedValidators {
/// If `true`, delete any validator keystore lockfiles that would prevent starting.
delete_lockfiles: bool,
/// A list of validator definitions which can be stored on-disk.
definitions: ValidatorDefinitions,
/// The directory that the `self.definitions` will be saved into.
@@ -328,47 +287,14 @@ pub struct InitializedValidators {
log: Logger,
}
pub struct LockedData<T> {
data: T,
lock_path: PathBuf,
}
impl<T> LockedData<T> {
fn new(data: T, lock_path: PathBuf) -> Self {
Self { data, lock_path }
}
}
impl<T> Deref for LockedData<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.data
}
}
impl<T> DerefMut for LockedData<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.data
}
}
impl<T> Drop for LockedData<T> {
fn drop(&mut self) {
remove_lock(&self.lock_path);
}
}
impl InitializedValidators {
/// Instantiates `Self`, initializing all validators in `definitions`.
pub async fn from_definitions(
definitions: ValidatorDefinitions,
validators_dir: PathBuf,
delete_lockfiles: bool,
log: Logger,
) -> Result<Self, Error> {
let mut this = Self {
delete_lockfiles,
validators_dir,
definitions,
validators: HashMap::default(),
@@ -566,23 +492,11 @@ impl InitializedValidators {
let key_cache_path = KeyCache::cache_file_path(&self.validators_dir);
let cache_lockfile_path = get_lockfile_path(&key_cache_path)
.ok_or_else(|| Error::BadKeyCachePath(key_cache_path))?;
create_lock_file(&cache_lockfile_path, self.delete_lockfiles, &self.log)?;
let _cache_lockfile = Lockfile::new(cache_lockfile_path)?;
let mut key_cache = LockedData::new(
{
let cache = KeyCache::open_or_create(&self.validators_dir).map_err(|e| {
remove_lock(&cache_lockfile_path);
Error::UnableToOpenKeyCache(e)
})?;
self.decrypt_key_cache(cache, &mut key_stores)
.await
.map_err(|e| {
remove_lock(&cache_lockfile_path);
e
})?
},
cache_lockfile_path,
);
let cache =
KeyCache::open_or_create(&self.validators_dir).map_err(Error::UnableToOpenKeyCache)?;
let mut key_cache = self.decrypt_key_cache(cache, &mut key_stores).await?;
let mut disabled_uuids = HashSet::new();
for def in self.definitions.as_slice() {
@@ -602,14 +516,18 @@ impl InitializedValidators {
match InitializedValidator::from_definition(
def.clone(),
self.delete_lockfiles,
&self.log,
&mut key_cache,
&mut key_stores,
)
.await
{
Ok(init) => {
let existing_lockfile_path = init
.keystore_lockfile()
.as_ref()
.filter(|l| l.file_existed())
.map(|l| l.path().to_owned());
self.validators
.insert(init.voting_public_key().clone(), init);
info!(
@@ -617,6 +535,17 @@ impl InitializedValidators {
"Enabled validator";
"voting_pubkey" => format!("{:?}", def.voting_public_key)
);
if let Some(lockfile_path) = existing_lockfile_path {
warn!(
self.log,
"Ignored stale lockfile";
"path" => lockfile_path.display(),
"cause" => "Ungraceful shutdown (harmless) OR \
non-Lighthouse client using this keystore \
(risky)"
);
}
}
Err(e) => {
error!(

1
validator_client/src/lib.rs
View File

@@ -138,7 +138,6 @@ impl<T: EthSpec> ProductionValidatorClient<T> {
let validators = InitializedValidators::from_definitions(
validator_defs,
config.validator_dir.clone(),
config.delete_lockfiles,
log.clone(),
)
.await