Fix race condition in seen caches (#1937)

## Issue Addressed

Closes #1719

## Proposed Changes

Lift the internal `RwLock`s and `Mutex`es from the `Observed*` data structures to resolve the race conditions described in #1719.

Most of this work was done by @paulhauner on his `lift-locks` branch, I merely updated it for the current `master` and checked over it.

## Additional Info

I think it would be prudent to test this on a testnet or two before mainnet launch, just to be sure that the extra lock contention doesn't negatively impact performance.

beacon_node/beacon_chain/src/attestation_verification.rs

@@ -316,6 +316,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         let attestation_root = attestation.tree_hash_root();
         if chain
             .observed_attestations
+            .write()
             .is_known(attestation, attestation_root)
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
@@ -329,6 +330,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         // Note: do not observe yet, only observe once the attestation has been verfied.
         match chain
             .observed_aggregators
+            .read()
             .validator_has_been_observed(attestation, aggregator_index as usize)
         {
             Ok(true) => Err(Error::AggregatorAlreadyKnown(aggregator_index)),
@@ -400,6 +402,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         // attestations processed at the same time could be published.
         if let ObserveOutcome::AlreadyKnown = chain
             .observed_attestations
+            .write()
             .observe_attestation(attestation, Some(attestation_root))
             .map_err(|e| Error::BeaconChainError(e.into()))?
         {
@@ -412,6 +415,7 @@ impl<T: BeaconChainTypes> VerifiedAggregatedAttestation<T> {
         // attestations processed at the same time could be published.
         if chain
             .observed_aggregators
+            .write()
             .observe_validator(&attestation, aggregator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -518,6 +522,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
          */
         if chain
             .observed_attesters
+            .read()
             .validator_has_been_observed(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {
@@ -538,6 +543,7 @@ impl<T: BeaconChainTypes> VerifiedUnaggregatedAttestation<T> {
         // process them in different threads.
         if chain
             .observed_attesters
+            .write()
             .observe_validator(&attestation, validator_index as usize)
             .map_err(BeaconChainError::from)?
         {