Rework block processing (#4092)

* introduce availability pending block

* add intoavailableblock trait

* small fixes

* add 'gossip blob cache' and start to clean up processing and transition types

* shard memory blob cache

* Initial commit

* Fix after rebase

* Add gossip verification conditions

* cache cleanup

* general chaos

* extended chaos

* cargo fmt

* more progress

* more progress

* tons of changes, just tryna compile

* everything, everywhere, all at once

* Reprocess an ExecutedBlock on unavailable blobs

* Add sus gossip verification for blobs

* Merge stuff

* Remove reprocessing cache stuff

* lint

* Add a wrapper to allow construction of only valid `AvailableBlock`s

* rename blob arc list to blob list

* merge cleanuo

* Revert "merge cleanuo"

This reverts commit 5e98326878.

* Revert "Revert "merge cleanuo""

This reverts commit 3a4009443a.

* fix rpc methods

* move beacon block and blob to eth2/types

* rename gossip blob cache to data availability checker

* lots of changes

* fix some compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* cargo fmt

* use a common data structure for block import types

* fix availability check on proposal import

* refactor the blob cache and split the block wrapper into two types

* add type conversion for signed block and block wrapper

* fix beacon chain tests and do some renaming, add some comments

* Partial processing (#4)

* move beacon block and blob to eth2/types

* rename gossip blob cache to data availability checker

* lots of changes

* fix some compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* fix compilation issues

* cargo fmt

* use a common data structure for block import types

* fix availability check on proposal import

* refactor the blob cache and split the block wrapper into two types

* add type conversion for signed block and block wrapper

* fix beacon chain tests and do some renaming, add some comments

* cargo update (#6)

---------

Co-authored-by: realbigsean <sean@sigmaprime.io>
Co-authored-by: realbigsean <seananderson33@gmail.com>

beacon_node/beacon_chain/tests/attestation_production.rs

@@ -1,9 +1,7 @@
 #![cfg(not(debug_assertions))]
 
-use beacon_chain::{
-    blob_verification::{BlockWrapper, IntoAvailableBlock},
-    test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy},
-};
+use beacon_chain::blob_verification::BlockWrapper;
+use beacon_chain::test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy};
 use beacon_chain::{StateSkipConfig, WhenSlotSkipped};
 use lazy_static::lazy_static;
 use std::sync::Arc;
@@ -135,6 +133,10 @@ async fn produces_attestations() {
             assert_eq!(data.target.root, target_root, "bad target root");
 
             let block_wrapper: BlockWrapper<MainnetEthSpec> = Arc::new(block.clone()).into();
+            let available_block = chain
+                .data_availability_checker
+                .try_check_availability(block_wrapper)
+                .unwrap();
 
             let early_attestation = {
                 let proto_block = chain
@@ -146,9 +148,7 @@ async fn produces_attestations() {
                     .early_attester_cache
                     .add_head_block(
                         block_root,
-                        block_wrapper
-                            .into_available_block(block_root, chain)
-                            .expect("should wrap into available block"),
+                        available_block,
                         proto_block,
                         &state,
                         &chain.spec,
@@ -199,18 +199,19 @@ async fn early_attester_cache_old_request() {
         .get_block(&head.beacon_block_root)
         .unwrap();
 
-    let block: BlockWrapper<MainnetEthSpec> = head.beacon_block.clone().into();
+    let block_wrapper: BlockWrapper<MainnetEthSpec> = head.beacon_block.clone().into();
+    let available_block = harness
+        .chain
+        .data_availability_checker
+        .try_check_availability(block_wrapper)
+        .unwrap();
 
-    let chain = &harness.chain;
     harness
         .chain
         .early_attester_cache
         .add_head_block(
             head.beacon_block_root,
-            block
-                .clone()
-                .into_available_block(head.beacon_block_root, &chain)
-                .expect("should wrap into available block"),
+            available_block,
             head_proto_block,
             &head.beacon_state,
             &harness.chain.spec,

beacon_node/beacon_chain/tests/block_verification.rs

@@ -1,7 +1,8 @@
 #![cfg(not(debug_assertions))]
 
+use beacon_chain::blob_verification::BlockWrapper;
 use beacon_chain::{
-    blob_verification::{AsBlock, BlockWrapper},
+    blob_verification::AsBlock,
     test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType},
 };
 use beacon_chain::{BeaconSnapshot, BlockError, ChainSegmentResult, NotifyExecutionLayer};

beacon_node/beacon_chain/tests/payload_invalidation.rs

@@ -702,6 +702,8 @@ async fn invalidates_all_descendants() {
             NotifyExecutionLayer::Yes,
         )
         .await
+        .unwrap()
+        .try_into()
         .unwrap();
     rig.recompute_head().await;
 
@@ -799,6 +801,8 @@ async fn switches_heads() {
             NotifyExecutionLayer::Yes,
         )
         .await
+        .unwrap()
+        .try_into()
         .unwrap();
     rig.recompute_head().await;
 

beacon_node/beacon_chain/tests/tests.rs

@@ -681,19 +681,20 @@ async fn run_skip_slot_test(skip_slots: u64) {
         Slot::new(0)
     );
 
-    assert_eq!(
-        harness_b
-            .chain
-            .process_block(
-                harness_a.chain.head_snapshot().beacon_block_root,
-                harness_a.chain.head_snapshot().beacon_block.clone(),
-                CountUnrealized::True,
-                NotifyExecutionLayer::Yes,
-            )
-            .await
-            .unwrap(),
-        harness_a.chain.head_snapshot().beacon_block_root
-    );
+    let status = harness_b
+        .chain
+        .process_block(
+            harness_a.chain.head_snapshot().beacon_block_root,
+            harness_a.chain.head_snapshot().beacon_block.clone(),
+            CountUnrealized::True,
+            NotifyExecutionLayer::Yes,
+        )
+        .await
+        .unwrap();
+
+    let root: Hash256 = status.try_into().unwrap();
+
+    assert_eq!(root, harness_a.chain.head_snapshot().beacon_block_root);
 
     harness_b.chain.recompute_head_at_current_slot().await;