Abstract SSZ vec decoding

Cargo.lock

@@ -1674,6 +1674,7 @@ version = "0.4.1"
 dependencies = [
  "eth2_ssz_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "ethereum-types 0.12.1",
+ "itertools",
  "smallvec",
 ]
 

consensus/ssz/Cargo.toml

@@ -14,7 +14,8 @@ eth2_ssz_derive = "0.3.0"
 
 [dependencies]
 ethereum-types = "0.12.1"
-smallvec = "1.6.1"
+smallvec = { version = "1.6.1", features = ["const_generics"] }
+itertools = "0.10.3"
 
 [features]
 arbitrary = ["ethereum-types/arbitrary"]

consensus/ssz/src/decode.rs

@@ -5,6 +5,7 @@ use std::cmp::Ordering;
 type SmallVec8<T> = SmallVec<[T; 8]>;
 
 pub mod impls;
+pub mod try_from_iter;
 
 /// Returned when SSZ decoding fails.
 #[derive(Debug, PartialEq, Clone)]

consensus/ssz/src/decode/impls.rs

@@ -1,6 +1,8 @@
 use super::*;
+use crate::decode::try_from_iter::{TryCollect, TryFromIter};
 use core::num::NonZeroUsize;
 use ethereum_types::{H160, H256, U128, U256};
+use itertools::process_results;
 use smallvec::SmallVec;
 use std::collections::BTreeMap;
 use std::iter::{self, FromIterator};
@@ -397,14 +399,14 @@ macro_rules! impl_for_vec {
 }
 
 impl_for_vec!(Vec<T>, None);
-impl_for_vec!(SmallVec<[T; 1]>, Some(1));
+impl_for_vec!(SmallVec<[T; 1]>, None);
-impl_for_vec!(SmallVec<[T; 2]>, Some(2));
+impl_for_vec!(SmallVec<[T; 2]>, None);
-impl_for_vec!(SmallVec<[T; 3]>, Some(3));
+impl_for_vec!(SmallVec<[T; 3]>, None);
-impl_for_vec!(SmallVec<[T; 4]>, Some(4));
+impl_for_vec!(SmallVec<[T; 4]>, None);
-impl_for_vec!(SmallVec<[T; 5]>, Some(5));
+impl_for_vec!(SmallVec<[T; 5]>, None);
-impl_for_vec!(SmallVec<[T; 6]>, Some(6));
+impl_for_vec!(SmallVec<[T; 6]>, None);
-impl_for_vec!(SmallVec<[T; 7]>, Some(7));
+impl_for_vec!(SmallVec<[T; 7]>, None);
-impl_for_vec!(SmallVec<[T; 8]>, Some(8));
+impl_for_vec!(SmallVec<[T; 8]>, None);
 
 impl<K, V> Decode for BTreeMap<K, V>
 where
@@ -434,12 +436,14 @@ where
 /// The `ssz::SszDecoder` can also perform this functionality, however this function is
 /// significantly faster as it is optimized to read same-typed items whilst `ssz::SszDecoder`
 /// supports reading items of differing types.
-pub fn decode_list_of_variable_length_items<T: Decode, Container: FromIterator<T>>(
+pub fn decode_list_of_variable_length_items<T: Decode, Container: TryFromIter<T>>(
     bytes: &[u8],
     max_len: Option<usize>,
 ) -> Result<Container, DecodeError> {
     if bytes.is_empty() {
-        return Ok(Container::from_iter(iter::empty()));
+        return Container::try_from_iter(iter::empty()).map_err(|e| {
+            DecodeError::BytesInvalid(format!("Error trying to collect empty list: {:?}", e))
+        });
     }
 
     let first_offset = read_offset(bytes)?;
@@ -459,8 +463,8 @@ pub fn decode_list_of_variable_length_items<T: Decode, Container: FromIterator<T
     }
 
     let mut offset = first_offset;
-    (1..=num_items)
+    process_results(
-        .map(|i| {
+        (1..=num_items).map(|i| {
             let slice_option = if i == num_items {
                 bytes.get(offset..)
             } else {
@@ -475,8 +479,10 @@ pub fn decode_list_of_variable_length_items<T: Decode, Container: FromIterator<T
 
             let slice = slice_option.ok_or(DecodeError::OutOfBoundsByte { i: offset })?;
             T::from_ssz_bytes(slice)
-        })
+        }),
-        .collect()
+        |iter| iter.try_collect(),
+    )?
+    .map_err(|e| DecodeError::BytesInvalid(format!("Error collecting into container: {:?}", e)))
 }
 
 #[cfg(test)]

consensus/ssz/src/decode/try_from_iter.rs

@@ -0,0 +1,82 @@
+use smallvec::SmallVec;
+use std::collections::BTreeMap;
+use std::convert::Infallible;
+use std::fmt::Debug;
+
+/// Partial variant of `std::iter::FromIterator`.
+///
+/// This trait is implemented for types which can be constructed from an iterator of decoded SSZ
+/// values, but which may refuse values once a length limit is reached.
+pub trait TryFromIter<T>: Sized {
+    type Error: Debug;
+
+    fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
+    where
+        I: IntoIterator<Item = T>;
+}
+
+// It would be nice to be able to do a blanket impl, e.g.
+//
+// `impl TryFromIter<T> for C where C: FromIterator<T>`
+//
+// However this runs into trait coherence issues due to the type parameter `T` on `TryFromIter`.
+//
+// E.g. If we added an impl downstream for `List<T, N>` then another crate downstream of that
+// could legally add an impl of `FromIterator<Local> for List<Local, N>` which would create
+// two conflicting implementations for `List<Local, N>`. Hence the `List<T, N>` impl is disallowed
+// by the compiler in the presence of the blanket impl. That's obviously annoying, so we opt to
+// abandon the blanket impl in favour of impls for selected types.
+impl<T> TryFromIter<T> for Vec<T> {
+    type Error = Infallible;
+
+    fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
+    where
+        I: IntoIterator<Item = T>,
+    {
+        Ok(Self::from_iter(iter))
+    }
+}
+
+impl<T, const N: usize> TryFromIter<T> for SmallVec<[T; N]> {
+    type Error = Infallible;
+
+    fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
+    where
+        I: IntoIterator<Item = T>,
+    {
+        Ok(Self::from_iter(iter))
+    }
+}
+
+impl<K, V> TryFromIter<(K, V)> for BTreeMap<K, V>
+where
+    K: Ord,
+{
+    type Error = Infallible;
+
+    fn try_from_iter<I>(iter: I) -> Result<Self, Self::Error>
+    where
+        I: IntoIterator<Item = (K, V)>,
+    {
+        Ok(Self::from_iter(iter))
+    }
+}
+
+/// Partial variant of `collect`.
+pub trait TryCollect: Iterator {
+    fn try_collect<C>(self) -> Result<C, C::Error>
+    where
+        C: TryFromIter<Self::Item>;
+}
+
+impl<I> TryCollect for I
+where
+    I: Iterator,
+{
+    fn try_collect<C>(self) -> Result<C, C::Error>
+    where
+        C: TryFromIter<Self::Item>,
+    {
+        C::try_from_iter(self)
+    }
+}

consensus/ssz/src/lib.rs

@@ -40,8 +40,8 @@ pub mod legacy;
 mod union_selector;
 
 pub use decode::{
-    impls::decode_list_of_variable_length_items, read_offset, split_union_bytes, Decode,
+    impls::decode_list_of_variable_length_items, read_offset, split_union_bytes,
-    DecodeError, SszDecoder, SszDecoderBuilder,
+    try_from_iter::TryFromIter, Decode, DecodeError, SszDecoder, SszDecoderBuilder,
 };
 pub use encode::{encode_length, Encode, SszEncoder};
 pub use union_selector::UnionSelector;