From 55e5b5b2df72657e273c4c98e701f6cb5d6ff9be Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 21 Sep 2021 16:56:52 +1000 Subject: [PATCH] Checkout serde_utils from rayonism --- consensus/serde_utils/src/hex.rs | 8 +- consensus/serde_utils/src/hex_vec.rs | 23 +++ consensus/serde_utils/src/lib.rs | 3 + .../serde_utils/src/list_of_bytes_lists.rs | 49 +++++++ consensus/serde_utils/src/quoted_int.rs | 11 -- consensus/serde_utils/src/u64_hex_be.rs | 134 ++++++++++++++++++ 6 files changed, 211 insertions(+), 17 deletions(-) create mode 100644 consensus/serde_utils/src/hex_vec.rs create mode 100644 consensus/serde_utils/src/list_of_bytes_lists.rs create mode 100644 consensus/serde_utils/src/u64_hex_be.rs diff --git a/consensus/serde_utils/src/hex.rs b/consensus/serde_utils/src/hex.rs index 647b0ecfb5..1e6c02427f 100644 --- a/consensus/serde_utils/src/hex.rs +++ b/consensus/serde_utils/src/hex.rs @@ -6,6 +6,7 @@ use std::fmt; /// Encode `data` as a 0x-prefixed hex string. pub fn encode>(data: T) -> String { let hex = hex::encode(data); + let mut s = "0x".to_string(); s.push_str(hex.as_str()); s @@ -33,12 +34,7 @@ impl<'de> Visitor<'de> for PrefixedHexVisitor { where E: de::Error, { - if let Some(stripped) = value.strip_prefix("0x") { - Ok(hex::decode(stripped) - .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e)))?) - } else { - Err(de::Error::custom("missing 0x prefix")) - } + decode(value).map_err(de::Error::custom) } } diff --git a/consensus/serde_utils/src/hex_vec.rs b/consensus/serde_utils/src/hex_vec.rs new file mode 100644 index 0000000000..09c6b9f932 --- /dev/null +++ b/consensus/serde_utils/src/hex_vec.rs @@ -0,0 +1,23 @@ +//! Formats `Vec` as a 0x-prefixed hex string. +//! +//! E.g., `vec![0, 1, 2, 3]` serializes as `"0x00010203"`. + +use crate::hex::PrefixedHexVisitor; +use serde::{Deserializer, Serializer}; + +pub fn serialize(bytes: &Vec, serializer: S) -> Result +where + S: Serializer, +{ + let mut hex_string: String = "0x".to_string(); + hex_string.push_str(&hex::encode(&bytes)); + + serializer.serialize_str(&hex_string) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_str(PrefixedHexVisitor) +} diff --git a/consensus/serde_utils/src/lib.rs b/consensus/serde_utils/src/lib.rs index 0016e67a3d..541a86d897 100644 --- a/consensus/serde_utils/src/lib.rs +++ b/consensus/serde_utils/src/lib.rs @@ -2,8 +2,11 @@ mod quoted_int; pub mod bytes_4_hex; pub mod hex; +pub mod hex_vec; +pub mod list_of_bytes_lists; pub mod quoted_u64_vec; pub mod u32_hex; +pub mod u64_hex_be; pub mod u8_hex; pub use quoted_int::{quoted_u32, quoted_u64, quoted_u8}; diff --git a/consensus/serde_utils/src/list_of_bytes_lists.rs b/consensus/serde_utils/src/list_of_bytes_lists.rs new file mode 100644 index 0000000000..b93321aa06 --- /dev/null +++ b/consensus/serde_utils/src/list_of_bytes_lists.rs @@ -0,0 +1,49 @@ +//! Formats `Vec` using quotes. +//! +//! E.g., `vec![0, 1, 2]` serializes as `["0", "1", "2"]`. +//! +//! Quotes can be optional during decoding. + +use crate::hex; +use serde::ser::SerializeSeq; +use serde::{de, Deserializer, Serializer}; + +pub struct ListOfBytesListVisitor; +impl<'a> serde::de::Visitor<'a> for ListOfBytesListVisitor { + type Value = Vec>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(formatter, "a list of 0x-prefixed byte lists") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'a>, + { + let mut vec = vec![]; + + while let Some(val) = seq.next_element::()? { + vec.push(hex::decode(&val).map_err(de::Error::custom)?); + } + + Ok(vec) + } +} + +pub fn serialize(value: &[Vec], serializer: S) -> Result +where + S: Serializer, +{ + let mut seq = serializer.serialize_seq(Some(value.len()))?; + for val in value { + seq.serialize_element(&hex::encode(val))?; + } + seq.end() +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + deserializer.deserialize_any(ListOfBytesListVisitor) +} diff --git a/consensus/serde_utils/src/quoted_int.rs b/consensus/serde_utils/src/quoted_int.rs index 5c3fa0f0aa..24edf1ebee 100644 --- a/consensus/serde_utils/src/quoted_int.rs +++ b/consensus/serde_utils/src/quoted_int.rs @@ -70,17 +70,6 @@ macro_rules! define_mod { pub value: T, } - /// Compositional wrapper type that allows quotes or no quotes. - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)] - #[serde(transparent)] - pub struct MaybeQuoted - where - T: From<$int> + Into<$int> + Copy + TryFrom, - { - #[serde(with = "self")] - pub value: T, - } - /// Serialize with quotes. pub fn serialize(value: &T, serializer: S) -> Result where diff --git a/consensus/serde_utils/src/u64_hex_be.rs b/consensus/serde_utils/src/u64_hex_be.rs new file mode 100644 index 0000000000..ce6941c937 --- /dev/null +++ b/consensus/serde_utils/src/u64_hex_be.rs @@ -0,0 +1,134 @@ +//! Formats `u64` as a 0x-prefixed, big-endian hex string. +//! +//! E.g., `0` serializes as `"0x0000000000000000"`. + +use serde::de::{self, Error, Visitor}; +use serde::{Deserializer, Serializer}; +use std::fmt; + +const BYTES_LEN: usize = 8; + +pub struct QuantityVisitor; +impl<'de> Visitor<'de> for QuantityVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a hex string") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + if !value.starts_with("0x") { + return Err(de::Error::custom("must start with 0x")); + } + + let stripped = value.trim_start_matches("0x"); + + if stripped.is_empty() { + Err(de::Error::custom(format!( + "quantity cannot be {}", + stripped + ))) + } else if stripped == "0" { + Ok(vec![0]) + } else if stripped.starts_with('0') { + Err(de::Error::custom("cannot have leading zero")) + } else if stripped.len() % 2 != 0 { + hex::decode(&format!("0{}", stripped)) + .map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e))) + } else { + hex::decode(&stripped).map_err(|e| de::Error::custom(format!("invalid hex ({:?})", e))) + } + } +} + +pub fn serialize(num: &u64, serializer: S) -> Result +where + S: Serializer, +{ + let raw = hex::encode(num.to_be_bytes()); + let trimmed = raw.trim_start_matches('0'); + + let hex = if trimmed == "" { "0" } else { &trimmed }; + + serializer.serialize_str(&format!("0x{}", &hex)) +} + +pub fn deserialize<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let decoded = deserializer.deserialize_str(QuantityVisitor)?; + + // TODO: this is not strict about byte length like other methods. + if decoded.len() > BYTES_LEN { + return Err(D::Error::custom(format!( + "expected max {} bytes for array, got {}", + BYTES_LEN, + decoded.len() + ))); + } + + let mut array = [0; BYTES_LEN]; + array[BYTES_LEN - decoded.len()..].copy_from_slice(&decoded); + Ok(u64::from_be_bytes(array)) +} + +#[cfg(test)] +mod test { + use serde::{Deserialize, Serialize}; + use serde_json; + + #[derive(Debug, PartialEq, Serialize, Deserialize)] + #[serde(transparent)] + struct Wrapper { + #[serde(with = "super")] + val: u64, + } + + #[test] + fn encoding() { + assert_eq!( + &serde_json::to_string(&Wrapper { val: 0 }).unwrap(), + "\"0x0\"" + ); + assert_eq!( + &serde_json::to_string(&Wrapper { val: 1 }).unwrap(), + "\"0x1\"" + ); + assert_eq!( + &serde_json::to_string(&Wrapper { val: 256 }).unwrap(), + "\"0x100\"" + ); + assert_eq!( + &serde_json::to_string(&Wrapper { val: 65 }).unwrap(), + "\"0x41\"" + ); + assert_eq!( + &serde_json::to_string(&Wrapper { val: 1024 }).unwrap(), + "\"0x400\"" + ); + } + + #[test] + fn decoding() { + assert_eq!( + serde_json::from_str::("\"0x0\"").unwrap(), + Wrapper { val: 0 }, + ); + assert_eq!( + serde_json::from_str::("\"0x41\"").unwrap(), + Wrapper { val: 65 }, + ); + assert_eq!( + serde_json::from_str::("\"0x400\"").unwrap(), + Wrapper { val: 1024 }, + ); + serde_json::from_str::("\"0x\"").unwrap_err(); + serde_json::from_str::("\"0x0400\"").unwrap_err(); + serde_json::from_str::("\"400\"").unwrap_err(); + serde_json::from_str::("\"ff\"").unwrap_err(); + } +}