poanetwork
/
parity-common
mirror of https://github.com/poanetwork/parity-common.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Avoid heap allocations in deserialization.

This commit is contained in:
Tomasz Drwięga 2018-02-02 12:42:32 +01:00
parent c43a3bfed9
commit 942fbf392b
No known key found for this signature in database
GPG Key ID: B2BA26B1C688F8FC
7 changed files with 162 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ethbloom/src/lib.rs
View File

@ -241,8 +241,9 @@ impl Serialize for Bloom {
#[cfg(feature="serialize")]
impl<'de> Deserialize<'de> for Bloom {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Exact(256))
.map(|x| (&*x).into())
let mut bytes = [0; 256];
ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Exact(&mut bytes))?;
Ok(Bloom(bytes))
}
}

5
ethereum-types/src/hash.rs
View File

@ -88,8 +88,9 @@ macro_rules! impl_serde {
#[cfg(feature="serialize")]
impl<'de> Deserialize<'de> for $name {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Exact($len))
.map(|x| (&*x).into())
let mut bytes = [0u8; $len];
ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Exact(&mut bytes))?;
Ok($name(bytes))
}
}
}

5
ethereum-types/src/uint.rs
View File

@ -343,8 +343,9 @@ macro_rules! impl_serde {
#[cfg(feature="serialize")]
impl<'de> Deserialize<'de> for $name {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> where D: Deserializer<'de> {
ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Between(0, $len * 8))
.map(|x| (&*x).into())
let mut bytes = [0u8; $len * 8];
let wrote = ethereum_types_serialize::deserialize_check_len(deserializer, ethereum_types_serialize::ExpectedLen::Between(0, &mut bytes))?;
Ok(bytes[0..wrote].into())
}
}
}

84
serialize/src/lib.rs
View File

@ -4,7 +4,7 @@ extern crate rustc_hex;
use std::fmt;
use serde::{de, Serializer, Deserializer};
use rustc_hex::{ToHex, FromHex};
use rustc_hex::{ToHex};
/// Serializes a slice of bytes.
pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error> where
@ -35,42 +35,33 @@ pub fn serialize_uint<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
/// Expected length of bytes vector.
#[derive(Debug, PartialEq, Eq)]
pub enum ExpectedLen {
/// Any length in bytes.
Any,
pub enum ExpectedLen<'a> {
/// Exact length in bytes.
Exact(usize),
/// A bytes length between (min; max].
Between(usize, usize),
Exact(&'a mut [u8]),
/// A bytes length between (min; slice.len()].
Between(usize, &'a mut [u8]),
}
impl fmt::Display for ExpectedLen {
impl<'a> fmt::Display for ExpectedLen<'a> {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
match *self {
ExpectedLen::Any => write!(fmt, "even length"),
ExpectedLen::Exact(v) => write!(fmt, "length of {}", v * 2),
ExpectedLen::Between(min, max) => write!(fmt, "length between ({}; {}]", min * 2, max * 2),
ExpectedLen::Exact(ref v) => write!(fmt, "length of {}", v.len() * 2),
ExpectedLen::Between(min, ref v) => write!(fmt, "length between ({}; {}]", min * 2, v.len() * 2),
}
}
}
/// Deserialize into vector of bytes.
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error> where
D: Deserializer<'de>,
{
deserialize_check_len(deserializer, ExpectedLen::Any)
}
/// Deserialize into vector of bytes with additional size check.
pub fn deserialize_check_len<'de, D>(deserializer: D, len: ExpectedLen) -> Result<Vec<u8>, D::Error> where
/// Returns number of bytes written.
pub fn deserialize_check_len<'a, 'de, D>(deserializer: D, len: ExpectedLen<'a>) -> Result<usize, D::Error> where
D: Deserializer<'de>,
{
struct Visitor {
len: ExpectedLen,
struct Visitor<'a> {
len: ExpectedLen<'a>,
}
impl<'a> de::Visitor<'a> for Visitor {
type Value = Vec<u8>;
impl<'a, 'b> de::Visitor<'b> for Visitor<'a> {
type Value = usize;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
write!(formatter, "a 0x-prefixed hex string with {}", self.len)
@ -82,10 +73,8 @@ pub fn deserialize_check_len<'de, D>(deserializer: D, len: ExpectedLen) -> Resul
}
let is_len_valid = match self.len {
// just make sure that we have all nibbles
ExpectedLen::Any => v.len() % 2 == 0,
ExpectedLen::Exact(len) => v.len() == 2 * len + 2,
ExpectedLen::Between(min, max) => v.len() <= 2 * max + 2 && v.len() > 2 * min + 2,
ExpectedLen::Exact(ref slice) => v.len() == 2 * slice.len() + 2,
ExpectedLen::Between(min, ref slice) => v.len() <= 2 * slice.len() + 2 && v.len() > 2 * min + 2,
};
if !is_len_valid {
@ -93,20 +82,45 @@ pub fn deserialize_check_len<'de, D>(deserializer: D, len: ExpectedLen) -> Resul
}
let bytes = match self.len {
ExpectedLen::Between(..) if v.len() % 2 != 0 => {
FromHex::from_hex(&*format!("0{}", &v[2..]))
},
_ => FromHex::from_hex(&v[2..])
};
ExpectedLen::Exact(slice) => slice,
ExpectedLen::Between(_, slice) => slice,
};
bytes.map_err(|e| E::custom(&format!("invalid hex value: {:?}", e)))
let mut modulus = v.len() % 2;
let mut buf = 0;
let mut pos = 0;
for (idx, byte) in v.bytes().enumerate().skip(2) {
buf <<= 4;
match byte {
b'A'...b'F' => buf |= byte - b'A' + 10,
b'a'...b'f' => buf |= byte - b'a' + 10,
b'0'...b'9' => buf |= byte - b'0',
b' '|b'\r'|b'\n'|b'\t' => {
buf >>= 4;
continue
}
_ => {
let ch = v[idx..].chars().next().unwrap();
return Err(E::custom(&format!("invalid hex character: {}, at {}", ch, idx)))
}
}
modulus += 1;
if modulus == 2 {
modulus = 0;
bytes[pos] = buf;
pos += 1;
}
}
Ok(pos)
}
fn visit_string<E: de::Error>(self, v: String) -> Result<Self::Value, E> {
self.visit_str(&v)
}
}
// TODO [ToDr] Use raw bytes if we switch to RLP / binencoding
// (visit_bytes, visit_bytes_buf)
deserializer.deserialize_str(Visitor { len })
}

3
tests/Cargo.toml
View File

@ -10,7 +10,8 @@ rustc_version = "0.2"
crunchy = "0.1.5"
ethereum-types = { path ="../ethereum-types", features = ["std", "heapsizeof"] }
quickcheck = "0.6"
serde_json = "1.0"
uint = { path = "../uint" }
[features]
use_asm = ["uint/use_asm", "ethereum-types/use_asm"]
use_asm = ["uint/use_asm", "ethereum-types/use_asm"]

9
tests/src/lib.rs
View File

@ -1,14 +1,19 @@
extern crate core;
extern crate ethereum_types;
#[cfg(test)]
#[macro_use]
extern crate uint;
extern crate ethereum_types;
#[cfg(test)]
#[macro_use]
extern crate crunchy;
#[cfg(test)]
#[macro_use]
extern crate quickcheck;
#[cfg(test)]
extern crate serde_json;
#[cfg(test)]
pub mod uint_tests;
pub mod uint_tests;
#[cfg(test)]
pub mod serialization;

95
tests/src/serialization.rs Normal file
View File

@ -0,0 +1,95 @@
use ethereum_types::{U256, U512, H160, H256};
use serde_json as ser;
macro_rules! test {
($name: ident, $test_name: ident) => {
#[test]
fn $test_name() {
let tests = vec![
($name::from(0), "0x0"),
($name::from(1), "0x1"),
($name::from(2), "0x2"),
($name::from(10), "0xa"),
($name::from(15), "0xf"),
($name::from(15), "0xf"),
($name::from(16), "0x10"),
($name::from(1_000), "0x3e8"),
($name::from(100_000), "0x186a0"),
($name::from(u64::max_value()), "0xffffffffffffffff"),
($name::from(u64::max_value()) + 1.into(), "0x10000000000000000"),
];
for (number, expected) in tests {
assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number).unwrap());
assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
}
// Invalid examples
assert!(ser::from_str::<$name>("\"0x\"").unwrap_err().is_data());
assert!(ser::from_str::<$name>("\"0xg\"").unwrap_err().is_data());
assert!(ser::from_str::<$name>("\"\"").unwrap_err().is_data());
assert!(ser::from_str::<$name>("\"10\"").unwrap_err().is_data());
assert!(ser::from_str::<$name>("\"0\"").unwrap_err().is_data());
}
}
}
test!(U256, test_u256);
test!(U512, test_u512);
#[test]
fn test_large_values() {
assert_eq!(
ser::to_string_pretty(&!U256::zero()).unwrap(),
"\"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\""
);
assert!(
ser::from_str::<U256>("\"0x1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff\"").unwrap_err().is_data()
);
}
#[test]
fn test_h160() {
let tests = vec![
(H160::from(0), "0x0000000000000000000000000000000000000000"),
(H160::from(2), "0x0000000000000000000000000000000000000002"),
(H160::from(15), "0x000000000000000000000000000000000000000f"),
(H160::from(16), "0x0000000000000000000000000000000000000010"),
(H160::from(1_000), "0x00000000000000000000000000000000000003e8"),
(H160::from(100_000), "0x00000000000000000000000000000000000186a0"),
(H160::from(u64::max_value()), "0x000000000000000000000000ffffffffffffffff"),
];
for (number, expected) in tests {
assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number).unwrap());
assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
}
}
#[test]
fn test_h256() {
let tests = vec![
(H256::from(0), "0x0000000000000000000000000000000000000000000000000000000000000000"),
(H256::from(2), "0x0000000000000000000000000000000000000000000000000000000000000002"),
(H256::from(15), "0x000000000000000000000000000000000000000000000000000000000000000f"),
(H256::from(16), "0x0000000000000000000000000000000000000000000000000000000000000010"),
(H256::from(1_000), "0x00000000000000000000000000000000000000000000000000000000000003e8"),
(H256::from(100_000), "0x00000000000000000000000000000000000000000000000000000000000186a0"),
(H256::from(u64::max_value()), "0x000000000000000000000000000000000000000000000000ffffffffffffffff"),
];
for (number, expected) in tests {
assert_eq!(format!("{:?}", expected), ser::to_string_pretty(&number).unwrap());
assert_eq!(number, ser::from_str(&format!("{:?}", expected)).unwrap());
}
}
#[test]
fn test_invalid() {
assert!(ser::from_str::<H256>("\"0x000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data());
assert!(ser::from_str::<H256>("\"0x000000000000000000000000000000000000000000000000000000000000000g\"").unwrap_err().is_data());
assert!(ser::from_str::<H256>("\"0x00000000000000000000000000000000000000000000000000000000000000000\"").unwrap_err().is_data());
assert!(ser::from_str::<H256>("\"\"").unwrap_err().is_data());
assert!(ser::from_str::<H256>("\"0\"").unwrap_err().is_data());
assert!(ser::from_str::<H256>("\"10\"").unwrap_err().is_data());
}