Optimize Transaction/Instruction serialization with custom routine (#2515)

* Optimize transaction serialization with custom routine to reduce the serialized size.

* Update serialized_size to accept self as parameter

* Optimize serialize / deserialize operations
This commit is contained in:
Sathish 2019-01-24 21:14:15 -08:00 committed by GitHub
parent 9845aec007
commit e0f046b7a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 681 additions and 69 deletions

View File

@ -11,6 +11,7 @@ edition = "2018"
[dependencies]
bincode = "1.0.0"
bs58 = "0.2.0"
byteorder = "1.2.1"
chrono = { version = "0.4.0", features = ["serde"] }
generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
log = "0.4.2"

View File

@ -12,6 +12,7 @@ pub mod native_program;
pub mod packet;
pub mod payment_plan;
pub mod pubkey;
pub mod shortvec;
pub mod signature;
pub mod storage_program;
pub mod system_instruction;

373
sdk/src/shortvec.rs Normal file
View File

@ -0,0 +1,373 @@
use bincode::{deserialize_from, serialize_into, Error};
use serde::Serialize;
use std::io::{Cursor, Read, Write};
use std::mem::size_of;
pub fn encode_len<W: Write>(writer: &mut W, len: usize) -> Result<(), Error> {
let mut rem_len = len;
loop {
let mut elem = (rem_len & 0x7f) as u8;
rem_len >>= 7;
if rem_len == 0 {
writer.write_all(&[elem])?;
break;
} else {
elem |= 0x80;
writer.write_all(&[elem])?;
}
}
Ok(())
}
pub fn decode_len<R: Read>(reader: &mut R) -> Result<usize, Error> {
let mut len: usize = 0;
let mut size: usize = 0;
loop {
let mut elem = [0u8; 1];
reader.read_exact(&mut elem)?;
len |= (elem[0] as usize & 0x7f) << (size * 7);
size += 1;
if elem[0] as usize & 0x80 == 0 {
break;
}
assert!(size <= size_of::<usize>() + 1);
}
Ok(len)
}
pub fn serialize_vec_with<T>(
mut writer: &mut Cursor<&mut [u8]>,
input: &[T],
ser_fn: fn(&mut Cursor<&mut [u8]>, &T) -> Result<(), Error>,
) -> Result<(), Error> {
encode_len(&mut writer, input.len())?;
input.iter().for_each(|e| ser_fn(&mut writer, &e).unwrap());
Ok(())
}
pub fn serialize_vec_bytes<W: Write>(mut writer: W, input: &[u8]) -> Result<(), Error> {
let len = input.len();
encode_len(&mut writer, len)?;
writer.write_all(input)?;
Ok(())
}
pub fn serialize_vec<W: Write, T>(mut writer: W, input: &[T]) -> Result<(), Error>
where
T: Serialize,
{
let len = input.len();
encode_len(&mut writer, len)?;
input
.iter()
.for_each(|e| serialize_into(&mut writer, &e).unwrap());
Ok(())
}
pub fn deserialize_vec_bytes<R: Read>(mut reader: &mut R) -> Result<Vec<u8>, Error> {
let vec_len = decode_len(&mut reader)?;
let mut buf = vec![0; vec_len];
reader.read_exact(&mut buf[..])?;
Ok(buf)
}
pub fn deserialize_vec<R: Read, T>(mut reader: &mut R) -> Result<Vec<T>, Error>
where
T: serde::de::DeserializeOwned,
{
let vec_len = decode_len(&mut reader)?;
let mut vec: Vec<T> = Vec::with_capacity(vec_len);
for _ in 0..vec_len {
let t: T = deserialize_from(&mut reader)?;
vec.push(t);
}
Ok(vec)
}
pub fn deserialize_vec_with<T>(
mut reader: &mut Cursor<&[u8]>,
deser_fn: fn(&mut Cursor<&[u8]>) -> Result<T, Error>,
) -> Result<Vec<T>, Error> {
let vec_len = decode_len(&mut reader)?;
let mut vec: Vec<T> = Vec::with_capacity(vec_len);
for _ in 0..vec_len {
let t: T = deser_fn(&mut reader)?;
vec.push(t);
}
Ok(vec)
}
#[cfg(test)]
mod tests {
use super::*;
use bincode::{deserialize, serialize, serialized_size};
use serde::ser::Serializer;
use serde::Deserialize;
use std::fmt;
use std::io::Cursor;
#[test]
fn test_shortvec_encode_len() {
let mut buf = vec![0u8; size_of::<u64>() + 1];
let mut wr = Cursor::new(&mut buf[..]);
encode_len(&mut wr, 0x0).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0u8]);
wr.set_position(0);
encode_len(&mut wr, 0x5).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0x5u8]);
wr.set_position(0);
encode_len(&mut wr, 0x7f).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0x7fu8]);
wr.set_position(0);
encode_len(&mut wr, 0x80).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0x80u8, 0x01u8]);
wr.set_position(0);
encode_len(&mut wr, 0xff).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0xffu8, 0x01u8]);
wr.set_position(0);
encode_len(&mut wr, 0x100).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0x80u8, 0x02u8]);
wr.set_position(0);
encode_len(&mut wr, 0x7fff).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0xffu8, 0xffu8, 0x01u8]);
wr.set_position(0);
encode_len(&mut wr, 0x200000).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0x80u8, 0x80u8, 0x80u8, 0x01u8]);
wr.set_position(0);
encode_len(&mut wr, 0x7ffffffff).unwrap();
let vec = wr.get_ref()[..wr.position() as usize].to_vec();
assert_eq!(vec, vec![0xffu8, 0xffu8, 0xffu8, 0xffu8, 0x7fu8]);
}
#[test]
#[should_panic]
fn test_shortvec_decode_zero_len() {
let mut buf = vec![];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0);
assert_eq!(rd.position(), 0);
}
#[test]
fn test_shortvec_decode_len() {
let mut buf = vec![0u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0);
assert_eq!(rd.position(), 1);
let mut buf = vec![5u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 5);
assert_eq!(rd.position(), 1);
let mut buf = vec![0x7fu8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x7f);
assert_eq!(rd.position(), 1);
let mut buf = vec![0x80u8, 0x01u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x80);
assert_eq!(rd.position(), 2);
let mut buf = vec![0xffu8, 0x01u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0xff);
assert_eq!(rd.position(), 2);
let mut buf = vec![0x80u8, 0x02u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x100);
assert_eq!(rd.position(), 2);
let mut buf = vec![0xffu8, 0xffu8, 0x01u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x7fff);
assert_eq!(rd.position(), 3);
let mut buf = vec![0x80u8, 0x80u8, 0x80u8, 0x01u8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x200000);
assert_eq!(rd.position(), 4);
let mut buf = vec![0xffu8, 0xffu8, 0xffu8, 0xffu8, 0x7fu8];
let mut rd = Cursor::new(&mut buf[..]);
assert_eq!(decode_len(&mut rd).unwrap(), 0x7ffffffff);
assert_eq!(rd.position(), 5);
}
#[test]
fn test_shortvec_u8() {
let vec: Vec<u8> = vec![4; 32];
let mut buf = vec![0u8; serialized_size(&vec).unwrap() as usize + 1];
let mut wr = Cursor::new(&mut buf[..]);
serialize_vec_bytes(&mut wr, &vec).unwrap();
let size = wr.position() as usize;
let ser = &wr.into_inner()[..size];
assert_eq!(ser.len(), vec.len() + 1);
let mut rd = Cursor::new(&ser[..]);
let deser: Vec<u8> = deserialize_vec_bytes(&mut rd).unwrap();
assert_eq!(vec, deser);
}
#[derive(Debug, Eq, PartialEq)]
struct TestVec {
vec_u8: Vec<u8>,
id: u8,
vec_u32: Vec<u32>,
}
impl TestVec {
pub fn serialize_with(
mut writer: &mut Cursor<&mut [u8]>,
tv: &TestVec,
) -> Result<(), Error> {
serialize_vec(&mut writer, &tv.vec_u8).unwrap();
serialize_into(&mut writer, &tv.id).unwrap();
serialize_vec(&mut writer, &tv.vec_u32).unwrap();
Ok(())
}
pub fn from_bytes(mut reader: &mut Cursor<&[u8]>) -> Result<Self, Error> {
let vec_u8: Vec<u8> = deserialize_vec(&mut reader).unwrap();
let id: u8 = deserialize_from(&mut reader).unwrap();
let vec_u32: Vec<u32> = deserialize_vec(&mut reader).unwrap();
Ok(TestVec {
vec_u8,
id,
vec_u32,
})
}
pub fn serialized_size(&self) -> u64 {
let mut buf = vec![0u8; size_of::<u64>() + 1];
let mut size = size_of::<u64>();
let mut wr = Cursor::new(&mut buf[..]);
encode_len(&mut wr, self.vec_u8.len()).unwrap();
size += wr.position() as usize + self.vec_u8.len() * size_of::<u8>();
size += size_of::<u8>();
wr.set_position(0);
encode_len(&mut wr, self.vec_u32.len()).unwrap();
size += wr.position() as usize + self.vec_u32.len() * size_of::<u32>();
size as u64
}
}
impl Serialize for TestVec {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
use serde::ser::Error;
let mut buf = vec![0u8; self.serialized_size() as usize];
let mut wr = Cursor::new(&mut buf[..]);
serialize_vec(&mut wr, &self.vec_u8).map_err(Error::custom)?;
serialize_into(&mut wr, &self.id).map_err(Error::custom)?;
serialize_vec(&mut wr, &self.vec_u32).map_err(Error::custom)?;
let len = wr.position() as usize;
serializer.serialize_bytes(&wr.into_inner()[..len])
}
}
struct TestVecVisitor;
impl<'a> serde::de::Visitor<'a> for TestVecVisitor {
type Value = TestVec;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("Expecting TestVec")
}
fn visit_bytes<E>(self, data: &[u8]) -> Result<TestVec, E>
where
E: serde::de::Error,
{
use serde::de::Error;
let mut rd = Cursor::new(&data[..]);
let v1: Vec<u8> = deserialize_vec(&mut rd).map_err(Error::custom)?;
let id: u8 = deserialize_from(&mut rd).map_err(Error::custom)?;
let v2: Vec<u32> = deserialize_vec(&mut rd).map_err(Error::custom)?;
Ok(TestVec {
vec_u8: v1,
id,
vec_u32: v2,
})
}
}
impl<'de> Deserialize<'de> for TestVec {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: ::serde::Deserializer<'de>,
{
deserializer.deserialize_bytes(TestVecVisitor)
}
}
#[test]
fn test_shortvec_testvec() {
let tvec: TestVec = TestVec {
vec_u8: vec![4; 32],
id: 5,
vec_u32: vec![6; 32],
};
let size = tvec.serialized_size() as usize;
assert_eq!(
size,
tvec.vec_u8.len() + 1 + 1 + (tvec.vec_u32.len() * 4) + 1 + 8
);
let ser = serialize(&tvec).unwrap();
assert_eq!(
ser.len(),
tvec.vec_u8.len() + 1 + 1 + (tvec.vec_u32.len() * 4) + 1 + 8
);
let deser = deserialize(&ser).unwrap();
assert_eq!(tvec, deser);
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
struct TestVecArr {
data: Vec<TestVec>,
}
impl TestVecArr {
fn new() -> Self {
TestVecArr { data: vec![] }
}
fn serialized_size(tvec: &TestVecArr) -> u64 {
let mut buf = vec![0u8; size_of::<u64>() + 1];
let mut wr = Cursor::new(&mut buf[..]);
encode_len(&mut wr, tvec.data.len()).unwrap();
let size: u64 = tvec
.data
.iter()
.map(|tv| TestVec::serialized_size(&tv))
.sum();
size_of::<u64>() as u64 + size + wr.position()
}
}
#[test]
fn test_shortvec_testvec_with() {
let tvec1 = TestVec {
vec_u8: vec![4; 32],
id: 5,
vec_u32: vec![6; 32],
};
let tvec2 = TestVec {
vec_u8: vec![7; 32],
id: 8,
vec_u32: vec![9; 32],
};
let tvec3 = TestVec {
vec_u8: vec![],
id: 10,
vec_u32: vec![11; 32],
};
let mut tvecarr: TestVecArr = TestVecArr::new();
tvecarr.data.push(tvec1);
tvecarr.data.push(tvec2);
tvecarr.data.push(tvec3);
let mut buf = vec![0u8; TestVecArr::serialized_size(&tvecarr) as usize];
let mut wr = Cursor::new(&mut buf[..]);
serialize_vec_with(&mut wr, &tvecarr.data, TestVec::serialize_with).unwrap();
let size = wr.position() as usize;
let ser = &wr.into_inner()[..size];
let mut rd = Cursor::new(&ser[..]);
let deser = deserialize_vec_with(&mut rd, TestVec::from_bytes).unwrap();
assert_eq!(tvecarr.data, deser);
}
}

View File

@ -1,14 +1,20 @@
//! The `transaction` module provides functionality for creating log transactions.
use crate::hash::{Hash, Hasher};
use crate::packet::PACKET_DATA_SIZE;
use crate::pubkey::Pubkey;
use crate::shortvec::{
deserialize_vec_bytes, deserialize_vec_with, encode_len, serialize_vec_bytes,
serialize_vec_with,
};
use crate::signature::{Keypair, KeypairUtil, Signature};
use bincode::serialize;
use serde::Serialize;
use bincode::{serialize, Error};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use serde::{Deserialize, Serialize, Serializer};
use std::fmt;
use std::io::{Cursor, Read, Write};
use std::mem::size_of;
pub const SIG_OFFSET: usize = size_of::<u64>();
/// An instruction to execute a program
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
pub struct Instruction {
@ -29,10 +35,50 @@ impl Instruction {
accounts,
}
}
pub fn serialize_with(
mut writer: &mut Cursor<&mut [u8]>,
ix: &Instruction,
) -> Result<(), Error> {
writer.write_all(&[ix.program_ids_index])?;
serialize_vec_bytes(&mut writer, &ix.accounts[..])?;
serialize_vec_bytes(&mut writer, &ix.userdata[..])?;
Ok(())
}
pub fn deserialize_from(mut reader: &mut Cursor<&[u8]>) -> Result<Self, Error> {
let mut buf = [0];
reader.read_exact(&mut buf)?;
let program_ids_index = buf[0];
let accounts = deserialize_vec_bytes(&mut reader)?;
let userdata = deserialize_vec_bytes(&mut reader)?;
Ok(Instruction {
program_ids_index,
accounts,
userdata,
})
}
pub fn serialized_size(&self) -> Result<u64, Error> {
let mut buf = [0; size_of::<u64>() + 1];
let mut wr = Cursor::new(&mut buf[..]);
let mut size = size_of::<u8>();
let len = self.accounts.len();
encode_len(&mut wr, len)?;
size += wr.position() as usize + (len * size_of::<u8>());
let len = self.userdata.len();
wr.set_position(0);
encode_len(&mut wr, len)?;
size += wr.position() as usize + (len * size_of::<u8>());
Ok(size as u64)
}
}
/// An atomic transaction
#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)]
#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Transaction {
/// A set of digital signatures of `account_keys`, `program_ids`, `last_id`, `fee` and `instructions`, signed by the first
/// signatures.len() keys of account_keys
@ -148,20 +194,20 @@ impl Transaction {
}
/// Get the transaction data to sign.
pub fn get_sign_data(&self) -> Vec<u8> {
let mut data = serialize(&self.account_keys).expect("serialize account_keys");
let last_id_data = serialize(&self.last_id).expect("serialize last_id");
data.extend_from_slice(&last_id_data);
let fee_data = serialize(&self.fee).expect("serialize fee");
data.extend_from_slice(&fee_data);
let program_ids = serialize(&self.program_ids).expect("serialize program_ids");
data.extend_from_slice(&program_ids);
let instructions = serialize(&self.instructions).expect("serialize instructions");
data.extend_from_slice(&instructions);
data
let mut buf = vec![0u8; PACKET_DATA_SIZE];
let mut wr = Cursor::new(&mut buf[..]);
serialize_vec_with(&mut wr, &self.account_keys, Transaction::serialize_pubkey)
.expect("serialize account_keys");
wr.write_all(self.last_id.as_ref())
.expect("serialize last_id");
wr.write_u64::<LittleEndian>(self.fee)
.expect("serialize fee");
serialize_vec_with(&mut wr, &self.program_ids, Transaction::serialize_pubkey)
.expect("serialize program_ids");
serialize_vec_with(&mut wr, &self.instructions, Instruction::serialize_with)
.expect("serialize instructions");
let len = wr.position() as usize;
wr.into_inner()[..len].to_vec()
}
/// Sign this transaction.
@ -208,12 +254,141 @@ impl Transaction {
.for_each(|tx| hasher.hash(&tx.signatures[0].as_ref()));
hasher.result()
}
pub fn serialized_size(&self) -> Result<u64, Error> {
let mut buf = [0u8; size_of::<u64>() + 1];
let mut wr = Cursor::new(&mut buf[..]);
let mut size = size_of::<u64>();
let len = self.signatures.len();
encode_len(&mut wr, len)?;
size += wr.position() as usize + (len * size_of::<Signature>());
let len = self.account_keys.len();
wr.set_position(0);
encode_len(&mut wr, len)?;
size += wr.position() as usize + (len * size_of::<Pubkey>());
size += size_of::<Hash>();
size += size_of::<u64>();
let len = self.program_ids.len();
wr.set_position(0);
encode_len(&mut wr, len)?;
size += wr.position() as usize + (len * size_of::<Pubkey>());
let len = self.instructions.len();
wr.set_position(0);
encode_len(&mut wr, len)?;
size += wr.position() as usize;
let inst_size: u64 = self
.instructions
.iter()
.map(|ix| ix.serialized_size().unwrap())
.sum();
Ok(size as u64 + inst_size)
}
fn serialize_signature(writer: &mut Cursor<&mut [u8]>, sig: &Signature) -> Result<(), Error> {
writer.write_all(sig.as_ref())?;
Ok(())
}
fn serialize_pubkey(writer: &mut Cursor<&mut [u8]>, key: &Pubkey) -> Result<(), Error> {
writer.write_all(key.as_ref())?;
Ok(())
}
fn deserialize_signature(reader: &mut Cursor<&[u8]>) -> Result<Signature, Error> {
let mut buf = [0; size_of::<Signature>()];
reader.read_exact(&mut buf)?;
Ok(Signature::new(&buf))
}
fn deserialize_pubkey(reader: &mut Cursor<&[u8]>) -> Result<Pubkey, Error> {
let mut buf = [0; size_of::<Pubkey>()];
reader.read_exact(&mut buf)?;
Ok(Pubkey::new(&buf))
}
}
impl Serialize for Transaction {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
use serde::ser::Error;
let mut buf = vec![0u8; self.serialized_size().unwrap() as usize];
let mut wr = Cursor::new(&mut buf[..]);
serialize_vec_with(&mut wr, &self.signatures, Transaction::serialize_signature)
.map_err(Error::custom)?;
serialize_vec_with(&mut wr, &self.account_keys, Transaction::serialize_pubkey)
.map_err(Error::custom)?;
wr.write_all(self.last_id.as_ref()).map_err(Error::custom)?;
wr.write_u64::<LittleEndian>(self.fee)
.map_err(Error::custom)?;
serialize_vec_with(&mut wr, &self.program_ids, Transaction::serialize_pubkey)
.map_err(Error::custom)?;
serialize_vec_with(&mut wr, &self.instructions, Instruction::serialize_with)
.map_err(Error::custom)?;
let size = wr.position() as usize;
serializer.serialize_bytes(&wr.into_inner()[..size])
}
}
struct TransactionVisitor;
impl<'a> serde::de::Visitor<'a> for TransactionVisitor {
type Value = Transaction;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("Expecting Instruction")
}
fn visit_bytes<E>(self, data: &[u8]) -> Result<Transaction, E>
where
E: serde::de::Error,
{
use serde::de::Error;
let mut rd = Cursor::new(&data[..]);
let signatures: Vec<Signature> =
deserialize_vec_with(&mut rd, Transaction::deserialize_signature)
.map_err(Error::custom)?;
let account_keys: Vec<Pubkey> =
deserialize_vec_with(&mut rd, Transaction::deserialize_pubkey)
.map_err(Error::custom)?;
let mut buf = [0; size_of::<Hash>()];
rd.read_exact(&mut buf).map_err(Error::custom)?;
let last_id: Hash = Hash::new(&buf);
let fee = rd.read_u64::<LittleEndian>().map_err(Error::custom)?;
let program_ids: Vec<Pubkey> =
deserialize_vec_with(&mut rd, Transaction::deserialize_pubkey)
.map_err(Error::custom)?;
let instructions: Vec<Instruction> =
deserialize_vec_with(&mut rd, Instruction::deserialize_from).map_err(Error::custom)?;
Ok(Transaction {
signatures,
account_keys,
last_id,
fee,
program_ids,
instructions,
})
}
}
impl<'de> Deserialize<'de> for Transaction {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: ::serde::Deserializer<'de>,
{
deserializer.deserialize_bytes(TransactionVisitor)
}
}
#[cfg(test)]
mod tests {
use super::*;
use bincode::serialize;
use bincode::deserialize;
#[test]
fn test_refs() {
@ -287,6 +462,53 @@ mod tests {
assert!(!tx.verify_refs());
}
#[test]
fn test_transaction_serialize() {
let keypair = Keypair::new();
let program_id = Pubkey::new(&[4; 32]);
let to = Pubkey::new(&[5; 32]);
let tx = Transaction::new(
&keypair,
&[keypair.pubkey(), to],
program_id,
&(1u8, 2u8, 3u8),
Hash::default(),
99,
);
let ser = serialize(&tx).unwrap();
let deser = deserialize(&ser).unwrap();
assert_eq!(tx, deser);
}
#[test]
fn test_transaction_serialized_size() {
let keypair = Keypair::new();
let program_id = Pubkey::new(&[4; 32]);
let to = Pubkey::new(&[5; 32]);
let tx = Transaction::new(
&keypair,
&[keypair.pubkey(), to],
program_id,
&(1u8, 2u8, 3u8),
Hash::default(),
99,
);
let req_size = size_of::<u64>()
+ 1
+ (tx.signatures.len() * size_of::<Signature>())
+ 1
+ (tx.account_keys.len() * size_of::<Pubkey>())
+ size_of::<Hash>()
+ size_of::<u64>()
+ 1
+ (tx.program_ids.len() * size_of::<Pubkey>())
+ 1
+ tx.instructions[0].serialized_size().unwrap() as usize;
let size = tx.serialized_size().unwrap() as usize;
assert_eq!(req_size, size);
}
/// Detect binary changes in the serialized transaction userdata, which could have a downstream
/// affect on SDKs and DApps
#[test]
@ -321,19 +543,18 @@ mod tests {
assert_eq!(
serialize(&tx).unwrap(),
vec![
1, 0, 0, 0, 0, 0, 0, 0, 213, 248, 255, 179, 219, 217, 130, 31, 27, 85, 33, 217, 62,
28, 180, 204, 186, 141, 178, 150, 153, 184, 205, 87, 123, 128, 101, 254, 222, 111,
152, 17, 153, 210, 169, 1, 81, 208, 254, 64, 229, 205, 145, 10, 213, 241, 255, 31,
184, 52, 242, 148, 213, 131, 241, 165, 144, 181, 18, 4, 58, 171, 44, 11, 3, 0, 0,
0, 0, 0, 0, 0, 36, 100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187,
189, 171, 118, 241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 36,
100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189, 171, 118,
241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 1, 1, 1, 4, 5, 6, 7,
8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 7, 6, 5, 4, 1, 1, 1, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 4, 5, 6, 7, 8, 9, 1,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 8, 7, 6, 5, 4, 2, 2, 2, 1, 0, 0, 0, 0, 0,
0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3
245, 0, 0, 0, 0, 0, 0, 0, 1, 151, 224, 239, 74, 248, 111, 129, 62, 193, 150, 178,
53, 242, 136, 228, 153, 16, 245, 127, 217, 6, 122, 114, 165, 224, 243, 191, 164,
197, 107, 71, 41, 57, 132, 240, 19, 166, 239, 109, 168, 225, 215, 1, 59, 120, 57,
141, 103, 243, 182, 221, 176, 161, 153, 217, 129, 87, 178, 228, 151, 57, 163, 75,
13, 3, 36, 100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189,
171, 118, 241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 36, 100,
158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189, 171, 118, 241, 90,
248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 1, 1, 1, 4, 5, 6, 7, 8, 9, 9,
9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 7, 6, 5, 4, 1, 1, 1, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99,
0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 4, 5, 6, 7, 8, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 9, 8, 7, 6, 5, 4, 2, 2, 2, 1, 0, 3, 0, 1, 2, 3, 1, 2, 3
],
);
}

View File

@ -166,7 +166,7 @@ mod tests {
assert_eq!(
hasher.result(),
Hash::new(&hex!(
"1ef70b5491a5f2b05ebeb0f92a03c131a7a78622f3643064d6d3c52a0c083175"
"16b1159b112b11d7a2fb7b0471797ab079bce7e0e86b8a879474616abb61e5aa"
)),
);
remove_file(out_path).unwrap();

View File

@ -96,7 +96,7 @@ impl Entry {
}
};
let size = serialized_size(&entry).unwrap();
let size = Entry::serialized_size(&entry.transactions[..]);
if size > BLOB_DATA_SIZE as u64 {
panic!(
"Serialized entry size too large: {} ({} transactions):",
@ -125,11 +125,13 @@ impl Entry {
/// Estimate serialized_size of Entry without creating an Entry.
pub fn serialized_size(transactions: &[Transaction]) -> u64 {
let txs_size = serialized_size(transactions).unwrap();
let txs_size: u64 = transactions
.iter()
.map(|tx| tx.serialized_size().unwrap())
.sum();
// tick_height+num_hashes + id + txs
(2 * size_of::<u64>() + size_of::<Hash>()) as u64 + txs_size
(3 * size_of::<u64>() + size_of::<Hash>()) as u64 + txs_size
}
pub fn num_will_fit(transactions: &[Transaction]) -> usize {
@ -436,7 +438,7 @@ pub fn make_large_test_entries(num_entries: usize) -> Vec<Entry> {
one,
);
let serialized_size = serialized_size(&vec![&tx]).unwrap();
let serialized_size = tx.serialized_size().unwrap();
let num_txs = BLOB_DATA_SIZE / serialized_size as usize;
let txs = vec![tx; num_txs];
let entry = next_entries(&one, 1, txs)[0].clone();
@ -676,8 +678,8 @@ mod tests {
};
let tx_large = Transaction::budget_new(&keypair, keypair.pubkey(), 1, next_id);
let tx_small_size = serialized_size(&tx_small).unwrap() as usize;
let tx_large_size = serialized_size(&tx_large).unwrap() as usize;
let tx_small_size = tx_small.serialized_size().unwrap() as usize;
let tx_large_size = tx_large.serialized_size().unwrap() as usize;
let entry_size = serialized_size(&Entry {
tick_height: 0,
num_hashes: 0,

View File

@ -2,7 +2,7 @@
use crate::counter::Counter;
use crate::recvmmsg::{recv_mmsg, NUM_RCVMMSGS};
use crate::result::{Error, Result};
use bincode::{deserialize, serialize};
use bincode::{deserialize, serialize, serialize_into};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use log::Level;
use serde::Serialize;
@ -226,9 +226,9 @@ pub fn to_packets_chunked<T: Serialize>(xs: &[T], chunks: usize) -> Vec<SharedPa
.packets
.resize(x.len(), Default::default());
for (i, o) in x.iter().zip(p.write().unwrap().packets.iter_mut()) {
let v = serialize(&i).expect("serialize request");
let len = v.len();
o.data[..len].copy_from_slice(&v);
let mut wr = io::Cursor::new(&mut o.data[..]);
serialize_into(&mut wr, &i).expect("serialize request");
let len = wr.position() as usize;
o.meta.size = len;
}
out.push(p);

View File

@ -7,13 +7,13 @@
use crate::counter::Counter;
use crate::packet::{Packet, SharedPackets};
use crate::result::Result;
use byteorder::{LittleEndian, ReadBytesExt};
use log::Level;
use solana_sdk::pubkey::Pubkey;
use solana_sdk::shortvec::decode_len;
use solana_sdk::signature::Signature;
#[cfg(test)]
use solana_sdk::transaction::Transaction;
use std::io;
use std::io::Cursor;
use std::mem::size_of;
use std::sync::atomic::AtomicUsize;
@ -126,17 +126,25 @@ pub fn ed25519_verify(batches: &[SharedPackets]) -> Vec<Vec<u8>> {
}
pub fn get_packet_offsets(packet: &Packet, current_offset: u32) -> (u32, u32, u32, u32) {
// Read in u64 as the size of signatures array
let mut rdr = io::Cursor::new(&packet.data[TX_OFFSET..size_of::<u64>()]);
let sig_len = rdr.read_u64::<LittleEndian>().unwrap() as u32;
// Read in the size of signatures array
let start_offset = TX_OFFSET + size_of::<u64>();
let mut rd = Cursor::new(&packet.data[start_offset..]);
let sig_len = decode_len(&mut rd).unwrap();
let sig_size = rd.position() as usize;
let msg_start_offset = start_offset + sig_size + sig_len * size_of::<Signature>();
let mut rd = Cursor::new(&packet.data[msg_start_offset..]);
let _ = decode_len(&mut rd).unwrap();
let pubkey_size = rd.position() as usize;
let pubkey_offset = current_offset as usize + msg_start_offset + pubkey_size;
let msg_start_offset =
current_offset + size_of::<u64>() as u32 + sig_len * size_of::<Signature>() as u32;
let pubkey_offset = msg_start_offset + size_of::<u64>() as u32;
let sig_start = start_offset + current_offset as usize + sig_size;
let sig_start = TX_OFFSET as u32 + size_of::<u64>() as u32;
(sig_len, sig_start, msg_start_offset, pubkey_offset)
(
sig_len as u32,
sig_start as u32,
current_offset + msg_start_offset as u32,
pubkey_offset as u32,
)
}
pub fn generate_offsets(batches: &[SharedPackets]) -> Result<TxOffsets> {
@ -151,14 +159,14 @@ pub fn generate_offsets(batches: &[SharedPackets]) -> Result<TxOffsets> {
p.read().unwrap().packets.iter().for_each(|packet| {
let current_offset = current_packet as u32 * size_of::<Packet>() as u32;
let (sig_len, _sig_start, msg_start_offset, pubkey_offset) =
let (sig_len, sig_start, msg_start_offset, pubkey_offset) =
get_packet_offsets(packet, current_offset);
let mut pubkey_offset = pubkey_offset;
sig_lens.push(sig_len);
trace!("pubkey_offset: {}", pubkey_offset);
let mut sig_offset = current_offset + size_of::<u64>() as u32;
let mut sig_offset = sig_start;
for _ in 0..sig_len {
signature_offsets.push(sig_offset);
sig_offset += size_of::<Signature>() as u32;
@ -334,7 +342,9 @@ mod tests {
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::system_instruction::SystemInstruction;
use solana_sdk::system_program;
use solana_sdk::transaction::{Instruction, Transaction, SIG_OFFSET};
use solana_sdk::transaction::{Instruction, Transaction};
const SIG_OFFSET: usize = std::mem::size_of::<u64>() + 1;
pub fn memfind<A: Eq>(a: &[A], b: &[A]) -> Option<usize> {
assert!(a.len() >= b.len());
@ -414,9 +424,9 @@ mod tests {
let (sig_len, sig_start, msg_start_offset, pubkey_offset) =
sigverify::get_packet_offsets(&packet, 0);
assert_eq!(sig_len, 1);
assert_eq!(sig_start, 8);
assert_eq!(msg_start_offset, 72);
assert_eq!(pubkey_offset, 80);
assert_eq!(sig_start, 9);
assert_eq!(msg_start_offset, 73);
assert_eq!(pubkey_offset, 74);
}
fn generate_packet_vec(

View File

@ -9,7 +9,7 @@ use crate::gossip_service::GossipService;
use crate::packet::PACKET_DATA_SIZE;
use crate::result::{Error, Result};
use crate::rpc_request::{RpcClient, RpcRequest, RpcRequestHandler};
use bincode::serialize;
use bincode::serialize_into;
use bs58;
use hashbrown::HashMap;
use log::Level;
@ -94,10 +94,12 @@ impl ThinClient {
/// Send a signed Transaction to the server for processing. This method
/// does not wait for a response.
pub fn transfer_signed(&self, tx: &Transaction) -> io::Result<Signature> {
let data = serialize(&tx).expect("serialize Transaction in pub fn transfer_signed");
assert!(data.len() < PACKET_DATA_SIZE);
let mut buf = vec![0; tx.serialized_size().unwrap() as usize];
let mut wr = std::io::Cursor::new(&mut buf[..]);
serialize_into(&mut wr, &tx).expect("serialize Transaction in pub fn transfer_signed");
assert!(buf.len() < PACKET_DATA_SIZE);
self.transactions_socket
.send_to(&data, &self.transactions_addr)?;
.send_to(&buf[..], &self.transactions_addr)?;
Ok(tx.signatures[0])
}
@ -110,9 +112,11 @@ impl ThinClient {
) -> io::Result<Signature> {
for x in 0..tries {
tx.sign(&[&keypair], self.get_last_id());
let data = serialize(&tx).expect("serialize Transaction in pub fn transfer_signed");
let mut buf = vec![0; tx.serialized_size().unwrap() as usize];
let mut wr = std::io::Cursor::new(&mut buf[..]);
serialize_into(&mut wr, &tx).expect("serialize Transaction in pub fn transfer_signed");
self.transactions_socket
.send_to(&data, &self.transactions_addr)?;
.send_to(&buf[..], &self.transactions_addr)?;
if self.poll_for_signature(&tx.signatures[0]).is_ok() {
return Ok(tx.signatures[0]);
}
@ -443,7 +447,7 @@ mod tests {
use crate::mint::Mint;
use crate::storage_stage::STORAGE_ROTATE_TEST_COUNT;
use crate::vote_signer_proxy::VoteSignerProxy;
use bincode::deserialize;
use bincode::{deserialize, serialize};
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::system_instruction::SystemInstruction;
use solana_sdk::vote_program::VoteProgram;