diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index 652bfeaf8d..c11ce46060 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" [dependencies] bincode = "1.0.0" bs58 = "0.2.0" +byteorder = "1.2.1" chrono = { version = "0.4.0", features = ["serde"] } generic-array = { version = "0.12.0", default-features = false, features = ["serde"] } log = "0.4.2" diff --git a/sdk/src/lib.rs b/sdk/src/lib.rs index d6c67b6975..dd6c482c37 100644 --- a/sdk/src/lib.rs +++ b/sdk/src/lib.rs @@ -12,6 +12,7 @@ pub mod native_program; pub mod packet; pub mod payment_plan; pub mod pubkey; +pub mod shortvec; pub mod signature; pub mod storage_program; pub mod system_instruction; diff --git a/sdk/src/shortvec.rs b/sdk/src/shortvec.rs new file mode 100644 index 0000000000..09b7fb28e6 --- /dev/null +++ b/sdk/src/shortvec.rs @@ -0,0 +1,373 @@ +use bincode::{deserialize_from, serialize_into, Error}; +use serde::Serialize; +use std::io::{Cursor, Read, Write}; +use std::mem::size_of; + +pub fn encode_len(writer: &mut W, len: usize) -> Result<(), Error> { + let mut rem_len = len; + loop { + let mut elem = (rem_len & 0x7f) as u8; + rem_len >>= 7; + if rem_len == 0 { + writer.write_all(&[elem])?; + break; + } else { + elem |= 0x80; + writer.write_all(&[elem])?; + } + } + Ok(()) +} + +pub fn decode_len(reader: &mut R) -> Result { + let mut len: usize = 0; + let mut size: usize = 0; + loop { + let mut elem = [0u8; 1]; + reader.read_exact(&mut elem)?; + len |= (elem[0] as usize & 0x7f) << (size * 7); + size += 1; + if elem[0] as usize & 0x80 == 0 { + break; + } + assert!(size <= size_of::() + 1); + } + Ok(len) +} + +pub fn serialize_vec_with( + mut writer: &mut Cursor<&mut [u8]>, + input: &[T], + ser_fn: fn(&mut Cursor<&mut [u8]>, &T) -> Result<(), Error>, +) -> Result<(), Error> { + encode_len(&mut writer, input.len())?; + input.iter().for_each(|e| ser_fn(&mut writer, &e).unwrap()); + Ok(()) +} + +pub fn serialize_vec_bytes(mut writer: W, input: &[u8]) -> Result<(), Error> { + let len = input.len(); + encode_len(&mut writer, len)?; + writer.write_all(input)?; + Ok(()) +} + +pub fn serialize_vec(mut writer: W, input: &[T]) -> Result<(), Error> +where + T: Serialize, +{ + let len = input.len(); + encode_len(&mut writer, len)?; + input + .iter() + .for_each(|e| serialize_into(&mut writer, &e).unwrap()); + Ok(()) +} + +pub fn deserialize_vec_bytes(mut reader: &mut R) -> Result, Error> { + let vec_len = decode_len(&mut reader)?; + let mut buf = vec![0; vec_len]; + reader.read_exact(&mut buf[..])?; + Ok(buf) +} + +pub fn deserialize_vec(mut reader: &mut R) -> Result, Error> +where + T: serde::de::DeserializeOwned, +{ + let vec_len = decode_len(&mut reader)?; + let mut vec: Vec = Vec::with_capacity(vec_len); + for _ in 0..vec_len { + let t: T = deserialize_from(&mut reader)?; + vec.push(t); + } + Ok(vec) +} + +pub fn deserialize_vec_with( + mut reader: &mut Cursor<&[u8]>, + deser_fn: fn(&mut Cursor<&[u8]>) -> Result, +) -> Result, Error> { + let vec_len = decode_len(&mut reader)?; + let mut vec: Vec = Vec::with_capacity(vec_len); + for _ in 0..vec_len { + let t: T = deser_fn(&mut reader)?; + vec.push(t); + } + Ok(vec) +} + +#[cfg(test)] +mod tests { + use super::*; + use bincode::{deserialize, serialize, serialized_size}; + use serde::ser::Serializer; + use serde::Deserialize; + use std::fmt; + use std::io::Cursor; + + #[test] + fn test_shortvec_encode_len() { + let mut buf = vec![0u8; size_of::() + 1]; + let mut wr = Cursor::new(&mut buf[..]); + encode_len(&mut wr, 0x0).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0u8]); + wr.set_position(0); + encode_len(&mut wr, 0x5).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0x5u8]); + wr.set_position(0); + encode_len(&mut wr, 0x7f).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0x7fu8]); + wr.set_position(0); + encode_len(&mut wr, 0x80).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0x80u8, 0x01u8]); + wr.set_position(0); + encode_len(&mut wr, 0xff).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0xffu8, 0x01u8]); + wr.set_position(0); + encode_len(&mut wr, 0x100).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0x80u8, 0x02u8]); + wr.set_position(0); + encode_len(&mut wr, 0x7fff).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0xffu8, 0xffu8, 0x01u8]); + wr.set_position(0); + encode_len(&mut wr, 0x200000).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0x80u8, 0x80u8, 0x80u8, 0x01u8]); + wr.set_position(0); + encode_len(&mut wr, 0x7ffffffff).unwrap(); + let vec = wr.get_ref()[..wr.position() as usize].to_vec(); + assert_eq!(vec, vec![0xffu8, 0xffu8, 0xffu8, 0xffu8, 0x7fu8]); + } + + #[test] + #[should_panic] + fn test_shortvec_decode_zero_len() { + let mut buf = vec![]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0); + assert_eq!(rd.position(), 0); + } + + #[test] + fn test_shortvec_decode_len() { + let mut buf = vec![0u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0); + assert_eq!(rd.position(), 1); + let mut buf = vec![5u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 5); + assert_eq!(rd.position(), 1); + let mut buf = vec![0x7fu8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x7f); + assert_eq!(rd.position(), 1); + let mut buf = vec![0x80u8, 0x01u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x80); + assert_eq!(rd.position(), 2); + let mut buf = vec![0xffu8, 0x01u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0xff); + assert_eq!(rd.position(), 2); + let mut buf = vec![0x80u8, 0x02u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x100); + assert_eq!(rd.position(), 2); + let mut buf = vec![0xffu8, 0xffu8, 0x01u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x7fff); + assert_eq!(rd.position(), 3); + let mut buf = vec![0x80u8, 0x80u8, 0x80u8, 0x01u8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x200000); + assert_eq!(rd.position(), 4); + let mut buf = vec![0xffu8, 0xffu8, 0xffu8, 0xffu8, 0x7fu8]; + let mut rd = Cursor::new(&mut buf[..]); + assert_eq!(decode_len(&mut rd).unwrap(), 0x7ffffffff); + assert_eq!(rd.position(), 5); + } + + #[test] + fn test_shortvec_u8() { + let vec: Vec = vec![4; 32]; + let mut buf = vec![0u8; serialized_size(&vec).unwrap() as usize + 1]; + let mut wr = Cursor::new(&mut buf[..]); + serialize_vec_bytes(&mut wr, &vec).unwrap(); + let size = wr.position() as usize; + let ser = &wr.into_inner()[..size]; + assert_eq!(ser.len(), vec.len() + 1); + let mut rd = Cursor::new(&ser[..]); + let deser: Vec = deserialize_vec_bytes(&mut rd).unwrap(); + assert_eq!(vec, deser); + } + + #[derive(Debug, Eq, PartialEq)] + struct TestVec { + vec_u8: Vec, + id: u8, + vec_u32: Vec, + } + + impl TestVec { + pub fn serialize_with( + mut writer: &mut Cursor<&mut [u8]>, + tv: &TestVec, + ) -> Result<(), Error> { + serialize_vec(&mut writer, &tv.vec_u8).unwrap(); + serialize_into(&mut writer, &tv.id).unwrap(); + serialize_vec(&mut writer, &tv.vec_u32).unwrap(); + Ok(()) + } + pub fn from_bytes(mut reader: &mut Cursor<&[u8]>) -> Result { + let vec_u8: Vec = deserialize_vec(&mut reader).unwrap(); + let id: u8 = deserialize_from(&mut reader).unwrap(); + let vec_u32: Vec = deserialize_vec(&mut reader).unwrap(); + Ok(TestVec { + vec_u8, + id, + vec_u32, + }) + } + pub fn serialized_size(&self) -> u64 { + let mut buf = vec![0u8; size_of::() + 1]; + let mut size = size_of::(); + let mut wr = Cursor::new(&mut buf[..]); + encode_len(&mut wr, self.vec_u8.len()).unwrap(); + size += wr.position() as usize + self.vec_u8.len() * size_of::(); + size += size_of::(); + wr.set_position(0); + encode_len(&mut wr, self.vec_u32.len()).unwrap(); + size += wr.position() as usize + self.vec_u32.len() * size_of::(); + size as u64 + } + } + + impl Serialize for TestVec { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::Error; + let mut buf = vec![0u8; self.serialized_size() as usize]; + let mut wr = Cursor::new(&mut buf[..]); + serialize_vec(&mut wr, &self.vec_u8).map_err(Error::custom)?; + serialize_into(&mut wr, &self.id).map_err(Error::custom)?; + serialize_vec(&mut wr, &self.vec_u32).map_err(Error::custom)?; + let len = wr.position() as usize; + serializer.serialize_bytes(&wr.into_inner()[..len]) + } + } + + struct TestVecVisitor; + impl<'a> serde::de::Visitor<'a> for TestVecVisitor { + type Value = TestVec; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Expecting TestVec") + } + fn visit_bytes(self, data: &[u8]) -> Result + where + E: serde::de::Error, + { + use serde::de::Error; + let mut rd = Cursor::new(&data[..]); + let v1: Vec = deserialize_vec(&mut rd).map_err(Error::custom)?; + let id: u8 = deserialize_from(&mut rd).map_err(Error::custom)?; + let v2: Vec = deserialize_vec(&mut rd).map_err(Error::custom)?; + Ok(TestVec { + vec_u8: v1, + id, + vec_u32: v2, + }) + } + } + impl<'de> Deserialize<'de> for TestVec { + fn deserialize(deserializer: D) -> Result + where + D: ::serde::Deserializer<'de>, + { + deserializer.deserialize_bytes(TestVecVisitor) + } + } + + #[test] + fn test_shortvec_testvec() { + let tvec: TestVec = TestVec { + vec_u8: vec![4; 32], + id: 5, + vec_u32: vec![6; 32], + }; + let size = tvec.serialized_size() as usize; + assert_eq!( + size, + tvec.vec_u8.len() + 1 + 1 + (tvec.vec_u32.len() * 4) + 1 + 8 + ); + let ser = serialize(&tvec).unwrap(); + assert_eq!( + ser.len(), + tvec.vec_u8.len() + 1 + 1 + (tvec.vec_u32.len() * 4) + 1 + 8 + ); + let deser = deserialize(&ser).unwrap(); + assert_eq!(tvec, deser); + } + + #[derive(Serialize, Deserialize, Debug, Eq, PartialEq)] + struct TestVecArr { + data: Vec, + } + impl TestVecArr { + fn new() -> Self { + TestVecArr { data: vec![] } + } + fn serialized_size(tvec: &TestVecArr) -> u64 { + let mut buf = vec![0u8; size_of::() + 1]; + let mut wr = Cursor::new(&mut buf[..]); + encode_len(&mut wr, tvec.data.len()).unwrap(); + let size: u64 = tvec + .data + .iter() + .map(|tv| TestVec::serialized_size(&tv)) + .sum(); + size_of::() as u64 + size + wr.position() + } + } + #[test] + fn test_shortvec_testvec_with() { + let tvec1 = TestVec { + vec_u8: vec![4; 32], + id: 5, + vec_u32: vec![6; 32], + }; + let tvec2 = TestVec { + vec_u8: vec![7; 32], + id: 8, + vec_u32: vec![9; 32], + }; + let tvec3 = TestVec { + vec_u8: vec![], + id: 10, + vec_u32: vec![11; 32], + }; + let mut tvecarr: TestVecArr = TestVecArr::new(); + tvecarr.data.push(tvec1); + tvecarr.data.push(tvec2); + tvecarr.data.push(tvec3); + let mut buf = vec![0u8; TestVecArr::serialized_size(&tvecarr) as usize]; + let mut wr = Cursor::new(&mut buf[..]); + serialize_vec_with(&mut wr, &tvecarr.data, TestVec::serialize_with).unwrap(); + let size = wr.position() as usize; + let ser = &wr.into_inner()[..size]; + let mut rd = Cursor::new(&ser[..]); + let deser = deserialize_vec_with(&mut rd, TestVec::from_bytes).unwrap(); + assert_eq!(tvecarr.data, deser); + } +} diff --git a/sdk/src/transaction.rs b/sdk/src/transaction.rs index 3f99c49288..d595f14dbe 100644 --- a/sdk/src/transaction.rs +++ b/sdk/src/transaction.rs @@ -1,14 +1,20 @@ //! The `transaction` module provides functionality for creating log transactions. use crate::hash::{Hash, Hasher}; +use crate::packet::PACKET_DATA_SIZE; use crate::pubkey::Pubkey; +use crate::shortvec::{ + deserialize_vec_bytes, deserialize_vec_with, encode_len, serialize_vec_bytes, + serialize_vec_with, +}; use crate::signature::{Keypair, KeypairUtil, Signature}; -use bincode::serialize; -use serde::Serialize; +use bincode::{serialize, Error}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use serde::{Deserialize, Serialize, Serializer}; +use std::fmt; +use std::io::{Cursor, Read, Write}; use std::mem::size_of; -pub const SIG_OFFSET: usize = size_of::(); - /// An instruction to execute a program #[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] pub struct Instruction { @@ -29,10 +35,50 @@ impl Instruction { accounts, } } + + pub fn serialize_with( + mut writer: &mut Cursor<&mut [u8]>, + ix: &Instruction, + ) -> Result<(), Error> { + writer.write_all(&[ix.program_ids_index])?; + serialize_vec_bytes(&mut writer, &ix.accounts[..])?; + serialize_vec_bytes(&mut writer, &ix.userdata[..])?; + Ok(()) + } + + pub fn deserialize_from(mut reader: &mut Cursor<&[u8]>) -> Result { + let mut buf = [0]; + reader.read_exact(&mut buf)?; + let program_ids_index = buf[0]; + let accounts = deserialize_vec_bytes(&mut reader)?; + let userdata = deserialize_vec_bytes(&mut reader)?; + Ok(Instruction { + program_ids_index, + accounts, + userdata, + }) + } + + pub fn serialized_size(&self) -> Result { + let mut buf = [0; size_of::() + 1]; + let mut wr = Cursor::new(&mut buf[..]); + let mut size = size_of::(); + + let len = self.accounts.len(); + encode_len(&mut wr, len)?; + size += wr.position() as usize + (len * size_of::()); + + let len = self.userdata.len(); + wr.set_position(0); + encode_len(&mut wr, len)?; + size += wr.position() as usize + (len * size_of::()); + + Ok(size as u64) + } } /// An atomic transaction -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Clone)] +#[derive(Debug, PartialEq, Eq, Clone)] pub struct Transaction { /// A set of digital signatures of `account_keys`, `program_ids`, `last_id`, `fee` and `instructions`, signed by the first /// signatures.len() keys of account_keys @@ -148,20 +194,20 @@ impl Transaction { } /// Get the transaction data to sign. pub fn get_sign_data(&self) -> Vec { - let mut data = serialize(&self.account_keys).expect("serialize account_keys"); - - let last_id_data = serialize(&self.last_id).expect("serialize last_id"); - data.extend_from_slice(&last_id_data); - - let fee_data = serialize(&self.fee).expect("serialize fee"); - data.extend_from_slice(&fee_data); - - let program_ids = serialize(&self.program_ids).expect("serialize program_ids"); - data.extend_from_slice(&program_ids); - - let instructions = serialize(&self.instructions).expect("serialize instructions"); - data.extend_from_slice(&instructions); - data + let mut buf = vec![0u8; PACKET_DATA_SIZE]; + let mut wr = Cursor::new(&mut buf[..]); + serialize_vec_with(&mut wr, &self.account_keys, Transaction::serialize_pubkey) + .expect("serialize account_keys"); + wr.write_all(self.last_id.as_ref()) + .expect("serialize last_id"); + wr.write_u64::(self.fee) + .expect("serialize fee"); + serialize_vec_with(&mut wr, &self.program_ids, Transaction::serialize_pubkey) + .expect("serialize program_ids"); + serialize_vec_with(&mut wr, &self.instructions, Instruction::serialize_with) + .expect("serialize instructions"); + let len = wr.position() as usize; + wr.into_inner()[..len].to_vec() } /// Sign this transaction. @@ -208,12 +254,141 @@ impl Transaction { .for_each(|tx| hasher.hash(&tx.signatures[0].as_ref())); hasher.result() } + + pub fn serialized_size(&self) -> Result { + let mut buf = [0u8; size_of::() + 1]; + let mut wr = Cursor::new(&mut buf[..]); + let mut size = size_of::(); + + let len = self.signatures.len(); + encode_len(&mut wr, len)?; + size += wr.position() as usize + (len * size_of::()); + + let len = self.account_keys.len(); + wr.set_position(0); + encode_len(&mut wr, len)?; + size += wr.position() as usize + (len * size_of::()); + + size += size_of::(); + + size += size_of::(); + + let len = self.program_ids.len(); + wr.set_position(0); + encode_len(&mut wr, len)?; + size += wr.position() as usize + (len * size_of::()); + + let len = self.instructions.len(); + wr.set_position(0); + encode_len(&mut wr, len)?; + size += wr.position() as usize; + let inst_size: u64 = self + .instructions + .iter() + .map(|ix| ix.serialized_size().unwrap()) + .sum(); + Ok(size as u64 + inst_size) + } + + fn serialize_signature(writer: &mut Cursor<&mut [u8]>, sig: &Signature) -> Result<(), Error> { + writer.write_all(sig.as_ref())?; + Ok(()) + } + + fn serialize_pubkey(writer: &mut Cursor<&mut [u8]>, key: &Pubkey) -> Result<(), Error> { + writer.write_all(key.as_ref())?; + Ok(()) + } + + fn deserialize_signature(reader: &mut Cursor<&[u8]>) -> Result { + let mut buf = [0; size_of::()]; + reader.read_exact(&mut buf)?; + Ok(Signature::new(&buf)) + } + + fn deserialize_pubkey(reader: &mut Cursor<&[u8]>) -> Result { + let mut buf = [0; size_of::()]; + reader.read_exact(&mut buf)?; + Ok(Pubkey::new(&buf)) + } +} + +impl Serialize for Transaction { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + use serde::ser::Error; + let mut buf = vec![0u8; self.serialized_size().unwrap() as usize]; + let mut wr = Cursor::new(&mut buf[..]); + serialize_vec_with(&mut wr, &self.signatures, Transaction::serialize_signature) + .map_err(Error::custom)?; + serialize_vec_with(&mut wr, &self.account_keys, Transaction::serialize_pubkey) + .map_err(Error::custom)?; + wr.write_all(self.last_id.as_ref()).map_err(Error::custom)?; + wr.write_u64::(self.fee) + .map_err(Error::custom)?; + serialize_vec_with(&mut wr, &self.program_ids, Transaction::serialize_pubkey) + .map_err(Error::custom)?; + serialize_vec_with(&mut wr, &self.instructions, Instruction::serialize_with) + .map_err(Error::custom)?; + let size = wr.position() as usize; + serializer.serialize_bytes(&wr.into_inner()[..size]) + } +} + +struct TransactionVisitor; +impl<'a> serde::de::Visitor<'a> for TransactionVisitor { + type Value = Transaction; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Expecting Instruction") + } + fn visit_bytes(self, data: &[u8]) -> Result + where + E: serde::de::Error, + { + use serde::de::Error; + let mut rd = Cursor::new(&data[..]); + let signatures: Vec = + deserialize_vec_with(&mut rd, Transaction::deserialize_signature) + .map_err(Error::custom)?; + let account_keys: Vec = + deserialize_vec_with(&mut rd, Transaction::deserialize_pubkey) + .map_err(Error::custom)?; + let mut buf = [0; size_of::()]; + rd.read_exact(&mut buf).map_err(Error::custom)?; + let last_id: Hash = Hash::new(&buf); + let fee = rd.read_u64::().map_err(Error::custom)?; + let program_ids: Vec = + deserialize_vec_with(&mut rd, Transaction::deserialize_pubkey) + .map_err(Error::custom)?; + let instructions: Vec = + deserialize_vec_with(&mut rd, Instruction::deserialize_from).map_err(Error::custom)?; + Ok(Transaction { + signatures, + account_keys, + last_id, + fee, + program_ids, + instructions, + }) + } +} + +impl<'de> Deserialize<'de> for Transaction { + fn deserialize(deserializer: D) -> Result + where + D: ::serde::Deserializer<'de>, + { + deserializer.deserialize_bytes(TransactionVisitor) + } } #[cfg(test)] mod tests { use super::*; - use bincode::serialize; + use bincode::deserialize; #[test] fn test_refs() { @@ -287,6 +462,53 @@ mod tests { assert!(!tx.verify_refs()); } + #[test] + fn test_transaction_serialize() { + let keypair = Keypair::new(); + let program_id = Pubkey::new(&[4; 32]); + let to = Pubkey::new(&[5; 32]); + let tx = Transaction::new( + &keypair, + &[keypair.pubkey(), to], + program_id, + &(1u8, 2u8, 3u8), + Hash::default(), + 99, + ); + + let ser = serialize(&tx).unwrap(); + let deser = deserialize(&ser).unwrap(); + assert_eq!(tx, deser); + } + + #[test] + fn test_transaction_serialized_size() { + let keypair = Keypair::new(); + let program_id = Pubkey::new(&[4; 32]); + let to = Pubkey::new(&[5; 32]); + let tx = Transaction::new( + &keypair, + &[keypair.pubkey(), to], + program_id, + &(1u8, 2u8, 3u8), + Hash::default(), + 99, + ); + let req_size = size_of::() + + 1 + + (tx.signatures.len() * size_of::()) + + 1 + + (tx.account_keys.len() * size_of::()) + + size_of::() + + size_of::() + + 1 + + (tx.program_ids.len() * size_of::()) + + 1 + + tx.instructions[0].serialized_size().unwrap() as usize; + let size = tx.serialized_size().unwrap() as usize; + assert_eq!(req_size, size); + } + /// Detect binary changes in the serialized transaction userdata, which could have a downstream /// affect on SDKs and DApps #[test] @@ -321,19 +543,18 @@ mod tests { assert_eq!( serialize(&tx).unwrap(), vec![ - 1, 0, 0, 0, 0, 0, 0, 0, 213, 248, 255, 179, 219, 217, 130, 31, 27, 85, 33, 217, 62, - 28, 180, 204, 186, 141, 178, 150, 153, 184, 205, 87, 123, 128, 101, 254, 222, 111, - 152, 17, 153, 210, 169, 1, 81, 208, 254, 64, 229, 205, 145, 10, 213, 241, 255, 31, - 184, 52, 242, 148, 213, 131, 241, 165, 144, 181, 18, 4, 58, 171, 44, 11, 3, 0, 0, - 0, 0, 0, 0, 0, 36, 100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, - 189, 171, 118, 241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 36, - 100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189, 171, 118, - 241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 1, 1, 1, 4, 5, 6, 7, - 8, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 7, 6, 5, 4, 1, 1, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 99, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 4, 5, 6, 7, 8, 9, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 8, 7, 6, 5, 4, 2, 2, 2, 1, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3 + 245, 0, 0, 0, 0, 0, 0, 0, 1, 151, 224, 239, 74, 248, 111, 129, 62, 193, 150, 178, + 53, 242, 136, 228, 153, 16, 245, 127, 217, 6, 122, 114, 165, 224, 243, 191, 164, + 197, 107, 71, 41, 57, 132, 240, 19, 166, 239, 109, 168, 225, 215, 1, 59, 120, 57, + 141, 103, 243, 182, 221, 176, 161, 153, 217, 129, 87, 178, 228, 151, 57, 163, 75, + 13, 3, 36, 100, 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189, + 171, 118, 241, 90, 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 36, 100, + 158, 252, 33, 161, 97, 185, 62, 89, 99, 195, 250, 249, 187, 189, 171, 118, 241, 90, + 248, 14, 68, 219, 231, 62, 157, 5, 142, 27, 210, 117, 1, 1, 1, 4, 5, 6, 7, 8, 9, 9, + 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 8, 7, 6, 5, 4, 1, 1, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, + 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 2, 4, 5, 6, 7, 8, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 9, 8, 7, 6, 5, 4, 2, 2, 2, 1, 0, 3, 0, 1, 2, 3, 1, 2, 3 ], ); } diff --git a/src/chacha.rs b/src/chacha.rs index cecc8f3fcc..08d673b597 100644 --- a/src/chacha.rs +++ b/src/chacha.rs @@ -166,7 +166,7 @@ mod tests { assert_eq!( hasher.result(), Hash::new(&hex!( - "1ef70b5491a5f2b05ebeb0f92a03c131a7a78622f3643064d6d3c52a0c083175" + "16b1159b112b11d7a2fb7b0471797ab079bce7e0e86b8a879474616abb61e5aa" )), ); remove_file(out_path).unwrap(); diff --git a/src/entry.rs b/src/entry.rs index e469c3f1b9..6840e85f89 100644 --- a/src/entry.rs +++ b/src/entry.rs @@ -96,7 +96,7 @@ impl Entry { } }; - let size = serialized_size(&entry).unwrap(); + let size = Entry::serialized_size(&entry.transactions[..]); if size > BLOB_DATA_SIZE as u64 { panic!( "Serialized entry size too large: {} ({} transactions):", @@ -125,11 +125,13 @@ impl Entry { /// Estimate serialized_size of Entry without creating an Entry. pub fn serialized_size(transactions: &[Transaction]) -> u64 { - let txs_size = serialized_size(transactions).unwrap(); - + let txs_size: u64 = transactions + .iter() + .map(|tx| tx.serialized_size().unwrap()) + .sum(); // tick_height+num_hashes + id + txs - (2 * size_of::() + size_of::()) as u64 + txs_size + (3 * size_of::() + size_of::()) as u64 + txs_size } pub fn num_will_fit(transactions: &[Transaction]) -> usize { @@ -436,7 +438,7 @@ pub fn make_large_test_entries(num_entries: usize) -> Vec { one, ); - let serialized_size = serialized_size(&vec![&tx]).unwrap(); + let serialized_size = tx.serialized_size().unwrap(); let num_txs = BLOB_DATA_SIZE / serialized_size as usize; let txs = vec![tx; num_txs]; let entry = next_entries(&one, 1, txs)[0].clone(); @@ -676,8 +678,8 @@ mod tests { }; let tx_large = Transaction::budget_new(&keypair, keypair.pubkey(), 1, next_id); - let tx_small_size = serialized_size(&tx_small).unwrap() as usize; - let tx_large_size = serialized_size(&tx_large).unwrap() as usize; + let tx_small_size = tx_small.serialized_size().unwrap() as usize; + let tx_large_size = tx_large.serialized_size().unwrap() as usize; let entry_size = serialized_size(&Entry { tick_height: 0, num_hashes: 0, diff --git a/src/packet.rs b/src/packet.rs index f44235795e..32e5419727 100644 --- a/src/packet.rs +++ b/src/packet.rs @@ -2,7 +2,7 @@ use crate::counter::Counter; use crate::recvmmsg::{recv_mmsg, NUM_RCVMMSGS}; use crate::result::{Error, Result}; -use bincode::{deserialize, serialize}; +use bincode::{deserialize, serialize, serialize_into}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use log::Level; use serde::Serialize; @@ -226,9 +226,9 @@ pub fn to_packets_chunked(xs: &[T], chunks: usize) -> Vec Vec> { } pub fn get_packet_offsets(packet: &Packet, current_offset: u32) -> (u32, u32, u32, u32) { - // Read in u64 as the size of signatures array - let mut rdr = io::Cursor::new(&packet.data[TX_OFFSET..size_of::()]); - let sig_len = rdr.read_u64::().unwrap() as u32; + // Read in the size of signatures array + let start_offset = TX_OFFSET + size_of::(); + let mut rd = Cursor::new(&packet.data[start_offset..]); + let sig_len = decode_len(&mut rd).unwrap(); + let sig_size = rd.position() as usize; + let msg_start_offset = start_offset + sig_size + sig_len * size_of::(); + let mut rd = Cursor::new(&packet.data[msg_start_offset..]); + let _ = decode_len(&mut rd).unwrap(); + let pubkey_size = rd.position() as usize; + let pubkey_offset = current_offset as usize + msg_start_offset + pubkey_size; - let msg_start_offset = - current_offset + size_of::() as u32 + sig_len * size_of::() as u32; - let pubkey_offset = msg_start_offset + size_of::() as u32; + let sig_start = start_offset + current_offset as usize + sig_size; - let sig_start = TX_OFFSET as u32 + size_of::() as u32; - - (sig_len, sig_start, msg_start_offset, pubkey_offset) + ( + sig_len as u32, + sig_start as u32, + current_offset + msg_start_offset as u32, + pubkey_offset as u32, + ) } pub fn generate_offsets(batches: &[SharedPackets]) -> Result { @@ -151,14 +159,14 @@ pub fn generate_offsets(batches: &[SharedPackets]) -> Result { p.read().unwrap().packets.iter().for_each(|packet| { let current_offset = current_packet as u32 * size_of::() as u32; - let (sig_len, _sig_start, msg_start_offset, pubkey_offset) = + let (sig_len, sig_start, msg_start_offset, pubkey_offset) = get_packet_offsets(packet, current_offset); let mut pubkey_offset = pubkey_offset; sig_lens.push(sig_len); trace!("pubkey_offset: {}", pubkey_offset); - let mut sig_offset = current_offset + size_of::() as u32; + let mut sig_offset = sig_start; for _ in 0..sig_len { signature_offsets.push(sig_offset); sig_offset += size_of::() as u32; @@ -334,7 +342,9 @@ mod tests { use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_instruction::SystemInstruction; use solana_sdk::system_program; - use solana_sdk::transaction::{Instruction, Transaction, SIG_OFFSET}; + use solana_sdk::transaction::{Instruction, Transaction}; + + const SIG_OFFSET: usize = std::mem::size_of::() + 1; pub fn memfind(a: &[A], b: &[A]) -> Option { assert!(a.len() >= b.len()); @@ -414,9 +424,9 @@ mod tests { let (sig_len, sig_start, msg_start_offset, pubkey_offset) = sigverify::get_packet_offsets(&packet, 0); assert_eq!(sig_len, 1); - assert_eq!(sig_start, 8); - assert_eq!(msg_start_offset, 72); - assert_eq!(pubkey_offset, 80); + assert_eq!(sig_start, 9); + assert_eq!(msg_start_offset, 73); + assert_eq!(pubkey_offset, 74); } fn generate_packet_vec( diff --git a/src/thin_client.rs b/src/thin_client.rs index e7ae5081fd..b2ea5758ca 100644 --- a/src/thin_client.rs +++ b/src/thin_client.rs @@ -9,7 +9,7 @@ use crate::gossip_service::GossipService; use crate::packet::PACKET_DATA_SIZE; use crate::result::{Error, Result}; use crate::rpc_request::{RpcClient, RpcRequest, RpcRequestHandler}; -use bincode::serialize; +use bincode::serialize_into; use bs58; use hashbrown::HashMap; use log::Level; @@ -94,10 +94,12 @@ impl ThinClient { /// Send a signed Transaction to the server for processing. This method /// does not wait for a response. pub fn transfer_signed(&self, tx: &Transaction) -> io::Result { - let data = serialize(&tx).expect("serialize Transaction in pub fn transfer_signed"); - assert!(data.len() < PACKET_DATA_SIZE); + let mut buf = vec![0; tx.serialized_size().unwrap() as usize]; + let mut wr = std::io::Cursor::new(&mut buf[..]); + serialize_into(&mut wr, &tx).expect("serialize Transaction in pub fn transfer_signed"); + assert!(buf.len() < PACKET_DATA_SIZE); self.transactions_socket - .send_to(&data, &self.transactions_addr)?; + .send_to(&buf[..], &self.transactions_addr)?; Ok(tx.signatures[0]) } @@ -110,9 +112,11 @@ impl ThinClient { ) -> io::Result { for x in 0..tries { tx.sign(&[&keypair], self.get_last_id()); - let data = serialize(&tx).expect("serialize Transaction in pub fn transfer_signed"); + let mut buf = vec![0; tx.serialized_size().unwrap() as usize]; + let mut wr = std::io::Cursor::new(&mut buf[..]); + serialize_into(&mut wr, &tx).expect("serialize Transaction in pub fn transfer_signed"); self.transactions_socket - .send_to(&data, &self.transactions_addr)?; + .send_to(&buf[..], &self.transactions_addr)?; if self.poll_for_signature(&tx.signatures[0]).is_ok() { return Ok(tx.signatures[0]); } @@ -443,7 +447,7 @@ mod tests { use crate::mint::Mint; use crate::storage_stage::STORAGE_ROTATE_TEST_COUNT; use crate::vote_signer_proxy::VoteSignerProxy; - use bincode::deserialize; + use bincode::{deserialize, serialize}; use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_instruction::SystemInstruction; use solana_sdk::vote_program::VoteProgram;