Version transaction message and add new message format (#18725)

* Version transaction message and add new message format

* Update abi digest due to message path change

* Update v0.rs

Fix comment

* Update original.rs

* Update message versions name and address map indexes field name

* s/original/legacy

* update comment

* cargo fmt

* Update abi digest due to legacy rename
This commit is contained in:
Justin Starry 2021-08-09 22:03:39 -07:00 committed by GitHub
parent f506da5e19
commit 8817f59b6e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 689 additions and 25 deletions

View File

@ -252,7 +252,7 @@ pub fn make_accounts_hashes_message(
pub(crate) type Ping = ping_pong::Ping<[u8; GOSSIP_PING_TOKEN_SIZE]>;
// TODO These messages should go through the gpu pipeline for spam filtering
#[frozen_abi(digest = "4khbdefBamDC8XpdahkW4bzkGX6N5c8PcHp3kBXJGg46")]
#[frozen_abi(digest = "AqKhoLDkFr85WPiZnXG4bcRwHU4qSSyDZ3MQZLk3cnJf")]
#[derive(Serialize, Deserialize, Debug, AbiEnumVisitor, AbiExample)]
#[allow(clippy::large_enum_variant)]
pub(crate) enum Protocol {

View File

@ -9,6 +9,7 @@ use crate::{
bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable,
hash::Hash,
instruction::{AccountMeta, CompiledInstruction, Instruction},
message::MessageHeader,
pubkey::Pubkey,
short_vec, system_instruction, system_program, sysvar,
};
@ -18,7 +19,7 @@ use std::{convert::TryFrom, str::FromStr};
lazy_static! {
// Copied keys over since direct references create cyclical dependency.
static ref BUILTIN_PROGRAMS_KEYS: [Pubkey; 10] = {
pub static ref BUILTIN_PROGRAMS_KEYS: [Pubkey; 10] = {
let parse = |s| Pubkey::from_str(s).unwrap();
[
parse("Config1111111111111111111111111111111111111"),
@ -163,27 +164,8 @@ fn get_program_ids(instructions: &[Instruction]) -> Vec<Pubkey> {
.collect()
}
pub const MESSAGE_HEADER_LENGTH: usize = 3;
#[frozen_abi(digest = "BVC5RhetsNpheGipt5rUrkR6RDDUHtD5sCLK1UjymL4S")]
#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)]
#[serde(rename_all = "camelCase")]
pub struct MessageHeader {
/// The number of signatures required for this message to be considered valid. The
/// signatures must match the first `num_required_signatures` of `account_keys`.
/// NOTE: Serialization-related changes must be paired with the direct read at sigverify.
pub num_required_signatures: u8,
/// The last num_readonly_signed_accounts of the signed keys are read-only accounts. Programs
/// may process multiple transactions that load read-only accounts within a single PoH entry,
/// but are not permitted to credit or debit lamports or modify account data. Transactions
/// targeting the same read-write account are evaluated sequentially.
pub num_readonly_signed_accounts: u8,
/// The last num_readonly_unsigned_accounts of the unsigned keys are read-only accounts.
pub num_readonly_unsigned_accounts: u8,
}
// NOTE: Serialization-related changes must be paired with the custom serialization
// for versioned messages in the `RemainingLegacyMessage` struct.
#[frozen_abi(digest = "2KnLEqfLcTBQqitE22Pp8JYkaqVVbAkGbCfdeHoyxcAU")]
#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)]
#[serde(rename_all = "camelCase")]
@ -508,7 +490,7 @@ impl Message {
#[cfg(test)]
mod tests {
use super::*;
use crate::{hash, instruction::AccountMeta};
use crate::{hash, instruction::AccountMeta, message::MESSAGE_HEADER_LENGTH};
use std::collections::HashSet;
#[test]

View File

@ -0,0 +1,27 @@
//! A library for generating a message from a sequence of instructions
mod legacy;
mod v0;
mod versions;
pub use legacy::Message;
pub const MESSAGE_HEADER_LENGTH: usize = 3;
#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)]
#[serde(rename_all = "camelCase")]
pub struct MessageHeader {
/// The number of signatures required for this message to be considered valid. The
/// signatures must match the first `num_required_signatures` of `account_keys`.
/// NOTE: Serialization-related changes must be paired with the direct read at sigverify.
pub num_required_signatures: u8,
/// The last num_readonly_signed_accounts of the signed keys are read-only accounts. Programs
/// may process multiple transactions that load read-only accounts within a single PoH entry,
/// but are not permitted to credit or debit lamports or modify account data. Transactions
/// targeting the same read-write account are evaluated sequentially.
pub num_readonly_signed_accounts: u8,
/// The last num_readonly_unsigned_accounts of the unsigned keys are read-only accounts.
pub num_readonly_unsigned_accounts: u8,
}

View File

@ -0,0 +1,384 @@
#![allow(clippy::integer_arithmetic)]
use crate::{
hash::Hash,
instruction::CompiledInstruction,
message::MessageHeader,
pubkey::Pubkey,
sanitize::{Sanitize, SanitizeError},
short_vec,
};
/// Indexes that are mapped to addresses using an on-chain address map for
/// succinctly loading readonly and writable accounts.
#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)]
#[serde(rename_all = "camelCase")]
pub struct AddressMapIndexes {
#[serde(with = "short_vec")]
pub writable: Vec<u8>,
#[serde(with = "short_vec")]
pub readonly: Vec<u8>,
}
/// Transaction message format which supports succinct account loading with
/// indexes for on-chain address maps.
#[derive(Serialize, Deserialize, Default, Debug, PartialEq, Eq, Clone, AbiExample)]
#[serde(rename_all = "camelCase")]
pub struct Message {
/// The message header, identifying signed and read-only `account_keys`
pub header: MessageHeader,
/// List of accounts loaded by this transaction.
#[serde(with = "short_vec")]
pub account_keys: Vec<Pubkey>,
/// List of address map indexes used to succinctly load additional accounts
/// for this transaction.
///
/// # Notes
///
/// The last `address_map_indexes.len()` accounts of the read-only unsigned
/// accounts are loaded as address maps.
#[serde(with = "short_vec")]
pub address_map_indexes: Vec<AddressMapIndexes>,
/// The blockhash of a recent block.
pub recent_blockhash: Hash,
/// Instructions that invoke a designated program, are executed in sequence,
/// and committed in one atomic transaction if all succeed.
///
/// # Notes
///
/// Account and program indexes will index into the list of addresses
/// constructed from the concatenation of `account_keys`, flattened list of
/// `writable` address map indexes, and the flattened `readonly` address
/// map indexes.
#[serde(with = "short_vec")]
pub instructions: Vec<CompiledInstruction>,
}
impl Sanitize for Message {
fn sanitize(&self) -> Result<(), SanitizeError> {
// signing area and read-only non-signing area should not
// overlap
if self.header.num_required_signatures as usize
+ self.header.num_readonly_unsigned_accounts as usize
> self.account_keys.len()
{
return Err(SanitizeError::IndexOutOfBounds);
}
// there should be at least 1 RW fee-payer account.
if self.header.num_readonly_signed_accounts >= self.header.num_required_signatures {
return Err(SanitizeError::IndexOutOfBounds);
}
// there cannot be more address maps than read-only unsigned accounts.
let num_address_map_indexes = self.address_map_indexes.len();
if num_address_map_indexes > self.header.num_readonly_unsigned_accounts as usize {
return Err(SanitizeError::IndexOutOfBounds);
}
// each map must load at least one entry
let mut num_loaded_accounts = self.account_keys.len();
for indexes in &self.address_map_indexes {
let num_loaded_map_entries = indexes
.writable
.len()
.saturating_add(indexes.readonly.len());
if num_loaded_map_entries == 0 {
return Err(SanitizeError::InvalidValue);
}
num_loaded_accounts = num_loaded_accounts.saturating_add(num_loaded_map_entries);
}
// the number of loaded accounts must be <= 256 since account indices are
// encoded as `u8`
if num_loaded_accounts > 256 {
return Err(SanitizeError::IndexOutOfBounds);
}
for ci in &self.instructions {
if ci.program_id_index as usize >= num_loaded_accounts {
return Err(SanitizeError::IndexOutOfBounds);
}
// A program cannot be a payer.
if ci.program_id_index == 0 {
return Err(SanitizeError::IndexOutOfBounds);
}
for ai in &ci.accounts {
if *ai as usize >= num_loaded_accounts {
return Err(SanitizeError::IndexOutOfBounds);
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn simple_message() -> Message {
Message {
header: MessageHeader {
num_required_signatures: 1,
num_readonly_signed_accounts: 0,
num_readonly_unsigned_accounts: 1,
},
account_keys: vec![Pubkey::new_unique(), Pubkey::new_unique()],
address_map_indexes: vec![AddressMapIndexes {
writable: vec![],
readonly: vec![0],
}],
..Message::default()
}
}
fn two_map_message() -> Message {
Message {
header: MessageHeader {
num_required_signatures: 1,
num_readonly_signed_accounts: 0,
num_readonly_unsigned_accounts: 2,
},
account_keys: vec![
Pubkey::new_unique(),
Pubkey::new_unique(),
Pubkey::new_unique(),
],
address_map_indexes: vec![
AddressMapIndexes {
writable: vec![1],
readonly: vec![0],
},
AddressMapIndexes {
writable: vec![0],
readonly: vec![1],
},
],
..Message::default()
}
}
#[test]
fn test_sanitize_account_indices() {
assert!(Message {
account_keys: (0..=u8::MAX).map(|_| Pubkey::new_unique()).collect(),
address_map_indexes: vec![],
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(),
address_map_indexes: vec![],
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..simple_message()
}
.sanitize()
.is_err());
assert!(Message {
account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(),
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
account_keys: (0..u8::MAX - 1).map(|_| Pubkey::new_unique()).collect(),
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..simple_message()
}
.sanitize()
.is_err());
assert!(Message {
address_map_indexes: vec![
AddressMapIndexes {
writable: (0..200).step_by(2).collect(),
readonly: (1..200).step_by(2).collect(),
},
AddressMapIndexes {
writable: (0..53).step_by(2).collect(),
readonly: (1..53).step_by(2).collect(),
},
],
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..two_map_message()
}
.sanitize()
.is_ok());
assert!(Message {
address_map_indexes: vec![
AddressMapIndexes {
writable: (0..200).step_by(2).collect(),
readonly: (1..200).step_by(2).collect(),
},
AddressMapIndexes {
writable: (0..52).step_by(2).collect(),
readonly: (1..52).step_by(2).collect(),
},
],
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![u8::MAX],
data: vec![],
}],
..two_map_message()
}
.sanitize()
.is_err());
}
#[test]
fn test_sanitize_excessive_loaded_accounts() {
assert!(Message {
account_keys: (0..=u8::MAX).map(|_| Pubkey::new_unique()).collect(),
address_map_indexes: vec![],
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
account_keys: (0..257).map(|_| Pubkey::new_unique()).collect(),
address_map_indexes: vec![],
..simple_message()
}
.sanitize()
.is_err());
assert!(Message {
account_keys: (0..u8::MAX).map(|_| Pubkey::new_unique()).collect(),
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
account_keys: (0..256).map(|_| Pubkey::new_unique()).collect(),
..simple_message()
}
.sanitize()
.is_err());
assert!(Message {
address_map_indexes: vec![
AddressMapIndexes {
writable: (0..200).step_by(2).collect(),
readonly: (1..200).step_by(2).collect(),
},
AddressMapIndexes {
writable: (0..53).step_by(2).collect(),
readonly: (1..53).step_by(2).collect(),
}
],
..two_map_message()
}
.sanitize()
.is_ok());
assert!(Message {
address_map_indexes: vec![
AddressMapIndexes {
writable: (0..200).step_by(2).collect(),
readonly: (1..200).step_by(2).collect(),
},
AddressMapIndexes {
writable: (0..200).step_by(2).collect(),
readonly: (1..200).step_by(2).collect(),
}
],
..two_map_message()
}
.sanitize()
.is_err());
}
#[test]
fn test_sanitize_excessive_maps() {
assert!(Message {
header: MessageHeader {
num_readonly_unsigned_accounts: 1,
..simple_message().header
},
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
header: MessageHeader {
num_readonly_unsigned_accounts: 0,
..simple_message().header
},
..simple_message()
}
.sanitize()
.is_err());
}
#[test]
fn test_sanitize_address_map() {
assert!(Message {
address_map_indexes: vec![AddressMapIndexes {
writable: vec![0],
readonly: vec![],
}],
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
address_map_indexes: vec![AddressMapIndexes {
writable: vec![],
readonly: vec![0],
}],
..simple_message()
}
.sanitize()
.is_ok());
assert!(Message {
address_map_indexes: vec![AddressMapIndexes {
writable: vec![],
readonly: vec![],
}],
..simple_message()
}
.sanitize()
.is_err());
}
}

View File

@ -0,0 +1,271 @@
use {
crate::{
hash::Hash,
instruction::CompiledInstruction,
message::{v0, Message, MessageHeader},
pubkey::Pubkey,
sanitize::{Sanitize, SanitizeError},
short_vec,
},
serde::{
de::{self, Deserializer, SeqAccess, Visitor},
ser::{SerializeTuple, Serializer},
{Deserialize, Serialize},
},
std::fmt,
};
/// Bit mask that indicates whether a serialized message is versioned.
pub const MESSAGE_VERSION_PREFIX: u8 = 0x80;
/// Message versions supported by the Solana runtime.
///
/// # Serialization
///
/// If the first bit is set, the remaining 7 bits will be used to determine
/// which message version is serialized starting from version `0`. If the first
/// is bit is not set, all bytes are used to encode the legacy `Message`
/// format.
#[frozen_abi(digest = "C4MZ7qztFJHUp1bVcuh7Gn43PQExadzEGyEb8UMn9unz")]
#[derive(Debug, PartialEq, Eq, Clone, AbiEnumVisitor, AbiExample)]
pub enum VersionedMessage {
Legacy(Message),
V0(v0::Message),
}
impl Default for VersionedMessage {
fn default() -> Self {
Self::Legacy(Message::default())
}
}
impl Sanitize for VersionedMessage {
fn sanitize(&self) -> Result<(), SanitizeError> {
match self {
Self::Legacy(message) => message.sanitize(),
Self::V0(message) => message.sanitize(),
}
}
}
impl Serialize for VersionedMessage {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match self {
Self::Legacy(message) => {
let mut seq = serializer.serialize_tuple(1)?;
seq.serialize_element(message)?;
seq.end()
}
Self::V0(message) => {
let mut seq = serializer.serialize_tuple(2)?;
seq.serialize_element(&MESSAGE_VERSION_PREFIX)?;
seq.serialize_element(message)?;
seq.end()
}
}
}
}
enum MessagePrefix {
Legacy(u8),
Versioned(u8),
}
impl<'de> Deserialize<'de> for MessagePrefix {
fn deserialize<D>(deserializer: D) -> Result<MessagePrefix, D::Error>
where
D: Deserializer<'de>,
{
struct PrefixVisitor;
impl<'de> Visitor<'de> for PrefixVisitor {
type Value = MessagePrefix;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("message prefix byte")
}
fn visit_u8<E>(self, byte: u8) -> Result<MessagePrefix, E> {
if byte & MESSAGE_VERSION_PREFIX != 0 {
Ok(MessagePrefix::Versioned(byte & !MESSAGE_VERSION_PREFIX))
} else {
Ok(MessagePrefix::Legacy(byte))
}
}
}
deserializer.deserialize_u8(PrefixVisitor)
}
}
impl<'de> Deserialize<'de> for VersionedMessage {
fn deserialize<D>(deserializer: D) -> Result<VersionedMessage, D::Error>
where
D: Deserializer<'de>,
{
struct MessageVisitor;
impl<'de> Visitor<'de> for MessageVisitor {
type Value = VersionedMessage;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("message bytes")
}
fn visit_seq<A>(self, mut seq: A) -> Result<VersionedMessage, A::Error>
where
A: SeqAccess<'de>,
{
let prefix: MessagePrefix = seq
.next_element()?
.ok_or_else(|| de::Error::invalid_length(0, &self))?;
match prefix {
MessagePrefix::Legacy(num_required_signatures) => {
// The remaining fields of the legacy Message struct after the first byte.
#[derive(Serialize, Deserialize)]
struct RemainingLegacyMessage {
pub num_readonly_signed_accounts: u8,
pub num_readonly_unsigned_accounts: u8,
#[serde(with = "short_vec")]
pub account_keys: Vec<Pubkey>,
pub recent_blockhash: Hash,
#[serde(with = "short_vec")]
pub instructions: Vec<CompiledInstruction>,
}
let message: RemainingLegacyMessage =
seq.next_element()?.ok_or_else(|| {
// will never happen since tuple length is always 2
de::Error::invalid_length(1, &self)
})?;
Ok(VersionedMessage::Legacy(Message {
header: MessageHeader {
num_required_signatures,
num_readonly_signed_accounts: message.num_readonly_signed_accounts,
num_readonly_unsigned_accounts: message
.num_readonly_unsigned_accounts,
},
account_keys: message.account_keys,
recent_blockhash: message.recent_blockhash,
instructions: message.instructions,
}))
}
MessagePrefix::Versioned(version) => {
if version == 0 {
Ok(VersionedMessage::V0(seq.next_element()?.ok_or_else(
|| {
// will never happen since tuple length is always 2
de::Error::invalid_length(1, &self)
},
)?))
} else {
Err(de::Error::invalid_value(
de::Unexpected::Unsigned(version as u64),
&"supported versions: [0]",
))
}
}
}
}
}
deserializer.deserialize_tuple(2, MessageVisitor)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{
instruction::{AccountMeta, Instruction},
message::v0::AddressMapIndexes,
};
#[test]
fn test_legacy_message_serialization() {
let program_id0 = Pubkey::new_unique();
let program_id1 = Pubkey::new_unique();
let id0 = Pubkey::new_unique();
let id1 = Pubkey::new_unique();
let id2 = Pubkey::new_unique();
let id3 = Pubkey::new_unique();
let instructions = vec![
Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id0, false)]),
Instruction::new_with_bincode(program_id0, &0, vec![AccountMeta::new(id1, true)]),
Instruction::new_with_bincode(
program_id1,
&0,
vec![AccountMeta::new_readonly(id2, false)],
),
Instruction::new_with_bincode(
program_id1,
&0,
vec![AccountMeta::new_readonly(id3, true)],
),
];
let mut message = Message::new(&instructions, Some(&id1));
message.recent_blockhash = Hash::new_unique();
let bytes1 = bincode::serialize(&message).unwrap();
let bytes2 = bincode::serialize(&VersionedMessage::Legacy(message.clone())).unwrap();
assert_eq!(bytes1, bytes2);
let message1: Message = bincode::deserialize(&bytes1).unwrap();
let message2: VersionedMessage = bincode::deserialize(&bytes2).unwrap();
if let VersionedMessage::Legacy(message2) = message2 {
assert_eq!(message, message1);
assert_eq!(message1, message2);
} else {
panic!("should deserialize to legacy message");
}
}
#[test]
fn test_versioned_message_serialization() {
let message = v0::Message {
header: MessageHeader {
num_required_signatures: 1,
num_readonly_signed_accounts: 0,
num_readonly_unsigned_accounts: 2,
},
recent_blockhash: Hash::new_unique(),
account_keys: vec![
Pubkey::new_unique(),
Pubkey::new_unique(),
Pubkey::new_unique(),
],
address_map_indexes: vec![
AddressMapIndexes {
writable: vec![1],
readonly: vec![0],
},
AddressMapIndexes {
writable: vec![0],
readonly: vec![1],
},
],
instructions: vec![CompiledInstruction {
program_id_index: 1,
accounts: vec![0],
data: vec![],
}],
};
let bytes = bincode::serialize(&VersionedMessage::V0(message.clone())).unwrap();
let message_from_bytes: VersionedMessage = bincode::deserialize(&bytes).unwrap();
if let VersionedMessage::V0(message_from_bytes) = message_from_bytes {
assert_eq!(message, message_from_bytes);
} else {
panic!("should deserialize to versioned message");
}
}
}

View File

@ -110,7 +110,7 @@ impl From<SanitizeError> for TransactionError {
}
/// An atomic transaction
#[frozen_abi(digest = "AAeVxvWiiotwxDLxKLxsfgkA6ndW74nVbaAEb6cwJYqR")]
#[frozen_abi(digest = "FZtncnS1Xk8ghHfKiXE5oGiUbw2wJhmfXQuNgQR3K6Mc")]
#[derive(Debug, PartialEq, Default, Eq, Clone, Serialize, Deserialize, AbiExample)]
pub struct Transaction {
/// A set of digital signatures of a serialized [`Message`], signed by the