From 731a9432392a1a034b9da33189a39bd0cdba3ac4 Mon Sep 17 00:00:00 2001 From: Justin Starry Date: Thu, 24 Sep 2020 13:10:29 +0800 Subject: [PATCH] Remove transaction encoding from storage layer (#12404) --- client/src/rpc_client.rs | 8 +- core/src/banking_stage.rs | 42 +++----- core/src/replay_stage.rs | 42 +++----- core/src/rpc.rs | 49 +++++---- ledger-tool/src/bigtable.rs | 39 +++---- ledger/src/bigtable_upload.rs | 5 +- ledger/src/blockstore.rs | 192 ++++++++++++---------------------- stake-monitor/src/lib.rs | 8 +- storage-bigtable/src/lib.rs | 147 ++++++++++---------------- transaction-status/src/lib.rs | 63 ++++++++++- watchtower/src/main.rs | 10 +- 11 files changed, 270 insertions(+), 335 deletions(-) diff --git a/client/src/rpc_client.rs b/client/src/rpc_client.rs index 1a1394739d..cfb031afb1 100644 --- a/client/src/rpc_client.rs +++ b/client/src/rpc_client.rs @@ -35,7 +35,7 @@ use solana_sdk::{ transaction::{self, Transaction}, }; use solana_transaction_status::{ - ConfirmedBlock, ConfirmedTransaction, TransactionStatus, UiTransactionEncoding, + EncodedConfirmedBlock, EncodedConfirmedTransaction, TransactionStatus, UiTransactionEncoding, }; use solana_vote_program::vote_state::MAX_LOCKOUT_HISTORY; use std::{ @@ -248,7 +248,7 @@ impl RpcClient { self.send(RpcRequest::GetClusterNodes, Value::Null) } - pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult { + pub fn get_confirmed_block(&self, slot: Slot) -> ClientResult { self.get_confirmed_block_with_encoding(slot, UiTransactionEncoding::Json) } @@ -256,7 +256,7 @@ impl RpcClient { &self, slot: Slot, encoding: UiTransactionEncoding, - ) -> ClientResult { + ) -> ClientResult { self.send(RpcRequest::GetConfirmedBlock, json!([slot, encoding])) } @@ -326,7 +326,7 @@ impl RpcClient { &self, signature: &Signature, encoding: UiTransactionEncoding, - ) -> ClientResult { + ) -> ClientResult { self.send( RpcRequest::GetConfirmedTransaction, json!([signature.to_string(), encoding]), diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index 7ada128160..499305d58f 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -1092,7 +1092,7 @@ mod tests { system_transaction, transaction::TransactionError, }; - use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta}; + use solana_transaction_status::TransactionWithStatusMeta; use std::{sync::atomic::Ordering, thread::sleep}; #[test] @@ -2038,36 +2038,26 @@ mod tests { transaction_status_service.join().unwrap(); - let confirmed_block = blockstore.get_confirmed_block(bank.slot(), None).unwrap(); + let confirmed_block = blockstore.get_confirmed_block(bank.slot()).unwrap(); assert_eq!(confirmed_block.transactions.len(), 3); for TransactionWithStatusMeta { transaction, meta } in confirmed_block.transactions.into_iter() { - if let EncodedTransaction::Json(transaction) = transaction { - if transaction.signatures[0] == success_signature.to_string() { - let meta = meta.unwrap(); - assert_eq!(meta.err, None); - assert_eq!(meta.status, Ok(())); - } else if transaction.signatures[0] == ix_error_signature.to_string() { - let meta = meta.unwrap(); - assert_eq!( - meta.err, - Some(TransactionError::InstructionError( - 0, - InstructionError::Custom(1) - )) - ); - assert_eq!( - meta.status, - Err(TransactionError::InstructionError( - 0, - InstructionError::Custom(1) - )) - ); - } else { - assert_eq!(meta, None); - } + if transaction.signatures[0] == success_signature { + let meta = meta.unwrap(); + assert_eq!(meta.status, Ok(())); + } else if transaction.signatures[0] == ix_error_signature { + let meta = meta.unwrap(); + assert_eq!( + meta.status, + Err(TransactionError::InstructionError( + 0, + InstructionError::Custom(1) + )) + ); + } else { + assert_eq!(meta, None); } } } diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs index 1c96ed4509..09852ced6a 100644 --- a/core/src/replay_stage.rs +++ b/core/src/replay_stage.rs @@ -1992,7 +1992,7 @@ pub(crate) mod tests { system_transaction, transaction::TransactionError, }; - use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta}; + use solana_transaction_status::TransactionWithStatusMeta; use solana_vote_program::{ vote_state::{VoteState, VoteStateVersions}, vote_transaction, @@ -2732,36 +2732,26 @@ pub(crate) mod tests { blockstore.clone(), ); - let confirmed_block = blockstore.get_confirmed_block(slot, None).unwrap(); + let confirmed_block = blockstore.get_confirmed_block(slot).unwrap(); assert_eq!(confirmed_block.transactions.len(), 3); for TransactionWithStatusMeta { transaction, meta } in confirmed_block.transactions.into_iter() { - if let EncodedTransaction::Json(transaction) = transaction { - if transaction.signatures[0] == signatures[0].to_string() { - let meta = meta.unwrap(); - assert_eq!(meta.err, None); - assert_eq!(meta.status, Ok(())); - } else if transaction.signatures[0] == signatures[1].to_string() { - let meta = meta.unwrap(); - assert_eq!( - meta.err, - Some(TransactionError::InstructionError( - 0, - InstructionError::Custom(1) - )) - ); - assert_eq!( - meta.status, - Err(TransactionError::InstructionError( - 0, - InstructionError::Custom(1) - )) - ); - } else { - assert_eq!(meta, None); - } + if transaction.signatures[0] == signatures[0] { + let meta = meta.unwrap(); + assert_eq!(meta.status, Ok(())); + } else if transaction.signatures[0] == signatures[1] { + let meta = meta.unwrap(); + assert_eq!( + meta.status, + Err(TransactionError::InstructionError( + 0, + InstructionError::Custom(1) + )) + ); + } else { + assert_eq!(meta, None); } } } diff --git a/core/src/rpc.rs b/core/src/rpc.rs index 9e1a770fcc..b3611e9489 100644 --- a/core/src/rpc.rs +++ b/core/src/rpc.rs @@ -61,7 +61,7 @@ use solana_sdk::{ }; use solana_stake_program::stake_state::StakeState; use solana_transaction_status::{ - ConfirmedBlock, ConfirmedTransaction, TransactionStatus, UiTransactionEncoding, + EncodedConfirmedBlock, EncodedConfirmedTransaction, TransactionStatus, UiTransactionEncoding, }; use solana_vote_program::vote_state::{VoteState, MAX_LOCKOUT_HISTORY}; use spl_token_v2_0::{ @@ -612,7 +612,7 @@ impl JsonRpcRequestProcessor { &self, slot: Slot, encoding: Option, - ) -> Result> { + ) -> Result> { let encoding = encoding.unwrap_or(UiTransactionEncoding::Json); if self.config.enable_rpc_transaction_history && slot @@ -622,17 +622,20 @@ impl JsonRpcRequestProcessor { .unwrap() .highest_confirmed_root() { - let result = self.blockstore.get_confirmed_block(slot, Some(encoding)); + let result = self.blockstore.get_confirmed_block(slot); if result.is_err() { if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage { return Ok(self .runtime_handle - .block_on(bigtable_ledger_storage.get_confirmed_block(slot, encoding)) - .ok()); + .block_on(bigtable_ledger_storage.get_confirmed_block(slot)) + .ok() + .map(|confirmed_block| confirmed_block.encode(encoding))); } } self.check_slot_cleaned_up(&result, slot)?; - Ok(result.ok()) + Ok(result + .ok() + .map(|confirmed_block| confirmed_block.encode(encoding))) } else { Err(RpcCustomError::BlockNotAvailable { slot }.into()) } @@ -808,12 +811,12 @@ impl JsonRpcRequestProcessor { &self, signature: Signature, encoding: Option, - ) -> Option { + ) -> Option { let encoding = encoding.unwrap_or(UiTransactionEncoding::Json); if self.config.enable_rpc_transaction_history { match self .blockstore - .get_confirmed_transaction(signature, Some(encoding)) + .get_confirmed_transaction(signature) .unwrap_or(None) { Some(confirmed_transaction) => { @@ -824,18 +827,16 @@ impl JsonRpcRequestProcessor { .unwrap() .highest_confirmed_root() { - return Some(confirmed_transaction); + return Some(confirmed_transaction.encode(encoding)); } } None => { if let Some(bigtable_ledger_storage) = &self.bigtable_ledger_storage { return self .runtime_handle - .block_on( - bigtable_ledger_storage - .get_confirmed_transaction(&signature, encoding), - ) - .unwrap_or(None); + .block_on(bigtable_ledger_storage.get_confirmed_transaction(&signature)) + .unwrap_or(None) + .map(|confirmed| confirmed.encode(encoding)); } } } @@ -1660,7 +1661,7 @@ pub trait RpcSol { meta: Self::Metadata, slot: Slot, encoding: Option, - ) -> Result>; + ) -> Result>; #[rpc(meta, name = "getBlockTime")] fn get_block_time(&self, meta: Self::Metadata, slot: Slot) -> Result>; @@ -1679,7 +1680,7 @@ pub trait RpcSol { meta: Self::Metadata, signature_str: String, encoding: Option, - ) -> Result>; + ) -> Result>; #[rpc(meta, name = "getConfirmedSignaturesForAddress")] fn get_confirmed_signatures_for_address( @@ -2308,7 +2309,7 @@ impl RpcSol for RpcSolImpl { meta: Self::Metadata, slot: Slot, encoding: Option, - ) -> Result> { + ) -> Result> { debug!("get_confirmed_block rpc request received: {:?}", slot); meta.get_confirmed_block(slot, encoding) } @@ -2335,7 +2336,7 @@ impl RpcSol for RpcSolImpl { meta: Self::Metadata, signature_str: String, encoding: Option, - ) -> Result> { + ) -> Result> { debug!( "get_confirmed_transaction rpc request received: {:?}", signature_str @@ -2563,7 +2564,9 @@ pub mod tests { timing::slot_duration_from_slots_per_year, transaction::{self, TransactionError}, }; - use solana_transaction_status::{EncodedTransaction, TransactionWithStatusMeta, UiMessage}; + use solana_transaction_status::{ + EncodedTransaction, EncodedTransactionWithStatusMeta, UiMessage, + }; use solana_vote_program::{ vote_instruction, vote_state::{Vote, VoteInit, MAX_LOCKOUT_HISTORY}, @@ -4532,12 +4535,12 @@ pub mod tests { let res = io.handle_request_sync(&req, meta.clone()); let result: Value = serde_json::from_str(&res.expect("actual response")) .expect("actual response deserialization"); - let confirmed_block: Option = + let confirmed_block: Option = serde_json::from_value(result["result"].clone()).unwrap(); let confirmed_block = confirmed_block.unwrap(); assert_eq!(confirmed_block.transactions.len(), 3); - for TransactionWithStatusMeta { transaction, meta } in + for EncodedTransactionWithStatusMeta { transaction, meta } in confirmed_block.transactions.into_iter() { if let EncodedTransaction::Json(transaction) = transaction { @@ -4576,12 +4579,12 @@ pub mod tests { let res = io.handle_request_sync(&req, meta); let result: Value = serde_json::from_str(&res.expect("actual response")) .expect("actual response deserialization"); - let confirmed_block: Option = + let confirmed_block: Option = serde_json::from_value(result["result"].clone()).unwrap(); let confirmed_block = confirmed_block.unwrap(); assert_eq!(confirmed_block.transactions.len(), 3); - for TransactionWithStatusMeta { transaction, meta } in + for EncodedTransactionWithStatusMeta { transaction, meta } in confirmed_block.transactions.into_iter() { if let EncodedTransaction::LegacyBinary(transaction) = transaction { diff --git a/ledger-tool/src/bigtable.rs b/ledger-tool/src/bigtable.rs index 9479dc4151..ae177a9cfc 100644 --- a/ledger-tool/src/bigtable.rs +++ b/ledger-tool/src/bigtable.rs @@ -7,7 +7,7 @@ use solana_clap_utils::{ use solana_cli_output::display::println_transaction; use solana_ledger::{blockstore::Blockstore, blockstore_db::AccessType}; use solana_sdk::{clock::Slot, pubkey::Pubkey, signature::Signature}; -use solana_transaction_status::UiTransactionEncoding; +use solana_transaction_status::ConfirmedBlock; use std::{ path::Path, process::exit, @@ -51,9 +51,7 @@ async fn block(slot: Slot) -> Result<(), Box> { .await .map_err(|err| format!("Failed to connect to storage: {:?}", err))?; - let block = bigtable - .get_confirmed_block(slot, UiTransactionEncoding::Base64) - .await?; + let block = bigtable.get_confirmed_block(slot).await?; println!("Slot: {}", slot); println!("Parent Slot: {}", block.parent_slot); @@ -65,11 +63,11 @@ async fn block(slot: Slot) -> Result<(), Box> { if !block.rewards.is_empty() { println!("Rewards: {:?}", block.rewards); } - for (index, transaction_with_meta) in block.transactions.iter().enumerate() { + for (index, transaction_with_meta) in block.transactions.into_iter().enumerate() { println!("Transaction {}:", index); println_transaction( - &transaction_with_meta.transaction.decode().unwrap(), - &transaction_with_meta.meta, + &transaction_with_meta.transaction, + &transaction_with_meta.meta.map(|meta| meta.into()), " ", ); } @@ -96,22 +94,15 @@ async fn confirm(signature: &Signature, verbose: bool) -> Result<(), Box { println!( "\nTransaction executed in slot {}:", confirmed_transaction.slot ); println_transaction( - &confirmed_transaction - .transaction - .transaction - .decode() - .expect("Successful decode"), - &confirmed_transaction.transaction.meta, + &confirmed_transaction.transaction.transaction, + &confirmed_transaction.transaction.meta.map(|m| m.into()), " ", ); } @@ -138,7 +129,7 @@ pub async fn transaction_history( ) -> Result<(), Box> { let bigtable = solana_storage_bigtable::LedgerStorage::new(true).await?; - let mut loaded_block: Option<(Slot, solana_transaction_status::ConfirmedBlock)> = None; + let mut loaded_block: Option<(Slot, ConfirmedBlock)> = None; while limit > 0 { let results = bigtable .get_confirmed_signatures_for_address( @@ -188,11 +179,8 @@ pub async fn transaction_history( } Some(transaction_with_meta) => { println_transaction( - &transaction_with_meta - .transaction - .decode() - .expect("Successful decode"), - &transaction_with_meta.meta, + &transaction_with_meta.transaction, + &transaction_with_meta.meta.clone().map(|m| m.into()), " ", ); } @@ -200,10 +188,7 @@ pub async fn transaction_history( break; } } - match bigtable - .get_confirmed_block(result.slot, UiTransactionEncoding::Base64) - .await - { + match bigtable.get_confirmed_block(result.slot).await { Err(err) => { println!(" Unable to get confirmed transaction details: {}", err); break; diff --git a/ledger/src/bigtable_upload.rs b/ledger/src/bigtable_upload.rs index 9d4120ef59..dd43975531 100644 --- a/ledger/src/bigtable_upload.rs +++ b/ledger/src/bigtable_upload.rs @@ -135,10 +135,7 @@ pub async fn upload_confirmed_blocks( break; } - let _ = match blockstore.get_confirmed_block( - *slot, - Some(solana_transaction_status::UiTransactionEncoding::Base64), - ) { + let _ = match blockstore.get_confirmed_block(*slot) { Ok(confirmed_block) => sender.send((*slot, Some(confirmed_block))), Err(err) => { warn!( diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 3b6b18caec..e8fa6d2086 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -37,9 +37,8 @@ use solana_sdk::{ transaction::Transaction, }; use solana_transaction_status::{ - ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, - EncodedTransaction, Rewards, TransactionStatusMeta, TransactionWithStatusMeta, - UiTransactionEncoding, UiTransactionStatusMeta, + ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards, + TransactionStatusMeta, TransactionWithStatusMeta, }; use solana_vote_program::vote_instruction::VoteInstruction; use std::{ @@ -1646,11 +1645,7 @@ impl Blockstore { Ok(root_iterator.next().unwrap_or_default()) } - pub fn get_confirmed_block( - &self, - slot: Slot, - encoding: Option, - ) -> Result { + pub fn get_confirmed_block(&self, slot: Slot) -> Result { datapoint_info!( "blockstore-rpc-api", ("method", "get_confirmed_block".to_string(), String) @@ -1661,7 +1656,6 @@ impl Blockstore { if *lowest_cleanup_slot > 0 && *lowest_cleanup_slot >= slot { return Err(BlockstoreError::SlotCleanedUp); } - let encoding = encoding.unwrap_or(UiTransactionEncoding::Json); if self.is_root(slot) { let slot_meta_cf = self.db.column::(); let slot_meta = match slot_meta_cf.get(slot)? { @@ -1697,11 +1691,8 @@ impl Blockstore { previous_blockhash: previous_blockhash.to_string(), blockhash: blockhash.to_string(), parent_slot: slot_meta.parent_slot, - transactions: self.map_transactions_to_statuses( - slot, - encoding, - slot_transaction_iterator, - ), + transactions: self + .map_transactions_to_statuses(slot, slot_transaction_iterator), rewards, block_time, }; @@ -1714,19 +1705,16 @@ impl Blockstore { fn map_transactions_to_statuses<'a>( &self, slot: Slot, - encoding: UiTransactionEncoding, iterator: impl Iterator + 'a, ) -> Vec { iterator .map(|transaction| { let signature = transaction.signatures[0]; - let encoded_transaction = EncodedTransaction::encode(transaction, encoding); TransactionWithStatusMeta { - transaction: encoded_transaction, + transaction, meta: self .read_transaction_status((signature, slot)) - .expect("Expect database get to succeed") - .map(UiTransactionStatusMeta::from), + .expect("Expect database get to succeed"), } }) .collect() @@ -1902,7 +1890,6 @@ impl Blockstore { pub fn get_confirmed_transaction( &self, signature: Signature, - encoding: Option, ) -> Result> { datapoint_info!( "blockstore-rpc-api", @@ -1912,13 +1899,11 @@ impl Blockstore { let transaction = self .find_transaction_in_slot(slot, signature)? .ok_or(BlockstoreError::TransactionStatusSlotMismatch)?; // Should not happen - let encoding = encoding.unwrap_or(UiTransactionEncoding::Json); - let encoded_transaction = EncodedTransaction::encode(transaction, encoding); Ok(Some(ConfirmedTransaction { slot, transaction: TransactionWithStatusMeta { - transaction: encoded_transaction, - meta: Some(status.into()), + transaction, + meta: Some(status), }, })) } else { @@ -2019,27 +2004,23 @@ impl Blockstore { match transaction_status { None => return Ok(vec![]), Some((slot, _)) => { - let confirmed_block = self - .get_confirmed_block(slot, Some(UiTransactionEncoding::Base64)) - .map_err(|err| { - BlockstoreError::IO(IOError::new( - ErrorKind::Other, - format!("Unable to get confirmed block: {}", err), - )) - })?; + let confirmed_block = self.get_confirmed_block(slot).map_err(|err| { + BlockstoreError::IO(IOError::new( + ErrorKind::Other, + format!("Unable to get confirmed block: {}", err), + )) + })?; // Load all signatures for the block let mut slot_signatures: Vec<_> = confirmed_block .transactions - .iter() + .into_iter() .filter_map(|transaction_with_meta| { - if let Some(transaction) = - transaction_with_meta.transaction.decode() - { - transaction.signatures.into_iter().next() - } else { - None - } + transaction_with_meta + .transaction + .signatures + .into_iter() + .next() }) .collect(); @@ -2073,27 +2054,23 @@ impl Blockstore { match transaction_status { None => (0, HashSet::new()), Some((slot, _)) => { - let confirmed_block = self - .get_confirmed_block(slot, Some(UiTransactionEncoding::Base64)) - .map_err(|err| { - BlockstoreError::IO(IOError::new( - ErrorKind::Other, - format!("Unable to get confirmed block: {}", err), - )) - })?; + let confirmed_block = self.get_confirmed_block(slot).map_err(|err| { + BlockstoreError::IO(IOError::new( + ErrorKind::Other, + format!("Unable to get confirmed block: {}", err), + )) + })?; // Load all signatures for the block let mut slot_signatures: Vec<_> = confirmed_block .transactions - .iter() + .into_iter() .filter_map(|transaction_with_meta| { - if let Some(transaction) = - transaction_with_meta.transaction.decode() - { - transaction.signatures.into_iter().next() - } else { - None - } + transaction_with_meta + .transaction + .signatures + .into_iter() + .next() }) .collect(); @@ -5672,7 +5649,7 @@ pub mod tests { .put_meta_bytes(slot - 1, &serialize(&parent_meta).unwrap()) .unwrap(); - let expected_transactions: Vec<(Transaction, Option)> = entries + let expected_transactions: Vec = entries .iter() .cloned() .filter(|entry| !entry.is_tick()) @@ -5709,37 +5686,27 @@ pub mod tests { }, ) .unwrap(); - ( + TransactionWithStatusMeta { transaction, - Some( - TransactionStatusMeta { - status: Ok(()), - fee: 42, - pre_balances, - post_balances, - } - .into(), - ), - ) + meta: Some(TransactionStatusMeta { + status: Ok(()), + fee: 42, + pre_balances, + post_balances, + }), + } }) .collect(); // Even if marked as root, a slot that is empty of entries should return an error - let confirmed_block_err = ledger.get_confirmed_block(slot - 1, None).unwrap_err(); + let confirmed_block_err = ledger.get_confirmed_block(slot - 1).unwrap_err(); assert_matches!(confirmed_block_err, BlockstoreError::SlotNotRooted); - let confirmed_block = ledger.get_confirmed_block(slot, None).unwrap(); + let confirmed_block = ledger.get_confirmed_block(slot).unwrap(); assert_eq!(confirmed_block.transactions.len(), 100); let expected_block = ConfirmedBlock { - transactions: expected_transactions - .iter() - .cloned() - .map(|(tx, meta)| TransactionWithStatusMeta { - transaction: EncodedTransaction::encode(tx, UiTransactionEncoding::Json), - meta, - }) - .collect(), + transactions: expected_transactions.clone(), parent_slot: slot - 1, blockhash: blockhash.to_string(), previous_blockhash: Hash::default().to_string(), @@ -5750,18 +5717,11 @@ pub mod tests { // root, but empty of entries. This is special handling for snapshot root slots. assert_eq!(confirmed_block, expected_block); - let confirmed_block = ledger.get_confirmed_block(slot + 1, None).unwrap(); + let confirmed_block = ledger.get_confirmed_block(slot + 1).unwrap(); assert_eq!(confirmed_block.transactions.len(), 100); let mut expected_block = ConfirmedBlock { - transactions: expected_transactions - .iter() - .cloned() - .map(|(tx, meta)| TransactionWithStatusMeta { - transaction: EncodedTransaction::encode(tx, UiTransactionEncoding::Json), - meta, - }) - .collect(), + transactions: expected_transactions, parent_slot: slot, blockhash: blockhash.to_string(), previous_blockhash: blockhash.to_string(), @@ -5770,7 +5730,7 @@ pub mod tests { }; assert_eq!(confirmed_block, expected_block); - let not_root = ledger.get_confirmed_block(slot + 2, None).unwrap_err(); + let not_root = ledger.get_confirmed_block(slot + 2).unwrap_err(); assert_matches!(not_root, BlockstoreError::SlotNotRooted); // Test block_time returns, if available @@ -5778,7 +5738,7 @@ pub mod tests { ledger.blocktime_cf.put(slot + 1, ×tamp).unwrap(); expected_block.block_time = Some(timestamp); - let confirmed_block = ledger.get_confirmed_block(slot + 1, None).unwrap(); + let confirmed_block = ledger.get_confirmed_block(slot + 1).unwrap(); assert_eq!(confirmed_block, expected_block); drop(ledger); @@ -6434,7 +6394,7 @@ pub mod tests { blockstore.insert_shreds(shreds, None, false).unwrap(); blockstore.set_roots(&[slot - 1, slot]).unwrap(); - let expected_transactions: Vec<(Transaction, Option)> = entries + let expected_transactions: Vec = entries .iter() .cloned() .filter(|entry| !entry.is_tick()) @@ -6459,48 +6419,32 @@ pub mod tests { }, ) .unwrap(); - ( + TransactionWithStatusMeta { transaction, - Some( - TransactionStatusMeta { - status: Ok(()), - fee: 42, - pre_balances, - post_balances, - } - .into(), - ), - ) + meta: Some(TransactionStatusMeta { + status: Ok(()), + fee: 42, + pre_balances, + post_balances, + }), + } }) .collect(); - for (transaction, status) in expected_transactions.clone() { - let signature = transaction.signatures[0]; - let encoded_transaction = - EncodedTransaction::encode(transaction, UiTransactionEncoding::Json); - let expected_transaction = ConfirmedTransaction { - slot, - transaction: TransactionWithStatusMeta { - transaction: encoded_transaction, - meta: status, - }, - }; + for transaction in expected_transactions.clone() { + let signature = transaction.transaction.signatures[0]; assert_eq!( - blockstore - .get_confirmed_transaction(signature, None) - .unwrap(), - Some(expected_transaction) + blockstore.get_confirmed_transaction(signature).unwrap(), + Some(ConfirmedTransaction { slot, transaction }) ); } blockstore.run_purge(0, 2, PurgeType::PrimaryIndex).unwrap(); *blockstore.lowest_cleanup_slot.write().unwrap() = slot; - for (transaction, _) in expected_transactions { + for TransactionWithStatusMeta { transaction, .. } in expected_transactions { let signature = transaction.signatures[0]; assert_eq!( - blockstore - .get_confirmed_transaction(signature, None) - .unwrap(), + blockstore.get_confirmed_transaction(signature).unwrap(), None, ); } @@ -6513,7 +6457,7 @@ pub mod tests { blockstore.set_roots(&[0]).unwrap(); assert_eq!( blockstore - .get_confirmed_transaction(Signature::default(), None) + .get_confirmed_transaction(Signature::default()) .unwrap(), None ); @@ -6935,11 +6879,7 @@ pub mod tests { vec![CompiledInstruction::new(1, &(), vec![0])], )); - let map = blockstore.map_transactions_to_statuses( - slot, - UiTransactionEncoding::Json, - transactions.into_iter(), - ); + let map = blockstore.map_transactions_to_statuses(slot, transactions.into_iter()); assert_eq!(map.len(), 5); for (x, m) in map.iter().take(4).enumerate() { assert_eq!(m.meta.as_ref().unwrap().fee, x as u64); diff --git a/stake-monitor/src/lib.rs b/stake-monitor/src/lib.rs index aa26290a38..08defc3568 100644 --- a/stake-monitor/src/lib.rs +++ b/stake-monitor/src/lib.rs @@ -7,7 +7,9 @@ use solana_sdk::{ pubkey::Pubkey, signature::Signature, transaction::Transaction, }; use solana_stake_program::{stake_instruction::StakeInstruction, stake_state::Lockup}; -use solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding, UiTransactionStatusMeta}; +use solana_transaction_status::{ + EncodedConfirmedBlock, UiTransactionEncoding, UiTransactionStatusMeta, +}; use std::{collections::HashMap, thread::sleep, time::Duration}; pub type PubkeyString = String; @@ -244,7 +246,7 @@ fn process_transaction( fn process_confirmed_block( slot: Slot, - confirmed_block: ConfirmedBlock, + confirmed_block: EncodedConfirmedBlock, accounts: &mut HashMap, ) { for rpc_transaction in confirmed_block.transactions { @@ -281,7 +283,7 @@ fn load_blocks( rpc_client: &RpcClient, start_slot: Slot, end_slot: Slot, -) -> ClientResult> { +) -> ClientResult> { info!( "Loading confirmed blocks between slots: {} - {}", start_slot, end_slot diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 0d48b62597..1133a5a6ca 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -8,14 +8,10 @@ use solana_sdk::{ transaction::{Transaction, TransactionError}, }; use solana_transaction_status::{ - ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, - EncodedTransaction, Rewards, TransactionStatus, TransactionWithStatusMeta, - UiTransactionEncoding, UiTransactionStatusMeta, -}; -use std::{ - collections::HashMap, - convert::{TryFrom, TryInto}, + ConfirmedBlock, ConfirmedTransaction, ConfirmedTransactionStatusWithSignature, Rewards, + TransactionStatus, TransactionStatusMeta, TransactionWithStatusMeta, }; +use std::collections::HashMap; use thiserror::Error; #[macro_use] @@ -91,35 +87,8 @@ struct StoredConfirmedBlock { block_time: Option, } -impl StoredConfirmedBlock { - fn into_confirmed_block(self, encoding: UiTransactionEncoding) -> ConfirmedBlock { - let StoredConfirmedBlock { - previous_blockhash, - blockhash, - parent_slot, - transactions, - rewards, - block_time, - } = self; - - ConfirmedBlock { - previous_blockhash, - blockhash, - parent_slot, - transactions: transactions - .into_iter() - .map(|transaction| transaction.into_transaction_with_status_meta(encoding)) - .collect(), - rewards, - block_time, - } - } -} - -impl TryFrom for StoredConfirmedBlock { - type Error = Error; - - fn try_from(confirmed_block: ConfirmedBlock) -> Result { +impl From for StoredConfirmedBlock { + fn from(confirmed_block: ConfirmedBlock) -> Self { let ConfirmedBlock { previous_blockhash, blockhash, @@ -129,19 +98,36 @@ impl TryFrom for StoredConfirmedBlock { block_time, } = confirmed_block; - let mut encoded_transactions = vec![]; - for transaction in transactions.into_iter() { - encoded_transactions.push(transaction.try_into()?); - } - - Ok(Self { + Self { previous_blockhash, blockhash, parent_slot, - transactions: encoded_transactions, + transactions: transactions.into_iter().map(|tx| tx.into()).collect(), rewards, block_time, - }) + } + } +} + +impl From for ConfirmedBlock { + fn from(confirmed_block: StoredConfirmedBlock) -> Self { + let StoredConfirmedBlock { + previous_blockhash, + blockhash, + parent_slot, + transactions, + rewards, + block_time, + } = confirmed_block; + + Self { + previous_blockhash, + blockhash, + parent_slot, + transactions: transactions.into_iter().map(|tx| tx.into()).collect(), + rewards, + block_time, + } } } @@ -151,31 +137,21 @@ struct StoredConfirmedBlockTransaction { meta: Option, } -impl StoredConfirmedBlockTransaction { - fn into_transaction_with_status_meta( - self, - encoding: UiTransactionEncoding, - ) -> TransactionWithStatusMeta { - let StoredConfirmedBlockTransaction { transaction, meta } = self; - TransactionWithStatusMeta { - transaction: EncodedTransaction::encode(transaction, encoding), - meta: meta.map(|meta| meta.into()), +impl From for StoredConfirmedBlockTransaction { + fn from(value: TransactionWithStatusMeta) -> Self { + Self { + transaction: value.transaction, + meta: value.meta.map(|meta| meta.into()), } } } -impl TryFrom for StoredConfirmedBlockTransaction { - type Error = Error; - - fn try_from(value: TransactionWithStatusMeta) -> Result { - let TransactionWithStatusMeta { transaction, meta } = value; - - Ok(Self { - transaction: transaction - .decode() - .ok_or(Error::UnsupportedTransactionEncoding)?, - meta: meta.map(|meta| meta.into()), - }) +impl From for TransactionWithStatusMeta { + fn from(value: StoredConfirmedBlockTransaction) -> Self { + Self { + transaction: value.transaction, + meta: value.meta.map(|meta| meta.into()), + } } } @@ -187,7 +163,7 @@ struct StoredConfirmedBlockTransactionStatusMeta { post_balances: Vec, } -impl From for UiTransactionStatusMeta { +impl From for TransactionStatusMeta { fn from(value: StoredConfirmedBlockTransactionStatusMeta) -> Self { let StoredConfirmedBlockTransactionStatusMeta { err, @@ -200,7 +176,6 @@ impl From for UiTransactionStatusMeta Some(err) => Err(err.clone()), }; Self { - err, status, fee, pre_balances, @@ -209,17 +184,17 @@ impl From for UiTransactionStatusMeta } } -impl From for StoredConfirmedBlockTransactionStatusMeta { - fn from(value: UiTransactionStatusMeta) -> Self { - let UiTransactionStatusMeta { - err, +impl From for StoredConfirmedBlockTransactionStatusMeta { + fn from(value: TransactionStatusMeta) -> Self { + let TransactionStatusMeta { + status, fee, pre_balances, post_balances, .. } = value; Self { - err, + err: status.err(), fee, pre_balances, post_balances, @@ -296,16 +271,12 @@ impl LedgerStorage { } /// Fetch the confirmed block from the desired slot - pub async fn get_confirmed_block( - &self, - slot: Slot, - encoding: UiTransactionEncoding, - ) -> Result { + pub async fn get_confirmed_block(&self, slot: Slot) -> Result { let mut bigtable = self.connection.client(); let block = bigtable .get_bincode_cell::("blocks", slot_to_key(slot)) .await?; - Ok(block.into_confirmed_block(encoding)) + Ok(block.into()) } pub async fn get_signature_status(&self, signature: &Signature) -> Result { @@ -320,7 +291,6 @@ impl LedgerStorage { pub async fn get_confirmed_transaction( &self, signature: &Signature, - encoding: UiTransactionEncoding, ) -> Result> { let mut bigtable = self.connection.client(); @@ -348,8 +318,7 @@ impl LedgerStorage { } else { Ok(Some(ConfirmedTransaction { slot, - transaction: bucket_block_transaction - .into_transaction_with_status_meta(encoding), + transaction: bucket_block_transaction.into(), })) } } @@ -467,22 +436,16 @@ impl LedgerStorage { ) -> Result<()> { let mut bytes_written = 0; - let mut by_addr: HashMap> = HashMap::new(); + let mut by_addr: HashMap<&Pubkey, Vec> = HashMap::new(); let mut tx_cells = vec![]; for (index, transaction_with_meta) in confirmed_block.transactions.iter().enumerate() { - let err = transaction_with_meta - .meta - .as_ref() - .and_then(|meta| meta.err.clone()); + let TransactionWithStatusMeta { meta, transaction } = transaction_with_meta; + let err = meta.as_ref().and_then(|meta| meta.status.clone().err()); let index = index as u32; - let transaction = transaction_with_meta - .transaction - .decode() - .expect("transaction decode failed"); let signature = transaction.signatures[0]; - for address in transaction.message.account_keys { + for address in &transaction.message.account_keys { if !is_sysvar_id(&address) { by_addr .entry(address) @@ -539,7 +502,7 @@ impl LedgerStorage { // Store the block itself last, after all other metadata about the block has been // successfully stored. This avoids partial uploaded blocks from becoming visible to // `get_confirmed_block()` and `get_confirmed_blocks()` - let blocks_cells = [(slot_to_key(slot), confirmed_block.try_into()?)]; + let blocks_cells = [(slot_to_key(slot), confirmed_block.into())]; bytes_written += self .connection .put_bincode_cells_with_retry::("blocks", &blocks_cells) diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 063ef61203..d61a020df4 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -165,6 +165,34 @@ pub struct ConfirmedBlock { pub block_time: Option, } +impl ConfirmedBlock { + pub fn encode(self, encoding: UiTransactionEncoding) -> EncodedConfirmedBlock { + EncodedConfirmedBlock { + previous_blockhash: self.previous_blockhash, + blockhash: self.blockhash, + parent_slot: self.parent_slot, + transactions: self + .transactions + .into_iter() + .map(|tx| tx.encode(encoding)) + .collect(), + rewards: self.rewards, + block_time: self.block_time, + } + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncodedConfirmedBlock { + pub previous_blockhash: String, + pub blockhash: String, + pub parent_slot: Slot, + pub transactions: Vec, + pub rewards: Rewards, + pub block_time: Option, +} + #[derive(Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConfirmedTransaction { @@ -173,6 +201,23 @@ pub struct ConfirmedTransaction { pub transaction: TransactionWithStatusMeta, } +impl ConfirmedTransaction { + pub fn encode(self, encoding: UiTransactionEncoding) -> EncodedConfirmedTransaction { + EncodedConfirmedTransaction { + slot: self.slot, + transaction: self.transaction.encode(encoding), + } + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncodedConfirmedTransaction { + pub slot: Slot, + #[serde(flatten)] + pub transaction: EncodedTransactionWithStatusMeta, +} + /// A duplicate representation of a Transaction for pretty JSON serialization #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -207,9 +252,25 @@ pub struct UiParsedMessage { pub instructions: Vec, } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransactionWithStatusMeta { + pub transaction: Transaction, + pub meta: Option, +} + +impl TransactionWithStatusMeta { + fn encode(self, encoding: UiTransactionEncoding) -> EncodedTransactionWithStatusMeta { + EncodedTransactionWithStatusMeta { + transaction: EncodedTransaction::encode(self.transaction, encoding), + meta: self.meta.map(|meta| meta.into()), + } + } +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EncodedTransactionWithStatusMeta { pub transaction: EncodedTransaction, pub meta: Option, } diff --git a/watchtower/src/main.rs b/watchtower/src/main.rs index 6b72f28f69..1e1120141b 100644 --- a/watchtower/src/main.rs +++ b/watchtower/src/main.rs @@ -16,7 +16,7 @@ use solana_sdk::{ clock::Slot, hash::Hash, native_token::lamports_to_sol, program_utils::limited_deserialize, pubkey::Pubkey, }; -use solana_transaction_status::{ConfirmedBlock, UiTransactionEncoding}; +use solana_transaction_status::{EncodedConfirmedBlock, UiTransactionEncoding}; use solana_vote_program::vote_instruction::VoteInstruction; use std::{ collections::HashMap, @@ -157,7 +157,11 @@ fn get_config() -> Config { config } -fn process_confirmed_block(notifier: &Notifier, slot: Slot, confirmed_block: ConfirmedBlock) { +fn process_confirmed_block( + notifier: &Notifier, + slot: Slot, + confirmed_block: EncodedConfirmedBlock, +) { let break_program_id = "BrEAK7zGZ6dM71zUDACDqJnekihmwF15noTddWTsknjC" .parse::() .unwrap(); @@ -215,7 +219,7 @@ fn load_blocks( rpc_client: &RpcClient, start_slot: Slot, end_slot: Slot, -) -> ClientResult> { +) -> ClientResult> { info!( "Loading confirmed blocks between slots: {} - {}", start_slot, end_slot