blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move error counters to be reported by leader only at end of slot (#24581) 

* Add error counters to leader metrics only

* Add dependencies
This commit is contained in:
carllin 2022-04-23 18:10:47 -05:00 committed by GitHub
parent 92ad767738
commit 8a062273de
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 175 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
core/src/banking_stage.rs
View File

@ -29,13 +29,13 @@ use {
solana_poh::poh_recorder::{BankStart, PohRecorder, PohRecorderError, TransactionRecorder}, solana_poh::poh_recorder::{BankStart, PohRecorder, PohRecorderError, TransactionRecorder},
solana_program_runtime::timings::ExecuteTimings, solana_program_runtime::timings::ExecuteTimings,
solana_runtime::{ solana_runtime::{
accounts_db::ErrorCounters,
bank::{ bank::{
Bank, LoadAndExecuteTransactionsOutput, TransactionBalancesSet, TransactionCheckResult, Bank, LoadAndExecuteTransactionsOutput, TransactionBalancesSet, TransactionCheckResult,
}, },
bank_utils, bank_utils,
cost_model::{CostModel, TransactionCost}, cost_model::{CostModel, TransactionCost},
transaction_batch::TransactionBatch, transaction_batch::TransactionBatch,
transaction_error_metrics::TransactionErrorMetrics,
vote_sender_types::ReplayVoteSender, vote_sender_types::ReplayVoteSender,
}, },
solana_sdk::{ solana_sdk::{
@ -121,6 +121,7 @@ pub struct ExecuteAndCommitTransactionsOutput {
// committed into the Poh stream. // committed into the Poh stream.
commit_transactions_result: Result<Vec<CommitTransactionDetails>, PohRecorderError>, commit_transactions_result: Result<Vec<CommitTransactionDetails>, PohRecorderError>,
execute_and_commit_timings: LeaderExecuteAndCommitTimings, execute_and_commit_timings: LeaderExecuteAndCommitTimings,
error_counters: TransactionErrorMetrics,
} }
#[derive(Debug, Default)] #[derive(Debug, Default)]
@ -1172,6 +1173,7 @@ impl BankingStage {
executed_transactions_count, executed_transactions_count,
executed_with_successful_result_count, executed_with_successful_result_count,
signature_count, signature_count,
error_counters,
.. ..
} = load_and_execute_transactions_output; } = load_and_execute_transactions_output;
@ -1232,6 +1234,7 @@ impl BankingStage {
retryable_transaction_indexes, retryable_transaction_indexes,
commit_transactions_result: Err(recorder_err), commit_transactions_result: Err(recorder_err),
execute_and_commit_timings, execute_and_commit_timings,
error_counters,
}; };
} }
@ -1338,6 +1341,7 @@ impl BankingStage {
retryable_transaction_indexes, retryable_transaction_indexes,
commit_transactions_result: Ok(commit_transaction_statuses), commit_transactions_result: Ok(commit_transaction_statuses),
execute_and_commit_timings, execute_and_commit_timings,
error_counters,
} }
} }
@ -1556,6 +1560,7 @@ impl BankingStage {
let mut total_cost_model_throttled_transactions_count: usize = 0; let mut total_cost_model_throttled_transactions_count: usize = 0;
let mut total_cost_model_us: u64 = 0; let mut total_cost_model_us: u64 = 0;
let mut total_execute_and_commit_timings = LeaderExecuteAndCommitTimings::default(); let mut total_execute_and_commit_timings = LeaderExecuteAndCommitTimings::default();
let mut total_error_counters = TransactionErrorMetrics::default();
let mut reached_max_poh_height = false; let mut reached_max_poh_height = false;
while chunk_start != transactions.len() { while chunk_start != transactions.len() {
let chunk_end = std::cmp::min( let chunk_end = std::cmp::min(
@ -1589,10 +1594,12 @@ impl BankingStage {
retryable_transaction_indexes: new_retryable_transaction_indexes, retryable_transaction_indexes: new_retryable_transaction_indexes,
commit_transactions_result: new_commit_transactions_result, commit_transactions_result: new_commit_transactions_result,
execute_and_commit_timings: new_execute_and_commit_timings, execute_and_commit_timings: new_execute_and_commit_timings,
error_counters: new_error_counters,
.. ..
} = execute_and_commit_transactions_output; } = execute_and_commit_transactions_output;
total_execute_and_commit_timings.accumulate(&new_execute_and_commit_timings); total_execute_and_commit_timings.accumulate(&new_execute_and_commit_timings);
total_error_counters.accumulate(&new_error_counters);
total_transactions_attempted_execution_count = total_transactions_attempted_execution_count =
total_transactions_attempted_execution_count total_transactions_attempted_execution_count
.saturating_add(new_transactions_attempted_execution_count); .saturating_add(new_transactions_attempted_execution_count);
@ -1655,6 +1662,7 @@ impl BankingStage {
cost_model_throttled_transactions_count: total_cost_model_throttled_transactions_count, cost_model_throttled_transactions_count: total_cost_model_throttled_transactions_count,
cost_model_us: total_cost_model_us, cost_model_us: total_cost_model_us,
execute_and_commit_timings: total_execute_and_commit_timings, execute_and_commit_timings: total_execute_and_commit_timings,
error_counters: total_error_counters,
} }
} }
@ -1727,7 +1735,7 @@ impl BankingStage {
let filter = let filter =
Self::prepare_filter_for_pending_transactions(transactions.len(), pending_indexes); Self::prepare_filter_for_pending_transactions(transactions.len(), pending_indexes);
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
// The following code also checks if the blockhash for a transaction is too old // The following code also checks if the blockhash for a transaction is too old
// The check accounts for // The check accounts for
// 1. Transaction forwarding delay // 1. Transaction forwarding delay
@ -1808,10 +1816,12 @@ impl BankingStage {
let ProcessTransactionsSummary { let ProcessTransactionsSummary {
ref retryable_transaction_indexes, ref retryable_transaction_indexes,
ref error_counters,
.. ..
} = process_transactions_summary; } = process_transactions_summary;
slot_metrics_tracker.accumulate_process_transactions_summary(&process_transactions_summary); slot_metrics_tracker.accumulate_process_transactions_summary(&process_transactions_summary);
slot_metrics_tracker.accumulate_transaction_errors(error_counters);
let retryable_tx_count = retryable_transaction_indexes.len(); let retryable_tx_count = retryable_transaction_indexes.len();
inc_new_counter_info!("banking_stage-unprocessed_transactions", retryable_tx_count); inc_new_counter_info!("banking_stage-unprocessed_transactions", retryable_tx_count);

19
core/src/leader_slot_banking_stage_metrics.rs
View File

@ -1,6 +1,7 @@
use { use {
crate::leader_slot_banking_stage_timing_metrics::*, crate::leader_slot_banking_stage_timing_metrics::*,
solana_poh::poh_recorder::BankStart, solana_poh::poh_recorder::BankStart,
solana_runtime::transaction_error_metrics::*,
solana_sdk::{clock::Slot, saturating_add_assign}, solana_sdk::{clock::Slot, saturating_add_assign},
std::time::Instant, std::time::Instant,
}; };
@ -45,6 +46,9 @@ pub(crate) struct ProcessTransactionsSummary {
// Breakdown of time spent executing and comitting transactions // Breakdown of time spent executing and comitting transactions
pub execute_and_commit_timings: LeaderExecuteAndCommitTimings, pub execute_and_commit_timings: LeaderExecuteAndCommitTimings,
// Breakdown of all the transaction errors from transactions passed for execution
pub error_counters: TransactionErrorMetrics,
} }
// Metrics describing packets ingested/processed in various parts of BankingStage during this // Metrics describing packets ingested/processed in various parts of BankingStage during this
@ -226,6 +230,8 @@ pub(crate) struct LeaderSlotMetrics {
packet_count_metrics: LeaderSlotPacketCountMetrics, packet_count_metrics: LeaderSlotPacketCountMetrics,
transaction_error_metrics: TransactionErrorMetrics,
timing_metrics: LeaderSlotTimingMetrics, timing_metrics: LeaderSlotTimingMetrics,
// Used by tests to check if the `self.report()` method was called // Used by tests to check if the `self.report()` method was called
@ -238,6 +244,7 @@ impl LeaderSlotMetrics {
id, id,
slot, slot,
packet_count_metrics: LeaderSlotPacketCountMetrics::new(), packet_count_metrics: LeaderSlotPacketCountMetrics::new(),
transaction_error_metrics: TransactionErrorMetrics::new(),
timing_metrics: LeaderSlotTimingMetrics::new(bank_creation_time), timing_metrics: LeaderSlotTimingMetrics::new(bank_creation_time),
is_reported: false, is_reported: false,
} }
@ -247,6 +254,7 @@ impl LeaderSlotMetrics {
self.is_reported = true; self.is_reported = true;
self.timing_metrics.report(self.id, self.slot); self.timing_metrics.report(self.id, self.slot);
self.transaction_error_metrics.report(self.id, self.slot);
self.packet_count_metrics.report(self.id, self.slot); self.packet_count_metrics.report(self.id, self.slot);
} }
@ -405,6 +413,17 @@ impl LeaderSlotMetricsTracker {
} }
} }
pub(crate) fn accumulate_transaction_errors(
&mut self,
error_metrics: &TransactionErrorMetrics,
) {
if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {
leader_slot_metrics
.transaction_error_metrics
.accumulate(error_metrics);
}
}
// Packet inflow/outflow/processing metrics // Packet inflow/outflow/processing metrics
pub(crate) fn increment_total_new_valid_packets(&mut self, count: u64) { pub(crate) fn increment_total_new_valid_packets(&mut self, count: u64) {
if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics { if let Some(leader_slot_metrics) = &mut self.leader_slot_metrics {

45
runtime/src/accounts.rs
View File

@ -4,7 +4,7 @@ use {
account_rent_state::{check_rent_state_with_account, RentState}, account_rent_state::{check_rent_state_with_account, RentState},
accounts_db::{ accounts_db::{
AccountShrinkThreshold, AccountsAddRootTiming, AccountsDb, AccountsDbConfig, AccountShrinkThreshold, AccountsAddRootTiming, AccountsDb, AccountsDbConfig,
BankHashInfo, ErrorCounters, LoadHint, LoadedAccount, ScanStorageResult, BankHashInfo, LoadHint, LoadedAccount, ScanStorageResult,
ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING,
}, },
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ScanResult}, accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ScanResult},
@ -17,6 +17,7 @@ use {
blockhash_queue::BlockhashQueue, blockhash_queue::BlockhashQueue,
rent_collector::RentCollector, rent_collector::RentCollector,
system_instruction_processor::{get_system_account_kind, SystemAccountKind}, system_instruction_processor::{get_system_account_kind, SystemAccountKind},
transaction_error_metrics::TransactionErrorMetrics,
}, },
dashmap::{ dashmap::{
mapref::entry::Entry::{Occupied, Vacant}, mapref::entry::Entry::{Occupied, Vacant},
@ -238,7 +239,7 @@ impl Accounts {
ancestors: &Ancestors, ancestors: &Ancestors,
tx: &SanitizedTransaction, tx: &SanitizedTransaction,
fee: u64, fee: u64,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
rent_collector: &RentCollector, rent_collector: &RentCollector,
feature_set: &FeatureSet, feature_set: &FeatureSet,
account_overrides: Option<&AccountOverrides>, account_overrides: Option<&AccountOverrides>,
@ -422,7 +423,7 @@ impl Accounts {
ancestors: &Ancestors, ancestors: &Ancestors,
accounts: &mut Vec<TransactionAccount>, accounts: &mut Vec<TransactionAccount>,
mut program_account_index: usize, mut program_account_index: usize,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Result<Vec<usize>> { ) -> Result<Vec<usize>> {
let mut account_indices = Vec::new(); let mut account_indices = Vec::new();
let mut program_id = match accounts.get(program_account_index) { let mut program_id = match accounts.get(program_account_index) {
@ -502,7 +503,7 @@ impl Accounts {
txs: &[SanitizedTransaction], txs: &[SanitizedTransaction],
lock_results: Vec<TransactionCheckResult>, lock_results: Vec<TransactionCheckResult>,
hash_queue: &BlockhashQueue, hash_queue: &BlockhashQueue,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
rent_collector: &RentCollector, rent_collector: &RentCollector,
feature_set: &FeatureSet, feature_set: &FeatureSet,
fee_structure: &FeeStructure, fee_structure: &FeeStructure,
@ -1417,7 +1418,7 @@ mod tests {
ka: &[TransactionAccount], ka: &[TransactionAccount],
lamports_per_signature: u64, lamports_per_signature: u64,
rent_collector: &RentCollector, rent_collector: &RentCollector,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
feature_set: &FeatureSet, feature_set: &FeatureSet,
fee_structure: &FeeStructure, fee_structure: &FeeStructure,
) -> Vec<TransactionLoadResult> { ) -> Vec<TransactionLoadResult> {
@ -1453,7 +1454,7 @@ mod tests {
tx: Transaction, tx: Transaction,
ka: &[TransactionAccount], ka: &[TransactionAccount],
lamports_per_signature: u64, lamports_per_signature: u64,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Vec<TransactionLoadResult> { ) -> Vec<TransactionLoadResult> {
load_accounts_with_fee_and_rent( load_accounts_with_fee_and_rent(
tx, tx,
@ -1469,7 +1470,7 @@ mod tests {
fn load_accounts( fn load_accounts(
tx: Transaction, tx: Transaction,
ka: &[TransactionAccount], ka: &[TransactionAccount],
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Vec<TransactionLoadResult> { ) -> Vec<TransactionLoadResult> {
load_accounts_with_fee(tx, ka, 0, error_counters) load_accounts_with_fee(tx, ka, 0, error_counters)
} }
@ -1534,7 +1535,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_no_account_0_exists() { fn test_load_accounts_no_account_0_exists() {
let accounts: Vec<TransactionAccount> = Vec::new(); let accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
@ -1560,7 +1561,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_unknown_program_id() { fn test_load_accounts_unknown_program_id() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1594,7 +1595,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_insufficient_funds() { fn test_load_accounts_insufficient_funds() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1632,7 +1633,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_invalid_account_for_fee() { fn test_load_accounts_invalid_account_for_fee() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1661,7 +1662,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_fee_payer_is_nonce() { fn test_load_accounts_fee_payer_is_nonce() {
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let mut feature_set = FeatureSet::all_enabled(); let mut feature_set = FeatureSet::all_enabled();
feature_set.deactivate(&tx_wide_compute_cap::id()); feature_set.deactivate(&tx_wide_compute_cap::id());
let rent_collector = RentCollector::new( let rent_collector = RentCollector::new(
@ -1742,7 +1743,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_no_loaders() { fn test_load_accounts_no_loaders() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1783,7 +1784,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_max_call_depth() { fn test_load_accounts_max_call_depth() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1849,7 +1850,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_bad_owner() { fn test_load_accounts_bad_owner() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1884,7 +1885,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_not_executable() { fn test_load_accounts_not_executable() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -1918,7 +1919,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_multiple_loaders() { fn test_load_accounts_multiple_loaders() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -2152,7 +2153,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_executable_with_write_lock() { fn test_load_accounts_executable_with_write_lock() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -2208,7 +2209,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_upgradeable_with_write_lock() { fn test_load_accounts_upgradeable_with_write_lock() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -2305,7 +2306,7 @@ mod tests {
#[test] #[test]
fn test_load_accounts_programdata_with_write_lock() { fn test_load_accounts_programdata_with_write_lock() {
let mut accounts: Vec<TransactionAccount> = Vec::new(); let mut accounts: Vec<TransactionAccount> = Vec::new();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let keypair = Keypair::new(); let keypair = Keypair::new();
let key0 = keypair.pubkey(); let key0 = keypair.pubkey();
@ -2394,7 +2395,7 @@ mod tests {
false, false,
AccountShrinkThreshold::default(), AccountShrinkThreshold::default(),
); );
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
let keypair = Keypair::new(); let keypair = Keypair::new();
@ -3047,7 +3048,7 @@ mod tests {
hash_queue.register_hash(tx.message().recent_blockhash(), 10); hash_queue.register_hash(tx.message().recent_blockhash(), 10);
let ancestors = vec![(0, 0)].into_iter().collect(); let ancestors = vec![(0, 0)].into_iter().collect();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
accounts.load_accounts( accounts.load_accounts(
&ancestors, &ancestors,
&[tx], &[tx],

20
runtime/src/accounts_db.rs
View File

@ -224,26 +224,6 @@ pub enum ScanStorageResult<R, B> {
Stored(B), Stored(B),
} }
#[derive(Debug, Default)]
pub struct ErrorCounters {
pub total: usize,
pub account_in_use: usize,
pub account_loaded_twice: usize,
pub account_not_found: usize,
pub blockhash_not_found: usize,
pub blockhash_too_old: usize,
pub call_chain_too_deep: usize,
pub already_processed: usize,
pub instruction_error: usize,
pub insufficient_funds: usize,
pub invalid_account_for_fee: usize,
pub invalid_account_index: usize,
pub invalid_program_for_execution: usize,
pub not_allowed_during_cluster_maintenance: usize,
pub invalid_writable_account: usize,
pub invalid_rent_paying_account: usize,
}
#[derive(Debug, Default, Clone, Copy)] #[derive(Debug, Default, Clone, Copy)]
pub struct IndexGenerationInfo { pub struct IndexGenerationInfo {
pub accounts_data_len: u64, pub accounts_data_len: u64,

100
runtime/src/bank.rs
View File

@ -43,7 +43,7 @@ use {
TransactionLoadResult, TransactionLoadResult,
}, },
accounts_db::{ accounts_db::{
AccountShrinkThreshold, AccountsDbConfig, ErrorCounters, SnapshotStorages, AccountShrinkThreshold, AccountsDbConfig, SnapshotStorages,
ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING, ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS, ACCOUNTS_DB_CONFIG_FOR_TESTING,
}, },
accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult}, accounts_index::{AccountSecondaryIndexes, IndexKey, ScanConfig, ScanResult},
@ -66,6 +66,7 @@ use {
status_cache::{SlotDelta, StatusCache}, status_cache::{SlotDelta, StatusCache},
system_instruction_processor::{get_system_account_kind, SystemAccountKind}, system_instruction_processor::{get_system_account_kind, SystemAccountKind},
transaction_batch::TransactionBatch, transaction_batch::TransactionBatch,
transaction_error_metrics::TransactionErrorMetrics,
vote_account::{VoteAccount, VoteAccountsHashMap}, vote_account::{VoteAccount, VoteAccountsHashMap},
vote_parser, vote_parser,
}, },
@ -664,6 +665,7 @@ pub struct LoadAndExecuteTransactionsOutput {
// an error. // an error.
pub executed_with_successful_result_count: usize, pub executed_with_successful_result_count: usize,
pub signature_count: u64, pub signature_count: u64,
pub error_counters: TransactionErrorMetrics,
} }
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
@ -3886,7 +3888,7 @@ impl Bank {
txs: impl Iterator<Item = &'a SanitizedTransaction>, txs: impl Iterator<Item = &'a SanitizedTransaction>,
lock_results: &[Result<()>], lock_results: &[Result<()>],
max_age: usize, max_age: usize,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Vec<TransactionCheckResult> { ) -> Vec<TransactionCheckResult> {
let hash_queue = self.blockhash_queue.read().unwrap(); let hash_queue = self.blockhash_queue.read().unwrap();
txs.zip(lock_results) txs.zip(lock_results)
@ -3923,7 +3925,7 @@ impl Bank {
&self, &self,
sanitized_txs: &[SanitizedTransaction], sanitized_txs: &[SanitizedTransaction],
lock_results: Vec<TransactionCheckResult>, lock_results: Vec<TransactionCheckResult>,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Vec<TransactionCheckResult> { ) -> Vec<TransactionCheckResult> {
let rcache = self.src.status_cache.read().unwrap(); let rcache = self.src.status_cache.read().unwrap();
sanitized_txs sanitized_txs
@ -3977,7 +3979,7 @@ impl Bank {
sanitized_txs: &[SanitizedTransaction], sanitized_txs: &[SanitizedTransaction],
lock_results: &[Result<()>], lock_results: &[Result<()>],
max_age: usize, max_age: usize,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> Vec<TransactionCheckResult> { ) -> Vec<TransactionCheckResult> {
let age_results = let age_results =
self.check_age(sanitized_txs.iter(), lock_results, max_age, error_counters); self.check_age(sanitized_txs.iter(), lock_results, max_age, error_counters);
@ -3996,88 +3998,6 @@ impl Bank {
balances balances
} }
#[allow(clippy::cognitive_complexity)]
fn update_error_counters(error_counters: &ErrorCounters) {
if 0 != error_counters.total {
inc_new_counter_info!(
"bank-process_transactions-error_count",
error_counters.total
);
}
if 0 != error_counters.account_not_found {
inc_new_counter_info!(
"bank-process_transactions-account_not_found",
error_counters.account_not_found
);
}
if 0 != error_counters.account_in_use {
inc_new_counter_info!(
"bank-process_transactions-account_in_use",
error_counters.account_in_use
);
}
if 0 != error_counters.account_loaded_twice {
inc_new_counter_info!(
"bank-process_transactions-account_loaded_twice",
error_counters.account_loaded_twice
);
}
if 0 != error_counters.blockhash_not_found {
inc_new_counter_info!(
"bank-process_transactions-error-blockhash_not_found",
error_counters.blockhash_not_found
);
}
if 0 != error_counters.blockhash_too_old {
inc_new_counter_info!(
"bank-process_transactions-error-blockhash_too_old",
error_counters.blockhash_too_old
);
}
if 0 != error_counters.invalid_account_index {
inc_new_counter_info!(
"bank-process_transactions-error-invalid_account_index",
error_counters.invalid_account_index
);
}
if 0 != error_counters.invalid_account_for_fee {
inc_new_counter_info!(
"bank-process_transactions-error-invalid_account_for_fee",
error_counters.invalid_account_for_fee
);
}
if 0 != error_counters.insufficient_funds {
inc_new_counter_info!(
"bank-process_transactions-error-insufficient_funds",
error_counters.insufficient_funds
);
}
if 0 != error_counters.instruction_error {
inc_new_counter_info!(
"bank-process_transactions-error-instruction_error",
error_counters.instruction_error
);
}
if 0 != error_counters.already_processed {
inc_new_counter_info!(
"bank-process_transactions-error-already_processed",
error_counters.already_processed
);
}
if 0 != error_counters.not_allowed_during_cluster_maintenance {
inc_new_counter_info!(
"bank-process_transactions-error-cluster-maintenance",
error_counters.not_allowed_during_cluster_maintenance
);
}
if 0 != error_counters.invalid_writable_account {
inc_new_counter_info!(
"bank-process_transactions-error-invalid_writable_account",
error_counters.invalid_writable_account
);
}
}
/// Get any cached executors needed by the transaction /// Get any cached executors needed by the transaction
fn get_executors(&self, accounts: &[TransactionAccount]) -> Rc<RefCell<Executors>> { fn get_executors(&self, accounts: &[TransactionAccount]) -> Rc<RefCell<Executors>> {
let executable_keys: Vec<_> = accounts let executable_keys: Vec<_> = accounts
@ -4148,7 +4068,7 @@ impl Bank {
enable_log_recording: bool, enable_log_recording: bool,
enable_return_data_recording: bool, enable_return_data_recording: bool,
timings: &mut ExecuteTimings, timings: &mut ExecuteTimings,
error_counters: &mut ErrorCounters, error_counters: &mut TransactionErrorMetrics,
) -> TransactionExecutionResult { ) -> TransactionExecutionResult {
let mut get_executors_time = Measure::start("get_executors_time"); let mut get_executors_time = Measure::start("get_executors_time");
let executors = self.get_executors(&loaded_transaction.accounts); let executors = self.get_executors(&loaded_transaction.accounts);
@ -4297,7 +4217,7 @@ impl Bank {
let sanitized_txs = batch.sanitized_transactions(); let sanitized_txs = batch.sanitized_transactions();
debug!("processing transactions: {}", sanitized_txs.len()); debug!("processing transactions: {}", sanitized_txs.len());
inc_new_counter_info!("bank-process_transactions", sanitized_txs.len()); inc_new_counter_info!("bank-process_transactions", sanitized_txs.len());
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let retryable_transaction_indexes: Vec<_> = batch let retryable_transaction_indexes: Vec<_> = batch
.lock_results() .lock_results()
@ -4516,7 +4436,6 @@ impl Bank {
*err_count + executed_with_successful_result_count *err_count + executed_with_successful_result_count
); );
} }
Self::update_error_counters(&error_counters);
LoadAndExecuteTransactionsOutput { LoadAndExecuteTransactionsOutput {
loaded_transactions, loaded_transactions,
execution_results, execution_results,
@ -4524,6 +4443,7 @@ impl Bank {
executed_transactions_count, executed_transactions_count,
executed_with_successful_result_count, executed_with_successful_result_count,
signature_count, signature_count,
error_counters,
} }
} }
@ -17507,7 +17427,7 @@ pub(crate) mod tests {
let number_of_instructions_at_transaction_level = tx.message().instructions.len(); let number_of_instructions_at_transaction_level = tx.message().instructions.len();
let num_accounts = tx.message().account_keys.len(); let num_accounts = tx.message().account_keys.len();
let sanitized_tx = SanitizedTransaction::try_from_legacy_transaction(tx).unwrap(); let sanitized_tx = SanitizedTransaction::try_from_legacy_transaction(tx).unwrap();
let mut error_counters = ErrorCounters::default(); let mut error_counters = TransactionErrorMetrics::default();
let loaded_txs = bank.rc.accounts.load_accounts( let loaded_txs = bank.rc.accounts.load_accounts(
&bank.ancestors, &bank.ancestors,
&[sanitized_tx.clone()], &[sanitized_tx.clone()],

1
runtime/src/lib.rs
View File

@ -69,6 +69,7 @@ mod storable_accounts;
mod system_instruction_processor; mod system_instruction_processor;
pub mod transaction_batch; pub mod transaction_batch;
pub mod transaction_cost_metrics_sender; pub mod transaction_cost_metrics_sender;
pub mod transaction_error_metrics;
pub mod vote_account; pub mod vote_account;
pub mod vote_parser; pub mod vote_parser;
pub mod vote_sender_types; pub mod vote_sender_types;

110
runtime/src/transaction_error_metrics.rs Normal file
View File

@ -0,0 +1,110 @@
use solana_sdk::{clock::Slot, saturating_add_assign};
#[derive(Debug, Default)]
pub struct TransactionErrorMetrics {
pub total: usize,
pub account_in_use: usize,
pub account_loaded_twice: usize,
pub account_not_found: usize,
pub blockhash_not_found: usize,
pub blockhash_too_old: usize,
pub call_chain_too_deep: usize,
pub already_processed: usize,
pub instruction_error: usize,
pub insufficient_funds: usize,
pub invalid_account_for_fee: usize,
pub invalid_account_index: usize,
pub invalid_program_for_execution: usize,
pub not_allowed_during_cluster_maintenance: usize,
pub invalid_writable_account: usize,
pub invalid_rent_paying_account: usize,
}
impl TransactionErrorMetrics {
pub fn new() -> Self {
Self { ..Self::default() }
}
pub fn accumulate(&mut self, other: &TransactionErrorMetrics) {
saturating_add_assign!(self.total, other.total);
saturating_add_assign!(self.account_in_use, other.account_in_use);
saturating_add_assign!(self.account_loaded_twice, other.account_loaded_twice);
saturating_add_assign!(self.account_not_found, other.account_not_found);
saturating_add_assign!(self.blockhash_not_found, other.blockhash_not_found);
saturating_add_assign!(self.blockhash_too_old, other.blockhash_too_old);
saturating_add_assign!(self.call_chain_too_deep, other.call_chain_too_deep);
saturating_add_assign!(self.already_processed, other.already_processed);
saturating_add_assign!(self.instruction_error, other.instruction_error);
saturating_add_assign!(self.insufficient_funds, other.insufficient_funds);
saturating_add_assign!(self.invalid_account_for_fee, other.invalid_account_for_fee);
saturating_add_assign!(self.invalid_account_index, other.invalid_account_index);
saturating_add_assign!(
self.invalid_program_for_execution,
other.invalid_program_for_execution
);
saturating_add_assign!(
self.not_allowed_during_cluster_maintenance,
other.not_allowed_during_cluster_maintenance
);
saturating_add_assign!(
self.invalid_writable_account,
other.invalid_writable_account
);
saturating_add_assign!(
self.invalid_rent_paying_account,
other.invalid_rent_paying_account
);
}
pub fn report(&self, id: u32, slot: Slot) {
datapoint_info!(
"banking_stage-leader_slot_transaction_errors",
("id", id as i64, i64),
("slot", slot as i64, i64),
("total", self.total as i64, i64),
("account_in_use", self.account_in_use as i64, i64),
(
"account_loaded_twice",
self.account_loaded_twice as i64,
i64
),
("account_not_found", self.account_not_found as i64, i64),
("blockhash_not_found", self.blockhash_not_found as i64, i64),
("blockhash_too_old", self.blockhash_too_old as i64, i64),
("call_chain_too_deep", self.call_chain_too_deep as i64, i64),
("already_processed", self.already_processed as i64, i64),
("instruction_error", self.instruction_error as i64, i64),
("insufficient_funds", self.insufficient_funds as i64, i64),
(
"invalid_account_for_fee",
self.invalid_account_for_fee as i64,
i64
),
(
"invalid_account_index",
self.invalid_account_index as i64,
i64
),
(
"invalid_program_for_execution",
self.invalid_program_for_execution as i64,
i64
),
(
"not_allowed_during_cluster_maintenance",
self.not_allowed_during_cluster_maintenance as i64,
i64
),
(
"invalid_writable_account",
self.invalid_writable_account as i64,
i64
),
(
"invalid_rent_paying_account",
self.invalid_rent_paying_account as i64,
i64
),
);
}
}