Cost Model to limit transactions which are not parallelizeable (#16694)

* * Add following to banking_stage:
  1. CostModel as immutable ref shared between threads, to provide estimated cost for transactions.
  2. CostTracker which is shared between threads, tracks transaction costs for each block.

* replace hard coded program ID with id() calls

* Add Account Access Cost as part of TransactionCost. Account Access cost are weighted differently between read and write, signed and non-signed.

* Establish instruction_execution_cost_table, add function to update or insert instruction cost, unit tested. It is read-only for now; it allows Replay to insert realtime instruction execution costs to the table.

* add test for cost_tracker atomically try_add operation, serves as safety guard for future changes

* check cost against local copy of cost_tracker, return transactions that would exceed limit as unprocessed transaction to be buffered; only apply bank processed transactions cost to tracker;

* bencher to new banking_stage with max cost limit to allow cost model being hit consistently during bench iterations

Cargo.lock

@@ -4421,6 +4421,7 @@ dependencies = [
  "solana-banks-server",
  "solana-clap-utils",
  "solana-client",
+ "solana-config-program",
  "solana-faucet",
  "solana-frozen-abi 1.8.0",
  "solana-frozen-abi-macro 1.8.0",

core/Cargo.toml

@@ -54,6 +54,7 @@ solana-account-decoder = { path = "../account-decoder", version = "=1.8.0" }
 solana-banks-server = { path = "../banks-server", version = "=1.8.0" }
 solana-clap-utils = { path = "../clap-utils", version = "=1.8.0" }
 solana-client = { path = "../client", version = "=1.8.0" }
+solana-config-program = { path = "../programs/config", version = "=1.8.0" }
 solana-faucet = { path = "../faucet", version = "=1.8.0" }
 solana-gossip = { path = "../gossip", version = "=1.8.0" }
 solana-ledger = { path = "../ledger", version = "=1.8.0" }

core/benches/banking_stage.rs

@@ -8,6 +8,8 @@ use log::*;
 use rand::{thread_rng, Rng};
 use rayon::prelude::*;
 use solana_core::banking_stage::{create_test_recorder, BankingStage, BankingStageStats};
+use solana_core::cost_model::CostModel;
+use solana_core::cost_tracker::CostTracker;
 use solana_core::poh_recorder::WorkingBankEntry;
 use solana_gossip::cluster_info::ClusterInfo;
 use solana_gossip::cluster_info::Node;
@@ -32,7 +34,7 @@ use solana_sdk::transaction::Transaction;
 use std::collections::VecDeque;
 use std::sync::atomic::Ordering;
 use std::sync::mpsc::Receiver;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
 use test::Bencher;
 
@@ -91,6 +93,8 @@ fn bench_consume_buffered(bencher: &mut Bencher) {
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
+                &Arc::new(CostModel::default()),
+                &Arc::new(Mutex::new(CostTracker::new(std::u32::MAX, std::u32::MAX))),
             );
         });
 
@@ -204,13 +208,15 @@ fn bench_banking(bencher: &mut Bencher, tx_type: TransactionType) {
         let cluster_info = ClusterInfo::new_with_invalid_keypair(Node::new_localhost().info);
         let cluster_info = Arc::new(cluster_info);
         let (s, _r) = unbounded();
-        let _banking_stage = BankingStage::new(
+        let _banking_stage = BankingStage::new_with_cost_limit(
             &cluster_info,
             &poh_recorder,
             verified_receiver,
             vote_receiver,
             None,
             s,
+            std::u32::MAX,
+            std::u32::MAX,
         );
         poh_recorder.lock().unwrap().set_bank(&bank);
 

core/src/banking_stage.rs

@@ -2,6 +2,8 @@
 //! to contruct a software pipeline. The stage uses all available CPU cores and
 //! can do its processing in parallel with signature verification on the GPU.
 use crate::{
+    cost_model::{CostModel, ACCOUNT_MAX_COST, BLOCK_MAX_COST},
+    cost_tracker::CostTracker,
     packet_hasher::PacketHasher,
     poh_recorder::{PohRecorder, PohRecorderError, TransactionRecorder, WorkingBankEntry},
     poh_service::{self, PohService},
@@ -231,6 +233,35 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: ReplayVoteSender,
     ) -> Self {
+        Self::new_with_cost_limit(
+            cluster_info,
+            poh_recorder,
+            verified_receiver,
+            verified_vote_receiver,
+            transaction_status_sender,
+            gossip_vote_sender,
+            ACCOUNT_MAX_COST,
+            BLOCK_MAX_COST,
+        )
+    }
+
+    pub fn new_with_cost_limit(
+        cluster_info: &Arc<ClusterInfo>,
+        poh_recorder: &Arc<Mutex<PohRecorder>>,
+        verified_receiver: CrossbeamReceiver<Vec<Packets>>,
+        verified_vote_receiver: CrossbeamReceiver<Vec<Packets>>,
+        transaction_status_sender: Option<TransactionStatusSender>,
+        gossip_vote_sender: ReplayVoteSender,
+        account_cost_limit: u32,
+        block_cost_limit: u32,
+    ) -> Self {
+        // shared immutable 'cost_model' that calcuates transaction costs
+        // shared mutex guarded 'cost_tracker' tracks bank's cost against configured limits.
+        let cost_model = Arc::new(CostModel::new(account_cost_limit, block_cost_limit));
+        let cost_tracker = Arc::new(Mutex::new(CostTracker::new(
+            cost_model.get_account_cost_limit(),
+            cost_model.get_block_cost_limit(),
+        )));
         Self::new_num_threads(
             cluster_info,
             poh_recorder,
@@ -239,6 +270,8 @@ impl BankingStage {
             Self::num_threads(),
             transaction_status_sender,
             gossip_vote_sender,
+            &cost_model,
+            &cost_tracker,
         )
     }
 
@@ -250,6 +283,8 @@ impl BankingStage {
         num_threads: u32,
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: ReplayVoteSender,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) -> Self {
         let batch_limit = TOTAL_BUFFERED_PACKETS / ((num_threads - 1) as usize * PACKETS_PER_BATCH);
         // Single thread to generate entries from many banks.
@@ -276,6 +311,8 @@ impl BankingStage {
                 let transaction_status_sender = transaction_status_sender.clone();
                 let gossip_vote_sender = gossip_vote_sender.clone();
                 let duplicates = duplicates.clone();
+                let cost_model = cost_model.clone();
+                let cost_tracker = cost_tracker.clone();
                 Builder::new()
                     .name("solana-banking-stage-tx".to_string())
                     .spawn(move || {
@@ -291,6 +328,8 @@ impl BankingStage {
                             transaction_status_sender,
                             gossip_vote_sender,
                             &duplicates,
+                            &cost_model,
+                            &cost_tracker,
                         );
                     })
                     .unwrap()
@@ -338,6 +377,11 @@ impl BankingStage {
         has_more_unprocessed_transactions
     }
 
+    fn reset_cost_tracker_if_new_bank(cost_tracker: &Arc<Mutex<CostTracker>>, bank_slot: Slot) {
+        cost_tracker.lock().unwrap().reset_if_new_bank(bank_slot);
+    }
+
+    #[allow(clippy::too_many_arguments)]
     pub fn consume_buffered_packets(
         my_pubkey: &Pubkey,
         max_tx_ingestion_ns: u128,
@@ -348,6 +392,8 @@ impl BankingStage {
         test_fn: Option<impl Fn()>,
         banking_stage_stats: &BankingStageStats,
         recorder: &TransactionRecorder,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) {
         let mut rebuffered_packets_len = 0;
         let mut new_tx_count = 0;
@@ -365,6 +411,8 @@ impl BankingStage {
                     &original_unprocessed_indexes,
                     my_pubkey,
                     *next_leader,
+                    cost_model,
+                    cost_tracker,
                 );
                 Self::update_buffered_packets_with_new_unprocessed(
                     original_unprocessed_indexes,
@@ -373,6 +421,7 @@ impl BankingStage {
             } else {
                 let bank_start = poh_recorder.lock().unwrap().bank_start();
                 if let Some((bank, bank_creation_time)) = bank_start {
+                    Self::reset_cost_tracker_if_new_bank(cost_tracker, bank.slot());
                     let (processed, verified_txs_len, new_unprocessed_indexes) =
                         Self::process_packets_transactions(
                             &bank,
@@ -383,6 +432,8 @@ impl BankingStage {
                             transaction_status_sender.clone(),
                             gossip_vote_sender,
                             banking_stage_stats,
+                            cost_model,
+                            cost_tracker,
                         );
                     if processed < verified_txs_len
                         || !Bank::should_bank_still_be_processing_txs(
@@ -485,6 +536,8 @@ impl BankingStage {
         gossip_vote_sender: &ReplayVoteSender,
         banking_stage_stats: &BankingStageStats,
         recorder: &TransactionRecorder,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) -> BufferedPacketsDecision {
         let bank_start;
         let (
@@ -495,6 +548,9 @@ impl BankingStage {
         ) = {
             let poh = poh_recorder.lock().unwrap();
             bank_start = poh.bank_start();
+            if let Some((ref bank, _)) = bank_start {
+                Self::reset_cost_tracker_if_new_bank(cost_tracker, bank.slot());
+            };
             (
                 poh.leader_after_n_slots(FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET),
                 PohRecorder::get_bank_still_processing_txs(&bank_start),
@@ -525,6 +581,8 @@ impl BankingStage {
                     None::<Box<dyn Fn()>>,
                     banking_stage_stats,
                     recorder,
+                    cost_model,
+                    cost_tracker,
                 );
             }
             BufferedPacketsDecision::Forward => {
@@ -595,6 +653,8 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: ReplayVoteSender,
         duplicates: &Arc<Mutex<(LruCache<u64, ()>, PacketHasher)>>,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) {
         let recorder = poh_recorder.lock().unwrap().recorder();
         let socket = UdpSocket::bind("0.0.0.0:0").unwrap();
@@ -613,6 +673,8 @@ impl BankingStage {
                     &gossip_vote_sender,
                     &banking_stage_stats,
                     &recorder,
+                    cost_model,
+                    cost_tracker,
                 );
                 if matches!(decision, BufferedPacketsDecision::Hold)
                     || matches!(decision, BufferedPacketsDecision::ForwardAndHold)
@@ -647,6 +709,8 @@ impl BankingStage {
                 &banking_stage_stats,
                 duplicates,
                 &recorder,
+                cost_model,
+                cost_tracker,
             ) {
                 Ok(()) | Err(RecvTimeoutError::Timeout) => (),
                 Err(RecvTimeoutError::Disconnected) => break,
@@ -893,12 +957,12 @@ impl BankingStage {
     ) -> (usize, Vec<usize>) {
         let mut chunk_start = 0;
         let mut unprocessed_txs = vec![];
+
         while chunk_start != transactions.len() {
             let chunk_end = std::cmp::min(
                 transactions.len(),
                 chunk_start + MAX_NUM_TRANSACTIONS_PER_BATCH,
             );
-
             let (result, retryable_txs_in_chunk) = Self::process_and_record_transactions(
                 bank,
                 &transactions[chunk_start..chunk_end],
@@ -981,12 +1045,21 @@ impl BankingStage {
     // This function deserializes packets into transactions, computes the blake3 hash of transaction messages,
     // and verifies secp256k1 instructions. A list of valid transactions are returned with their message hashes
     // and packet indexes.
+    // Also returned is packet indexes for transaction should be retried due to cost limits.
     fn transactions_from_packets(
         msgs: &Packets,
         transaction_indexes: &[usize],
         secp256k1_program_enabled: bool,
-    ) -> (Vec<HashedTransaction<'static>>, Vec<usize>) {
+        cost_model: &Arc<CostModel>,
-        transaction_indexes
+        cost_tracker: &Arc<Mutex<CostTracker>>,
+    ) -> (Vec<HashedTransaction<'static>>, Vec<usize>, Vec<usize>) {
+        // Making a snapshot of shared cost_tracker by clone(), drop lock immediately.
+        // Local copy `cost_tracker` is used to filter transactions by cost.
+        // Shared cost_tracker is updated later by processed transactions confirmed by bank.
+        let mut cost_tracker = cost_tracker.lock().unwrap().clone();
+
+        let mut retryable_transaction_packet_indexes: Vec<usize> = vec![];
+        let (filtered_transactions, filter_transaction_packet_indexes) = transaction_indexes
             .iter()
             .filter_map(|tx_index| {
                 let p = &msgs.packets[*tx_index];
@@ -994,6 +1067,19 @@ impl BankingStage {
                 if secp256k1_program_enabled {
                     tx.verify_precompiles().ok()?;
                 }
+
+                // Get transaction cost via immutable cost_model; try to add cost to
+                // local copy of cost_tracker, if suceeded, local copy is updated
+                // and transaction added to valid list; otherwise, transaction is
+                // added to retry list. No locking here.
+                let tx_cost = cost_model.calculate_cost(&tx);
+                let result = cost_tracker.try_add(tx_cost);
+                if result.is_err() {
+                    debug!("transaction {:?} would exceed limit: {:?}", tx, result);
+                    retryable_transaction_packet_indexes.push(*tx_index);
+                    return None;
+                }
+
                 let message_bytes = Self::packet_message(p)?;
                 let message_hash = Message::hash_raw_message(message_bytes);
                 Some((
@@ -1001,7 +1087,13 @@ impl BankingStage {
                     tx_index,
                 ))
             })
-            .unzip()
+            .unzip();
+
+        (
+            filtered_transactions,
+            filter_transaction_packet_indexes,
+            retryable_transaction_packet_indexes,
+        )
     }
 
     /// This function filters pending packets that are still valid
@@ -1043,6 +1135,7 @@ impl BankingStage {
         Self::filter_valid_transaction_indexes(&results, transaction_to_packet_indexes)
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn process_packets_transactions(
         bank: &Arc<Bank>,
         bank_creation_time: &Instant,
@@ -1052,19 +1145,25 @@ impl BankingStage {
         transaction_status_sender: Option<TransactionStatusSender>,
         gossip_vote_sender: &ReplayVoteSender,
         banking_stage_stats: &BankingStageStats,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) -> (usize, usize, Vec<usize>) {
         let mut packet_conversion_time = Measure::start("packet_conversion");
-        let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
+        let (transactions, transaction_to_packet_indexes, retryable_packet_indexes) =
-            msgs,
+            Self::transactions_from_packets(
-            &packet_indexes,
+                msgs,
-            bank.secp256k1_program_enabled(),
+                &packet_indexes,
-        );
+                bank.secp256k1_program_enabled(),
+                cost_model,
+                cost_tracker,
+            );
         packet_conversion_time.stop();
 
         debug!(
-            "bank: {} filtered transactions {}",
+            "bank: {} filtered transactions {} cost limited transactions {}",
             bank.slot(),
-            transactions.len()
+            transactions.len(),
+            retryable_packet_indexes.len()
         );
 
         let tx_len = transactions.len();
@@ -1079,11 +1178,20 @@ impl BankingStage {
             gossip_vote_sender,
         );
         process_tx_time.stop();
-
         let unprocessed_tx_count = unprocessed_tx_indexes.len();
 
+        // applying cost of processed transactions to shared cost_tracker
+        transactions.iter().enumerate().for_each(|(index, tx)| {
+            if !unprocessed_tx_indexes.iter().any(|&i| i == index) {
+                let tx_cost = cost_model.calculate_cost(&tx.transaction());
+                let mut guard = cost_tracker.lock().unwrap();
+                let _result = guard.try_add(tx_cost);
+                drop(guard);
+            }
+        });
+
         let mut filter_pending_packets_time = Measure::start("filter_pending_packets_time");
-        let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
+        let mut filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
             bank,
             &transactions,
             &transaction_to_packet_indexes,
@@ -1091,6 +1199,10 @@ impl BankingStage {
         );
         filter_pending_packets_time.stop();
 
+        // combine cost-related unprocessed transactions with bank determined unprocessed for
+        // buffering
+        filtered_unprocessed_packet_indexes.extend(retryable_packet_indexes);
+
         inc_new_counter_info!(
             "banking_stage-dropped_tx_before_forwarding",
             unprocessed_tx_count.saturating_sub(filtered_unprocessed_packet_indexes.len())
@@ -1115,6 +1227,8 @@ impl BankingStage {
         transaction_indexes: &[usize],
         my_pubkey: &Pubkey,
         next_leader: Option<Pubkey>,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) -> Vec<usize> {
         // Check if we are the next leader. If so, let's not filter the packets
         // as we'll filter it again while processing the packets.
@@ -1125,22 +1239,27 @@ impl BankingStage {
             }
         }
 
-        let (transactions, transaction_to_packet_indexes) = Self::transactions_from_packets(
+        let (transactions, transaction_to_packet_indexes, retry_packet_indexes) =
-            msgs,
+            Self::transactions_from_packets(
-            &transaction_indexes,
+                msgs,
-            bank.secp256k1_program_enabled(),
+                &transaction_indexes,
-        );
+                bank.secp256k1_program_enabled(),
+                cost_model,
+                cost_tracker,
+            );
 
         let tx_count = transaction_to_packet_indexes.len();
 
         let unprocessed_tx_indexes = (0..transactions.len()).collect_vec();
-        let filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
+        let mut filtered_unprocessed_packet_indexes = Self::filter_pending_packets_from_pending_txs(
             bank,
             &transactions,
             &transaction_to_packet_indexes,
             &unprocessed_tx_indexes,
         );
 
+        filtered_unprocessed_packet_indexes.extend(retry_packet_indexes);
+
         inc_new_counter_info!(
             "banking_stage-dropped_tx_before_forwarding",
             tx_count.saturating_sub(filtered_unprocessed_packet_indexes.len())
@@ -1180,6 +1299,8 @@ impl BankingStage {
         banking_stage_stats: &BankingStageStats,
         duplicates: &Arc<Mutex<(LruCache<u64, ()>, PacketHasher)>>,
         recorder: &TransactionRecorder,
+        cost_model: &Arc<CostModel>,
+        cost_tracker: &Arc<Mutex<CostTracker>>,
     ) -> Result<(), RecvTimeoutError> {
         let mut recv_time = Measure::start("process_packets_recv");
         let mms = verified_receiver.recv_timeout(recv_timeout)?;
@@ -1218,6 +1339,7 @@ impl BankingStage {
                 continue;
             }
             let (bank, bank_creation_time) = bank_start.unwrap();
+            Self::reset_cost_tracker_if_new_bank(cost_tracker, bank.slot());
 
             let (processed, verified_txs_len, unprocessed_indexes) =
                 Self::process_packets_transactions(
@@ -1229,6 +1351,8 @@ impl BankingStage {
                     transaction_status_sender.clone(),
                     gossip_vote_sender,
                     banking_stage_stats,
+                    cost_model,
+                    cost_tracker,
                 );
 
             new_tx_count += processed;
@@ -1259,6 +1383,8 @@ impl BankingStage {
                         &packet_indexes,
                         &my_pubkey,
                         next_leader,
+                        cost_model,
+                        cost_tracker,
                     );
                     Self::push_unprocessed(
                         buffered_packets,
@@ -1751,6 +1877,11 @@ mod tests {
                     2,
                     None,
                     gossip_vote_sender,
+                    &Arc::new(CostModel::default()),
+                    &Arc::new(Mutex::new(CostTracker::new(
+                        ACCOUNT_MAX_COST,
+                        BLOCK_MAX_COST,
+                    ))),
                 );
 
                 // wait for banking_stage to eat the packets
@@ -2571,6 +2702,11 @@ mod tests {
                 None::<Box<dyn Fn()>>,
                 &BankingStageStats::default(),
                 &recorder,
+                &Arc::new(CostModel::default()),
+                &Arc::new(Mutex::new(CostTracker::new(
+                    ACCOUNT_MAX_COST,
+                    BLOCK_MAX_COST,
+                ))),
             );
             assert_eq!(buffered_packets[0].1.len(), num_conflicting_transactions);
             // When the poh recorder has a bank, should process all non conflicting buffered packets.
@@ -2587,6 +2723,11 @@ mod tests {
                     None::<Box<dyn Fn()>>,
                     &BankingStageStats::default(),
                     &recorder,
+                    &Arc::new(CostModel::default()),
+                    &Arc::new(Mutex::new(CostTracker::new(
+                        ACCOUNT_MAX_COST,
+                        BLOCK_MAX_COST,
+                    ))),
                 );
                 if num_expected_unprocessed == 0 {
                     assert!(buffered_packets.is_empty())
@@ -2652,6 +2793,11 @@ mod tests {
                         test_fn,
                         &BankingStageStats::default(),
                         &recorder,
+                        &Arc::new(CostModel::default()),
+                        &Arc::new(Mutex::new(CostTracker::new(
+                            ACCOUNT_MAX_COST,
+                            BLOCK_MAX_COST,
+                        ))),
                     );
 
                     // Check everything is correct. All indexes after `interrupted_iteration`

core/src/cost_model.rs

@@ -0,0 +1,554 @@
+//! 'cost_model` provides service to estimate a transaction's cost
+//! It does so by analyzing accounts the transaction touches, and instructions
+//! it includes. Using historical data as guideline, it estimates cost of
+//! reading/writing account, the sum of that comes up to "account access cost";
+//! Instructions take time to execute, both historical and runtime data are
+//! used to determine each instruction's execution time, the sum of that
+//! is transaction's "execution cost"
+//! The main function is `calculate_cost` which returns a TransactionCost struct.
+//!
+use log::*;
+use solana_sdk::{
+    bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, feature, incinerator,
+    message::Message, native_loader, pubkey::Pubkey, secp256k1_program, system_program,
+    transaction::Transaction,
+};
+use std::collections::HashMap;
+
+// from mainnet-beta data, taking `vote program` as 1 COST_UNIT to load and execute
+// amount all type programs, the costs are:
+// min: 0.9 COST_UNIT
+// max: 110 COST UNIT
+// Median: 12 COST_UNIT
+// Average: 19 COST_UNIT
+const COST_UNIT: u32 = 1;
+const DEFAULT_PROGRAM_COST: u32 = COST_UNIT * 100;
+// re-adjust these numbers if needed
+const SIGNED_WRITABLE_ACCOUNT_ACCESS_COST: u32 = COST_UNIT * 10;
+const SIGNED_READONLY_ACCOUNT_ACCESS_COST: u32 = COST_UNIT * 2;
+const NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST: u32 = COST_UNIT * 5;
+const NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST: u32 = COST_UNIT;
+// running 'ledger-tool compute-cost' over mainnet ledger, the largest block cost
+// is 575_687, and the largest chain cost (eg account cost) is 559_000
+// Configuring cost model to have larger block limit and smaller account limit
+// to encourage packing parallelizable transactions in block.
+pub const ACCOUNT_MAX_COST: u32 = COST_UNIT * 10_000;
+pub const BLOCK_MAX_COST: u32 = COST_UNIT * 10_000_000;
+
+// cost of transaction is made of account_access_cost and instruction execution_cost
+// where
+// account_access_cost is the sum of read/write/sign all accounts included in the transaction
+//     read is cheaper than write.
+// execution_cost is the sum of all instructions execution cost, which is
+//     observed during runtime and feedback by Replay
+#[derive(Default, Debug)]
+pub struct TransactionCost {
+    pub writable_accounts: Vec<Pubkey>,
+    pub account_access_cost: u32,
+    pub execution_cost: u32,
+}
+
+// instruction execution code table is initialized with default values, and
+// updated with realtime information (by Replay)
+#[derive(Debug)]
+struct InstructionExecutionCostTable {
+    pub table: HashMap<Pubkey, u32>,
+}
+
+macro_rules! costmetrics {
+    ($( $key: expr => $val: expr ),*) => {{
+        let mut hashmap: HashMap< Pubkey, u32 > = HashMap::new();
+        $( hashmap.insert( $key, $val); )*
+        hashmap
+    }}
+}
+
+impl InstructionExecutionCostTable {
+    // build cost table with default value
+    pub fn new() -> Self {
+        Self {
+            table: costmetrics![
+                solana_config_program::id()        => COST_UNIT,
+                feature::id()                      => COST_UNIT * 2,
+                incinerator::id()                  => COST_UNIT * 2,
+                native_loader::id()                => COST_UNIT * 2,
+                solana_stake_program::id()         => COST_UNIT * 2,
+                solana_stake_program::config::id() => COST_UNIT,
+                solana_vote_program::id()          => COST_UNIT,
+                secp256k1_program::id()            => COST_UNIT,
+                system_program::id()               => COST_UNIT * 8,
+                bpf_loader::id()                   => COST_UNIT * 500,
+                bpf_loader_deprecated::id()        => COST_UNIT * 500,
+                bpf_loader_upgradeable::id()       => COST_UNIT * 500
+            ],
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct CostModel {
+    account_cost_limit: u32,
+    block_cost_limit: u32,
+    instruction_execution_cost_table: InstructionExecutionCostTable,
+}
+
+impl Default for CostModel {
+    fn default() -> Self {
+        CostModel::new(ACCOUNT_MAX_COST, BLOCK_MAX_COST)
+    }
+}
+
+impl CostModel {
+    pub fn new(chain_max: u32, block_max: u32) -> Self {
+        Self {
+            account_cost_limit: chain_max,
+            block_cost_limit: block_max,
+            instruction_execution_cost_table: InstructionExecutionCostTable::new(),
+        }
+    }
+
+    pub fn get_account_cost_limit(&self) -> u32 {
+        self.account_cost_limit
+    }
+
+    pub fn get_block_cost_limit(&self) -> u32 {
+        self.block_cost_limit
+    }
+
+    pub fn calculate_cost(&self, transaction: &Transaction) -> TransactionCost {
+        let (
+            signed_writable_accounts,
+            signed_readonly_accounts,
+            non_signed_writable_accounts,
+            non_signed_readonly_accounts,
+        ) = CostModel::sort_accounts_by_type(transaction.message());
+
+        let mut cost = TransactionCost {
+            writable_accounts: vec![],
+            account_access_cost: CostModel::find_account_access_cost(
+                &signed_writable_accounts,
+                &signed_readonly_accounts,
+                &non_signed_writable_accounts,
+                &non_signed_readonly_accounts,
+            ),
+            execution_cost: self.find_transaction_cost(&transaction),
+        };
+        cost.writable_accounts.extend(&signed_writable_accounts);
+        cost.writable_accounts.extend(&non_signed_writable_accounts);
+        debug!("transaction {:?} has cost {:?}", transaction, cost);
+        cost
+    }
+
+    // To update or insert instruction cost to table.
+    // When updating, uses the average of new and old values to smooth out outliers
+    pub fn upsert_instruction_cost(
+        &mut self,
+        program_key: &Pubkey,
+        cost: &u32,
+    ) -> Result<u32, &'static str> {
+        let instruction_cost = self
+            .instruction_execution_cost_table
+            .table
+            .entry(*program_key)
+            .or_insert(*cost);
+        *instruction_cost = (*instruction_cost + *cost) / 2;
+        Ok(*instruction_cost)
+    }
+
+    fn find_instruction_cost(&self, program_key: &Pubkey) -> u32 {
+        match self
+            .instruction_execution_cost_table
+            .table
+            .get(&program_key)
+        {
+            Some(cost) => *cost,
+            None => {
+                debug!(
+                    "Program key {:?} does not have assigned cost, using default {}",
+                    program_key, DEFAULT_PROGRAM_COST
+                );
+                DEFAULT_PROGRAM_COST
+            }
+        }
+    }
+
+    fn find_transaction_cost(&self, transaction: &Transaction) -> u32 {
+        let mut cost: u32 = 0;
+
+        for instruction in &transaction.message().instructions {
+            let program_id =
+                transaction.message().account_keys[instruction.program_id_index as usize];
+            let instruction_cost = self.find_instruction_cost(&program_id);
+            trace!(
+                "instruction {:?} has cost of {}",
+                instruction,
+                instruction_cost
+            );
+            cost += instruction_cost;
+        }
+        cost
+    }
+
+    fn find_account_access_cost(
+        signed_writable_accounts: &[Pubkey],
+        signed_readonly_accounts: &[Pubkey],
+        non_signed_writable_accounts: &[Pubkey],
+        non_signed_readonly_accounts: &[Pubkey],
+    ) -> u32 {
+        let mut cost = 0;
+        cost += signed_writable_accounts.len() as u32 * SIGNED_WRITABLE_ACCOUNT_ACCESS_COST;
+        cost += signed_readonly_accounts.len() as u32 * SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+        cost += non_signed_writable_accounts.len() as u32 * NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST;
+        cost += non_signed_readonly_accounts.len() as u32 * NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+        cost
+    }
+
+    fn sort_accounts_by_type(
+        message: &Message,
+    ) -> (Vec<Pubkey>, Vec<Pubkey>, Vec<Pubkey>, Vec<Pubkey>) {
+        let demote_sysvar_write_locks = true;
+        let mut signer_writable: Vec<Pubkey> = vec![];
+        let mut signer_readonly: Vec<Pubkey> = vec![];
+        let mut non_signer_writable: Vec<Pubkey> = vec![];
+        let mut non_signer_readonly: Vec<Pubkey> = vec![];
+        message.account_keys.iter().enumerate().for_each(|(i, k)| {
+            let is_signer = message.is_signer(i);
+            let is_writable = message.is_writable(i, demote_sysvar_write_locks);
+
+            if is_signer && is_writable {
+                signer_writable.push(*k);
+            } else if is_signer && !is_writable {
+                signer_readonly.push(*k);
+            } else if !is_signer && is_writable {
+                non_signer_writable.push(*k);
+            } else {
+                non_signer_readonly.push(*k);
+            }
+        });
+        (
+            signer_writable,
+            signer_readonly,
+            non_signer_writable,
+            non_signer_readonly,
+        )
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::{
+        bank::Bank,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    };
+    use solana_sdk::{
+        hash::Hash,
+        instruction::CompiledInstruction,
+        message::Message,
+        signature::{Keypair, Signer},
+        system_instruction::{self},
+        system_transaction,
+    };
+    use std::{
+        str::FromStr,
+        sync::{Arc, RwLock},
+        thread::{self, JoinHandle},
+    };
+
+    fn test_setup() -> (Keypair, Hash) {
+        solana_logger::setup();
+        let GenesisConfigInfo {
+            genesis_config,
+            mint_keypair,
+            ..
+        } = create_genesis_config(10);
+        let bank = Arc::new(Bank::new_no_wallclock_throttle(&genesis_config));
+        let start_hash = bank.last_blockhash();
+        (mint_keypair, start_hash)
+    }
+
+    #[test]
+    fn test_cost_model_instruction_cost() {
+        let testee = CostModel::default();
+
+        // find cost for known programs
+        assert_eq!(
+            COST_UNIT,
+            testee.find_instruction_cost(
+                &Pubkey::from_str("Vote111111111111111111111111111111111111111").unwrap()
+            )
+        );
+        assert_eq!(
+            COST_UNIT * 500,
+            testee.find_instruction_cost(&bpf_loader::id())
+        );
+
+        // unknown program is assigned with default cost
+        assert_eq!(
+            DEFAULT_PROGRAM_COST,
+            testee.find_instruction_cost(
+                &Pubkey::from_str("unknown111111111111111111111111111111111111").unwrap()
+            )
+        );
+    }
+
+    #[test]
+    fn test_cost_model_simple_transaction() {
+        let (mint_keypair, start_hash) = test_setup();
+
+        let keypair = Keypair::new();
+        let simple_transaction =
+            system_transaction::transfer(&mint_keypair, &keypair.pubkey(), 2, start_hash);
+        debug!(
+            "system_transaction simple_transaction {:?}",
+            simple_transaction
+        );
+
+        // expected cost for one system transfer instructions
+        let expected_cost = COST_UNIT * 8;
+
+        let testee = CostModel::default();
+        assert_eq!(
+            expected_cost,
+            testee.find_transaction_cost(&simple_transaction)
+        );
+    }
+
+    #[test]
+    fn test_cost_model_transaction_many_transfer_instructions() {
+        let (mint_keypair, start_hash) = test_setup();
+
+        let key1 = solana_sdk::pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
+        let instructions =
+            system_instruction::transfer_many(&mint_keypair.pubkey(), &[(key1, 1), (key2, 1)]);
+        let message = Message::new(&instructions, Some(&mint_keypair.pubkey()));
+        let tx = Transaction::new(&[&mint_keypair], message, start_hash);
+        debug!("many transfer transaction {:?}", tx);
+
+        // expected cost for two system transfer instructions
+        let expected_cost = COST_UNIT * 8 * 2;
+
+        let testee = CostModel::default();
+        assert_eq!(expected_cost, testee.find_transaction_cost(&tx));
+    }
+
+    #[test]
+    fn test_cost_model_message_many_different_instructions() {
+        let (mint_keypair, start_hash) = test_setup();
+
+        // construct a transaction with multiple random instructions
+        let key1 = solana_sdk::pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
+        let prog1 = solana_sdk::pubkey::new_rand();
+        let prog2 = solana_sdk::pubkey::new_rand();
+        let instructions = vec![
+            CompiledInstruction::new(3, &(), vec![0, 1]),
+            CompiledInstruction::new(4, &(), vec![0, 2]),
+        ];
+        let tx = Transaction::new_with_compiled_instructions(
+            &[&mint_keypair],
+            &[key1, key2],
+            start_hash,
+            vec![prog1, prog2],
+            instructions,
+        );
+        debug!("many random transaction {:?}", tx);
+
+        // expected cost for two random/unknown program is
+        let expected_cost = DEFAULT_PROGRAM_COST * 2;
+
+        let testee = CostModel::default();
+        assert_eq!(expected_cost, testee.find_transaction_cost(&tx));
+    }
+
+    #[test]
+    fn test_cost_model_sort_message_accounts_by_type() {
+        // construct a transaction with two random instructions with same signer
+        let signer1 = Keypair::new();
+        let signer2 = Keypair::new();
+        let key1 = Pubkey::new_unique();
+        let key2 = Pubkey::new_unique();
+        let prog1 = Pubkey::new_unique();
+        let prog2 = Pubkey::new_unique();
+        let instructions = vec![
+            CompiledInstruction::new(4, &(), vec![0, 2]),
+            CompiledInstruction::new(5, &(), vec![1, 3]),
+        ];
+        let tx = Transaction::new_with_compiled_instructions(
+            &[&signer1, &signer2],
+            &[key1, key2],
+            Hash::new_unique(),
+            vec![prog1, prog2],
+            instructions,
+        );
+        debug!("many random transaction {:?}", tx);
+
+        let (
+            signed_writable_accounts,
+            signed_readonly_accounts,
+            non_signed_writable_accounts,
+            non_signed_readonly_accounts,
+        ) = CostModel::sort_accounts_by_type(tx.message());
+
+        assert_eq!(2, signed_writable_accounts.len());
+        assert_eq!(signer1.pubkey(), signed_writable_accounts[0]);
+        assert_eq!(signer2.pubkey(), signed_writable_accounts[1]);
+        assert_eq!(0, signed_readonly_accounts.len());
+        assert_eq!(2, non_signed_writable_accounts.len());
+        assert_eq!(key1, non_signed_writable_accounts[0]);
+        assert_eq!(key2, non_signed_writable_accounts[1]);
+        assert_eq!(2, non_signed_readonly_accounts.len());
+        assert_eq!(prog1, non_signed_readonly_accounts[0]);
+        assert_eq!(prog2, non_signed_readonly_accounts[1]);
+    }
+
+    #[test]
+    fn test_cost_model_insert_instruction_cost() {
+        let key1 = Pubkey::new_unique();
+        let cost1 = 100;
+
+        let mut cost_model = CostModel::default();
+        // Using default cost for unknown instruction
+        assert_eq!(
+            DEFAULT_PROGRAM_COST,
+            cost_model.find_instruction_cost(&key1)
+        );
+
+        // insert instruction cost to table
+        assert!(cost_model.upsert_instruction_cost(&key1, &cost1).is_ok());
+
+        // now it is known insturction with known cost
+        assert_eq!(cost1, cost_model.find_instruction_cost(&key1));
+    }
+
+    #[test]
+    fn test_cost_model_calculate_cost() {
+        let (mint_keypair, start_hash) = test_setup();
+        let tx =
+            system_transaction::transfer(&mint_keypair, &Keypair::new().pubkey(), 2, start_hash);
+
+        let expected_account_cost = SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+        let expected_execution_cost = COST_UNIT * 8;
+
+        let cost_model = CostModel::default();
+        let tx_cost = cost_model.calculate_cost(&tx);
+        assert_eq!(expected_account_cost, tx_cost.account_access_cost);
+        assert_eq!(expected_execution_cost, tx_cost.execution_cost);
+        assert_eq!(2, tx_cost.writable_accounts.len());
+    }
+
+    #[test]
+    fn test_cost_model_update_instruction_cost() {
+        let key1 = Pubkey::new_unique();
+        let cost1 = 100;
+        let cost2 = 200;
+        let updated_cost = (cost1 + cost2) / 2;
+
+        let mut cost_model = CostModel::default();
+
+        // insert instruction cost to table
+        assert!(cost_model.upsert_instruction_cost(&key1, &cost1).is_ok());
+        assert_eq!(cost1, cost_model.find_instruction_cost(&key1));
+
+        // update instruction cost
+        assert!(cost_model.upsert_instruction_cost(&key1, &cost2).is_ok());
+        assert_eq!(updated_cost, cost_model.find_instruction_cost(&key1));
+    }
+
+    #[test]
+    fn test_cost_model_can_be_shared_concurrently_as_immutable() {
+        let (mint_keypair, start_hash) = test_setup();
+        let number_threads = 10;
+        let expected_account_cost = SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST;
+        let expected_execution_cost = COST_UNIT * 8;
+
+        let cost_model = Arc::new(CostModel::default());
+
+        let thread_handlers: Vec<JoinHandle<()>> = (0..number_threads)
+            .map(|_| {
+                // each thread creates its own simple transaction
+                let simple_transaction = system_transaction::transfer(
+                    &mint_keypair,
+                    &Keypair::new().pubkey(),
+                    2,
+                    start_hash,
+                );
+                let cost_model = cost_model.clone();
+                thread::spawn(move || {
+                    let tx_cost = cost_model.calculate_cost(&simple_transaction);
+                    assert_eq!(2, tx_cost.writable_accounts.len());
+                    assert_eq!(expected_account_cost, tx_cost.account_access_cost);
+                    assert_eq!(expected_execution_cost, tx_cost.execution_cost);
+                })
+            })
+            .collect();
+
+        for th in thread_handlers {
+            th.join().unwrap();
+        }
+    }
+
+    #[test]
+    fn test_cost_model_can_be_shared_concurrently_with_rwlock() {
+        let (mint_keypair, start_hash) = test_setup();
+        // construct a transaction with multiple random instructions
+        let key1 = solana_sdk::pubkey::new_rand();
+        let key2 = solana_sdk::pubkey::new_rand();
+        let prog1 = solana_sdk::pubkey::new_rand();
+        let prog2 = solana_sdk::pubkey::new_rand();
+        let instructions = vec![
+            CompiledInstruction::new(3, &(), vec![0, 1]),
+            CompiledInstruction::new(4, &(), vec![0, 2]),
+        ];
+        let tx = Arc::new(Transaction::new_with_compiled_instructions(
+            &[&mint_keypair],
+            &[key1, key2],
+            start_hash,
+            vec![prog1, prog2],
+            instructions,
+        ));
+
+        let number_threads = 10;
+        let expected_account_cost = SIGNED_WRITABLE_ACCOUNT_ACCESS_COST
+            + NON_SIGNED_WRITABLE_ACCOUNT_ACCESS_COST * 2
+            + NON_SIGNED_READONLY_ACCOUNT_ACCESS_COST * 2;
+        let cost1 = 100;
+        let cost2 = 200;
+        // execution cost can be either 2 * Default (before write) or cost1+cost2 (after write)
+        let expected_execution_cost = Arc::new(vec![cost1 + cost2, DEFAULT_PROGRAM_COST * 2]);
+
+        let cost_model: Arc<RwLock<CostModel>> = Arc::new(RwLock::new(CostModel::default()));
+
+        let thread_handlers: Vec<JoinHandle<()>> = (0..number_threads)
+            .map(|i| {
+                let cost_model = cost_model.clone();
+                let tx = tx.clone();
+                let expected_execution_cost = expected_execution_cost.clone();
+
+                if i == 5 {
+                    thread::spawn(move || {
+                        let mut cost_model = cost_model.write().unwrap();
+                        assert!(cost_model.upsert_instruction_cost(&prog1, &cost1).is_ok());
+                        assert!(cost_model.upsert_instruction_cost(&prog2, &cost2).is_ok());
+                    })
+                } else {
+                    thread::spawn(move || {
+                        let tx_cost = cost_model.read().unwrap().calculate_cost(&tx);
+                        assert_eq!(3, tx_cost.writable_accounts.len());
+                        assert_eq!(expected_account_cost, tx_cost.account_access_cost);
+                        assert!(expected_execution_cost.contains(&tx_cost.execution_cost));
+                    })
+                }
+            })
+            .collect();
+
+        for th in thread_handlers {
+            th.join().unwrap();
+        }
+    }
+}

core/src/cost_tracker.rs

@@ -0,0 +1,356 @@
+//! `cost_tracker` keeps tracking tranasction cost per chained accounts as well as for entire block
+//! The main entry function is 'try_add', if success, it returns new block cost.
+//!
+use crate::cost_model::TransactionCost;
+use solana_sdk::{clock::Slot, pubkey::Pubkey};
+use std::collections::HashMap;
+
+#[derive(Debug, Clone)]
+pub struct CostTracker {
+    account_cost_limit: u32,
+    block_cost_limit: u32,
+    current_bank_slot: Slot,
+    cost_by_writable_accounts: HashMap<Pubkey, u32>,
+    block_cost: u32,
+}
+
+impl CostTracker {
+    pub fn new(chain_max: u32, package_max: u32) -> Self {
+        assert!(chain_max <= package_max);
+        Self {
+            account_cost_limit: chain_max,
+            block_cost_limit: package_max,
+            current_bank_slot: 0,
+            cost_by_writable_accounts: HashMap::new(),
+            block_cost: 0,
+        }
+    }
+
+    pub fn reset_if_new_bank(&mut self, slot: Slot) {
+        if slot != self.current_bank_slot {
+            self.current_bank_slot = slot;
+            self.cost_by_writable_accounts.clear();
+            self.block_cost = 0;
+        }
+    }
+
+    pub fn try_add(&mut self, transaction_cost: TransactionCost) -> Result<u32, &'static str> {
+        let cost = transaction_cost.account_access_cost + transaction_cost.execution_cost;
+        self.would_fit(&transaction_cost.writable_accounts, &cost)?;
+
+        self.add_transaction(&transaction_cost.writable_accounts, &cost);
+        Ok(self.block_cost)
+    }
+
+    fn would_fit(&self, keys: &[Pubkey], cost: &u32) -> Result<(), &'static str> {
+        // check against the total package cost
+        if self.block_cost + cost > self.block_cost_limit {
+            return Err("would exceed block cost limit");
+        }
+
+        // check if the transaction itself is more costly than the account_cost_limit
+        if *cost > self.account_cost_limit {
+            return Err("Transaction is too expansive, exceeds account cost limit");
+        }
+
+        // check each account against account_cost_limit,
+        for account_key in keys.iter() {
+            match self.cost_by_writable_accounts.get(&account_key) {
+                Some(chained_cost) => {
+                    if chained_cost + cost > self.account_cost_limit {
+                        return Err("would exceed account cost limit");
+                    } else {
+                        continue;
+                    }
+                }
+                None => continue,
+            }
+        }
+
+        Ok(())
+    }
+
+    fn add_transaction(&mut self, keys: &[Pubkey], cost: &u32) {
+        for account_key in keys.iter() {
+            *self
+                .cost_by_writable_accounts
+                .entry(*account_key)
+                .or_insert(0) += cost;
+        }
+        self.block_cost += cost;
+    }
+}
+
+// CostStats can be collected by util, such as ledger_tool
+#[derive(Default, Debug)]
+pub struct CostStats {
+    pub total_cost: u32,
+    pub number_of_accounts: usize,
+    pub costliest_account: Pubkey,
+    pub costliest_account_cost: u32,
+}
+
+impl CostTracker {
+    pub fn get_stats(&self) -> CostStats {
+        let mut stats = CostStats {
+            total_cost: self.block_cost,
+            number_of_accounts: self.cost_by_writable_accounts.len(),
+            costliest_account: Pubkey::default(),
+            costliest_account_cost: 0,
+        };
+
+        for (key, cost) in self.cost_by_writable_accounts.iter() {
+            if cost > &stats.costliest_account_cost {
+                stats.costliest_account = *key;
+                stats.costliest_account_cost = *cost;
+            }
+        }
+
+        stats
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use solana_runtime::{
+        bank::Bank,
+        genesis_utils::{create_genesis_config, GenesisConfigInfo},
+    };
+    use solana_sdk::{
+        hash::Hash,
+        signature::{Keypair, Signer},
+        system_transaction,
+        transaction::Transaction,
+    };
+    use std::{cmp, sync::Arc};
+
+    fn test_setup() -> (Keypair, Hash) {
+        solana_logger::setup();
+        let GenesisConfigInfo {
+            genesis_config,
+            mint_keypair,
+            ..
+        } = create_genesis_config(10);
+        let bank = Arc::new(Bank::new_no_wallclock_throttle(&genesis_config));
+        let start_hash = bank.last_blockhash();
+        (mint_keypair, start_hash)
+    }
+
+    fn build_simple_transaction(
+        mint_keypair: &Keypair,
+        start_hash: &Hash,
+    ) -> (Transaction, Vec<Pubkey>, u32) {
+        let keypair = Keypair::new();
+        let simple_transaction =
+            system_transaction::transfer(&mint_keypair, &keypair.pubkey(), 2, *start_hash);
+
+        (simple_transaction, vec![mint_keypair.pubkey()], 5)
+    }
+
+    #[test]
+    fn test_cost_tracker_initialization() {
+        let testee = CostTracker::new(10, 11);
+        assert_eq!(10, testee.account_cost_limit);
+        assert_eq!(11, testee.block_cost_limit);
+        assert_eq!(0, testee.cost_by_writable_accounts.len());
+        assert_eq!(0, testee.block_cost);
+    }
+
+    #[test]
+    fn test_cost_tracker_ok_add_one() {
+        let (mint_keypair, start_hash) = test_setup();
+        let (_tx, keys, cost) = build_simple_transaction(&mint_keypair, &start_hash);
+
+        // build testee to have capacity for one simple transaction
+        let mut testee = CostTracker::new(cost, cost);
+        assert!(testee.would_fit(&keys, &cost).is_ok());
+        testee.add_transaction(&keys, &cost);
+        assert_eq!(cost, testee.block_cost);
+    }
+
+    #[test]
+    fn test_cost_tracker_ok_add_two_same_accounts() {
+        let (mint_keypair, start_hash) = test_setup();
+        // build two transactions with same signed account
+        let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash);
+        let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash);
+
+        // build testee to have capacity for two simple transactions, with same accounts
+        let mut testee = CostTracker::new(cost1 + cost2, cost1 + cost2);
+        {
+            assert!(testee.would_fit(&keys1, &cost1).is_ok());
+            testee.add_transaction(&keys1, &cost1);
+        }
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_ok());
+            testee.add_transaction(&keys2, &cost2);
+        }
+        assert_eq!(cost1 + cost2, testee.block_cost);
+        assert_eq!(1, testee.cost_by_writable_accounts.len());
+    }
+
+    #[test]
+    fn test_cost_tracker_ok_add_two_diff_accounts() {
+        let (mint_keypair, start_hash) = test_setup();
+        // build two transactions with diff accounts
+        let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash);
+        let second_account = Keypair::new();
+        let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash);
+
+        // build testee to have capacity for two simple transactions, with same accounts
+        let mut testee = CostTracker::new(cmp::max(cost1, cost2), cost1 + cost2);
+        {
+            assert!(testee.would_fit(&keys1, &cost1).is_ok());
+            testee.add_transaction(&keys1, &cost1);
+        }
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_ok());
+            testee.add_transaction(&keys2, &cost2);
+        }
+        assert_eq!(cost1 + cost2, testee.block_cost);
+        assert_eq!(2, testee.cost_by_writable_accounts.len());
+    }
+
+    #[test]
+    fn test_cost_tracker_chain_reach_limit() {
+        let (mint_keypair, start_hash) = test_setup();
+        // build two transactions with same signed account
+        let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash);
+        let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash);
+
+        // build testee to have capacity for two simple transactions, but not for same accounts
+        let mut testee = CostTracker::new(cmp::min(cost1, cost2), cost1 + cost2);
+        // should have room for first transaction
+        {
+            assert!(testee.would_fit(&keys1, &cost1).is_ok());
+            testee.add_transaction(&keys1, &cost1);
+        }
+        // but no more sapce on the same chain (same signer account)
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_err());
+        }
+    }
+
+    #[test]
+    fn test_cost_tracker_reach_limit() {
+        let (mint_keypair, start_hash) = test_setup();
+        // build two transactions with diff accounts
+        let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash);
+        let second_account = Keypair::new();
+        let (_tx2, keys2, cost2) = build_simple_transaction(&second_account, &start_hash);
+
+        // build testee to have capacity for each chain, but not enough room for both transactions
+        let mut testee = CostTracker::new(cmp::max(cost1, cost2), cost1 + cost2 - 1);
+        // should have room for first transaction
+        {
+            assert!(testee.would_fit(&keys1, &cost1).is_ok());
+            testee.add_transaction(&keys1, &cost1);
+        }
+        // but no more room for package as whole
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_err());
+        }
+    }
+
+    #[test]
+    fn test_cost_tracker_reset() {
+        let (mint_keypair, start_hash) = test_setup();
+        // build two transactions with same signed account
+        let (_tx1, keys1, cost1) = build_simple_transaction(&mint_keypair, &start_hash);
+        let (_tx2, keys2, cost2) = build_simple_transaction(&mint_keypair, &start_hash);
+
+        // build testee to have capacity for two simple transactions, but not for same accounts
+        let mut testee = CostTracker::new(cmp::min(cost1, cost2), cost1 + cost2);
+        // should have room for first transaction
+        {
+            assert!(testee.would_fit(&keys1, &cost1).is_ok());
+            testee.add_transaction(&keys1, &cost1);
+            assert_eq!(1, testee.cost_by_writable_accounts.len());
+            assert_eq!(cost1, testee.block_cost);
+        }
+        // but no more sapce on the same chain (same signer account)
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_err());
+        }
+        // reset the tracker
+        {
+            testee.reset_if_new_bank(100);
+            assert_eq!(0, testee.cost_by_writable_accounts.len());
+            assert_eq!(0, testee.block_cost);
+        }
+        //now the second transaction can be added
+        {
+            assert!(testee.would_fit(&keys2, &cost2).is_ok());
+        }
+    }
+
+    #[test]
+    fn test_cost_tracker_try_add_is_atomic() {
+        let acct1 = Pubkey::new_unique();
+        let acct2 = Pubkey::new_unique();
+        let acct3 = Pubkey::new_unique();
+        let cost = 100;
+        let account_max = cost * 2;
+        let block_max = account_max * 3; // for three accts
+
+        let mut testee = CostTracker::new(account_max, block_max);
+
+        // case 1: a tx writes to 3 accounts, should success, we will have:
+        // | acct1 | $cost |
+        // | acct2 | $cost |
+        // | acct2 | $cost |
+        // and block_cost = $cost
+        {
+            let tx_cost = TransactionCost {
+                writable_accounts: vec![acct1, acct2, acct3],
+                account_access_cost: 0,
+                execution_cost: cost,
+            };
+            assert!(testee.try_add(tx_cost).is_ok());
+            let stat = testee.get_stats();
+            assert_eq!(cost, stat.total_cost);
+            assert_eq!(3, stat.number_of_accounts);
+            assert_eq!(cost, stat.costliest_account_cost);
+        }
+
+        // case 2: add tx writes to acct2 with $cost, should succeed, result to
+        // | acct1 | $cost |
+        // | acct2 | $cost * 2 |
+        // | acct2 | $cost |
+        // and block_cost = $cost * 2
+        {
+            let tx_cost = TransactionCost {
+                writable_accounts: vec![acct2],
+                account_access_cost: 0,
+                execution_cost: cost,
+            };
+            assert!(testee.try_add(tx_cost).is_ok());
+            let stat = testee.get_stats();
+            assert_eq!(cost * 2, stat.total_cost);
+            assert_eq!(3, stat.number_of_accounts);
+            assert_eq!(cost * 2, stat.costliest_account_cost);
+            assert_eq!(acct2, stat.costliest_account);
+        }
+
+        // case 3: add tx writes to [acct1, acct2], acct2 exceeds limit, should failed atomically,
+        // we shoudl still have:
+        // | acct1 | $cost |
+        // | acct2 | $cost |
+        // | acct2 | $cost |
+        // and block_cost = $cost
+        {
+            let tx_cost = TransactionCost {
+                writable_accounts: vec![acct1, acct2],
+                account_access_cost: 0,
+                execution_cost: cost,
+            };
+            assert!(testee.try_add(tx_cost).is_err());
+            let stat = testee.get_stats();
+            assert_eq!(cost * 2, stat.total_cost);
+            assert_eq!(3, stat.number_of_accounts);
+            assert_eq!(cost * 2, stat.costliest_account_cost);
+            assert_eq!(acct2, stat.costliest_account);
+        }
+    }
+}

core/src/lib.rs

@@ -19,6 +19,8 @@ pub mod cluster_slots_service;
 pub mod commitment_service;
 pub mod completed_data_sets_service;
 pub mod consensus;
+pub mod cost_model;
+pub mod cost_tracker;
 pub mod fetch_stage;
 pub mod fork_choice;
 pub mod gen_keys;