diff --git a/Cargo.lock b/Cargo.lock index 5207f5f3f9..141db9f030 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5480,6 +5480,7 @@ dependencies = [ "ed25519-dalek", "flate2", "fnv", + "histogram", "itertools 0.10.1", "lazy_static", "libsecp256k1 0.6.0", diff --git a/banking-bench/src/main.rs b/banking-bench/src/main.rs index 36b096f087..baec1c05f3 100644 --- a/banking-bench/src/main.rs +++ b/banking-bench/src/main.rs @@ -4,7 +4,7 @@ use crossbeam_channel::unbounded; use log::*; use rand::{thread_rng, Rng}; use rayon::prelude::*; -use solana_core::{banking_stage::BankingStage, cost_model::CostModel, cost_tracker::CostTracker}; +use solana_core::banking_stage::BankingStage; use solana_gossip::{cluster_info::ClusterInfo, cluster_info::Node}; use solana_ledger::{ blockstore::Blockstore, @@ -16,6 +16,7 @@ use solana_perf::packet::to_packets_chunked; use solana_poh::poh_recorder::{create_test_recorder, PohRecorder, WorkingBankEntry}; use solana_runtime::{ accounts_background_service::AbsRequestSender, bank::Bank, bank_forks::BankForks, + cost_model::CostModel, cost_tracker::CostTracker, }; use solana_sdk::{ hash::Hash, diff --git a/core/benches/banking_stage.rs b/core/benches/banking_stage.rs index 992160dbc7..479966538e 100644 --- a/core/benches/banking_stage.rs +++ b/core/benches/banking_stage.rs @@ -8,9 +8,6 @@ use log::*; use rand::{thread_rng, Rng}; use rayon::prelude::*; use solana_core::banking_stage::{BankingStage, BankingStageStats}; -use solana_core::cost_model::CostModel; -use solana_core::cost_tracker::CostTracker; -use solana_core::cost_tracker_stats::CostTrackerStats; use solana_entry::entry::{next_hash, Entry}; use solana_gossip::cluster_info::ClusterInfo; use solana_gossip::cluster_info::Node; @@ -21,6 +18,9 @@ use solana_perf::packet::to_packets_chunked; use solana_perf::test_tx::test_tx; use solana_poh::poh_recorder::{create_test_recorder, WorkingBankEntry}; use solana_runtime::bank::Bank; +use solana_runtime::cost_model::CostModel; +use solana_runtime::cost_tracker::CostTracker; +use solana_runtime::cost_tracker_stats::CostTrackerStats; use solana_sdk::genesis_config::GenesisConfig; use solana_sdk::hash::Hash; use solana_sdk::message::Message; diff --git a/core/src/banking_stage.rs b/core/src/banking_stage.rs index dd583461bd..5d242f7f78 100644 --- a/core/src/banking_stage.rs +++ b/core/src/banking_stage.rs @@ -1,9 +1,7 @@ //! The `banking_stage` processes Transaction messages. It is intended to be used //! to contruct a software pipeline. The stage uses all available CPU cores and //! can do its processing in parallel with signature verification on the GPU. -use crate::{ - cost_tracker::CostTracker, cost_tracker_stats::CostTrackerStats, packet_hasher::PacketHasher, -}; +use crate::packet_hasher::PacketHasher; use crossbeam_channel::{Receiver as CrossbeamReceiver, RecvTimeoutError}; use itertools::Itertools; use lru::LruCache; @@ -27,6 +25,8 @@ use solana_runtime::{ TransactionExecutionResult, }, bank_utils, + cost_tracker::CostTracker, + cost_tracker_stats::CostTrackerStats, transaction_batch::TransactionBatch, vote_sender_types::ReplayVoteSender, }; @@ -1697,7 +1697,6 @@ where #[cfg(test)] mod tests { use super::*; - use crate::cost_model::CostModel; use crossbeam_channel::unbounded; use itertools::Itertools; use solana_entry::entry::{next_entry, Entry, EntrySlice}; @@ -1714,6 +1713,7 @@ mod tests { poh_service::PohService, }; use solana_rpc::transaction_status_service::TransactionStatusService; + use solana_runtime::cost_model::CostModel; use solana_sdk::{ hash::Hash, instruction::InstructionError, diff --git a/core/src/cost_update_service.rs b/core/src/cost_update_service.rs index 798b62206f..0698a187fa 100644 --- a/core/src/cost_update_service.rs +++ b/core/src/cost_update_service.rs @@ -3,10 +3,9 @@ //! packing transactions into block; it also triggers persisting cost //! table to blockstore. -use crate::cost_model::CostModel; use solana_ledger::blockstore::Blockstore; use solana_measure::measure::Measure; -use solana_runtime::bank::ExecuteTimings; +use solana_runtime::{bank::ExecuteTimings, cost_model::CostModel}; use solana_sdk::timing::timestamp; use std::{ sync::{ diff --git a/core/src/lib.rs b/core/src/lib.rs index 91309e4422..99c351e204 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -20,12 +20,8 @@ pub mod cluster_slots_service; pub mod commitment_service; pub mod completed_data_sets_service; pub mod consensus; -pub mod cost_model; -pub mod cost_tracker; -pub mod cost_tracker_stats; pub mod cost_update_service; pub mod duplicate_repair_status; -pub mod execute_cost_table; pub mod fetch_stage; pub mod fork_choice; pub mod gen_keys; diff --git a/core/src/tpu.rs b/core/src/tpu.rs index 245e873eb9..0c45edbde5 100644 --- a/core/src/tpu.rs +++ b/core/src/tpu.rs @@ -8,8 +8,6 @@ use crate::{ ClusterInfoVoteListener, GossipDuplicateConfirmedSlotsSender, GossipVerifiedVoteHashSender, VerifiedVoteSender, VoteTracker, }, - cost_model::CostModel, - cost_tracker::CostTracker, fetch_stage::FetchStage, sigverify::TransactionSigVerifier, sigverify_stage::SigVerifyStage, @@ -24,6 +22,8 @@ use solana_rpc::{ }; use solana_runtime::{ bank_forks::BankForks, + cost_model::CostModel, + cost_tracker::CostTracker, vote_sender_types::{ReplayVoteReceiver, ReplayVoteSender}, }; use std::{ diff --git a/core/src/tvu.rs b/core/src/tvu.rs index 939103e879..5da7738331 100644 --- a/core/src/tvu.rs +++ b/core/src/tvu.rs @@ -12,7 +12,6 @@ use crate::{ cluster_slots::ClusterSlots, completed_data_sets_service::CompletedDataSetsSender, consensus::Tower, - cost_model::CostModel, cost_update_service::CostUpdateService, ledger_cleanup_service::LedgerCleanupService, replay_stage::{ReplayStage, ReplayStageConfig}, @@ -43,6 +42,7 @@ use solana_runtime::{ bank::ExecuteTimings, bank_forks::BankForks, commitment::BlockCommitmentCache, + cost_model::CostModel, snapshot_config::SnapshotConfig, snapshot_package::{AccountsPackageReceiver, AccountsPackageSender, PendingSnapshotPackage}, vote_sender_types::ReplayVoteSender, diff --git a/core/src/validator.rs b/core/src/validator.rs index cae64f2f61..bc6b3d3caf 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -8,7 +8,6 @@ use { cluster_info_vote_listener::VoteTracker, completed_data_sets_service::CompletedDataSetsService, consensus::{reconcile_blockstore_roots_with_tower, Tower}, - cost_model::CostModel, rewards_recorder_service::{RewardsRecorderSender, RewardsRecorderService}, sample_performance_service::SamplePerformanceService, serve_repair::ServeRepair, @@ -69,6 +68,7 @@ use { bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache, + cost_model::CostModel, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig, diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 282cc1290b..69966eae80 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -15,9 +15,6 @@ use solana_clap_utils::{ is_bin, is_parsable, is_pubkey, is_pubkey_or_keypair, is_slot, is_valid_percentage, }, }; -use solana_core::cost_model::CostModel; -use solana_core::cost_tracker::CostTracker; -use solana_core::cost_tracker_stats::CostTrackerStats; use solana_entry::entry::Entry; use solana_ledger::{ ancestor_iterator::AncestorIterator, @@ -32,6 +29,9 @@ use solana_runtime::{ accounts_index::AccountsIndexConfig, bank::{Bank, RewardCalculationEvent}, bank_forks::BankForks, + cost_model::CostModel, + cost_tracker::CostTracker, + cost_tracker_stats::CostTrackerStats, hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_config::SnapshotConfig, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index cf155331c5..9755c28ebb 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -1,7 +1,6 @@ use crate::{ - block_cost_limits::*, block_error::BlockError, blockstore::Blockstore, - blockstore_db::BlockstoreError, blockstore_meta::SlotMeta, - leader_schedule_cache::LeaderScheduleCache, + block_error::BlockError, blockstore::Blockstore, blockstore_db::BlockstoreError, + blockstore_meta::SlotMeta, leader_schedule_cache::LeaderScheduleCache, }; use chrono_humanize::{Accuracy, HumanTime, Tense}; use crossbeam_channel::Sender; @@ -25,6 +24,7 @@ use solana_runtime::{ }, bank_forks::BankForks, bank_utils, + block_cost_limits::*, commitment::VOTE_THRESHOLD_SIZE, snapshot_config::SnapshotConfig, snapshot_package::{AccountsPackageSender, SnapshotType}, diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs index 4baccf5362..75b51a77ff 100644 --- a/ledger/src/lib.rs +++ b/ledger/src/lib.rs @@ -11,7 +11,6 @@ pub mod block_error; #[macro_use] pub mod blockstore; pub mod ancestor_iterator; -pub mod block_cost_limits; pub mod blockstore_db; pub mod blockstore_meta; pub mod blockstore_processor; diff --git a/programs/bpf/Cargo.lock b/programs/bpf/Cargo.lock index 53c88fa699..a60504eaaa 100644 --- a/programs/bpf/Cargo.lock +++ b/programs/bpf/Cargo.lock @@ -1112,6 +1112,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "histogram" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" + [[package]] name = "hmac" version = "0.8.1" @@ -3242,6 +3248,7 @@ dependencies = [ "dir-diff", "flate2", "fnv", + "histogram", "itertools 0.10.1", "lazy_static", "log", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index cdb3dc85fe..e4c8006d68 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,6 +21,7 @@ crossbeam-channel = "0.5" dir-diff = "0.3.2" flate2 = "1.0.22" fnv = "1.0.7" +histogram = "0.6.9" itertools = "0.10.1" lazy_static = "1.4.0" log = "0.4.14" diff --git a/ledger/src/block_cost_limits.rs b/runtime/src/block_cost_limits.rs similarity index 100% rename from ledger/src/block_cost_limits.rs rename to runtime/src/block_cost_limits.rs diff --git a/core/src/cost_model.rs b/runtime/src/cost_model.rs similarity index 99% rename from core/src/cost_model.rs rename to runtime/src/cost_model.rs index 43c41cacc4..a49f7900e7 100644 --- a/core/src/cost_model.rs +++ b/runtime/src/cost_model.rs @@ -4,9 +4,8 @@ //! //! The main function is `calculate_cost` which returns &TransactionCost. //! -use crate::execute_cost_table::ExecuteCostTable; +use crate::{block_cost_limits::*, execute_cost_table::ExecuteCostTable}; use log::*; -use solana_ledger::block_cost_limits::*; use solana_sdk::{pubkey::Pubkey, transaction::SanitizedTransaction}; use std::collections::HashMap; @@ -25,7 +24,7 @@ pub enum CostModelError { WouldExceedAccountMaxLimit, } -#[derive(Default, Debug)] +#[derive(AbiExample, Default, Debug)] pub struct TransactionCost { pub writable_accounts: Vec, pub signature_cost: u64, @@ -55,7 +54,7 @@ impl TransactionCost { } } -#[derive(Debug)] +#[derive(AbiExample, Debug)] pub struct CostModel { account_cost_limit: u64, block_cost_limit: u64, @@ -219,7 +218,7 @@ impl CostModel { #[cfg(test)] mod tests { use super::*; - use solana_runtime::{ + use crate::{ bank::Bank, genesis_utils::{create_genesis_config, GenesisConfigInfo}, }; diff --git a/core/src/cost_tracker.rs b/runtime/src/cost_tracker.rs similarity index 98% rename from core/src/cost_tracker.rs rename to runtime/src/cost_tracker.rs index 645a81469c..bb26856b21 100644 --- a/core/src/cost_tracker.rs +++ b/runtime/src/cost_tracker.rs @@ -14,7 +14,7 @@ use std::{ const WRITABLE_ACCOUNTS_PER_BLOCK: usize = 512; -#[derive(Debug)] +#[derive(AbiExample, Debug)] pub struct CostTracker { cost_model: Arc>, account_cost_limit: u64, @@ -24,6 +24,12 @@ pub struct CostTracker { block_cost: u64, } +impl Default for CostTracker { + fn default() -> Self { + CostTracker::new(Arc::new(RwLock::new(CostModel::default()))) + } +} + impl CostTracker { pub fn new(cost_model: Arc>) -> Self { let (account_cost_limit, block_cost_limit) = { @@ -191,7 +197,7 @@ impl CostTracker { #[cfg(test)] mod tests { use super::*; - use solana_runtime::{ + use crate::{ bank::Bank, genesis_utils::{create_genesis_config, GenesisConfigInfo}, }; diff --git a/core/src/cost_tracker_stats.rs b/runtime/src/cost_tracker_stats.rs similarity index 100% rename from core/src/cost_tracker_stats.rs rename to runtime/src/cost_tracker_stats.rs diff --git a/core/src/execute_cost_table.rs b/runtime/src/execute_cost_table.rs similarity index 94% rename from core/src/execute_cost_table.rs rename to runtime/src/execute_cost_table.rs index cef757a026..f89fc427d7 100644 --- a/core/src/execute_cost_table.rs +++ b/runtime/src/execute_cost_table.rs @@ -5,7 +5,7 @@ /// to make room for new ones. use log::*; use solana_sdk::pubkey::Pubkey; -use std::{collections::HashMap, time::SystemTime}; +use std::collections::HashMap; // prune is rather expensive op, free up bulk space in each operation // would be more efficient. PRUNE_RATIO defines the after prune table @@ -16,11 +16,11 @@ const OCCURRENCES_WEIGHT: i64 = 100; const DEFAULT_CAPACITY: usize = 1024; -#[derive(Debug)] +#[derive(AbiExample, Debug)] pub struct ExecuteCostTable { capacity: usize, table: HashMap, - occurrences: HashMap, + occurrences: HashMap, } impl Default for ExecuteCostTable { @@ -91,9 +91,9 @@ impl ExecuteCostTable { let (count, timestamp) = self .occurrences .entry(*key) - .or_insert((0, SystemTime::now())); + .or_insert((0, Self::micros_since_epoch())); *count += 1; - *timestamp = SystemTime::now(); + *timestamp = Self::micros_since_epoch(); Some(*program_cost) } @@ -120,12 +120,12 @@ impl ExecuteCostTable { return; } - let now = SystemTime::now(); + let now = Self::micros_since_epoch(); let mut sorted_by_weighted_age: Vec<_> = self .occurrences .iter() .map(|(key, (count, timestamp))| { - let age = now.duration_since(*timestamp).unwrap().as_micros(); + let age = now - timestamp; let weighted_age = *count as i64 * OCCURRENCES_WEIGHT + -(age as i64); (weighted_age, *key) }) @@ -140,6 +140,13 @@ impl ExecuteCostTable { } } } + + fn micros_since_epoch() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_micros() + } } #[cfg(test)] diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 52a6251a2a..6fa37a60f3 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -14,6 +14,7 @@ pub mod bank; pub mod bank_client; pub mod bank_forks; pub mod bank_utils; +pub mod block_cost_limits; pub mod blockhash_queue; pub mod bloom; pub mod bucket_map_holder; @@ -23,7 +24,11 @@ pub mod cache_hash_data; pub mod cache_hash_data_stats; pub mod commitment; pub mod contains; +pub mod cost_model; +pub mod cost_tracker; +pub mod cost_tracker_stats; pub mod epoch_stakes; +pub mod execute_cost_table; pub mod genesis_utils; pub mod hardened_unpack; pub mod in_mem_accounts_index;