From 461342cdd402907790ea757f755663c66ba5f31d Mon Sep 17 00:00:00 2001 From: "Jeff Washington (jwash)" Date: Wed, 24 May 2023 12:54:09 -0500 Subject: [PATCH] add cli args and use of partitioned rewards config (#31800) * add cli args and use of partitioned rewards config * update comments --- ledger-tool/src/args.rs | 11 +++++++++++ ledger-tool/src/main.rs | 15 +++++++++++++++ runtime/src/accounts_db.rs | 19 +++++++++++++++++++ runtime/src/partitioned_rewards.rs | 5 ++++- validator/src/cli.rs | 15 +++++++++++++++ validator/src/main.rs | 11 +++++++++++ 6 files changed, 75 insertions(+), 1 deletion(-) diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 9881e2851..25f2a0abe 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -3,6 +3,7 @@ use { solana_runtime::{ accounts_db::{AccountsDb, AccountsDbConfig, FillerAccountsConfig}, accounts_index::{AccountsIndexConfig, IndexLimitMb}, + partitioned_rewards::TestPartitionedEpochRewards, }, solana_sdk::clock::Slot, std::path::{Path, PathBuf}, @@ -25,6 +26,15 @@ pub fn get_accounts_db_config( } else { IndexLimitMb::Unspecified }; + let test_partitioned_epoch_rewards = + if arg_matches.is_present("partitioned_epoch_rewards_compare_calculation") { + TestPartitionedEpochRewards::CompareResults + } else if arg_matches.is_present("partitioned_epoch_rewards_force_enable_single_slot") { + TestPartitionedEpochRewards::ForcePartitionedEpochRewardsInOneBlock + } else { + TestPartitionedEpochRewards::None + }; + let accounts_index_drives: Vec = if arg_matches.is_present("accounts_index_path") { values_t_or_exit!(arg_matches, "accounts_index_path", String) .into_iter() @@ -53,6 +63,7 @@ pub fn get_accounts_db_config( .ok(), exhaustively_verify_refcounts: arg_matches.is_present("accounts_db_verify_refcounts"), skip_initial_hash_calc: arg_matches.is_present("accounts_db_skip_initial_hash_calculation"), + test_partitioned_epoch_rewards, ..AccountsDbConfig::default() } } diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index adca02497..d79244a77 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1540,6 +1540,21 @@ fn main() { .takes_value(false) .help("After 'verify' completes, run a final accounts hash calculation. Final hash calculation could race with accounts background service tasks and assert."), ) + .arg( + Arg::with_name("partitioned-epoch-rewards-compare-calculation") + .long("partitioned_epoch_rewards_compare_calculation") + .takes_value(false) + .help("Do normal epoch rewards distribution, but also calculate rewards using the partitioned rewards code path and compare the resulting vote and stake accounts") + .hidden(hidden_unless_forced()) + ) + .arg( + Arg::with_name("partitioned-epoch-rewards-force-enable-single-slot") + .long("partitioned_epoch_rewards_force_enable_single_slot") + .takes_value(false) + .help("Force the partitioned rewards distribution, but distribute all rewards in the first slot in the epoch. This should match consensus with the normal rewards distribution.") + .conflicts_with("partitioned_epoch_rewards_compare_calculation") + .hidden(hidden_unless_forced()) + ) .arg( Arg::with_name("print_accounts_stats") .long("print-accounts-stats") diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index c6ebaa572..a2cf8547d 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -59,6 +59,7 @@ use { cache_hash_data::{CacheHashData, CacheHashDataFile}, contains::Contains, epoch_accounts_hash::EpochAccountsHashManager, + partitioned_rewards::{PartitionedEpochRewardsConfig, TestPartitionedEpochRewards}, pubkey_bins::PubkeyBinCalculator24, read_only_accounts_cache::ReadOnlyAccountsCache, rent_collector::RentCollector, @@ -477,6 +478,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig { exhaustively_verify_refcounts: false, assert_stakes_cache_consistency: true, create_ancient_storage: CreateAncientStorage::Pack, + test_partitioned_epoch_rewards: TestPartitionedEpochRewards::CompareResults, }; pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig { index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS), @@ -488,6 +490,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig exhaustively_verify_refcounts: false, assert_stakes_cache_consistency: false, create_ancient_storage: CreateAncientStorage::Pack, + test_partitioned_epoch_rewards: TestPartitionedEpochRewards::None, }; pub type BinnedHashData = Vec>; @@ -551,6 +554,7 @@ pub struct AccountsDbConfig { pub assert_stakes_cache_consistency: bool, /// how to create ancient storages pub create_ancient_storage: CreateAncientStorage, + pub test_partitioned_epoch_rewards: TestPartitionedEpochRewards, } #[cfg(not(test))] @@ -1496,6 +1500,11 @@ pub struct AccountsDb { /// debug feature to scan every append vec and verify refcounts are equal exhaustively_verify_refcounts: bool, + /// this will live here until the feature for partitioned epoch rewards is activated. + /// At that point, this and other code can be deleted. + #[allow(dead_code)] + pub(crate) partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig, + /// the full accounts hash calculation as of a predetermined block height 'N' /// to be included in the bank hash at a predetermined block height 'M' /// The cadence is once per epoch, all nodes calculate a full accounts hash as of a known slot calculated using 'N' @@ -2440,6 +2449,7 @@ impl AccountsDb { filler_account_suffix: None, log_dead_slots: AtomicBool::new(true), exhaustively_verify_refcounts: false, + partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig::default(), epoch_accounts_hash_manager: EpochAccountsHashManager::new_invalid(), } } @@ -2514,6 +2524,14 @@ impl AccountsDb { .map(|config| config.create_ancient_storage) .unwrap_or(CreateAncientStorage::Append); + let test_partitioned_epoch_rewards = accounts_db_config + .as_ref() + .map(|config| config.test_partitioned_epoch_rewards) + .unwrap_or_default(); + + let partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig = + PartitionedEpochRewardsConfig::new(test_partitioned_epoch_rewards); + let filler_account_suffix = if filler_accounts_config.count > 0 { Some(solana_sdk::pubkey::new_rand()) } else { @@ -2535,6 +2553,7 @@ impl AccountsDb { write_cache_limit_bytes: accounts_db_config .as_ref() .and_then(|x| x.write_cache_limit_bytes), + partitioned_epoch_rewards_config, exhaustively_verify_refcounts, ..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path) }; diff --git a/runtime/src/partitioned_rewards.rs b/runtime/src/partitioned_rewards.rs index 125580e0a..41458356e 100644 --- a/runtime/src/partitioned_rewards.rs +++ b/runtime/src/partitioned_rewards.rs @@ -9,9 +9,12 @@ use solana_sdk::clock::Slot; pub(crate) struct PartitionedEpochRewardsConfig { /// Number of blocks for reward calculation and storing vote accounts. /// Distributing rewards to stake accounts begins AFTER this many blocks. + /// Normally, this will be 1. + /// if force_one_slot_partitioned_rewards, this will be 0 (ie. we take 0 blocks just for reward calculation) pub(crate) reward_calculation_num_blocks: Slot, /// number of stake accounts to store in one block during partititioned reward interval - /// if force_one_slot_partitioned_rewards, this will usually be 1 + /// normally, this is a number tuned for reasonable performance, such as 4096 accounts/block + /// if force_one_slot_partitioned_rewards, this will usually be u64::MAX so that all stake accounts are written in the first block pub(crate) stake_account_stores_per_block: Slot, /// if true, end of epoch bank rewards will force using partitioned rewards distribution. /// see `set_test_enable_partitioned_rewards` diff --git a/validator/src/cli.rs b/validator/src/cli.rs index 0f4c02a58..72df65043 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -1243,6 +1243,21 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .takes_value(true) .help("Number of bins to divide the accounts index into"), ) + .arg( + Arg::with_name("partitioned-epoch-rewards-compare-calculation") + .long("partitioned_epoch_rewards_compare_calculation") + .takes_value(false) + .help("Do normal epoch rewards distribution, but also calculate rewards using the partitioned rewards code path and compare the resulting vote and stake accounts") + .hidden(hidden_unless_forced()) + ) + .arg( + Arg::with_name("partitioned-epoch-rewards-force-enable-single-slot") + .long("partitioned_epoch_rewards_force_enable_single_slot") + .takes_value(false) + .help("Force the partitioned rewards distribution, but distribute all rewards in the first slot in the epoch. This should match consensus with the normal rewards distribution.") + .conflicts_with("partitioned_epoch_rewards_compare_calculation") + .hidden(hidden_unless_forced()) + ) .arg( Arg::with_name("accounts_index_path") .long("accounts-index-path") diff --git a/validator/src/main.rs b/validator/src/main.rs index b54f90060..573d512b5 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -40,6 +40,7 @@ use { AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, AccountsIndexConfig, IndexLimitMb, }, + partitioned_rewards::TestPartitionedEpochRewards, runtime_config::RuntimeConfig, snapshot_config::{SnapshotConfig, SnapshotUsage}, snapshot_utils::{ @@ -1119,6 +1120,15 @@ pub fn main() { accounts_index_config.bins = Some(bins); } + let test_partitioned_epoch_rewards = + if matches.is_present("partitioned_epoch_rewards_compare_calculation") { + TestPartitionedEpochRewards::CompareResults + } else if matches.is_present("partitioned_epoch_rewards_force_enable_single_slot") { + TestPartitionedEpochRewards::ForcePartitionedEpochRewardsInOneBlock + } else { + TestPartitionedEpochRewards::None + }; + accounts_index_config.index_limit_mb = if let Ok(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize) { IndexLimitMb::Limit(limit) @@ -1167,6 +1177,7 @@ pub fn main() { .is_present("accounts_db_create_ancient_storage_packed") .then_some(CreateAncientStorage::Pack) .unwrap_or_default(), + test_partitioned_epoch_rewards, ..AccountsDbConfig::default() };