add cli args and use of partitioned rewards config (#31800)

* add cli args and use of partitioned rewards config

* update comments

ledger-tool/src/args.rs

@@ -3,6 +3,7 @@ use {
     solana_runtime::{
         accounts_db::{AccountsDb, AccountsDbConfig, FillerAccountsConfig},
         accounts_index::{AccountsIndexConfig, IndexLimitMb},
+        partitioned_rewards::TestPartitionedEpochRewards,
     },
     solana_sdk::clock::Slot,
     std::path::{Path, PathBuf},
@@ -25,6 +26,15 @@ pub fn get_accounts_db_config(
         } else {
             IndexLimitMb::Unspecified
         };
+    let test_partitioned_epoch_rewards =
+        if arg_matches.is_present("partitioned_epoch_rewards_compare_calculation") {
+            TestPartitionedEpochRewards::CompareResults
+        } else if arg_matches.is_present("partitioned_epoch_rewards_force_enable_single_slot") {
+            TestPartitionedEpochRewards::ForcePartitionedEpochRewardsInOneBlock
+        } else {
+            TestPartitionedEpochRewards::None
+        };
+
     let accounts_index_drives: Vec<PathBuf> = if arg_matches.is_present("accounts_index_path") {
         values_t_or_exit!(arg_matches, "accounts_index_path", String)
             .into_iter()
@@ -53,6 +63,7 @@ pub fn get_accounts_db_config(
             .ok(),
         exhaustively_verify_refcounts: arg_matches.is_present("accounts_db_verify_refcounts"),
         skip_initial_hash_calc: arg_matches.is_present("accounts_db_skip_initial_hash_calculation"),
+        test_partitioned_epoch_rewards,
         ..AccountsDbConfig::default()
     }
 }

ledger-tool/src/main.rs

@@ -1540,6 +1540,21 @@ fn main() {
                     .takes_value(false)
                     .help("After 'verify' completes, run a final accounts hash calculation. Final hash calculation could race with accounts background service tasks and assert."),
             )
+            .arg(
+                Arg::with_name("partitioned-epoch-rewards-compare-calculation")
+                    .long("partitioned_epoch_rewards_compare_calculation")
+                    .takes_value(false)
+                    .help("Do normal epoch rewards distribution, but also calculate rewards using the partitioned rewards code path and compare the resulting vote and stake accounts")
+                    .hidden(hidden_unless_forced())
+            )
+            .arg(
+                Arg::with_name("partitioned-epoch-rewards-force-enable-single-slot")
+                    .long("partitioned_epoch_rewards_force_enable_single_slot")
+                    .takes_value(false)
+                    .help("Force the partitioned rewards distribution, but distribute all rewards in the first slot in the epoch. This should match consensus with the normal rewards distribution.")
+                    .conflicts_with("partitioned_epoch_rewards_compare_calculation")
+                    .hidden(hidden_unless_forced())
+            )
             .arg(
                 Arg::with_name("print_accounts_stats")
                     .long("print-accounts-stats")

runtime/src/accounts_db.rs

@@ -59,6 +59,7 @@ use {
         cache_hash_data::{CacheHashData, CacheHashDataFile},
         contains::Contains,
         epoch_accounts_hash::EpochAccountsHashManager,
+        partitioned_rewards::{PartitionedEpochRewardsConfig, TestPartitionedEpochRewards},
         pubkey_bins::PubkeyBinCalculator24,
         read_only_accounts_cache::ReadOnlyAccountsCache,
         rent_collector::RentCollector,
@@ -477,6 +478,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
     exhaustively_verify_refcounts: false,
     assert_stakes_cache_consistency: true,
     create_ancient_storage: CreateAncientStorage::Pack,
+    test_partitioned_epoch_rewards: TestPartitionedEpochRewards::CompareResults,
 };
 pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
     index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
@@ -488,6 +490,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig
     exhaustively_verify_refcounts: false,
     assert_stakes_cache_consistency: false,
     create_ancient_storage: CreateAncientStorage::Pack,
+    test_partitioned_epoch_rewards: TestPartitionedEpochRewards::None,
 };
 
 pub type BinnedHashData = Vec<Vec<CalculateHashIntermediate>>;
@@ -551,6 +554,7 @@ pub struct AccountsDbConfig {
     pub assert_stakes_cache_consistency: bool,
     /// how to create ancient storages
     pub create_ancient_storage: CreateAncientStorage,
+    pub test_partitioned_epoch_rewards: TestPartitionedEpochRewards,
 }
 
 #[cfg(not(test))]
@@ -1496,6 +1500,11 @@ pub struct AccountsDb {
     /// debug feature to scan every append vec and verify refcounts are equal
     exhaustively_verify_refcounts: bool,
 
+    /// this will live here until the feature for partitioned epoch rewards is activated.
+    /// At that point, this and other code can be deleted.
+    #[allow(dead_code)]
+    pub(crate) partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig,
+
     /// the full accounts hash calculation as of a predetermined block height 'N'
     /// to be included in the bank hash at a predetermined block height 'M'
     /// The cadence is once per epoch, all nodes calculate a full accounts hash as of a known slot calculated using 'N'
@@ -2440,6 +2449,7 @@ impl AccountsDb {
             filler_account_suffix: None,
             log_dead_slots: AtomicBool::new(true),
             exhaustively_verify_refcounts: false,
+            partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig::default(),
             epoch_accounts_hash_manager: EpochAccountsHashManager::new_invalid(),
         }
     }
@@ -2514,6 +2524,14 @@ impl AccountsDb {
             .map(|config| config.create_ancient_storage)
             .unwrap_or(CreateAncientStorage::Append);
 
+        let test_partitioned_epoch_rewards = accounts_db_config
+            .as_ref()
+            .map(|config| config.test_partitioned_epoch_rewards)
+            .unwrap_or_default();
+
+        let partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig =
+            PartitionedEpochRewardsConfig::new(test_partitioned_epoch_rewards);
+
         let filler_account_suffix = if filler_accounts_config.count > 0 {
             Some(solana_sdk::pubkey::new_rand())
         } else {
@@ -2535,6 +2553,7 @@ impl AccountsDb {
             write_cache_limit_bytes: accounts_db_config
                 .as_ref()
                 .and_then(|x| x.write_cache_limit_bytes),
+            partitioned_epoch_rewards_config,
             exhaustively_verify_refcounts,
             ..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
         };

runtime/src/partitioned_rewards.rs

@@ -9,9 +9,12 @@ use solana_sdk::clock::Slot;
 pub(crate) struct PartitionedEpochRewardsConfig {
     /// Number of blocks for reward calculation and storing vote accounts.
     /// Distributing rewards to stake accounts begins AFTER this many blocks.
+    /// Normally, this will be 1.
+    /// if force_one_slot_partitioned_rewards, this will be 0 (ie. we take 0 blocks just for reward calculation)
     pub(crate) reward_calculation_num_blocks: Slot,
     /// number of stake accounts to store in one block during partititioned reward interval
-    /// if force_one_slot_partitioned_rewards, this will usually be 1
+    /// normally, this is a number tuned for reasonable performance, such as 4096 accounts/block
+    /// if force_one_slot_partitioned_rewards, this will usually be u64::MAX so that all stake accounts are written in the first block
     pub(crate) stake_account_stores_per_block: Slot,
     /// if true, end of epoch bank rewards will force using partitioned rewards distribution.
     /// see `set_test_enable_partitioned_rewards`

validator/src/cli.rs

@@ -1243,6 +1243,21 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> {
                 .takes_value(true)
                 .help("Number of bins to divide the accounts index into"),
         )
+        .arg(
+            Arg::with_name("partitioned-epoch-rewards-compare-calculation")
+                .long("partitioned_epoch_rewards_compare_calculation")
+                .takes_value(false)
+                .help("Do normal epoch rewards distribution, but also calculate rewards using the partitioned rewards code path and compare the resulting vote and stake accounts")
+                .hidden(hidden_unless_forced())
+        )
+        .arg(
+            Arg::with_name("partitioned-epoch-rewards-force-enable-single-slot")
+                .long("partitioned_epoch_rewards_force_enable_single_slot")
+                .takes_value(false)
+                .help("Force the partitioned rewards distribution, but distribute all rewards in the first slot in the epoch. This should match consensus with the normal rewards distribution.")
+                .conflicts_with("partitioned_epoch_rewards_compare_calculation")
+                .hidden(hidden_unless_forced())
+        )
         .arg(
             Arg::with_name("accounts_index_path")
                 .long("accounts-index-path")

validator/src/main.rs

@@ -40,6 +40,7 @@ use {
             AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
             AccountsIndexConfig, IndexLimitMb,
         },
+        partitioned_rewards::TestPartitionedEpochRewards,
         runtime_config::RuntimeConfig,
         snapshot_config::{SnapshotConfig, SnapshotUsage},
         snapshot_utils::{
@@ -1119,6 +1120,15 @@ pub fn main() {
         accounts_index_config.bins = Some(bins);
     }
 
+    let test_partitioned_epoch_rewards =
+        if matches.is_present("partitioned_epoch_rewards_compare_calculation") {
+            TestPartitionedEpochRewards::CompareResults
+        } else if matches.is_present("partitioned_epoch_rewards_force_enable_single_slot") {
+            TestPartitionedEpochRewards::ForcePartitionedEpochRewardsInOneBlock
+        } else {
+            TestPartitionedEpochRewards::None
+        };
+
     accounts_index_config.index_limit_mb =
         if let Ok(limit) = value_t!(matches, "accounts_index_memory_limit_mb", usize) {
             IndexLimitMb::Limit(limit)
@@ -1167,6 +1177,7 @@ pub fn main() {
             .is_present("accounts_db_create_ancient_storage_packed")
             .then_some(CreateAncientStorage::Pack)
             .unwrap_or_default(),
+        test_partitioned_epoch_rewards,
         ..AccountsDbConfig::default()
     };