From 23fa84b322b09defe1447e439aa60bcc69eff539 Mon Sep 17 00:00:00 2001 From: Ryo Onodera Date: Mon, 17 Aug 2020 14:22:16 +0900 Subject: [PATCH] Re-do rent collection check on rent-exempt account (#11349) * wip: re-do rent collection check on rent-exempt account * Let's see how the ci goes * Restore previous code * Well, almost all new changes are revertable * Update doc * Add test and gating * Fix tests * Fix tests, especially avoid to change abi... * Fix more tests... * Fix snapshot restore * Align to _new_ with better uninitialized detection --- core/src/rpc_pubsub.rs | 6 +- core/src/rpc_subscriptions.rs | 8 +-- core/tests/bank_forks.rs | 51 ++++++++++++----- docs/src/implemented-proposals/rent.md | 6 +- programs/bpf/c/src/invoked/invoked.c | 6 +- programs/bpf/rust/invoked/src/lib.rs | 6 +- runtime/src/accounts.rs | 2 + runtime/src/bank.rs | 19 +++++-- runtime/src/rent_collector.rs | 76 ++++++++++++++++++++++++-- 9 files changed, 138 insertions(+), 42 deletions(-) diff --git a/core/src/rpc_pubsub.rs b/core/src/rpc_pubsub.rs index 2c4300dad7..a0c003011b 100644 --- a/core/src/rpc_pubsub.rs +++ b/core/src/rpc_pubsub.rs @@ -582,7 +582,7 @@ mod tests { "lamports": 51, "data": bs58::encode(expected_data).into_string(), "executable": false, - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 0, @@ -692,7 +692,7 @@ mod tests { "lamports": 100, "data": expected_data, "executable": false, - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 0, @@ -855,7 +855,7 @@ mod tests { "lamports": 100, "data": "", "executable": false, - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 0, diff --git a/core/src/rpc_subscriptions.rs b/core/src/rpc_subscriptions.rs index 207daf22ae..075f2344f8 100644 --- a/core/src/rpc_subscriptions.rs +++ b/core/src/rpc_subscriptions.rs @@ -1076,7 +1076,7 @@ pub(crate) mod tests { "executable": false, "lamports": 1, "owner": "Stake11111111111111111111111111111111111111", - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 0, @@ -1158,7 +1158,7 @@ pub(crate) mod tests { "executable": false, "lamports": 1, "owner": "Stake11111111111111111111111111111111111111", - "rentEpoch": 1, + "rentEpoch": 0, }, "pubkey": alice.pubkey().to_string(), }, @@ -1570,7 +1570,7 @@ pub(crate) mod tests { "executable": false, "lamports": 1, "owner": "Stake11111111111111111111111111111111111111", - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 0, @@ -1604,7 +1604,7 @@ pub(crate) mod tests { "executable": false, "lamports": 1, "owner": "Stake11111111111111111111111111111111111111", - "rentEpoch": 1, + "rentEpoch": 0, }, }, "subscription": 1, diff --git a/core/tests/bank_forks.rs b/core/tests/bank_forks.rs index 3332d7f5c0..3db80cd7ef 100644 --- a/core/tests/bank_forks.rs +++ b/core/tests/bank_forks.rs @@ -1,31 +1,32 @@ // Long-running bank_forks tests macro_rules! DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS { - ($x:ident) => { + ($x:ident, $y:ident, $z:ident) => { #[allow(non_snake_case)] - mod $x { + mod $z { use super::*; const SNAPSHOT_VERSION: SnapshotVersion = SnapshotVersion::$x; + const OPERATING_MODE: OperatingMode = OperatingMode::$y; #[test] fn test_bank_forks_status_cache_snapshot_n() { - run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION) + run_test_bank_forks_status_cache_snapshot_n(SNAPSHOT_VERSION, OPERATING_MODE) } #[test] fn test_bank_forks_snapshot_n() { - run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION) + run_test_bank_forks_snapshot_n(SNAPSHOT_VERSION, OPERATING_MODE) } #[test] fn test_concurrent_snapshot_packaging() { - run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION) + run_test_concurrent_snapshot_packaging(SNAPSHOT_VERSION, OPERATING_MODE) } #[test] fn test_slots_to_snapshot() { - run_test_slots_to_snapshot(SNAPSHOT_VERSION) + run_test_slots_to_snapshot(SNAPSHOT_VERSION, OPERATING_MODE) } } }; @@ -49,7 +50,7 @@ mod tests { }; use solana_sdk::{ clock::Slot, - genesis_config::GenesisConfig, + genesis_config::{GenesisConfig, OperatingMode}, hash::hashv, pubkey::Pubkey, signature::{Keypair, Signer}, @@ -58,7 +59,9 @@ mod tests { use std::{fs, path::PathBuf, sync::atomic::AtomicBool, sync::mpsc::channel, sync::Arc}; use tempfile::TempDir; - DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0); + DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Development, V1_2_0_Development); + DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Preview, V1_2_0_Preview); + DEFINE_SNAPSHOT_VERSION_PARAMETERIZED_TEST_FUNCTIONS!(V1_2_0, Stable, V1_2_0_Stable); struct SnapshotTestConfig { accounts_dir: TempDir, @@ -72,12 +75,14 @@ mod tests { impl SnapshotTestConfig { fn new( snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, snapshot_interval_slots: u64, ) -> SnapshotTestConfig { let accounts_dir = TempDir::new().unwrap(); let snapshot_dir = TempDir::new().unwrap(); let snapshot_output_path = TempDir::new().unwrap(); - let genesis_config_info = create_genesis_config(10_000); + let mut genesis_config_info = create_genesis_config(10_000); + genesis_config_info.genesis_config.operating_mode = operating_mode; let bank0 = Bank::new_with_paths( &genesis_config_info.genesis_config, vec![accounts_dir.path().to_path_buf()], @@ -158,6 +163,7 @@ mod tests { // `last_slot` bank fn run_bank_forks_snapshot_n( snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, last_slot: Slot, f: F, set_root_interval: u64, @@ -166,7 +172,7 @@ mod tests { { solana_logger::setup(); // Set up snapshotting config - let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); + let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, operating_mode, 1); let bank_forks = &mut snapshot_test_config.bank_forks; let mint_keypair = &snapshot_test_config.genesis_config_info.mint_keypair; @@ -211,11 +217,15 @@ mod tests { restore_from_snapshot(bank_forks, last_slot, genesis_config, account_paths); } - fn run_test_bank_forks_snapshot_n(snapshot_version: SnapshotVersion) { + fn run_test_bank_forks_snapshot_n( + snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, + ) { // create banks up to slot 4 and create 1 new account in each bank. test that bank 4 snapshots // and restores correctly run_bank_forks_snapshot_n( snapshot_version, + operating_mode, 4, |bank, mint_keypair| { let key1 = Keypair::new().pubkey(); @@ -246,11 +256,14 @@ mod tests { } } - fn run_test_concurrent_snapshot_packaging(snapshot_version: SnapshotVersion) { + fn run_test_concurrent_snapshot_packaging( + snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, + ) { solana_logger::setup(); // Set up snapshotting config - let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, 1); + let mut snapshot_test_config = SnapshotTestConfig::new(snapshot_version, operating_mode, 1); let bank_forks = &mut snapshot_test_config.bank_forks; let accounts_dir = &snapshot_test_config.accounts_dir; @@ -395,7 +408,10 @@ mod tests { ); } - fn run_test_slots_to_snapshot(snapshot_version: SnapshotVersion) { + fn run_test_slots_to_snapshot( + snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, + ) { solana_logger::setup(); let num_set_roots = MAX_CACHE_ENTRIES * 2; @@ -404,6 +420,7 @@ mod tests { // Make sure this test never clears bank.slots_since_snapshot let mut snapshot_test_config = SnapshotTestConfig::new( snapshot_version, + operating_mode, (*add_root_interval * num_set_roots * 2) as u64, ); let mut current_bank = snapshot_test_config.bank_forks[0].clone(); @@ -437,7 +454,10 @@ mod tests { } } - fn run_test_bank_forks_status_cache_snapshot_n(snapshot_version: SnapshotVersion) { + fn run_test_bank_forks_status_cache_snapshot_n( + snapshot_version: SnapshotVersion, + operating_mode: OperatingMode, + ) { // create banks up to slot (MAX_CACHE_ENTRIES * 2) + 1 while transferring 1 lamport into 2 different accounts each time // this is done to ensure the AccountStorageEntries keep getting cleaned up as the root moves // ahead. Also tests the status_cache purge and status cache snapshotting. @@ -447,6 +467,7 @@ mod tests { for set_root_interval in &[1, 4] { run_bank_forks_snapshot_n( snapshot_version, + operating_mode, (MAX_CACHE_ENTRIES * 2 + 1) as u64, |bank, mint_keypair| { let tx = system_transaction::transfer( diff --git a/docs/src/implemented-proposals/rent.md b/docs/src/implemented-proposals/rent.md index d3d357a898..649d20b8e5 100644 --- a/docs/src/implemented-proposals/rent.md +++ b/docs/src/implemented-proposals/rent.md @@ -24,11 +24,11 @@ Even if those processes are out of scope of rent collection, all of manipulated ## Actual processing of collecting rent -Rent is due for one epoch's worth of time, and accounts always have `Account::rent_epoch` of `current_epoch + 1`. +Rent is due for one epoch's worth of time, and accounts have `Account::rent_epoch` of `current_epoch` or `current_epoch + 1` depending on the rent regime. -If the account is in the exempt regime, `Account::rent_epoch` is simply pushed to `current_epoch + 1`. +If the account is in the exempt regime, `Account::rent_epoch` is simply updated to `current_epoch`. -If the account is non-exempt, the difference between the next epoch and `Account::rent_epoch` is used to calculate the amount of rent owed by this account \(via `Rent::due()`\). Any fractional lamports of the calculation are truncated. Rent due is deducted from `Account::lamports` and `Account::rent_epoch` is updated to the next epoch. If the amount of rent due is less than one lamport, no changes are made to the account. +If the account is non-exempt, the difference between the next epoch and `Account::rent_epoch` is used to calculate the amount of rent owed by this account \(via `Rent::due()`\). Any fractional lamports of the calculation are truncated. Rent due is deducted from `Account::lamports` and `Account::rent_epoch` is updated to `current_epoch + 1` (= next epoch). If the amount of rent due is less than one lamport, no changes are made to the account. Accounts whose balance is insufficient to satisfy the rent that would be due simply fail to load. diff --git a/programs/bpf/c/src/invoked/invoked.c b/programs/bpf/c/src/invoked/invoked.c index bdd5482244..db6473f826 100644 --- a/programs/bpf/c/src/invoked/invoked.c +++ b/programs/bpf/c/src/invoked/invoked.c @@ -41,7 +41,7 @@ extern uint64_t entrypoint(const uint8_t *input) { sol_assert(accounts[ARGUMENT_INDEX].data_len == 100); sol_assert(accounts[ARGUMENT_INDEX].is_signer); sol_assert(accounts[ARGUMENT_INDEX].is_writable); - sol_assert(accounts[ARGUMENT_INDEX].rent_epoch == 1); + sol_assert(accounts[ARGUMENT_INDEX].rent_epoch == 0); sol_assert(!accounts[ARGUMENT_INDEX].executable); for (int i = 0; i < accounts[ARGUMENT_INDEX].data_len; i++) { sol_assert(accounts[ARGUMENT_INDEX].data[i] == i); @@ -53,7 +53,7 @@ extern uint64_t entrypoint(const uint8_t *input) { sol_assert(accounts[INVOKED_ARGUMENT_INDEX].data_len == 10); sol_assert(accounts[INVOKED_ARGUMENT_INDEX].is_signer); sol_assert(accounts[INVOKED_ARGUMENT_INDEX].is_writable); - sol_assert(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch == 1); + sol_assert(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch == 0); sol_assert(!accounts[INVOKED_ARGUMENT_INDEX].executable); sol_assert( @@ -62,7 +62,7 @@ extern uint64_t entrypoint(const uint8_t *input) { &bpf_loader_id)); sol_assert(!accounts[INVOKED_PROGRAM_INDEX].is_signer); sol_assert(!accounts[INVOKED_PROGRAM_INDEX].is_writable); - sol_assert(accounts[INVOKED_PROGRAM_INDEX].rent_epoch == 1); + sol_assert(accounts[INVOKED_PROGRAM_INDEX].rent_epoch == 0); sol_assert(accounts[INVOKED_PROGRAM_INDEX].executable); sol_assert(SolPubkey_same(accounts[INVOKED_PROGRAM_INDEX].key, diff --git a/programs/bpf/rust/invoked/src/lib.rs b/programs/bpf/rust/invoked/src/lib.rs index 11a1f11c47..b976697a88 100644 --- a/programs/bpf/rust/invoked/src/lib.rs +++ b/programs/bpf/rust/invoked/src/lib.rs @@ -42,7 +42,7 @@ fn process_instruction( assert_eq!(accounts[ARGUMENT_INDEX].data_len(), 100); assert!(accounts[ARGUMENT_INDEX].is_signer); assert!(accounts[ARGUMENT_INDEX].is_writable); - assert_eq!(accounts[ARGUMENT_INDEX].rent_epoch, 1); + assert_eq!(accounts[ARGUMENT_INDEX].rent_epoch, 0); assert!(!accounts[ARGUMENT_INDEX].executable); { let data = accounts[ARGUMENT_INDEX].try_borrow_data()?; @@ -59,14 +59,14 @@ fn process_instruction( assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].data_len(), 10); assert!(accounts[INVOKED_ARGUMENT_INDEX].is_signer); assert!(accounts[INVOKED_ARGUMENT_INDEX].is_writable); - assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch, 1); + assert_eq!(accounts[INVOKED_ARGUMENT_INDEX].rent_epoch, 0); assert!(!accounts[INVOKED_ARGUMENT_INDEX].executable); assert_eq!(accounts[INVOKED_PROGRAM_INDEX].key, program_id); assert_eq!(accounts[INVOKED_PROGRAM_INDEX].owner, &bpf_loader::id()); assert!(!accounts[INVOKED_PROGRAM_INDEX].is_signer); assert!(!accounts[INVOKED_PROGRAM_INDEX].is_writable); - assert_eq!(accounts[INVOKED_PROGRAM_INDEX].rent_epoch, 1); + assert_eq!(accounts[INVOKED_PROGRAM_INDEX].rent_epoch, 0); assert!(accounts[INVOKED_PROGRAM_INDEX].executable); assert_eq!( diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index 1a2e10ae19..50dd2d1284 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -778,6 +778,7 @@ mod tests { account::Account, epoch_schedule::EpochSchedule, fee_calculator::FeeCalculator, + genesis_config::OperatingMode, hash::Hash, instruction::CompiledInstruction, message::Message, @@ -1011,6 +1012,7 @@ mod tests { lamports_per_byte_year: 42, ..Rent::default() }, + OperatingMode::Development, ); let min_balance = rent_collector.rent.minimum_balance(nonce::State::size()); let fee_calculator = FeeCalculator::new(min_balance); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 45db8e65c4..6d80a75d48 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -519,7 +519,9 @@ impl Bank { slots_per_year: parent.slots_per_year, epoch_schedule, collected_rent: AtomicU64::new(0), - rent_collector: parent.rent_collector.clone_with_epoch(epoch), + rent_collector: parent + .rent_collector + .clone_with_epoch(epoch, parent.operating_mode()), max_tick_height: (slot + 1) * parent.ticks_per_slot, block_height: parent.block_height + 1, fee_calculator: fee_rate_governor.create_fee_calculator(), @@ -640,7 +642,10 @@ impl Bank { fee_calculator: fields.fee_calculator, fee_rate_governor: fields.fee_rate_governor, collected_rent: AtomicU64::new(fields.collected_rent), - rent_collector: fields.rent_collector, + // clone()-ing is needed to consider a gated behavior in rent_collector + rent_collector: fields + .rent_collector + .clone_with_epoch(fields.epoch, genesis_config.operating_mode), epoch_schedule: fields.epoch_schedule, inflation: Arc::new(RwLock::new(fields.inflation)), stakes: RwLock::new(fields.stakes), @@ -689,6 +694,7 @@ impl Bank { &bank.epoch_schedule, bank.slots_per_year, &genesis_config.rent, + genesis_config.operating_mode, ) ); @@ -1198,6 +1204,7 @@ impl Bank { &self.epoch_schedule, self.slots_per_year, &genesis_config.rent, + self.operating_mode(), ); // Add additional native programs specified in the genesis config @@ -4748,7 +4755,7 @@ mod tests { bank.get_account(&rent_exempt_pubkey).unwrap().lamports, large_lamports ); - assert_eq!(bank.get_account(&rent_exempt_pubkey).unwrap().rent_epoch, 6); + assert_eq!(bank.get_account(&rent_exempt_pubkey).unwrap().rent_epoch, 5); assert_eq!( bank.slots_by_pubkey(&rent_due_pubkey, &ancestors), vec![genesis_slot, some_slot] @@ -7889,19 +7896,19 @@ mod tests { if bank.slot == 32 { assert_eq!( bank.hash().to_string(), - "5yZDar5HaXypoeNnE9mEfVwJEEyDCP5W1pLwhuouPjqN" + "GDH7kUpcQuMT23pPeU9vZdmyMSPQPwzoqdNgFaLga7x3" ); } if bank.slot == 64 { assert_eq!( bank.hash().to_string(), - "FmPBRC6AAZgXu1QiqZ445FhLYZXun9KTtjMMKqCim9Hd" + "J4L6bT3KnMMXSufcUSy6Lg9TNi2pFVsYNvQ1Fzms2j1Z" ); } if bank.slot == 128 { assert_eq!( bank.hash().to_string(), - "Aqi2QcSoxaYu2QJHKaMHi9G8J2vNEtjpuKzjj5rHP9wr" + "BiCUyj8PsbsLW79waf1ifr3wDuZSFwLBhTkdbgHFjrtJ" ); break; } diff --git a/runtime/src/rent_collector.rs b/runtime/src/rent_collector.rs index 5f370902e9..ef2a8e7149 100644 --- a/runtime/src/rent_collector.rs +++ b/runtime/src/rent_collector.rs @@ -1,7 +1,13 @@ //! calculate and collect rent from Accounts use solana_sdk::{ - account::Account, clock::Epoch, epoch_schedule::EpochSchedule, genesis_config::GenesisConfig, - incinerator, pubkey::Pubkey, rent::Rent, sysvar, + account::Account, + clock::Epoch, + epoch_schedule::EpochSchedule, + genesis_config::{GenesisConfig, OperatingMode}, + incinerator, + pubkey::Pubkey, + rent::Rent, + sysvar, }; #[derive(Serialize, Deserialize, Clone, PartialEq, Debug, AbiExample)] @@ -10,6 +16,11 @@ pub struct RentCollector { pub epoch_schedule: EpochSchedule, pub slots_per_year: f64, pub rent: Rent, + // serde(skip) is needed not to break abi + // Also, wrap this with Option so that we can spot any uninitialized codepath (like + // snapshot restore) + #[serde(skip)] + pub operating_mode: Option, } impl Default for RentCollector { @@ -20,6 +31,7 @@ impl Default for RentCollector { // derive default value using GenesisConfig::default() slots_per_year: GenesisConfig::default().slots_per_year(), rent: Rent::default(), + operating_mode: Option::default(), } } } @@ -30,21 +42,33 @@ impl RentCollector { epoch_schedule: &EpochSchedule, slots_per_year: f64, rent: &Rent, + operating_mode: OperatingMode, ) -> Self { Self { epoch, epoch_schedule: *epoch_schedule, slots_per_year, rent: *rent, + operating_mode: Some(operating_mode), } } - pub fn clone_with_epoch(&self, epoch: Epoch) -> Self { + pub fn clone_with_epoch(&self, epoch: Epoch, operating_mode: OperatingMode) -> Self { Self { epoch, + operating_mode: Some(operating_mode), ..self.clone() } } + + fn enable_new_behavior(&self) -> bool { + match self.operating_mode.unwrap() { + OperatingMode::Development => true, + OperatingMode::Preview => self.epoch >= Epoch::max_value(), + OperatingMode::Stable => self.epoch >= Epoch::max_value(), + } + } + // updates this account's lamports and status and returns // the account rent collected, if any // @@ -74,7 +98,15 @@ impl RentCollector { if exempt || rent_due != 0 { if account.lamports > rent_due { - account.rent_epoch = self.epoch + 1; + account.rent_epoch = self.epoch + + if self.enable_new_behavior() && exempt { + // Rent isn't collected for the next epoch + // Make sure to check exempt status later in curent epoch again + 0 + } else { + // Rent is collected for next epoch + 1 + }; account.lamports -= rent_due; rent_due } else { @@ -115,21 +147,55 @@ mod tests { (account.clone(), account) }; - let rent_collector = RentCollector::default().clone_with_epoch(new_epoch); + let rent_collector = + RentCollector::default().clone_with_epoch(new_epoch, OperatingMode::Development); + // collect rent on a newly-created account let collected = rent_collector.collect_from_created_account(&Pubkey::new_rand(), &mut created_account); assert!(created_account.lamports < old_lamports); assert_eq!(created_account.lamports + collected, old_lamports); assert_ne!(created_account.rent_epoch, old_epoch); + // collect rent on a already-existing account let collected = rent_collector .collect_from_existing_account(&Pubkey::new_rand(), &mut existing_account); assert!(existing_account.lamports < old_lamports); assert_eq!(existing_account.lamports + collected, old_lamports); assert_ne!(existing_account.rent_epoch, old_epoch); + // newly created account should be collected for less rent; thus more remaining balance assert!(created_account.lamports > existing_account.lamports); assert_eq!(created_account.rent_epoch, existing_account.rent_epoch); } + + #[test] + fn test_rent_exempt_temporal_escape() { + let mut account = Account::default(); + let epoch = 3; + let huge_lamports = 123_456_789_012; + let tiny_lamports = 789_012; + let mut collected; + let pubkey = Pubkey::new_rand(); + + account.lamports = huge_lamports; + assert_eq!(account.rent_epoch, 0); + + // create a tested rent collector + let rent_collector = + RentCollector::default().clone_with_epoch(epoch, OperatingMode::Development); + + // first mark account as being collected while being rent-exempt + collected = rent_collector.collect_from_existing_account(&pubkey, &mut account); + assert_eq!(account.lamports, huge_lamports); + assert_eq!(collected, 0); + + // decrease the balance not to be rent-exempt + account.lamports = tiny_lamports; + + // ... and trigger another rent collection on the same epoch and check that rent is working + collected = rent_collector.collect_from_existing_account(&pubkey, &mut account); + assert_eq!(account.lamports, tiny_lamports - collected); + assert_ne!(collected, 0); + } }