diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 7c9959d17..240c5d016 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -1282,10 +1282,10 @@ pub struct Bank { /// Deprecated, do not use /// Latest transaction fees for transactions processed by this bank - fee_calculator: FeeCalculator, + pub(crate) fee_calculator: FeeCalculator, /// Track cluster signature throughput and adjust fee rate - fee_rate_governor: FeeRateGovernor, + pub(crate) fee_rate_governor: FeeRateGovernor, /// Rent that has been collected collected_rent: AtomicU64, diff --git a/runtime/src/genesis_utils.rs b/runtime/src/genesis_utils.rs index 27aee6089..5fec8cea1 100644 --- a/runtime/src/genesis_utils.rs +++ b/runtime/src/genesis_utils.rs @@ -196,18 +196,22 @@ pub fn create_genesis_config_with_leader( pub fn activate_all_features(genesis_config: &mut GenesisConfig) { // Activate all features at genesis in development mode for feature_id in FeatureSet::default().inactive { - genesis_config.accounts.insert( - feature_id, - Account::from(feature::create_account( - &Feature { - activated_at: Some(0), - }, - std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1), - )), - ); + activate_feature(genesis_config, feature_id); } } +pub fn activate_feature(genesis_config: &mut GenesisConfig, feature_id: Pubkey) { + genesis_config.accounts.insert( + feature_id, + Account::from(feature::create_account( + &Feature { + activated_at: Some(0), + }, + std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1), + )), + ); +} + #[allow(clippy::too_many_arguments)] pub fn create_genesis_config_with_leader_ex( mint_lamports: u64, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 53a43bb9a..27953bc3a 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -151,6 +151,14 @@ trait TypeContext<'a>: PartialEq { where Self: std::marker::Sized; + #[cfg(test)] + fn serialize_bank_and_storage_without_extra_fields( + serializer: S, + serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>, + ) -> std::result::Result + where + Self: std::marker::Sized; + fn serialize_accounts_db_fields( serializer: S, serializable_db: &SerializableAccountsDb<'a, Self>, @@ -315,6 +323,37 @@ where }) } +#[cfg(test)] +pub(crate) fn bank_to_stream_no_extra_fields( + serde_style: SerdeStyle, + stream: &mut BufWriter, + bank: &Bank, + snapshot_storages: &[SnapshotStorage], +) -> Result<(), Error> +where + W: Write, +{ + macro_rules! INTO { + ($style:ident) => { + bincode::serialize_into( + stream, + &SerializableBankAndStorageNoExtra::<$style::Context> { + bank, + snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + ) + }; + } + match serde_style { + SerdeStyle::Newer => INTO!(newer), + } + .map_err(|err| { + warn!("bankrc_to_stream error: {:?}", err); + err + }) +} + /// deserialize the bank from 'stream_reader' /// modify the accounts_hash /// reserialize the bank to 'stream_writer' @@ -381,6 +420,39 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> { } } +#[cfg(test)] +struct SerializableBankAndStorageNoExtra<'a, C> { + bank: &'a Bank, + snapshot_storages: &'a [SnapshotStorage], + phantom: std::marker::PhantomData, +} + +#[cfg(test)] +impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorageNoExtra<'a, C> { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::ser::Serializer, + { + C::serialize_bank_and_storage_without_extra_fields(serializer, self) + } +} + +#[cfg(test)] +impl<'a, C> From> for SerializableBankAndStorage<'a, C> { + fn from(s: SerializableBankAndStorageNoExtra<'a, C>) -> SerializableBankAndStorage<'a, C> { + let SerializableBankAndStorageNoExtra { + bank, + snapshot_storages, + phantom, + } = s; + SerializableBankAndStorage { + bank, + snapshot_storages, + phantom, + } + } +} + struct SerializableAccountsDb<'a, C> { accounts_db: &'a AccountsDb, slot: Slot, diff --git a/runtime/src/serde_snapshot/newer.rs b/runtime/src/serde_snapshot/newer.rs index 999b8ac88..6ab9a1b23 100644 --- a/runtime/src/serde_snapshot/newer.rs +++ b/runtime/src/serde_snapshot/newer.rs @@ -191,6 +191,33 @@ impl<'a> TypeContext<'a> for Context { serializer: S, serializable_bank: &SerializableBankAndStorage<'a, Self>, ) -> std::result::Result + where + Self: std::marker::Sized, + { + let ancestors = HashMap::from(&serializable_bank.bank.ancestors); + let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors); + let lamports_per_signature = fields.fee_rate_governor.lamports_per_signature; + ( + SerializableVersionedBank::from(fields), + SerializableAccountsDb::<'a, Self> { + accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db, + slot: serializable_bank.bank.rc.slot, + account_storage_entries: serializable_bank.snapshot_storages, + phantom: std::marker::PhantomData::default(), + }, + // Additional fields, we manually store the lamps per signature here so that + // we can grab it on restart. + // TODO: if we do a snapshot version bump, consider moving this out. + lamports_per_signature, + ) + .serialize(serializer) + } + + #[cfg(test)] + fn serialize_bank_and_storage_without_extra_fields( + serializer: S, + serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>, + ) -> std::result::Result where Self: std::marker::Sized, { @@ -279,8 +306,18 @@ impl<'a> TypeContext<'a> for Context { where R: Read, { - let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); + let mut bank_fields: BankFieldsToDeserialize = + deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into(); let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?; + // Process extra fields + let lamports_per_signature: u64 = match deserialize_from(stream) { + Err(err) if err.to_string() == "io error: unexpected end of file" => Ok(0), + Err(err) if err.to_string() == "io error: failed to fill whole buffer" => Ok(0), + result => result, + }?; + bank_fields.fee_rate_governor = bank_fields + .fee_rate_governor + .clone_with_lamports_per_signature(lamports_per_signature); Ok((bank_fields, accounts_db_fields)) } @@ -311,6 +348,7 @@ impl<'a> TypeContext<'a> for Context { let rhs = bank_fields; let blockhash_queue = RwLock::new(rhs.blockhash_queue.clone()); let hard_forks = RwLock::new(rhs.hard_forks.clone()); + let lamports_per_signature = rhs.fee_rate_governor.lamports_per_signature; let bank = SerializableVersionedBank { blockhash_queue: &blockhash_queue, ancestors: &rhs.ancestors, @@ -346,6 +384,9 @@ impl<'a> TypeContext<'a> for Context { is_delta: rhs.is_delta, }; - bincode::serialize_into(stream_writer, &(bank, accounts_db_fields)) + bincode::serialize_into( + stream_writer, + &(bank, accounts_db_fields, lamports_per_signature), + ) } } diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 1ac1a55ce..2d4fca051 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -5,13 +5,16 @@ use { accounts::{test_utils::create_test_accounts, Accounts}, accounts_db::{get_temp_accounts_paths, AccountShrinkThreshold}, bank::{Bank, Rewrites, StatusCacheRc}, + genesis_utils::{activate_all_features, activate_feature}, hardened_unpack::UnpackedAppendVecMap, + snapshot_utils::ArchiveFormat, }, bincode::serialize_into, rand::{thread_rng, Rng}, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::Slot, + feature_set::disable_fee_calculator, genesis_config::{create_genesis_config, ClusterType}, pubkey::Pubkey, signature::{Keypair, Signer}, @@ -361,13 +364,192 @@ fn test_bank_serialize_newer() { } } +#[test] +fn test_extra_fields_eof() { + solana_logger::setup(); + let (mut genesis_config, _) = create_genesis_config(500); + activate_feature(&mut genesis_config, disable_fee_calculator::id()); + + let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); + bank0.squash(); + let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); + + // Set extra fields + bank.fee_rate_governor.lamports_per_signature = 7000; + + // Serialize + let snapshot_storages = bank.get_snapshot_storages(None); + let mut buf = vec![]; + let mut writer = Cursor::new(&mut buf); + crate::serde_snapshot::bank_to_stream( + SerdeStyle::Newer, + &mut std::io::BufWriter::new(&mut writer), + &bank, + &snapshot_storages, + ) + .unwrap(); + + // Deserialize + let rdr = Cursor::new(&buf[..]); + let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]); + let mut snapshot_streams = SnapshotStreams { + full_snapshot_stream: &mut reader, + incremental_snapshot_stream: None, + }; + let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap(); + let copied_accounts = TempDir::new().unwrap(); + let unpacked_append_vec_map = + copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); + let dbank = crate::serde_snapshot::bank_from_streams( + SerdeStyle::Newer, + &mut snapshot_streams, + &dbank_paths, + unpacked_append_vec_map, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + None, + ) + .unwrap(); + + assert_eq!( + bank.fee_rate_governor.lamports_per_signature, + dbank.fee_rate_governor.lamports_per_signature + ); +} + +#[test] +fn test_extra_fields_full_snapshot_archive() { + solana_logger::setup(); + + let (mut genesis_config, _) = create_genesis_config(500); + activate_all_features(&mut genesis_config); + + let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); + let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); + while !bank.is_complete() { + bank.fill_bank_with_ticks_for_tests(); + } + + // Set extra field + bank.fee_rate_governor.lamports_per_signature = 7000; + + let accounts_dir = TempDir::new().unwrap(); + let bank_snapshots_dir = TempDir::new().unwrap(); + let full_snapshot_archives_dir = TempDir::new().unwrap(); + let incremental_snapshot_archives_dir = TempDir::new().unwrap(); + + // Serialize + let snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive( + &bank_snapshots_dir, + &bank, + None, + full_snapshot_archives_dir.path(), + incremental_snapshot_archives_dir.path(), + ArchiveFormat::TarBzip2, + 1, + 0, + ) + .unwrap(); + + // Deserialize + let (dbank, _) = snapshot_utils::bank_from_snapshot_archives( + &[PathBuf::from(accounts_dir.path())], + bank_snapshots_dir.path(), + &snapshot_archive_info, + None, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + false, + false, + Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + None, + ) + .unwrap(); + + assert_eq!( + bank.fee_rate_governor.lamports_per_signature, + dbank.fee_rate_governor.lamports_per_signature + ); +} + +#[test] +fn test_blank_extra_fields() { + solana_logger::setup(); + let (mut genesis_config, _) = create_genesis_config(500); + activate_feature(&mut genesis_config, disable_fee_calculator::id()); + + let bank0 = Arc::new(Bank::new_for_tests(&genesis_config)); + bank0.squash(); + let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1); + + // Set extra fields + bank.fee_rate_governor.lamports_per_signature = 7000; + + // Serialize, but don't serialize the extra fields + let snapshot_storages = bank.get_snapshot_storages(None); + let mut buf = vec![]; + let mut writer = Cursor::new(&mut buf); + crate::serde_snapshot::bank_to_stream_no_extra_fields( + SerdeStyle::Newer, + &mut std::io::BufWriter::new(&mut writer), + &bank, + &snapshot_storages, + ) + .unwrap(); + + // Deserialize + let rdr = Cursor::new(&buf[..]); + let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]); + let mut snapshot_streams = SnapshotStreams { + full_snapshot_stream: &mut reader, + incremental_snapshot_stream: None, + }; + let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap(); + let copied_accounts = TempDir::new().unwrap(); + let unpacked_append_vec_map = + copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap(); + let dbank = crate::serde_snapshot::bank_from_streams( + SerdeStyle::Newer, + &mut snapshot_streams, + &dbank_paths, + unpacked_append_vec_map, + &genesis_config, + None, + None, + AccountSecondaryIndexes::default(), + false, + None, + AccountShrinkThreshold::default(), + false, + Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + None, + ) + .unwrap(); + + // Defaults to 0 + assert_eq!(0, dbank.fee_rate_governor.lamports_per_signature); +} + #[cfg(RUSTC_WITH_SPECIALIZATION)] mod test_bank_serialize { use super::*; // This some what long test harness is required to freeze the ABI of // Bank's serialization due to versioned nature - #[frozen_abi(digest = "HT9yewU4zJ6ZAgJ7aDSbHPtzZGZqASpq6rkq6ET42Kki")] + #[frozen_abi(digest = "9vGBt7YfymKUTPWLHVVpQbDtPD7dFDwXRMFkCzwujNqJ")] #[derive(Serialize, AbiExample)] pub struct BankAbiTestWrapperNewer { #[serde(serialize_with = "wrapper_newer")] diff --git a/sdk/program/src/fee_calculator.rs b/sdk/program/src/fee_calculator.rs index 5c91cd94c..69ec783bc 100644 --- a/sdk/program/src/fee_calculator.rs +++ b/sdk/program/src/fee_calculator.rs @@ -165,6 +165,13 @@ impl FeeRateGovernor { me } + pub fn clone_with_lamports_per_signature(&self, lamports_per_signature: u64) -> Self { + Self { + lamports_per_signature, + ..*self + } + } + /// calculate unburned fee from a fee total, returns (unburned, burned) pub fn burn(&self, fees: u64) -> (u64, u64) { let burned = fees * u64::from(self.burn_percent) / 100;