From af08221aecc613eaf695a032a44ef66ed7fa9618 Mon Sep 17 00:00:00 2001 From: carllin Date: Wed, 2 Sep 2020 00:37:36 -0700 Subject: [PATCH] Switch account hashing to blake3 (#11969) * Switch account hashing to blake3 Co-authored-by: Carl --- Cargo.lock | 47 ++++- accounts-bench/src/main.rs | 4 +- programs/bpf/Cargo.lock | 63 ++++++- runtime/Cargo.toml | 1 + runtime/benches/accounts.rs | 26 ++- runtime/src/accounts.rs | 21 +-- runtime/src/accounts_db.rs | 257 ++++++++++++++++++++-------- runtime/src/bank.rs | 2 +- runtime/src/serde_snapshot.rs | 12 +- runtime/src/serde_snapshot/tests.rs | 5 +- sdk/src/hash.rs | 2 +- 11 files changed, 330 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 071d325d1..7d385bbff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -268,6 +268,21 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "blake3" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -625,6 +640,16 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.3", + "subtle 2.2.2", +] + [[package]] name = "csv" version = "1.1.3" @@ -664,7 +689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" dependencies = [ "byteorder", - "digest", + "digest 0.8.1", "rand_core 0.5.1", "subtle 2.2.2", "zeroize", @@ -707,6 +732,15 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.3", +] + [[package]] name = "dir-diff" version = "0.3.2" @@ -1288,8 +1322,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" dependencies = [ - "crypto-mac", - "digest", + "crypto-mac 0.7.0", + "digest 0.8.1", ] [[package]] @@ -2253,7 +2287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ "byteorder", - "crypto-mac", + "crypto-mac 0.7.0", ] [[package]] @@ -3094,7 +3128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -3112,7 +3146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -4137,6 +4171,7 @@ version = "1.4.0" dependencies = [ "assert_matches", "bincode", + "blake3", "bv", "byteorder", "bzip2", diff --git a/accounts-bench/src/main.rs b/accounts-bench/src/main.rs index 82dfbd325..3f2ab3779 100644 --- a/accounts-bench/src/main.rs +++ b/accounts-bench/src/main.rs @@ -5,7 +5,7 @@ use solana_runtime::{ accounts::{create_test_accounts, update_accounts, Accounts}, accounts_index::Ancestors, }; -use solana_sdk::pubkey::Pubkey; +use solana_sdk::{genesis_config::OperatingMode, pubkey::Pubkey}; use std::fs; use std::path::PathBuf; @@ -54,7 +54,7 @@ fn main() { if fs::remove_dir_all(path.clone()).is_err() { println!("Warning: Couldn't remove {:?}", path); } - let accounts = Accounts::new(vec![path]); + let accounts = Accounts::new(vec![path], &OperatingMode::Preview); println!("Creating {} accounts", num_accounts); let mut create_time = Measure::start("create accounts"); let pubkeys: Vec<_> = (0..num_slots) diff --git a/programs/bpf/Cargo.lock b/programs/bpf/Cargo.lock index 9f709b057..23bed2a0d 100644 --- a/programs/bpf/Cargo.lock +++ b/programs/bpf/Cargo.lock @@ -24,6 +24,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + [[package]] name = "ascii" version = "0.7.1" @@ -95,6 +107,21 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake3" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "crypto-mac 0.8.0", + "digest 0.9.0", +] + [[package]] name = "block-buffer" version = "0.7.3" @@ -241,6 +268,12 @@ dependencies = [ "byteorder 1.3.4", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "crc32fast" version = "1.2.0" @@ -317,6 +350,16 @@ dependencies = [ "subtle 1.0.0", ] +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array 0.14.3", + "subtle 2.2.2", +] + [[package]] name = "curve25519-dalek" version = "2.1.0" @@ -324,7 +367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" dependencies = [ "byteorder 1.3.4", - "digest", + "digest 0.8.1", "rand_core", "subtle 2.2.2", "zeroize", @@ -339,6 +382,15 @@ dependencies = [ "generic-array 0.12.3", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array 0.14.3", +] + [[package]] name = "dir-diff" version = "0.3.2" @@ -686,8 +738,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695" dependencies = [ - "crypto-mac", - "digest", + "crypto-mac 0.7.0", + "digest 0.8.1", ] [[package]] @@ -1151,7 +1203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9" dependencies = [ "byteorder 1.3.4", - "crypto-mac", + "crypto-mac 0.7.0", ] [[package]] @@ -1578,7 +1630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69" dependencies = [ "block-buffer", - "digest", + "digest 0.8.1", "fake-simd", "opaque-debug", ] @@ -1853,6 +1905,7 @@ name = "solana-runtime" version = "1.4.0" dependencies = [ "bincode", + "blake3", "bv", "byteorder 1.3.4", "bzip2", diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index e2253c0ab..46d29d1a2 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -10,6 +10,7 @@ edition = "2018" [dependencies] bincode = "1.3.1" +blake3 = "0.3.6" bv = { version = "0.11.1", features = ["serde"] } byteorder = "1.3.4" bzip2 = "0.3.3" diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index 4b9e99bc5..a92125b3a 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -6,7 +6,11 @@ use solana_runtime::{ accounts::{create_test_accounts, Accounts}, bank::*, }; -use solana_sdk::{account::Account, genesis_config::create_genesis_config, pubkey::Pubkey}; +use solana_sdk::{ + account::Account, + genesis_config::{create_genesis_config, OperatingMode}, + pubkey::Pubkey, +}; use std::{path::PathBuf, sync::Arc}; use test::Bencher; @@ -65,7 +69,10 @@ fn test_accounts_squash(bencher: &mut Bencher) { #[bench] fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { - let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]); + let accounts = Accounts::new( + vec![PathBuf::from("bench_accounts_hash_internal")], + &OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 60000, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -76,7 +83,10 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) { #[bench] fn test_update_accounts_hash(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("update_accounts_hash")]); + let accounts = Accounts::new( + vec![PathBuf::from("update_accounts_hash")], + &OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 50_000, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -88,7 +98,10 @@ fn test_update_accounts_hash(bencher: &mut Bencher) { #[bench] fn test_accounts_delta_hash(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("accounts_delta_hash")]); + let accounts = Accounts::new( + vec![PathBuf::from("accounts_delta_hash")], + &OperatingMode::Development, + ); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 100_000, 0); bencher.iter(|| { @@ -99,7 +112,10 @@ fn test_accounts_delta_hash(bencher: &mut Bencher) { #[bench] fn bench_delete_dependencies(bencher: &mut Bencher) { solana_logger::setup(); - let accounts = Accounts::new(vec![PathBuf::from("accounts_delete_deps")]); + let accounts = Accounts::new( + vec![PathBuf::from("accounts_delete_deps")], + &OperatingMode::Development, + ); let mut old_pubkey = Pubkey::default(); let zero_account = Account::new(0, 0, &Account::default().owner); for i in 0..1000 { diff --git a/runtime/src/accounts.rs b/runtime/src/accounts.rs index bab8b3de5..48abe30c4 100644 --- a/runtime/src/accounts.rs +++ b/runtime/src/accounts.rs @@ -18,6 +18,7 @@ use solana_sdk::{ account::Account, clock::Slot, fee_calculator::FeeCalculator, + genesis_config::OperatingMode, hash::Hash, message::Message, native_loader, nonce, @@ -66,10 +67,10 @@ pub enum AccountAddressFilter { } impl Accounts { - pub fn new(paths: Vec) -> Self { + pub fn new(paths: Vec, operating_mode: &OperatingMode) -> Self { Self { slot: 0, - accounts_db: Arc::new(AccountsDB::new(paths)), + accounts_db: Arc::new(AccountsDB::new(paths, operating_mode)), account_locks: Mutex::new(HashSet::new()), readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))), } @@ -819,7 +820,7 @@ mod tests { ) -> Vec<(Result, Option)> { let mut hash_queue = BlockhashQueue::new(100); hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); for ka in ka.iter() { accounts.store_slow(0, &ka.0, &ka.1); } @@ -1385,7 +1386,7 @@ mod tests { #[test] fn test_load_by_program_slot() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); // Load accounts owned by various programs into AccountsDB let pubkey0 = Pubkey::new_rand(); @@ -1408,7 +1409,7 @@ mod tests { #[test] fn test_accounts_account_not_found() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); let mut error_counters = ErrorCounters::default(); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -1430,7 +1431,7 @@ mod tests { #[test] #[should_panic] fn test_accounts_empty_bank_hash() { - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); accounts.bank_hash_at(1); } @@ -1446,7 +1447,7 @@ mod tests { let account2 = Account::new(3, 0, &Pubkey::default()); let account3 = Account::new(4, 0, &Pubkey::default()); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1558,7 +1559,7 @@ mod tests { let account1 = Account::new(2, 0, &Pubkey::default()); let account2 = Account::new(3, 0, &Pubkey::default()); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); accounts.store_slow(0, &keypair0.pubkey(), &account0); accounts.store_slow(0, &keypair1.pubkey(), &account1); accounts.store_slow(0, &keypair2.pubkey(), &account2); @@ -1688,7 +1689,7 @@ mod tests { let mut loaded = vec![loaded0, loaded1]; - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); { let mut readonly_locks = accounts.readonly_locks.write().unwrap(); let readonly_locks = readonly_locks.as_mut().unwrap(); @@ -1739,7 +1740,7 @@ mod tests { #[test] fn huge_clean() { solana_logger::setup(); - let accounts = Accounts::new(Vec::new()); + let accounts = Accounts::new(Vec::new(), &OperatingMode::Development); let mut old_pubkey = Pubkey::default(); let zero_account = Account::new(0, 0, &Account::default().owner); info!("storing.."); diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 16521b84d..0b921aa69 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -22,7 +22,7 @@ use crate::{ accounts_index::{AccountsIndex, Ancestors, SlotList, SlotSlice}, append_vec::{AppendVec, StoredAccount, StoredMeta}, }; -use byteorder::{ByteOrder, LittleEndian}; +use blake3::traits::digest::Digest; use lazy_static::lazy_static; use log::*; use rand::{thread_rng, Rng}; @@ -33,9 +33,11 @@ use solana_rayon_threadlimit::get_thread_count; use solana_sdk::{ account::Account, clock::{Epoch, Slot}, + genesis_config::OperatingMode, hash::{Hash, Hasher}, pubkey::Pubkey, }; +use std::convert::TryFrom; use std::{ collections::{HashMap, HashSet}, io::{Error as IOError, Result as IOResult}, @@ -413,6 +415,8 @@ pub struct AccountsDB { dead_slots: RwLock>, stats: AccountsStats, + + operating_mode: Option, } #[derive(Debug, Default)] @@ -474,16 +478,18 @@ impl Default for AccountsDB { frozen_accounts: HashMap::new(), dead_slots: RwLock::new(HashSet::new()), stats: AccountsStats::default(), + operating_mode: None, } } } impl AccountsDB { - pub fn new(paths: Vec) -> Self { + pub fn new(paths: Vec, operating_mode: &OperatingMode) -> Self { let new = if !paths.is_empty() { Self { paths, temp_paths: None, + operating_mode: Some(*operating_mode), ..Self::default() } } else { @@ -493,6 +499,7 @@ impl AccountsDB { Self { paths, temp_paths: Some(temp_dirs), + operating_mode: Some(*operating_mode), ..Self::default() } }; @@ -508,14 +515,14 @@ impl AccountsDB { pub fn new_single() -> Self { AccountsDB { min_num_stores: 0, - ..AccountsDB::new(Vec::new()) + ..AccountsDB::new(Vec::new(), &OperatingMode::Development) } } #[cfg(test)] pub fn new_sized(paths: Vec, file_size: u64) -> Self { AccountsDB { file_size, - ..AccountsDB::new(paths) + ..AccountsDB::new(paths, &OperatingMode::Development) } } @@ -1273,28 +1280,61 @@ impl AccountsDB { assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none()); } - pub fn hash_stored_account(slot: Slot, account: &StoredAccount) -> Hash { - Self::hash_account_data( - slot, - account.account_meta.lamports, - &account.account_meta.owner, - account.account_meta.executable, - account.account_meta.rent_epoch, - account.data, - &account.meta.pubkey, - ) + pub fn hash_stored_account( + slot: Slot, + account: &StoredAccount, + operating_mode: &OperatingMode, + ) -> Hash { + if slot > Self::get_blake3_slot(operating_mode) { + Self::blake3_hash_account_data( + slot, + account.account_meta.lamports, + &account.account_meta.owner, + account.account_meta.executable, + account.account_meta.rent_epoch, + account.data, + &account.meta.pubkey, + ) + } else { + Self::hash_account_data( + slot, + account.account_meta.lamports, + &account.account_meta.owner, + account.account_meta.executable, + account.account_meta.rent_epoch, + account.data, + &account.meta.pubkey, + ) + } } - pub fn hash_account(slot: Slot, account: &Account, pubkey: &Pubkey) -> Hash { - Self::hash_account_data( - slot, - account.lamports, - &account.owner, - account.executable, - account.rent_epoch, - &account.data, - pubkey, - ) + pub fn hash_account( + slot: Slot, + account: &Account, + pubkey: &Pubkey, + operating_mode: &OperatingMode, + ) -> Hash { + if slot > Self::get_blake3_slot(operating_mode) { + Self::blake3_hash_account_data( + slot, + account.lamports, + &account.owner, + account.executable, + account.rent_epoch, + &account.data, + pubkey, + ) + } else { + Self::hash_account_data( + slot, + account.lamports, + &account.owner, + account.executable, + account.rent_epoch, + &account.data, + pubkey, + ) + } } fn hash_frozen_account_data(account: &Account) -> Hash { @@ -1326,16 +1366,12 @@ impl AccountsDB { } let mut hasher = Hasher::default(); - let mut buf = [0u8; 8]; - LittleEndian::write_u64(&mut buf[..], lamports); - hasher.hash(&buf); + hasher.hash(&lamports.to_le_bytes()); - LittleEndian::write_u64(&mut buf[..], slot); - hasher.hash(&buf); + hasher.hash(&slot.to_le_bytes()); - LittleEndian::write_u64(&mut buf[..], rent_epoch); - hasher.hash(&buf); + hasher.hash(&rent_epoch.to_le_bytes()); hasher.hash(&data); @@ -1351,6 +1387,50 @@ impl AccountsDB { hasher.result() } + pub fn blake3_hash_account_data( + slot: Slot, + lamports: u64, + owner: &Pubkey, + executable: bool, + rent_epoch: Epoch, + data: &[u8], + pubkey: &Pubkey, + ) -> Hash { + if lamports == 0 { + return Hash::default(); + } + + let mut hasher = blake3::Hasher::new(); + + hasher.update(&lamports.to_le_bytes()); + + hasher.update(&slot.to_le_bytes()); + + hasher.update(&rent_epoch.to_le_bytes()); + + hasher.update(&data); + + if executable { + hasher.update(&[1u8; 1]); + } else { + hasher.update(&[0u8; 1]); + } + + hasher.update(&owner.as_ref()); + hasher.update(&pubkey.as_ref()); + + Hash(<[u8; solana_sdk::hash::HASH_BYTES]>::try_from(hasher.finalize().as_slice()).unwrap()) + } + + fn get_blake3_slot(operating_mode: &OperatingMode) -> Slot { + match operating_mode { + OperatingMode::Development => 0, + // Epoch 75 + OperatingMode::Stable => 32_400_000, + OperatingMode::Preview => std::u64::MAX, + } + } + fn bulk_assign_write_version(&self, count: usize) -> u64 { self.write_version .fetch_add(count as u64, Ordering::Relaxed) @@ -1566,7 +1646,13 @@ impl AccountsDB { let account = store.accounts.get_account(account_info.offset)?.0; if check_hash { - let hash = Self::hash_stored_account(*slot, &account); + let hash = Self::hash_stored_account( + *slot, + &account, + &self + .operating_mode + .expect("Operating mode must be set at initialization"), + ); if hash != *account.hash { mismatch_found.fetch_add(1, Ordering::Relaxed); return None; @@ -1801,13 +1887,18 @@ impl AccountsDB { } } - fn hash_accounts(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) -> Vec { + fn hash_accounts( + &self, + slot: Slot, + accounts: &[(&Pubkey, &Account)], + operating_mode: &OperatingMode, + ) -> Vec { let mut stats = BankHashStats::default(); let hashes: Vec<_> = accounts .iter() .map(|(pubkey, account)| { stats.update(account); - Self::hash_account(slot, account, pubkey) + Self::hash_account(slot, account, pubkey, operating_mode) }) .collect(); @@ -1872,7 +1963,13 @@ impl AccountsDB { /// Store the account update. pub fn store(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) { self.assert_frozen_accounts(accounts); - let hashes = self.hash_accounts(slot, accounts); + let hashes = self.hash_accounts( + slot, + accounts, + &self + .operating_mode + .expect("Operating mode must be set at initialization"), + ); self.store_with_hashes(slot, accounts, &hashes); } @@ -2083,7 +2180,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2096,7 +2193,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2123,7 +2220,7 @@ pub mod tests { #[test] fn test_accountsdb_latest_ancestor_with_root() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2143,7 +2240,7 @@ pub mod tests { #[test] fn test_accountsdb_root_one_slot() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); @@ -2184,7 +2281,7 @@ pub mod tests { #[test] fn test_accountsdb_add_root_many() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); @@ -2273,7 +2370,7 @@ pub mod tests { let key = Pubkey::default(); // 1 token in the "root", i.e. db zero - let db0 = AccountsDB::new(Vec::new()); + let db0 = AccountsDB::new(Vec::new(), &OperatingMode::Development); let account0 = Account::new(1, 0, &key); db0.store(0, &[(&key, &account0)]); @@ -2292,7 +2389,7 @@ pub mod tests { #[test] fn test_remove_unrooted_slot() { let unrooted_slot = 9; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account0 = Account::new(1, 0, &key); let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect(); @@ -2338,7 +2435,7 @@ pub mod tests { #[test] fn test_remove_unrooted_slot_snapshot() { let unrooted_slot = 9; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::new_rand(); let account0 = Account::new(1, 0, &key); db.store(unrooted_slot, &[(&key, &account0)]); @@ -2465,7 +2562,7 @@ pub mod tests { #[test] fn test_account_one() { let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap(); - let db = AccountsDB::new(paths); + let db = AccountsDB::new(paths, &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2478,7 +2575,7 @@ pub mod tests { #[test] fn test_account_many() { let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap(); - let db = AccountsDB::new(paths); + let db = AccountsDB::new(paths, &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&db, &mut pubkeys, 0, 100, 0, 0); check_accounts(&db, &pubkeys, 0, 100, 1); @@ -2596,7 +2693,7 @@ pub mod tests { #[test] fn test_purge_slot_not_root() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2607,7 +2704,7 @@ pub mod tests { #[test] fn test_purge_slot_after_root() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_account(&accounts, &mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -2622,7 +2719,7 @@ pub mod tests { //This test is pedantic //A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is //not root, it means we are retaining dead banks. - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2700,7 +2797,7 @@ pub mod tests { fn test_clean_old_with_normal_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2726,7 +2823,7 @@ pub mod tests { fn test_clean_old_with_zero_lamport_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey1 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand(); let normal_account = Account::new(1, 0, &Account::default().owner); @@ -2756,7 +2853,7 @@ pub mod tests { fn test_clean_old_with_both_normal_and_zero_lamport_accounts() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey1 = Pubkey::new_rand(); let pubkey2 = Pubkey::new_rand(); let normal_account = Account::new(1, 0, &Account::default().owner); @@ -2789,7 +2886,7 @@ pub mod tests { fn test_uncleaned_roots_with_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); //store an account @@ -2809,7 +2906,7 @@ pub mod tests { fn test_uncleaned_roots_with_no_account() { solana_logger::setup(); - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); assert_eq!(accounts.uncleaned_root_count(), 0); @@ -3243,7 +3340,7 @@ pub mod tests { #[test] fn test_accountsdb_scan_accounts() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let key0 = Pubkey::new_rand(); let account0 = Account::new(1, 0, &key); @@ -3307,7 +3404,7 @@ pub mod tests { #[test] fn test_store_large_account() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let data_len = DEFAULT_FILE_SIZE as usize + 7; @@ -3392,7 +3489,7 @@ pub mod tests { fn test_frozen_account_lamport_increase() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3427,7 +3524,7 @@ pub mod tests { fn test_frozen_account_lamport_decrease() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3447,7 +3544,7 @@ pub mod tests { fn test_frozen_account_nonexistent() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let ancestors = vec![(0, 0)].into_iter().collect(); db.freeze_accounts(&ancestors, &[frozen_pubkey]); @@ -3460,7 +3557,7 @@ pub mod tests { fn test_frozen_account_data_modified() { let frozen_pubkey = Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap(); - let mut db = AccountsDB::new(Vec::new()); + let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let mut account = Account::new(1, 42, &frozen_pubkey); db.store(0, &[(&frozen_pubkey, &account)]); @@ -3510,15 +3607,20 @@ pub mod tests { }; let account = stored_account.clone_account(); let expected_account_hash = - Hash::from_str("5iRNZVcAnq9JLYjSF2ibFhGEeq48r9Eq9HXxwm3BxywN").unwrap(); + Hash::from_str("4StuvYHFd7xuShVXB94uHHvpqGMCaacdZnYB74QQkPA1").unwrap(); assert_eq!( - AccountsDB::hash_stored_account(slot, &stored_account), + AccountsDB::hash_stored_account(slot, &stored_account, &OperatingMode::Development), expected_account_hash, "StoredAccount's data layout might be changed; update hashing if needed." ); assert_eq!( - AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey), + AccountsDB::hash_account( + slot, + &account, + &stored_account.meta.pubkey, + &OperatingMode::Development + ), expected_account_hash, "Account-based hashing must be consistent with StoredAccount-based one." ); @@ -3527,7 +3629,7 @@ pub mod tests { #[test] fn test_bank_hash_stats() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 5; @@ -3555,7 +3657,7 @@ pub mod tests { fn test_verify_bank_hash() { use BankHashVerificationError::*; solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 0; @@ -3593,7 +3695,7 @@ pub mod tests { #[test] fn test_verify_bank_hash_no_account() { solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let some_slot: Slot = 0; let ancestors = vec![(some_slot, 0)].into_iter().collect(); @@ -3611,7 +3713,7 @@ pub mod tests { fn test_verify_bank_hash_bad_account_hash() { use BankHashVerificationError::*; solana_logger::setup(); - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let some_data_len = 0; @@ -3621,7 +3723,7 @@ pub mod tests { let accounts = &[(&key, &account)]; // update AccountsDB's bank hash but discard real account hashes - db.hash_accounts(some_slot, accounts); + db.hash_accounts(some_slot, accounts, &OperatingMode::Development); // provide bogus account hashes let some_hash = Hash::new(&[0xca; HASH_BYTES]); db.store_with_hashes(some_slot, accounts, &[some_hash]); @@ -3635,7 +3737,7 @@ pub mod tests { #[test] fn test_bad_bank_hash() { use solana_sdk::signature::{Keypair, Signer}; - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let some_slot: Slot = 0; let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect(); @@ -3660,7 +3762,12 @@ pub mod tests { for (key, account) in &accounts_keys { assert_eq!( db.load_account_hash(&ancestors, key), - AccountsDB::hash_account(some_slot, &account, &key) + AccountsDB::hash_account( + some_slot, + &account, + &key, + &OperatingMode::Development + ) ); } } @@ -3668,13 +3775,13 @@ pub mod tests { #[test] fn test_get_snapshot_storages_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); assert!(db.get_snapshot_storages(0).is_empty()); } #[test] fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3692,7 +3799,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_only_non_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3716,7 +3823,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_only_roots() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3732,7 +3839,7 @@ pub mod tests { #[test] fn test_get_snapshot_storages_exclude_empty() { - let db = AccountsDB::new(Vec::new()); + let db = AccountsDB::new(Vec::new(), &OperatingMode::Development); let key = Pubkey::default(); let account = Account::new(1, 0, &key); @@ -3751,7 +3858,7 @@ pub mod tests { #[test] #[should_panic(expected = "double remove of account in slot: 0/store: 0!!")] fn test_storage_remove_account_double_remove() { - let accounts = AccountsDB::new(Vec::new()); + let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development); let pubkey = Pubkey::new_rand(); let account = Account::new(1, 0, &Account::default().owner); accounts.store(0, &[(&pubkey, &account)]); diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index e5dcf4acc..1ba09d75e 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -477,7 +477,7 @@ impl Bank { bank.operating_mode = Some(genesis_config.operating_mode); bank.ancestors.insert(bank.slot(), 0); - bank.rc.accounts = Arc::new(Accounts::new(paths)); + bank.rc.accounts = Arc::new(Accounts::new(paths, &genesis_config.operating_mode)); bank.process_genesis_config(genesis_config); bank.finish_init(); diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index 5a40685a8..4df8a0f4b 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -22,6 +22,7 @@ use { epoch_schedule::EpochSchedule, fee_calculator::{FeeCalculator, FeeRateGovernor}, genesis_config::GenesisConfig, + genesis_config::OperatingMode, hard_forks::HardForks, hash::Hash, inflation::Inflation, @@ -228,8 +229,12 @@ where E: Into, P: AsRef, { - let mut accounts_db = - reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?; + let mut accounts_db = reconstruct_accountsdb_from_fields( + accounts_db_fields, + account_paths, + append_vecs_path, + &genesis_config.operating_mode, + )?; accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys); let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot); @@ -242,12 +247,13 @@ fn reconstruct_accountsdb_from_fields( accounts_db_fields: AccountsDbFields, account_paths: &[PathBuf], stream_append_vecs_path: P, + operating_mode: &OperatingMode, ) -> Result where E: Into, P: AsRef, { - let accounts_db = AccountsDB::new(account_paths.to_vec()); + let accounts_db = AccountsDB::new(account_paths.to_vec(), operating_mode); let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields; diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index 85ca57721..a4d6a8d7d 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -11,7 +11,7 @@ use { solana_sdk::{ account::Account, clock::Slot, - genesis_config::create_genesis_config, + genesis_config::{create_genesis_config, OperatingMode}, pubkey::Pubkey, signature::{Keypair, Signer}, }, @@ -69,6 +69,7 @@ where C::deserialize_accounts_db_fields(stream)?, account_paths, stream_append_vecs_path, + &OperatingMode::Development, ) } @@ -120,7 +121,7 @@ where fn test_accounts_serialize_style(serde_style: SerdeStyle) { solana_logger::setup(); let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap(); - let accounts = Accounts::new(paths); + let accounts = Accounts::new(paths, &OperatingMode::Development); let mut pubkeys: Vec = vec![]; create_test_accounts(&accounts, &mut pubkeys, 100, 0); diff --git a/sdk/src/hash.rs b/sdk/src/hash.rs index 959741897..0f5497d87 100644 --- a/sdk/src/hash.rs +++ b/sdk/src/hash.rs @@ -10,7 +10,7 @@ pub const HASH_BYTES: usize = 32; Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash, AbiExample, )] #[repr(transparent)] -pub struct Hash([u8; HASH_BYTES]); +pub struct Hash(pub [u8; HASH_BYTES]); #[derive(Clone, Default)] pub struct Hasher {