Switch account hashing to blake3 (#11969)

* Switch account hashing to blake3

Co-authored-by: Carl <carl@solana.com>
This commit is contained in:
carllin 2020-09-02 00:37:36 -07:00 committed by GitHub
parent 2ed6a457b4
commit af08221aec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 330 additions and 110 deletions

47
Cargo.lock generated
View File

@ -268,6 +268,21 @@ dependencies = [
"constant_time_eq",
]
[[package]]
name = "blake3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
"crypto-mac 0.8.0",
"digest 0.9.0",
]
[[package]]
name = "block-buffer"
version = "0.7.3"
@ -625,6 +640,16 @@ dependencies = [
"subtle 1.0.0",
]
[[package]]
name = "crypto-mac"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array 0.14.3",
"subtle 2.2.2",
]
[[package]]
name = "csv"
version = "1.1.3"
@ -664,7 +689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5"
dependencies = [
"byteorder",
"digest",
"digest 0.8.1",
"rand_core 0.5.1",
"subtle 2.2.2",
"zeroize",
@ -707,6 +732,15 @@ dependencies = [
"generic-array 0.12.3",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array 0.14.3",
]
[[package]]
name = "dir-diff"
version = "0.3.2"
@ -1288,8 +1322,8 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
dependencies = [
"crypto-mac",
"digest",
"crypto-mac 0.7.0",
"digest 0.8.1",
]
[[package]]
@ -2253,7 +2287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
dependencies = [
"byteorder",
"crypto-mac",
"crypto-mac 0.7.0",
]
[[package]]
@ -3094,7 +3128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df"
dependencies = [
"block-buffer",
"digest",
"digest 0.8.1",
"fake-simd",
"opaque-debug",
]
@ -3112,7 +3146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
dependencies = [
"block-buffer",
"digest",
"digest 0.8.1",
"fake-simd",
"opaque-debug",
]
@ -4137,6 +4171,7 @@ version = "1.4.0"
dependencies = [
"assert_matches",
"bincode",
"blake3",
"bv",
"byteorder",
"bzip2",

View File

@ -5,7 +5,7 @@ use solana_runtime::{
accounts::{create_test_accounts, update_accounts, Accounts},
accounts_index::Ancestors,
};
use solana_sdk::pubkey::Pubkey;
use solana_sdk::{genesis_config::OperatingMode, pubkey::Pubkey};
use std::fs;
use std::path::PathBuf;
@ -54,7 +54,7 @@ fn main() {
if fs::remove_dir_all(path.clone()).is_err() {
println!("Warning: Couldn't remove {:?}", path);
}
let accounts = Accounts::new(vec![path]);
let accounts = Accounts::new(vec![path], &OperatingMode::Preview);
println!("Creating {} accounts", num_accounts);
let mut create_time = Measure::start("create accounts");
let pubkeys: Vec<_> = (0..num_slots)

View File

@ -24,6 +24,18 @@ dependencies = [
"memchr",
]
[[package]]
name = "arrayref"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
[[package]]
name = "ascii"
version = "0.7.1"
@ -95,6 +107,21 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "blake3"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce4f9586c9a3151c4b49b19e82ba163dd073614dd057e53c969e1a4db5b52720"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
"crypto-mac 0.8.0",
"digest 0.9.0",
]
[[package]]
name = "block-buffer"
version = "0.7.3"
@ -241,6 +268,12 @@ dependencies = [
"byteorder 1.3.4",
]
[[package]]
name = "constant_time_eq"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"
[[package]]
name = "crc32fast"
version = "1.2.0"
@ -317,6 +350,16 @@ dependencies = [
"subtle 1.0.0",
]
[[package]]
name = "crypto-mac"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab"
dependencies = [
"generic-array 0.14.3",
"subtle 2.2.2",
]
[[package]]
name = "curve25519-dalek"
version = "2.1.0"
@ -324,7 +367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5"
dependencies = [
"byteorder 1.3.4",
"digest",
"digest 0.8.1",
"rand_core",
"subtle 2.2.2",
"zeroize",
@ -339,6 +382,15 @@ dependencies = [
"generic-array 0.12.3",
]
[[package]]
name = "digest"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
dependencies = [
"generic-array 0.14.3",
]
[[package]]
name = "dir-diff"
version = "0.3.2"
@ -686,8 +738,8 @@ version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dcb5e64cda4c23119ab41ba960d1e170a774c8e4b9d9e6a9bc18aabf5e59695"
dependencies = [
"crypto-mac",
"digest",
"crypto-mac 0.7.0",
"digest 0.8.1",
]
[[package]]
@ -1151,7 +1203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
dependencies = [
"byteorder 1.3.4",
"crypto-mac",
"crypto-mac 0.7.0",
]
[[package]]
@ -1578,7 +1630,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a256f46ea78a0c0d9ff00077504903ac881a1dafdc20da66545699e7776b3e69"
dependencies = [
"block-buffer",
"digest",
"digest 0.8.1",
"fake-simd",
"opaque-debug",
]
@ -1853,6 +1905,7 @@ name = "solana-runtime"
version = "1.4.0"
dependencies = [
"bincode",
"blake3",
"bv",
"byteorder 1.3.4",
"bzip2",

View File

@ -10,6 +10,7 @@ edition = "2018"
[dependencies]
bincode = "1.3.1"
blake3 = "0.3.6"
bv = { version = "0.11.1", features = ["serde"] }
byteorder = "1.3.4"
bzip2 = "0.3.3"

View File

@ -6,7 +6,11 @@ use solana_runtime::{
accounts::{create_test_accounts, Accounts},
bank::*,
};
use solana_sdk::{account::Account, genesis_config::create_genesis_config, pubkey::Pubkey};
use solana_sdk::{
account::Account,
genesis_config::{create_genesis_config, OperatingMode},
pubkey::Pubkey,
};
use std::{path::PathBuf, sync::Arc};
use test::Bencher;
@ -65,7 +69,10 @@ fn test_accounts_squash(bencher: &mut Bencher) {
#[bench]
fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
let accounts = Accounts::new(vec![PathBuf::from("bench_accounts_hash_internal")]);
let accounts = Accounts::new(
vec![PathBuf::from("bench_accounts_hash_internal")],
&OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 60000, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -76,7 +83,10 @@ fn test_accounts_hash_bank_hash(bencher: &mut Bencher) {
#[bench]
fn test_update_accounts_hash(bencher: &mut Bencher) {
solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("update_accounts_hash")]);
let accounts = Accounts::new(
vec![PathBuf::from("update_accounts_hash")],
&OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 50_000, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -88,7 +98,10 @@ fn test_update_accounts_hash(bencher: &mut Bencher) {
#[bench]
fn test_accounts_delta_hash(bencher: &mut Bencher) {
solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("accounts_delta_hash")]);
let accounts = Accounts::new(
vec![PathBuf::from("accounts_delta_hash")],
&OperatingMode::Development,
);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100_000, 0);
bencher.iter(|| {
@ -99,7 +112,10 @@ fn test_accounts_delta_hash(bencher: &mut Bencher) {
#[bench]
fn bench_delete_dependencies(bencher: &mut Bencher) {
solana_logger::setup();
let accounts = Accounts::new(vec![PathBuf::from("accounts_delete_deps")]);
let accounts = Accounts::new(
vec![PathBuf::from("accounts_delete_deps")],
&OperatingMode::Development,
);
let mut old_pubkey = Pubkey::default();
let zero_account = Account::new(0, 0, &Account::default().owner);
for i in 0..1000 {

View File

@ -18,6 +18,7 @@ use solana_sdk::{
account::Account,
clock::Slot,
fee_calculator::FeeCalculator,
genesis_config::OperatingMode,
hash::Hash,
message::Message,
native_loader, nonce,
@ -66,10 +67,10 @@ pub enum AccountAddressFilter {
}
impl Accounts {
pub fn new(paths: Vec<PathBuf>) -> Self {
pub fn new(paths: Vec<PathBuf>, operating_mode: &OperatingMode) -> Self {
Self {
slot: 0,
accounts_db: Arc::new(AccountsDB::new(paths)),
accounts_db: Arc::new(AccountsDB::new(paths, operating_mode)),
account_locks: Mutex::new(HashSet::new()),
readonly_locks: Arc::new(RwLock::new(Some(HashMap::new()))),
}
@ -819,7 +820,7 @@ mod tests {
) -> Vec<(Result<TransactionLoadResult>, Option<HashAgeKind>)> {
let mut hash_queue = BlockhashQueue::new(100);
hash_queue.register_hash(&tx.message().recent_blockhash, &fee_calculator);
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
for ka in ka.iter() {
accounts.store_slow(0, &ka.0, &ka.1);
}
@ -1385,7 +1386,7 @@ mod tests {
#[test]
fn test_load_by_program_slot() {
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
// Load accounts owned by various programs into AccountsDB
let pubkey0 = Pubkey::new_rand();
@ -1408,7 +1409,7 @@ mod tests {
#[test]
fn test_accounts_account_not_found() {
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
let mut error_counters = ErrorCounters::default();
let ancestors = vec![(0, 0)].into_iter().collect();
@ -1430,7 +1431,7 @@ mod tests {
#[test]
#[should_panic]
fn test_accounts_empty_bank_hash() {
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
accounts.bank_hash_at(1);
}
@ -1446,7 +1447,7 @@ mod tests {
let account2 = Account::new(3, 0, &Pubkey::default());
let account3 = Account::new(4, 0, &Pubkey::default());
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1558,7 +1559,7 @@ mod tests {
let account1 = Account::new(2, 0, &Pubkey::default());
let account2 = Account::new(3, 0, &Pubkey::default());
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
accounts.store_slow(0, &keypair0.pubkey(), &account0);
accounts.store_slow(0, &keypair1.pubkey(), &account1);
accounts.store_slow(0, &keypair2.pubkey(), &account2);
@ -1688,7 +1689,7 @@ mod tests {
let mut loaded = vec![loaded0, loaded1];
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
{
let mut readonly_locks = accounts.readonly_locks.write().unwrap();
let readonly_locks = readonly_locks.as_mut().unwrap();
@ -1739,7 +1740,7 @@ mod tests {
#[test]
fn huge_clean() {
solana_logger::setup();
let accounts = Accounts::new(Vec::new());
let accounts = Accounts::new(Vec::new(), &OperatingMode::Development);
let mut old_pubkey = Pubkey::default();
let zero_account = Account::new(0, 0, &Account::default().owner);
info!("storing..");

View File

@ -22,7 +22,7 @@ use crate::{
accounts_index::{AccountsIndex, Ancestors, SlotList, SlotSlice},
append_vec::{AppendVec, StoredAccount, StoredMeta},
};
use byteorder::{ByteOrder, LittleEndian};
use blake3::traits::digest::Digest;
use lazy_static::lazy_static;
use log::*;
use rand::{thread_rng, Rng};
@ -33,9 +33,11 @@ use solana_rayon_threadlimit::get_thread_count;
use solana_sdk::{
account::Account,
clock::{Epoch, Slot},
genesis_config::OperatingMode,
hash::{Hash, Hasher},
pubkey::Pubkey,
};
use std::convert::TryFrom;
use std::{
collections::{HashMap, HashSet},
io::{Error as IOError, Result as IOResult},
@ -413,6 +415,8 @@ pub struct AccountsDB {
dead_slots: RwLock<HashSet<Slot>>,
stats: AccountsStats,
operating_mode: Option<OperatingMode>,
}
#[derive(Debug, Default)]
@ -474,16 +478,18 @@ impl Default for AccountsDB {
frozen_accounts: HashMap::new(),
dead_slots: RwLock::new(HashSet::new()),
stats: AccountsStats::default(),
operating_mode: None,
}
}
}
impl AccountsDB {
pub fn new(paths: Vec<PathBuf>) -> Self {
pub fn new(paths: Vec<PathBuf>, operating_mode: &OperatingMode) -> Self {
let new = if !paths.is_empty() {
Self {
paths,
temp_paths: None,
operating_mode: Some(*operating_mode),
..Self::default()
}
} else {
@ -493,6 +499,7 @@ impl AccountsDB {
Self {
paths,
temp_paths: Some(temp_dirs),
operating_mode: Some(*operating_mode),
..Self::default()
}
};
@ -508,14 +515,14 @@ impl AccountsDB {
pub fn new_single() -> Self {
AccountsDB {
min_num_stores: 0,
..AccountsDB::new(Vec::new())
..AccountsDB::new(Vec::new(), &OperatingMode::Development)
}
}
#[cfg(test)]
pub fn new_sized(paths: Vec<PathBuf>, file_size: u64) -> Self {
AccountsDB {
file_size,
..AccountsDB::new(paths)
..AccountsDB::new(paths, &OperatingMode::Development)
}
}
@ -1273,28 +1280,61 @@ impl AccountsDB {
assert!(self.storage.read().unwrap().0.get(&remove_slot).is_none());
}
pub fn hash_stored_account(slot: Slot, account: &StoredAccount) -> Hash {
Self::hash_account_data(
slot,
account.account_meta.lamports,
&account.account_meta.owner,
account.account_meta.executable,
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
)
pub fn hash_stored_account(
slot: Slot,
account: &StoredAccount,
operating_mode: &OperatingMode,
) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
Self::blake3_hash_account_data(
slot,
account.account_meta.lamports,
&account.account_meta.owner,
account.account_meta.executable,
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
)
} else {
Self::hash_account_data(
slot,
account.account_meta.lamports,
&account.account_meta.owner,
account.account_meta.executable,
account.account_meta.rent_epoch,
account.data,
&account.meta.pubkey,
)
}
}
pub fn hash_account(slot: Slot, account: &Account, pubkey: &Pubkey) -> Hash {
Self::hash_account_data(
slot,
account.lamports,
&account.owner,
account.executable,
account.rent_epoch,
&account.data,
pubkey,
)
pub fn hash_account(
slot: Slot,
account: &Account,
pubkey: &Pubkey,
operating_mode: &OperatingMode,
) -> Hash {
if slot > Self::get_blake3_slot(operating_mode) {
Self::blake3_hash_account_data(
slot,
account.lamports,
&account.owner,
account.executable,
account.rent_epoch,
&account.data,
pubkey,
)
} else {
Self::hash_account_data(
slot,
account.lamports,
&account.owner,
account.executable,
account.rent_epoch,
&account.data,
pubkey,
)
}
}
fn hash_frozen_account_data(account: &Account) -> Hash {
@ -1326,16 +1366,12 @@ impl AccountsDB {
}
let mut hasher = Hasher::default();
let mut buf = [0u8; 8];
LittleEndian::write_u64(&mut buf[..], lamports);
hasher.hash(&buf);
hasher.hash(&lamports.to_le_bytes());
LittleEndian::write_u64(&mut buf[..], slot);
hasher.hash(&buf);
hasher.hash(&slot.to_le_bytes());
LittleEndian::write_u64(&mut buf[..], rent_epoch);
hasher.hash(&buf);
hasher.hash(&rent_epoch.to_le_bytes());
hasher.hash(&data);
@ -1351,6 +1387,50 @@ impl AccountsDB {
hasher.result()
}
pub fn blake3_hash_account_data(
slot: Slot,
lamports: u64,
owner: &Pubkey,
executable: bool,
rent_epoch: Epoch,
data: &[u8],
pubkey: &Pubkey,
) -> Hash {
if lamports == 0 {
return Hash::default();
}
let mut hasher = blake3::Hasher::new();
hasher.update(&lamports.to_le_bytes());
hasher.update(&slot.to_le_bytes());
hasher.update(&rent_epoch.to_le_bytes());
hasher.update(&data);
if executable {
hasher.update(&[1u8; 1]);
} else {
hasher.update(&[0u8; 1]);
}
hasher.update(&owner.as_ref());
hasher.update(&pubkey.as_ref());
Hash(<[u8; solana_sdk::hash::HASH_BYTES]>::try_from(hasher.finalize().as_slice()).unwrap())
}
fn get_blake3_slot(operating_mode: &OperatingMode) -> Slot {
match operating_mode {
OperatingMode::Development => 0,
// Epoch 75
OperatingMode::Stable => 32_400_000,
OperatingMode::Preview => std::u64::MAX,
}
}
fn bulk_assign_write_version(&self, count: usize) -> u64 {
self.write_version
.fetch_add(count as u64, Ordering::Relaxed)
@ -1566,7 +1646,13 @@ impl AccountsDB {
let account = store.accounts.get_account(account_info.offset)?.0;
if check_hash {
let hash = Self::hash_stored_account(*slot, &account);
let hash = Self::hash_stored_account(
*slot,
&account,
&self
.operating_mode
.expect("Operating mode must be set at initialization"),
);
if hash != *account.hash {
mismatch_found.fetch_add(1, Ordering::Relaxed);
return None;
@ -1801,13 +1887,18 @@ impl AccountsDB {
}
}
fn hash_accounts(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) -> Vec<Hash> {
fn hash_accounts(
&self,
slot: Slot,
accounts: &[(&Pubkey, &Account)],
operating_mode: &OperatingMode,
) -> Vec<Hash> {
let mut stats = BankHashStats::default();
let hashes: Vec<_> = accounts
.iter()
.map(|(pubkey, account)| {
stats.update(account);
Self::hash_account(slot, account, pubkey)
Self::hash_account(slot, account, pubkey, operating_mode)
})
.collect();
@ -1872,7 +1963,13 @@ impl AccountsDB {
/// Store the account update.
pub fn store(&self, slot: Slot, accounts: &[(&Pubkey, &Account)]) {
self.assert_frozen_accounts(accounts);
let hashes = self.hash_accounts(slot, accounts);
let hashes = self.hash_accounts(
slot,
accounts,
&self
.operating_mode
.expect("Operating mode must be set at initialization"),
);
self.store_with_hashes(slot, accounts, &hashes);
}
@ -2083,7 +2180,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -2096,7 +2193,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -2123,7 +2220,7 @@ pub mod tests {
#[test]
fn test_accountsdb_latest_ancestor_with_root() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -2143,7 +2240,7 @@ pub mod tests {
#[test]
fn test_accountsdb_root_one_slot() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
@ -2184,7 +2281,7 @@ pub mod tests {
#[test]
fn test_accountsdb_add_root_many() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
@ -2273,7 +2370,7 @@ pub mod tests {
let key = Pubkey::default();
// 1 token in the "root", i.e. db zero
let db0 = AccountsDB::new(Vec::new());
let db0 = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let account0 = Account::new(1, 0, &key);
db0.store(0, &[(&key, &account0)]);
@ -2292,7 +2389,7 @@ pub mod tests {
#[test]
fn test_remove_unrooted_slot() {
let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account0 = Account::new(1, 0, &key);
let ancestors: HashMap<_, _> = vec![(unrooted_slot, 1)].into_iter().collect();
@ -2338,7 +2435,7 @@ pub mod tests {
#[test]
fn test_remove_unrooted_slot_snapshot() {
let unrooted_slot = 9;
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key);
db.store(unrooted_slot, &[(&key, &account0)]);
@ -2465,7 +2562,7 @@ pub mod tests {
#[test]
fn test_account_one() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let db = AccountsDB::new(paths);
let db = AccountsDB::new(paths, &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -2478,7 +2575,7 @@ pub mod tests {
#[test]
fn test_account_many() {
let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap();
let db = AccountsDB::new(paths);
let db = AccountsDB::new(paths, &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&db, &mut pubkeys, 0, 100, 0, 0);
check_accounts(&db, &pubkeys, 0, 100, 1);
@ -2596,7 +2693,7 @@ pub mod tests {
#[test]
fn test_purge_slot_not_root() {
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -2607,7 +2704,7 @@ pub mod tests {
#[test]
fn test_purge_slot_after_root() {
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_account(&accounts, &mut pubkeys, 0, 1, 0, 0);
let ancestors = vec![(0, 0)].into_iter().collect();
@ -2622,7 +2719,7 @@ pub mod tests {
//This test is pedantic
//A slot is purged when a non root bank is cleaned up. If a slot is behind root but it is
//not root, it means we are retaining dead banks.
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@ -2700,7 +2797,7 @@ pub mod tests {
fn test_clean_old_with_normal_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@ -2726,7 +2823,7 @@ pub mod tests {
fn test_clean_old_with_zero_lamport_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner);
@ -2756,7 +2853,7 @@ pub mod tests {
fn test_clean_old_with_both_normal_and_zero_lamport_accounts() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey1 = Pubkey::new_rand();
let pubkey2 = Pubkey::new_rand();
let normal_account = Account::new(1, 0, &Account::default().owner);
@ -2789,7 +2886,7 @@ pub mod tests {
fn test_uncleaned_roots_with_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
//store an account
@ -2809,7 +2906,7 @@ pub mod tests {
fn test_uncleaned_roots_with_no_account() {
solana_logger::setup();
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
assert_eq!(accounts.uncleaned_root_count(), 0);
@ -3243,7 +3340,7 @@ pub mod tests {
#[test]
fn test_accountsdb_scan_accounts() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let key0 = Pubkey::new_rand();
let account0 = Account::new(1, 0, &key);
@ -3307,7 +3404,7 @@ pub mod tests {
#[test]
fn test_store_large_account() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let data_len = DEFAULT_FILE_SIZE as usize + 7;
@ -3392,7 +3489,7 @@ pub mod tests {
fn test_frozen_account_lamport_increase() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new());
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@ -3427,7 +3524,7 @@ pub mod tests {
fn test_frozen_account_lamport_decrease() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new());
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@ -3447,7 +3544,7 @@ pub mod tests {
fn test_frozen_account_nonexistent() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new());
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let ancestors = vec![(0, 0)].into_iter().collect();
db.freeze_accounts(&ancestors, &[frozen_pubkey]);
@ -3460,7 +3557,7 @@ pub mod tests {
fn test_frozen_account_data_modified() {
let frozen_pubkey =
Pubkey::from_str("My11111111111111111111111111111111111111111").unwrap();
let mut db = AccountsDB::new(Vec::new());
let mut db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let mut account = Account::new(1, 42, &frozen_pubkey);
db.store(0, &[(&frozen_pubkey, &account)]);
@ -3510,15 +3607,20 @@ pub mod tests {
};
let account = stored_account.clone_account();
let expected_account_hash =
Hash::from_str("5iRNZVcAnq9JLYjSF2ibFhGEeq48r9Eq9HXxwm3BxywN").unwrap();
Hash::from_str("4StuvYHFd7xuShVXB94uHHvpqGMCaacdZnYB74QQkPA1").unwrap();
assert_eq!(
AccountsDB::hash_stored_account(slot, &stored_account),
AccountsDB::hash_stored_account(slot, &stored_account, &OperatingMode::Development),
expected_account_hash,
"StoredAccount's data layout might be changed; update hashing if needed."
);
assert_eq!(
AccountsDB::hash_account(slot, &account, &stored_account.meta.pubkey),
AccountsDB::hash_account(
slot,
&account,
&stored_account.meta.pubkey,
&OperatingMode::Development
),
expected_account_hash,
"Account-based hashing must be consistent with StoredAccount-based one."
);
@ -3527,7 +3629,7 @@ pub mod tests {
#[test]
fn test_bank_hash_stats() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let some_data_len = 5;
@ -3555,7 +3657,7 @@ pub mod tests {
fn test_verify_bank_hash() {
use BankHashVerificationError::*;
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let some_data_len = 0;
@ -3593,7 +3695,7 @@ pub mod tests {
#[test]
fn test_verify_bank_hash_no_account() {
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let some_slot: Slot = 0;
let ancestors = vec![(some_slot, 0)].into_iter().collect();
@ -3611,7 +3713,7 @@ pub mod tests {
fn test_verify_bank_hash_bad_account_hash() {
use BankHashVerificationError::*;
solana_logger::setup();
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let some_data_len = 0;
@ -3621,7 +3723,7 @@ pub mod tests {
let accounts = &[(&key, &account)];
// update AccountsDB's bank hash but discard real account hashes
db.hash_accounts(some_slot, accounts);
db.hash_accounts(some_slot, accounts, &OperatingMode::Development);
// provide bogus account hashes
let some_hash = Hash::new(&[0xca; HASH_BYTES]);
db.store_with_hashes(some_slot, accounts, &[some_hash]);
@ -3635,7 +3737,7 @@ pub mod tests {
#[test]
fn test_bad_bank_hash() {
use solana_sdk::signature::{Keypair, Signer};
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let some_slot: Slot = 0;
let ancestors: Ancestors = [(some_slot, 0)].iter().copied().collect();
@ -3660,7 +3762,12 @@ pub mod tests {
for (key, account) in &accounts_keys {
assert_eq!(
db.load_account_hash(&ancestors, key),
AccountsDB::hash_account(some_slot, &account, &key)
AccountsDB::hash_account(
some_slot,
&account,
&key,
&OperatingMode::Development
)
);
}
}
@ -3668,13 +3775,13 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_empty() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
assert!(db.get_snapshot_storages(0).is_empty());
}
#[test]
fn test_get_snapshot_storages_only_older_than_or_equal_to_snapshot_slot() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@ -3692,7 +3799,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_only_non_empty() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@ -3716,7 +3823,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_only_roots() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@ -3732,7 +3839,7 @@ pub mod tests {
#[test]
fn test_get_snapshot_storages_exclude_empty() {
let db = AccountsDB::new(Vec::new());
let db = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let key = Pubkey::default();
let account = Account::new(1, 0, &key);
@ -3751,7 +3858,7 @@ pub mod tests {
#[test]
#[should_panic(expected = "double remove of account in slot: 0/store: 0!!")]
fn test_storage_remove_account_double_remove() {
let accounts = AccountsDB::new(Vec::new());
let accounts = AccountsDB::new(Vec::new(), &OperatingMode::Development);
let pubkey = Pubkey::new_rand();
let account = Account::new(1, 0, &Account::default().owner);
accounts.store(0, &[(&pubkey, &account)]);

View File

@ -477,7 +477,7 @@ impl Bank {
bank.operating_mode = Some(genesis_config.operating_mode);
bank.ancestors.insert(bank.slot(), 0);
bank.rc.accounts = Arc::new(Accounts::new(paths));
bank.rc.accounts = Arc::new(Accounts::new(paths, &genesis_config.operating_mode));
bank.process_genesis_config(genesis_config);
bank.finish_init();

View File

@ -22,6 +22,7 @@ use {
epoch_schedule::EpochSchedule,
fee_calculator::{FeeCalculator, FeeRateGovernor},
genesis_config::GenesisConfig,
genesis_config::OperatingMode,
hard_forks::HardForks,
hash::Hash,
inflation::Inflation,
@ -228,8 +229,12 @@ where
E: Into<AccountStorageEntry>,
P: AsRef<Path>,
{
let mut accounts_db =
reconstruct_accountsdb_from_fields(accounts_db_fields, account_paths, append_vecs_path)?;
let mut accounts_db = reconstruct_accountsdb_from_fields(
accounts_db_fields,
account_paths,
append_vecs_path,
&genesis_config.operating_mode,
)?;
accounts_db.freeze_accounts(&bank_fields.ancestors, frozen_account_pubkeys);
let bank_rc = BankRc::new(Accounts::new_empty(accounts_db), bank_fields.slot);
@ -242,12 +247,13 @@ fn reconstruct_accountsdb_from_fields<E, P>(
accounts_db_fields: AccountsDbFields<E>,
account_paths: &[PathBuf],
stream_append_vecs_path: P,
operating_mode: &OperatingMode,
) -> Result<AccountsDB, Error>
where
E: Into<AccountStorageEntry>,
P: AsRef<Path>,
{
let accounts_db = AccountsDB::new(account_paths.to_vec());
let accounts_db = AccountsDB::new(account_paths.to_vec(), operating_mode);
let AccountsDbFields(storage, version, slot, bank_hash_info) = accounts_db_fields;

View File

@ -11,7 +11,7 @@ use {
solana_sdk::{
account::Account,
clock::Slot,
genesis_config::create_genesis_config,
genesis_config::{create_genesis_config, OperatingMode},
pubkey::Pubkey,
signature::{Keypair, Signer},
},
@ -69,6 +69,7 @@ where
C::deserialize_accounts_db_fields(stream)?,
account_paths,
stream_append_vecs_path,
&OperatingMode::Development,
)
}
@ -120,7 +121,7 @@ where
fn test_accounts_serialize_style(serde_style: SerdeStyle) {
solana_logger::setup();
let (_accounts_dir, paths) = get_temp_accounts_paths(4).unwrap();
let accounts = Accounts::new(paths);
let accounts = Accounts::new(paths, &OperatingMode::Development);
let mut pubkeys: Vec<Pubkey> = vec![];
create_test_accounts(&accounts, &mut pubkeys, 100, 0);

View File

@ -10,7 +10,7 @@ pub const HASH_BYTES: usize = 32;
Serialize, Deserialize, Clone, Copy, Default, Eq, PartialEq, Ord, PartialOrd, Hash, AbiExample,
)]
#[repr(transparent)]
pub struct Hash([u8; HASH_BYTES]);
pub struct Hash(pub [u8; HASH_BYTES]);
#[derive(Clone, Default)]
pub struct Hasher {