blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add ability to output components that go into Bank hash (#32632) 

When a consensus divergance occurs, the current workflow involves a
handful of manual steps to hone in on the offending slot and
transaction. This process isn't overly difficult to execute; however, it
is tedious and currently involves creating and parsing logs.

This change introduces functionality to output a debug file that
contains the components go into the bank hash. The file can be generated
in two ways:
- Via solana-validator when the node realizes it has diverged
- Via solana-ledger-tool verify by passing a flag

When a divergance occurs now, the steps to debug would be:
- Grab the file from the node that diverged
- Generate a file for the same slot with ledger-tool with a known good
  version
- Diff the files, they are pretty-printed json
This commit is contained in:
steviez 2023-08-15 01:12:05 -04:00 committed by GitHub
parent de4eee15c8
commit 6bbf514e78
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 390 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -6880,6 +6880,7 @@ version = "1.17.0"
dependencies = [
"arrayref",
"assert_matches",
"base64 0.21.2",
"bincode",
"blake3",
"bv",
@ -6918,6 +6919,7 @@ dependencies = [
"rustc_version 0.4.0",
"serde",
"serde_derive",
"serde_json",
"siphasher",
"solana-accounts-db",
"solana-address-lookup-table-program",
@ -6939,6 +6941,7 @@ dependencies = [
"solana-sdk",
"solana-stake-program",
"solana-system-program",
"solana-version",
"solana-vote-program",
"solana-zk-token-proof-program",
"solana-zk-token-sdk",

99
accounts-db/src/accounts_db.rs
View File

@ -469,7 +469,7 @@ pub(crate) struct ShrinkCollect<'a, T: ShrinkCollectRefs<'a>> {
pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
accounts_hash_cache_path: None,
base_working_path: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
write_cache_limit_bytes: None,
ancient_append_vec_offset: None,
@ -480,7 +480,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
};
pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
accounts_hash_cache_path: None,
base_working_path: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
write_cache_limit_bytes: None,
ancient_append_vec_offset: None,
@ -539,7 +539,8 @@ const ANCIENT_APPEND_VEC_DEFAULT_OFFSET: Option<i64> = Some(-10_000);
#[derive(Debug, Default, Clone)]
pub struct AccountsDbConfig {
pub index: Option<AccountsIndexConfig>,
pub accounts_hash_cache_path: Option<PathBuf>,
/// Base directory for various necessary files
pub base_working_path: Option<PathBuf>,
pub filler_accounts_config: FillerAccountsConfig,
pub write_cache_limit_bytes: Option<u64>,
/// if None, ancient append vecs are set to ANCIENT_APPEND_VEC_DEFAULT_OFFSET
@ -1467,6 +1468,9 @@ pub struct AccountsDb {
/// Set of storage paths to pick from
pub paths: Vec<PathBuf>,
/// Base directory for various necessary files
base_working_path: PathBuf,
/// Directories for account hash calculations, within base_working_path
full_accounts_hash_cache_path: PathBuf,
incremental_accounts_hash_cache_path: PathBuf,
transient_accounts_hash_cache_path: PathBuf,
@ -2413,6 +2417,13 @@ impl<'a> AppendVecScan for ScanState<'a> {
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PubkeyHashAccount {
pub pubkey: Pubkey,
pub hash: Hash,
pub account: AccountSharedData,
}
impl AccountsDb {
pub const ACCOUNTS_HASH_CACHE_DIR: &str = "accounts_hash_cache";
@ -2422,20 +2433,34 @@ impl AccountsDb {
fn default_with_accounts_index(
accounts_index: AccountInfoAccountsIndex,
accounts_hash_cache_path: Option<PathBuf>,
base_working_path: Option<PathBuf>,
) -> Self {
let num_threads = get_thread_count();
const MAX_READ_ONLY_CACHE_DATA_SIZE: usize = 400_000_000; // 400M bytes
let mut temp_accounts_hash_cache_path = None;
let accounts_hash_cache_path = accounts_hash_cache_path.unwrap_or_else(|| {
temp_accounts_hash_cache_path = Some(TempDir::new().unwrap());
temp_accounts_hash_cache_path
.as_ref()
.unwrap()
.path()
.to_path_buf()
});
let (base_working_path, accounts_hash_cache_path, temp_accounts_hash_cache_path) =
match base_working_path {
Some(base_working_path) => {
let accounts_hash_cache_path =
base_working_path.join(Self::ACCOUNTS_HASH_CACHE_DIR);
(base_working_path, accounts_hash_cache_path, None)
}
None => {
let temp_accounts_hash_cache_path = Some(TempDir::new().unwrap());
let base_working_path = temp_accounts_hash_cache_path
.as_ref()
.unwrap()
.path()
.to_path_buf();
let accounts_hash_cache_path =
base_working_path.join(Self::ACCOUNTS_HASH_CACHE_DIR);
(
base_working_path,
accounts_hash_cache_path,
temp_accounts_hash_cache_path,
)
}
};
let mut bank_hash_stats = HashMap::new();
bank_hash_stats.insert(0, BankHashStats::default());
@ -2464,6 +2489,7 @@ impl AccountsDb {
write_cache_limit_bytes: None,
write_version: AtomicU64::new(0),
paths: vec![],
base_working_path,
full_accounts_hash_cache_path: accounts_hash_cache_path.join("full"),
incremental_accounts_hash_cache_path: accounts_hash_cache_path.join("incremental"),
transient_accounts_hash_cache_path: accounts_hash_cache_path.join("transient"),
@ -2545,9 +2571,9 @@ impl AccountsDb {
accounts_db_config.as_mut().and_then(|x| x.index.take()),
exit,
);
let accounts_hash_cache_path = accounts_db_config
let base_working_path = accounts_db_config
.as_ref()
.and_then(|x| x.accounts_hash_cache_path.clone());
.and_then(|x| x.base_working_path.clone());
let filler_accounts_config = accounts_db_config
.as_ref()
@ -2603,7 +2629,7 @@ impl AccountsDb {
.and_then(|x| x.write_cache_limit_bytes),
partitioned_epoch_rewards_config,
exhaustively_verify_refcounts,
..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
..Self::default_with_accounts_index(accounts_index, base_working_path)
};
if paths_is_empty {
// Create a temporary set of accounts directories, used primarily
@ -2650,6 +2676,11 @@ impl AccountsDb {
self.file_size
}
/// Get the base working directory
pub fn get_base_working_path(&self) -> PathBuf {
self.base_working_path.clone()
}
pub fn new_single_for_tests() -> Self {
AccountsDb::new_for_tests(Vec::new(), &ClusterType::Development)
}
@ -7856,6 +7887,42 @@ impl AccountsDb {
(hashes, scan.as_us(), accumulate)
}
/// Return all of the accounts for a given slot
pub fn get_pubkey_hash_account_for_slot(&self, slot: Slot) -> Vec<PubkeyHashAccount> {
type ScanResult =
ScanStorageResult<PubkeyHashAccount, DashMap<Pubkey, (Hash, AccountSharedData)>>;
let scan_result: ScanResult = self.scan_account_storage(
slot,
|loaded_account: LoadedAccount| {
// Cache only has one version per key, don't need to worry about versioning
Some(PubkeyHashAccount {
pubkey: *loaded_account.pubkey(),
hash: loaded_account.loaded_hash(),
account: loaded_account.take_account(),
})
},
|accum: &DashMap<Pubkey, (Hash, AccountSharedData)>, loaded_account: LoadedAccount| {
// Storage may have duplicates so only keep the latest version for each key
accum.insert(
*loaded_account.pubkey(),
(loaded_account.loaded_hash(), loaded_account.take_account()),
);
},
);
match scan_result {
ScanStorageResult::Cached(cached_result) => cached_result,
ScanStorageResult::Stored(stored_result) => stored_result
.into_iter()
.map(|(pubkey, (hash, account))| PubkeyHashAccount {
pubkey,
hash,
account,
})
.collect(),
}
}
/// Calculate accounts delta hash for `slot`
///
/// As part of calculating the accounts delta hash, get a list of accounts modified this slot

3
core/src/replay_stage.rs
View File

@ -59,7 +59,7 @@ use {
solana_rpc_client_api::response::SlotUpdate,
solana_runtime::{
accounts_background_service::AbsRequestSender,
bank::{Bank, NewBankOptions},
bank::{bank_hash_details, Bank, NewBankOptions},
bank_forks::{BankForks, MAX_ROOT_DISTANCE_FOR_VOTE_ONLY},
commitment::BlockCommitmentCache,
prioritization_fee_cache::PrioritizationFeeCache,
@ -1500,6 +1500,7 @@ impl ReplayStage {
let bank = w_bank_forks
.remove(*slot)
.expect("BankForks should not have been purged yet");
let _ = bank_hash_details::write_bank_hash_details_file(&bank);
((*slot, bank.bank_id()), bank)
})
.unzip()

4
ledger-tool/src/args.rs
View File

@ -1,7 +1,7 @@
use {
clap::{value_t, values_t_or_exit, ArgMatches},
solana_accounts_db::{
accounts_db::{AccountsDb, AccountsDbConfig, FillerAccountsConfig},
accounts_db::{AccountsDbConfig, FillerAccountsConfig},
accounts_index::{AccountsIndexConfig, IndexLimitMb},
partitioned_rewards::TestPartitionedEpochRewards,
},
@ -57,7 +57,7 @@ pub fn get_accounts_db_config(
AccountsDbConfig {
index: Some(accounts_index_config),
accounts_hash_cache_path: Some(ledger_path.join(AccountsDb::ACCOUNTS_HASH_CACHE_DIR)),
base_working_path: Some(ledger_path.to_path_buf()),
filler_accounts_config,
ancient_append_vec_offset: value_t!(arg_matches, "accounts_db_ancient_append_vecs", i64)
.ok(),

15
ledger-tool/src/main.rs
View File

@ -50,7 +50,7 @@ use {
},
solana_measure::{measure, measure::Measure},
solana_runtime::{
bank::{Bank, RewardCalculationEvent, TotalAccountsStats},
bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
bank_forks::BankForks,
runtime_config::RuntimeConfig,
snapshot_archive_info::SnapshotArchiveInfoGetter,
@ -1663,6 +1663,14 @@ fn main() {
.takes_value(false)
.help("After verifying the ledger, print some information about the account stores"),
)
.arg(
Arg::with_name("write_bank_file")
.long("write-bank-file")
.takes_value(false)
.help("After verifying the ledger, write a file that contains the information \
that went into computing the completed bank's bank hash. The file will be \
written within <LEDGER_DIR>/bank_hash_details/"),
)
).subcommand(
SubCommand::with_name("graph")
.about("Create a Graphviz rendering of the ledger")
@ -2645,6 +2653,7 @@ fn main() {
..ProcessOptions::default()
};
let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
let write_bank_file = arg_matches.is_present("write_bank_file");
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
info!("genesis hash: {}", genesis_config.hash());
@ -2671,6 +2680,10 @@ fn main() {
let working_bank = bank_forks.read().unwrap().working_bank();
working_bank.print_accounts_stats();
}
if write_bank_file {
let working_bank = bank_forks.read().unwrap().working_bank();
let _ = bank_hash_details::write_bank_hash_details_file(&working_bank);
}
exit_signal.store(true, Ordering::Relaxed);
system_monitor_service.join().unwrap();
}

3
programs/sbf/Cargo.lock generated
View File

@ -5597,6 +5597,7 @@ name = "solana-runtime"
version = "1.17.0"
dependencies = [
"arrayref",
"base64 0.21.2",
"bincode",
"blake3",
"bv",
@ -5631,6 +5632,7 @@ dependencies = [
"rustc_version",
"serde",
"serde_derive",
"serde_json",
"siphasher",
"solana-accounts-db",
"solana-address-lookup-table-program",
@ -5650,6 +5652,7 @@ dependencies = [
"solana-sdk",
"solana-stake-program",
"solana-system-program",
"solana-version",
"solana-vote-program",
"solana-zk-token-proof-program",
"solana-zk-token-sdk",

3
runtime/Cargo.toml
View File

@ -11,6 +11,7 @@ edition = { workspace = true }
[dependencies]
arrayref = { workspace = true }
base64 = { workspace = true }
bincode = { workspace = true }
blake3 = { workspace = true }
bv = { workspace = true, features = ["serde"] }
@ -44,6 +45,7 @@ rayon = { workspace = true }
regex = { workspace = true }
serde = { workspace = true, features = ["rc"] }
serde_derive = { workspace = true }
serde_json = { workspace = true }
siphasher = { workspace = true }
solana-accounts-db = { workspace = true }
solana-address-lookup-table-program = { workspace = true }
@ -63,6 +65,7 @@ solana-rayon-threadlimit = { workspace = true }
solana-sdk = { workspace = true }
solana-stake-program = { workspace = true }
solana-system-program = { workspace = true }
solana-version = { workspace = true }
solana-vote-program = { workspace = true }
solana-zk-token-proof-program = { workspace = true }
solana-zk-token-sdk = { workspace = true }

1
runtime/src/bank.rs
View File

@ -212,6 +212,7 @@ struct VerifyAccountsHashConfig {
}
mod address_lookup_table;
pub mod bank_hash_details;
mod builtin_programs;
pub mod epoch_accounts_hash_utils;
mod metrics;

277
runtime/src/bank/bank_hash_details.rs Normal file
View File

@ -0,0 +1,277 @@
//! Container to capture information relevant to computing a bank hash
use {
super::Bank,
base64::{prelude::BASE64_STANDARD, Engine},
log::*,
serde::{
de::{self, Deserialize, Deserializer},
ser::{Serialize, SerializeSeq, Serializer},
},
solana_accounts_db::{accounts_db::PubkeyHashAccount, accounts_hash::AccountsDeltaHash},
solana_sdk::{
account::{Account, AccountSharedData, ReadableAccount},
clock::{Epoch, Slot},
hash::Hash,
pubkey::Pubkey,
},
std::str::FromStr,
};
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub(crate) struct BankHashDetails {
/// client version
pub version: String,
pub account_data_encoding: String,
pub slot: Slot,
pub bank_hash: String,
pub parent_bank_hash: String,
pub accounts_delta_hash: String,
pub signature_count: u64,
pub last_blockhash: String,
pub accounts: BankHashAccounts,
}
impl BankHashDetails {
pub fn new(
slot: Slot,
bank_hash: Hash,
parent_bank_hash: Hash,
accounts_delta_hash: Hash,
signature_count: u64,
last_blockhash: Hash,
accounts: BankHashAccounts,
) -> Self {
Self {
version: solana_version::version!().to_string(),
account_data_encoding: "base64".to_string(),
slot,
bank_hash: bank_hash.to_string(),
parent_bank_hash: parent_bank_hash.to_string(),
accounts_delta_hash: accounts_delta_hash.to_string(),
signature_count,
last_blockhash: last_blockhash.to_string(),
accounts,
}
}
}
impl TryFrom<&Bank> for BankHashDetails {
type Error = String;
fn try_from(bank: &Bank) -> Result<Self, Self::Error> {
let slot = bank.slot();
if !bank.is_frozen() {
return Err(format!(
"Bank {slot} must be frozen in order to get bank hash details"
));
}
// This bank is frozen; as a result, we know that the state has been
// hashed which means the delta hash is Some(). So, .unwrap() is safe
let AccountsDeltaHash(accounts_delta_hash) = bank
.rc
.accounts
.accounts_db
.get_accounts_delta_hash(slot)
.unwrap();
let mut accounts = bank
.rc
.accounts
.accounts_db
.get_pubkey_hash_account_for_slot(slot);
// get_pubkey_hash_account_for_slot() returns an arbitrary ordering;
// sort by pubkey to match the ordering used for accounts delta hash
accounts.sort_by_key(|account| account.pubkey);
Ok(Self::new(
slot,
bank.hash(),
bank.parent_hash(),
accounts_delta_hash,
bank.signature_count(),
bank.last_blockhash(),
BankHashAccounts { accounts },
))
}
}
// Wrap the Vec<...> so we can implement custom Serialize/Deserialize traits on the wrapper type
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct BankHashAccounts {
pub accounts: Vec<PubkeyHashAccount>,
}
#[derive(Deserialize, Serialize)]
/// Used as an intermediate for serializing and deserializing account fields
/// into a human readable format.
struct SerdeAccount {
pubkey: String,
hash: String,
owner: String,
lamports: u64,
rent_epoch: Epoch,
executable: bool,
data: String,
}
impl From<&PubkeyHashAccount> for SerdeAccount {
fn from(pubkey_hash_account: &PubkeyHashAccount) -> Self {
let PubkeyHashAccount {
pubkey,
hash,
account,
} = pubkey_hash_account;
Self {
pubkey: pubkey.to_string(),
hash: hash.to_string(),
owner: account.owner().to_string(),
lamports: account.lamports(),
rent_epoch: account.rent_epoch(),
executable: account.executable(),
data: BASE64_STANDARD.encode(account.data()),
}
}
}
impl TryFrom<SerdeAccount> for PubkeyHashAccount {
type Error = String;
fn try_from(temp_account: SerdeAccount) -> Result<Self, Self::Error> {
let pubkey = Pubkey::from_str(&temp_account.pubkey).map_err(|err| err.to_string())?;
let hash = Hash::from_str(&temp_account.hash).map_err(|err| err.to_string())?;
let account = AccountSharedData::from(Account {
lamports: temp_account.lamports,
data: BASE64_STANDARD
.decode(temp_account.data)
.map_err(|err| err.to_string())?,
owner: Pubkey::from_str(&temp_account.owner).map_err(|err| err.to_string())?,
executable: temp_account.executable,
rent_epoch: temp_account.rent_epoch,
});
Ok(Self {
pubkey,
hash,
account,
})
}
}
impl Serialize for BankHashAccounts {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.accounts.len()))?;
for account in self.accounts.iter() {
let temp_account = SerdeAccount::from(account);
seq.serialize_element(&temp_account)?;
}
seq.end()
}
}
impl<'de> Deserialize<'de> for BankHashAccounts {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: Deserializer<'de>,
{
let temp_accounts: Vec<SerdeAccount> = Deserialize::deserialize(deserializer)?;
let pubkey_hash_accounts: Result<Vec<_>, _> = temp_accounts
.into_iter()
.map(PubkeyHashAccount::try_from)
.collect();
let pubkey_hash_accounts = pubkey_hash_accounts.map_err(de::Error::custom)?;
Ok(BankHashAccounts {
accounts: pubkey_hash_accounts,
})
}
}
/// Output the components that comprise bank hash
pub fn write_bank_hash_details_file(bank: &Bank) -> std::result::Result<(), String> {
let details = BankHashDetails::try_from(bank)?;
let slot = details.slot;
let hash = &details.bank_hash;
let file_name = format!("{slot}-{hash}.json");
let parent_dir = bank
.rc
.accounts
.accounts_db
.get_base_working_path()
.join("bank_hash_details");
let path = parent_dir.join(file_name);
// A file with the same name implies the same hash for this slot. Skip
// rewriting a duplicate file in this scenario
if !path.exists() {
info!("writing details of bank {} to {}", slot, path.display());
// std::fs::write may fail (depending on platform) if the full directory
// path does not exist. So, call std::fs_create_dir_all first.
// https://doc.rust-lang.org/std/fs/fn.write.html
_ = std::fs::create_dir_all(parent_dir);
let file = std::fs::File::create(&path).map_err(|err| {
format!(
"Unable to create bank hash file at {}: {err}",
path.display()
)
})?;
serde_json::to_writer_pretty(file, &details)
.map_err(|err| format!("Unable to write bank hash file contents: {err}"))?;
}
Ok(())
}
#[cfg(test)]
pub mod tests {
use super::*;
#[test]
fn test_serde_bank_hash_details() {
use solana_sdk::hash::hash;
let slot = 123_456_789;
let signature_count = 314;
let account = AccountSharedData::from(Account {
lamports: 123_456_789,
data: vec![0, 9, 1, 8, 2, 7, 3, 6, 4, 5],
owner: Pubkey::new_unique(),
executable: true,
rent_epoch: 123,
});
let account_pubkey = Pubkey::new_unique();
let account_hash = hash("account".as_bytes());
let accounts = BankHashAccounts {
accounts: vec![PubkeyHashAccount {
pubkey: account_pubkey,
hash: account_hash,
account,
}],
};
let bank_hash = hash("bank".as_bytes());
let parent_bank_hash = hash("parent_bank".as_bytes());
let accounts_delta_hash = hash("accounts_delta".as_bytes());
let last_blockhash = hash("last_blockhash".as_bytes());
let bank_hash_details = BankHashDetails::new(
slot,
bank_hash,
parent_bank_hash,
accounts_delta_hash,
signature_count,
last_blockhash,
accounts,
);
let serialized_bytes = serde_json::to_vec(&bank_hash_details).unwrap();
let deserialized_bank_hash_details: BankHashDetails =
serde_json::from_slice(&serialized_bytes).unwrap();
assert_eq!(bank_hash_details, deserialized_bank_hash_details);
}
}

5
validator/src/main.rs
View File

@ -9,8 +9,7 @@ use {
rand::{seq::SliceRandom, thread_rng},
solana_accounts_db::{
accounts_db::{
AccountShrinkThreshold, AccountsDb, AccountsDbConfig, CreateAncientStorage,
FillerAccountsConfig,
AccountShrinkThreshold, AccountsDbConfig, CreateAncientStorage, FillerAccountsConfig,
},
accounts_index::{
AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
@ -1175,7 +1174,7 @@ pub fn main() {
let accounts_db_config = AccountsDbConfig {
index: Some(accounts_index_config),
accounts_hash_cache_path: Some(ledger_path.join(AccountsDb::ACCOUNTS_HASH_CACHE_DIR)),
base_working_path: Some(ledger_path.clone()),
filler_accounts_config,
write_cache_limit_bytes: value_t!(matches, "accounts_db_cache_limit_mb", u64)
.ok()