blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract process_ledger from Bank 

Fullnode was the only real consumer of process_ledger and it was
only there to process a Blocktree. Blocktree is a tree, and a
ledger is a sequence, so something's clearly not right here.
Drop all other dependencies on process_ledger (only one test) so
that it can be fixed up in isolation.
This commit is contained in:
Greg Fitzgerald 2019-02-16 03:26:21 -07:00
parent 38aed0c886
commit 4467d5eb4c
4 changed files with 155 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
src/bank.rs
View File

@ -6,7 +6,6 @@
use crate::accounts::{Accounts, ErrorCounters, InstructionAccounts, InstructionLoaders};
use crate::counter::Counter;
use crate::entry::Entry;
use crate::entry::EntrySlice;
use crate::genesis_block::GenesisBlock;
use crate::last_id_queue::{LastIdQueue, MAX_ENTRY_IDS};
use crate::leader_scheduler::{LeaderScheduler, LeaderSchedulerConfig};
@ -15,7 +14,6 @@ use crate::result::Error;
use crate::rpc_pubsub::RpcSubscriptions;
use crate::status_cache::StatusCache;
use bincode::deserialize;
use itertools::Itertools;
use log::Level;
use rayon::prelude::*;
use solana_runtime::{self, RuntimeError};
@ -87,8 +85,6 @@ pub enum BankError {
pub type Result<T> = result::Result<T, BankError>;
pub const VERIFY_BLOCK_SIZE: usize = 16;
pub trait BankSubscriptions {
fn check_account(&self, pubkey: &Pubkey, account: &Account);
fn check_signature(&self, signature: &Signature, status: &Result<()>);
@ -740,58 +736,6 @@ impl Bank {
Ok(())
}
/// Process an ordered list of entries, populating a circular buffer "tail"
/// as we go.
fn process_block(&self, entries: &[Entry]) -> Result<()> {
for entry in entries {
self.process_entry(entry)?;
}
Ok(())
}
/// Starting from the genesis block, append the provided entries to the ledger verifying them
/// along the way.
pub fn process_ledger<I>(&mut self, entries: I) -> Result<(u64, Hash)>
where
I: IntoIterator<Item = Entry>,
{
let mut last_entry_id = self.last_id();
let mut entries_iter = entries.into_iter();
trace!("genesis last_id={}", last_entry_id);
// The first entry in the ledger is a pseudo-tick used only to ensure the number of ticks
// in slot 0 is the same as the number of ticks in all subsequent slots. It is not
// registered as a tick and thus cannot be used as a last_id
let entry0 = entries_iter
.next()
.ok_or(BankError::LedgerVerificationFailed)?;
if !(entry0.is_tick() && entry0.verify(&last_entry_id)) {
warn!("Ledger proof of history failed at entry0");
return Err(BankError::LedgerVerificationFailed);
}
last_entry_id = entry0.id;
let mut entry_height = 1;
// Ledger verification needs to be parallelized, but we can't pull the whole
// thing into memory. We therefore chunk it.
for block in &entries_iter.chunks(VERIFY_BLOCK_SIZE) {
let block: Vec<_> = block.collect();
if !block.verify(&last_entry_id) {
warn!("Ledger proof of history failed at entry: {}", entry_height);
return Err(BankError::LedgerVerificationFailed);
}
self.process_block(&block)?;
last_entry_id = block.last().unwrap().id;
entry_height += block.len() as u64;
}
Ok((entry_height, last_entry_id))
}
/// Create, sign, and process a Transaction from `keypair` to `to` of
/// `n` tokens where `last_id` is the last Entry ID observed by the client.
pub fn transfer(
@ -1197,7 +1141,7 @@ mod tests {
genesis_block: &GenesisBlock,
mint_keypair: &Keypair,
keypairs: &[Keypair],
) -> impl Iterator<Item = Entry> {
) -> Vec<Entry> {
let mut entries: Vec<Entry> = vec![];
let mut last_id = genesis_block.last_id();
@ -1220,83 +1164,7 @@ mod tests {
last_id = hash;
entries.push(tick);
}
entries.into_iter()
}
// create a ledger with a tick every `tick_interval` entries and a couple other transactions
fn create_sample_block_with_ticks(
genesis_block: &GenesisBlock,
mint_keypair: &Keypair,
num_one_token_transfers: usize,
tick_interval: usize,
) -> impl Iterator<Item = Entry> {
let mut entries = vec![];
let mut last_id = genesis_block.last_id();
// Start off the ledger with the psuedo-tick linked to the genesis block
// (see entry0 in `process_ledger`)
let tick = Entry::new(&genesis_block.last_id(), 0, 1, vec![]);
let mut hash = tick.id;
entries.push(tick);
for i in 0..num_one_token_transfers {
// Transfer one token from the mint to a random account
let keypair = Keypair::new();
let tx = SystemTransaction::new_account(mint_keypair, keypair.pubkey(), 1, last_id, 0);
let entry = Entry::new(&hash, 0, 1, vec![tx]);
hash = entry.id;
entries.push(entry);
// Add a second Transaction that will produce a
// ProgramError<0, ResultWithNegativeTokens> error when processed
let keypair2 = Keypair::new();
let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0);
let entry = Entry::new(&hash, 0, 1, vec![tx]);
hash = entry.id;
entries.push(entry);
if (i + 1) % tick_interval == 0 {
let tick = Entry::new(&hash, 0, 1, vec![]);
hash = tick.id;
last_id = hash;
entries.push(tick);
}
}
entries.into_iter()
}
fn create_sample_ledger(
tokens: u64,
num_one_token_transfers: usize,
) -> (GenesisBlock, Keypair, impl Iterator<Item = Entry>) {
let (genesis_block, mint_keypair) = GenesisBlock::new(tokens);
let block = create_sample_block_with_ticks(
&genesis_block,
&mint_keypair,
num_one_token_transfers,
num_one_token_transfers,
);
(genesis_block, mint_keypair, block)
}
#[test]
fn test_process_ledger_simple() {
let (genesis_block, mint_keypair, ledger) = create_sample_ledger(100, 3);
let mut bank = Bank::default();
bank.add_builtin_programs();
bank.process_genesis_block(&genesis_block);
assert_eq!(bank.tick_height(), 0);
assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100);
assert_eq!(
bank.get_current_leader(),
Some(genesis_block.bootstrap_leader_id)
);
let (ledger_height, last_id) = bank.process_ledger(ledger).unwrap();
assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100 - 3);
assert_eq!(ledger_height, 8);
assert_eq!(bank.tick_height(), 1);
assert_eq!(bank.last_id(), last_id);
entries
}
#[test]
@ -1305,25 +1173,25 @@ mod tests {
let seed = [0u8; 32];
let mut rnd = GenKeys::new(seed);
let keypairs = rnd.gen_n_keypairs(5);
let ledger0 = create_sample_block_with_next_entries_using_keypairs(
let entries0 = create_sample_block_with_next_entries_using_keypairs(
&genesis_block,
&mint_keypair,
&keypairs,
);
let ledger1 = create_sample_block_with_next_entries_using_keypairs(
let entries1 = create_sample_block_with_next_entries_using_keypairs(
&genesis_block,
&mint_keypair,
&keypairs,
);
let mut bank0 = Bank::default();
let bank0 = Bank::default();
bank0.add_builtin_programs();
bank0.process_genesis_block(&genesis_block);
bank0.process_ledger(ledger0).unwrap();
let mut bank1 = Bank::default();
bank0.process_entries(&entries0).unwrap();
let bank1 = Bank::default();
bank1.add_builtin_programs();
bank1.process_genesis_block(&genesis_block);
bank1.process_ledger(ledger1).unwrap();
bank1.process_entries(&entries1).unwrap();
let initial_state = bank0.hash_internal_state();

142
src/blocktree_processor.rs Normal file
View File

@ -0,0 +1,142 @@
use crate::bank::{Bank, BankError, Result};
use crate::blocktree::Blocktree;
use crate::entry::{Entry, EntrySlice};
use itertools::Itertools;
use solana_sdk::hash::Hash;
pub const VERIFY_BLOCK_SIZE: usize = 16;
/// Process an ordered list of entries, populating a circular buffer "tail"
/// as we go.
fn process_block(bank: &Bank, entries: &[Entry]) -> Result<()> {
for entry in entries {
bank.process_entry(entry)?;
}
Ok(())
}
/// Starting from the genesis block, append the provided entries to the ledger verifying them
/// along the way.
fn process_ledger<I>(bank: &Bank, entries: I) -> Result<(u64, Hash)>
where
I: IntoIterator<Item = Entry>,
{
let mut last_entry_id = bank.last_id();
let mut entries_iter = entries.into_iter();
trace!("genesis last_id={}", last_entry_id);
// The first entry in the ledger is a pseudo-tick used only to ensure the number of ticks
// in slot 0 is the same as the number of ticks in all subsequent slots. It is not
// registered as a tick and thus cannot be used as a last_id
let entry0 = entries_iter
.next()
.ok_or(BankError::LedgerVerificationFailed)?;
if !(entry0.is_tick() && entry0.verify(&last_entry_id)) {
warn!("Ledger proof of history failed at entry0");
return Err(BankError::LedgerVerificationFailed);
}
last_entry_id = entry0.id;
let mut entry_height = 1;
// Ledger verification needs to be parallelized, but we can't pull the whole
// thing into memory. We therefore chunk it.
for block in &entries_iter.chunks(VERIFY_BLOCK_SIZE) {
let block: Vec<_> = block.collect();
if !block.verify(&last_entry_id) {
warn!("Ledger proof of history failed at entry: {}", entry_height);
return Err(BankError::LedgerVerificationFailed);
}
process_block(bank, &block)?;
last_entry_id = block.last().unwrap().id;
entry_height += block.len() as u64;
}
Ok((entry_height, last_entry_id))
}
pub fn process_blocktree(bank: &Bank, blocktree: &Blocktree) -> Result<(u64, Hash)> {
let entries = blocktree.read_ledger().expect("opening ledger");
process_ledger(&bank, entries)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::genesis_block::GenesisBlock;
use solana_sdk::signature::{Keypair, KeypairUtil};
use solana_sdk::system_transaction::SystemTransaction;
// create a ledger with a tick every `tick_interval` entries and a couple other transactions
fn create_sample_block_with_ticks(
genesis_block: &GenesisBlock,
mint_keypair: &Keypair,
num_one_token_transfers: usize,
tick_interval: usize,
) -> impl Iterator<Item = Entry> {
let mut entries = vec![];
let mut last_id = genesis_block.last_id();
// Start off the ledger with the psuedo-tick linked to the genesis block
// (see entry0 in `process_ledger`)
let tick = Entry::new(&genesis_block.last_id(), 0, 1, vec![]);
let mut hash = tick.id;
entries.push(tick);
for i in 0..num_one_token_transfers {
// Transfer one token from the mint to a random account
let keypair = Keypair::new();
let tx = SystemTransaction::new_account(mint_keypair, keypair.pubkey(), 1, last_id, 0);
let entry = Entry::new(&hash, 0, 1, vec![tx]);
hash = entry.id;
entries.push(entry);
// Add a second Transaction that will produce a
// ProgramError<0, ResultWithNegativeTokens> error when processed
let keypair2 = Keypair::new();
let tx = SystemTransaction::new_account(&keypair, keypair2.pubkey(), 42, last_id, 0);
let entry = Entry::new(&hash, 0, 1, vec![tx]);
hash = entry.id;
entries.push(entry);
if (i + 1) % tick_interval == 0 {
let tick = Entry::new(&hash, 0, 1, vec![]);
hash = tick.id;
last_id = hash;
entries.push(tick);
}
}
entries.into_iter()
}
fn create_sample_ledger(
tokens: u64,
num_one_token_transfers: usize,
) -> (GenesisBlock, Keypair, impl Iterator<Item = Entry>) {
let (genesis_block, mint_keypair) = GenesisBlock::new(tokens);
let block = create_sample_block_with_ticks(
&genesis_block,
&mint_keypair,
num_one_token_transfers,
num_one_token_transfers,
);
(genesis_block, mint_keypair, block)
}
#[test]
fn test_process_ledger_simple() {
let (genesis_block, mint_keypair, ledger) = create_sample_ledger(100, 3);
let bank = Bank::new(&genesis_block);
assert_eq!(bank.tick_height(), 0);
assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100);
let (ledger_height, last_id) = process_ledger(&bank, ledger).unwrap();
assert_eq!(bank.get_balance(&mint_keypair.pubkey()), 100 - 3);
assert_eq!(ledger_height, 8);
assert_eq!(bank.tick_height(), 1);
assert_eq!(bank.last_id(), last_id);
}
}

7
src/fullnode.rs
View File

@ -2,6 +2,7 @@
use crate::bank::Bank;
use crate::blocktree::{Blocktree, BlocktreeConfig};
use crate::blocktree_processor;
use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
use crate::counter::Counter;
use crate::genesis_block::GenesisBlock;
@ -462,12 +463,12 @@ pub fn new_bank_from_ledger(
.expect("Expected to successfully open database ledger");
let genesis_block =
GenesisBlock::load(ledger_path).expect("Expected to successfully open genesis block");
let mut bank = Bank::new_with_leader_scheduler_config(&genesis_block, leader_scheduler_config);
let bank = Bank::new_with_leader_scheduler_config(&genesis_block, leader_scheduler_config);
let now = Instant::now();
let entries = blocktree.read_ledger().expect("opening ledger");
info!("processing ledger...");
let (entry_height, last_entry_id) = bank.process_ledger(entries).expect("process_ledger");
let (entry_height, last_entry_id) =
blocktree_processor::process_blocktree(&bank, &blocktree).expect("process_blocktree");
info!(
"processed {} ledger entries in {}ms, tick_height={}...",
entry_height,

1
src/lib.rs
View File

@ -30,6 +30,7 @@ pub mod crds_value;
#[macro_use]
pub mod contact_info;
pub mod blocktree;
pub mod blocktree_processor;
pub mod cluster_info;
pub mod compute_leader_confirmation_service;
pub mod db_window;