2018-05-11 23:31:32 -07:00
|
|
|
//! The `tvu` module implements the Transaction Validation Unit, a
|
2019-01-08 08:21:39 -08:00
|
|
|
//! multi-stage transaction validation pipeline in software.
|
2018-06-14 17:05:12 -07:00
|
|
|
//!
|
2018-12-07 14:09:29 -08:00
|
|
|
//! 1. BlobFetchStage
|
|
|
|
//! - Incoming blobs are picked up from the TVU sockets and repair socket.
|
|
|
|
//! 2. RetransmitStage
|
2018-06-15 14:49:22 -07:00
|
|
|
//! - Blobs are windowed until a contiguous chunk is available. This stage also repairs and
|
|
|
|
//! retransmits blobs that are in the queue.
|
2018-12-07 14:09:29 -08:00
|
|
|
//! 3. ReplayStage
|
2018-06-15 14:49:22 -07:00
|
|
|
//! - Transactions in blobs are processed and applied to the bank.
|
|
|
|
//! - TODO We need to verify the signatures in the blobs.
|
2019-01-03 21:29:21 -08:00
|
|
|
//! 4. StorageStage
|
2018-12-07 14:09:29 -08:00
|
|
|
//! - Generating the keys used to encrypt the ledger and sample it for storage mining.
|
2018-05-11 23:31:32 -07:00
|
|
|
|
2019-02-21 11:19:45 -08:00
|
|
|
use crate::bank_forks::BankForks;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::blob_fetch_stage::BlobFetchStage;
|
2019-02-07 20:52:39 -08:00
|
|
|
use crate::blocktree::Blocktree;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::cluster_info::ClusterInfo;
|
2019-02-11 10:36:59 -08:00
|
|
|
use crate::entry_stream_stage::EntryStreamStage;
|
2019-02-16 08:23:29 -08:00
|
|
|
use crate::leader_scheduler::LeaderScheduler;
|
2019-01-26 00:28:08 -08:00
|
|
|
use crate::replay_stage::ReplayStage;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::retransmit_stage::RetransmitStage;
|
2019-02-18 18:08:54 -08:00
|
|
|
use crate::rpc_subscriptions::RpcSubscriptions;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::service::Service;
|
2019-01-28 14:53:50 -08:00
|
|
|
use crate::storage_stage::{StorageStage, StorageState};
|
2019-01-31 20:12:51 -08:00
|
|
|
use crate::voting_keypair::VotingKeypair;
|
2018-11-16 08:04:46 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2019-02-21 09:15:30 -08:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
2019-01-30 19:28:48 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
2019-01-11 12:58:31 -08:00
|
|
|
use std::net::UdpSocket;
|
2018-09-25 15:41:29 -07:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
2019-02-20 18:16:37 -08:00
|
|
|
use std::sync::mpsc::{channel, Receiver, Sender};
|
2018-05-11 23:31:32 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-09-13 14:00:17 -07:00
|
|
|
use std::thread;
|
2018-05-11 23:31:32 -07:00
|
|
|
|
2019-02-21 09:15:30 -08:00
|
|
|
pub type TvuReturnType = (
|
|
|
|
u64, // bank_id,
|
|
|
|
u64, // slot height to initiate a rotation
|
|
|
|
Pubkey, // leader upon rotation
|
|
|
|
);
|
2019-02-20 18:16:37 -08:00
|
|
|
pub type TvuRotationSender = Sender<TvuReturnType>;
|
|
|
|
pub type TvuRotationReceiver = Receiver<TvuReturnType>;
|
2019-02-06 21:09:46 -08:00
|
|
|
|
2018-05-11 23:31:32 -07:00
|
|
|
pub struct Tvu {
|
2018-07-03 21:14:08 -07:00
|
|
|
fetch_stage: BlobFetchStage,
|
2018-08-09 12:03:34 -07:00
|
|
|
retransmit_stage: RetransmitStage,
|
2018-12-07 14:09:29 -08:00
|
|
|
replay_stage: ReplayStage,
|
2019-02-11 13:51:14 -08:00
|
|
|
entry_stream_stage: Option<EntryStreamStage>,
|
2018-10-08 13:12:33 -07:00
|
|
|
storage_stage: StorageStage,
|
2018-09-25 15:41:29 -07:00
|
|
|
exit: Arc<AtomicBool>,
|
2018-05-11 23:31:32 -07:00
|
|
|
}
|
|
|
|
|
2018-12-12 18:57:48 -08:00
|
|
|
pub struct Sockets {
|
|
|
|
pub fetch: Vec<UdpSocket>,
|
|
|
|
pub repair: UdpSocket,
|
|
|
|
pub retransmit: UdpSocket,
|
|
|
|
}
|
|
|
|
|
2018-05-11 23:31:32 -07:00
|
|
|
impl Tvu {
|
|
|
|
/// This service receives messages from a leader in the network and processes the transactions
|
2018-05-14 14:33:11 -07:00
|
|
|
/// on the bank state.
|
2018-05-11 23:31:32 -07:00
|
|
|
/// # Arguments
|
2018-12-12 18:57:48 -08:00
|
|
|
/// * `bank` - The bank state.
|
2018-10-08 13:12:33 -07:00
|
|
|
/// * `entry_height` - Initial ledger height
|
2018-12-12 18:57:48 -08:00
|
|
|
/// * `last_entry_id` - Hash of the last entry
|
2018-10-08 19:55:54 -07:00
|
|
|
/// * `cluster_info` - The cluster_info state.
|
2018-12-12 18:57:48 -08:00
|
|
|
/// * `sockets` - My fetch, repair, and restransmit sockets
|
2019-02-07 20:52:39 -08:00
|
|
|
/// * `blocktree` - the ledger itself
|
2019-01-31 13:43:22 -08:00
|
|
|
#[allow(clippy::new_ret_no_self, clippy::too_many_arguments)]
|
2018-05-23 07:11:11 -07:00
|
|
|
pub fn new(
|
2019-01-31 20:12:51 -08:00
|
|
|
voting_keypair: Option<Arc<VotingKeypair>>,
|
2019-02-21 11:19:45 -08:00
|
|
|
bank_forks: &Arc<RwLock<BankForks>>,
|
2018-06-27 12:35:58 -07:00
|
|
|
entry_height: u64,
|
2018-11-12 12:41:19 -08:00
|
|
|
last_entry_id: Hash,
|
2018-12-10 11:38:29 -08:00
|
|
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
2018-12-12 18:57:48 -08:00
|
|
|
sockets: Sockets,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Arc<Blocktree>,
|
2019-01-02 11:02:15 -08:00
|
|
|
storage_rotate_count: u64,
|
2019-02-10 19:34:18 -08:00
|
|
|
to_leader_sender: &TvuRotationSender,
|
2019-01-28 14:53:50 -08:00
|
|
|
storage_state: &StorageState,
|
2019-01-31 20:41:03 -08:00
|
|
|
entry_stream: Option<&String>,
|
2019-02-04 15:33:43 -08:00
|
|
|
ledger_signal_receiver: Receiver<bool>,
|
2019-02-16 08:23:29 -08:00
|
|
|
leader_scheduler: Arc<RwLock<LeaderScheduler>>,
|
2019-02-18 18:08:54 -08:00
|
|
|
subscriptions: &Arc<RpcSubscriptions>,
|
2019-02-11 17:56:52 -08:00
|
|
|
) -> Self {
|
2018-09-25 15:41:29 -07:00
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
2018-12-12 18:57:48 -08:00
|
|
|
let keypair: Arc<Keypair> = cluster_info
|
|
|
|
.read()
|
|
|
|
.expect("Unable to read from cluster_info during Tvu creation")
|
|
|
|
.keypair
|
|
|
|
.clone();
|
|
|
|
|
|
|
|
let Sockets {
|
|
|
|
repair: repair_socket,
|
|
|
|
fetch: fetch_sockets,
|
|
|
|
retransmit: retransmit_socket,
|
|
|
|
} = sockets;
|
2018-09-25 15:41:29 -07:00
|
|
|
|
2019-01-31 13:43:22 -08:00
|
|
|
let (blob_fetch_sender, blob_fetch_receiver) = channel();
|
|
|
|
|
2018-08-30 12:07:54 -07:00
|
|
|
let repair_socket = Arc::new(repair_socket);
|
2018-09-14 16:56:06 -07:00
|
|
|
let mut blob_sockets: Vec<Arc<UdpSocket>> =
|
2018-12-07 14:09:29 -08:00
|
|
|
fetch_sockets.into_iter().map(Arc::new).collect();
|
2018-09-14 16:56:06 -07:00
|
|
|
blob_sockets.push(repair_socket.clone());
|
2019-01-31 13:43:22 -08:00
|
|
|
let fetch_stage =
|
|
|
|
BlobFetchStage::new_multi_socket(blob_sockets, &blob_fetch_sender, exit.clone());
|
2018-11-24 19:32:33 -08:00
|
|
|
|
2018-05-11 23:31:32 -07:00
|
|
|
//TODO
|
|
|
|
//the packets coming out of blob_receiver need to be sent to the GPU and verified
|
|
|
|
//then sent to the window, which does the erasure coding reconstruction
|
2019-02-04 15:33:43 -08:00
|
|
|
let retransmit_stage = RetransmitStage::new(
|
2019-02-21 11:19:45 -08:00
|
|
|
&bank_forks,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.clone(),
|
2018-10-08 19:55:54 -07:00
|
|
|
&cluster_info,
|
2018-08-28 16:32:40 -07:00
|
|
|
Arc::new(retransmit_socket),
|
2018-08-30 12:07:54 -07:00
|
|
|
repair_socket,
|
2018-07-05 12:01:40 -07:00
|
|
|
blob_fetch_receiver,
|
2019-02-16 08:23:29 -08:00
|
|
|
leader_scheduler.clone(),
|
2019-01-31 13:43:22 -08:00
|
|
|
exit.clone(),
|
2018-05-22 15:17:59 -07:00
|
|
|
);
|
2018-05-11 23:31:32 -07:00
|
|
|
|
2019-02-21 11:19:45 -08:00
|
|
|
let bank = bank_forks.read().unwrap().working_bank();
|
2019-02-11 13:51:14 -08:00
|
|
|
let (replay_stage, mut previous_receiver) = ReplayStage::new(
|
2019-01-30 19:28:48 -08:00
|
|
|
keypair.pubkey(),
|
2019-01-31 20:12:51 -08:00
|
|
|
voting_keypair,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree.clone(),
|
2018-07-13 12:32:20 -07:00
|
|
|
bank.clone(),
|
2018-12-10 11:38:29 -08:00
|
|
|
cluster_info.clone(),
|
2018-09-25 15:41:29 -07:00
|
|
|
exit.clone(),
|
2019-02-21 11:34:42 -08:00
|
|
|
last_entry_id,
|
2019-01-26 00:28:08 -08:00
|
|
|
to_leader_sender,
|
2019-02-04 15:33:43 -08:00
|
|
|
ledger_signal_receiver,
|
2019-02-16 08:23:29 -08:00
|
|
|
&leader_scheduler,
|
2019-02-18 18:08:54 -08:00
|
|
|
subscriptions,
|
2018-07-13 12:32:20 -07:00
|
|
|
);
|
2018-06-13 21:52:23 -07:00
|
|
|
|
2019-02-11 13:51:14 -08:00
|
|
|
let entry_stream_stage = if entry_stream.is_some() {
|
|
|
|
let (entry_stream_stage, entry_stream_receiver) = EntryStreamStage::new(
|
|
|
|
previous_receiver,
|
|
|
|
entry_stream.unwrap().to_string(),
|
2019-02-19 21:19:27 -08:00
|
|
|
bank.tick_height(),
|
2019-02-16 08:23:29 -08:00
|
|
|
leader_scheduler,
|
2019-02-11 13:51:14 -08:00
|
|
|
exit.clone(),
|
|
|
|
);
|
|
|
|
previous_receiver = entry_stream_receiver;
|
|
|
|
Some(entry_stream_stage)
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
};
|
2019-02-11 10:36:59 -08:00
|
|
|
|
2018-11-07 18:25:36 -08:00
|
|
|
let storage_stage = StorageStage::new(
|
2019-01-28 14:53:50 -08:00
|
|
|
storage_state,
|
2019-02-11 13:51:14 -08:00
|
|
|
previous_receiver,
|
2019-02-07 20:52:39 -08:00
|
|
|
Some(blocktree),
|
2019-01-17 14:41:48 -08:00
|
|
|
&keypair,
|
|
|
|
&exit.clone(),
|
2018-11-07 18:25:36 -08:00
|
|
|
entry_height,
|
2019-01-02 11:02:15 -08:00
|
|
|
storage_rotate_count,
|
2019-01-17 14:41:48 -08:00
|
|
|
&cluster_info,
|
2018-10-08 13:12:33 -07:00
|
|
|
);
|
2018-10-23 14:42:48 -07:00
|
|
|
|
2019-02-11 17:56:52 -08:00
|
|
|
Tvu {
|
|
|
|
fetch_stage,
|
|
|
|
retransmit_stage,
|
|
|
|
replay_stage,
|
|
|
|
entry_stream_stage,
|
|
|
|
storage_stage,
|
|
|
|
exit,
|
|
|
|
}
|
2018-05-11 23:31:32 -07:00
|
|
|
}
|
2018-07-09 13:53:18 -07:00
|
|
|
|
2018-10-10 16:49:41 -07:00
|
|
|
pub fn is_exited(&self) -> bool {
|
|
|
|
self.exit.load(Ordering::Relaxed)
|
|
|
|
}
|
|
|
|
|
2018-10-25 16:58:40 -07:00
|
|
|
pub fn exit(&self) {
|
2019-02-04 15:33:43 -08:00
|
|
|
// Call exit to make sure replay stage is unblocked from a channel it may be blocked on.
|
|
|
|
// Then replay stage will set the self.exit variable and cause the rest of the
|
|
|
|
// pipeline to exit
|
|
|
|
self.replay_stage.exit();
|
2018-09-25 15:41:29 -07:00
|
|
|
}
|
|
|
|
|
2019-02-08 08:06:27 -08:00
|
|
|
pub fn close(self) -> thread::Result<()> {
|
2019-02-04 15:33:43 -08:00
|
|
|
self.exit();
|
2018-07-09 13:53:18 -07:00
|
|
|
self.join()
|
|
|
|
}
|
2018-05-11 23:31:32 -07:00
|
|
|
}
|
|
|
|
|
2018-07-03 21:14:08 -07:00
|
|
|
impl Service for Tvu {
|
2019-02-08 08:06:27 -08:00
|
|
|
type JoinReturnType = ();
|
2018-07-03 21:14:08 -07:00
|
|
|
|
2019-02-08 08:06:27 -08:00
|
|
|
fn join(self) -> thread::Result<()> {
|
2018-10-10 16:49:41 -07:00
|
|
|
self.retransmit_stage.join()?;
|
2018-09-13 14:00:17 -07:00
|
|
|
self.fetch_stage.join()?;
|
2018-10-08 13:12:33 -07:00
|
|
|
self.storage_stage.join()?;
|
2019-02-11 13:51:14 -08:00
|
|
|
if self.entry_stream_stage.is_some() {
|
|
|
|
self.entry_stream_stage.unwrap().join()?;
|
|
|
|
}
|
2019-02-08 08:06:27 -08:00
|
|
|
self.replay_stage.join()?;
|
|
|
|
Ok(())
|
2018-07-03 21:14:08 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-05-11 23:31:32 -07:00
|
|
|
#[cfg(test)]
|
2018-05-23 10:49:48 -07:00
|
|
|
pub mod tests {
|
2019-02-08 08:10:28 -08:00
|
|
|
use super::*;
|
2019-02-18 07:48:43 -08:00
|
|
|
use crate::blocktree::get_tmp_ledger_path;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::cluster_info::{ClusterInfo, Node};
|
2019-02-14 09:48:14 -08:00
|
|
|
use crate::leader_scheduler::LeaderSchedulerConfig;
|
2019-02-08 08:10:28 -08:00
|
|
|
use crate::storage_stage::STORAGE_ROTATE_TEST_COUNT;
|
2019-02-21 11:19:45 -08:00
|
|
|
use solana_runtime::bank::Bank;
|
2019-02-18 22:26:22 -08:00
|
|
|
use solana_sdk::genesis_block::GenesisBlock;
|
2018-08-03 11:06:06 -07:00
|
|
|
|
2019-02-04 15:33:43 -08:00
|
|
|
#[test]
|
|
|
|
fn test_tvu_exit() {
|
|
|
|
solana_logger::setup();
|
|
|
|
let leader = Node::new_localhost();
|
|
|
|
let target1_keypair = Keypair::new();
|
|
|
|
let target1 = Node::new_localhost_with_pubkey(target1_keypair.pubkey());
|
|
|
|
|
|
|
|
let starting_balance = 10_000;
|
|
|
|
let (genesis_block, _mint_keypair) = GenesisBlock::new(starting_balance);
|
2019-02-21 11:19:45 -08:00
|
|
|
|
|
|
|
let bank_forks = BankForks::new(0, Bank::new(&genesis_block));
|
2019-02-16 11:36:22 -08:00
|
|
|
let leader_scheduler_config = LeaderSchedulerConfig::default();
|
2019-02-21 11:19:45 -08:00
|
|
|
let leader_scheduler =
|
|
|
|
LeaderScheduler::new_with_bank(&leader_scheduler_config, &bank_forks.working_bank());
|
2019-02-16 11:36:22 -08:00
|
|
|
let leader_scheduler = Arc::new(RwLock::new(leader_scheduler));
|
2019-02-04 15:33:43 -08:00
|
|
|
|
|
|
|
//start cluster_info1
|
|
|
|
let mut cluster_info1 = ClusterInfo::new(target1.info.clone());
|
|
|
|
cluster_info1.insert_info(leader.info.clone());
|
|
|
|
cluster_info1.set_leader(leader.info.id);
|
|
|
|
let cref1 = Arc::new(RwLock::new(cluster_info1));
|
|
|
|
|
|
|
|
let cur_hash = Hash::default();
|
2019-02-12 15:03:11 -08:00
|
|
|
let blocktree_path = get_tmp_ledger_path("test_tvu_exit");
|
2019-02-17 13:05:47 -08:00
|
|
|
let (blocktree, l_receiver) = Blocktree::open_with_signal(&blocktree_path)
|
2019-02-04 15:33:43 -08:00
|
|
|
.expect("Expected to successfully open ledger");
|
|
|
|
let vote_account_keypair = Arc::new(Keypair::new());
|
|
|
|
let voting_keypair = VotingKeypair::new_local(&vote_account_keypair);
|
|
|
|
let (sender, _receiver) = channel();
|
2019-02-11 17:56:52 -08:00
|
|
|
let tvu = Tvu::new(
|
2019-02-04 15:33:43 -08:00
|
|
|
Some(Arc::new(voting_keypair)),
|
2019-02-21 11:19:45 -08:00
|
|
|
&Arc::new(RwLock::new(bank_forks)),
|
2019-02-04 15:33:43 -08:00
|
|
|
0,
|
|
|
|
cur_hash,
|
|
|
|
&cref1,
|
|
|
|
{
|
|
|
|
Sockets {
|
|
|
|
repair: target1.sockets.repair,
|
|
|
|
retransmit: target1.sockets.retransmit,
|
|
|
|
fetch: target1.sockets.tvu,
|
|
|
|
}
|
|
|
|
},
|
2019-02-07 20:52:39 -08:00
|
|
|
Arc::new(blocktree),
|
2019-02-04 15:33:43 -08:00
|
|
|
STORAGE_ROTATE_TEST_COUNT,
|
2019-02-10 19:34:18 -08:00
|
|
|
&sender,
|
2019-02-04 15:33:43 -08:00
|
|
|
&StorageState::default(),
|
|
|
|
None,
|
|
|
|
l_receiver,
|
2019-02-16 08:23:29 -08:00
|
|
|
leader_scheduler,
|
2019-02-18 18:08:54 -08:00
|
|
|
&Arc::new(RpcSubscriptions::default()),
|
2019-02-04 15:33:43 -08:00
|
|
|
);
|
|
|
|
tvu.close().expect("close");
|
|
|
|
}
|
2018-05-11 23:31:32 -07:00
|
|
|
}
|