2018-10-08 13:12:33 -07:00
|
|
|
// A stage that handles generating the keys used to encrypt the ledger and sample it
|
|
|
|
// for storage mining. Replicators submit storage proofs, validator then bundles them
|
2018-10-24 11:20:37 -07:00
|
|
|
// to submit its proof for mining to be rewarded.
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2019-04-24 22:34:10 -07:00
|
|
|
use crate::bank_forks::BankForks;
|
2019-02-07 20:52:39 -08:00
|
|
|
use crate::blocktree::Blocktree;
|
2018-12-08 21:52:29 -08:00
|
|
|
#[cfg(all(feature = "chacha", feature = "cuda"))]
|
|
|
|
use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
|
2019-04-24 22:34:10 -07:00
|
|
|
use crate::cluster_info::ClusterInfo;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::result::{Error, Result};
|
|
|
|
use crate::service::Service;
|
2018-12-10 11:38:29 -08:00
|
|
|
use bincode::deserialize;
|
2018-12-05 14:12:10 -08:00
|
|
|
use rand::{Rng, SeedableRng};
|
|
|
|
use rand_chacha::ChaChaRng;
|
2018-11-16 08:04:46 -08:00
|
|
|
use solana_sdk::hash::Hash;
|
2019-04-24 22:34:10 -07:00
|
|
|
use solana_sdk::instruction::Instruction;
|
2019-05-07 15:01:10 -07:00
|
|
|
use solana_sdk::message::Message;
|
2018-12-10 11:38:29 -08:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
2019-03-22 21:02:00 -07:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil, Signature};
|
2019-04-24 22:34:10 -07:00
|
|
|
use solana_sdk::system_instruction;
|
2019-01-17 14:41:48 -08:00
|
|
|
use solana_sdk::transaction::Transaction;
|
2019-05-03 16:27:53 -07:00
|
|
|
use solana_storage_api::storage_instruction::StorageInstruction;
|
|
|
|
use solana_storage_api::{get_segment_from_slot, storage_instruction};
|
2018-12-10 11:38:29 -08:00
|
|
|
use std::collections::HashSet;
|
2019-01-17 14:41:48 -08:00
|
|
|
use std::io;
|
2018-12-23 13:40:52 -08:00
|
|
|
use std::mem::size_of;
|
2019-04-24 22:34:10 -07:00
|
|
|
use std::net::UdpSocket;
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::sync::atomic::{AtomicBool, Ordering};
|
2019-05-03 16:27:53 -07:00
|
|
|
use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender};
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::sync::{Arc, RwLock};
|
2019-02-01 07:36:35 -08:00
|
|
|
use std::thread::{self, sleep, Builder, JoinHandle};
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
// Block of hash answers to validate against
|
|
|
|
// Vec of [ledger blocks] x [keys]
|
|
|
|
type StorageResults = Vec<Hash>;
|
|
|
|
type StorageKeys = Vec<u8>;
|
2018-12-10 11:38:29 -08:00
|
|
|
type ReplicatorMap = Vec<HashSet<Pubkey>>;
|
|
|
|
|
|
|
|
#[derive(Default)]
|
|
|
|
pub struct StorageStateInner {
|
|
|
|
storage_results: StorageResults,
|
2018-12-23 13:40:52 -08:00
|
|
|
storage_keys: StorageKeys,
|
2018-12-10 11:38:29 -08:00
|
|
|
replicator_map: ReplicatorMap,
|
2019-03-02 10:25:16 -08:00
|
|
|
storage_blockhash: Hash,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot: u64,
|
2018-12-10 11:38:29 -08:00
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2019-01-28 14:53:50 -08:00
|
|
|
#[derive(Clone, Default)]
|
2018-10-08 13:12:33 -07:00
|
|
|
pub struct StorageState {
|
2018-12-10 11:38:29 -08:00
|
|
|
state: Arc<RwLock<StorageStateInner>>,
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
pub struct StorageStage {
|
|
|
|
t_storage_mining_verifier: JoinHandle<()>,
|
2019-01-17 14:41:48 -08:00
|
|
|
t_storage_create_accounts: JoinHandle<()>,
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
pub const STORAGE_ROTATE_TEST_COUNT: u64 = 2;
|
2018-10-08 13:12:33 -07:00
|
|
|
// TODO: some way to dynamically size NUM_IDENTITIES
|
|
|
|
const NUM_IDENTITIES: usize = 1024;
|
2019-01-02 11:02:15 -08:00
|
|
|
pub const NUM_STORAGE_SAMPLES: usize = 4;
|
2018-10-08 13:12:33 -07:00
|
|
|
const KEY_SIZE: usize = 64;
|
|
|
|
|
2019-04-24 22:34:10 -07:00
|
|
|
type InstructionSender = Sender<Instruction>;
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2018-11-02 08:40:29 -07:00
|
|
|
fn get_identity_index_from_signature(key: &Signature) -> usize {
|
2018-10-08 13:12:33 -07:00
|
|
|
let rkey = key.as_ref();
|
|
|
|
let mut res: usize = (rkey[0] as usize)
|
|
|
|
| ((rkey[1] as usize) << 8)
|
|
|
|
| ((rkey[2] as usize) << 16)
|
|
|
|
| ((rkey[3] as usize) << 24);
|
|
|
|
res &= NUM_IDENTITIES - 1;
|
|
|
|
res
|
|
|
|
}
|
|
|
|
|
|
|
|
impl StorageState {
|
|
|
|
pub fn new() -> Self {
|
2018-12-10 11:38:29 -08:00
|
|
|
let storage_keys = vec![0u8; KEY_SIZE * NUM_IDENTITIES];
|
|
|
|
let storage_results = vec![Hash::default(); NUM_IDENTITIES];
|
|
|
|
let replicator_map = vec![];
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2018-12-10 11:38:29 -08:00
|
|
|
let state = StorageStateInner {
|
2018-10-08 13:12:33 -07:00
|
|
|
storage_keys,
|
|
|
|
storage_results,
|
2018-12-10 11:38:29 -08:00
|
|
|
replicator_map,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot: 0,
|
2019-03-02 10:25:16 -08:00
|
|
|
storage_blockhash: Hash::default(),
|
2018-12-10 11:38:29 -08:00
|
|
|
};
|
|
|
|
|
|
|
|
StorageState {
|
|
|
|
state: Arc::new(RwLock::new(state)),
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
pub fn get_mining_key(&self, key: &Signature) -> Vec<u8> {
|
|
|
|
let idx = get_identity_index_from_signature(key);
|
|
|
|
self.state.read().unwrap().storage_keys[idx..idx + KEY_SIZE].to_vec()
|
|
|
|
}
|
|
|
|
|
2018-11-02 08:40:29 -07:00
|
|
|
pub fn get_mining_result(&self, key: &Signature) -> Hash {
|
|
|
|
let idx = get_identity_index_from_signature(key);
|
2018-12-10 11:38:29 -08:00
|
|
|
self.state.read().unwrap().storage_results[idx]
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
2018-11-02 08:40:29 -07:00
|
|
|
|
2019-03-02 10:25:16 -08:00
|
|
|
pub fn get_storage_blockhash(&self) -> Hash {
|
|
|
|
self.state.read().unwrap().storage_blockhash
|
2018-12-23 13:40:52 -08:00
|
|
|
}
|
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
pub fn get_slot(&self) -> u64 {
|
|
|
|
self.state.read().unwrap().slot
|
2018-12-23 13:40:52 -08:00
|
|
|
}
|
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
pub fn get_pubkeys_for_slot(&self, slot: u64) -> Vec<Pubkey> {
|
2018-12-10 11:38:29 -08:00
|
|
|
// TODO: keep track of age?
|
|
|
|
const MAX_PUBKEYS_TO_RETURN: usize = 5;
|
2019-05-03 16:27:53 -07:00
|
|
|
let index = get_segment_from_slot(slot) as usize;
|
2018-12-10 11:38:29 -08:00
|
|
|
let replicator_map = &self.state.read().unwrap().replicator_map;
|
|
|
|
if index < replicator_map.len() {
|
|
|
|
replicator_map[index]
|
|
|
|
.iter()
|
|
|
|
.cloned()
|
|
|
|
.take(MAX_PUBKEYS_TO_RETURN)
|
|
|
|
.collect::<Vec<_>>()
|
|
|
|
} else {
|
|
|
|
vec![]
|
|
|
|
}
|
2018-11-02 08:40:29 -07:00
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
impl StorageStage {
|
2019-04-24 22:34:10 -07:00
|
|
|
#[allow(clippy::too_many_arguments)]
|
2018-10-08 13:12:33 -07:00
|
|
|
pub fn new(
|
|
|
|
storage_state: &StorageState,
|
2019-05-18 15:24:50 -07:00
|
|
|
slot_receiver: Receiver<Vec<u64>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: Option<Arc<Blocktree>>,
|
2019-01-17 14:41:48 -08:00
|
|
|
keypair: &Arc<Keypair>,
|
2019-03-27 15:54:09 -07:00
|
|
|
storage_keypair: &Arc<Keypair>,
|
2019-01-17 14:41:48 -08:00
|
|
|
exit: &Arc<AtomicBool>,
|
2019-04-24 22:34:10 -07:00
|
|
|
bank_forks: &Arc<RwLock<BankForks>>,
|
2019-01-02 11:02:15 -08:00
|
|
|
storage_rotate_count: u64,
|
2019-01-17 14:41:48 -08:00
|
|
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
2018-10-08 13:12:33 -07:00
|
|
|
) -> Self {
|
2019-04-24 22:34:10 -07:00
|
|
|
let (instruction_sender, instruction_receiver) = channel();
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2019-05-07 08:05:12 -07:00
|
|
|
let t_storage_mining_verifier = {
|
|
|
|
let storage_state_inner = storage_state.state.clone();
|
|
|
|
let exit = exit.clone();
|
|
|
|
let storage_keypair = storage_keypair.clone();
|
|
|
|
Builder::new()
|
|
|
|
.name("solana-storage-mining-verify-stage".to_string())
|
|
|
|
.spawn(move || {
|
|
|
|
let mut current_key = 0;
|
|
|
|
let mut slot_count = 0;
|
2019-05-18 15:24:50 -07:00
|
|
|
let mut last_root = 0;
|
2019-05-07 08:05:12 -07:00
|
|
|
loop {
|
|
|
|
if let Some(ref some_blocktree) = blocktree {
|
|
|
|
if let Err(e) = Self::process_entries(
|
|
|
|
&storage_keypair,
|
|
|
|
&storage_state_inner,
|
|
|
|
&slot_receiver,
|
|
|
|
&some_blocktree,
|
|
|
|
&mut slot_count,
|
2019-05-18 15:24:50 -07:00
|
|
|
&mut last_root,
|
2019-05-07 08:05:12 -07:00
|
|
|
&mut current_key,
|
|
|
|
storage_rotate_count,
|
|
|
|
&instruction_sender,
|
|
|
|
) {
|
|
|
|
match e {
|
|
|
|
Error::RecvTimeoutError(RecvTimeoutError::Disconnected) => {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
Error::RecvTimeoutError(RecvTimeoutError::Timeout) => (),
|
|
|
|
_ => info!("Error from process_entries: {:?}", e),
|
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
}
|
2019-05-07 08:05:12 -07:00
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
break;
|
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
2019-05-07 08:05:12 -07:00
|
|
|
})
|
|
|
|
.unwrap()
|
|
|
|
};
|
|
|
|
|
|
|
|
let t_storage_create_accounts = {
|
|
|
|
let cluster_info = cluster_info.clone();
|
|
|
|
let exit = exit.clone();
|
|
|
|
let keypair = keypair.clone();
|
|
|
|
let storage_keypair = storage_keypair.clone();
|
|
|
|
let bank_forks = bank_forks.clone();
|
|
|
|
Builder::new()
|
|
|
|
.name("solana-storage-create-accounts".to_string())
|
|
|
|
.spawn(move || {
|
|
|
|
let transactions_socket = UdpSocket::bind("0.0.0.0:0").unwrap();
|
|
|
|
loop {
|
|
|
|
match instruction_receiver.recv_timeout(Duration::from_secs(1)) {
|
|
|
|
Ok(instruction) => {
|
|
|
|
Self::send_transaction(
|
|
|
|
&bank_forks,
|
|
|
|
&cluster_info,
|
|
|
|
instruction,
|
|
|
|
&keypair,
|
|
|
|
&storage_keypair,
|
|
|
|
&transactions_socket,
|
|
|
|
)
|
|
|
|
.unwrap_or_else(|err| {
|
|
|
|
info!("failed to send storage transaction: {:?}", err)
|
|
|
|
});
|
2019-04-24 22:34:10 -07:00
|
|
|
}
|
2019-05-07 08:05:12 -07:00
|
|
|
Err(e) => match e {
|
|
|
|
RecvTimeoutError::Disconnected => break,
|
|
|
|
RecvTimeoutError::Timeout => (),
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
|
|
|
if exit.load(Ordering::Relaxed) {
|
|
|
|
break;
|
2019-01-17 14:41:48 -08:00
|
|
|
}
|
2019-05-07 08:05:12 -07:00
|
|
|
sleep(Duration::from_millis(100));
|
2019-01-17 14:41:48 -08:00
|
|
|
}
|
2019-05-07 08:05:12 -07:00
|
|
|
})
|
|
|
|
.unwrap()
|
|
|
|
};
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2018-10-08 13:12:33 -07:00
|
|
|
StorageStage {
|
|
|
|
t_storage_mining_verifier,
|
2019-01-17 14:41:48 -08:00
|
|
|
t_storage_create_accounts,
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-08 10:36:56 -08:00
|
|
|
fn send_transaction(
|
2019-04-24 22:34:10 -07:00
|
|
|
bank_forks: &Arc<RwLock<BankForks>>,
|
2019-01-17 14:41:48 -08:00
|
|
|
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
2019-04-24 22:34:10 -07:00
|
|
|
instruction: Instruction,
|
2019-01-17 14:41:48 -08:00
|
|
|
keypair: &Arc<Keypair>,
|
2019-03-27 15:54:09 -07:00
|
|
|
storage_keypair: &Arc<Keypair>,
|
2019-04-24 22:34:10 -07:00
|
|
|
transactions_socket: &UdpSocket,
|
2019-01-17 14:41:48 -08:00
|
|
|
) -> io::Result<()> {
|
2019-04-24 22:34:10 -07:00
|
|
|
let working_bank = bank_forks.read().unwrap().working_bank();
|
|
|
|
let blockhash = working_bank.confirmed_last_blockhash();
|
|
|
|
let mut instructions = vec![];
|
2019-05-07 15:01:10 -07:00
|
|
|
let signer_keys = vec![keypair.as_ref(), storage_keypair.as_ref()];
|
2019-05-07 08:05:12 -07:00
|
|
|
if working_bank
|
|
|
|
.get_account(&storage_keypair.pubkey())
|
|
|
|
.is_none()
|
|
|
|
{
|
|
|
|
let create_instruction = system_instruction::create_account(
|
|
|
|
&keypair.pubkey(),
|
|
|
|
&storage_keypair.pubkey(),
|
2019-05-20 08:55:45 -07:00
|
|
|
1,
|
|
|
|
1024 * 4, // TODO the account space needs to be well defined somewhere
|
2019-05-07 08:05:12 -07:00
|
|
|
&solana_storage_api::id(),
|
|
|
|
);
|
|
|
|
instructions.push(create_instruction);
|
|
|
|
info!("storage account requested");
|
2019-01-17 14:41:48 -08:00
|
|
|
}
|
2019-04-24 22:34:10 -07:00
|
|
|
instructions.push(instruction);
|
2019-05-07 15:01:10 -07:00
|
|
|
let message = Message::new_with_payer(instructions, Some(&signer_keys[0].pubkey()));
|
|
|
|
let transaction = Transaction::new(&signer_keys, message, blockhash);
|
|
|
|
|
2019-04-24 22:34:10 -07:00
|
|
|
transactions_socket.send_to(
|
|
|
|
&bincode::serialize(&transaction).unwrap(),
|
|
|
|
cluster_info.read().unwrap().my_data().tpu,
|
|
|
|
)?;
|
|
|
|
Ok(())
|
2019-01-17 14:41:48 -08:00
|
|
|
}
|
|
|
|
|
2019-03-22 21:02:00 -07:00
|
|
|
fn process_entry_crossing(
|
2019-05-07 08:05:12 -07:00
|
|
|
storage_keypair: &Arc<Keypair>,
|
2018-12-23 13:40:52 -08:00
|
|
|
state: &Arc<RwLock<StorageStateInner>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
_blocktree: &Arc<Blocktree>,
|
2018-10-08 13:12:33 -07:00
|
|
|
entry_id: Hash,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot: u64,
|
2019-04-24 22:34:10 -07:00
|
|
|
instruction_sender: &InstructionSender,
|
2018-10-08 13:12:33 -07:00
|
|
|
) -> Result<()> {
|
|
|
|
let mut seed = [0u8; 32];
|
2019-05-07 08:05:12 -07:00
|
|
|
let signature = storage_keypair.sign(&entry_id.as_ref());
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2019-05-07 08:05:12 -07:00
|
|
|
let ix = storage_instruction::advertise_recent_blockhash(
|
|
|
|
&storage_keypair.pubkey(),
|
|
|
|
entry_id,
|
|
|
|
slot,
|
|
|
|
);
|
2019-04-24 22:34:10 -07:00
|
|
|
instruction_sender.send(ix)?;
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2019-04-18 13:37:20 -07:00
|
|
|
seed.copy_from_slice(&signature.to_bytes()[..32]);
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
let mut rng = ChaChaRng::from_seed(seed);
|
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
state.write().unwrap().slot = slot;
|
2018-12-10 11:38:29 -08:00
|
|
|
|
2018-10-08 13:12:33 -07:00
|
|
|
// Regenerate the answers
|
2019-05-03 16:27:53 -07:00
|
|
|
let num_segments = get_segment_from_slot(slot) as usize;
|
2018-12-10 11:38:29 -08:00
|
|
|
if num_segments == 0 {
|
|
|
|
info!("Ledger has 0 segments!");
|
2018-10-08 13:12:33 -07:00
|
|
|
return Ok(());
|
|
|
|
}
|
2018-12-10 11:38:29 -08:00
|
|
|
// TODO: what if the validator does not have this segment
|
2019-04-18 13:37:20 -07:00
|
|
|
let segment = signature.to_bytes()[0] as usize % num_segments;
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
debug!(
|
2018-12-10 11:38:29 -08:00
|
|
|
"storage verifying: segment: {} identities: {}",
|
|
|
|
segment, NUM_IDENTITIES,
|
2018-10-08 13:12:33 -07:00
|
|
|
);
|
|
|
|
|
|
|
|
let mut samples = vec![];
|
2019-01-02 11:02:15 -08:00
|
|
|
for _ in 0..NUM_STORAGE_SAMPLES {
|
2018-10-08 13:12:33 -07:00
|
|
|
samples.push(rng.gen_range(0, 10));
|
|
|
|
}
|
|
|
|
debug!("generated samples: {:?}", samples);
|
|
|
|
// TODO: cuda required to generate the reference values
|
|
|
|
// but if it is missing, then we need to take care not to
|
|
|
|
// process storage mining results.
|
2018-10-31 15:42:39 -07:00
|
|
|
#[cfg(all(feature = "chacha", feature = "cuda"))]
|
2018-10-08 13:12:33 -07:00
|
|
|
{
|
|
|
|
// Lock the keys, since this is the IV memory,
|
|
|
|
// it will be updated in-place by the encryption.
|
2019-03-15 11:44:10 -07:00
|
|
|
// Should be overwritten by the proof signatures which replace the
|
2018-10-08 13:12:33 -07:00
|
|
|
// key values by the time it runs again.
|
2018-12-10 11:38:29 -08:00
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
let mut statew = state.write().unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
match chacha_cbc_encrypt_file_many_keys(
|
2019-02-07 20:52:39 -08:00
|
|
|
_blocktree,
|
2018-12-10 11:38:29 -08:00
|
|
|
segment as u64,
|
|
|
|
&mut statew.storage_keys,
|
2018-10-08 13:12:33 -07:00
|
|
|
&samples,
|
|
|
|
) {
|
|
|
|
Ok(hashes) => {
|
2018-12-10 11:38:29 -08:00
|
|
|
debug!("Success! encrypted ledger segment: {}", segment);
|
|
|
|
statew.storage_results.copy_from_slice(&hashes);
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
Err(e) => {
|
|
|
|
info!("error encrypting file: {:?}", e);
|
|
|
|
Err(e)?;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
// TODO: bundle up mining submissions from replicators
|
|
|
|
// and submit them in a tx to the leader to get reward.
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
fn process_storage_transaction(
|
|
|
|
data: &[u8],
|
|
|
|
slot: u64,
|
|
|
|
storage_state: &Arc<RwLock<StorageStateInner>>,
|
|
|
|
current_key_idx: &mut usize,
|
|
|
|
transaction_key0: Pubkey,
|
|
|
|
) {
|
|
|
|
match deserialize(data) {
|
|
|
|
Ok(StorageInstruction::SubmitMiningProof {
|
|
|
|
slot: proof_slot,
|
|
|
|
signature,
|
|
|
|
..
|
|
|
|
}) => {
|
|
|
|
if proof_slot < slot {
|
|
|
|
{
|
|
|
|
debug!(
|
|
|
|
"generating storage_keys from storage txs current_key_idx: {}",
|
|
|
|
*current_key_idx
|
|
|
|
);
|
|
|
|
let storage_keys = &mut storage_state.write().unwrap().storage_keys;
|
|
|
|
storage_keys[*current_key_idx..*current_key_idx + size_of::<Signature>()]
|
|
|
|
.copy_from_slice(signature.as_ref());
|
|
|
|
*current_key_idx += size_of::<Signature>();
|
|
|
|
*current_key_idx %= storage_keys.len();
|
|
|
|
}
|
|
|
|
|
|
|
|
let mut statew = storage_state.write().unwrap();
|
|
|
|
let max_segment_index = get_segment_from_slot(slot) as usize;
|
|
|
|
if statew.replicator_map.len() <= max_segment_index {
|
|
|
|
statew
|
|
|
|
.replicator_map
|
|
|
|
.resize(max_segment_index, HashSet::new());
|
|
|
|
}
|
|
|
|
let proof_segment_index = get_segment_from_slot(proof_slot) as usize;
|
|
|
|
if proof_segment_index < statew.replicator_map.len() {
|
|
|
|
statew.replicator_map[proof_segment_index].insert(transaction_key0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
debug!("storage proof: slot: {}", slot);
|
|
|
|
}
|
|
|
|
Ok(_) => {}
|
|
|
|
Err(e) => {
|
|
|
|
info!("error: {:?}", e);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-03-22 21:02:00 -07:00
|
|
|
fn process_entries(
|
2019-05-07 08:05:12 -07:00
|
|
|
storage_keypair: &Arc<Keypair>,
|
2018-12-10 11:38:29 -08:00
|
|
|
storage_state: &Arc<RwLock<StorageStateInner>>,
|
2019-05-18 15:24:50 -07:00
|
|
|
slot_receiver: &Receiver<Vec<u64>>,
|
2019-02-07 20:52:39 -08:00
|
|
|
blocktree: &Arc<Blocktree>,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot_count: &mut u64,
|
2019-05-18 15:24:50 -07:00
|
|
|
last_root: &mut u64,
|
2018-12-23 13:40:52 -08:00
|
|
|
current_key_idx: &mut usize,
|
2019-01-02 11:02:15 -08:00
|
|
|
storage_rotate_count: u64,
|
2019-04-24 22:34:10 -07:00
|
|
|
instruction_sender: &InstructionSender,
|
2018-10-08 13:12:33 -07:00
|
|
|
) -> Result<()> {
|
|
|
|
let timeout = Duration::new(1, 0);
|
2019-05-18 15:24:50 -07:00
|
|
|
let slots: Vec<u64> = slot_receiver.recv_timeout(timeout)?;
|
|
|
|
// check if any rooted slots were missed leading up to this one and bump slot count and process proofs for each missed root
|
|
|
|
for slot in slots.into_iter().rev() {
|
|
|
|
if slot > *last_root {
|
|
|
|
*slot_count += 1;
|
|
|
|
*last_root = slot;
|
|
|
|
|
|
|
|
if let Ok(entries) = blocktree.get_slot_entries(slot, 0, None) {
|
|
|
|
for entry in &entries {
|
|
|
|
// Go through the transactions, find proofs, and use them to update
|
|
|
|
// the storage_keys with their signatures
|
|
|
|
for tx in &entry.transactions {
|
|
|
|
for (i, program_id) in tx.message.program_ids().iter().enumerate() {
|
|
|
|
if solana_storage_api::check_id(&program_id) {
|
|
|
|
Self::process_storage_transaction(
|
|
|
|
&tx.message().instructions[i].data,
|
|
|
|
slot,
|
|
|
|
storage_state,
|
|
|
|
current_key_idx,
|
|
|
|
tx.message.account_keys[0],
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2018-12-23 13:40:52 -08:00
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
2019-05-18 15:24:50 -07:00
|
|
|
if *slot_count % storage_rotate_count == 0 {
|
|
|
|
// assume the last entry in the slot is the blockhash for that slot
|
|
|
|
let entry_hash = entries.last().unwrap().hash;
|
|
|
|
debug!(
|
|
|
|
"crosses sending at root slot: {}! with last entry's hash {}",
|
|
|
|
slot_count, entry_hash
|
|
|
|
);
|
|
|
|
Self::process_entry_crossing(
|
|
|
|
&storage_keypair,
|
|
|
|
&storage_state,
|
|
|
|
&blocktree,
|
|
|
|
entries.last().unwrap().hash,
|
|
|
|
slot,
|
|
|
|
instruction_sender,
|
|
|
|
)?;
|
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
2019-05-17 14:52:54 -07:00
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Service for StorageStage {
|
|
|
|
type JoinReturnType = ();
|
|
|
|
|
|
|
|
fn join(self) -> thread::Result<()> {
|
2019-01-17 14:41:48 -08:00
|
|
|
self.t_storage_create_accounts.join().unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
self.t_storage_mining_verifier.join()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
2019-03-22 13:30:18 -07:00
|
|
|
use super::*;
|
2019-02-26 16:35:00 -08:00
|
|
|
use crate::blocktree::{create_new_tmp_ledger, Blocktree};
|
2019-03-08 17:23:07 -08:00
|
|
|
use crate::cluster_info::ClusterInfo;
|
|
|
|
use crate::contact_info::ContactInfo;
|
2019-02-28 01:14:37 -08:00
|
|
|
use crate::entry::{make_tiny_test_entries, Entry};
|
2019-05-07 11:16:22 -07:00
|
|
|
use crate::genesis_utils::create_genesis_block;
|
2018-12-07 19:16:27 -08:00
|
|
|
use crate::service::Service;
|
2018-11-29 15:50:03 -08:00
|
|
|
use rayon::prelude::*;
|
2019-04-24 22:34:10 -07:00
|
|
|
use solana_runtime::bank::Bank;
|
2019-03-22 13:30:18 -07:00
|
|
|
use solana_sdk::hash::{Hash, Hasher};
|
2019-01-17 14:41:48 -08:00
|
|
|
use solana_sdk::pubkey::Pubkey;
|
2019-01-25 22:41:20 -08:00
|
|
|
use solana_sdk::signature::{Keypair, KeypairUtil};
|
2019-05-15 13:28:56 -07:00
|
|
|
use solana_storage_api::SLOTS_PER_SEGMENT;
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::cmp::{max, min};
|
|
|
|
use std::fs::remove_dir_all;
|
2018-11-29 15:50:03 -08:00
|
|
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::sync::mpsc::channel;
|
2019-01-17 14:41:48 -08:00
|
|
|
use std::sync::{Arc, RwLock};
|
2018-10-08 13:12:33 -07:00
|
|
|
use std::thread::sleep;
|
|
|
|
use std::time::Duration;
|
|
|
|
|
|
|
|
#[test]
|
|
|
|
fn test_storage_stage_none_ledger() {
|
|
|
|
let keypair = Arc::new(Keypair::new());
|
2019-03-27 15:54:09 -07:00
|
|
|
let storage_keypair = Arc::new(Keypair::new());
|
2018-10-08 13:12:33 -07:00
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
|
|
|
|
2019-03-09 19:28:43 -08:00
|
|
|
let cluster_info = test_cluster_info(&keypair.pubkey());
|
2019-05-07 11:16:22 -07:00
|
|
|
let (genesis_block, _mint_keypair) = create_genesis_block(1000);
|
2019-04-24 22:34:10 -07:00
|
|
|
let bank = Arc::new(Bank::new(&genesis_block));
|
2019-05-07 23:34:10 -07:00
|
|
|
let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(&[bank], 0)));
|
2019-05-03 16:27:53 -07:00
|
|
|
let (_slot_sender, slot_receiver) = channel();
|
2018-10-08 13:12:33 -07:00
|
|
|
let storage_state = StorageState::new();
|
|
|
|
let storage_stage = StorageStage::new(
|
|
|
|
&storage_state,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot_receiver,
|
2018-10-08 13:12:33 -07:00
|
|
|
None,
|
2019-01-17 14:41:48 -08:00
|
|
|
&keypair,
|
2019-03-27 15:54:09 -07:00
|
|
|
&storage_keypair,
|
2019-01-17 14:41:48 -08:00
|
|
|
&exit.clone(),
|
2019-04-24 22:34:10 -07:00
|
|
|
&bank_forks,
|
2019-01-02 11:02:15 -08:00
|
|
|
STORAGE_ROTATE_TEST_COUNT,
|
2019-01-17 14:41:48 -08:00
|
|
|
&cluster_info,
|
2018-10-08 13:12:33 -07:00
|
|
|
);
|
|
|
|
exit.store(true, Ordering::Relaxed);
|
|
|
|
storage_stage.join().unwrap();
|
|
|
|
}
|
|
|
|
|
2019-03-09 19:28:43 -08:00
|
|
|
fn test_cluster_info(id: &Pubkey) -> Arc<RwLock<ClusterInfo>> {
|
2019-03-08 17:23:07 -08:00
|
|
|
let contact_info = ContactInfo::new_localhost(id, 0);
|
|
|
|
let cluster_info = ClusterInfo::new_with_invalid_keypair(contact_info);
|
2019-01-17 14:41:48 -08:00
|
|
|
Arc::new(RwLock::new(cluster_info))
|
|
|
|
}
|
|
|
|
|
2018-10-08 13:12:33 -07:00
|
|
|
#[test]
|
|
|
|
fn test_storage_stage_process_entries() {
|
2018-12-14 12:36:50 -08:00
|
|
|
solana_logger::setup();
|
2018-10-08 13:12:33 -07:00
|
|
|
let keypair = Arc::new(Keypair::new());
|
2019-03-27 15:54:09 -07:00
|
|
|
let storage_keypair = Arc::new(Keypair::new());
|
2018-10-08 13:12:33 -07:00
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
|
|
|
|
2019-05-07 11:16:22 -07:00
|
|
|
let (genesis_block, _mint_keypair) = create_genesis_block(1000);
|
2019-02-22 21:55:46 -08:00
|
|
|
let ticks_per_slot = genesis_block.ticks_per_slot;
|
2019-03-02 10:25:16 -08:00
|
|
|
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2019-01-02 11:02:15 -08:00
|
|
|
let entries = make_tiny_test_entries(64);
|
2019-05-03 16:27:53 -07:00
|
|
|
let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
|
|
|
|
let slot = 1;
|
2019-04-24 22:34:10 -07:00
|
|
|
let bank = Arc::new(Bank::new(&genesis_block));
|
2019-05-07 23:34:10 -07:00
|
|
|
let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(&[bank], 0)));
|
2019-03-14 15:18:37 -07:00
|
|
|
blocktree
|
2019-05-03 16:27:53 -07:00
|
|
|
.write_entries(slot, 0, 0, ticks_per_slot, &entries)
|
2019-03-14 15:18:37 -07:00
|
|
|
.unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2019-03-09 19:28:43 -08:00
|
|
|
let cluster_info = test_cluster_info(&keypair.pubkey());
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
let (slot_sender, slot_receiver) = channel();
|
2018-10-08 13:12:33 -07:00
|
|
|
let storage_state = StorageState::new();
|
|
|
|
let storage_stage = StorageStage::new(
|
|
|
|
&storage_state,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot_receiver,
|
|
|
|
Some(blocktree.clone()),
|
2019-01-17 14:41:48 -08:00
|
|
|
&keypair,
|
2019-03-27 15:54:09 -07:00
|
|
|
&storage_keypair,
|
2019-01-17 14:41:48 -08:00
|
|
|
&exit.clone(),
|
2019-04-24 22:34:10 -07:00
|
|
|
&bank_forks,
|
2019-01-02 11:02:15 -08:00
|
|
|
STORAGE_ROTATE_TEST_COUNT,
|
2019-01-17 14:41:48 -08:00
|
|
|
&cluster_info,
|
2018-10-08 13:12:33 -07:00
|
|
|
);
|
2019-05-18 15:24:50 -07:00
|
|
|
slot_sender.send(vec![slot]).unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
let keypair = Keypair::new();
|
2018-11-02 08:40:29 -07:00
|
|
|
let hash = Hash::default();
|
2019-01-25 22:41:20 -08:00
|
|
|
let signature = keypair.sign_message(&hash.as_ref());
|
2018-11-02 08:40:29 -07:00
|
|
|
let mut result = storage_state.get_mining_result(&signature);
|
2018-10-08 13:12:33 -07:00
|
|
|
assert_eq!(result, Hash::default());
|
|
|
|
|
2019-05-18 15:24:50 -07:00
|
|
|
let rooted_slots = (slot..slot + SLOTS_PER_SEGMENT + 1)
|
|
|
|
.map(|i| {
|
|
|
|
blocktree
|
|
|
|
.write_entries(i, 0, 0, ticks_per_slot, &entries)
|
|
|
|
.unwrap();
|
|
|
|
i
|
|
|
|
})
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
slot_sender.send(rooted_slots).unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
for _ in 0..5 {
|
2018-11-02 08:40:29 -07:00
|
|
|
result = storage_state.get_mining_result(&signature);
|
2018-10-08 13:12:33 -07:00
|
|
|
if result != Hash::default() {
|
|
|
|
info!("found result = {:?} sleeping..", result);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
info!("result = {:?} sleeping..", result);
|
|
|
|
sleep(Duration::new(1, 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
info!("joining..?");
|
|
|
|
exit.store(true, Ordering::Relaxed);
|
|
|
|
storage_stage.join().unwrap();
|
|
|
|
|
2018-10-31 15:42:39 -07:00
|
|
|
#[cfg(not(all(feature = "cuda", feature = "chacha")))]
|
2018-10-08 13:12:33 -07:00
|
|
|
assert_eq!(result, Hash::default());
|
|
|
|
|
2018-10-31 15:42:39 -07:00
|
|
|
#[cfg(all(feature = "cuda", feature = "chacha"))]
|
2018-10-08 13:12:33 -07:00
|
|
|
assert_ne!(result, Hash::default());
|
|
|
|
|
|
|
|
remove_dir_all(ledger_path).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2019-03-15 11:44:10 -07:00
|
|
|
fn test_storage_stage_process_proof_entries() {
|
2018-12-14 12:36:50 -08:00
|
|
|
solana_logger::setup();
|
2018-10-08 13:12:33 -07:00
|
|
|
let keypair = Arc::new(Keypair::new());
|
2019-03-27 15:54:09 -07:00
|
|
|
let storage_keypair = Arc::new(Keypair::new());
|
2018-10-08 13:12:33 -07:00
|
|
|
let exit = Arc::new(AtomicBool::new(false));
|
|
|
|
|
2019-05-07 11:16:22 -07:00
|
|
|
let (genesis_block, _mint_keypair) = create_genesis_block(1000);
|
2019-02-22 21:55:46 -08:00
|
|
|
let ticks_per_slot = genesis_block.ticks_per_slot;;
|
2019-03-02 10:25:16 -08:00
|
|
|
let (ledger_path, _blockhash) = create_new_tmp_ledger!(&genesis_block);
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
let entries = make_tiny_test_entries(128);
|
2019-05-03 16:27:53 -07:00
|
|
|
let blocktree = Arc::new(Blocktree::open(&ledger_path).unwrap());
|
2019-03-14 15:18:37 -07:00
|
|
|
blocktree
|
|
|
|
.write_entries(1, 0, 0, ticks_per_slot, &entries)
|
|
|
|
.unwrap();
|
2019-04-24 22:34:10 -07:00
|
|
|
let bank = Arc::new(Bank::new(&genesis_block));
|
2019-05-07 23:34:10 -07:00
|
|
|
let bank_forks = Arc::new(RwLock::new(BankForks::new_from_banks(&[bank], 0)));
|
2019-03-09 19:28:43 -08:00
|
|
|
let cluster_info = test_cluster_info(&keypair.pubkey());
|
2019-01-17 14:41:48 -08:00
|
|
|
|
2019-05-03 16:27:53 -07:00
|
|
|
let (slot_sender, slot_receiver) = channel();
|
2018-10-08 13:12:33 -07:00
|
|
|
let storage_state = StorageState::new();
|
|
|
|
let storage_stage = StorageStage::new(
|
|
|
|
&storage_state,
|
2019-05-03 16:27:53 -07:00
|
|
|
slot_receiver,
|
|
|
|
Some(blocktree.clone()),
|
2019-01-17 14:41:48 -08:00
|
|
|
&keypair,
|
2019-03-27 15:54:09 -07:00
|
|
|
&storage_keypair,
|
2019-01-17 14:41:48 -08:00
|
|
|
&exit.clone(),
|
2019-04-24 22:34:10 -07:00
|
|
|
&bank_forks,
|
2019-01-02 11:02:15 -08:00
|
|
|
STORAGE_ROTATE_TEST_COUNT,
|
2019-01-17 14:41:48 -08:00
|
|
|
&cluster_info,
|
2018-10-08 13:12:33 -07:00
|
|
|
);
|
2019-05-18 15:24:50 -07:00
|
|
|
slot_sender.send(vec![1]).unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
let mut reference_keys;
|
|
|
|
{
|
|
|
|
let keys = &storage_state.state.read().unwrap().storage_keys;
|
|
|
|
reference_keys = vec![0; keys.len()];
|
|
|
|
reference_keys.copy_from_slice(keys);
|
|
|
|
}
|
2019-03-22 21:02:00 -07:00
|
|
|
|
2018-10-08 13:12:33 -07:00
|
|
|
let keypair = Keypair::new();
|
2019-04-03 08:45:57 -07:00
|
|
|
let mining_proof_ix = storage_instruction::mining_proof(
|
2019-03-22 21:02:00 -07:00
|
|
|
&keypair.pubkey(),
|
2019-03-15 11:44:10 -07:00
|
|
|
Hash::default(),
|
|
|
|
0,
|
|
|
|
keypair.sign_message(b"test"),
|
|
|
|
);
|
2019-03-26 16:14:28 -07:00
|
|
|
let mining_proof_tx = Transaction::new_unsigned_instructions(vec![mining_proof_ix]);
|
2019-03-22 21:02:00 -07:00
|
|
|
let mining_txs = vec![mining_proof_tx];
|
|
|
|
|
2019-03-15 11:44:10 -07:00
|
|
|
let proof_entries = vec![Entry::new(&Hash::default(), 1, mining_txs)];
|
2019-05-03 16:27:53 -07:00
|
|
|
blocktree
|
|
|
|
.write_entries(2, 0, 0, ticks_per_slot, &proof_entries)
|
|
|
|
.unwrap();
|
2019-05-18 15:24:50 -07:00
|
|
|
slot_sender.send(vec![2]).unwrap();
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
for _ in 0..5 {
|
2018-12-23 13:40:52 -08:00
|
|
|
{
|
|
|
|
let keys = &storage_state.state.read().unwrap().storage_keys;
|
|
|
|
if keys[..] != *reference_keys.as_slice() {
|
|
|
|
break;
|
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
sleep(Duration::new(1, 0));
|
|
|
|
}
|
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
debug!("joining..?");
|
2018-10-08 13:12:33 -07:00
|
|
|
exit.store(true, Ordering::Relaxed);
|
|
|
|
storage_stage.join().unwrap();
|
|
|
|
|
2018-12-23 13:40:52 -08:00
|
|
|
{
|
|
|
|
let keys = &storage_state.state.read().unwrap().storage_keys;
|
|
|
|
assert_ne!(keys[..], *reference_keys);
|
|
|
|
}
|
2018-10-08 13:12:33 -07:00
|
|
|
|
|
|
|
remove_dir_all(ledger_path).unwrap();
|
|
|
|
}
|
|
|
|
|
|
|
|
#[test]
|
2018-11-02 08:40:29 -07:00
|
|
|
fn test_signature_distribution() {
|
|
|
|
// See that signatures have an even-ish distribution..
|
2018-11-29 15:50:03 -08:00
|
|
|
let mut hist = Arc::new(vec![]);
|
|
|
|
for _ in 0..NUM_IDENTITIES {
|
|
|
|
Arc::get_mut(&mut hist).unwrap().push(AtomicUsize::new(0));
|
|
|
|
}
|
2018-11-02 08:40:29 -07:00
|
|
|
let hasher = Hasher::default();
|
2018-11-29 15:50:03 -08:00
|
|
|
{
|
|
|
|
let hist = hist.clone();
|
|
|
|
(0..(32 * NUM_IDENTITIES))
|
|
|
|
.into_par_iter()
|
|
|
|
.for_each(move |_| {
|
|
|
|
let keypair = Keypair::new();
|
2018-11-02 08:40:29 -07:00
|
|
|
let hash = hasher.clone().result();
|
2019-01-25 22:41:20 -08:00
|
|
|
let signature = keypair.sign_message(&hash.as_ref());
|
2018-11-02 08:40:29 -07:00
|
|
|
let ix = get_identity_index_from_signature(&signature);
|
2018-11-29 15:50:03 -08:00
|
|
|
hist[ix].fetch_add(1, Ordering::Relaxed);
|
|
|
|
});
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
2018-11-29 15:50:03 -08:00
|
|
|
|
2018-10-08 13:12:33 -07:00
|
|
|
let mut hist_max = 0;
|
|
|
|
let mut hist_min = NUM_IDENTITIES;
|
|
|
|
for x in hist.iter() {
|
2018-11-29 15:50:03 -08:00
|
|
|
let val = x.load(Ordering::Relaxed);
|
|
|
|
hist_max = max(val, hist_max);
|
|
|
|
hist_min = min(val, hist_min);
|
2018-10-08 13:12:33 -07:00
|
|
|
}
|
|
|
|
info!("min: {} max: {}", hist_min, hist_max);
|
|
|
|
assert_ne!(hist_min, 0);
|
|
|
|
}
|
|
|
|
}
|