Refactor chacha cuda to be able to test cuda crate but not in OpenCL (#7685)
* Refactor chacha cuda to be able to test cuda crate but not in OpenCL chacha not implemeted in OpenCL * Get off core::Error
This commit is contained in:
sakridge
GitHub
parent
ed0129f881
commit
8572b57834
|
|
@ -3409,6 +3409,7 @@ version = "0.23.0"
|
|||
dependencies = [
|
||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-archiver-lib 0.23.0",
|
||||
"solana-clap-utils 0.23.0",
|
||||
"solana-core 0.23.0",
|
||||
"solana-logger 0.23.0",
|
||||
|
|
@ -3417,6 +3418,51 @@ dependencies = [
|
|||
"solana-sdk 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-archiver-lib"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-archiver-utils 0.23.0",
|
||||
"solana-chacha 0.23.0",
|
||||
"solana-chacha-sys 0.23.0",
|
||||
"solana-client 0.23.0",
|
||||
"solana-core 0.23.0",
|
||||
"solana-ledger 0.23.0",
|
||||
"solana-logger 0.23.0",
|
||||
"solana-metrics 0.23.0",
|
||||
"solana-net-utils 0.23.0",
|
||||
"solana-perf 0.23.0",
|
||||
"solana-sdk 0.23.0",
|
||||
"solana-storage-program 0.23.0",
|
||||
"thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-archiver-utils"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-chacha 0.23.0",
|
||||
"solana-chacha-sys 0.23.0",
|
||||
"solana-ledger 0.23.0",
|
||||
"solana-logger 0.23.0",
|
||||
"solana-perf 0.23.0",
|
||||
"solana-sdk 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-banking-bench"
|
||||
version = "0.23.0"
|
||||
|
|
@ -3553,6 +3599,35 @@ dependencies = [
|
|||
"thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-chacha"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-chacha-sys 0.23.0",
|
||||
"solana-ledger 0.23.0",
|
||||
"solana-logger 0.23.0",
|
||||
"solana-perf 0.23.0",
|
||||
"solana-sdk 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-chacha-cuda"
|
||||
version = "0.23.0"
|
||||
dependencies = [
|
||||
"hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-archiver-utils 0.23.0",
|
||||
"solana-chacha 0.23.0",
|
||||
"solana-ledger 0.23.0",
|
||||
"solana-logger 0.23.0",
|
||||
"solana-perf 0.23.0",
|
||||
"solana-sdk 0.23.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-chacha-sys"
|
||||
version = "0.23.0"
|
||||
|
|
@ -3661,7 +3736,6 @@ dependencies = [
|
|||
"crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"jsonrpc-core 14.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -3685,7 +3759,7 @@ dependencies = [
|
|||
"serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-budget-program 0.23.0",
|
||||
"solana-chacha-sys 0.23.0",
|
||||
"solana-chacha-cuda 0.23.0",
|
||||
"solana-clap-utils 0.23.0",
|
||||
"solana-client 0.23.0",
|
||||
"solana-faucet 0.23.0",
|
||||
|
|
@ -3972,6 +4046,7 @@ dependencies = [
|
|||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-archiver-lib 0.23.0",
|
||||
"solana-client 0.23.0",
|
||||
"solana-config-program 0.23.0",
|
||||
"solana-core 0.23.0",
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@ members = [
|
|||
"bench-streamer",
|
||||
"bench-tps",
|
||||
"banking-bench",
|
||||
"chacha",
|
||||
"chacha-cuda",
|
||||
"chacha-sys",
|
||||
"client",
|
||||
"core",
|
||||
|
|
@ -38,6 +40,8 @@ members = [
|
|||
"programs/vest",
|
||||
"programs/vote",
|
||||
"archiver",
|
||||
"archiver-lib",
|
||||
"archiver-utils",
|
||||
"runtime",
|
||||
"sdk",
|
||||
"sdk-c",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
[package]
|
||||
name = "solana-archiver-lib"
|
||||
version = "0.23.0"
|
||||
description = "Solana Archiver Library"
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://solana.com/"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
bincode = "1.2.1"
|
||||
crossbeam-channel = "0.3"
|
||||
ed25519-dalek = "=1.0.0-pre.1"
|
||||
log = "0.4.8"
|
||||
rand = "0.6.5"
|
||||
rand_chacha = "0.1.1"
|
||||
solana-client = { path = "../client", version = "0.23.0" }
|
||||
solana-storage-program = { path = "../programs/storage", version = "0.23.0" }
|
||||
thiserror = "1.0"
|
||||
serde = "1.0.104"
|
||||
serde_json = "1.0.44"
|
||||
serde_derive = "1.0.103"
|
||||
solana-net-utils = { path = "../net-utils", version = "0.23.0" }
|
||||
solana-chacha = { path = "../chacha", version = "0.23.0" }
|
||||
solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
|
||||
solana-ledger = { path = "../ledger", version = "0.23.0" }
|
||||
solana-logger = { path = "../logger", version = "0.23.0" }
|
||||
solana-perf = { path = "../perf", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
solana-core = { path = "../core", version = "0.23.0" }
|
||||
solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" }
|
||||
solana-metrics = { path = "../metrics", version = "0.23.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
hex = "0.4.0"
|
||||
|
||||
[lib]
|
||||
name = "solana_archiver_lib"
|
||||
|
|
@ -1,26 +1,27 @@
|
|||
use crate::{
|
||||
chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE},
|
||||
use crate::result::ArchiverError;
|
||||
use crossbeam_channel::unbounded;
|
||||
use ed25519_dalek;
|
||||
use rand::{thread_rng, Rng, SeedableRng};
|
||||
use rand_chacha::ChaChaRng;
|
||||
use solana_archiver_utils::sample_file;
|
||||
use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
|
||||
use solana_client::{
|
||||
rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
|
||||
thin_client::ThinClient,
|
||||
};
|
||||
use solana_core::{
|
||||
cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
|
||||
contact_info::ContactInfo,
|
||||
gossip_service::GossipService,
|
||||
packet::{limited_deserialize, PACKET_DATA_SIZE},
|
||||
repair_service,
|
||||
repair_service::{RepairService, RepairSlotRange, RepairStrategy},
|
||||
result::{Error, Result},
|
||||
shred_fetch_stage::ShredFetchStage,
|
||||
sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
|
||||
storage_stage::NUM_STORAGE_SAMPLES,
|
||||
streamer::{receiver, responder, PacketReceiver},
|
||||
window_service::WindowService,
|
||||
};
|
||||
use crossbeam_channel::unbounded;
|
||||
use ed25519_dalek;
|
||||
use rand::{thread_rng, Rng, SeedableRng};
|
||||
use rand_chacha::ChaChaRng;
|
||||
use solana_client::{
|
||||
rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
|
||||
thin_client::ThinClient,
|
||||
};
|
||||
use solana_ledger::{
|
||||
blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
|
||||
};
|
||||
|
|
@ -33,7 +34,7 @@ use solana_sdk::{
|
|||
client::{AsyncClient, SyncClient},
|
||||
clock::{get_complete_segment_from_slot, get_segment_from_slot, Slot},
|
||||
commitment_config::CommitmentConfig,
|
||||
hash::{Hash, Hasher},
|
||||
hash::Hash,
|
||||
message::Message,
|
||||
signature::{Keypair, KeypairUtil, Signature},
|
||||
timing::timestamp,
|
||||
|
|
@ -45,9 +46,7 @@ use solana_storage_program::{
|
|||
storage_instruction::{self, StorageAccountType},
|
||||
};
|
||||
use std::{
|
||||
fs::File,
|
||||
io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom},
|
||||
mem::size_of,
|
||||
io::{self, ErrorKind},
|
||||
net::{SocketAddr, UdpSocket},
|
||||
path::{Path, PathBuf},
|
||||
result,
|
||||
|
|
@ -58,6 +57,8 @@ use std::{
|
|||
time::Duration,
|
||||
};
|
||||
|
||||
type Result<T> = std::result::Result<T, ArchiverError>;
|
||||
|
||||
static ENCRYPTED_FILENAME: &str = "ledger.enc";
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
|
|
@ -85,41 +86,6 @@ struct ArchiverMeta {
|
|||
client_commitment: CommitmentConfig,
|
||||
}
|
||||
|
||||
pub(crate) fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
|
||||
let in_file = File::open(in_path)?;
|
||||
let metadata = in_file.metadata()?;
|
||||
let mut buffer_file = BufReader::new(in_file);
|
||||
|
||||
let mut hasher = Hasher::default();
|
||||
let sample_size = size_of::<Hash>();
|
||||
let sample_size64 = sample_size as u64;
|
||||
let mut buf = vec![0; sample_size];
|
||||
|
||||
let file_len = metadata.len();
|
||||
if file_len < sample_size64 {
|
||||
return Err(io::Error::new(ErrorKind::Other, "file too short!"));
|
||||
}
|
||||
for offset in sample_offsets {
|
||||
if *offset > (file_len - sample_size64) / sample_size64 {
|
||||
return Err(io::Error::new(ErrorKind::Other, "offset too large"));
|
||||
}
|
||||
buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
|
||||
trace!("sampling @ {} ", *offset);
|
||||
match buffer_file.read(&mut buf) {
|
||||
Ok(size) => {
|
||||
assert_eq!(size, buf.len());
|
||||
hasher.hash(&buf);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Error sampling file");
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hasher.result())
|
||||
}
|
||||
|
||||
fn get_slot_from_signature(
|
||||
signature: &ed25519_dalek::Signature,
|
||||
storage_turn: u64,
|
||||
|
|
@ -239,16 +205,16 @@ impl Archiver {
|
|||
|
||||
info!("Connecting to the cluster via {:?}", cluster_entrypoint);
|
||||
let (nodes, _) =
|
||||
match crate::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
|
||||
match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
|
||||
Ok(nodes_and_archivers) => nodes_and_archivers,
|
||||
Err(e) => {
|
||||
//shutdown services before exiting
|
||||
exit.store(true, Ordering::Relaxed);
|
||||
gossip_service.join()?;
|
||||
return Err(Error::from(e));
|
||||
return Err(e.into());
|
||||
}
|
||||
};
|
||||
let client = crate::gossip_service::get_client(&nodes);
|
||||
let client = solana_core::gossip_service::get_client(&nodes);
|
||||
|
||||
info!("Setting up mining account...");
|
||||
if let Err(e) = Self::setup_mining_account(
|
||||
|
|
@ -411,7 +377,7 @@ impl Archiver {
|
|||
client_commitment: CommitmentConfig,
|
||||
) {
|
||||
let nodes = cluster_info.read().unwrap().tvu_peers();
|
||||
let client = crate::gossip_service::get_client(&nodes);
|
||||
let client = solana_core::gossip_service::get_client(&nodes);
|
||||
|
||||
if let Ok(Some(account)) =
|
||||
client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment.clone())
|
||||
|
|
@ -622,9 +588,7 @@ impl Archiver {
|
|||
client_commitment.clone(),
|
||||
)? == 0
|
||||
{
|
||||
return Err(
|
||||
io::Error::new(io::ErrorKind::Other, "keypair account has no balance").into(),
|
||||
);
|
||||
return Err(ArchiverError::EmptyStorageAccountBalance);
|
||||
}
|
||||
|
||||
info!("checking storage account keypair...");
|
||||
|
|
@ -635,11 +599,8 @@ impl Archiver {
|
|||
let blockhash =
|
||||
match client.get_recent_blockhash_with_commitment(client_commitment.clone()) {
|
||||
Ok((blockhash, _)) => blockhash,
|
||||
Err(_) => {
|
||||
return Err(Error::IO(<io::Error>::new(
|
||||
io::ErrorKind::Other,
|
||||
"unable to get recent blockhash, can't submit proof",
|
||||
)));
|
||||
Err(e) => {
|
||||
return Err(ArchiverError::TransportError(e));
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -673,7 +634,7 @@ impl Archiver {
|
|||
) {
|
||||
// No point if we've got no storage account...
|
||||
let nodes = cluster_info.read().unwrap().tvu_peers();
|
||||
let client = crate::gossip_service::get_client(&nodes);
|
||||
let client = solana_core::gossip_service::get_client(&nodes);
|
||||
let storage_balance = client.poll_get_balance_with_commitment(
|
||||
&storage_keypair.pubkey(),
|
||||
meta.client_commitment.clone(),
|
||||
|
|
@ -737,7 +698,7 @@ impl Archiver {
|
|||
fn get_segment_config(
|
||||
cluster_info: &Arc<RwLock<ClusterInfo>>,
|
||||
client_commitment: CommitmentConfig,
|
||||
) -> result::Result<u64, Error> {
|
||||
) -> Result<u64> {
|
||||
let rpc_peers = {
|
||||
let cluster_info = cluster_info.read().unwrap();
|
||||
cluster_info.rpc_peers()
|
||||
|
|
@ -756,12 +717,12 @@ impl Archiver {
|
|||
)
|
||||
.map_err(|err| {
|
||||
warn!("Error while making rpc request {:?}", err);
|
||||
Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
|
||||
ArchiverError::ClientError(err)
|
||||
})?
|
||||
.as_u64()
|
||||
.unwrap())
|
||||
} else {
|
||||
Err(io::Error::new(io::ErrorKind::Other, "No RPC peers...".to_string()).into())
|
||||
Err(ArchiverError::NoRpcPeers)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -771,7 +732,7 @@ impl Archiver {
|
|||
slots_per_segment: u64,
|
||||
previous_blockhash: &Hash,
|
||||
exit: &Arc<AtomicBool>,
|
||||
) -> result::Result<(Hash, u64), Error> {
|
||||
) -> Result<(Hash, u64)> {
|
||||
loop {
|
||||
let (blockhash, turn_slot) = Self::poll_for_blockhash_and_slot(
|
||||
cluster_info,
|
||||
|
|
@ -791,7 +752,7 @@ impl Archiver {
|
|||
slots_per_segment: u64,
|
||||
previous_blockhash: &Hash,
|
||||
exit: &Arc<AtomicBool>,
|
||||
) -> result::Result<(Hash, u64), Error> {
|
||||
) -> Result<(Hash, u64)> {
|
||||
info!("waiting for the next turn...");
|
||||
loop {
|
||||
let rpc_peers = {
|
||||
|
|
@ -812,17 +773,13 @@ impl Archiver {
|
|||
)
|
||||
.map_err(|err| {
|
||||
warn!("Error while making rpc request {:?}", err);
|
||||
Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
|
||||
ArchiverError::ClientError(err)
|
||||
})?;
|
||||
let RpcStorageTurn {
|
||||
blockhash: storage_blockhash,
|
||||
slot: turn_slot,
|
||||
} = serde_json::from_value::<RpcStorageTurn>(response).map_err(|err| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("Couldn't parse response: {:?}", err),
|
||||
)
|
||||
})?;
|
||||
} = serde_json::from_value::<RpcStorageTurn>(response)
|
||||
.map_err(ArchiverError::JsonError)?;
|
||||
let turn_blockhash = storage_blockhash.parse().map_err(|err| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
|
|
@ -840,7 +797,7 @@ impl Archiver {
|
|||
}
|
||||
}
|
||||
if exit.load(Ordering::Relaxed) {
|
||||
return Err(Error::IO(io::Error::new(
|
||||
return Err(ArchiverError::IO(io::Error::new(
|
||||
ErrorKind::Other,
|
||||
"exit signalled...",
|
||||
)));
|
||||
|
|
@ -948,9 +905,7 @@ impl Archiver {
|
|||
|
||||
// check if all the slots in the segment are complete
|
||||
if !Self::segment_complete(start_slot, slots_per_segment, blockstore) {
|
||||
return Err(
|
||||
io::Error::new(ErrorKind::Other, "Unable to download the full segment").into(),
|
||||
);
|
||||
return Err(ArchiverError::SegmentDownloadError);
|
||||
}
|
||||
Ok(start_slot)
|
||||
}
|
||||
|
|
@ -993,74 +948,3 @@ impl Archiver {
|
|||
panic!("Couldn't get segment slot from archiver!");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs::{create_dir_all, remove_file};
|
||||
use std::io::Write;
|
||||
|
||||
fn tmp_file_path(name: &str) -> PathBuf {
|
||||
use std::env;
|
||||
let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
||||
let keypair = Keypair::new();
|
||||
|
||||
let mut path = PathBuf::new();
|
||||
path.push(out_dir);
|
||||
path.push("tmp");
|
||||
create_dir_all(&path).unwrap();
|
||||
|
||||
path.push(format!("{}-{}", name, keypair.pubkey()));
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file() {
|
||||
solana_logger::setup();
|
||||
let in_path = tmp_file_path("test_sample_file_input.txt");
|
||||
let num_strings = 4096;
|
||||
let string = "12foobar";
|
||||
{
|
||||
let mut in_file = File::create(&in_path).unwrap();
|
||||
for _ in 0..num_strings {
|
||||
in_file.write(string.as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
|
||||
let samples: Vec<_> = (0..num_samples).collect();
|
||||
let res = sample_file(&in_path, samples.as_slice());
|
||||
let ref_hash: Hash = Hash::new(&[
|
||||
173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
|
||||
151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
|
||||
]);
|
||||
let res = res.unwrap();
|
||||
assert_eq!(res, ref_hash);
|
||||
|
||||
// Sample just past the end
|
||||
assert!(sample_file(&in_path, &[num_samples]).is_err());
|
||||
remove_file(&in_path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file_invalid_offset() {
|
||||
let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
|
||||
{
|
||||
let mut in_file = File::create(&in_path).unwrap();
|
||||
for _ in 0..4096 {
|
||||
in_file.write("123456foobar".as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
let samples = [0, 200000];
|
||||
let res = sample_file(&in_path, &samples);
|
||||
assert!(res.is_err());
|
||||
remove_file(in_path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file_missing_file() {
|
||||
let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
|
||||
let samples = [0, 5];
|
||||
let res = sample_file(&in_path, &samples);
|
||||
assert!(res.is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
#[macro_use]
|
||||
extern crate serde_derive;
|
||||
|
||||
#[macro_use]
|
||||
extern crate solana_metrics;
|
||||
|
||||
pub mod archiver;
|
||||
mod result;
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
use serde_json;
|
||||
use solana_client::client_error;
|
||||
use solana_ledger::blockstore;
|
||||
use solana_sdk::transport;
|
||||
use std::any::Any;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum ArchiverError {
|
||||
#[error("IO error")]
|
||||
IO(#[from] std::io::Error),
|
||||
|
||||
#[error("blockstore error")]
|
||||
BlockstoreError(#[from] blockstore::BlockstoreError),
|
||||
|
||||
#[error("crossbeam error")]
|
||||
CrossbeamSendError(#[from] crossbeam_channel::SendError<u64>),
|
||||
|
||||
#[error("send error")]
|
||||
SendError(#[from] std::sync::mpsc::SendError<u64>),
|
||||
|
||||
#[error("join error")]
|
||||
JoinError(Box<dyn Any + Send + 'static>),
|
||||
|
||||
#[error("transport error")]
|
||||
TransportError(#[from] transport::TransportError),
|
||||
|
||||
#[error("client error")]
|
||||
ClientError(#[from] client_error::ClientError),
|
||||
|
||||
#[error("Json parsing error")]
|
||||
JsonError(#[from] serde_json::error::Error),
|
||||
|
||||
#[error("Storage account has no balance")]
|
||||
EmptyStorageAccountBalance,
|
||||
|
||||
#[error("No RPC peers..")]
|
||||
NoRpcPeers,
|
||||
|
||||
#[error("Couldn't download full segment")]
|
||||
SegmentDownloadError,
|
||||
}
|
||||
|
||||
impl std::convert::From<Box<dyn Any + Send + 'static>> for ArchiverError {
|
||||
fn from(e: Box<dyn Any + Send + 'static>) -> ArchiverError {
|
||||
ArchiverError::JoinError(e)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
[package]
|
||||
name = "solana-archiver-utils"
|
||||
version = "0.23.0"
|
||||
description = "Solana Archiver Utils"
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://solana.com/"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
log = "0.4.8"
|
||||
rand = "0.6.5"
|
||||
rand_chacha = "0.1.1"
|
||||
solana-chacha = { path = "../chacha", version = "0.23.0" }
|
||||
solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
|
||||
solana-ledger = { path = "../ledger", version = "0.23.0" }
|
||||
solana-logger = { path = "../logger", version = "0.23.0" }
|
||||
solana-perf = { path = "../perf", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
hex = "0.4.0"
|
||||
|
||||
[lib]
|
||||
name = "solana_archiver_utils"
|
||||
|
|
@ -0,0 +1,120 @@
|
|||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
use solana_sdk::hash::{Hash, Hasher};
|
||||
use std::fs::File;
|
||||
use std::io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom};
|
||||
use std::mem::size_of;
|
||||
use std::path::Path;
|
||||
|
||||
pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
|
||||
let in_file = File::open(in_path)?;
|
||||
let metadata = in_file.metadata()?;
|
||||
let mut buffer_file = BufReader::new(in_file);
|
||||
|
||||
let mut hasher = Hasher::default();
|
||||
let sample_size = size_of::<Hash>();
|
||||
let sample_size64 = sample_size as u64;
|
||||
let mut buf = vec![0; sample_size];
|
||||
|
||||
let file_len = metadata.len();
|
||||
if file_len < sample_size64 {
|
||||
return Err(io::Error::new(ErrorKind::Other, "file too short!"));
|
||||
}
|
||||
for offset in sample_offsets {
|
||||
if *offset > (file_len - sample_size64) / sample_size64 {
|
||||
return Err(io::Error::new(ErrorKind::Other, "offset too large"));
|
||||
}
|
||||
buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
|
||||
trace!("sampling @ {} ", *offset);
|
||||
match buffer_file.read(&mut buf) {
|
||||
Ok(size) => {
|
||||
assert_eq!(size, buf.len());
|
||||
hasher.hash(&buf);
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Error sampling file");
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hasher.result())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rand::{thread_rng, Rng};
|
||||
use std::fs::{create_dir_all, remove_file};
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
|
||||
extern crate hex;
|
||||
|
||||
fn tmp_file_path(name: &str) -> PathBuf {
|
||||
use std::env;
|
||||
let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
|
||||
let mut rand_bits = [0u8; 32];
|
||||
thread_rng().fill(&mut rand_bits[..]);
|
||||
|
||||
let mut path = PathBuf::new();
|
||||
path.push(out_dir);
|
||||
path.push("tmp");
|
||||
create_dir_all(&path).unwrap();
|
||||
|
||||
path.push(format!("{}-{:?}", name, hex::encode(rand_bits)));
|
||||
println!("path: {:?}", path);
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file() {
|
||||
solana_logger::setup();
|
||||
let in_path = tmp_file_path("test_sample_file_input.txt");
|
||||
let num_strings = 4096;
|
||||
let string = "12foobar";
|
||||
{
|
||||
let mut in_file = File::create(&in_path).unwrap();
|
||||
for _ in 0..num_strings {
|
||||
in_file.write(string.as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
|
||||
let samples: Vec<_> = (0..num_samples).collect();
|
||||
let res = sample_file(&in_path, samples.as_slice());
|
||||
let ref_hash: Hash = Hash::new(&[
|
||||
173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
|
||||
151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
|
||||
]);
|
||||
let res = res.unwrap();
|
||||
assert_eq!(res, ref_hash);
|
||||
|
||||
// Sample just past the end
|
||||
assert!(sample_file(&in_path, &[num_samples]).is_err());
|
||||
remove_file(&in_path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file_invalid_offset() {
|
||||
let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
|
||||
{
|
||||
let mut in_file = File::create(&in_path).unwrap();
|
||||
for _ in 0..4096 {
|
||||
in_file.write("123456foobar".as_bytes()).unwrap();
|
||||
}
|
||||
}
|
||||
let samples = [0, 200000];
|
||||
let res = sample_file(&in_path, &samples);
|
||||
assert!(res.is_err());
|
||||
remove_file(in_path).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sample_file_missing_file() {
|
||||
let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
|
||||
let samples = [0, 5];
|
||||
let res = sample_file(&in_path, &samples);
|
||||
assert!(res.is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@ solana-clap-utils = { path = "../clap-utils", version = "0.23.0" }
|
|||
solana-core = { path = "../core", version = "0.23.0" }
|
||||
solana-logger = { path = "../logger", version = "0.23.0" }
|
||||
solana-metrics = { path = "../metrics", version = "0.23.0" }
|
||||
solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" }
|
||||
solana-net-utils = { path = "../net-utils", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use clap::{crate_description, crate_name, App, Arg};
|
||||
use console::style;
|
||||
use solana_archiver_lib::archiver::Archiver;
|
||||
use solana_clap_utils::{
|
||||
input_validators::is_keypair,
|
||||
keypair::{
|
||||
|
|
@ -8,7 +9,6 @@ use solana_clap_utils::{
|
|||
},
|
||||
};
|
||||
use solana_core::{
|
||||
archiver::Archiver,
|
||||
cluster_info::{Node, VALIDATOR_PORT_RANGE},
|
||||
contact_info::ContactInfo,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
[package]
|
||||
name = "solana-chacha-cuda"
|
||||
version = "0.23.0"
|
||||
description = "Solana Chacha Cuda APIs"
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://solana.com/"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
log = "0.4.8"
|
||||
solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" }
|
||||
solana-chacha = { path = "../chacha", version = "0.23.0" }
|
||||
solana-ledger = { path = "../ledger", version = "0.23.0" }
|
||||
solana-logger = { path = "../logger", version = "0.23.0" }
|
||||
solana-perf = { path = "../perf", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
hex-literal = "0.2.1"
|
||||
|
||||
[lib]
|
||||
name = "solana_chacha_cuda"
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
// Module used by validators to approve storage mining proofs in parallel using the GPU
|
||||
|
||||
use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
|
||||
use solana_chacha::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
|
||||
use solana_ledger::blockstore::Blockstore;
|
||||
use solana_perf::perf_libs;
|
||||
use solana_sdk::hash::Hash;
|
||||
|
|
@ -113,8 +113,8 @@ pub fn chacha_cbc_encrypt_file_many_keys(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::archiver::sample_file;
|
||||
use crate::chacha::chacha_cbc_encrypt_ledger;
|
||||
use solana_archiver_utils::sample_file;
|
||||
use solana_chacha::chacha::chacha_cbc_encrypt_ledger;
|
||||
use solana_ledger::entry::create_ticks;
|
||||
use solana_ledger::get_tmp_ledger_path;
|
||||
use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT;
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
#[cfg(test)]
|
||||
#[macro_use]
|
||||
extern crate hex_literal;
|
||||
|
||||
pub mod chacha_cuda;
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "solana-chacha"
|
||||
version = "0.23.0"
|
||||
description = "Solana Chacha APIs"
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://solana.com/"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
log = "0.4.8"
|
||||
rand = "0.6.5"
|
||||
rand_chacha = "0.1.1"
|
||||
solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
|
||||
solana-ledger = { path = "../ledger", version = "0.23.0" }
|
||||
solana-logger = { path = "../logger", version = "0.23.0" }
|
||||
solana-perf = { path = "../perf", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
|
||||
[dev-dependencies]
|
||||
hex-literal = "0.2.1"
|
||||
|
||||
[lib]
|
||||
name = "solana_chacha"
|
||||
|
|
@ -74,13 +74,14 @@ pub fn chacha_cbc_encrypt_ledger(
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chacha::chacha_cbc_encrypt_ledger;
|
||||
use crate::gen_keys::GenKeys;
|
||||
use rand::SeedableRng;
|
||||
use rand_chacha::ChaChaRng;
|
||||
use solana_ledger::blockstore::Blockstore;
|
||||
use solana_ledger::entry::Entry;
|
||||
use solana_ledger::get_tmp_ledger_path;
|
||||
use solana_sdk::hash::{hash, Hash, Hasher};
|
||||
use solana_sdk::pubkey::Pubkey;
|
||||
use solana_sdk::signature::KeypairUtil;
|
||||
use solana_sdk::signature::{Keypair, KeypairUtil};
|
||||
use solana_sdk::system_transaction;
|
||||
use std::fs::remove_file;
|
||||
use std::fs::File;
|
||||
|
|
@ -92,8 +93,9 @@ mod tests {
|
|||
let one = hash(&zero.as_ref());
|
||||
|
||||
let seed = [2u8; 32];
|
||||
let mut rnd = GenKeys::new(seed);
|
||||
let keypair = rnd.gen_keypair();
|
||||
|
||||
let mut generator = ChaChaRng::from_seed(seed);
|
||||
let keypair = Keypair::generate(&mut generator);
|
||||
|
||||
let mut id = one;
|
||||
let mut num_hashes = 0;
|
||||
|
|
@ -135,8 +137,9 @@ mod tests {
|
|||
let out_path = tmp_file_path("test_encrypt_ledger");
|
||||
|
||||
let seed = [2u8; 32];
|
||||
let mut rnd = GenKeys::new(seed);
|
||||
let keypair = rnd.gen_keypair();
|
||||
|
||||
let mut generator = ChaChaRng::from_seed(seed);
|
||||
let keypair = Keypair::generate(&mut generator);
|
||||
|
||||
let entries = make_tiny_deterministic_test_entries(slots_per_segment);
|
||||
blockstore
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
#[cfg(test)]
|
||||
#[macro_use]
|
||||
extern crate hex_literal;
|
||||
|
||||
pub mod chacha;
|
||||
|
|
@ -86,7 +86,7 @@ test-stable-perf)
|
|||
fi
|
||||
|
||||
_ cargo +"$rust_stable" build --bins ${V:+--verbose}
|
||||
_ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
|
||||
_ cargo +"$rust_stable" test --package solana-chacha-cuda --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
|
||||
;;
|
||||
test-move)
|
||||
ci/affects-files.sh \
|
||||
|
|
|
|||
|
|
@ -42,7 +42,6 @@ serde_derive = "1.0.103"
|
|||
serde_json = "1.0.44"
|
||||
solana-budget-program = { path = "../programs/budget", version = "0.23.0" }
|
||||
solana-clap-utils = { path = "../clap-utils", version = "0.23.0" }
|
||||
solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
|
||||
solana-client = { path = "../client", version = "0.23.0" }
|
||||
solana-faucet = { path = "../faucet", version = "0.23.0" }
|
||||
ed25519-dalek = "=1.0.0-pre.1"
|
||||
|
|
@ -52,6 +51,7 @@ solana-merkle-tree = { path = "../merkle-tree", version = "0.23.0" }
|
|||
solana-metrics = { path = "../metrics", version = "0.23.0" }
|
||||
solana-measure = { path = "../measure", version = "0.23.0" }
|
||||
solana-net-utils = { path = "../net-utils", version = "0.23.0" }
|
||||
solana-chacha-cuda = { path = "../chacha-cuda", version = "0.23.0" }
|
||||
solana-perf = { path = "../perf", version = "0.23.0" }
|
||||
solana-runtime = { path = "../runtime", version = "0.23.0" }
|
||||
solana-sdk = { path = "../sdk", version = "0.23.0" }
|
||||
|
|
@ -73,7 +73,6 @@ solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.23.0" }
|
|||
reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
|
||||
|
||||
[dev-dependencies]
|
||||
hex-literal = "0.2.1"
|
||||
matches = "0.1.6"
|
||||
reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
|
||||
serial_test = "0.3.2"
|
||||
|
|
|
|||
|
|
@ -7,14 +7,11 @@
|
|||
|
||||
pub mod banking_stage;
|
||||
pub mod broadcast_stage;
|
||||
pub mod chacha;
|
||||
pub mod chacha_cuda;
|
||||
pub mod cluster_info_vote_listener;
|
||||
pub mod commitment;
|
||||
pub mod shred_fetch_stage;
|
||||
#[macro_use]
|
||||
pub mod contact_info;
|
||||
pub mod archiver;
|
||||
pub mod blockstream;
|
||||
pub mod blockstream_service;
|
||||
pub mod cluster_info;
|
||||
|
|
@ -63,10 +60,6 @@ pub mod window_service;
|
|||
#[macro_use]
|
||||
extern crate solana_budget_program;
|
||||
|
||||
#[cfg(test)]
|
||||
#[macro_use]
|
||||
extern crate hex_literal;
|
||||
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
|
|
|
|||
|
|
@ -3,13 +3,13 @@
|
|||
// to submit its proof for mining to be rewarded.
|
||||
|
||||
use crate::{
|
||||
chacha_cuda::chacha_cbc_encrypt_file_many_keys,
|
||||
cluster_info::ClusterInfo,
|
||||
contact_info::ContactInfo,
|
||||
result::{Error, Result},
|
||||
};
|
||||
use rand::{Rng, SeedableRng};
|
||||
use rand_chacha::ChaChaRng;
|
||||
use solana_chacha_cuda::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
|
||||
use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
|
||||
use solana_runtime::{bank::Bank, storage_utils::archiver_accounts};
|
||||
use solana_sdk::{
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ homepage = "https://solana.com/"
|
|||
itertools = "0.8.1"
|
||||
log = "0.4.8"
|
||||
rand = "0.6.5"
|
||||
solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" }
|
||||
solana-config-program = { path = "../programs/config", version = "0.23.0" }
|
||||
solana-core = { path = "../core", version = "0.23.0" }
|
||||
solana-client = { path = "../client", version = "0.23.0" }
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
use crate::cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo};
|
||||
use itertools::izip;
|
||||
use log::*;
|
||||
use solana_archiver_lib::archiver::Archiver;
|
||||
use solana_client::thin_client::{create_client, ThinClient};
|
||||
use solana_core::{
|
||||
archiver::Archiver,
|
||||
cluster_info::{Node, VALIDATOR_PORT_RANGE},
|
||||
contact_info::ContactInfo,
|
||||
genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo},
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
use log::*;
|
||||
use serial_test_derive::serial;
|
||||
use solana_archiver_lib::archiver::Archiver;
|
||||
use solana_client::thin_client::create_client;
|
||||
use solana_core::{
|
||||
archiver::Archiver,
|
||||
cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
|
||||
contact_info::ContactInfo,
|
||||
gossip_service::discover_cluster,
|
||||
|
|
|
|||
Loading…
Reference in New Issue