From 8572b578343f4bb3f74e0dd39879fe3831488aa4 Mon Sep 17 00:00:00 2001 From: sakridge Date: Thu, 16 Jan 2020 08:29:36 -0800 Subject: [PATCH] Refactor chacha cuda to be able to test cuda crate but not in OpenCL (#7685) * Refactor chacha cuda to be able to test cuda crate but not in OpenCL chacha not implemeted in OpenCL * Get off core::Error --- Cargo.lock | 79 +++++++++- Cargo.toml | 4 + archiver-lib/Cargo.toml | 39 +++++ {core => archiver-lib}/src/archiver.rs | 184 +++++------------------ archiver-lib/src/lib.rs | 11 ++ archiver-lib/src/result.rs | 48 ++++++ archiver-utils/Cargo.toml | 26 ++++ archiver-utils/src/lib.rs | 120 +++++++++++++++ archiver/Cargo.toml | 1 + archiver/src/main.rs | 2 +- chacha-cuda/Cargo.toml | 24 +++ {core => chacha-cuda}/src/chacha_cuda.rs | 6 +- chacha-cuda/src/lib.rs | 8 + chacha/Cargo.toml | 25 +++ {core => chacha}/src/chacha.rs | 15 +- chacha/src/lib.rs | 8 + ci/test-stable.sh | 2 +- core/Cargo.toml | 3 +- core/src/lib.rs | 7 - core/src/storage_stage.rs | 2 +- local-cluster/Cargo.toml | 1 + local-cluster/src/local_cluster.rs | 2 +- local-cluster/tests/archiver.rs | 2 +- 23 files changed, 444 insertions(+), 175 deletions(-) create mode 100644 archiver-lib/Cargo.toml rename {core => archiver-lib}/src/archiver.rs (86%) create mode 100644 archiver-lib/src/lib.rs create mode 100644 archiver-lib/src/result.rs create mode 100644 archiver-utils/Cargo.toml create mode 100644 archiver-utils/src/lib.rs create mode 100644 chacha-cuda/Cargo.toml rename {core => chacha-cuda}/src/chacha_cuda.rs (98%) create mode 100644 chacha-cuda/src/lib.rs create mode 100644 chacha/Cargo.toml rename {core => chacha}/src/chacha.rs (94%) create mode 100644 chacha/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 282377c78..dfe5beba3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3409,6 +3409,7 @@ version = "0.23.0" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-archiver-lib 0.23.0", "solana-clap-utils 0.23.0", "solana-core 0.23.0", "solana-logger 0.23.0", @@ -3417,6 +3418,51 @@ dependencies = [ "solana-sdk 0.23.0", ] +[[package]] +name = "solana-archiver-lib" +version = "0.23.0" +dependencies = [ + "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-archiver-utils 0.23.0", + "solana-chacha 0.23.0", + "solana-chacha-sys 0.23.0", + "solana-client 0.23.0", + "solana-core 0.23.0", + "solana-ledger 0.23.0", + "solana-logger 0.23.0", + "solana-metrics 0.23.0", + "solana-net-utils 0.23.0", + "solana-perf 0.23.0", + "solana-sdk 0.23.0", + "solana-storage-program 0.23.0", + "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "solana-archiver-utils" +version = "0.23.0" +dependencies = [ + "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-chacha 0.23.0", + "solana-chacha-sys 0.23.0", + "solana-ledger 0.23.0", + "solana-logger 0.23.0", + "solana-perf 0.23.0", + "solana-sdk 0.23.0", +] + [[package]] name = "solana-banking-bench" version = "0.23.0" @@ -3553,6 +3599,35 @@ dependencies = [ "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "solana-chacha" +version = "0.23.0" +dependencies = [ + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-chacha-sys 0.23.0", + "solana-ledger 0.23.0", + "solana-logger 0.23.0", + "solana-perf 0.23.0", + "solana-sdk 0.23.0", +] + +[[package]] +name = "solana-chacha-cuda" +version = "0.23.0" +dependencies = [ + "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-archiver-utils 0.23.0", + "solana-chacha 0.23.0", + "solana-ledger 0.23.0", + "solana-logger 0.23.0", + "solana-perf 0.23.0", + "solana-sdk 0.23.0", +] + [[package]] name = "solana-chacha-sys" version = "0.23.0" @@ -3661,7 +3736,6 @@ dependencies = [ "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)", "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-core 14.0.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3685,7 +3759,7 @@ dependencies = [ "serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "solana-budget-program 0.23.0", - "solana-chacha-sys 0.23.0", + "solana-chacha-cuda 0.23.0", "solana-clap-utils 0.23.0", "solana-client 0.23.0", "solana-faucet 0.23.0", @@ -3972,6 +4046,7 @@ dependencies = [ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", "serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-archiver-lib 0.23.0", "solana-client 0.23.0", "solana-config-program 0.23.0", "solana-core 0.23.0", diff --git a/Cargo.toml b/Cargo.toml index ac80ab09f..b39e09a54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,8 @@ members = [ "bench-streamer", "bench-tps", "banking-bench", + "chacha", + "chacha-cuda", "chacha-sys", "client", "core", @@ -38,6 +40,8 @@ members = [ "programs/vest", "programs/vote", "archiver", + "archiver-lib", + "archiver-utils", "runtime", "sdk", "sdk-c", diff --git a/archiver-lib/Cargo.toml b/archiver-lib/Cargo.toml new file mode 100644 index 000000000..134627e7c --- /dev/null +++ b/archiver-lib/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "solana-archiver-lib" +version = "0.23.0" +description = "Solana Archiver Library" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" +edition = "2018" + +[dependencies] +bincode = "1.2.1" +crossbeam-channel = "0.3" +ed25519-dalek = "=1.0.0-pre.1" +log = "0.4.8" +rand = "0.6.5" +rand_chacha = "0.1.1" +solana-client = { path = "../client", version = "0.23.0" } +solana-storage-program = { path = "../programs/storage", version = "0.23.0" } +thiserror = "1.0" +serde = "1.0.104" +serde_json = "1.0.44" +serde_derive = "1.0.103" +solana-net-utils = { path = "../net-utils", version = "0.23.0" } +solana-chacha = { path = "../chacha", version = "0.23.0" } +solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" } +solana-ledger = { path = "../ledger", version = "0.23.0" } +solana-logger = { path = "../logger", version = "0.23.0" } +solana-perf = { path = "../perf", version = "0.23.0" } +solana-sdk = { path = "../sdk", version = "0.23.0" } +solana-core = { path = "../core", version = "0.23.0" } +solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" } +solana-metrics = { path = "../metrics", version = "0.23.0" } + +[dev-dependencies] +hex = "0.4.0" + +[lib] +name = "solana_archiver_lib" diff --git a/core/src/archiver.rs b/archiver-lib/src/archiver.rs similarity index 86% rename from core/src/archiver.rs rename to archiver-lib/src/archiver.rs index 2b642cb2f..015e7445e 100644 --- a/core/src/archiver.rs +++ b/archiver-lib/src/archiver.rs @@ -1,26 +1,27 @@ -use crate::{ - chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE}, +use crate::result::ArchiverError; +use crossbeam_channel::unbounded; +use ed25519_dalek; +use rand::{thread_rng, Rng, SeedableRng}; +use rand_chacha::ChaChaRng; +use solana_archiver_utils::sample_file; +use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE}; +use solana_client::{ + rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn, + thin_client::ThinClient, +}; +use solana_core::{ cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, gossip_service::GossipService, packet::{limited_deserialize, PACKET_DATA_SIZE}, repair_service, repair_service::{RepairService, RepairSlotRange, RepairStrategy}, - result::{Error, Result}, shred_fetch_stage::ShredFetchStage, sigverify_stage::{DisabledSigVerifier, SigVerifyStage}, storage_stage::NUM_STORAGE_SAMPLES, streamer::{receiver, responder, PacketReceiver}, window_service::WindowService, }; -use crossbeam_channel::unbounded; -use ed25519_dalek; -use rand::{thread_rng, Rng, SeedableRng}; -use rand_chacha::ChaChaRng; -use solana_client::{ - rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn, - thin_client::ThinClient, -}; use solana_ledger::{ blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred, }; @@ -33,7 +34,7 @@ use solana_sdk::{ client::{AsyncClient, SyncClient}, clock::{get_complete_segment_from_slot, get_segment_from_slot, Slot}, commitment_config::CommitmentConfig, - hash::{Hash, Hasher}, + hash::Hash, message::Message, signature::{Keypair, KeypairUtil, Signature}, timing::timestamp, @@ -45,9 +46,7 @@ use solana_storage_program::{ storage_instruction::{self, StorageAccountType}, }; use std::{ - fs::File, - io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom}, - mem::size_of, + io::{self, ErrorKind}, net::{SocketAddr, UdpSocket}, path::{Path, PathBuf}, result, @@ -58,6 +57,8 @@ use std::{ time::Duration, }; +type Result = std::result::Result; + static ENCRYPTED_FILENAME: &str = "ledger.enc"; #[derive(Serialize, Deserialize)] @@ -85,41 +86,6 @@ struct ArchiverMeta { client_commitment: CommitmentConfig, } -pub(crate) fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result { - let in_file = File::open(in_path)?; - let metadata = in_file.metadata()?; - let mut buffer_file = BufReader::new(in_file); - - let mut hasher = Hasher::default(); - let sample_size = size_of::(); - let sample_size64 = sample_size as u64; - let mut buf = vec![0; sample_size]; - - let file_len = metadata.len(); - if file_len < sample_size64 { - return Err(io::Error::new(ErrorKind::Other, "file too short!")); - } - for offset in sample_offsets { - if *offset > (file_len - sample_size64) / sample_size64 { - return Err(io::Error::new(ErrorKind::Other, "offset too large")); - } - buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?; - trace!("sampling @ {} ", *offset); - match buffer_file.read(&mut buf) { - Ok(size) => { - assert_eq!(size, buf.len()); - hasher.hash(&buf); - } - Err(e) => { - warn!("Error sampling file"); - return Err(e); - } - } - } - - Ok(hasher.result()) -} - fn get_slot_from_signature( signature: &ed25519_dalek::Signature, storage_turn: u64, @@ -239,16 +205,16 @@ impl Archiver { info!("Connecting to the cluster via {:?}", cluster_entrypoint); let (nodes, _) = - match crate::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) { + match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) { Ok(nodes_and_archivers) => nodes_and_archivers, Err(e) => { //shutdown services before exiting exit.store(true, Ordering::Relaxed); gossip_service.join()?; - return Err(Error::from(e)); + return Err(e.into()); } }; - let client = crate::gossip_service::get_client(&nodes); + let client = solana_core::gossip_service::get_client(&nodes); info!("Setting up mining account..."); if let Err(e) = Self::setup_mining_account( @@ -411,7 +377,7 @@ impl Archiver { client_commitment: CommitmentConfig, ) { let nodes = cluster_info.read().unwrap().tvu_peers(); - let client = crate::gossip_service::get_client(&nodes); + let client = solana_core::gossip_service::get_client(&nodes); if let Ok(Some(account)) = client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment.clone()) @@ -622,9 +588,7 @@ impl Archiver { client_commitment.clone(), )? == 0 { - return Err( - io::Error::new(io::ErrorKind::Other, "keypair account has no balance").into(), - ); + return Err(ArchiverError::EmptyStorageAccountBalance); } info!("checking storage account keypair..."); @@ -635,11 +599,8 @@ impl Archiver { let blockhash = match client.get_recent_blockhash_with_commitment(client_commitment.clone()) { Ok((blockhash, _)) => blockhash, - Err(_) => { - return Err(Error::IO(::new( - io::ErrorKind::Other, - "unable to get recent blockhash, can't submit proof", - ))); + Err(e) => { + return Err(ArchiverError::TransportError(e)); } }; @@ -673,7 +634,7 @@ impl Archiver { ) { // No point if we've got no storage account... let nodes = cluster_info.read().unwrap().tvu_peers(); - let client = crate::gossip_service::get_client(&nodes); + let client = solana_core::gossip_service::get_client(&nodes); let storage_balance = client.poll_get_balance_with_commitment( &storage_keypair.pubkey(), meta.client_commitment.clone(), @@ -737,7 +698,7 @@ impl Archiver { fn get_segment_config( cluster_info: &Arc>, client_commitment: CommitmentConfig, - ) -> result::Result { + ) -> Result { let rpc_peers = { let cluster_info = cluster_info.read().unwrap(); cluster_info.rpc_peers() @@ -756,12 +717,12 @@ impl Archiver { ) .map_err(|err| { warn!("Error while making rpc request {:?}", err); - Error::IO(io::Error::new(ErrorKind::Other, "rpc error")) + ArchiverError::ClientError(err) })? .as_u64() .unwrap()) } else { - Err(io::Error::new(io::ErrorKind::Other, "No RPC peers...".to_string()).into()) + Err(ArchiverError::NoRpcPeers) } } @@ -771,7 +732,7 @@ impl Archiver { slots_per_segment: u64, previous_blockhash: &Hash, exit: &Arc, - ) -> result::Result<(Hash, u64), Error> { + ) -> Result<(Hash, u64)> { loop { let (blockhash, turn_slot) = Self::poll_for_blockhash_and_slot( cluster_info, @@ -791,7 +752,7 @@ impl Archiver { slots_per_segment: u64, previous_blockhash: &Hash, exit: &Arc, - ) -> result::Result<(Hash, u64), Error> { + ) -> Result<(Hash, u64)> { info!("waiting for the next turn..."); loop { let rpc_peers = { @@ -812,17 +773,13 @@ impl Archiver { ) .map_err(|err| { warn!("Error while making rpc request {:?}", err); - Error::IO(io::Error::new(ErrorKind::Other, "rpc error")) + ArchiverError::ClientError(err) })?; let RpcStorageTurn { blockhash: storage_blockhash, slot: turn_slot, - } = serde_json::from_value::(response).map_err(|err| { - io::Error::new( - io::ErrorKind::Other, - format!("Couldn't parse response: {:?}", err), - ) - })?; + } = serde_json::from_value::(response) + .map_err(ArchiverError::JsonError)?; let turn_blockhash = storage_blockhash.parse().map_err(|err| { io::Error::new( io::ErrorKind::Other, @@ -840,7 +797,7 @@ impl Archiver { } } if exit.load(Ordering::Relaxed) { - return Err(Error::IO(io::Error::new( + return Err(ArchiverError::IO(io::Error::new( ErrorKind::Other, "exit signalled...", ))); @@ -948,9 +905,7 @@ impl Archiver { // check if all the slots in the segment are complete if !Self::segment_complete(start_slot, slots_per_segment, blockstore) { - return Err( - io::Error::new(ErrorKind::Other, "Unable to download the full segment").into(), - ); + return Err(ArchiverError::SegmentDownloadError); } Ok(start_slot) } @@ -993,74 +948,3 @@ impl Archiver { panic!("Couldn't get segment slot from archiver!"); } } - -#[cfg(test)] -mod tests { - use super::*; - use std::fs::{create_dir_all, remove_file}; - use std::io::Write; - - fn tmp_file_path(name: &str) -> PathBuf { - use std::env; - let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()); - let keypair = Keypair::new(); - - let mut path = PathBuf::new(); - path.push(out_dir); - path.push("tmp"); - create_dir_all(&path).unwrap(); - - path.push(format!("{}-{}", name, keypair.pubkey())); - path - } - - #[test] - fn test_sample_file() { - solana_logger::setup(); - let in_path = tmp_file_path("test_sample_file_input.txt"); - let num_strings = 4096; - let string = "12foobar"; - { - let mut in_file = File::create(&in_path).unwrap(); - for _ in 0..num_strings { - in_file.write(string.as_bytes()).unwrap(); - } - } - let num_samples = (string.len() * num_strings / size_of::()) as u64; - let samples: Vec<_> = (0..num_samples).collect(); - let res = sample_file(&in_path, samples.as_slice()); - let ref_hash: Hash = Hash::new(&[ - 173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144, - 151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52, - ]); - let res = res.unwrap(); - assert_eq!(res, ref_hash); - - // Sample just past the end - assert!(sample_file(&in_path, &[num_samples]).is_err()); - remove_file(&in_path).unwrap(); - } - - #[test] - fn test_sample_file_invalid_offset() { - let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt"); - { - let mut in_file = File::create(&in_path).unwrap(); - for _ in 0..4096 { - in_file.write("123456foobar".as_bytes()).unwrap(); - } - } - let samples = [0, 200000]; - let res = sample_file(&in_path, &samples); - assert!(res.is_err()); - remove_file(in_path).unwrap(); - } - - #[test] - fn test_sample_file_missing_file() { - let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt"); - let samples = [0, 5]; - let res = sample_file(&in_path, &samples); - assert!(res.is_err()); - } -} diff --git a/archiver-lib/src/lib.rs b/archiver-lib/src/lib.rs new file mode 100644 index 000000000..660b11fde --- /dev/null +++ b/archiver-lib/src/lib.rs @@ -0,0 +1,11 @@ +#[macro_use] +extern crate log; + +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate solana_metrics; + +pub mod archiver; +mod result; diff --git a/archiver-lib/src/result.rs b/archiver-lib/src/result.rs new file mode 100644 index 000000000..cb7228de6 --- /dev/null +++ b/archiver-lib/src/result.rs @@ -0,0 +1,48 @@ +use serde_json; +use solana_client::client_error; +use solana_ledger::blockstore; +use solana_sdk::transport; +use std::any::Any; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum ArchiverError { + #[error("IO error")] + IO(#[from] std::io::Error), + + #[error("blockstore error")] + BlockstoreError(#[from] blockstore::BlockstoreError), + + #[error("crossbeam error")] + CrossbeamSendError(#[from] crossbeam_channel::SendError), + + #[error("send error")] + SendError(#[from] std::sync::mpsc::SendError), + + #[error("join error")] + JoinError(Box), + + #[error("transport error")] + TransportError(#[from] transport::TransportError), + + #[error("client error")] + ClientError(#[from] client_error::ClientError), + + #[error("Json parsing error")] + JsonError(#[from] serde_json::error::Error), + + #[error("Storage account has no balance")] + EmptyStorageAccountBalance, + + #[error("No RPC peers..")] + NoRpcPeers, + + #[error("Couldn't download full segment")] + SegmentDownloadError, +} + +impl std::convert::From> for ArchiverError { + fn from(e: Box) -> ArchiverError { + ArchiverError::JoinError(e) + } +} diff --git a/archiver-utils/Cargo.toml b/archiver-utils/Cargo.toml new file mode 100644 index 000000000..5e43f6973 --- /dev/null +++ b/archiver-utils/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "solana-archiver-utils" +version = "0.23.0" +description = "Solana Archiver Utils" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" +edition = "2018" + +[dependencies] +log = "0.4.8" +rand = "0.6.5" +rand_chacha = "0.1.1" +solana-chacha = { path = "../chacha", version = "0.23.0" } +solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" } +solana-ledger = { path = "../ledger", version = "0.23.0" } +solana-logger = { path = "../logger", version = "0.23.0" } +solana-perf = { path = "../perf", version = "0.23.0" } +solana-sdk = { path = "../sdk", version = "0.23.0" } + +[dev-dependencies] +hex = "0.4.0" + +[lib] +name = "solana_archiver_utils" diff --git a/archiver-utils/src/lib.rs b/archiver-utils/src/lib.rs new file mode 100644 index 000000000..b38a36dbe --- /dev/null +++ b/archiver-utils/src/lib.rs @@ -0,0 +1,120 @@ +#[macro_use] +extern crate log; + +use solana_sdk::hash::{Hash, Hasher}; +use std::fs::File; +use std::io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom}; +use std::mem::size_of; +use std::path::Path; + +pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result { + let in_file = File::open(in_path)?; + let metadata = in_file.metadata()?; + let mut buffer_file = BufReader::new(in_file); + + let mut hasher = Hasher::default(); + let sample_size = size_of::(); + let sample_size64 = sample_size as u64; + let mut buf = vec![0; sample_size]; + + let file_len = metadata.len(); + if file_len < sample_size64 { + return Err(io::Error::new(ErrorKind::Other, "file too short!")); + } + for offset in sample_offsets { + if *offset > (file_len - sample_size64) / sample_size64 { + return Err(io::Error::new(ErrorKind::Other, "offset too large")); + } + buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?; + trace!("sampling @ {} ", *offset); + match buffer_file.read(&mut buf) { + Ok(size) => { + assert_eq!(size, buf.len()); + hasher.hash(&buf); + } + Err(e) => { + warn!("Error sampling file"); + return Err(e); + } + } + } + + Ok(hasher.result()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rand::{thread_rng, Rng}; + use std::fs::{create_dir_all, remove_file}; + use std::io::Write; + use std::path::PathBuf; + + extern crate hex; + + fn tmp_file_path(name: &str) -> PathBuf { + use std::env; + let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string()); + let mut rand_bits = [0u8; 32]; + thread_rng().fill(&mut rand_bits[..]); + + let mut path = PathBuf::new(); + path.push(out_dir); + path.push("tmp"); + create_dir_all(&path).unwrap(); + + path.push(format!("{}-{:?}", name, hex::encode(rand_bits))); + println!("path: {:?}", path); + path + } + + #[test] + fn test_sample_file() { + solana_logger::setup(); + let in_path = tmp_file_path("test_sample_file_input.txt"); + let num_strings = 4096; + let string = "12foobar"; + { + let mut in_file = File::create(&in_path).unwrap(); + for _ in 0..num_strings { + in_file.write(string.as_bytes()).unwrap(); + } + } + let num_samples = (string.len() * num_strings / size_of::()) as u64; + let samples: Vec<_> = (0..num_samples).collect(); + let res = sample_file(&in_path, samples.as_slice()); + let ref_hash: Hash = Hash::new(&[ + 173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144, + 151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52, + ]); + let res = res.unwrap(); + assert_eq!(res, ref_hash); + + // Sample just past the end + assert!(sample_file(&in_path, &[num_samples]).is_err()); + remove_file(&in_path).unwrap(); + } + + #[test] + fn test_sample_file_invalid_offset() { + let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt"); + { + let mut in_file = File::create(&in_path).unwrap(); + for _ in 0..4096 { + in_file.write("123456foobar".as_bytes()).unwrap(); + } + } + let samples = [0, 200000]; + let res = sample_file(&in_path, &samples); + assert!(res.is_err()); + remove_file(in_path).unwrap(); + } + + #[test] + fn test_sample_file_missing_file() { + let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt"); + let samples = [0, 5]; + let res = sample_file(&in_path, &samples); + assert!(res.is_err()); + } +} diff --git a/archiver/Cargo.toml b/archiver/Cargo.toml index 2c3c22247..ea6b0445c 100644 --- a/archiver/Cargo.toml +++ b/archiver/Cargo.toml @@ -14,6 +14,7 @@ solana-clap-utils = { path = "../clap-utils", version = "0.23.0" } solana-core = { path = "../core", version = "0.23.0" } solana-logger = { path = "../logger", version = "0.23.0" } solana-metrics = { path = "../metrics", version = "0.23.0" } +solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" } solana-net-utils = { path = "../net-utils", version = "0.23.0" } solana-sdk = { path = "../sdk", version = "0.23.0" } diff --git a/archiver/src/main.rs b/archiver/src/main.rs index 28d4c65a0..ed6ba4bb8 100644 --- a/archiver/src/main.rs +++ b/archiver/src/main.rs @@ -1,5 +1,6 @@ use clap::{crate_description, crate_name, App, Arg}; use console::style; +use solana_archiver_lib::archiver::Archiver; use solana_clap_utils::{ input_validators::is_keypair, keypair::{ @@ -8,7 +9,6 @@ use solana_clap_utils::{ }, }; use solana_core::{ - archiver::Archiver, cluster_info::{Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, }; diff --git a/chacha-cuda/Cargo.toml b/chacha-cuda/Cargo.toml new file mode 100644 index 000000000..e47a2742e --- /dev/null +++ b/chacha-cuda/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "solana-chacha-cuda" +version = "0.23.0" +description = "Solana Chacha Cuda APIs" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" +edition = "2018" + +[dependencies] +log = "0.4.8" +solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" } +solana-chacha = { path = "../chacha", version = "0.23.0" } +solana-ledger = { path = "../ledger", version = "0.23.0" } +solana-logger = { path = "../logger", version = "0.23.0" } +solana-perf = { path = "../perf", version = "0.23.0" } +solana-sdk = { path = "../sdk", version = "0.23.0" } + +[dev-dependencies] +hex-literal = "0.2.1" + +[lib] +name = "solana_chacha_cuda" diff --git a/core/src/chacha_cuda.rs b/chacha-cuda/src/chacha_cuda.rs similarity index 98% rename from core/src/chacha_cuda.rs rename to chacha-cuda/src/chacha_cuda.rs index 3eb579be4..b49c9228e 100644 --- a/core/src/chacha_cuda.rs +++ b/chacha-cuda/src/chacha_cuda.rs @@ -1,6 +1,6 @@ // Module used by validators to approve storage mining proofs in parallel using the GPU -use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE}; +use solana_chacha::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE}; use solana_ledger::blockstore::Blockstore; use solana_perf::perf_libs; use solana_sdk::hash::Hash; @@ -113,8 +113,8 @@ pub fn chacha_cbc_encrypt_file_many_keys( #[cfg(test)] mod tests { use super::*; - use crate::archiver::sample_file; - use crate::chacha::chacha_cbc_encrypt_ledger; + use solana_archiver_utils::sample_file; + use solana_chacha::chacha::chacha_cbc_encrypt_ledger; use solana_ledger::entry::create_ticks; use solana_ledger::get_tmp_ledger_path; use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT; diff --git a/chacha-cuda/src/lib.rs b/chacha-cuda/src/lib.rs new file mode 100644 index 000000000..bffbdceed --- /dev/null +++ b/chacha-cuda/src/lib.rs @@ -0,0 +1,8 @@ +#[macro_use] +extern crate log; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +pub mod chacha_cuda; diff --git a/chacha/Cargo.toml b/chacha/Cargo.toml new file mode 100644 index 000000000..0b017d0fd --- /dev/null +++ b/chacha/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "solana-chacha" +version = "0.23.0" +description = "Solana Chacha APIs" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +license = "Apache-2.0" +homepage = "https://solana.com/" +edition = "2018" + +[dependencies] +log = "0.4.8" +rand = "0.6.5" +rand_chacha = "0.1.1" +solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" } +solana-ledger = { path = "../ledger", version = "0.23.0" } +solana-logger = { path = "../logger", version = "0.23.0" } +solana-perf = { path = "../perf", version = "0.23.0" } +solana-sdk = { path = "../sdk", version = "0.23.0" } + +[dev-dependencies] +hex-literal = "0.2.1" + +[lib] +name = "solana_chacha" diff --git a/core/src/chacha.rs b/chacha/src/chacha.rs similarity index 94% rename from core/src/chacha.rs rename to chacha/src/chacha.rs index baa29fdbe..d6ab1ddf2 100644 --- a/core/src/chacha.rs +++ b/chacha/src/chacha.rs @@ -74,13 +74,14 @@ pub fn chacha_cbc_encrypt_ledger( #[cfg(test)] mod tests { use crate::chacha::chacha_cbc_encrypt_ledger; - use crate::gen_keys::GenKeys; + use rand::SeedableRng; + use rand_chacha::ChaChaRng; use solana_ledger::blockstore::Blockstore; use solana_ledger::entry::Entry; use solana_ledger::get_tmp_ledger_path; use solana_sdk::hash::{hash, Hash, Hasher}; use solana_sdk::pubkey::Pubkey; - use solana_sdk::signature::KeypairUtil; + use solana_sdk::signature::{Keypair, KeypairUtil}; use solana_sdk::system_transaction; use std::fs::remove_file; use std::fs::File; @@ -92,8 +93,9 @@ mod tests { let one = hash(&zero.as_ref()); let seed = [2u8; 32]; - let mut rnd = GenKeys::new(seed); - let keypair = rnd.gen_keypair(); + + let mut generator = ChaChaRng::from_seed(seed); + let keypair = Keypair::generate(&mut generator); let mut id = one; let mut num_hashes = 0; @@ -135,8 +137,9 @@ mod tests { let out_path = tmp_file_path("test_encrypt_ledger"); let seed = [2u8; 32]; - let mut rnd = GenKeys::new(seed); - let keypair = rnd.gen_keypair(); + + let mut generator = ChaChaRng::from_seed(seed); + let keypair = Keypair::generate(&mut generator); let entries = make_tiny_deterministic_test_entries(slots_per_segment); blockstore diff --git a/chacha/src/lib.rs b/chacha/src/lib.rs new file mode 100644 index 000000000..5f1e27d34 --- /dev/null +++ b/chacha/src/lib.rs @@ -0,0 +1,8 @@ +#[macro_use] +extern crate log; + +#[cfg(test)] +#[macro_use] +extern crate hex_literal; + +pub mod chacha; diff --git a/ci/test-stable.sh b/ci/test-stable.sh index 1c039a6e2..4c4ae41b5 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -86,7 +86,7 @@ test-stable-perf) fi _ cargo +"$rust_stable" build --bins ${V:+--verbose} - _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture + _ cargo +"$rust_stable" test --package solana-chacha-cuda --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture ;; test-move) ci/affects-files.sh \ diff --git a/core/Cargo.toml b/core/Cargo.toml index d5a50ba01..86c370136 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -42,7 +42,6 @@ serde_derive = "1.0.103" serde_json = "1.0.44" solana-budget-program = { path = "../programs/budget", version = "0.23.0" } solana-clap-utils = { path = "../clap-utils", version = "0.23.0" } -solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" } solana-client = { path = "../client", version = "0.23.0" } solana-faucet = { path = "../faucet", version = "0.23.0" } ed25519-dalek = "=1.0.0-pre.1" @@ -52,6 +51,7 @@ solana-merkle-tree = { path = "../merkle-tree", version = "0.23.0" } solana-metrics = { path = "../metrics", version = "0.23.0" } solana-measure = { path = "../measure", version = "0.23.0" } solana-net-utils = { path = "../net-utils", version = "0.23.0" } +solana-chacha-cuda = { path = "../chacha-cuda", version = "0.23.0" } solana-perf = { path = "../perf", version = "0.23.0" } solana-runtime = { path = "../runtime", version = "0.23.0" } solana-sdk = { path = "../sdk", version = "0.23.0" } @@ -73,7 +73,6 @@ solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.23.0" } reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] } [dev-dependencies] -hex-literal = "0.2.1" matches = "0.1.6" reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] } serial_test = "0.3.2" diff --git a/core/src/lib.rs b/core/src/lib.rs index 9639978f8..caa18dad6 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -7,14 +7,11 @@ pub mod banking_stage; pub mod broadcast_stage; -pub mod chacha; -pub mod chacha_cuda; pub mod cluster_info_vote_listener; pub mod commitment; pub mod shred_fetch_stage; #[macro_use] pub mod contact_info; -pub mod archiver; pub mod blockstream; pub mod blockstream_service; pub mod cluster_info; @@ -63,10 +60,6 @@ pub mod window_service; #[macro_use] extern crate solana_budget_program; -#[cfg(test)] -#[macro_use] -extern crate hex_literal; - #[macro_use] extern crate log; diff --git a/core/src/storage_stage.rs b/core/src/storage_stage.rs index 146e01106..bbb20408c 100644 --- a/core/src/storage_stage.rs +++ b/core/src/storage_stage.rs @@ -3,13 +3,13 @@ // to submit its proof for mining to be rewarded. use crate::{ - chacha_cuda::chacha_cbc_encrypt_file_many_keys, cluster_info::ClusterInfo, contact_info::ContactInfo, result::{Error, Result}, }; use rand::{Rng, SeedableRng}; use rand_chacha::ChaChaRng; +use solana_chacha_cuda::chacha_cuda::chacha_cbc_encrypt_file_many_keys; use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore}; use solana_runtime::{bank::Bank, storage_utils::archiver_accounts}; use solana_sdk::{ diff --git a/local-cluster/Cargo.toml b/local-cluster/Cargo.toml index 09a5232d1..b95ebd713 100644 --- a/local-cluster/Cargo.toml +++ b/local-cluster/Cargo.toml @@ -12,6 +12,7 @@ homepage = "https://solana.com/" itertools = "0.8.1" log = "0.4.8" rand = "0.6.5" +solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" } solana-config-program = { path = "../programs/config", version = "0.23.0" } solana-core = { path = "../core", version = "0.23.0" } solana-client = { path = "../client", version = "0.23.0" } diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index 7bbcc6ed0..a7bedf2b3 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -1,9 +1,9 @@ use crate::cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo}; use itertools::izip; use log::*; +use solana_archiver_lib::archiver::Archiver; use solana_client::thin_client::{create_client, ThinClient}; use solana_core::{ - archiver::Archiver, cluster_info::{Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, diff --git a/local-cluster/tests/archiver.rs b/local-cluster/tests/archiver.rs index 51874b744..91562a44c 100644 --- a/local-cluster/tests/archiver.rs +++ b/local-cluster/tests/archiver.rs @@ -1,8 +1,8 @@ use log::*; use serial_test_derive::serial; +use solana_archiver_lib::archiver::Archiver; use solana_client::thin_client::create_client; use solana_core::{ - archiver::Archiver, cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE}, contact_info::ContactInfo, gossip_service::discover_cluster,