Refactor chacha cuda to be able to test cuda crate but not in OpenCL (#7685)

* Refactor chacha cuda to be able to test cuda crate but not in OpenCL

chacha not implemeted in OpenCL

* Get off core::Error

Cargo.lock

@@ -3409,6 +3409,7 @@ version = "0.23.0"
 dependencies = [
  "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "console 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-archiver-lib 0.23.0",
  "solana-clap-utils 0.23.0",
  "solana-core 0.23.0",
  "solana-logger 0.23.0",
@@ -3417,6 +3418,51 @@ dependencies = [
  "solana-sdk 0.23.0",
 ]
 
+[[package]]
+name = "solana-archiver-lib"
+version = "0.23.0"
+dependencies = [
+ "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
+ "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.44 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-archiver-utils 0.23.0",
+ "solana-chacha 0.23.0",
+ "solana-chacha-sys 0.23.0",
+ "solana-client 0.23.0",
+ "solana-core 0.23.0",
+ "solana-ledger 0.23.0",
+ "solana-logger 0.23.0",
+ "solana-metrics 0.23.0",
+ "solana-net-utils 0.23.0",
+ "solana-perf 0.23.0",
+ "solana-sdk 0.23.0",
+ "solana-storage-program 0.23.0",
+ "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "solana-archiver-utils"
+version = "0.23.0"
+dependencies = [
+ "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-chacha 0.23.0",
+ "solana-chacha-sys 0.23.0",
+ "solana-ledger 0.23.0",
+ "solana-logger 0.23.0",
+ "solana-perf 0.23.0",
+ "solana-sdk 0.23.0",
+]
+
 [[package]]
 name = "solana-banking-bench"
 version = "0.23.0"
@@ -3553,6 +3599,35 @@ dependencies = [
  "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "solana-chacha"
+version = "0.23.0"
+dependencies = [
+ "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-chacha-sys 0.23.0",
+ "solana-ledger 0.23.0",
+ "solana-logger 0.23.0",
+ "solana-perf 0.23.0",
+ "solana-sdk 0.23.0",
+]
+
+[[package]]
+name = "solana-chacha-cuda"
+version = "0.23.0"
+dependencies = [
+ "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-archiver-utils 0.23.0",
+ "solana-chacha 0.23.0",
+ "solana-ledger 0.23.0",
+ "solana-logger 0.23.0",
+ "solana-perf 0.23.0",
+ "solana-sdk 0.23.0",
+]
+
 [[package]]
 name = "solana-chacha-sys"
 version = "0.23.0"
@@ -3661,7 +3736,6 @@ dependencies = [
  "crossbeam-channel 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
  "ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "fs_extra 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "hex-literal 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "jsonrpc-core 14.0.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3685,7 +3759,7 @@ dependencies = [
  "serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "solana-budget-program 0.23.0",
- "solana-chacha-sys 0.23.0",
+ "solana-chacha-cuda 0.23.0",
  "solana-clap-utils 0.23.0",
  "solana-client 0.23.0",
  "solana-faucet 0.23.0",
@@ -3972,6 +4046,7 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "serial_test 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "serial_test_derive 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "solana-archiver-lib 0.23.0",
  "solana-client 0.23.0",
  "solana-config-program 0.23.0",
  "solana-core 0.23.0",

Cargo.toml

@@ -4,6 +4,8 @@ members = [
     "bench-streamer",
     "bench-tps",
     "banking-bench",
+    "chacha",
+    "chacha-cuda",
     "chacha-sys",
     "client",
     "core",
@@ -38,6 +40,8 @@ members = [
     "programs/vest",
     "programs/vote",
     "archiver",
+    "archiver-lib",
+    "archiver-utils",
     "runtime",
     "sdk",
     "sdk-c",

archiver-lib/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "solana-archiver-lib"
+version = "0.23.0"
+description = "Solana Archiver Library"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+bincode = "1.2.1"
+crossbeam-channel = "0.3"
+ed25519-dalek = "=1.0.0-pre.1"
+log = "0.4.8"
+rand = "0.6.5"
+rand_chacha = "0.1.1"
+solana-client = { path = "../client", version = "0.23.0" }
+solana-storage-program = { path = "../programs/storage", version = "0.23.0" }
+thiserror = "1.0"
+serde = "1.0.104"
+serde_json = "1.0.44"
+serde_derive = "1.0.103"
+solana-net-utils = { path = "../net-utils", version = "0.23.0" }
+solana-chacha = { path = "../chacha", version = "0.23.0" }
+solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
+solana-ledger = { path = "../ledger", version = "0.23.0" }
+solana-logger = { path = "../logger", version = "0.23.0" }
+solana-perf = { path = "../perf", version = "0.23.0" }
+solana-sdk = { path = "../sdk", version = "0.23.0" }
+solana-core = { path = "../core", version = "0.23.0" }
+solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" }
+solana-metrics = { path = "../metrics", version = "0.23.0" }
+
+[dev-dependencies]
+hex = "0.4.0"
+
+[lib]
+name = "solana_archiver_lib"

archiver-lib/src/archiver.rs

@@ -1,26 +1,27 @@
-use crate::{
+use crate::result::ArchiverError;
-    chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE},
+use crossbeam_channel::unbounded;
+use ed25519_dalek;
+use rand::{thread_rng, Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use solana_archiver_utils::sample_file;
+use solana_chacha::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
+use solana_client::{
+    rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
+    thin_client::ThinClient,
+};
+use solana_core::{
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
     gossip_service::GossipService,
     packet::{limited_deserialize, PACKET_DATA_SIZE},
     repair_service,
     repair_service::{RepairService, RepairSlotRange, RepairStrategy},
-    result::{Error, Result},
     shred_fetch_stage::ShredFetchStage,
     sigverify_stage::{DisabledSigVerifier, SigVerifyStage},
     storage_stage::NUM_STORAGE_SAMPLES,
     streamer::{receiver, responder, PacketReceiver},
     window_service::WindowService,
 };
-use crossbeam_channel::unbounded;
-use ed25519_dalek;
-use rand::{thread_rng, Rng, SeedableRng};
-use rand_chacha::ChaChaRng;
-use solana_client::{
-    rpc_client::RpcClient, rpc_request::RpcRequest, rpc_response::RpcStorageTurn,
-    thin_client::ThinClient,
-};
 use solana_ledger::{
     blockstore::Blockstore, leader_schedule_cache::LeaderScheduleCache, shred::Shred,
 };
@@ -33,7 +34,7 @@ use solana_sdk::{
     client::{AsyncClient, SyncClient},
     clock::{get_complete_segment_from_slot, get_segment_from_slot, Slot},
     commitment_config::CommitmentConfig,
-    hash::{Hash, Hasher},
+    hash::Hash,
     message::Message,
     signature::{Keypair, KeypairUtil, Signature},
     timing::timestamp,
@@ -45,9 +46,7 @@ use solana_storage_program::{
     storage_instruction::{self, StorageAccountType},
 };
 use std::{
-    fs::File,
+    io::{self, ErrorKind},
-    io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom},
-    mem::size_of,
     net::{SocketAddr, UdpSocket},
     path::{Path, PathBuf},
     result,
@@ -58,6 +57,8 @@ use std::{
     time::Duration,
 };
 
+type Result<T> = std::result::Result<T, ArchiverError>;
+
 static ENCRYPTED_FILENAME: &str = "ledger.enc";
 
 #[derive(Serialize, Deserialize)]
@@ -85,41 +86,6 @@ struct ArchiverMeta {
     client_commitment: CommitmentConfig,
 }
 
-pub(crate) fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
-    let in_file = File::open(in_path)?;
-    let metadata = in_file.metadata()?;
-    let mut buffer_file = BufReader::new(in_file);
-
-    let mut hasher = Hasher::default();
-    let sample_size = size_of::<Hash>();
-    let sample_size64 = sample_size as u64;
-    let mut buf = vec![0; sample_size];
-
-    let file_len = metadata.len();
-    if file_len < sample_size64 {
-        return Err(io::Error::new(ErrorKind::Other, "file too short!"));
-    }
-    for offset in sample_offsets {
-        if *offset > (file_len - sample_size64) / sample_size64 {
-            return Err(io::Error::new(ErrorKind::Other, "offset too large"));
-        }
-        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
-        trace!("sampling @ {} ", *offset);
-        match buffer_file.read(&mut buf) {
-            Ok(size) => {
-                assert_eq!(size, buf.len());
-                hasher.hash(&buf);
-            }
-            Err(e) => {
-                warn!("Error sampling file");
-                return Err(e);
-            }
-        }
-    }
-
-    Ok(hasher.result())
-}
-
 fn get_slot_from_signature(
     signature: &ed25519_dalek::Signature,
     storage_turn: u64,
@@ -239,16 +205,16 @@ impl Archiver {
 
         info!("Connecting to the cluster via {:?}", cluster_entrypoint);
         let (nodes, _) =
-            match crate::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
+            match solana_core::gossip_service::discover_cluster(&cluster_entrypoint.gossip, 1) {
                 Ok(nodes_and_archivers) => nodes_and_archivers,
                 Err(e) => {
                     //shutdown services before exiting
                     exit.store(true, Ordering::Relaxed);
                     gossip_service.join()?;
-                    return Err(Error::from(e));
+                    return Err(e.into());
                 }
             };
-        let client = crate::gossip_service::get_client(&nodes);
+        let client = solana_core::gossip_service::get_client(&nodes);
 
         info!("Setting up mining account...");
         if let Err(e) = Self::setup_mining_account(
@@ -411,7 +377,7 @@ impl Archiver {
         client_commitment: CommitmentConfig,
     ) {
         let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = crate::gossip_service::get_client(&nodes);
+        let client = solana_core::gossip_service::get_client(&nodes);
 
         if let Ok(Some(account)) =
             client.get_account_with_commitment(&storage_keypair.pubkey(), client_commitment.clone())
@@ -622,9 +588,7 @@ impl Archiver {
             client_commitment.clone(),
         )? == 0
         {
-            return Err(
+            return Err(ArchiverError::EmptyStorageAccountBalance);
-                io::Error::new(io::ErrorKind::Other, "keypair account has no balance").into(),
-            );
         }
 
         info!("checking storage account keypair...");
@@ -635,11 +599,8 @@ impl Archiver {
             let blockhash =
                 match client.get_recent_blockhash_with_commitment(client_commitment.clone()) {
                     Ok((blockhash, _)) => blockhash,
-                    Err(_) => {
+                    Err(e) => {
-                        return Err(Error::IO(<io::Error>::new(
+                        return Err(ArchiverError::TransportError(e));
-                            io::ErrorKind::Other,
-                            "unable to get recent blockhash, can't submit proof",
-                        )));
                     }
                 };
 
@@ -673,7 +634,7 @@ impl Archiver {
     ) {
         // No point if we've got no storage account...
         let nodes = cluster_info.read().unwrap().tvu_peers();
-        let client = crate::gossip_service::get_client(&nodes);
+        let client = solana_core::gossip_service::get_client(&nodes);
         let storage_balance = client.poll_get_balance_with_commitment(
             &storage_keypair.pubkey(),
             meta.client_commitment.clone(),
@@ -737,7 +698,7 @@ impl Archiver {
     fn get_segment_config(
         cluster_info: &Arc<RwLock<ClusterInfo>>,
         client_commitment: CommitmentConfig,
-    ) -> result::Result<u64, Error> {
+    ) -> Result<u64> {
         let rpc_peers = {
             let cluster_info = cluster_info.read().unwrap();
             cluster_info.rpc_peers()
@@ -756,12 +717,12 @@ impl Archiver {
                 )
                 .map_err(|err| {
                     warn!("Error while making rpc request {:?}", err);
-                    Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
+                    ArchiverError::ClientError(err)
                 })?
                 .as_u64()
                 .unwrap())
         } else {
-            Err(io::Error::new(io::ErrorKind::Other, "No RPC peers...".to_string()).into())
+            Err(ArchiverError::NoRpcPeers)
         }
     }
 
@@ -771,7 +732,7 @@ impl Archiver {
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
-    ) -> result::Result<(Hash, u64), Error> {
+    ) -> Result<(Hash, u64)> {
         loop {
             let (blockhash, turn_slot) = Self::poll_for_blockhash_and_slot(
                 cluster_info,
@@ -791,7 +752,7 @@ impl Archiver {
         slots_per_segment: u64,
         previous_blockhash: &Hash,
         exit: &Arc<AtomicBool>,
-    ) -> result::Result<(Hash, u64), Error> {
+    ) -> Result<(Hash, u64)> {
         info!("waiting for the next turn...");
         loop {
             let rpc_peers = {
@@ -812,17 +773,13 @@ impl Archiver {
                     )
                     .map_err(|err| {
                         warn!("Error while making rpc request {:?}", err);
-                        Error::IO(io::Error::new(ErrorKind::Other, "rpc error"))
+                        ArchiverError::ClientError(err)
                     })?;
                 let RpcStorageTurn {
                     blockhash: storage_blockhash,
                     slot: turn_slot,
-                } = serde_json::from_value::<RpcStorageTurn>(response).map_err(|err| {
+                } = serde_json::from_value::<RpcStorageTurn>(response)
-                    io::Error::new(
+                    .map_err(ArchiverError::JsonError)?;
-                        io::ErrorKind::Other,
-                        format!("Couldn't parse response: {:?}", err),
-                    )
-                })?;
                 let turn_blockhash = storage_blockhash.parse().map_err(|err| {
                     io::Error::new(
                         io::ErrorKind::Other,
@@ -840,7 +797,7 @@ impl Archiver {
                 }
             }
             if exit.load(Ordering::Relaxed) {
-                return Err(Error::IO(io::Error::new(
+                return Err(ArchiverError::IO(io::Error::new(
                     ErrorKind::Other,
                     "exit signalled...",
                 )));
@@ -948,9 +905,7 @@ impl Archiver {
 
         // check if all the slots in the segment are complete
         if !Self::segment_complete(start_slot, slots_per_segment, blockstore) {
-            return Err(
+            return Err(ArchiverError::SegmentDownloadError);
-                io::Error::new(ErrorKind::Other, "Unable to download the full segment").into(),
-            );
         }
         Ok(start_slot)
     }
@@ -993,74 +948,3 @@ impl Archiver {
         panic!("Couldn't get segment slot from archiver!");
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::fs::{create_dir_all, remove_file};
-    use std::io::Write;
-
-    fn tmp_file_path(name: &str) -> PathBuf {
-        use std::env;
-        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
-        let keypair = Keypair::new();
-
-        let mut path = PathBuf::new();
-        path.push(out_dir);
-        path.push("tmp");
-        create_dir_all(&path).unwrap();
-
-        path.push(format!("{}-{}", name, keypair.pubkey()));
-        path
-    }
-
-    #[test]
-    fn test_sample_file() {
-        solana_logger::setup();
-        let in_path = tmp_file_path("test_sample_file_input.txt");
-        let num_strings = 4096;
-        let string = "12foobar";
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..num_strings {
-                in_file.write(string.as_bytes()).unwrap();
-            }
-        }
-        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
-        let samples: Vec<_> = (0..num_samples).collect();
-        let res = sample_file(&in_path, samples.as_slice());
-        let ref_hash: Hash = Hash::new(&[
-            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
-            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
-        ]);
-        let res = res.unwrap();
-        assert_eq!(res, ref_hash);
-
-        // Sample just past the end
-        assert!(sample_file(&in_path, &[num_samples]).is_err());
-        remove_file(&in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_invalid_offset() {
-        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
-        {
-            let mut in_file = File::create(&in_path).unwrap();
-            for _ in 0..4096 {
-                in_file.write("123456foobar".as_bytes()).unwrap();
-            }
-        }
-        let samples = [0, 200000];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-        remove_file(in_path).unwrap();
-    }
-
-    #[test]
-    fn test_sample_file_missing_file() {
-        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
-        let samples = [0, 5];
-        let res = sample_file(&in_path, &samples);
-        assert!(res.is_err());
-    }
-}

archiver-lib/src/lib.rs

@@ -0,0 +1,11 @@
+#[macro_use]
+extern crate log;
+
+#[macro_use]
+extern crate serde_derive;
+
+#[macro_use]
+extern crate solana_metrics;
+
+pub mod archiver;
+mod result;

archiver-lib/src/result.rs

@@ -0,0 +1,48 @@
+use serde_json;
+use solana_client::client_error;
+use solana_ledger::blockstore;
+use solana_sdk::transport;
+use std::any::Any;
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum ArchiverError {
+    #[error("IO error")]
+    IO(#[from] std::io::Error),
+
+    #[error("blockstore error")]
+    BlockstoreError(#[from] blockstore::BlockstoreError),
+
+    #[error("crossbeam error")]
+    CrossbeamSendError(#[from] crossbeam_channel::SendError<u64>),
+
+    #[error("send error")]
+    SendError(#[from] std::sync::mpsc::SendError<u64>),
+
+    #[error("join error")]
+    JoinError(Box<dyn Any + Send + 'static>),
+
+    #[error("transport error")]
+    TransportError(#[from] transport::TransportError),
+
+    #[error("client error")]
+    ClientError(#[from] client_error::ClientError),
+
+    #[error("Json parsing error")]
+    JsonError(#[from] serde_json::error::Error),
+
+    #[error("Storage account has no balance")]
+    EmptyStorageAccountBalance,
+
+    #[error("No RPC peers..")]
+    NoRpcPeers,
+
+    #[error("Couldn't download full segment")]
+    SegmentDownloadError,
+}
+
+impl std::convert::From<Box<dyn Any + Send + 'static>> for ArchiverError {
+    fn from(e: Box<dyn Any + Send + 'static>) -> ArchiverError {
+        ArchiverError::JoinError(e)
+    }
+}

archiver-utils/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "solana-archiver-utils"
+version = "0.23.0"
+description = "Solana Archiver Utils"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+rand = "0.6.5"
+rand_chacha = "0.1.1"
+solana-chacha = { path = "../chacha", version = "0.23.0" }
+solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
+solana-ledger = { path = "../ledger", version = "0.23.0" }
+solana-logger = { path = "../logger", version = "0.23.0" }
+solana-perf = { path = "../perf", version = "0.23.0" }
+solana-sdk = { path = "../sdk", version = "0.23.0" }
+
+[dev-dependencies]
+hex = "0.4.0"
+
+[lib]
+name = "solana_archiver_utils"

archiver-utils/src/lib.rs

@@ -0,0 +1,120 @@
+#[macro_use]
+extern crate log;
+
+use solana_sdk::hash::{Hash, Hasher};
+use std::fs::File;
+use std::io::{self, BufReader, ErrorKind, Read, Seek, SeekFrom};
+use std::mem::size_of;
+use std::path::Path;
+
+pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
+    let in_file = File::open(in_path)?;
+    let metadata = in_file.metadata()?;
+    let mut buffer_file = BufReader::new(in_file);
+
+    let mut hasher = Hasher::default();
+    let sample_size = size_of::<Hash>();
+    let sample_size64 = sample_size as u64;
+    let mut buf = vec![0; sample_size];
+
+    let file_len = metadata.len();
+    if file_len < sample_size64 {
+        return Err(io::Error::new(ErrorKind::Other, "file too short!"));
+    }
+    for offset in sample_offsets {
+        if *offset > (file_len - sample_size64) / sample_size64 {
+            return Err(io::Error::new(ErrorKind::Other, "offset too large"));
+        }
+        buffer_file.seek(SeekFrom::Start(*offset * sample_size64))?;
+        trace!("sampling @ {} ", *offset);
+        match buffer_file.read(&mut buf) {
+            Ok(size) => {
+                assert_eq!(size, buf.len());
+                hasher.hash(&buf);
+            }
+            Err(e) => {
+                warn!("Error sampling file");
+                return Err(e);
+            }
+        }
+    }
+
+    Ok(hasher.result())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use rand::{thread_rng, Rng};
+    use std::fs::{create_dir_all, remove_file};
+    use std::io::Write;
+    use std::path::PathBuf;
+
+    extern crate hex;
+
+    fn tmp_file_path(name: &str) -> PathBuf {
+        use std::env;
+        let out_dir = env::var("FARF_DIR").unwrap_or_else(|_| "farf".to_string());
+        let mut rand_bits = [0u8; 32];
+        thread_rng().fill(&mut rand_bits[..]);
+
+        let mut path = PathBuf::new();
+        path.push(out_dir);
+        path.push("tmp");
+        create_dir_all(&path).unwrap();
+
+        path.push(format!("{}-{:?}", name, hex::encode(rand_bits)));
+        println!("path: {:?}", path);
+        path
+    }
+
+    #[test]
+    fn test_sample_file() {
+        solana_logger::setup();
+        let in_path = tmp_file_path("test_sample_file_input.txt");
+        let num_strings = 4096;
+        let string = "12foobar";
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..num_strings {
+                in_file.write(string.as_bytes()).unwrap();
+            }
+        }
+        let num_samples = (string.len() * num_strings / size_of::<Hash>()) as u64;
+        let samples: Vec<_> = (0..num_samples).collect();
+        let res = sample_file(&in_path, samples.as_slice());
+        let ref_hash: Hash = Hash::new(&[
+            173, 251, 182, 165, 10, 54, 33, 150, 133, 226, 106, 150, 99, 192, 179, 1, 230, 144,
+            151, 126, 18, 191, 54, 67, 249, 140, 230, 160, 56, 30, 170, 52,
+        ]);
+        let res = res.unwrap();
+        assert_eq!(res, ref_hash);
+
+        // Sample just past the end
+        assert!(sample_file(&in_path, &[num_samples]).is_err());
+        remove_file(&in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_invalid_offset() {
+        let in_path = tmp_file_path("test_sample_file_invalid_offset_input.txt");
+        {
+            let mut in_file = File::create(&in_path).unwrap();
+            for _ in 0..4096 {
+                in_file.write("123456foobar".as_bytes()).unwrap();
+            }
+        }
+        let samples = [0, 200000];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+        remove_file(in_path).unwrap();
+    }
+
+    #[test]
+    fn test_sample_file_missing_file() {
+        let in_path = tmp_file_path("test_sample_file_that_doesnt_exist.txt");
+        let samples = [0, 5];
+        let res = sample_file(&in_path, &samples);
+        assert!(res.is_err());
+    }
+}

archiver/Cargo.toml

@@ -14,6 +14,7 @@ solana-clap-utils = { path = "../clap-utils", version = "0.23.0" }
 solana-core = { path = "../core", version = "0.23.0" }
 solana-logger = { path = "../logger", version = "0.23.0" }
 solana-metrics = { path = "../metrics", version = "0.23.0" }
+solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" }
 solana-net-utils = { path = "../net-utils", version = "0.23.0" }
 solana-sdk = { path = "../sdk", version = "0.23.0" }
 

archiver/src/main.rs

@@ -1,5 +1,6 @@
 use clap::{crate_description, crate_name, App, Arg};
 use console::style;
+use solana_archiver_lib::archiver::Archiver;
 use solana_clap_utils::{
     input_validators::is_keypair,
     keypair::{
@@ -8,7 +9,6 @@ use solana_clap_utils::{
     },
 };
 use solana_core::{
-    archiver::Archiver,
     cluster_info::{Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
 };

chacha-cuda/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "solana-chacha-cuda"
+version = "0.23.0"
+description = "Solana Chacha Cuda APIs"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+solana-archiver-utils = { path = "../archiver-utils", version = "0.23.0" }
+solana-chacha = { path = "../chacha", version = "0.23.0" }
+solana-ledger = { path = "../ledger", version = "0.23.0" }
+solana-logger = { path = "../logger", version = "0.23.0" }
+solana-perf = { path = "../perf", version = "0.23.0" }
+solana-sdk = { path = "../sdk", version = "0.23.0" }
+
+[dev-dependencies]
+hex-literal = "0.2.1"
+
+[lib]
+name = "solana_chacha_cuda"

chacha-cuda/src/chacha_cuda.rs

@@ -1,6 +1,6 @@
 // Module used by validators to approve storage mining proofs in parallel using the GPU
 
-use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
+use solana_chacha::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
 use solana_ledger::blockstore::Blockstore;
 use solana_perf::perf_libs;
 use solana_sdk::hash::Hash;
@@ -113,8 +113,8 @@ pub fn chacha_cbc_encrypt_file_many_keys(
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::archiver::sample_file;
+    use solana_archiver_utils::sample_file;
-    use crate::chacha::chacha_cbc_encrypt_ledger;
+    use solana_chacha::chacha::chacha_cbc_encrypt_ledger;
     use solana_ledger::entry::create_ticks;
     use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::clock::DEFAULT_SLOTS_PER_SEGMENT;

chacha-cuda/src/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+#[macro_use]
+extern crate hex_literal;
+
+pub mod chacha_cuda;

chacha/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "solana-chacha"
+version = "0.23.0"
+description = "Solana Chacha APIs"
+authors = ["Solana Maintainers <maintainers@solana.com>"]
+repository = "https://github.com/solana-labs/solana"
+license = "Apache-2.0"
+homepage = "https://solana.com/"
+edition = "2018"
+
+[dependencies]
+log = "0.4.8"
+rand = "0.6.5"
+rand_chacha = "0.1.1"
+solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
+solana-ledger = { path = "../ledger", version = "0.23.0" }
+solana-logger = { path = "../logger", version = "0.23.0" }
+solana-perf = { path = "../perf", version = "0.23.0" }
+solana-sdk = { path = "../sdk", version = "0.23.0" }
+
+[dev-dependencies]
+hex-literal = "0.2.1"
+
+[lib]
+name = "solana_chacha"

chacha/src/chacha.rs

@@ -74,13 +74,14 @@ pub fn chacha_cbc_encrypt_ledger(
 #[cfg(test)]
 mod tests {
     use crate::chacha::chacha_cbc_encrypt_ledger;
-    use crate::gen_keys::GenKeys;
+    use rand::SeedableRng;
+    use rand_chacha::ChaChaRng;
     use solana_ledger::blockstore::Blockstore;
     use solana_ledger::entry::Entry;
     use solana_ledger::get_tmp_ledger_path;
     use solana_sdk::hash::{hash, Hash, Hasher};
     use solana_sdk::pubkey::Pubkey;
-    use solana_sdk::signature::KeypairUtil;
+    use solana_sdk::signature::{Keypair, KeypairUtil};
     use solana_sdk::system_transaction;
     use std::fs::remove_file;
     use std::fs::File;
@@ -92,8 +93,9 @@ mod tests {
         let one = hash(&zero.as_ref());
 
         let seed = [2u8; 32];
-        let mut rnd = GenKeys::new(seed);
+
-        let keypair = rnd.gen_keypair();
+        let mut generator = ChaChaRng::from_seed(seed);
+        let keypair = Keypair::generate(&mut generator);
 
         let mut id = one;
         let mut num_hashes = 0;
@@ -135,8 +137,9 @@ mod tests {
         let out_path = tmp_file_path("test_encrypt_ledger");
 
         let seed = [2u8; 32];
-        let mut rnd = GenKeys::new(seed);
+
-        let keypair = rnd.gen_keypair();
+        let mut generator = ChaChaRng::from_seed(seed);
+        let keypair = Keypair::generate(&mut generator);
 
         let entries = make_tiny_deterministic_test_entries(slots_per_segment);
         blockstore

chacha/src/lib.rs

@@ -0,0 +1,8 @@
+#[macro_use]
+extern crate log;
+
+#[cfg(test)]
+#[macro_use]
+extern crate hex_literal;
+
+pub mod chacha;

ci/test-stable.sh

@@ -86,7 +86,7 @@ test-stable-perf)
   fi
 
   _ cargo +"$rust_stable" build --bins ${V:+--verbose}
-  _ cargo +"$rust_stable" test --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
+  _ cargo +"$rust_stable" test --package solana-chacha-cuda --package solana-perf --package solana-ledger --package solana-core --lib ${V:+--verbose} -- --nocapture
   ;;
 test-move)
   ci/affects-files.sh \

core/Cargo.toml

@@ -42,7 +42,6 @@ serde_derive = "1.0.103"
 serde_json = "1.0.44"
 solana-budget-program = { path = "../programs/budget", version = "0.23.0" }
 solana-clap-utils = { path = "../clap-utils", version = "0.23.0" }
-solana-chacha-sys = { path = "../chacha-sys", version = "0.23.0" }
 solana-client = { path = "../client", version = "0.23.0" }
 solana-faucet = { path = "../faucet", version = "0.23.0" }
 ed25519-dalek = "=1.0.0-pre.1"
@@ -52,6 +51,7 @@ solana-merkle-tree = { path = "../merkle-tree", version = "0.23.0" }
 solana-metrics = { path = "../metrics", version = "0.23.0" }
 solana-measure = { path = "../measure", version = "0.23.0" }
 solana-net-utils = { path = "../net-utils", version = "0.23.0" }
+solana-chacha-cuda = { path = "../chacha-cuda", version = "0.23.0" }
 solana-perf = { path = "../perf", version = "0.23.0" }
 solana-runtime = { path = "../runtime", version = "0.23.0" }
 solana-sdk = { path = "../sdk", version = "0.23.0" }
@@ -73,7 +73,6 @@ solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.23.0" }
 reed-solomon-erasure = { package = "solana-reed-solomon-erasure", version = "4.0.1-3", features = ["simd-accel"] }
 
 [dev-dependencies]
-hex-literal = "0.2.1"
 matches = "0.1.6"
 reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] }
 serial_test = "0.3.2"

core/src/lib.rs

@@ -7,14 +7,11 @@
 
 pub mod banking_stage;
 pub mod broadcast_stage;
-pub mod chacha;
-pub mod chacha_cuda;
 pub mod cluster_info_vote_listener;
 pub mod commitment;
 pub mod shred_fetch_stage;
 #[macro_use]
 pub mod contact_info;
-pub mod archiver;
 pub mod blockstream;
 pub mod blockstream_service;
 pub mod cluster_info;
@@ -63,10 +60,6 @@ pub mod window_service;
 #[macro_use]
 extern crate solana_budget_program;
 
-#[cfg(test)]
-#[macro_use]
-extern crate hex_literal;
-
 #[macro_use]
 extern crate log;
 

core/src/storage_stage.rs

@@ -3,13 +3,13 @@
 // to submit its proof for mining to be rewarded.
 
 use crate::{
-    chacha_cuda::chacha_cbc_encrypt_file_many_keys,
     cluster_info::ClusterInfo,
     contact_info::ContactInfo,
     result::{Error, Result},
 };
 use rand::{Rng, SeedableRng};
 use rand_chacha::ChaChaRng;
+use solana_chacha_cuda::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
 use solana_ledger::{bank_forks::BankForks, blockstore::Blockstore};
 use solana_runtime::{bank::Bank, storage_utils::archiver_accounts};
 use solana_sdk::{

local-cluster/Cargo.toml

@@ -12,6 +12,7 @@ homepage = "https://solana.com/"
 itertools = "0.8.1"
 log = "0.4.8"
 rand = "0.6.5"
+solana-archiver-lib = { path = "../archiver-lib", version = "0.23.0" }
 solana-config-program = { path = "../programs/config", version = "0.23.0" }
 solana-core = { path = "../core", version = "0.23.0" }
 solana-client = { path = "../client", version = "0.23.0" }

local-cluster/src/local_cluster.rs

@@ -1,9 +1,9 @@
 use crate::cluster::{Cluster, ClusterValidatorInfo, ValidatorInfo};
 use itertools::izip;
 use log::*;
+use solana_archiver_lib::archiver::Archiver;
 use solana_client::thin_client::{create_client, ThinClient};
 use solana_core::{
-    archiver::Archiver,
     cluster_info::{Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
     genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo},

local-cluster/tests/archiver.rs

@@ -1,8 +1,8 @@
 use log::*;
 use serial_test_derive::serial;
+use solana_archiver_lib::archiver::Archiver;
 use solana_client::thin_client::create_client;
 use solana_core::{
-    archiver::Archiver,
     cluster_info::{ClusterInfo, Node, VALIDATOR_PORT_RANGE},
     contact_info::ContactInfo,
     gossip_service::discover_cluster,