Initial validator code for rust side hooks for chacha cuda parallel encrypt
This commit is contained in:
parent
c30b605047
commit
37a0b7b132
3
build.rs
3
build.rs
|
@ -4,6 +4,7 @@ use std::process::Command;
|
|||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-changed=target/perf-libs");
|
||||
println!("cargo:rerun-if-changed=target/perf-libs/libcuda-crypt.a");
|
||||
println!("cargo:rerun-if-changed=build.rs");
|
||||
|
||||
// Ensure target/perf-libs/ exists. It's been observed that
|
||||
|
@ -54,7 +55,7 @@ fn main() {
|
|||
println!("cargo:rustc-link-search=native=target/perf-libs");
|
||||
}
|
||||
if cuda {
|
||||
println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
|
||||
println!("cargo:rustc-link-lib=static=cuda-crypt");
|
||||
println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
|
||||
println!("cargo:rustc-link-lib=dylib=cudart");
|
||||
println!("cargo:rustc-link-lib=dylib=cuda");
|
||||
|
|
|
@ -15,7 +15,7 @@ mkdir -p target/perf-libs
|
|||
cd target/perf-libs
|
||||
(
|
||||
set -x
|
||||
curl https://solana-perf.s3.amazonaws.com/v0.9.1/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
||||
curl https://solana-perf.s3.amazonaws.com/v0.10.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
||||
)
|
||||
|
||||
if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
|
||||
|
|
|
@ -7,8 +7,8 @@ extern crate solana;
|
|||
|
||||
use clap::{App, Arg};
|
||||
use solana::chacha::chacha_cbc_encrypt_files;
|
||||
use solana::cluster_info::Node;
|
||||
use solana::client::mk_client;
|
||||
use solana::cluster_info::Node;
|
||||
use solana::drone::DRONE_PORT;
|
||||
use solana::fullnode::Config;
|
||||
use solana::ledger::LEDGER_DATA_FILE;
|
||||
|
|
|
@ -39,7 +39,7 @@ pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) ->
|
|||
let mut ivec = [0; CHACHA_IVEC_SIZE];
|
||||
|
||||
while let Ok(size) = in_file.read(&mut buffer) {
|
||||
info!("read {} bytes", size);
|
||||
debug!("read {} bytes", size);
|
||||
if size == 0 {
|
||||
break;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,144 @@
|
|||
use hash::Hash;
|
||||
use ledger::LedgerWindow;
|
||||
use sigverify::chacha_cbc_encrypt_many_sample;
|
||||
use std::io;
|
||||
|
||||
const CHACHA_BLOCK_SIZE: usize = 64;
|
||||
const SHA256_BLOCK_SIZE: usize = 32;
|
||||
const CHACHA_KEY_LEN: usize = 32;
|
||||
const ENTRIES_PER_SLICE: u64 = 16;
|
||||
|
||||
pub fn chacha_cbc_encrypt_file_many_keys(
|
||||
in_path: &str,
|
||||
slice: u64,
|
||||
keys: &[u8],
|
||||
samples: &[u64],
|
||||
) -> io::Result<Vec<Hash>> {
|
||||
if keys.len() % CHACHA_KEY_LEN != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"bad key length({}) not divisible by {} ",
|
||||
keys.len(),
|
||||
CHACHA_KEY_LEN
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let mut ledger_window = LedgerWindow::open(in_path)?;
|
||||
let mut buffer = [0; 8 * 1024];
|
||||
let num_keys = keys.len() / CHACHA_KEY_LEN;
|
||||
let mut sha_states = vec![0; num_keys * SHA256_BLOCK_SIZE];
|
||||
let mut ivecs: Vec<u8> = vec![0; num_keys * CHACHA_BLOCK_SIZE];
|
||||
let mut entry = slice;
|
||||
let mut total_entries = 0;
|
||||
let mut total_entry_len = 0;
|
||||
let mut time: f32 = 0.0;
|
||||
loop {
|
||||
match ledger_window.get_entries_bytes(entry, ENTRIES_PER_SLICE - total_entries, &mut buffer)
|
||||
{
|
||||
Ok((num_entries, entry_len)) => {
|
||||
debug!(
|
||||
"encrypting slice: {} num_entries: {} entry_len: {}",
|
||||
slice, num_entries, entry_len
|
||||
);
|
||||
let entry_len_usz = entry_len as usize;
|
||||
unsafe {
|
||||
chacha_cbc_encrypt_many_sample(
|
||||
buffer[..entry_len_usz].as_ptr(),
|
||||
sha_states.as_mut_ptr(),
|
||||
entry_len_usz,
|
||||
keys.as_ptr(),
|
||||
ivecs.as_mut_ptr(),
|
||||
num_keys as u32,
|
||||
samples.as_ptr(),
|
||||
samples.len() as u32,
|
||||
total_entry_len,
|
||||
&mut time,
|
||||
);
|
||||
}
|
||||
|
||||
total_entry_len += entry_len;
|
||||
total_entries += num_entries;
|
||||
entry += num_entries;
|
||||
debug!(
|
||||
"total entries: {} entry: {} slice: {} entries_per_slice: {}",
|
||||
total_entries, entry, slice, ENTRIES_PER_SLICE
|
||||
);
|
||||
if (entry - slice) >= ENTRIES_PER_SLICE {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
info!("Error encrypting file: {:?}", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut res = Vec::new();
|
||||
for x in 0..num_keys {
|
||||
let start = x * SHA256_BLOCK_SIZE;
|
||||
let end = start + SHA256_BLOCK_SIZE;
|
||||
res.push(Hash::new(&sha_states[start..end]));
|
||||
}
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chacha::chacha_cbc_encrypt_files;
|
||||
use chacha_cuda::chacha_cbc_encrypt_file_many_keys;
|
||||
use ledger::LedgerWriter;
|
||||
use ledger::{get_tmp_ledger_path, make_tiny_test_entries, LEDGER_DATA_FILE};
|
||||
use replicator::sample_file;
|
||||
use std::fs::{remove_dir_all, remove_file};
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_file_many_keys() {
|
||||
use logger;
|
||||
logger::setup();
|
||||
|
||||
let entries = make_tiny_test_entries(16);
|
||||
let ledger_dir = "test_encrypt_file_many_keys";
|
||||
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||
{
|
||||
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
|
||||
writer.write_entries(entries.clone()).unwrap();
|
||||
}
|
||||
|
||||
let out_path = Path::new("test_chacha_encrypt_file_many_keys_output.txt.enc");
|
||||
|
||||
let samples = [0];
|
||||
let key = "thetestkeyabcdefghijklmnopqrstuvwxyzthetestkeyabcdefghijklmnopqr";
|
||||
|
||||
assert!(
|
||||
chacha_cbc_encrypt_files(
|
||||
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
|
||||
out_path,
|
||||
key.to_string()
|
||||
).is_ok()
|
||||
);
|
||||
|
||||
let ref_hash = sample_file(&out_path, &samples).unwrap();
|
||||
|
||||
let hashes =
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, key.as_bytes(), &samples).unwrap();
|
||||
|
||||
assert_eq!(hashes[0], ref_hash);
|
||||
|
||||
let _ignored = remove_dir_all(&ledger_path);
|
||||
let _ignored = remove_file(out_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_file_many_keys_bad_key_length() {
|
||||
let keys = "thetestkey";
|
||||
let ledger_dir = "test_encrypt_file_many_keys_bad_key_length";
|
||||
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||
let samples = [0];
|
||||
assert!(
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, keys.as_bytes(), &samples,).is_err()
|
||||
);
|
||||
}
|
||||
}
|
|
@ -5,7 +5,11 @@
|
|||
use bincode::{self, deserialize, deserialize_from, serialize_into, serialized_size};
|
||||
use budget_instruction::Vote;
|
||||
use budget_transaction::BudgetTransaction;
|
||||
#[cfg(test)]
|
||||
use chrono::prelude::Utc;
|
||||
use entry::Entry;
|
||||
#[cfg(test)]
|
||||
use hash::hash;
|
||||
use hash::Hash;
|
||||
use log::Level::Trace;
|
||||
use mint::Mint;
|
||||
|
@ -638,13 +642,36 @@ pub fn create_tmp_sample_ledger(name: &str, num: i64) -> (Mint, String, Vec<Entr
|
|||
(mint, path, genesis)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn make_tiny_test_entries(num: usize) -> Vec<Entry> {
|
||||
let zero = Hash::default();
|
||||
let one = hash(&zero.as_ref());
|
||||
let keypair = Keypair::new();
|
||||
|
||||
let mut id = one;
|
||||
let mut num_hashes = 0;
|
||||
(0..num)
|
||||
.map(|_| {
|
||||
Entry::new_mut(
|
||||
&mut id,
|
||||
&mut num_hashes,
|
||||
vec![Transaction::budget_new_timestamp(
|
||||
&keypair,
|
||||
keypair.pubkey(),
|
||||
keypair.pubkey(),
|
||||
Utc::now(),
|
||||
one,
|
||||
)],
|
||||
)
|
||||
}).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bincode::serialized_size;
|
||||
use budget_instruction::Vote;
|
||||
use budget_transaction::BudgetTransaction;
|
||||
use chrono::prelude::*;
|
||||
use entry::{next_entry, Entry};
|
||||
use hash::hash;
|
||||
use packet::{to_blobs, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
|
||||
|
@ -669,29 +696,6 @@ mod tests {
|
|||
assert!(!bad_ticks.verify(&zero)); // inductive step, bad
|
||||
}
|
||||
|
||||
fn make_tiny_test_entries(num: usize) -> Vec<Entry> {
|
||||
let zero = Hash::default();
|
||||
let one = hash(&zero.as_ref());
|
||||
let keypair = Keypair::new();
|
||||
|
||||
let mut id = one;
|
||||
let mut num_hashes = 0;
|
||||
(0..num)
|
||||
.map(|_| {
|
||||
Entry::new_mut(
|
||||
&mut id,
|
||||
&mut num_hashes,
|
||||
vec![Transaction::budget_new_timestamp(
|
||||
&keypair,
|
||||
keypair.pubkey(),
|
||||
keypair.pubkey(),
|
||||
Utc::now(),
|
||||
one,
|
||||
)],
|
||||
)
|
||||
}).collect()
|
||||
}
|
||||
|
||||
fn make_test_entries() -> Vec<Entry> {
|
||||
let zero = Hash::default();
|
||||
let one = hash(&zero.as_ref());
|
||||
|
|
|
@ -18,6 +18,8 @@ pub mod budget_instruction;
|
|||
pub mod budget_transaction;
|
||||
#[cfg(feature = "chacha")]
|
||||
pub mod chacha;
|
||||
#[cfg(feature = "cuda")]
|
||||
pub mod chacha_cuda;
|
||||
pub mod choose_gossip_peer_strategy;
|
||||
pub mod client;
|
||||
#[macro_use]
|
||||
|
|
|
@ -45,6 +45,9 @@ pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
|
|||
let mut buf = vec![0; sample_size];
|
||||
|
||||
let file_len = metadata.len();
|
||||
if file_len < sample_size64 {
|
||||
return Err(Error::new(ErrorKind::Other, "file too short!"));
|
||||
}
|
||||
for offset in sample_offsets {
|
||||
if *offset > (file_len - sample_size64) / sample_size64 {
|
||||
return Err(Error::new(ErrorKind::Other, "offset too large"));
|
||||
|
|
|
@ -21,7 +21,7 @@ struct Elems {
|
|||
}
|
||||
|
||||
#[cfg(feature = "cuda")]
|
||||
#[link(name = "cuda_verify_ed25519")]
|
||||
#[link(name = "cuda-crypt")]
|
||||
extern "C" {
|
||||
fn ed25519_init() -> bool;
|
||||
fn ed25519_set_verbose(val: bool);
|
||||
|
@ -35,6 +35,19 @@ extern "C" {
|
|||
signed_message_len_offset: u32,
|
||||
out: *mut u8, //combined length of all the items in vecs
|
||||
) -> u32;
|
||||
|
||||
pub fn chacha_cbc_encrypt_many_sample(
|
||||
input: *const u8,
|
||||
output: *mut u8,
|
||||
in_len: usize,
|
||||
keys: *const u8,
|
||||
ivec: *mut u8,
|
||||
num_keys: u32,
|
||||
samples: *const u64,
|
||||
num_samples: u32,
|
||||
starting_block: u64,
|
||||
time_us: *mut f32,
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cuda"))]
|
||||
|
|
Loading…
Reference in New Issue