Initial validator code for rust side hooks for chacha cuda parallel encrypt
This commit is contained in:
parent
c30b605047
commit
37a0b7b132
3
build.rs
3
build.rs
|
@ -4,6 +4,7 @@ use std::process::Command;
|
||||||
|
|
||||||
fn main() {
|
fn main() {
|
||||||
println!("cargo:rerun-if-changed=target/perf-libs");
|
println!("cargo:rerun-if-changed=target/perf-libs");
|
||||||
|
println!("cargo:rerun-if-changed=target/perf-libs/libcuda-crypt.a");
|
||||||
println!("cargo:rerun-if-changed=build.rs");
|
println!("cargo:rerun-if-changed=build.rs");
|
||||||
|
|
||||||
// Ensure target/perf-libs/ exists. It's been observed that
|
// Ensure target/perf-libs/ exists. It's been observed that
|
||||||
|
@ -54,7 +55,7 @@ fn main() {
|
||||||
println!("cargo:rustc-link-search=native=target/perf-libs");
|
println!("cargo:rustc-link-search=native=target/perf-libs");
|
||||||
}
|
}
|
||||||
if cuda {
|
if cuda {
|
||||||
println!("cargo:rustc-link-lib=static=cuda_verify_ed25519");
|
println!("cargo:rustc-link-lib=static=cuda-crypt");
|
||||||
println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
|
println!("cargo:rustc-link-search=native=/usr/local/cuda/lib64");
|
||||||
println!("cargo:rustc-link-lib=dylib=cudart");
|
println!("cargo:rustc-link-lib=dylib=cudart");
|
||||||
println!("cargo:rustc-link-lib=dylib=cuda");
|
println!("cargo:rustc-link-lib=dylib=cuda");
|
||||||
|
|
|
@ -15,7 +15,7 @@ mkdir -p target/perf-libs
|
||||||
cd target/perf-libs
|
cd target/perf-libs
|
||||||
(
|
(
|
||||||
set -x
|
set -x
|
||||||
curl https://solana-perf.s3.amazonaws.com/v0.9.1/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
curl https://solana-perf.s3.amazonaws.com/v0.10.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
||||||
)
|
)
|
||||||
|
|
||||||
if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
|
if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
|
||||||
|
|
|
@ -7,8 +7,8 @@ extern crate solana;
|
||||||
|
|
||||||
use clap::{App, Arg};
|
use clap::{App, Arg};
|
||||||
use solana::chacha::chacha_cbc_encrypt_files;
|
use solana::chacha::chacha_cbc_encrypt_files;
|
||||||
use solana::cluster_info::Node;
|
|
||||||
use solana::client::mk_client;
|
use solana::client::mk_client;
|
||||||
|
use solana::cluster_info::Node;
|
||||||
use solana::drone::DRONE_PORT;
|
use solana::drone::DRONE_PORT;
|
||||||
use solana::fullnode::Config;
|
use solana::fullnode::Config;
|
||||||
use solana::ledger::LEDGER_DATA_FILE;
|
use solana::ledger::LEDGER_DATA_FILE;
|
||||||
|
|
|
@ -39,7 +39,7 @@ pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) ->
|
||||||
let mut ivec = [0; CHACHA_IVEC_SIZE];
|
let mut ivec = [0; CHACHA_IVEC_SIZE];
|
||||||
|
|
||||||
while let Ok(size) = in_file.read(&mut buffer) {
|
while let Ok(size) = in_file.read(&mut buffer) {
|
||||||
info!("read {} bytes", size);
|
debug!("read {} bytes", size);
|
||||||
if size == 0 {
|
if size == 0 {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,144 @@
|
||||||
|
use hash::Hash;
|
||||||
|
use ledger::LedgerWindow;
|
||||||
|
use sigverify::chacha_cbc_encrypt_many_sample;
|
||||||
|
use std::io;
|
||||||
|
|
||||||
|
const CHACHA_BLOCK_SIZE: usize = 64;
|
||||||
|
const SHA256_BLOCK_SIZE: usize = 32;
|
||||||
|
const CHACHA_KEY_LEN: usize = 32;
|
||||||
|
const ENTRIES_PER_SLICE: u64 = 16;
|
||||||
|
|
||||||
|
pub fn chacha_cbc_encrypt_file_many_keys(
|
||||||
|
in_path: &str,
|
||||||
|
slice: u64,
|
||||||
|
keys: &[u8],
|
||||||
|
samples: &[u64],
|
||||||
|
) -> io::Result<Vec<Hash>> {
|
||||||
|
if keys.len() % CHACHA_KEY_LEN != 0 {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::Other,
|
||||||
|
format!(
|
||||||
|
"bad key length({}) not divisible by {} ",
|
||||||
|
keys.len(),
|
||||||
|
CHACHA_KEY_LEN
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut ledger_window = LedgerWindow::open(in_path)?;
|
||||||
|
let mut buffer = [0; 8 * 1024];
|
||||||
|
let num_keys = keys.len() / CHACHA_KEY_LEN;
|
||||||
|
let mut sha_states = vec![0; num_keys * SHA256_BLOCK_SIZE];
|
||||||
|
let mut ivecs: Vec<u8> = vec![0; num_keys * CHACHA_BLOCK_SIZE];
|
||||||
|
let mut entry = slice;
|
||||||
|
let mut total_entries = 0;
|
||||||
|
let mut total_entry_len = 0;
|
||||||
|
let mut time: f32 = 0.0;
|
||||||
|
loop {
|
||||||
|
match ledger_window.get_entries_bytes(entry, ENTRIES_PER_SLICE - total_entries, &mut buffer)
|
||||||
|
{
|
||||||
|
Ok((num_entries, entry_len)) => {
|
||||||
|
debug!(
|
||||||
|
"encrypting slice: {} num_entries: {} entry_len: {}",
|
||||||
|
slice, num_entries, entry_len
|
||||||
|
);
|
||||||
|
let entry_len_usz = entry_len as usize;
|
||||||
|
unsafe {
|
||||||
|
chacha_cbc_encrypt_many_sample(
|
||||||
|
buffer[..entry_len_usz].as_ptr(),
|
||||||
|
sha_states.as_mut_ptr(),
|
||||||
|
entry_len_usz,
|
||||||
|
keys.as_ptr(),
|
||||||
|
ivecs.as_mut_ptr(),
|
||||||
|
num_keys as u32,
|
||||||
|
samples.as_ptr(),
|
||||||
|
samples.len() as u32,
|
||||||
|
total_entry_len,
|
||||||
|
&mut time,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
total_entry_len += entry_len;
|
||||||
|
total_entries += num_entries;
|
||||||
|
entry += num_entries;
|
||||||
|
debug!(
|
||||||
|
"total entries: {} entry: {} slice: {} entries_per_slice: {}",
|
||||||
|
total_entries, entry, slice, ENTRIES_PER_SLICE
|
||||||
|
);
|
||||||
|
if (entry - slice) >= ENTRIES_PER_SLICE {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
info!("Error encrypting file: {:?}", e);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut res = Vec::new();
|
||||||
|
for x in 0..num_keys {
|
||||||
|
let start = x * SHA256_BLOCK_SIZE;
|
||||||
|
let end = start + SHA256_BLOCK_SIZE;
|
||||||
|
res.push(Hash::new(&sha_states[start..end]));
|
||||||
|
}
|
||||||
|
Ok(res)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use chacha::chacha_cbc_encrypt_files;
|
||||||
|
use chacha_cuda::chacha_cbc_encrypt_file_many_keys;
|
||||||
|
use ledger::LedgerWriter;
|
||||||
|
use ledger::{get_tmp_ledger_path, make_tiny_test_entries, LEDGER_DATA_FILE};
|
||||||
|
use replicator::sample_file;
|
||||||
|
use std::fs::{remove_dir_all, remove_file};
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encrypt_file_many_keys() {
|
||||||
|
use logger;
|
||||||
|
logger::setup();
|
||||||
|
|
||||||
|
let entries = make_tiny_test_entries(16);
|
||||||
|
let ledger_dir = "test_encrypt_file_many_keys";
|
||||||
|
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||||
|
{
|
||||||
|
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
|
||||||
|
writer.write_entries(entries.clone()).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
let out_path = Path::new("test_chacha_encrypt_file_many_keys_output.txt.enc");
|
||||||
|
|
||||||
|
let samples = [0];
|
||||||
|
let key = "thetestkeyabcdefghijklmnopqrstuvwxyzthetestkeyabcdefghijklmnopqr";
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
chacha_cbc_encrypt_files(
|
||||||
|
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
|
||||||
|
out_path,
|
||||||
|
key.to_string()
|
||||||
|
).is_ok()
|
||||||
|
);
|
||||||
|
|
||||||
|
let ref_hash = sample_file(&out_path, &samples).unwrap();
|
||||||
|
|
||||||
|
let hashes =
|
||||||
|
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, key.as_bytes(), &samples).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(hashes[0], ref_hash);
|
||||||
|
|
||||||
|
let _ignored = remove_dir_all(&ledger_path);
|
||||||
|
let _ignored = remove_file(out_path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_encrypt_file_many_keys_bad_key_length() {
|
||||||
|
let keys = "thetestkey";
|
||||||
|
let ledger_dir = "test_encrypt_file_many_keys_bad_key_length";
|
||||||
|
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||||
|
let samples = [0];
|
||||||
|
assert!(
|
||||||
|
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, keys.as_bytes(), &samples,).is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
|
@ -5,7 +5,11 @@
|
||||||
use bincode::{self, deserialize, deserialize_from, serialize_into, serialized_size};
|
use bincode::{self, deserialize, deserialize_from, serialize_into, serialized_size};
|
||||||
use budget_instruction::Vote;
|
use budget_instruction::Vote;
|
||||||
use budget_transaction::BudgetTransaction;
|
use budget_transaction::BudgetTransaction;
|
||||||
|
#[cfg(test)]
|
||||||
|
use chrono::prelude::Utc;
|
||||||
use entry::Entry;
|
use entry::Entry;
|
||||||
|
#[cfg(test)]
|
||||||
|
use hash::hash;
|
||||||
use hash::Hash;
|
use hash::Hash;
|
||||||
use log::Level::Trace;
|
use log::Level::Trace;
|
||||||
use mint::Mint;
|
use mint::Mint;
|
||||||
|
@ -638,13 +642,36 @@ pub fn create_tmp_sample_ledger(name: &str, num: i64) -> (Mint, String, Vec<Entr
|
||||||
(mint, path, genesis)
|
(mint, path, genesis)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub fn make_tiny_test_entries(num: usize) -> Vec<Entry> {
|
||||||
|
let zero = Hash::default();
|
||||||
|
let one = hash(&zero.as_ref());
|
||||||
|
let keypair = Keypair::new();
|
||||||
|
|
||||||
|
let mut id = one;
|
||||||
|
let mut num_hashes = 0;
|
||||||
|
(0..num)
|
||||||
|
.map(|_| {
|
||||||
|
Entry::new_mut(
|
||||||
|
&mut id,
|
||||||
|
&mut num_hashes,
|
||||||
|
vec![Transaction::budget_new_timestamp(
|
||||||
|
&keypair,
|
||||||
|
keypair.pubkey(),
|
||||||
|
keypair.pubkey(),
|
||||||
|
Utc::now(),
|
||||||
|
one,
|
||||||
|
)],
|
||||||
|
)
|
||||||
|
}).collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use bincode::serialized_size;
|
use bincode::serialized_size;
|
||||||
use budget_instruction::Vote;
|
use budget_instruction::Vote;
|
||||||
use budget_transaction::BudgetTransaction;
|
use budget_transaction::BudgetTransaction;
|
||||||
use chrono::prelude::*;
|
|
||||||
use entry::{next_entry, Entry};
|
use entry::{next_entry, Entry};
|
||||||
use hash::hash;
|
use hash::hash;
|
||||||
use packet::{to_blobs, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
|
use packet::{to_blobs, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
|
||||||
|
@ -669,29 +696,6 @@ mod tests {
|
||||||
assert!(!bad_ticks.verify(&zero)); // inductive step, bad
|
assert!(!bad_ticks.verify(&zero)); // inductive step, bad
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_tiny_test_entries(num: usize) -> Vec<Entry> {
|
|
||||||
let zero = Hash::default();
|
|
||||||
let one = hash(&zero.as_ref());
|
|
||||||
let keypair = Keypair::new();
|
|
||||||
|
|
||||||
let mut id = one;
|
|
||||||
let mut num_hashes = 0;
|
|
||||||
(0..num)
|
|
||||||
.map(|_| {
|
|
||||||
Entry::new_mut(
|
|
||||||
&mut id,
|
|
||||||
&mut num_hashes,
|
|
||||||
vec![Transaction::budget_new_timestamp(
|
|
||||||
&keypair,
|
|
||||||
keypair.pubkey(),
|
|
||||||
keypair.pubkey(),
|
|
||||||
Utc::now(),
|
|
||||||
one,
|
|
||||||
)],
|
|
||||||
)
|
|
||||||
}).collect()
|
|
||||||
}
|
|
||||||
|
|
||||||
fn make_test_entries() -> Vec<Entry> {
|
fn make_test_entries() -> Vec<Entry> {
|
||||||
let zero = Hash::default();
|
let zero = Hash::default();
|
||||||
let one = hash(&zero.as_ref());
|
let one = hash(&zero.as_ref());
|
||||||
|
|
|
@ -18,6 +18,8 @@ pub mod budget_instruction;
|
||||||
pub mod budget_transaction;
|
pub mod budget_transaction;
|
||||||
#[cfg(feature = "chacha")]
|
#[cfg(feature = "chacha")]
|
||||||
pub mod chacha;
|
pub mod chacha;
|
||||||
|
#[cfg(feature = "cuda")]
|
||||||
|
pub mod chacha_cuda;
|
||||||
pub mod choose_gossip_peer_strategy;
|
pub mod choose_gossip_peer_strategy;
|
||||||
pub mod client;
|
pub mod client;
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
|
|
|
@ -45,6 +45,9 @@ pub fn sample_file(in_path: &Path, sample_offsets: &[u64]) -> io::Result<Hash> {
|
||||||
let mut buf = vec![0; sample_size];
|
let mut buf = vec![0; sample_size];
|
||||||
|
|
||||||
let file_len = metadata.len();
|
let file_len = metadata.len();
|
||||||
|
if file_len < sample_size64 {
|
||||||
|
return Err(Error::new(ErrorKind::Other, "file too short!"));
|
||||||
|
}
|
||||||
for offset in sample_offsets {
|
for offset in sample_offsets {
|
||||||
if *offset > (file_len - sample_size64) / sample_size64 {
|
if *offset > (file_len - sample_size64) / sample_size64 {
|
||||||
return Err(Error::new(ErrorKind::Other, "offset too large"));
|
return Err(Error::new(ErrorKind::Other, "offset too large"));
|
||||||
|
|
|
@ -21,7 +21,7 @@ struct Elems {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "cuda")]
|
#[cfg(feature = "cuda")]
|
||||||
#[link(name = "cuda_verify_ed25519")]
|
#[link(name = "cuda-crypt")]
|
||||||
extern "C" {
|
extern "C" {
|
||||||
fn ed25519_init() -> bool;
|
fn ed25519_init() -> bool;
|
||||||
fn ed25519_set_verbose(val: bool);
|
fn ed25519_set_verbose(val: bool);
|
||||||
|
@ -35,6 +35,19 @@ extern "C" {
|
||||||
signed_message_len_offset: u32,
|
signed_message_len_offset: u32,
|
||||||
out: *mut u8, //combined length of all the items in vecs
|
out: *mut u8, //combined length of all the items in vecs
|
||||||
) -> u32;
|
) -> u32;
|
||||||
|
|
||||||
|
pub fn chacha_cbc_encrypt_many_sample(
|
||||||
|
input: *const u8,
|
||||||
|
output: *mut u8,
|
||||||
|
in_len: usize,
|
||||||
|
keys: *const u8,
|
||||||
|
ivec: *mut u8,
|
||||||
|
num_keys: u32,
|
||||||
|
samples: *const u64,
|
||||||
|
num_samples: u32,
|
||||||
|
starting_block: u64,
|
||||||
|
time_us: *mut f32,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "cuda"))]
|
#[cfg(not(feature = "cuda"))]
|
||||||
|
|
Loading…
Reference in New Issue