From ead7f4287aa004ee0bb51875e4af1eb3b56e624d Mon Sep 17 00:00:00 2001 From: Stephen Akridge Date: Fri, 19 Oct 2018 10:39:13 -0700 Subject: [PATCH] Storage mining fixups... * Use IV to make unique identies * Use hex! macro for hex literal and not string converted to u8 slice * fix sha sampling to control init/end of sha state --- Cargo.toml | 1 + fetch-perf-libs.sh | 2 +- src/bin/replicator.rs | 13 ++--- src/chacha.rs | 28 ++++++---- src/chacha_cuda.rs | 127 ++++++++++++++++++++++++++++++++---------- src/lib.rs | 4 ++ src/sigverify.rs | 5 +- 7 files changed, 130 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 817982e46..1ce056a7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ elf = "0.0.10" env_logger = "0.5.12" generic-array = { version = "0.12.0", default-features = false, features = ["serde"] } getopts = "0.2" +hex-literal = "0.1.1" influx_db_client = "0.3.4" solana-jsonrpc-core = "0.3.0" solana-jsonrpc-http-server = "0.3.0" diff --git a/fetch-perf-libs.sh b/fetch-perf-libs.sh index 90e76da03..7821538da 100755 --- a/fetch-perf-libs.sh +++ b/fetch-perf-libs.sh @@ -15,7 +15,7 @@ mkdir -p target/perf-libs cd target/perf-libs ( set -x - curl https://solana-perf.s3.amazonaws.com/v0.10.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf - + curl https://solana-perf.s3.amazonaws.com/v0.10.1/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf - ) if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then diff --git a/src/bin/replicator.rs b/src/bin/replicator.rs index 09a760360..561cef058 100644 --- a/src/bin/replicator.rs +++ b/src/bin/replicator.rs @@ -6,7 +6,7 @@ extern crate serde_json; extern crate solana; use clap::{App, Arg}; -use solana::chacha::chacha_cbc_encrypt_files; +use solana::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE}; use solana::client::mk_client; use solana::cluster_info::Node; use solana::drone::DRONE_PORT; @@ -114,13 +114,12 @@ fn main() { let ledger_path = Path::new(ledger_path.unwrap()); let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE); let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE)); - let key = "abc123"; + let mut ivec = [0u8; CHACHA_BLOCK_SIZE]; + ivec[0..4].copy_from_slice(&[2, 3, 4, 5]); - if let Err(e) = chacha_cbc_encrypt_files( - &ledger_data_file, - &ledger_data_file_encrypted, - key.to_string(), - ) { + if let Err(e) = + chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec) + { println!("Error while encrypting ledger: {:?}", e); return; } diff --git a/src/chacha.rs b/src/chacha.rs index 766a03068..896a7d06f 100644 --- a/src/chacha.rs +++ b/src/chacha.rs @@ -5,7 +5,8 @@ use std::io::Write; use std::io::{BufReader, BufWriter}; use std::path::Path; -const CHACHA_IVEC_SIZE: usize = 64; +pub const CHACHA_BLOCK_SIZE: usize = 64; +pub const CHACHA_KEY_SIZE: usize = 32; #[link(name = "cpu-crypt")] extern "C" { @@ -30,25 +31,24 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu } } -pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) -> io::Result<()> { +pub fn chacha_cbc_encrypt_file( + in_path: &Path, + out_path: &Path, + ivec: &mut [u8; CHACHA_BLOCK_SIZE], +) -> io::Result<()> { let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file")); let mut out_file = BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file")); let mut buffer = [0; 4 * 1024]; let mut encrypted_buffer = [0; 4 * 1024]; - let mut ivec = [0; CHACHA_IVEC_SIZE]; + let key = [0; CHACHA_KEY_SIZE]; while let Ok(size) = in_file.read(&mut buffer) { debug!("read {} bytes", size); if size == 0 { break; } - chacha_cbc_encrypt( - &buffer[..size], - &mut encrypted_buffer[..size], - key.as_bytes(), - &mut ivec, - ); + chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec); if let Err(res) = out_file.write(&encrypted_buffer[..size]) { println!("Error writing file! {:?}", res); return Err(res); @@ -59,7 +59,7 @@ pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) -> #[cfg(test)] mod tests { - use chacha::chacha_cbc_encrypt_files; + use chacha::chacha_cbc_encrypt_file; use std::fs::remove_file; use std::fs::File; use std::io::Read; @@ -74,13 +74,17 @@ mod tests { let mut in_file = File::create(in_path).unwrap(); in_file.write("123456foobar".as_bytes()).unwrap(); } - assert!(chacha_cbc_encrypt_files(in_path, out_path, "thetestkey".to_string()).is_ok()); + let mut key = hex!( + "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234 + abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234" + ); + assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok()); let mut out_file = File::open(out_path).unwrap(); let mut buf = vec![]; let size = out_file.read_to_end(&mut buf).unwrap(); assert_eq!( buf[..size], - [106, 186, 59, 108, 165, 33, 118, 212, 70, 238, 205, 185] + [66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53] ); remove_file(in_path).unwrap(); remove_file(out_path).unwrap(); diff --git a/src/chacha_cuda.rs b/src/chacha_cuda.rs index c8585d673..01de3a9ac 100644 --- a/src/chacha_cuda.rs +++ b/src/chacha_cuda.rs @@ -1,44 +1,51 @@ +use chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE}; use hash::Hash; use ledger::LedgerWindow; -use sigverify::chacha_cbc_encrypt_many_sample; +use sigverify::{chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state}; use std::io; +use std::mem::size_of; -const CHACHA_BLOCK_SIZE: usize = 64; -const SHA256_BLOCK_SIZE: usize = 32; -const CHACHA_KEY_LEN: usize = 32; const ENTRIES_PER_SLICE: u64 = 16; +// Encrypt a file with multiple starting IV states, determined by ivecs.len() +// +// Then sample each block at the offsets provided by samples argument with sha256 +// and return the vec of sha states pub fn chacha_cbc_encrypt_file_many_keys( in_path: &str, slice: u64, - keys: &[u8], + ivecs: &mut [u8], samples: &[u64], ) -> io::Result> { - if keys.len() % CHACHA_KEY_LEN != 0 { + if ivecs.len() % CHACHA_BLOCK_SIZE != 0 { return Err(io::Error::new( io::ErrorKind::Other, format!( - "bad key length({}) not divisible by {} ", - keys.len(), - CHACHA_KEY_LEN + "bad IV length({}) not divisible by {} ", + ivecs.len(), + CHACHA_BLOCK_SIZE, ), )); } let mut ledger_window = LedgerWindow::open(in_path)?; let mut buffer = [0; 8 * 1024]; - let num_keys = keys.len() / CHACHA_KEY_LEN; - let mut sha_states = vec![0; num_keys * SHA256_BLOCK_SIZE]; - let mut ivecs: Vec = vec![0; num_keys * CHACHA_BLOCK_SIZE]; + let num_keys = ivecs.len() / CHACHA_BLOCK_SIZE; + let mut sha_states = vec![0; num_keys * size_of::()]; + let mut int_sha_states = vec![0; num_keys * 112]; + let keys: Vec = vec![0; num_keys * CHACHA_KEY_SIZE]; // keys not used ATM, uniqueness comes from IV let mut entry = slice; let mut total_entries = 0; let mut total_entry_len = 0; let mut time: f32 = 0.0; + unsafe { + chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32); + } loop { match ledger_window.get_entries_bytes(entry, ENTRIES_PER_SLICE - total_entries, &mut buffer) { Ok((num_entries, entry_len)) => { - debug!( + info!( "encrypting slice: {} num_entries: {} entry_len: {}", slice, num_entries, entry_len ); @@ -46,7 +53,7 @@ pub fn chacha_cbc_encrypt_file_many_keys( unsafe { chacha_cbc_encrypt_many_sample( buffer[..entry_len_usz].as_ptr(), - sha_states.as_mut_ptr(), + int_sha_states.as_mut_ptr(), entry_len_usz, keys.as_ptr(), ivecs.as_mut_ptr(), @@ -75,10 +82,18 @@ pub fn chacha_cbc_encrypt_file_many_keys( } } } + unsafe { + chacha_end_sha_state( + int_sha_states.as_ptr(), + sha_states.as_mut_ptr(), + num_keys as u32, + ); + } + info!("num_keys: {}", num_keys); let mut res = Vec::new(); for x in 0..num_keys { - let start = x * SHA256_BLOCK_SIZE; - let end = start + SHA256_BLOCK_SIZE; + let start = x * size_of::(); + let end = start + size_of::(); res.push(Hash::new(&sha_states[start..end])); } Ok(res) @@ -86,8 +101,9 @@ pub fn chacha_cbc_encrypt_file_many_keys( #[cfg(test)] mod tests { - use chacha::chacha_cbc_encrypt_files; + use chacha::chacha_cbc_encrypt_file; use chacha_cuda::chacha_cbc_encrypt_file_many_keys; + use hash::Hash; use ledger::LedgerWriter; use ledger::{get_tmp_ledger_path, make_tiny_test_entries, LEDGER_DATA_FILE}; use replicator::sample_file; @@ -95,35 +111,39 @@ mod tests { use std::path::Path; #[test] - fn test_encrypt_file_many_keys() { + fn test_encrypt_file_many_keys_single() { use logger; logger::setup(); - let entries = make_tiny_test_entries(16); - let ledger_dir = "test_encrypt_file_many_keys"; + let entries = make_tiny_test_entries(32); + let ledger_dir = "test_encrypt_file_many_keys_single"; let ledger_path = get_tmp_ledger_path(ledger_dir); { let mut writer = LedgerWriter::open(&ledger_path, true).unwrap(); writer.write_entries(&entries).unwrap(); } - let out_path = Path::new("test_chacha_encrypt_file_many_keys_output.txt.enc"); + let out_path = Path::new("test_chacha_encrypt_file_many_keys_single_output.txt.enc"); let samples = [0]; - let key = "thetestkeyabcdefghijklmnopqrstuvwxyzthetestkeyabcdefghijklmnopqr"; + let mut ivecs = hex!( + "abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234 + abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234" + ); + let mut cpu_iv = ivecs.clone(); assert!( - chacha_cbc_encrypt_files( + chacha_cbc_encrypt_file( &Path::new(&ledger_path).join(LEDGER_DATA_FILE), out_path, - key.to_string() + &mut cpu_iv, ).is_ok() ); let ref_hash = sample_file(&out_path, &samples).unwrap(); let hashes = - chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, key.as_bytes(), &samples).unwrap(); + chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap(); assert_eq!(hashes[0], ref_hash); @@ -131,14 +151,63 @@ mod tests { let _ignored = remove_file(out_path); } + #[test] + fn test_encrypt_file_many_keys_multiple_keys() { + use logger; + logger::setup(); + + let entries = make_tiny_test_entries(32); + let ledger_dir = "test_encrypt_file_many_keys_multiple"; + let ledger_path = get_tmp_ledger_path(ledger_dir); + { + let mut writer = LedgerWriter::open(&ledger_path, true).unwrap(); + writer.write_entries(entries.clone()).unwrap(); + } + + let out_path = Path::new("test_chacha_encrypt_file_many_keys_multiple_output.txt.enc"); + + let samples = [0, 1, 3, 4, 5, 150]; + let mut ivecs = Vec::new(); + let mut ref_hashes: Vec = vec![]; + for i in 0..2 { + let mut ivec = hex!( + "abc123abc123abc123abc123abc123abc123abababababababababababababab + abc123abc123abc123abc123abc123abc123abababababababababababababab" + ); + ivec[0] = i; + ivecs.extend(ivec.clone().iter()); + assert!( + chacha_cbc_encrypt_file( + &Path::new(&ledger_path).join(LEDGER_DATA_FILE), + out_path, + &mut ivec, + ).is_ok() + ); + + ref_hashes.push(sample_file(&out_path, &samples).unwrap()); + info!( + "ivec: {:?} hash: {:?} ivecs: {:?}", + ivec.to_vec(), + ref_hashes.last(), + ivecs + ); + } + + let hashes = + chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap(); + + assert_eq!(hashes, ref_hashes); + + let _ignored = remove_dir_all(&ledger_path); + let _ignored = remove_file(out_path); + } + #[test] fn test_encrypt_file_many_keys_bad_key_length() { - let keys = "thetestkey"; + let mut keys = hex!("abc123"); let ledger_dir = "test_encrypt_file_many_keys_bad_key_length"; let ledger_path = get_tmp_ledger_path(ledger_dir); let samples = [0]; - assert!( - chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, keys.as_bytes(), &samples,).is_err() - ); + assert!(chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut keys, &samples,).is_err()); } } diff --git a/src/lib.rs b/src/lib.rs index 1a9c72a38..0462b3ab6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,10 @@ extern crate chrono; extern crate clap; extern crate dirs; extern crate generic_array; +#[cfg(test)] +#[cfg(any(feature = "chacha", feature = "cuda"))] +#[macro_use] +extern crate hex_literal; extern crate ipnetwork; extern crate itertools; extern crate libc; diff --git a/src/sigverify.rs b/src/sigverify.rs index c65a3d66e..6c8042dc3 100644 --- a/src/sigverify.rs +++ b/src/sigverify.rs @@ -38,7 +38,7 @@ extern "C" { pub fn chacha_cbc_encrypt_many_sample( input: *const u8, - output: *mut u8, + sha_state: *mut u8, in_len: usize, keys: *const u8, ivec: *mut u8, @@ -48,6 +48,9 @@ extern "C" { starting_block: u64, time_us: *mut f32, ); + + pub fn chacha_init_sha_state(sha_state: *mut u8, num_keys: u32); + pub fn chacha_end_sha_state(sha_state_in: *const u8, out: *mut u8, num_keys: u32); } #[cfg(not(feature = "cuda"))]