Storage mining fixups...
* Use IV to make unique identies * Use hex! macro for hex literal and not string converted to u8 slice * fix sha sampling to control init/end of sha state
This commit is contained in:
parent
3b33150cfb
commit
ead7f4287a
|
@ -79,6 +79,7 @@ elf = "0.0.10"
|
|||
env_logger = "0.5.12"
|
||||
generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
|
||||
getopts = "0.2"
|
||||
hex-literal = "0.1.1"
|
||||
influx_db_client = "0.3.4"
|
||||
solana-jsonrpc-core = "0.3.0"
|
||||
solana-jsonrpc-http-server = "0.3.0"
|
||||
|
|
|
@ -15,7 +15,7 @@ mkdir -p target/perf-libs
|
|||
cd target/perf-libs
|
||||
(
|
||||
set -x
|
||||
curl https://solana-perf.s3.amazonaws.com/v0.10.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
||||
curl https://solana-perf.s3.amazonaws.com/v0.10.1/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
|
||||
)
|
||||
|
||||
if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then
|
||||
|
|
|
@ -6,7 +6,7 @@ extern crate serde_json;
|
|||
extern crate solana;
|
||||
|
||||
use clap::{App, Arg};
|
||||
use solana::chacha::chacha_cbc_encrypt_files;
|
||||
use solana::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE};
|
||||
use solana::client::mk_client;
|
||||
use solana::cluster_info::Node;
|
||||
use solana::drone::DRONE_PORT;
|
||||
|
@ -114,13 +114,12 @@ fn main() {
|
|||
let ledger_path = Path::new(ledger_path.unwrap());
|
||||
let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE);
|
||||
let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE));
|
||||
let key = "abc123";
|
||||
let mut ivec = [0u8; CHACHA_BLOCK_SIZE];
|
||||
ivec[0..4].copy_from_slice(&[2, 3, 4, 5]);
|
||||
|
||||
if let Err(e) = chacha_cbc_encrypt_files(
|
||||
&ledger_data_file,
|
||||
&ledger_data_file_encrypted,
|
||||
key.to_string(),
|
||||
) {
|
||||
if let Err(e) =
|
||||
chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec)
|
||||
{
|
||||
println!("Error while encrypting ledger: {:?}", e);
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -5,7 +5,8 @@ use std::io::Write;
|
|||
use std::io::{BufReader, BufWriter};
|
||||
use std::path::Path;
|
||||
|
||||
const CHACHA_IVEC_SIZE: usize = 64;
|
||||
pub const CHACHA_BLOCK_SIZE: usize = 64;
|
||||
pub const CHACHA_KEY_SIZE: usize = 32;
|
||||
|
||||
#[link(name = "cpu-crypt")]
|
||||
extern "C" {
|
||||
|
@ -30,25 +31,24 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu
|
|||
}
|
||||
}
|
||||
|
||||
pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) -> io::Result<()> {
|
||||
pub fn chacha_cbc_encrypt_file(
|
||||
in_path: &Path,
|
||||
out_path: &Path,
|
||||
ivec: &mut [u8; CHACHA_BLOCK_SIZE],
|
||||
) -> io::Result<()> {
|
||||
let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file"));
|
||||
let mut out_file =
|
||||
BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
|
||||
let mut buffer = [0; 4 * 1024];
|
||||
let mut encrypted_buffer = [0; 4 * 1024];
|
||||
let mut ivec = [0; CHACHA_IVEC_SIZE];
|
||||
let key = [0; CHACHA_KEY_SIZE];
|
||||
|
||||
while let Ok(size) = in_file.read(&mut buffer) {
|
||||
debug!("read {} bytes", size);
|
||||
if size == 0 {
|
||||
break;
|
||||
}
|
||||
chacha_cbc_encrypt(
|
||||
&buffer[..size],
|
||||
&mut encrypted_buffer[..size],
|
||||
key.as_bytes(),
|
||||
&mut ivec,
|
||||
);
|
||||
chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
|
||||
if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
|
||||
println!("Error writing file! {:?}", res);
|
||||
return Err(res);
|
||||
|
@ -59,7 +59,7 @@ pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) ->
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chacha::chacha_cbc_encrypt_files;
|
||||
use chacha::chacha_cbc_encrypt_file;
|
||||
use std::fs::remove_file;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
|
@ -74,13 +74,17 @@ mod tests {
|
|||
let mut in_file = File::create(in_path).unwrap();
|
||||
in_file.write("123456foobar".as_bytes()).unwrap();
|
||||
}
|
||||
assert!(chacha_cbc_encrypt_files(in_path, out_path, "thetestkey".to_string()).is_ok());
|
||||
let mut key = hex!(
|
||||
"abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
|
||||
abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
|
||||
);
|
||||
assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok());
|
||||
let mut out_file = File::open(out_path).unwrap();
|
||||
let mut buf = vec![];
|
||||
let size = out_file.read_to_end(&mut buf).unwrap();
|
||||
assert_eq!(
|
||||
buf[..size],
|
||||
[106, 186, 59, 108, 165, 33, 118, 212, 70, 238, 205, 185]
|
||||
[66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53]
|
||||
);
|
||||
remove_file(in_path).unwrap();
|
||||
remove_file(out_path).unwrap();
|
||||
|
|
|
@ -1,44 +1,51 @@
|
|||
use chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
|
||||
use hash::Hash;
|
||||
use ledger::LedgerWindow;
|
||||
use sigverify::chacha_cbc_encrypt_many_sample;
|
||||
use sigverify::{chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state};
|
||||
use std::io;
|
||||
use std::mem::size_of;
|
||||
|
||||
const CHACHA_BLOCK_SIZE: usize = 64;
|
||||
const SHA256_BLOCK_SIZE: usize = 32;
|
||||
const CHACHA_KEY_LEN: usize = 32;
|
||||
const ENTRIES_PER_SLICE: u64 = 16;
|
||||
|
||||
// Encrypt a file with multiple starting IV states, determined by ivecs.len()
|
||||
//
|
||||
// Then sample each block at the offsets provided by samples argument with sha256
|
||||
// and return the vec of sha states
|
||||
pub fn chacha_cbc_encrypt_file_many_keys(
|
||||
in_path: &str,
|
||||
slice: u64,
|
||||
keys: &[u8],
|
||||
ivecs: &mut [u8],
|
||||
samples: &[u64],
|
||||
) -> io::Result<Vec<Hash>> {
|
||||
if keys.len() % CHACHA_KEY_LEN != 0 {
|
||||
if ivecs.len() % CHACHA_BLOCK_SIZE != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!(
|
||||
"bad key length({}) not divisible by {} ",
|
||||
keys.len(),
|
||||
CHACHA_KEY_LEN
|
||||
"bad IV length({}) not divisible by {} ",
|
||||
ivecs.len(),
|
||||
CHACHA_BLOCK_SIZE,
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
let mut ledger_window = LedgerWindow::open(in_path)?;
|
||||
let mut buffer = [0; 8 * 1024];
|
||||
let num_keys = keys.len() / CHACHA_KEY_LEN;
|
||||
let mut sha_states = vec![0; num_keys * SHA256_BLOCK_SIZE];
|
||||
let mut ivecs: Vec<u8> = vec![0; num_keys * CHACHA_BLOCK_SIZE];
|
||||
let num_keys = ivecs.len() / CHACHA_BLOCK_SIZE;
|
||||
let mut sha_states = vec![0; num_keys * size_of::<Hash>()];
|
||||
let mut int_sha_states = vec![0; num_keys * 112];
|
||||
let keys: Vec<u8> = vec![0; num_keys * CHACHA_KEY_SIZE]; // keys not used ATM, uniqueness comes from IV
|
||||
let mut entry = slice;
|
||||
let mut total_entries = 0;
|
||||
let mut total_entry_len = 0;
|
||||
let mut time: f32 = 0.0;
|
||||
unsafe {
|
||||
chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32);
|
||||
}
|
||||
loop {
|
||||
match ledger_window.get_entries_bytes(entry, ENTRIES_PER_SLICE - total_entries, &mut buffer)
|
||||
{
|
||||
Ok((num_entries, entry_len)) => {
|
||||
debug!(
|
||||
info!(
|
||||
"encrypting slice: {} num_entries: {} entry_len: {}",
|
||||
slice, num_entries, entry_len
|
||||
);
|
||||
|
@ -46,7 +53,7 @@ pub fn chacha_cbc_encrypt_file_many_keys(
|
|||
unsafe {
|
||||
chacha_cbc_encrypt_many_sample(
|
||||
buffer[..entry_len_usz].as_ptr(),
|
||||
sha_states.as_mut_ptr(),
|
||||
int_sha_states.as_mut_ptr(),
|
||||
entry_len_usz,
|
||||
keys.as_ptr(),
|
||||
ivecs.as_mut_ptr(),
|
||||
|
@ -75,10 +82,18 @@ pub fn chacha_cbc_encrypt_file_many_keys(
|
|||
}
|
||||
}
|
||||
}
|
||||
unsafe {
|
||||
chacha_end_sha_state(
|
||||
int_sha_states.as_ptr(),
|
||||
sha_states.as_mut_ptr(),
|
||||
num_keys as u32,
|
||||
);
|
||||
}
|
||||
info!("num_keys: {}", num_keys);
|
||||
let mut res = Vec::new();
|
||||
for x in 0..num_keys {
|
||||
let start = x * SHA256_BLOCK_SIZE;
|
||||
let end = start + SHA256_BLOCK_SIZE;
|
||||
let start = x * size_of::<Hash>();
|
||||
let end = start + size_of::<Hash>();
|
||||
res.push(Hash::new(&sha_states[start..end]));
|
||||
}
|
||||
Ok(res)
|
||||
|
@ -86,8 +101,9 @@ pub fn chacha_cbc_encrypt_file_many_keys(
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use chacha::chacha_cbc_encrypt_files;
|
||||
use chacha::chacha_cbc_encrypt_file;
|
||||
use chacha_cuda::chacha_cbc_encrypt_file_many_keys;
|
||||
use hash::Hash;
|
||||
use ledger::LedgerWriter;
|
||||
use ledger::{get_tmp_ledger_path, make_tiny_test_entries, LEDGER_DATA_FILE};
|
||||
use replicator::sample_file;
|
||||
|
@ -95,35 +111,39 @@ mod tests {
|
|||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_file_many_keys() {
|
||||
fn test_encrypt_file_many_keys_single() {
|
||||
use logger;
|
||||
logger::setup();
|
||||
|
||||
let entries = make_tiny_test_entries(16);
|
||||
let ledger_dir = "test_encrypt_file_many_keys";
|
||||
let entries = make_tiny_test_entries(32);
|
||||
let ledger_dir = "test_encrypt_file_many_keys_single";
|
||||
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||
{
|
||||
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
|
||||
writer.write_entries(&entries).unwrap();
|
||||
}
|
||||
|
||||
let out_path = Path::new("test_chacha_encrypt_file_many_keys_output.txt.enc");
|
||||
let out_path = Path::new("test_chacha_encrypt_file_many_keys_single_output.txt.enc");
|
||||
|
||||
let samples = [0];
|
||||
let key = "thetestkeyabcdefghijklmnopqrstuvwxyzthetestkeyabcdefghijklmnopqr";
|
||||
let mut ivecs = hex!(
|
||||
"abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
|
||||
abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
|
||||
);
|
||||
|
||||
let mut cpu_iv = ivecs.clone();
|
||||
assert!(
|
||||
chacha_cbc_encrypt_files(
|
||||
chacha_cbc_encrypt_file(
|
||||
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
|
||||
out_path,
|
||||
key.to_string()
|
||||
&mut cpu_iv,
|
||||
).is_ok()
|
||||
);
|
||||
|
||||
let ref_hash = sample_file(&out_path, &samples).unwrap();
|
||||
|
||||
let hashes =
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, key.as_bytes(), &samples).unwrap();
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap();
|
||||
|
||||
assert_eq!(hashes[0], ref_hash);
|
||||
|
||||
|
@ -131,14 +151,63 @@ mod tests {
|
|||
let _ignored = remove_file(out_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_file_many_keys_multiple_keys() {
|
||||
use logger;
|
||||
logger::setup();
|
||||
|
||||
let entries = make_tiny_test_entries(32);
|
||||
let ledger_dir = "test_encrypt_file_many_keys_multiple";
|
||||
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||
{
|
||||
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
|
||||
writer.write_entries(entries.clone()).unwrap();
|
||||
}
|
||||
|
||||
let out_path = Path::new("test_chacha_encrypt_file_many_keys_multiple_output.txt.enc");
|
||||
|
||||
let samples = [0, 1, 3, 4, 5, 150];
|
||||
let mut ivecs = Vec::new();
|
||||
let mut ref_hashes: Vec<Hash> = vec![];
|
||||
for i in 0..2 {
|
||||
let mut ivec = hex!(
|
||||
"abc123abc123abc123abc123abc123abc123abababababababababababababab
|
||||
abc123abc123abc123abc123abc123abc123abababababababababababababab"
|
||||
);
|
||||
ivec[0] = i;
|
||||
ivecs.extend(ivec.clone().iter());
|
||||
assert!(
|
||||
chacha_cbc_encrypt_file(
|
||||
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
|
||||
out_path,
|
||||
&mut ivec,
|
||||
).is_ok()
|
||||
);
|
||||
|
||||
ref_hashes.push(sample_file(&out_path, &samples).unwrap());
|
||||
info!(
|
||||
"ivec: {:?} hash: {:?} ivecs: {:?}",
|
||||
ivec.to_vec(),
|
||||
ref_hashes.last(),
|
||||
ivecs
|
||||
);
|
||||
}
|
||||
|
||||
let hashes =
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap();
|
||||
|
||||
assert_eq!(hashes, ref_hashes);
|
||||
|
||||
let _ignored = remove_dir_all(&ledger_path);
|
||||
let _ignored = remove_file(out_path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_encrypt_file_many_keys_bad_key_length() {
|
||||
let keys = "thetestkey";
|
||||
let mut keys = hex!("abc123");
|
||||
let ledger_dir = "test_encrypt_file_many_keys_bad_key_length";
|
||||
let ledger_path = get_tmp_ledger_path(ledger_dir);
|
||||
let samples = [0];
|
||||
assert!(
|
||||
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, keys.as_bytes(), &samples,).is_err()
|
||||
);
|
||||
assert!(chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut keys, &samples,).is_err());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -91,6 +91,10 @@ extern crate chrono;
|
|||
extern crate clap;
|
||||
extern crate dirs;
|
||||
extern crate generic_array;
|
||||
#[cfg(test)]
|
||||
#[cfg(any(feature = "chacha", feature = "cuda"))]
|
||||
#[macro_use]
|
||||
extern crate hex_literal;
|
||||
extern crate ipnetwork;
|
||||
extern crate itertools;
|
||||
extern crate libc;
|
||||
|
|
|
@ -38,7 +38,7 @@ extern "C" {
|
|||
|
||||
pub fn chacha_cbc_encrypt_many_sample(
|
||||
input: *const u8,
|
||||
output: *mut u8,
|
||||
sha_state: *mut u8,
|
||||
in_len: usize,
|
||||
keys: *const u8,
|
||||
ivec: *mut u8,
|
||||
|
@ -48,6 +48,9 @@ extern "C" {
|
|||
starting_block: u64,
|
||||
time_us: *mut f32,
|
||||
);
|
||||
|
||||
pub fn chacha_init_sha_state(sha_state: *mut u8, num_keys: u32);
|
||||
pub fn chacha_end_sha_state(sha_state_in: *const u8, out: *mut u8, num_keys: u32);
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "cuda"))]
|
||||
|
|
Loading…
Reference in New Issue