blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Storage mining fixups... 

* Use IV to make unique identies
* Use hex! macro for hex literal and not string converted to u8 slice
* fix sha sampling to control init/end of sha state
This commit is contained in:
Stephen Akridge 2018-10-19 10:39:13 -07:00 committed by sakridge
parent 3b33150cfb
commit ead7f4287a
7 changed files with 130 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.toml
View File

@ -79,6 +79,7 @@ elf = "0.0.10"
env_logger = "0.5.12"
generic-array = { version = "0.12.0", default-features = false, features = ["serde"] }
getopts = "0.2"
hex-literal = "0.1.1"
influx_db_client = "0.3.4"
solana-jsonrpc-core = "0.3.0"
solana-jsonrpc-http-server = "0.3.0"

2
fetch-perf-libs.sh
View File

@ -15,7 +15,7 @@ mkdir -p target/perf-libs
cd target/perf-libs
(
set -x
curl https://solana-perf.s3.amazonaws.com/v0.10.0/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
curl https://solana-perf.s3.amazonaws.com/v0.10.1/x86_64-unknown-linux-gnu/solana-perf.tgz | tar zxvf -
)
if [[ -r /usr/local/cuda/version.txt && -r cuda-version.txt ]]; then

13
src/bin/replicator.rs
View File

@ -6,7 +6,7 @@ extern crate serde_json;
extern crate solana;
use clap::{App, Arg};
use solana::chacha::chacha_cbc_encrypt_files;
use solana::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE};
use solana::client::mk_client;
use solana::cluster_info::Node;
use solana::drone::DRONE_PORT;
@ -114,13 +114,12 @@ fn main() {
let ledger_path = Path::new(ledger_path.unwrap());
let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE);
let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE));
let key = "abc123";
let mut ivec = [0u8; CHACHA_BLOCK_SIZE];
ivec[0..4].copy_from_slice(&[2, 3, 4, 5]);
if let Err(e) = chacha_cbc_encrypt_files(
&ledger_data_file,
&ledger_data_file_encrypted,
key.to_string(),
) {
if let Err(e) =
chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec)
{
println!("Error while encrypting ledger: {:?}", e);
return;
}

28
src/chacha.rs
View File

@ -5,7 +5,8 @@ use std::io::Write;
use std::io::{BufReader, BufWriter};
use std::path::Path;
const CHACHA_IVEC_SIZE: usize = 64;
pub const CHACHA_BLOCK_SIZE: usize = 64;
pub const CHACHA_KEY_SIZE: usize = 32;
#[link(name = "cpu-crypt")]
extern "C" {
@ -30,25 +31,24 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu
}
}
pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) -> io::Result<()> {
pub fn chacha_cbc_encrypt_file(
in_path: &Path,
out_path: &Path,
ivec: &mut [u8; CHACHA_BLOCK_SIZE],
) -> io::Result<()> {
let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file"));
let mut out_file =
BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
let mut buffer = [0; 4 * 1024];
let mut encrypted_buffer = [0; 4 * 1024];
let mut ivec = [0; CHACHA_IVEC_SIZE];
let key = [0; CHACHA_KEY_SIZE];
while let Ok(size) = in_file.read(&mut buffer) {
debug!("read {} bytes", size);
if size == 0 {
break;
}
chacha_cbc_encrypt(
&buffer[..size],
&mut encrypted_buffer[..size],
key.as_bytes(),
&mut ivec,
);
chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
println!("Error writing file! {:?}", res);
return Err(res);
@ -59,7 +59,7 @@ pub fn chacha_cbc_encrypt_files(in_path: &Path, out_path: &Path, key: String) ->
#[cfg(test)]
mod tests {
use chacha::chacha_cbc_encrypt_files;
use chacha::chacha_cbc_encrypt_file;
use std::fs::remove_file;
use std::fs::File;
use std::io::Read;
@ -74,13 +74,17 @@ mod tests {
let mut in_file = File::create(in_path).unwrap();
in_file.write("123456foobar".as_bytes()).unwrap();
}
assert!(chacha_cbc_encrypt_files(in_path, out_path, "thetestkey".to_string()).is_ok());
let mut key = hex!(
"abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
);
assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok());
let mut out_file = File::open(out_path).unwrap();
let mut buf = vec![];
let size = out_file.read_to_end(&mut buf).unwrap();
assert_eq!(
buf[..size],
[106, 186, 59, 108, 165, 33, 118, 212, 70, 238, 205, 185]
[66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53]
);
remove_file(in_path).unwrap();
remove_file(out_path).unwrap();

127
src/chacha_cuda.rs
View File

@ -1,44 +1,51 @@
use chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
use hash::Hash;
use ledger::LedgerWindow;
use sigverify::chacha_cbc_encrypt_many_sample;
use sigverify::{chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state};
use std::io;
use std::mem::size_of;
const CHACHA_BLOCK_SIZE: usize = 64;
const SHA256_BLOCK_SIZE: usize = 32;
const CHACHA_KEY_LEN: usize = 32;
const ENTRIES_PER_SLICE: u64 = 16;
// Encrypt a file with multiple starting IV states, determined by ivecs.len()
//
// Then sample each block at the offsets provided by samples argument with sha256
// and return the vec of sha states
pub fn chacha_cbc_encrypt_file_many_keys(
in_path: &str,
slice: u64,
keys: &[u8],
ivecs: &mut [u8],
samples: &[u64],
) -> io::Result<Vec<Hash>> {
if keys.len() % CHACHA_KEY_LEN != 0 {
if ivecs.len() % CHACHA_BLOCK_SIZE != 0 {
return Err(io::Error::new(
io::ErrorKind::Other,
format!(
"bad key length({}) not divisible by {} ",
keys.len(),
CHACHA_KEY_LEN
"bad IV length({}) not divisible by {} ",
ivecs.len(),
CHACHA_BLOCK_SIZE,
),
));
}
let mut ledger_window = LedgerWindow::open(in_path)?;
let mut buffer = [0; 8 * 1024];
let num_keys = keys.len() / CHACHA_KEY_LEN;
let mut sha_states = vec![0; num_keys * SHA256_BLOCK_SIZE];
let mut ivecs: Vec<u8> = vec![0; num_keys * CHACHA_BLOCK_SIZE];
let num_keys = ivecs.len() / CHACHA_BLOCK_SIZE;
let mut sha_states = vec![0; num_keys * size_of::<Hash>()];
let mut int_sha_states = vec![0; num_keys * 112];
let keys: Vec<u8> = vec![0; num_keys * CHACHA_KEY_SIZE]; // keys not used ATM, uniqueness comes from IV
let mut entry = slice;
let mut total_entries = 0;
let mut total_entry_len = 0;
let mut time: f32 = 0.0;
unsafe {
chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32);
}
loop {
match ledger_window.get_entries_bytes(entry, ENTRIES_PER_SLICE - total_entries, &mut buffer)
{
Ok((num_entries, entry_len)) => {
debug!(
info!(
"encrypting slice: {} num_entries: {} entry_len: {}",
slice, num_entries, entry_len
);
@ -46,7 +53,7 @@ pub fn chacha_cbc_encrypt_file_many_keys(
unsafe {
chacha_cbc_encrypt_many_sample(
buffer[..entry_len_usz].as_ptr(),
sha_states.as_mut_ptr(),
int_sha_states.as_mut_ptr(),
entry_len_usz,
keys.as_ptr(),
ivecs.as_mut_ptr(),
@ -75,10 +82,18 @@ pub fn chacha_cbc_encrypt_file_many_keys(
}
}
}
unsafe {
chacha_end_sha_state(
int_sha_states.as_ptr(),
sha_states.as_mut_ptr(),
num_keys as u32,
);
}
info!("num_keys: {}", num_keys);
let mut res = Vec::new();
for x in 0..num_keys {
let start = x * SHA256_BLOCK_SIZE;
let end = start + SHA256_BLOCK_SIZE;
let start = x * size_of::<Hash>();
let end = start + size_of::<Hash>();
res.push(Hash::new(&sha_states[start..end]));
}
Ok(res)
@ -86,8 +101,9 @@ pub fn chacha_cbc_encrypt_file_many_keys(
#[cfg(test)]
mod tests {
use chacha::chacha_cbc_encrypt_files;
use chacha::chacha_cbc_encrypt_file;
use chacha_cuda::chacha_cbc_encrypt_file_many_keys;
use hash::Hash;
use ledger::LedgerWriter;
use ledger::{get_tmp_ledger_path, make_tiny_test_entries, LEDGER_DATA_FILE};
use replicator::sample_file;
@ -95,35 +111,39 @@ mod tests {
use std::path::Path;
#[test]
fn test_encrypt_file_many_keys() {
fn test_encrypt_file_many_keys_single() {
use logger;
logger::setup();
let entries = make_tiny_test_entries(16);
let ledger_dir = "test_encrypt_file_many_keys";
let entries = make_tiny_test_entries(32);
let ledger_dir = "test_encrypt_file_many_keys_single";
let ledger_path = get_tmp_ledger_path(ledger_dir);
{
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
writer.write_entries(&entries).unwrap();
}
let out_path = Path::new("test_chacha_encrypt_file_many_keys_output.txt.enc");
let out_path = Path::new("test_chacha_encrypt_file_many_keys_single_output.txt.enc");
let samples = [0];
let key = "thetestkeyabcdefghijklmnopqrstuvwxyzthetestkeyabcdefghijklmnopqr";
let mut ivecs = hex!(
"abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
);
let mut cpu_iv = ivecs.clone();
assert!(
chacha_cbc_encrypt_files(
chacha_cbc_encrypt_file(
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
out_path,
key.to_string()
&mut cpu_iv,
).is_ok()
);
let ref_hash = sample_file(&out_path, &samples).unwrap();
let hashes =
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, key.as_bytes(), &samples).unwrap();
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap();
assert_eq!(hashes[0], ref_hash);
@ -131,14 +151,63 @@ mod tests {
let _ignored = remove_file(out_path);
}
#[test]
fn test_encrypt_file_many_keys_multiple_keys() {
use logger;
logger::setup();
let entries = make_tiny_test_entries(32);
let ledger_dir = "test_encrypt_file_many_keys_multiple";
let ledger_path = get_tmp_ledger_path(ledger_dir);
{
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
writer.write_entries(entries.clone()).unwrap();
}
let out_path = Path::new("test_chacha_encrypt_file_many_keys_multiple_output.txt.enc");
let samples = [0, 1, 3, 4, 5, 150];
let mut ivecs = Vec::new();
let mut ref_hashes: Vec<Hash> = vec![];
for i in 0..2 {
let mut ivec = hex!(
"abc123abc123abc123abc123abc123abc123abababababababababababababab
abc123abc123abc123abc123abc123abc123abababababababababababababab"
);
ivec[0] = i;
ivecs.extend(ivec.clone().iter());
assert!(
chacha_cbc_encrypt_file(
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
out_path,
&mut ivec,
).is_ok()
);
ref_hashes.push(sample_file(&out_path, &samples).unwrap());
info!(
"ivec: {:?} hash: {:?} ivecs: {:?}",
ivec.to_vec(),
ref_hashes.last(),
ivecs
);
}
let hashes =
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut ivecs, &samples).unwrap();
assert_eq!(hashes, ref_hashes);
let _ignored = remove_dir_all(&ledger_path);
let _ignored = remove_file(out_path);
}
#[test]
fn test_encrypt_file_many_keys_bad_key_length() {
let keys = "thetestkey";
let mut keys = hex!("abc123");
let ledger_dir = "test_encrypt_file_many_keys_bad_key_length";
let ledger_path = get_tmp_ledger_path(ledger_dir);
let samples = [0];
assert!(
chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, keys.as_bytes(), &samples,).is_err()
);
assert!(chacha_cbc_encrypt_file_many_keys(&ledger_path, 0, &mut keys, &samples,).is_err());
}
}

4
src/lib.rs
View File

@ -91,6 +91,10 @@ extern crate chrono;
extern crate clap;
extern crate dirs;
extern crate generic_array;
#[cfg(test)]
#[cfg(any(feature = "chacha", feature = "cuda"))]
#[macro_use]
extern crate hex_literal;
extern crate ipnetwork;
extern crate itertools;
extern crate libc;

5
src/sigverify.rs
View File

@ -38,7 +38,7 @@ extern "C" {
pub fn chacha_cbc_encrypt_many_sample(
input: *const u8,
output: *mut u8,
sha_state: *mut u8,
in_len: usize,
keys: *const u8,
ivec: *mut u8,
@ -48,6 +48,9 @@ extern "C" {
starting_block: u64,
time_us: *mut f32,
);
pub fn chacha_init_sha_state(sha_state: *mut u8, num_keys: u32);
pub fn chacha_end_sha_state(sha_state_in: *const u8, out: *mut u8, num_keys: u32);
}
#[cfg(not(feature = "cuda"))]