blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Convert chacha_encrypt_file to work with db_ledger blobs directly

This commit is contained in:
Stephen Akridge 2019-01-07 17:08:03 -08:00 committed by sakridge
parent a904e15ecc
commit 4b3d64ec9f
4 changed files with 120 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
src/chacha.rs
View File

@ -1,9 +1,11 @@
use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
use std::fs::File;
use std::io;
use std::io::Read;
use std::io::Write;
use std::io::{BufReader, BufWriter};
use std::io::{BufWriter, Write};
use std::path::Path;
use std::sync::Arc;
use crate::storage_stage::ENTRIES_PER_SEGMENT;
pub const CHACHA_BLOCK_SIZE: usize = 64;
pub const CHACHA_KEY_SIZE: usize = 32;
@ -31,27 +33,46 @@ pub fn chacha_cbc_encrypt(input: &[u8], output: &mut [u8], key: &[u8], ivec: &mu
}
}
pub fn chacha_cbc_encrypt_file(
in_path: &Path,
pub fn chacha_cbc_encrypt_ledger(
db_ledger: &Arc<DbLedger>,
slice: u64,
out_path: &Path,
ivec: &mut [u8; CHACHA_BLOCK_SIZE],
) -> io::Result<()> {
let mut in_file = BufReader::new(File::open(in_path).expect("Can't open ledger data file"));
let mut out_file =
BufWriter::new(File::create(out_path).expect("Can't open ledger encrypted data file"));
let mut buffer = [0; 4 * 1024];
let mut encrypted_buffer = [0; 4 * 1024];
let key = [0; CHACHA_KEY_SIZE];
let mut total_entries = 0;
let mut entry = slice;
while let Ok(size) = in_file.read(&mut buffer) {
debug!("read {} bytes", size);
if size == 0 {
break;
}
chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
println!("Error writing file! {:?}", res);
return Err(res);
loop {
match db_ledger.get_blob_bytes(
entry,
ENTRIES_PER_SEGMENT - total_entries,
&mut buffer,
DEFAULT_SLOT_HEIGHT,
) {
Ok((num_entries, entry_len)) => {
debug!("read {} bytes", entry_len);
let size = entry_len as usize;
if size == 0 {
break;
}
chacha_cbc_encrypt(&buffer[..size], &mut encrypted_buffer[..size], &key, ivec);
if let Err(res) = out_file.write(&encrypted_buffer[..size]) {
println!("Error writing file! {:?}", res);
return Err(res);
}
total_entries += num_entries;
entry += num_entries;
}
Err(e) => {
info!("Error encrypting file: {:?}", e);
break;
}
}
}
Ok(())
@ -59,34 +80,81 @@ pub fn chacha_cbc_encrypt_file(
#[cfg(test)]
mod tests {
use crate::chacha::chacha_cbc_encrypt_file;
use crate::chacha::chacha_cbc_encrypt_ledger;
use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
use crate::entry::Entry;
use crate::ledger::get_tmp_ledger_path;
use ring::signature::Ed25519KeyPair;
use solana_sdk::budget_transaction::BudgetTransaction;
use solana_sdk::hash::{hash, Hash, Hasher};
use solana_sdk::signature::KeypairUtil;
use solana_sdk::transaction::Transaction;
use std::fs::remove_file;
use std::fs::File;
use std::io::Read;
use std::io::Write;
use std::path::Path;
use std::sync::Arc;
use untrusted::Input;
fn make_tiny_deterministic_test_entries(num: usize) -> Vec<Entry> {
let zero = Hash::default();
let one = hash(&zero.as_ref());
let pkcs = [
48, 83, 2, 1, 1, 48, 5, 6, 3, 43, 101, 112, 4, 34, 4, 32, 109, 148, 235, 20, 97, 127,
43, 194, 109, 43, 121, 76, 54, 38, 234, 14, 108, 68, 209, 227, 137, 191, 167, 144, 177,
174, 57, 182, 79, 198, 196, 93, 161, 35, 3, 33, 0, 116, 121, 255, 78, 31, 95, 179, 172,
30, 125, 206, 87, 88, 78, 46, 145, 25, 154, 161, 252, 3, 58, 235, 116, 39, 148, 193,
150, 111, 61, 20, 226,
];
let keypair = Ed25519KeyPair::from_pkcs8(Input::from(&pkcs)).unwrap();
let mut id = one;
let mut num_hashes = 0;
(0..num)
.map(|_| {
Entry::new_mut(
&mut id,
&mut num_hashes,
vec![Transaction::budget_new_signature(
&keypair,
keypair.pubkey(),
keypair.pubkey(),
one,
)],
)
})
.collect()
}
#[test]
fn test_encrypt_file() {
let in_path = Path::new("test_chacha_encrypt_file_input.txt");
fn test_encrypt_ledger() {
solana_logger::setup();
let ledger_dir = "chacha_test_encrypt_file";
let ledger_path = get_tmp_ledger_path(ledger_dir);
let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
let out_path = Path::new("test_chacha_encrypt_file_output.txt.enc");
{
let mut in_file = File::create(in_path).unwrap();
in_file.write("123456foobar".as_bytes()).unwrap();
}
let entries = make_tiny_deterministic_test_entries(32);
db_ledger
.write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
.unwrap();
let mut key = hex!(
"abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234
abcd1234abcd1234abcd1234abcd1234 abcd1234abcd1234abcd1234abcd1234"
);
assert!(chacha_cbc_encrypt_file(in_path, out_path, &mut key).is_ok());
assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut key).is_ok());
let mut out_file = File::open(out_path).unwrap();
let mut buf = vec![];
let size = out_file.read_to_end(&mut buf).unwrap();
let mut hasher = Hasher::default();
hasher.hash(&buf[..size]);
assert_eq!(
buf[..size],
[66, 54, 56, 212, 142, 110, 105, 158, 116, 82, 120, 53]
hasher.result(),
Hash::new(&hex!(
"58433c941060af56b72bfeaca161f19ed6df531efb28961dc6b83f53fbf66ffe"
)),
);
remove_file(in_path).unwrap();
remove_file(out_path).unwrap();
}
}

40
src/chacha_cuda.rs
View File

@ -1,5 +1,5 @@
use crate::chacha::{CHACHA_BLOCK_SIZE, CHACHA_KEY_SIZE};
use crate::db_ledger::DbLedger;
use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
use crate::sigverify::{
chacha_cbc_encrypt_many_sample, chacha_end_sha_state, chacha_init_sha_state,
};
@ -44,7 +44,12 @@ pub fn chacha_cbc_encrypt_file_many_keys(
chacha_init_sha_state(int_sha_states.as_mut_ptr(), num_keys as u32);
}
loop {
match db_ledger.get_entries_bytes(entry, ENTRIES_PER_SEGMENT - total_entries, &mut buffer) {
match db_ledger.get_blob_bytes(
entry,
ENTRIES_PER_SEGMENT - total_entries,
&mut buffer,
DEFAULT_SLOT_HEIGHT,
) {
Ok((num_entries, entry_len)) => {
info!(
"encrypting slice: {} num_entries: {} entry_len: {}",
@ -101,12 +106,10 @@ pub fn chacha_cbc_encrypt_file_many_keys(
#[cfg(test)]
mod tests {
use crate::chacha::chacha_cbc_encrypt_file;
use crate::chacha::chacha_cbc_encrypt_ledger;
use crate::chacha_cuda::chacha_cbc_encrypt_file_many_keys;
use crate::db_ledger::{DbLedger, DEFAULT_SLOT_HEIGHT};
use crate::ledger::{
get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter, LEDGER_DATA_FILE,
};
use crate::ledger::{get_tmp_ledger_path, make_tiny_test_entries, LedgerWriter};
use crate::replicator::sample_file;
use solana_sdk::hash::Hash;
use std::fs::{remove_dir_all, remove_file};
@ -114,7 +117,6 @@ mod tests {
use std::sync::Arc;
#[test]
#[ignore]
fn test_encrypt_file_many_keys_single() {
solana_logger::setup();
@ -125,7 +127,7 @@ mod tests {
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
writer.write_entries(&entries).unwrap();
}
let db_ledger = DbLedger::open(&ledger_path).unwrap();
let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
db_ledger
.write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
.unwrap();
@ -139,18 +141,12 @@ mod tests {
);
let mut cpu_iv = ivecs.clone();
assert!(chacha_cbc_encrypt_file(
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
out_path,
&mut cpu_iv,
)
.is_ok());
assert!(chacha_cbc_encrypt_ledger(&db_ledger, 0, out_path, &mut cpu_iv,).is_ok());
let ref_hash = sample_file(&out_path, &samples).unwrap();
let hashes =
chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples)
.unwrap();
chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap();
assert_eq!(hashes[0], ref_hash);
@ -170,7 +166,7 @@ mod tests {
let mut writer = LedgerWriter::open(&ledger_path, true).unwrap();
writer.write_entries(&entries).unwrap();
}
let db_ledger = DbLedger::open(&ledger_path).unwrap();
let db_ledger = Arc::new(DbLedger::open(&ledger_path).unwrap());
db_ledger
.write_entries(DEFAULT_SLOT_HEIGHT, 0, &entries)
.unwrap();
@ -187,12 +183,7 @@ mod tests {
);
ivec[0] = i;
ivecs.extend(ivec.clone().iter());
assert!(chacha_cbc_encrypt_file(
&Path::new(&ledger_path).join(LEDGER_DATA_FILE),
out_path,
&mut ivec,
)
.is_ok());
assert!(chacha_cbc_encrypt_ledger(&db_ledger.clone(), 0, out_path, &mut ivec,).is_ok());
ref_hashes.push(sample_file(&out_path, &samples).unwrap());
info!(
@ -204,8 +195,7 @@ mod tests {
}
let hashes =
chacha_cbc_encrypt_file_many_keys(&Arc::new(db_ledger), 0, &mut ivecs, &samples)
.unwrap();
chacha_cbc_encrypt_file_many_keys(&db_ledger, 0, &mut ivecs, &samples).unwrap();
assert_eq!(hashes, ref_hashes);

2
src/ledger.rs
View File

@ -69,7 +69,7 @@ struct LedgerWindow {
data: BufReader<File>,
}
pub const LEDGER_DATA_FILE: &str = "data";
const LEDGER_DATA_FILE: &str = "data";
const LEDGER_INDEX_FILE: &str = "index";
// use a CONST because there's a cast, and we don't want "sizeof::<u64> as u64"...

15
src/replicator.rs
View File

@ -1,12 +1,11 @@
use crate::blob_fetch_stage::BlobFetchStage;
#[cfg(feature = "chacha")]
use crate::chacha::{chacha_cbc_encrypt_file, CHACHA_BLOCK_SIZE};
use crate::chacha::{chacha_cbc_encrypt_ledger, CHACHA_BLOCK_SIZE};
use crate::client::mk_client;
use crate::cluster_info::{ClusterInfo, Node, NodeInfo};
use crate::db_ledger::DbLedger;
use crate::gossip_service::GossipService;
use crate::leader_scheduler::LeaderScheduler;
use crate::ledger::LEDGER_DATA_FILE;
use crate::result::Result;
use crate::rpc_request::{RpcClient, RpcRequest};
use crate::service::Service;
@ -185,7 +184,7 @@ impl Replicator {
let (retransmit_sender, retransmit_receiver) = channel();
let t_window = window_service(
db_ledger,
db_ledger.clone(),
cluster_info.clone(),
0,
entry_height,
@ -243,14 +242,18 @@ impl Replicator {
info!("Done downloading ledger at {}", ledger_path.unwrap());
let ledger_path = Path::new(ledger_path.unwrap());
let ledger_data_file_encrypted = ledger_path.join(format!("{}.enc", LEDGER_DATA_FILE));
let ledger_data_file_encrypted = ledger_path.join("ledger.enc");
#[cfg(feature = "chacha")]
{
let ledger_data_file = ledger_path.join(LEDGER_DATA_FILE);
let mut ivec = [0u8; CHACHA_BLOCK_SIZE];
ivec[0..4].copy_from_slice(&[2, 3, 4, 5]);
chacha_cbc_encrypt_file(&ledger_data_file, &ledger_data_file_encrypted, &mut ivec)?;
chacha_cbc_encrypt_ledger(
&db_ledger,
entry_height,
&ledger_data_file_encrypted,
&mut ivec,
)?;
}
info!("Done encrypting the ledger");