add a persistent ledger of index and data files
This commit is contained in:
parent
1a8da769b6
commit
cae88c90b1
225
src/ledger.rs
225
src/ledger.rs
|
@ -1,64 +1,154 @@
|
||||||
//! The `ledger` module provides functions for parallel verification of the
|
//! The `ledger` module provides functions for parallel verification of the
|
||||||
//! Proof of History ledger.
|
//! Proof of History ledger.
|
||||||
|
|
||||||
use bincode::{deserialize, serialize_into};
|
use bincode::{deserialize, deserialize_from, serialize_into};
|
||||||
use entry::Entry;
|
use entry::Entry;
|
||||||
use hash::Hash;
|
use hash::Hash;
|
||||||
use packet::{self, SharedBlob, BLOB_DATA_SIZE};
|
use packet::{self, SharedBlob, BLOB_DATA_SIZE};
|
||||||
use rayon::prelude::*;
|
use rayon::prelude::*;
|
||||||
use result::{Error, Result};
|
use result::{Error, Result};
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use std::io::Cursor;
|
use std::fs::{create_dir_all, File, OpenOptions};
|
||||||
|
use std::io;
|
||||||
|
use std::io::prelude::*;
|
||||||
|
use std::io::{Cursor, ErrorKind, Seek, SeekFrom};
|
||||||
|
use std::mem::size_of;
|
||||||
|
use std::path::Path;
|
||||||
use transaction::Transaction;
|
use transaction::Transaction;
|
||||||
use std::fs::{File, OpenOptions};
|
|
||||||
|
|
||||||
// ledger
|
// ledger window
|
||||||
pub struct Ledger{
|
pub struct LedgerWindow {
|
||||||
entry_len_len: usize;
|
index: File,
|
||||||
|
data: File,
|
||||||
entry_height: u64; // current index
|
}
|
||||||
|
|
||||||
pub struct Files {
|
impl LedgerWindow {
|
||||||
index: File; // an array of usize elements
|
// opens a Ledger in directory, provides "infinite" window
|
||||||
data: File; // concatenated entries
|
pub fn new(directory: String) -> io::Result<Self> {
|
||||||
|
let directory = Path::new(&directory);
|
||||||
|
|
||||||
|
let index = File::open(directory.join("index"))?;
|
||||||
|
let data = File::open(directory.join("data"))?;
|
||||||
|
|
||||||
|
Ok(LedgerWindow { index, data })
|
||||||
}
|
}
|
||||||
|
|
||||||
reader: Files;
|
pub fn get_entry(&mut self, index: u64) -> io::Result<Entry> {
|
||||||
writer: Files;
|
fn u64_at(file: &mut File, at: u64) -> io::Result<u64> {
|
||||||
|
file.seek(SeekFrom::Start(at))?;
|
||||||
|
deserialize_from(file.take(size_of::<u64>() as u64))
|
||||||
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn new(directory: String) -> Self {
|
let end_offset = u64_at(&mut self.index, index * size_of::<u64>() as u64)?;
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_entry_height(&self) -> u64 {
|
let start_offset = if index != 0 {
|
||||||
entry_height
|
u64_at(&mut self.index, (index - 1) * size_of::<u64>() as u64)?
|
||||||
}
|
} else {
|
||||||
|
0u64
|
||||||
pub fn entry_at(&self, index: u64) -> Result<Entry> {
|
};
|
||||||
|
|
||||||
}
|
fn entry_at(file: &mut File, at: u64, len: u64) -> io::Result<Entry> {
|
||||||
pub fn append_entry(&self, entry: &Entry) -> io::Result<u64> {
|
file.seek(SeekFrom::Start(at))?;
|
||||||
Ok(0)
|
|
||||||
}
|
deserialize_from(file.take(len))
|
||||||
pub fn append_entries(&self, entries: &[Entry]) -> io::Result<u64> {
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))
|
||||||
Ok(0)
|
}
|
||||||
|
|
||||||
|
entry_at(&mut self.data, start_offset, end_offset - start_offset)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<R: BufRead> Iterator for Ledger<R> {
|
pub struct LedgerWriter {
|
||||||
|
index: File,
|
||||||
|
data: File,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LedgerWriter {
|
||||||
|
// opens or creates a LedgerWriter in directory
|
||||||
|
pub fn new(directory: String) -> io::Result<Self> {
|
||||||
|
let directory = Path::new(&directory);
|
||||||
|
|
||||||
|
create_dir_all(directory)?;
|
||||||
|
|
||||||
|
let index = OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.append(true)
|
||||||
|
.open(directory.join("index"))?;
|
||||||
|
|
||||||
|
let data = OpenOptions::new()
|
||||||
|
.create(true)
|
||||||
|
.append(true)
|
||||||
|
.open(directory.join("data"))?;
|
||||||
|
|
||||||
|
Ok(LedgerWriter { index, data })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_entry(&mut self, entry: &Entry) -> io::Result<()> {
|
||||||
|
serialize_into(&mut self.data, &entry)
|
||||||
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))?;
|
||||||
|
self.data.flush()?;
|
||||||
|
|
||||||
|
let offset = self.data.seek(SeekFrom::Current(0))?;
|
||||||
|
serialize_into(&mut self.index, &offset)
|
||||||
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))?;
|
||||||
|
self.index.flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn write_entries<I>(&mut self, entries: I) -> io::Result<()>
|
||||||
|
where
|
||||||
|
I: IntoIterator<Item = Entry>,
|
||||||
|
{
|
||||||
|
for entry in entries {
|
||||||
|
self.write_entry(&entry)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct LedgerReader {
|
||||||
|
offset: u64, // next start_offset
|
||||||
|
index: File,
|
||||||
|
data: File,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Iterator for LedgerReader {
|
||||||
type Item = io::Result<Entry>;
|
type Item = io::Result<Entry>;
|
||||||
|
|
||||||
fn next(&mut self) -> Option<io::Result<Entry>> {
|
fn next(&mut self) -> Option<io::Result<Entry>> {
|
||||||
let mut entry_len_bytes = [0u8; sizeof(::<usize>()]; // TODO: sizeof()?
|
fn next_offset(file: &mut File) -> io::Result<u64> {
|
||||||
|
deserialize_from(file.take(size_of::<u64>() as u64))
|
||||||
let mut entry_len =
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))
|
||||||
if self.reader.index
|
}
|
||||||
.read_exact(&mut entry_len_bytes[..self.entry_len_len])
|
|
||||||
.is_ok()
|
|
||||||
|
|
||||||
|
fn next_entry(file: &mut File, len: u64) -> io::Result<Entry> {
|
||||||
|
deserialize_from(file.take(len))
|
||||||
|
.map_err(|e| io::Error::new(ErrorKind::Other, e.to_string()))
|
||||||
|
}
|
||||||
|
match next_offset(&mut self.index) {
|
||||||
|
Ok(end_offset) => {
|
||||||
|
let len = end_offset - self.offset;
|
||||||
|
self.offset = end_offset;
|
||||||
|
Some(next_entry(&mut self.data, len))
|
||||||
|
}
|
||||||
|
Err(_) => None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return an iterator for all the entries in the given file.
|
||||||
|
pub fn read_ledger(directory: String) -> io::Result<impl Iterator<Item = io::Result<Entry>>> {
|
||||||
|
let directory = Path::new(&directory);
|
||||||
|
|
||||||
|
let index = File::open(directory.join("index"))?;
|
||||||
|
let data = File::open(directory.join("data"))?;
|
||||||
|
|
||||||
|
Ok(LedgerReader {
|
||||||
|
offset: 0,
|
||||||
|
index,
|
||||||
|
data,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// a Block is a slice of Entries
|
// a Block is a slice of Entries
|
||||||
pub trait Block {
|
pub trait Block {
|
||||||
|
@ -204,6 +294,7 @@ mod tests {
|
||||||
use hash::hash;
|
use hash::hash;
|
||||||
use packet::{BlobRecycler, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
|
use packet::{BlobRecycler, BLOB_DATA_SIZE, PACKET_DATA_SIZE};
|
||||||
use signature::{KeyPair, KeyPairUtil};
|
use signature::{KeyPair, KeyPairUtil};
|
||||||
|
use std;
|
||||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||||
use transaction::{Transaction, Vote};
|
use transaction::{Transaction, Vote};
|
||||||
|
|
||||||
|
@ -221,8 +312,7 @@ mod tests {
|
||||||
assert!(!bad_ticks.verify(&zero)); // inductive step, bad
|
assert!(!bad_ticks.verify(&zero)); // inductive step, bad
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
fn make_test_entries() -> Vec<Entry> {
|
||||||
fn test_entries_to_blobs() {
|
|
||||||
let zero = Hash::default();
|
let zero = Hash::default();
|
||||||
let one = hash(&zero);
|
let one = hash(&zero);
|
||||||
let keypair = KeyPair::new();
|
let keypair = KeyPair::new();
|
||||||
|
@ -248,7 +338,13 @@ mod tests {
|
||||||
// V
|
// V
|
||||||
let mut transactions = vec![tx0; 362];
|
let mut transactions = vec![tx0; 362];
|
||||||
transactions.extend(vec![tx1; 100]);
|
transactions.extend(vec![tx1; 100]);
|
||||||
let entries = next_entries(&zero, 0, transactions);
|
next_entries(&zero, 0, transactions);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_entries_to_blobs() {
|
||||||
|
let entries = make_test_entries();
|
||||||
|
|
||||||
let blob_recycler = BlobRecycler::default();
|
let blob_recycler = BlobRecycler::default();
|
||||||
let mut blob_q = VecDeque::new();
|
let mut blob_q = VecDeque::new();
|
||||||
entries.to_blobs(&blob_recycler, &mut blob_q);
|
entries.to_blobs(&blob_recycler, &mut blob_q);
|
||||||
|
@ -316,17 +412,46 @@ mod tests {
|
||||||
assert!(entries0[0].has_more);
|
assert!(entries0[0].has_more);
|
||||||
assert!(!entries0[entries0.len() - 1].has_more);
|
assert!(!entries0[entries0.len() - 1].has_more);
|
||||||
assert!(entries0.verify(&id));
|
assert!(entries0.verify(&id));
|
||||||
// test hand-construction... brittle, changes if split method changes... ?
|
|
||||||
// let mut entries1 = vec![];
|
|
||||||
// entries1.push(Entry::new(&id, 1, transactions[..threshold].to_vec(), true));
|
|
||||||
// id = entries1[0].id;
|
|
||||||
// entries1.push(Entry::new(
|
|
||||||
// &id,
|
|
||||||
// 1,
|
|
||||||
// transactions[threshold..].to_vec(),
|
|
||||||
// false,
|
|
||||||
// ));
|
|
||||||
//
|
|
||||||
// assert_eq!(entries0, entries1);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_ledger_reader_writer() {
|
||||||
|
let keypair = KeyPair::new();
|
||||||
|
|
||||||
|
let ledger_path = {
|
||||||
|
let id = {
|
||||||
|
let ids: Vec<_> = keypair
|
||||||
|
.pubkey()
|
||||||
|
.iter()
|
||||||
|
.map(|id| format!("{}", id))
|
||||||
|
.collect();
|
||||||
|
ids.join("")
|
||||||
|
};
|
||||||
|
|
||||||
|
format!("target/test_ledger_reader_writer_window-{}", id)
|
||||||
|
};
|
||||||
|
|
||||||
|
let entries = make_test_entries();
|
||||||
|
|
||||||
|
let mut writer = LedgerWriter::new(ledger_path.clone()).unwrap();
|
||||||
|
writer.write_entries(entries.clone()).unwrap();
|
||||||
|
|
||||||
|
let mut read_entries = vec![];
|
||||||
|
for x in read_ledger(ledger_path.clone()).unwrap() {
|
||||||
|
let entry = x.unwrap();
|
||||||
|
trace!("entry... {:?}", entry);
|
||||||
|
read_entries.push(entry);
|
||||||
|
}
|
||||||
|
assert_eq!(read_entries, entries);
|
||||||
|
|
||||||
|
let mut window = LedgerWindow::new(ledger_path.clone()).unwrap();
|
||||||
|
|
||||||
|
for (i, entry) in entries.iter().enumerate() {
|
||||||
|
let read_entry = window.get_entry(i as u64).unwrap();
|
||||||
|
assert_eq!(*entry, read_entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
std::fs::remove_dir_all(ledger_path).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue