Merge pull request #51 from ethcore/db-up

Database upgrade
This commit is contained in:
Nikolay Volf 2016-10-28 01:16:43 +03:00 committed by GitHub
commit fca0130c3e
3 changed files with 164 additions and 18 deletions

View File

@ -11,6 +11,7 @@ use serialization;
use chain::{self, RepresentH256};
use parking_lot::RwLock;
use transaction_meta::TransactionMeta;
use std::collections::HashMap;
const COL_COUNT: u32 = 10;
const COL_META: u32 = 0;
@ -222,16 +223,38 @@ impl Storage {
}
/// update transactions metadata in the specified database transaction
fn update_transactions_meta(&self, db_transaction: &mut DBTransaction, accepted_txs: &[chain::Transaction]) {
for accepted_tx in accepted_txs.iter() {
for (input_idx, input) in accepted_tx.inputs.iter().enumerate() {
// todo : hard error when no meta?
let mut meta = self.transaction_meta(&input.previous_output.hash)
.unwrap_or(TransactionMeta::new(0, input_idx+1));
meta.note_used(input_idx);
db_transaction.put(Some(COL_TRANSACTIONS_META), &*input.previous_output.hash, &meta.to_bytes());
fn update_transactions_meta(&self, db_transaction: &mut DBTransaction, number: u32, accepted_txs: &[chain::Transaction]) {
let mut meta_buf = HashMap::<H256, TransactionMeta>::new();
for (accepted_tx_idx, accepted_tx) in accepted_txs.iter().enumerate() {
// adding unspent transaction meta
let meta = TransactionMeta::new(number, accepted_tx.outputs.len());
db_transaction.put(Some(COL_TRANSACTIONS_META), &*accepted_tx.hash(), &meta.to_bytes());
// marking used transactions as spent
if accepted_tx_idx > 0 {
for (input_idx, input) in accepted_tx.inputs.iter().enumerate() {
meta_buf.entry(input.previous_output.hash.clone())
.or_insert(
self.transaction_meta(&input.previous_output.hash)
.unwrap_or_else(|| panic!(
"No transaction metadata for {}! Corrupted DB? Reindex?",
&input.previous_output.hash
))
);
// either panics on inserts in the previous line
let mut meta = meta_buf.get_mut(&input.previous_output.hash)
.expect("We just inserted transaction hash above or paniced there");
meta.note_used(input_idx);
}
}
}
// actually saving meta
for (hash, meta) in meta_buf.drain() {
db_transaction.put(Some(COL_TRANSACTIONS_META), &*hash, &meta.to_bytes());
}
}
}
@ -301,7 +324,7 @@ impl Store for Storage {
if block.hash() == new_best_hash { best_number + 1 }
else { best_number }
},
None => 1,
None => 0,
};
let mut transaction = self.database.transaction();
@ -326,8 +349,11 @@ impl Store for Storage {
);
if best_block.as_ref().map(|b| b.number) != Some(new_best_number) {
self.update_transactions_meta(&mut transaction, &block.transactions()[1..]);
self.update_transactions_meta(&mut transaction, new_best_number, block.transactions());
transaction.write_u32(Some(COL_META), KEY_BEST_BLOCK_NUMBER, new_best_number);
// updating main chain height reference
transaction.put(Some(COL_BLOCK_HASHES), &u32_key(new_best_number), std::ops::Deref::deref(&block_hash))
}
transaction.put(Some(COL_META), KEY_BEST_BLOCK_HASH, std::ops::Deref::deref(&new_best_hash));
@ -359,9 +385,34 @@ mod tests {
use super::{Storage, Store};
use devtools::RandomTempPath;
use chain::{Block, RepresentH256};
use chain::{
Block, BlockHeader, Transaction, RepresentH256, TransactionOutput,
TransactionInput, OutPoint,
};
use super::super::BlockRef;
use test_data;
use primitives::hash::H256;
use primitives::bytes::Bytes;
fn dummy_coinbase_tx() -> Transaction {
Transaction {
version: 0,
inputs: vec![
TransactionInput {
previous_output: OutPoint { hash: H256::from(0), index: 0xffffffff },
script_sig: Bytes::new_with_len(0),
sequence: 0
}
],
outputs: vec![
TransactionOutput {
value: 0,
script_pubkey: Bytes::new_with_len(0),
}
],
lock_time: 0,
}
}
#[test]
fn open_store() {
@ -374,19 +425,35 @@ mod tests {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let block: Block = test_data::block1();
let block: Block = test_data::block_h1();
store.insert_block(&block).unwrap();
let loaded_block = store.block(BlockRef::Hash(block.hash())).unwrap();
assert_eq!(loaded_block.hash(), block.hash());
}
#[test]
fn insert_genesis() {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let block: Block = test_data::genesis();
store.insert_block(&block).unwrap();
assert_eq!(store.best_block_number(), Some(0));
assert_eq!(store.block_hash(0), Some(block.hash()));
}
#[test]
fn best_block_update() {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let block: Block = test_data::block1();
let genesis: Block = test_data::genesis();
store.insert_block(&genesis).unwrap();
let block: Block = test_data::block_h1();
store.insert_block(&block).unwrap();
assert_eq!(store.best_block_number(), Some(1));
@ -397,16 +464,16 @@ mod tests {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let block: Block = test_data::block1();
let block: Block = test_data::block_h1();
store.insert_block(&block).unwrap();
let another_block: Block = test_data::block_h169();
let another_block: Block = test_data::block_h9();
store.insert_block(&another_block).unwrap();
// did not update because `another_block` is not child of `block`
assert_eq!(store.best_block_hash(), Some(block.hash()));
// number should not be update also
assert_eq!(store.best_block_number(), Some(1));
assert_eq!(store.best_block_number(), Some(0));
}
#[test]
@ -414,7 +481,10 @@ mod tests {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let block: Block = test_data::block1();
let block: Block = test_data::block_h9();
store.insert_block(&block).unwrap();
let block: Block = test_data::block_h170();
let tx1 = block.transactions()[0].hash();
store.insert_block(&block).unwrap();
@ -422,4 +492,54 @@ mod tests {
assert_eq!(loaded_transaction.hash(), block.transactions()[0].hash());
}
#[test]
fn transaction_meta_update() {
let path = RandomTempPath::create_dir();
let store = Storage::new(path.as_path()).unwrap();
let genesis = test_data::genesis();
store.insert_block(&genesis).unwrap();
let genesis_coinbase = genesis.transactions()[0].hash();
let genesis_meta = store.transaction_meta(&genesis_coinbase).unwrap();
assert!(!genesis_meta.is_spent(0));
let forged_block = Block::new(
BlockHeader {
version: 0,
previous_header_hash: genesis.hash(),
merkle_root_hash: H256::from(0),
nbits: 0,
time: 0,
nonce: 0,
},
vec![
dummy_coinbase_tx(),
Transaction {
version: 0,
inputs: vec![
TransactionInput {
previous_output: OutPoint { hash: genesis_coinbase.clone(), index: 0 },
script_sig: Bytes::new_with_len(0),
sequence: 0
}
],
outputs: vec![
TransactionOutput {
value: 0,
script_pubkey: Bytes::new_with_len(0),
}
],
lock_time: 0,
},
]
);
store.insert_block(&forged_block).unwrap();
let genesis_meta = store.transaction_meta(&genesis_coinbase).unwrap();
assert!(genesis_meta.is_spent(0));
assert_eq!(store.best_block_number(), Some(1));
}
}

View File

@ -40,8 +40,12 @@ impl TransactionMeta {
Ok(TransactionMeta {
block_height: LittleEndian::read_u32(&bytes[0..4]),
spent: BitVec::from_bytes(&bytes[5..]),
spent: BitVec::from_bytes(&bytes[4..]),
})
}
pub fn height(&self) -> u32 { self.block_height }
pub fn is_spent(&self, idx: usize) -> bool { self.spent.get(idx).expect("Index should be verified by the caller") }
}

View File

@ -36,6 +36,7 @@ pub fn genesis() -> Block {
// https://webbtc.com/block/00000000d1145790a8694403d4063f323d499e655c83426834d4ce2f8dd4a2ee
// block with the first transaction
// also is the source for 181
pub fn block_h170() -> Block {
let block: Block = "0100000055bd840a78798ad0da853f68974f3d183e2bd1db6a842c1feecf222a00000000ff104ccb05421ab93e63f8c3ce5c2c2e9dbb37de2764b3a3175c8166562cac7d51b96a49ffff001d283e9e700201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0102ffffffff0100f2052a01000000434104d46c4968bde02899d2aa0963367c7a6ce34eec332b32e42e5f3407e052d64ac625da6f0718e7b302140434bd725706957c092db53805b821a85b23a7ac61725bac000000000100000001c997a5e56e104102fa209c6a852dd90660a20b2d9c352423edce25857fcd3704000000004847304402204e45e16932b8af514961a1d3a1a25fdf3f4f7732e9d624c6c61548ab5fb8cd410220181522ec8eca07de4860a4acdd12909d831cc56cbbac4622082221a8768d1d0901ffffffff0200ca9a3b00000000434104ae1a62fe09c5f51b13905f07f06b99a2f7159b2225f374cd378d71302fa28414e7aab37397f554a7df5f142c21c1b7303b8a0626f1baded5c72a704f7e6cd84cac00286bee0000000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac00000000".into();
block
@ -55,3 +56,24 @@ pub fn block_h9() -> Block {
let block: Block = "01000000c60ddef1b7618ca2348a46e868afc26e3efc68226c78aa47f8488c4000000000c997a5e56e104102fa209c6a852dd90660a20b2d9c352423edce25857fcd37047fca6649ffff001d28404f530101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0134ffffffff0100f2052a0100000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac00000000".into();
block
}
// https://webbtc.com/block/0000000066356691a4353dd8bdc2c60da20d68ad34fff93d8839a133b2a6d42a
// block with some interesting test case
pub fn block_h221() -> Block {
let block: Block = "01000000581d2b080bc47372c06cc5de8eb40386b00c72d4bdfecdd239c56ab600000000079f89b6e0f19f8c29d6c648ff390c9af2cf7c1da8eab6ae168cd208c745f467cc516b49ffff001d0171a0690201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029e00ffffffff0100f2052a01000000434104b089ed9912db1edb2e7338154e2ba9c4eaac2ea1b9f95600005aac1fde07315ef3c16125cc387373232ac6fa6453f2046e317e2295321a58f244d6d2692d597eac00000000010000000173805864da01f15093f7837607ab8be7c3705e29a9d4a12c9116d709f8911e59000000004a493046022100f493d504a670e6280bc76ee285accf2796fd6a630659d4fa55dccb793fc9346402210080ecfbe069101993eb01320bb1d6029c138b27835e20ad54c232c36ffe10786301ffffffff0100e1f50500000000434104ea0d6650c8305f1213a89c65fc8f4343a5dac8e985c869e51d3aa02879b57c60cff49fcb99314d02dfc612d654e4333150ef61fa569c1c66415602cae387baf7ac00000000".into();
block
}
// https://webbtc.com/block/0000000054487811fc4ff7a95be738aa5ad9320c394c482b27c0da28b227ad5d
// block for with transaction source for 221
pub fn block_h182() -> Block {
let block: Block = "01000000e5c6af65c46bd826723a83c1c29d9efa189320458dc5298a0c8655dc0000000030c2a0d34bfb4a10d35e8166e0f5a37bce02fc1b85ff983739a191197f010f2f40df6a49ffff001d2ce7ac9e0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0129ffffffff0100f2052a01000000434104b10dd882c04204481116bd4b41510e98c05a869af51376807341fc7e3892c9034835954782295784bfc763d9736ed4122c8bb13d6e02c0882cb7502ce1ae8287ac000000000100000001be141eb442fbc446218b708f40caeb7507affe8acff58ed992eb5ddde43c6fa1010000004847304402201f27e51caeb9a0988a1e50799ff0af94a3902403c3ad4068b063e7b4d1b0a76702206713f69bd344058b0dee55a9798759092d0916dbbc3e592fee43060005ddc17401ffffffff0200e1f5050000000043410401518fa1d1e1e3e162852d68d9be1c0abad5e3d6297ec95f1f91b909dc1afe616d6876f92918451ca387c4387609ae1a895007096195a824baf9c38ea98c09c3ac007ddaac0000000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac00000000".into();
block
}
// https://webbtc.com/block/0000000054487811fc4ff7a95be738aa5ad9320c394c482b27c0da28b227ad5d
// block for with transaction source for 182
pub fn block_h181() -> Block {
let block: Block = "01000000f2c8a8d2af43a9cd05142654e56f41d159ce0274d9cabe15a20eefb500000000366c2a0915f05db4b450c050ce7165acd55f823fee51430a8c993e0bdbb192ede5dc6a49ffff001d192d3f2f0201000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0128ffffffff0100f2052a0100000043410435f0d8366085f73906a48309728155532f24293ea59fe0b33a245c4b8d75f82c3e70804457b7f49322aa822196a7521e4931f809d7e489bccb4ff14758d170e5ac000000000100000001169e1e83e930853391bc6f35f605c6754cfead57cf8387639d3b4096c54f18f40100000048473044022027542a94d6646c51240f23a76d33088d3dd8815b25e9ea18cac67d1171a3212e02203baf203c6e7b80ebd3e588628466ea28be572fe1aaa3f30947da4763dd3b3d2b01ffffffff0200ca9a3b00000000434104b5abd412d4341b45056d3e376cd446eca43fa871b51961330deebd84423e740daa520690e1d9e074654c59ff87b408db903649623e86f1ca5412786f61ade2bfac005ed0b20000000043410411db93e1dcdb8a016b49840f8c53bc1eb68a382e97b1482ecad7b148a6909a5cb2e0eaddfb84ccf9744464f82e160bfa9b8b64f9d4c03f999b8643f656b412a3ac00000000".into();
block
}