From bf6396f505a76cbcb813d632bf8ba6af06d529e3 Mon Sep 17 00:00:00 2001 From: NikVolf Date: Fri, 9 Dec 2016 13:10:52 +0100 Subject: [PATCH] transaction & meta lru cache --- Cargo.lock | 16 ++++++++++++ db/Cargo.toml | 1 + db/src/lib.rs | 1 + db/src/storage.rs | 66 +++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ada53b9..1daa0e0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,6 +164,7 @@ dependencies = [ "elastic-array 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "ethcore-devtools 1.3.0", "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "lru-cache 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "parking_lot 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "primitives 0.1.0", "rocksdb 0.4.5 (git+https://github.com/ethcore/rust-rocksdb)", @@ -304,6 +305,11 @@ name = "libc" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "linked-hash-map" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "linked-hash-map" version = "0.3.0" @@ -324,6 +330,14 @@ dependencies = [ "time 0.1.35 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "lru-cache" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "memchr" version = "0.1.11" @@ -879,8 +893,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lazy_static 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6abe0ee2e758cd6bc8a2cd56726359007748fbf4128da998b65d0b70f881e19b" "checksum lazycell 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ce12306c4739d86ee97c23139f3a34ddf0387bbf181bc7929d287025a8c3ef6b" "checksum libc 0.2.17 (registry+https://github.com/rust-lang/crates.io-index)" = "044d1360593a78f5c8e5e710beccdc24ab71d1f01bc19a29bcacdba22e8475d8" +"checksum linked-hash-map 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bda158e0dabeb97ee8a401f4d17e479d6b891a14de0bba79d5cc2d4d325b5e48" "checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" +"checksum lru-cache 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "656fa4dfcb02bcf1063c592ba3ff6a5303ee1f2afe98c8a889e8b1a77c6dfdb7" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" "checksum mio 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "410a1a0ff76f5a226f1e4e3ff1756128e65cd30166e39c3892283e2ac09d5b67" "checksum miow 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "d5bfc6782530ac8ace97af10a540054a37126b63b0702ddaaa243b73b5745b9a" diff --git a/db/Cargo.toml b/db/Cargo.toml index c75acc83..dd92e8d1 100644 --- a/db/Cargo.toml +++ b/db/Cargo.toml @@ -15,6 +15,7 @@ parking_lot = "0.3" test-data = { path = "../test-data" } bit-vec = "0.4" log = "0.3" +lru-cache = "0.1.0" [features] dev = [] diff --git a/db/src/lib.rs b/db/src/lib.rs index 8594a209..0caac780 100644 --- a/db/src/lib.rs +++ b/db/src/lib.rs @@ -9,6 +9,7 @@ extern crate chain; extern crate serialization; extern crate bit_vec; #[macro_use] extern crate log; +extern crate lru_cache; #[cfg(test)] extern crate ethcore_devtools as devtools; diff --git a/db/src/storage.rs b/db/src/storage.rs index 03a01d1b..3eb7a506 100644 --- a/db/src/storage.rs +++ b/db/src/storage.rs @@ -10,8 +10,9 @@ use super::{BlockRef, BestBlock, BlockLocation, IndexedBlock, IndexedTransaction use serialization::{serialize, deserialize}; use chain; use parking_lot::RwLock; -use transaction_meta::TransactionMeta; +use lru_cache::LruCache; +use transaction_meta::TransactionMeta; use error::{Error, ConsistencyError, MetaError}; use update_context::UpdateContext; use block_provider::{BlockProvider, BlockHeaderProvider, AsBlockHeaderProvider}; @@ -36,6 +37,7 @@ const DB_VERSION: u32 = 1; // TODO: check how bitcoin core deals with long forks const MAX_FORK_ROUTE_PRESET: usize = 2048; +const TRANSACTION_CACHE_SIZE: usize = 524288; /// Blockchain storage interface pub trait Store : BlockProvider + BlockStapler + TransactionProvider + TransactionMetaProvider + AsBlockHeaderProvider { @@ -50,6 +52,8 @@ pub trait Store : BlockProvider + BlockStapler + TransactionProvider + Transacti pub struct Storage { database: Database, best_block: RwLock>, + transaction_cache: RwLock>, + meta_cache: RwLock>, } const KEY_VERSION: &'static[u8] = b"version"; @@ -83,6 +87,8 @@ impl Storage { let storage = Storage { database: db, best_block: RwLock::default(), + transaction_cache: RwLock::new(LruCache::new(TRANSACTION_CACHE_SIZE)), + meta_cache: RwLock::new(LruCache::new(TRANSACTION_CACHE_SIZE)), }; match storage.read_meta_u32(KEY_VERSION) { @@ -586,7 +592,11 @@ impl BlockStapler for Storage { // we always update best hash even if it is not changed context.db_transaction.put(Some(COL_META), KEY_BEST_BLOCK_HASH, &*new_best_hash); - // write accumulated transactions meta + // write accumulated transactions meta and update cache + { + let mut cache = self.meta_cache.write(); + for (hash, meta) in context.meta.iter() { cache.insert(hash.clone(), meta.clone()); } + } try!(context.apply(&self.database)); trace!(target: "db", "Best block now ({}, {})", &new_best_hash.to_reversed_str(), &new_best_number); @@ -626,18 +636,57 @@ impl TransactionProvider for Storage { } fn transaction(&self, hash: &H256) -> Option { - self.transaction_bytes(hash).map(|tx_bytes| { - deserialize(tx_bytes.as_ref()).expect("Failed to deserialize transaction: db corrupted?") - }) + let mut cache = self.transaction_cache.write(); + + let (tx, is_cached) = { + let cached_transaction = cache.get_mut(hash); + match cached_transaction { + None => { + ( + self.transaction_bytes(hash).map(|tx_bytes| { + let tx: chain::Transaction = deserialize(tx_bytes.as_ref()) + .expect("Failed to deserialize transaction: db corrupted?"); + tx + }), + false + ) + }, + Some(tx) => (Some(tx.clone()), true) + } + }; + + match tx { + Some(ref tx) => { if !is_cached { cache.insert(hash.clone(), tx.clone()); } } + None => {} + }; + + tx } } impl TransactionMetaProvider for Storage { fn transaction_meta(&self, hash: &H256) -> Option { - self.get(COL_TRANSACTIONS_META, &**hash).map(|val| - TransactionMeta::from_bytes(&val).expect("Invalid transaction metadata: db corrupted?") - ) + let mut cache = self.meta_cache.write(); + + let (meta, is_cached) = { + let cached_meta = cache.get_mut(hash); + match cached_meta { + None => { + (self.get(COL_TRANSACTIONS_META, &**hash).map(|val| + TransactionMeta::from_bytes(&val).expect("Invalid transaction metadata: db corrupted?") + ), false) + }, + Some(meta) => (Some(meta.clone()), true) + } + }; + + match meta { + Some(ref meta) => { if !is_cached { cache.insert(hash.clone(), meta.clone()); } } + None => {} + }; + + meta } } @@ -1196,6 +1245,7 @@ mod tests { store.decanonize_block(&mut update_context, &block_hash) .expect("Decanonizing block #1 which was just inserted should not fail"); update_context.apply(&store.database).unwrap(); + store.meta_cache.write().clear(); let genesis_meta = store.transaction_meta(&genesis_coinbase) .expect("Transaction meta for the genesis coinbase transaction should exist");