zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

change(deps): bump rocksdb from 0.18.0 to 0.19.0 (#5071) 

* cargo upgrade --workspace rocksdb

* Add a deny.toml exception for bindgen 0.59

* Move `valid()` and `status()` methods to raw iterators

* Update some outdated comments

* Panic on iterator failures

We might want to change this if there are any common failure modes.

* allow(clippy::unwrap_in_result) in some methods
This commit is contained in:
teor 2022-09-06 04:58:45 +10:00 committed by GitHub
parent 524e9ab123
commit fc624d009d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 74 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
Cargo.lock generated
View File

@ -431,6 +431,25 @@ dependencies = [
"which",
]
[[package]]
name = "bindgen"
version = "0.60.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
dependencies = [
"bitflags",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"peeking_take_while",
"proc-macro2 1.0.42",
"quote 1.0.20",
"regex",
"rustc-hash",
"shlex",
]
[[package]]
name = "bip0039"
version = "0.9.0"
@ -2506,11 +2525,11 @@ checksum = "33a33a362ce288760ec6a508b94caaec573ae7d3bbbd91b87aa0bad4456839db"
[[package]]
name = "librocksdb-sys"
version = "0.6.1+6.28.2"
version = "0.8.0+7.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291"
checksum = "611804e4666a25136fcc5f8cf425ab4d26c7f74ea245ffe92ea23b85b6420b5d"
dependencies = [
"bindgen",
"bindgen 0.60.1",
"bzip2-sys",
"cc",
"glob",
@ -3997,9 +4016,9 @@ dependencies = [
[[package]]
name = "rocksdb"
version = "0.18.0"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290"
checksum = "7e9562ea1d70c0cc63a34a22d977753b50cca91cc6b6527750463bd5dd8697bc"
dependencies = [
"libc",
"librocksdb-sys",
@ -6291,7 +6310,7 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81e4255f320dead417a91cbf00a178b0d702b813d5a8c95a5f2e4cc7ccced17f"
dependencies = [
"bindgen",
"bindgen 0.59.2",
"blake2b_simd",
"cc",
"cxx",

4
deny.toml
View File

@ -78,6 +78,10 @@ skip-tree = [
# wait for primitive-types to upgrade
{ name = "proc-macro-crate", version = "=0.1.5" },
# wait for zcash_script to upgrade bindgen
# https://github.com/ZcashFoundation/zcash_script/issues/40
{ name = "bindgen", version = "=0.59.2" },
]
# This section is considered when running `cargo deny check sources`.

2
zebra-state/Cargo.toml
View File

@ -21,7 +21,7 @@ metrics = "0.18.1"
mset = "0.1.0"
regex = "1.6.0"
rlimit = "0.8.3"
rocksdb = { version = "0.18.0", default_features = false, features = ["lz4"] }
rocksdb = { version = "0.19.0", default_features = false, features = ["lz4"] }
serde = { version = "1.0.144", features = ["serde_derive"] }
tempfile = "3.3.0"
thiserror = "1.0.33"

4
zebra-state/src/service.rs
View File

@ -714,6 +714,7 @@ impl Service<Request> for StateService {
let timer = CodeTimer::start();
// TODO: move this work into the future, like Block and Transaction?
// move disk reads to a blocking thread (#2188)
let rsp = Ok(Response::Depth(self.best_depth(hash)));
// The work is all done, the future just returns the result.
@ -734,6 +735,7 @@ impl Service<Request> for StateService {
let timer = CodeTimer::start();
// TODO: move this work into the future, like Block and Transaction?
// move disk reads to a blocking thread (#2188)
let rsp = Ok(Response::Tip(self.best_tip()));
// The work is all done, the future just returns the result.
@ -752,6 +754,7 @@ impl Service<Request> for StateService {
let timer = CodeTimer::start();
// TODO: move this work into the future, like Block and Transaction?
// move disk reads to a blocking thread (#2188)
let rsp = Ok(Response::BlockLocator(
self.block_locator().unwrap_or_default(),
));
@ -836,6 +839,7 @@ impl Service<Request> for StateService {
let fut = self.pending_utxos.queue(outpoint);
// TODO: move disk reads (in `any_utxo()`) to a blocking thread (#2188)
if let Some(utxo) = self.any_utxo(&outpoint) {
self.pending_utxos.respond(&outpoint, utxo);
}

48
zebra-state/src/service/finalized_state/disk_db.rs
View File

@ -210,10 +210,10 @@ impl ReadDisk for DiskDb {
// Empty column families return invalid forward iterators.
//
// Checking iterator validity does not seem to cause database hangs.
!self
.db
.iterator_cf(cf, rocksdb::IteratorMode::Start)
.valid()
let iterator = self.db.iterator_cf(cf, rocksdb::IteratorMode::Start);
let raw_iterator: rocksdb::DBRawIteratorWithThreadMode<DB> = iterator.into();
!raw_iterator.valid()
}
#[allow(clippy::unwrap_in_result)]
@ -228,12 +228,10 @@ impl ReadDisk for DiskDb {
// We use `get_pinned_cf` to avoid taking ownership of the serialized
// value, because we're going to deserialize it anyways, which avoids an
// extra copy
//
// TODO: move disk reads to a blocking thread (#2188)
let value_bytes = self
.db
.get_pinned_cf(cf, key_bytes)
.expect("expected that disk errors would not occur");
.expect("unexpected database failure");
value_bytes.map(V::from_bytes)
}
@ -247,14 +245,13 @@ impl ReadDisk for DiskDb {
// We use `get_pinned_cf` to avoid taking ownership of the serialized
// value, because we don't use the value at all. This avoids an extra copy.
//
// TODO: move disk reads to a blocking thread (#2188)
self.db
.get_pinned_cf(cf, key_bytes)
.expect("expected that disk errors would not occur")
.expect("unexpected database failure")
.is_some()
}
#[allow(clippy::unwrap_in_result)]
fn zs_first_key_value<C, K, V>(&self, cf: &C) -> Option<(K, V)>
where
C: rocksdb::AsColumnFamilyRef,
@ -264,10 +261,14 @@ impl ReadDisk for DiskDb {
// Reading individual values from iterators does not seem to cause database hangs.
self.db
.iterator_cf(cf, rocksdb::IteratorMode::Start)
.next()
.map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
.next()?
.map(|(key_bytes, value_bytes)| {
Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
})
.expect("unexpected database failure")
}
#[allow(clippy::unwrap_in_result)]
fn zs_last_key_value<C, K, V>(&self, cf: &C) -> Option<(K, V)>
where
C: rocksdb::AsColumnFamilyRef,
@ -277,10 +278,14 @@ impl ReadDisk for DiskDb {
// Reading individual values from iterators does not seem to cause database hangs.
self.db
.iterator_cf(cf, rocksdb::IteratorMode::End)
.next()
.map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
.next()?
.map(|(key_bytes, value_bytes)| {
Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
})
.expect("unexpected database failure")
}
#[allow(clippy::unwrap_in_result)]
fn zs_next_key_value_from<C, K, V>(&self, cf: &C, lower_bound: &K) -> Option<(K, V)>
where
C: rocksdb::AsColumnFamilyRef,
@ -293,10 +298,14 @@ impl ReadDisk for DiskDb {
// Reading individual values from iterators does not seem to cause database hangs.
self.db
.iterator_cf(cf, from)
.next()
.map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
.next()?
.map(|(key_bytes, value_bytes)| {
Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
})
.expect("unexpected database failure")
}
#[allow(clippy::unwrap_in_result)]
fn zs_prev_key_value_back_from<C, K, V>(&self, cf: &C, upper_bound: &K) -> Option<(K, V)>
where
C: rocksdb::AsColumnFamilyRef,
@ -309,8 +318,11 @@ impl ReadDisk for DiskDb {
// Reading individual values from iterators does not seem to cause database hangs.
self.db
.iterator_cf(cf, from)
.next()
.map(|(key_bytes, value_bytes)| (K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
.next()?
.map(|(key_bytes, value_bytes)| {
Some((K::from_bytes(key_bytes), V::from_bytes(value_bytes)))
})
.expect("unexpected database failure")
}
}

11
zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
View File

@ -36,7 +36,11 @@ use zebra_chain::{
};
use crate::{
service::finalized_state::{disk_db::DiskDb, disk_format::tests::KV, FinalizedState},
service::finalized_state::{
disk_db::{DiskDb, DB},
disk_format::tests::KV,
FinalizedState,
},
Config,
};
@ -129,6 +133,7 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
// The default raw data serialization is very verbose, so we hex-encode the bytes.
let cf_data: Vec<KV> = cf_iter
.by_ref()
.map(|result| result.expect("unexpected database error"))
.map(|(key, value)| KV::new(key, value))
.collect();
@ -144,8 +149,10 @@ fn snapshot_raw_rocksdb_column_family_data(db: &DiskDb, original_cf_names: &[Str
insta::assert_ron_snapshot!(format!("{}_raw_data", cf_name), cf_data);
}
let raw_cf_iter: rocksdb::DBRawIteratorWithThreadMode<DB> = cf_iter.into();
assert_eq!(
cf_iter.status(),
raw_cf_iter.status(),
Ok(()),
"unexpected column family iterator error",
);

1
zebra-state/src/service/finalized_state/zebra_db/block/tests/snapshot.rs
View File

@ -462,6 +462,7 @@ fn snapshot_transparent_address_data(state: &FinalizedState, height: u32) {
.count();
let addresses: Vec<transparent::Address> = addresses
.map(|result| result.expect("unexpected database error"))
.map(|(key, _value)| transparent::Address::from_bytes(key))
.collect();