zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Temporarily upgrade subtrees in forward height order

This commit is contained in:
teor 2023-09-15 16:54:59 +10:00
parent 1d889aa3a8
commit a9558be214
3 changed files with 23 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
zebra-state/src/service/finalized_state/disk_db.rs
View File

@ -258,7 +258,6 @@ impl PartialEq for DiskDb {
self.ephemeral, other.ephemeral,
"database with same path but different ephemeral configs",
);
return true;
}
@ -432,46 +431,6 @@ impl DiskDb {
///
/// Holding this iterator open might delay block commit transactions.
pub fn zs_range_iter<C, K, V, R>(&self, cf: &C, range: R) -> impl Iterator<Item = (K, V)> + '_
where
C: rocksdb::AsColumnFamilyRef,
K: IntoDisk + FromDisk,
V: FromDisk,
R: RangeBounds<K>,
{
self.zs_range_iter_with_direction(cf, range, false)
}
/// Returns a reverse iterator over the items in `cf` in `range`.
///
/// Holding this iterator open might delay block commit transactions.
///
/// This code is copied from `zs_range_iter()`, but with the mode reversed.
pub fn zs_reverse_range_iter<C, K, V, R>(
&self,
cf: &C,
range: R,
) -> impl Iterator<Item = (K, V)> + '_
where
C: rocksdb::AsColumnFamilyRef,
K: IntoDisk + FromDisk,
V: FromDisk,
R: RangeBounds<K>,
{
self.zs_range_iter_with_direction(cf, range, true)
}
/// Returns an iterator over the items in `cf` in `range`.
///
/// RocksDB iterators are ordered by increasing key bytes by default.
/// Otherwise, if `reverse` is `true`, the iterator is ordered by decreasing key bytes.
///
/// Holding this iterator open might delay block commit transactions.
fn zs_range_iter_with_direction<C, K, V, R>(
&self,
cf: &C,
range: R,
reverse: bool,
) -> impl Iterator<Item = (K, V)> + '_
where
C: rocksdb::AsColumnFamilyRef,
K: IntoDisk + FromDisk,
@ -492,67 +451,40 @@ impl DiskDb {
let start_bound = map_to_vec(range.start_bound());
let end_bound = map_to_vec(range.end_bound());
let range = (start_bound, end_bound);
let range = (start_bound.clone(), end_bound);
let mode = Self::zs_iter_mode(&range, reverse);
let start_bound_vec =
if let Included(ref start_bound) | Excluded(ref start_bound) = start_bound {
start_bound.clone()
} else {
// Actually unused
Vec::new()
};
let start_mode = if matches!(start_bound, Unbounded) {
// Unbounded iterators start at the first item
rocksdb::IteratorMode::Start
} else {
rocksdb::IteratorMode::From(start_bound_vec.as_slice(), rocksdb::Direction::Forward)
};
// Reading multiple items from iterators has caused database hangs,
// in previous RocksDB versions
self.db
.iterator_cf(cf, mode)
.iterator_cf(cf, start_mode)
.map(|result| result.expect("unexpected database failure"))
.map(|(key, value)| (key.to_vec(), value))
// Skip excluded "from" bound and empty ranges. The `mode` already skips keys
// strictly before the "from" bound.
// Skip excluded start bound and empty ranges. The `start_mode` already skips keys
// before the start bound.
.skip_while({
let range = range.clone();
move |(key, _value)| !range.contains(key)
})
// Take until the excluded "to" bound is reached,
// or we're after the included "to" bound.
// Take until the excluded end bound is reached, or we're after the included end bound.
.take_while(move |(key, _value)| range.contains(key))
.map(|(key, value)| (K::from_bytes(key), V::from_bytes(value)))
}
/// Returns the RocksDB iterator "from" mode for `range`.
///
/// RocksDB iterators are ordered by increasing key bytes by default.
/// Otherwise, if `reverse` is `true`, the iterator is ordered by decreasing key bytes.
fn zs_iter_mode<R>(range: &R, reverse: bool) -> rocksdb::IteratorMode
where
R: RangeBounds<Vec<u8>>,
{
use std::ops::Bound::*;
let from_bound = if reverse {
range.end_bound()
} else {
range.start_bound()
};
match from_bound {
Unbounded => {
if reverse {
// Reversed unbounded iterators start from the last item
rocksdb::IteratorMode::End
} else {
// Unbounded iterators start from the first item
rocksdb::IteratorMode::Start
}
}
Included(bound) | Excluded(bound) => {
let direction = if reverse {
rocksdb::Direction::Reverse
} else {
rocksdb::Direction::Forward
};
rocksdb::IteratorMode::From(bound.as_slice(), direction)
}
}
}
/// The ideal open file limit for Zebra
const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;

17
zebra-state/src/service/finalized_state/disk_format/upgrade/add_subtrees.rs
View File

@ -44,23 +44,15 @@ pub fn run(
//
// Therefore, the first block with shielded note can't complete a subtree, which means we can
// skip the (genesis block, first shielded block) tree pair.
//
// # Compatibility
//
// Because wallets search backwards from the chain tip, subtrees need to be added to the
// database in reverse height order. (Tip first, genesis last.)
//
// Otherwise, wallets that sync during the upgrade will be missing some notes.
// Generate a list of sapling subtree inputs: previous and current trees, and their end heights.
let subtrees = upgrade_db
.sapling_tree_by_reversed_height_range(..=initial_tip_height)
.sapling_tree_by_height_range(..=initial_tip_height)
// The first block with sapling notes can't complete a subtree, see above for details.
.filter(|(height, _tree)| !height.is_min())
// We need both the tree and its previous tree for each shielded block.
.tuple_windows()
// Because the iterator is reversed, the larger tree is first.
.map(|((end_height, tree), (prev_end_height, prev_tree))| {
.map(|((prev_end_height, prev_tree), (end_height, tree))| {
(prev_end_height, prev_tree, end_height, tree)
})
// Find new subtrees.
@ -81,13 +73,12 @@ pub fn run(
// Generate a list of orchard subtree inputs: previous and current trees, and their end heights.
let subtrees = upgrade_db
.orchard_tree_by_reversed_height_range(..=initial_tip_height)
.orchard_tree_by_height_range(..=initial_tip_height)
// The first block with orchard notes can't complete a subtree, see above for details.
.filter(|(height, _tree)| !height.is_min())
// We need both the tree and its previous tree for each shielded block.
.tuple_windows()
// Because the iterator is reversed, the larger tree is first.
.map(|((end_height, tree), (prev_end_height, prev_tree))| {
.map(|((prev_end_height, prev_tree), (end_height, tree))| {
(prev_end_height, prev_tree, end_height, tree)
})
// Find new subtrees.

26
zebra-state/src/service/finalized_state/zebra_db/shielded.rs
View File

@ -180,19 +180,6 @@ impl ZebraDb {
self.db.zs_range_iter(&sapling_trees, range)
}
/// Returns the Sapling note commitment trees in the reversed range, in decreasing height order.
#[allow(clippy::unwrap_in_result)]
pub fn sapling_tree_by_reversed_height_range<R>(
&self,
range: R,
) -> impl Iterator<Item = (Height, Arc<sapling::tree::NoteCommitmentTree>)> + '_
where
R: std::ops::RangeBounds<Height>,
{
let sapling_trees = self.db.cf_handle("sapling_note_commitment_tree").unwrap();
self.db.zs_reverse_range_iter(&sapling_trees, range)
}
/// Returns the Sapling note commitment subtree at this `index`.
///
/// # Correctness
@ -326,19 +313,6 @@ impl ZebraDb {
self.db.zs_range_iter(&orchard_trees, range)
}
/// Returns the Orchard note commitment trees in the reversed range, in decreasing height order.
#[allow(clippy::unwrap_in_result)]
pub fn orchard_tree_by_reversed_height_range<R>(
&self,
range: R,
) -> impl Iterator<Item = (Height, Arc<orchard::tree::NoteCommitmentTree>)> + '_
where
R: std::ops::RangeBounds<Height>,
{
let orchard_trees = self.db.cf_handle("orchard_note_commitment_tree").unwrap();
self.db.zs_reverse_range_iter(&orchard_trees, range)
}
/// Returns the Orchard note commitment subtree at this `index`.
///
/// # Correctness