zec
/
librustzcash
mirror of https://github.com/zcash/librustzcash.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

zcash_client_sqlite: Enable forced re-scanning of previously scanned ranges. 

When `force_rescans` is set to `true` in a call to
`replace_queue_entries`, previously scanned ranges will have their
existing priority overwritten by the scan priority for a provided range;
otherwise, the existing scan priority dominance rule continues to be
enforced. This enables us to require previously scanned ranges be
re-scanned without interfering with higher-priority scan operations.
This commit is contained in:
Kris Nuttycombe 2023-08-16 11:15:10 -06:00
parent cf4c769609
commit c99c0fc884
2 changed files with 116 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
zcash_client_sqlite/src/wallet.rs
View File

@ -949,7 +949,7 @@ pub(crate) fn truncate_to_height<P: consensus::Parameters>(
// Prioritize the range starting at the height we just rewound to for verification // Prioritize the range starting at the height we just rewound to for verification
let query_range = block_height..(block_height + VERIFY_LOOKAHEAD); let query_range = block_height..(block_height + VERIFY_LOOKAHEAD);
let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify); let scan_range = ScanRange::from_parts(query_range.clone(), ScanPriority::Verify);
replace_queue_entries(conn, &query_range, Some(scan_range).into_iter())?; replace_queue_entries(conn, &query_range, Some(scan_range).into_iter(), false)?;
} }
Ok(()) Ok(())

157
zcash_client_sqlite/src/wallet/scanning.rs
View File

@ -17,18 +17,42 @@ use crate::{PRUNING_DEPTH, VERIFY_LOOKAHEAD};
use super::block_height_extrema; use super::block_height_extrema;
#[derive(Debug, Clone, Copy)] #[derive(Debug, Clone, Copy)]
enum Insert { enum InsertOn {
Left, Left,
Right, Right,
} }
struct Insert {
on: InsertOn,
force_rescan: bool,
}
impl Insert {
fn left(force_rescan: bool) -> Self {
Insert {
on: InsertOn::Left,
force_rescan,
}
}
fn right(force_rescan: bool) -> Self {
Insert {
on: InsertOn::Right,
force_rescan,
}
}
}
impl Not for Insert { impl Not for Insert {
type Output = Self; type Output = Self;
fn not(self) -> Self::Output { fn not(self) -> Self::Output {
match self { Insert {
Insert::Left => Insert::Right, on: match self.on {
Insert::Right => Insert::Left, InsertOn::Left => InsertOn::Right,
InsertOn::Right => InsertOn::Left,
},
force_rescan: self.force_rescan,
} }
} }
} }
@ -42,9 +66,9 @@ enum Dominance {
impl From<Insert> for Dominance { impl From<Insert> for Dominance {
fn from(value: Insert) -> Self { fn from(value: Insert) -> Self {
match value { match value.on {
Insert::Left => Dominance::Left, InsertOn::Left => Dominance::Left,
Insert::Right => Dominance::Right, InsertOn::Right => Dominance::Right,
} }
} }
} }
@ -115,7 +139,7 @@ fn dominance(current: &ScanPriority, inserted: &ScanPriority, insert: Insert) ->
match (current.cmp(inserted), (current, inserted)) { match (current.cmp(inserted), (current, inserted)) {
(Ordering::Equal, _) => Dominance::Equal, (Ordering::Equal, _) => Dominance::Equal,
(_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert), (_, (_, ScanPriority::Verify | ScanPriority::Scanned)) => Dominance::from(insert),
(_, (ScanPriority::Scanned, _)) => Dominance::from(!insert), (_, (ScanPriority::Scanned, _)) if !insert.force_rescan => Dominance::from(!insert),
(Ordering::Less, _) => Dominance::from(insert), (Ordering::Less, _) => Dominance::from(insert),
(Ordering::Greater, _) => Dominance::from(!insert), (Ordering::Greater, _) => Dominance::from(!insert),
} }
@ -197,7 +221,7 @@ fn join_nonoverlapping(left: ScanRange, right: ScanRange) -> Joined {
} }
} }
fn insert(current: ScanRange, to_insert: ScanRange) -> Joined { fn insert(current: ScanRange, to_insert: ScanRange, force_rescans: bool) -> Joined {
fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined { fn join_overlapping(left: ScanRange, right: ScanRange, insert: Insert) -> Joined {
assert!( assert!(
left.block_range().start <= right.block_range().start left.block_range().start <= right.block_range().start
@ -205,9 +229,9 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined {
); );
// recompute the range dominance based upon the queue entry priorities // recompute the range dominance based upon the queue entry priorities
let dominance = match insert { let dominance = match insert.on {
Insert::Left => dominance(&right.priority(), &left.priority(), insert), InsertOn::Left => dominance(&right.priority(), &left.priority(), insert),
Insert::Right => dominance(&left.priority(), &right.priority(), insert), InsertOn::Right => dominance(&left.priority(), &right.priority(), insert),
}; };
match dominance { match dominance {
@ -237,15 +261,23 @@ fn insert(current: ScanRange, to_insert: ScanRange) -> Joined {
use RangeOrdering::*; use RangeOrdering::*;
match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) { match RangeOrdering::cmp(to_insert.block_range(), current.block_range()) {
LeftFirstDisjoint => join_nonoverlapping(to_insert, current), LeftFirstDisjoint => join_nonoverlapping(to_insert, current),
LeftFirstOverlap | RightContained => join_overlapping(to_insert, current, Insert::Left), LeftFirstOverlap | RightContained => {
join_overlapping(to_insert, current, Insert::left(force_rescans))
}
Equal => Joined::One(ScanRange::from_parts( Equal => Joined::One(ScanRange::from_parts(
to_insert.block_range().clone(), to_insert.block_range().clone(),
match dominance(&current.priority(), &to_insert.priority(), Insert::Right) { match dominance(
&current.priority(),
&to_insert.priority(),
Insert::right(force_rescans),
) {
Dominance::Left | Dominance::Equal => current.priority(), Dominance::Left | Dominance::Equal => current.priority(),
Dominance::Right => to_insert.priority(), Dominance::Right => to_insert.priority(),
}, },
)), )),
RightFirstOverlap | LeftContained => join_overlapping(current, to_insert, Insert::Right), RightFirstOverlap | LeftContained => {
join_overlapping(current, to_insert, Insert::right(force_rescans))
}
RightFirstDisjoint => join_nonoverlapping(current, to_insert), RightFirstDisjoint => join_nonoverlapping(current, to_insert),
} }
} }
@ -294,9 +326,9 @@ impl SpanningTree {
to_insert: ScanRange, to_insert: ScanRange,
insert: Insert, insert: Insert,
) -> Self { ) -> Self {
let (left, right) = match insert { let (left, right) = match insert.on {
Insert::Left => (Box::new(left.insert(to_insert)), right), InsertOn::Left => (Box::new(left.insert(to_insert, insert.force_rescan)), right),
Insert::Right => (left, Box::new(right.insert(to_insert))), InsertOn::Right => (left, Box::new(right.insert(to_insert, insert.force_rescan))),
}; };
SpanningTree::Parent { SpanningTree::Parent {
span: left.span().start..right.span().end, span: left.span().start..right.span().end,
@ -305,12 +337,18 @@ impl SpanningTree {
} }
} }
fn from_split(left: Self, right: Self, to_insert: ScanRange, split_point: BlockHeight) -> Self { fn from_split(
left: Self,
right: Self,
to_insert: ScanRange,
split_point: BlockHeight,
force_rescans: bool,
) -> Self {
let (l_insert, r_insert) = to_insert let (l_insert, r_insert) = to_insert
.split_at(split_point) .split_at(split_point)
.expect("Split point is within the range of to_insert"); .expect("Split point is within the range of to_insert");
let left = Box::new(left.insert(l_insert)); let left = Box::new(left.insert(l_insert, force_rescans));
let right = Box::new(right.insert(r_insert)); let right = Box::new(right.insert(r_insert, force_rescans));
SpanningTree::Parent { SpanningTree::Parent {
span: left.span().start..right.span().end, span: left.span().start..right.span().end,
left, left,
@ -318,9 +356,9 @@ impl SpanningTree {
} }
} }
fn insert(self, to_insert: ScanRange) -> Self { fn insert(self, to_insert: ScanRange, force_rescans: bool) -> Self {
match self { match self {
SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert)), SpanningTree::Leaf(cur) => Self::from_joined(insert(cur, to_insert, force_rescans)),
SpanningTree::Parent { span, left, right } => { SpanningTree::Parent { span, left, right } => {
// This algorithm always preserves the existing partition point, and does not do // This algorithm always preserves the existing partition point, and does not do
// any rebalancing or unification of ranges within the tree. This should be okay // any rebalancing or unification of ranges within the tree. This should be okay
@ -331,15 +369,15 @@ impl SpanningTree {
match RangeOrdering::cmp(&span, to_insert.block_range()) { match RangeOrdering::cmp(&span, to_insert.block_range()) {
LeftFirstDisjoint => { LeftFirstDisjoint => {
// extend the right-hand branch // extend the right-hand branch
Self::from_insert(left, right, to_insert, Insert::Right) Self::from_insert(left, right, to_insert, Insert::right(force_rescans))
} }
LeftFirstOverlap => { LeftFirstOverlap => {
let split_point = left.span().end; let split_point = left.span().end;
if split_point > to_insert.block_range().start { if split_point > to_insert.block_range().start {
Self::from_split(*left, *right, to_insert, split_point) Self::from_split(*left, *right, to_insert, split_point, force_rescans)
} else { } else {
// to_insert is fully contained in or equals the right child // to_insert is fully contained in or equals the right child
Self::from_insert(left, right, to_insert, Insert::Right) Self::from_insert(left, right, to_insert, Insert::right(force_rescans))
} }
} }
RightContained => { RightContained => {
@ -348,42 +386,42 @@ impl SpanningTree {
let split_point = left.span().end; let split_point = left.span().end;
if to_insert.block_range().start >= split_point { if to_insert.block_range().start >= split_point {
// to_insert is fully contained in the right // to_insert is fully contained in the right
Self::from_insert(left, right, to_insert, Insert::Right) Self::from_insert(left, right, to_insert, Insert::right(force_rescans))
} else if to_insert.block_range().end <= split_point { } else if to_insert.block_range().end <= split_point {
// to_insert is fully contained in the left // to_insert is fully contained in the left
Self::from_insert(left, right, to_insert, Insert::Left) Self::from_insert(left, right, to_insert, Insert::left(force_rescans))
} else { } else {
// to_insert must be split. // to_insert must be split.
Self::from_split(*left, *right, to_insert, split_point) Self::from_split(*left, *right, to_insert, split_point, force_rescans)
} }
} }
Equal => { Equal => {
let split_point = left.span().end; let split_point = left.span().end;
if split_point > to_insert.block_range().start { if split_point > to_insert.block_range().start {
Self::from_split(*left, *right, to_insert, split_point) Self::from_split(*left, *right, to_insert, split_point, force_rescans)
} else { } else {
// to_insert is fully contained in the right subtree // to_insert is fully contained in the right subtree
right.insert(to_insert) right.insert(to_insert, force_rescans)
} }
} }
LeftContained => { LeftContained => {
// the current span is fully contained within to_insert, so we will extend // the current span is fully contained within to_insert, so we will extend
// or overwrite both sides // or overwrite both sides
let split_point = left.span().end; let split_point = left.span().end;
Self::from_split(*left, *right, to_insert, split_point) Self::from_split(*left, *right, to_insert, split_point, force_rescans)
} }
RightFirstOverlap => { RightFirstOverlap => {
let split_point = left.span().end; let split_point = left.span().end;
if split_point < to_insert.block_range().end { if split_point < to_insert.block_range().end {
Self::from_split(*left, *right, to_insert, split_point) Self::from_split(*left, *right, to_insert, split_point, force_rescans)
} else { } else {
// to_insert is fully contained in or equals the left child // to_insert is fully contained in or equals the left child
Self::from_insert(left, right, to_insert, Insert::Left) Self::from_insert(left, right, to_insert, Insert::left(force_rescans))
} }
} }
RightFirstDisjoint => { RightFirstDisjoint => {
// extend the left-hand branch // extend the left-hand branch
Self::from_insert(left, right, to_insert, Insert::Left) Self::from_insert(left, right, to_insert, Insert::left(force_rescans))
} }
} }
} }
@ -447,6 +485,7 @@ pub(crate) fn replace_queue_entries(
conn: &rusqlite::Transaction<'_>, conn: &rusqlite::Transaction<'_>,
query_range: &Range<BlockHeight>, query_range: &Range<BlockHeight>,
entries: impl Iterator<Item = ScanRange>, entries: impl Iterator<Item = ScanRange>,
force_rescans: bool,
) -> Result<(), SqliteClientError> { ) -> Result<(), SqliteClientError> {
let (to_create, to_delete_ends) = { let (to_create, to_delete_ends) = {
let mut suggested_stmt = conn.prepare_cached( let mut suggested_stmt = conn.prepare_cached(
@ -499,7 +538,7 @@ pub(crate) fn replace_queue_entries(
); );
to_delete_ends.push(Value::from(u32::from(entry.block_range().end))); to_delete_ends.push(Value::from(u32::from(entry.block_range().end)));
to_create = if let Some(cur) = to_create { to_create = if let Some(cur) = to_create {
Some(cur.insert(entry)) Some(cur.insert(entry, force_rescans))
} else { } else {
Some(SpanningTree::Leaf(entry)) Some(SpanningTree::Leaf(entry))
}; };
@ -509,7 +548,7 @@ pub(crate) fn replace_queue_entries(
// start with the scanned range. // start with the scanned range.
for entry in entries { for entry in entries {
to_create = if let Some(cur) = to_create { to_create = if let Some(cur) = to_create {
Some(cur.insert(entry)) Some(cur.insert(entry, force_rescans))
} else { } else {
Some(SpanningTree::Leaf(entry)) Some(SpanningTree::Leaf(entry))
}; };
@ -611,6 +650,7 @@ pub(crate) fn scan_complete<P: consensus::Parameters>(
conn, conn,
&query_range, &query_range,
Some(scanned).into_iter().chain(extensions.into_iter()), Some(scanned).into_iter().chain(extensions.into_iter()),
false,
)?; )?;
Ok(()) Ok(())
@ -714,6 +754,7 @@ pub(crate) fn update_chain_tip<P: consensus::Parameters>(
conn, conn,
&query_range, &query_range,
shard_entry.into_iter().chain(tip_entry.into_iter()), shard_entry.into_iter().chain(tip_entry.into_iter()),
false,
)?; )?;
} else { } else {
// If we have neither shard data nor any existing block data in the database, we should also // If we have neither shard data nor any existing block data in the database, we should also
@ -904,7 +945,7 @@ mod tests {
let scan_range = scan_range(range.clone(), *priority); let scan_range = scan_range(range.clone(), *priority);
match acc { match acc {
None => Some(SpanningTree::Leaf(scan_range)), None => Some(SpanningTree::Leaf(scan_range)),
Some(t) => Some(t.insert(scan_range)), Some(t) => Some(t.insert(scan_range, false)),
} }
}) })
} }
@ -1035,7 +1076,7 @@ mod tests {
// a `ChainTip` insertion should not overwrite a scanned range. // a `ChainTip` insertion should not overwrite a scanned range.
let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap(); let mut t = spanning_tree(&[(0..3, ChainTip), (3..5, Scanned), (5..7, ChainTip)]).unwrap();
t = t.insert(scan_range(0..7, ChainTip)); t = t.insert(scan_range(0..7, ChainTip), false);
assert_eq!( assert_eq!(
t.into_vec(), t.into_vec(),
vec![ vec![
@ -1054,7 +1095,7 @@ mod tests {
scan_range(280310..280320, Scanned) scan_range(280310..280320, Scanned)
] ]
); );
t = t.insert(scan_range(280300..280340, ChainTip)); t = t.insert(scan_range(280300..280340, ChainTip), false);
assert_eq!( assert_eq!(
t.into_vec(), t.into_vec(),
vec![ vec![
@ -1077,12 +1118,42 @@ mod tests {
]) ])
.unwrap(); .unwrap();
t = t.insert(scan_range(0..3, Scanned)); t = t.insert(scan_range(0..3, Scanned), false);
t = t.insert(scan_range(5..8, Scanned)); t = t.insert(scan_range(5..8, Scanned), false);
assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]); assert_eq!(t.into_vec(), vec![scan_range(0..10, Scanned)]);
} }
#[test]
fn spanning_tree_force_rescans() {
use ScanPriority::*;
let mut t = spanning_tree(&[
(0..3, Historic),
(3..5, Scanned),
(5..7, ChainTip),
(7..10, Scanned),
])
.unwrap();
t = t.insert(scan_range(4..9, OpenAdjacent), true);
let expected = vec![
scan_range(0..3, Historic),
scan_range(3..4, Scanned),
scan_range(4..5, OpenAdjacent),
scan_range(5..7, ChainTip),
scan_range(7..9, OpenAdjacent),
scan_range(9..10, Scanned),
];
assert_eq!(t.clone().into_vec(), expected);
// An insert of an ignored range should not override a scanned range; the existing
// priority should prevail, and so the expected state of the tree is unchanged.
t = t.insert(scan_range(2..5, Ignored), true);
assert_eq!(t.into_vec(), expected);
}
#[test] #[test]
fn scan_complete() { fn scan_complete() {
use ScanPriority::*; use ScanPriority::*;
@ -1326,6 +1397,7 @@ mod tests {
&tx, &tx,
&(BlockHeight::from(150)..BlockHeight::from(160)), &(BlockHeight::from(150)..BlockHeight::from(160)),
vec![scan_range(150..160, Scanned)].into_iter(), vec![scan_range(150..160, Scanned)].into_iter(),
false,
) )
.unwrap(); .unwrap();
tx.commit().unwrap(); tx.commit().unwrap();
@ -1368,6 +1440,7 @@ mod tests {
&tx, &tx,
&(BlockHeight::from(90)..BlockHeight::from(100)), &(BlockHeight::from(90)..BlockHeight::from(100)),
vec![scan_range(90..100, Scanned)].into_iter(), vec![scan_range(90..100, Scanned)].into_iter(),
false,
) )
.unwrap(); .unwrap();
tx.commit().unwrap(); tx.commit().unwrap();