2020-07-20 18:34:23 -07:00
//! Chain state updates for Zebra.
//!
//! Chain state updates occur in multiple stages:
//! - verify blocks (using `BlockVerifier` or `CheckpointVerifier`)
//! - update the list of verified blocks on disk
//! - create the chain state needed to verify child blocks
//! - choose the best tip from all the available chain tips
//! - update the mature chain state on disk
//! - prune orphaned side-chains
//!
//! Chain state updates are provided via a `tower::Service`, to support
//! backpressure and batch verification.
#[ cfg(test) ]
mod tests ;
2020-07-29 03:51:26 -07:00
use crate ::checkpoint ::{ CheckpointList , CheckpointVerifier } ;
2020-08-19 19:28:21 -07:00
use crate ::Config ;
2020-07-20 18:34:23 -07:00
use futures_util ::FutureExt ;
use std ::{
error ,
future ::Future ,
pin ::Pin ,
sync ::Arc ,
task ::{ Context , Poll } ,
} ;
use tower ::{ buffer ::Buffer , Service , ServiceExt } ;
2020-07-22 11:54:52 -07:00
use tracing_futures ::Instrument ;
2020-07-20 18:34:23 -07:00
2020-09-09 18:53:40 -07:00
use zebra_chain ::{
block ::{ self , Block } ,
parameters ::{ Network , NetworkUpgrade ::Sapling } ,
} ;
use zebra_state as zs ;
2020-07-20 18:34:23 -07:00
2020-07-21 06:02:41 -07:00
/// The maximum expected gap between blocks.
///
2020-09-02 19:50:16 -07:00
/// Used to identify unexpected out of order blocks.
2020-07-21 06:02:41 -07:00
const MAX_EXPECTED_BLOCK_GAP : u32 = 100_000 ;
2020-07-30 17:21:20 -07:00
/// A wrapper type that holds the `ChainVerifier`'s `CheckpointVerifier`, and
/// its associated state.
#[ derive(Clone) ]
2020-09-02 21:23:57 -07:00
struct ChainCheckpointVerifier < S >
where
2020-09-09 18:53:40 -07:00
S : Service < zs ::Request , Response = zs ::Response , Error = Error > + Send + Clone + 'static ,
2020-09-02 21:23:57 -07:00
S ::Future : Send + 'static ,
{
2020-07-30 17:21:20 -07:00
/// The underlying `CheckpointVerifier`, wrapped in a buffer, so we can
/// clone and share it with futures.
2020-09-02 21:23:57 -07:00
verifier : Buffer < CheckpointVerifier < S > , Arc < Block > > ,
2020-07-30 17:21:20 -07:00
/// The maximum checkpoint height for `checkpoint_verifier`.
2020-08-16 11:42:02 -07:00
max_height : block ::Height ,
2020-07-30 17:21:20 -07:00
}
/// A service that verifies the chain, using its associated `CheckpointVerifier`
/// and `BlockVerifier`.
2020-07-29 03:51:26 -07:00
struct ChainVerifier < BV , S >
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-29 03:51:26 -07:00
BV ::Future : Send + 'static ,
2020-09-09 18:53:40 -07:00
S : Service < zs ::Request , Response = zs ::Response , Error = Error > + Send + Clone + 'static ,
2020-07-29 03:51:26 -07:00
S ::Future : Send + 'static ,
{
2020-07-20 18:34:23 -07:00
/// The underlying `BlockVerifier`, possibly wrapped in other services.
block_verifier : BV ,
2020-07-30 17:21:20 -07:00
/// The `ChainVerifier`'s underlying `CheckpointVerifier`, and its
/// associated state.
2020-07-29 03:51:26 -07:00
///
/// None if all the checkpoints have been verified.
2020-09-02 21:23:57 -07:00
checkpoint : Option < ChainCheckpointVerifier < S > > ,
2020-07-21 06:02:41 -07:00
2020-07-29 03:51:26 -07:00
/// The most recent block height that was submitted to the verifier.
///
/// Used for debugging.
2020-07-21 06:02:41 -07:00
///
2020-07-29 03:51:26 -07:00
/// Updated before verification: the block at this height might not be valid.
2020-08-16 11:42:02 -07:00
last_block_height : Option < block ::Height > ,
2020-07-20 18:34:23 -07:00
}
/// The error type for the ChainVerifier Service.
// TODO(jlusby): Error = Report ?
type Error = Box < dyn error ::Error + Send + Sync + 'static > ;
/// The ChainVerifier service implementation.
///
/// After verification, blocks are added to the underlying state service.
impl < BV , S > Service < Arc < Block > > for ChainVerifier < BV , S >
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
BV ::Future : Send + 'static ,
2020-09-09 18:53:40 -07:00
S : Service < zs ::Request , Response = zs ::Response , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
S ::Future : Send + 'static ,
{
2020-08-15 23:20:01 -07:00
type Response = block ::Hash ;
2020-07-20 18:34:23 -07:00
type Error = Error ;
type Future =
Pin < Box < dyn Future < Output = Result < Self ::Response , Self ::Error > > + Send + 'static > > ;
fn poll_ready ( & mut self , _ : & mut Context < '_ > ) -> Poll < Result < ( ) , Self ::Error > > {
2020-09-02 21:23:57 -07:00
// We don't expect the verifiers to exert backpressure on our
// users, so we don't need to call the verifier's `poll_ready` here.
2020-07-20 18:34:23 -07:00
// (And we don't know which verifier to choose at this point, anyway.)
Poll ::Ready ( Ok ( ( ) ) )
}
fn call ( & mut self , block : Arc < Block > ) -> Self ::Future {
2020-09-02 21:23:57 -07:00
// TODO(jlusby): Error = Report
2020-09-03 15:06:21 -07:00
let height = block . coinbase_height ( ) ;
let hash = block . hash ( ) ;
let span = tracing ::debug_span! ( " block_verify " , ? height , ? hash , ) ;
2020-07-29 03:51:26 -07:00
let mut block_verifier = self . block_verifier . clone ( ) ;
2020-07-30 17:21:20 -07:00
let checkpoint_verifier = self . checkpoint . clone ( ) . map ( | c | c . verifier ) ;
let max_checkpoint_height = self . checkpoint . clone ( ) . map ( | c | c . max_height ) ;
2020-07-21 06:02:20 -07:00
2020-09-02 19:50:16 -07:00
// Log an info-level message on unexpected out of order blocks
let is_unexpected_gap = match ( height , self . last_block_height ) {
2020-09-03 15:06:21 -07:00
( Some ( block ::Height ( height ) ) , Some ( block ::Height ( last_block_height ) ) )
2020-09-02 19:50:16 -07:00
if ( height > last_block_height + MAX_EXPECTED_BLOCK_GAP )
| | ( height + MAX_EXPECTED_BLOCK_GAP < last_block_height ) = >
2020-07-21 06:02:41 -07:00
{
2020-09-03 15:06:21 -07:00
self . last_block_height = Some ( block ::Height ( height ) ) ;
2020-07-21 06:02:41 -07:00
true
}
2020-09-03 15:06:21 -07:00
( Some ( height ) , _ ) = > {
self . last_block_height = Some ( height ) ;
2020-07-21 06:02:41 -07:00
false
}
2020-07-29 03:51:26 -07:00
// The other cases are covered by the verifiers
2020-07-21 06:02:41 -07:00
_ = > false ,
} ;
2020-07-20 18:34:23 -07:00
async move {
2020-07-21 06:02:20 -07:00
// Call a verifier based on the block height and checkpoints.
2020-09-03 15:06:21 -07:00
if is_higher_than_max_checkpoint ( height , max_checkpoint_height ) {
2020-09-02 19:50:16 -07:00
// Log a message on unexpected out of order blocks.
//
2020-07-29 03:51:26 -07:00
// The sync service rejects most of these blocks, but we
// still want to know if a large number get through.
2020-09-02 19:50:16 -07:00
if is_unexpected_gap {
tracing ::debug! ( " large block height gap: this block or the previous block is out of order " ) ;
2020-07-20 18:34:23 -07:00
}
2020-09-02 21:23:57 -07:00
let verified_hash = block_verifier
2020-07-29 03:51:26 -07:00
. ready_and ( )
. await ?
. call ( block . clone ( ) )
. await ? ;
2020-09-02 21:23:57 -07:00
assert_eq! ( verified_hash , hash , " block verifier returned wrong hash: hashes must be equal " ) ;
2020-07-29 03:51:26 -07:00
} else {
2020-09-02 21:23:57 -07:00
let verified_hash = checkpoint_verifier
2020-09-02 19:50:16 -07:00
. expect ( " missing checkpoint verifier: verifier must be Some if max checkpoint height is Some " )
2020-07-29 03:51:26 -07:00
. ready_and ( )
. await ?
. call ( block . clone ( ) )
. await ? ;
2020-09-02 21:23:57 -07:00
assert_eq! ( verified_hash , hash , " checkpoint verifier returned wrong hash: hashes must be equal " ) ;
2020-07-29 03:51:26 -07:00
}
2020-07-20 18:34:23 -07:00
2020-09-02 19:50:16 -07:00
tracing ::trace! ( ? height , ? hash , " verified block " ) ;
2020-09-03 15:06:21 -07:00
metrics ::gauge! (
" chain.verified.block.height " ,
2020-09-02 19:50:16 -07:00
height . expect ( " valid blocks have a block height " ) . 0 as _
2020-09-03 15:06:21 -07:00
) ;
metrics ::counter! ( " chain.verified.block.count " , 1 ) ;
2020-09-02 21:23:57 -07:00
Ok ( hash )
2020-07-20 18:34:23 -07:00
}
2020-07-22 11:54:52 -07:00
. instrument ( span )
2020-07-20 18:34:23 -07:00
. boxed ( )
}
}
2020-07-29 03:51:26 -07:00
/// Returns true if block_height is higher than the maximum checkpoint
/// height. Also returns true if there is no maximum checkpoint height.
///
/// Returns false if the block does not have a height.
fn is_higher_than_max_checkpoint (
2020-08-16 11:42:02 -07:00
block_height : Option < block ::Height > ,
max_checkpoint_height : Option < block ::Height > ,
2020-07-29 03:51:26 -07:00
) -> bool {
match ( block_height , max_checkpoint_height ) {
( Some ( block_height ) , Some ( max_checkpoint_height ) ) = > block_height > max_checkpoint_height ,
( _ , None ) = > true ,
( None , _ ) = > false ,
}
}
2020-08-19 19:28:21 -07:00
/// Return a chain verification service, using `config`, `network` and
/// `state_service`.
2020-07-23 18:47:48 -07:00
///
/// Gets the initial tip from the state service, and uses it to create a block
2020-07-29 03:51:26 -07:00
/// verifier and checkpoint verifier (if needed).
2020-07-20 19:24:58 -07:00
///
/// This function should only be called once for a particular state service. If
/// you need shared block or checkpoint verfiers, create them yourself, and pass
/// them to `init_from_verifiers`.
//
// TODO: revise this interface when we generate our own blocks, or validate
2020-07-20 21:09:19 -07:00
// mempool transactions. We might want to share the BlockVerifier, and we
// might not want to add generated blocks to the state.
2020-07-23 18:47:48 -07:00
pub async fn init < S > (
2020-08-19 19:28:21 -07:00
config : Config ,
2020-07-20 19:24:58 -07:00
network : Network ,
state_service : S ,
) -> impl Service <
Arc < Block > ,
2020-08-15 23:20:01 -07:00
Response = block ::Hash ,
2020-07-20 19:24:58 -07:00
Error = Error ,
2020-08-15 23:20:01 -07:00
Future = impl Future < Output = Result < block ::Hash , Error > > ,
2020-07-20 19:24:58 -07:00
> + Send
+ Clone
+ 'static
where
2020-09-09 18:53:40 -07:00
S : Service < zs ::Request , Response = zs ::Response , Error = Error > + Send + Clone + 'static ,
2020-07-20 19:24:58 -07:00
S ::Future : Send + 'static ,
{
2020-09-09 18:53:40 -07:00
/*
let initial_tip = if let zs ::Response ::Tip ( tip ) = state_service
. ready_and ( )
2020-07-23 18:47:48 -07:00
. await
2020-09-09 18:53:40 -07:00
. unwrap ( )
. call ( zs ::Request ::Tip )
. await
. unwrap ( )
{
tip
} else {
unreachable! ( " wrong response to Request::Tip " ) ;
} ;
* /
// TODO: restore this after figuring out what data is required,
// after simplification of the chainverifier code.
let initial_tip = None ;
2020-07-21 06:02:20 -07:00
2020-07-20 19:24:58 -07:00
let block_verifier = crate ::block ::init ( state_service . clone ( ) ) ;
2020-08-19 19:28:21 -07:00
let checkpoint_list = match config . checkpoint_sync {
true = > CheckpointList ::new ( network ) ,
false = > CheckpointList ::new_up_to ( network , Sapling ) ,
} ;
2020-07-20 19:24:58 -07:00
2020-07-29 03:51:26 -07:00
init_from_verifiers (
network ,
block_verifier ,
Some ( checkpoint_list ) ,
state_service ,
initial_tip ,
)
2020-07-20 19:24:58 -07:00
}
2020-07-29 03:51:26 -07:00
/// Return a chain verification service, using the provided block verifier,
/// checkpoint list, and state service.
2020-07-20 18:34:23 -07:00
///
/// The returned type is opaque to allow instrumentation or other wrappers, but
/// can be boxed for storage. It is also `Clone` to allow sharing of a
/// verification service.
///
/// This function should only be called once for a particular state service and
2020-07-29 03:51:26 -07:00
/// block verifier (and the result be shared, cloning if needed). Constructing
2020-07-20 18:34:23 -07:00
/// multiple services from the same underlying state might cause synchronisation
/// bugs.
2020-07-29 03:51:26 -07:00
///
/// Panics:
///
2020-08-19 19:28:21 -07:00
/// Panics if the `checkpoint_list` is None, and the `initial_tip_height` is
2020-07-29 03:51:26 -07:00
/// below the Sapling network upgrade for `network`. (The `block_verifier` can't
/// verify all the constraints on pre-Sapling blocks, so they require
/// checkpoints.)
pub ( crate ) fn init_from_verifiers < BV , S > (
network : Network ,
2020-07-20 18:34:23 -07:00
block_verifier : BV ,
2020-07-29 03:51:26 -07:00
checkpoint_list : Option < CheckpointList > ,
2020-07-20 18:34:23 -07:00
state_service : S ,
2020-07-29 03:51:26 -07:00
initial_tip : Option < Arc < Block > > ,
2020-07-20 18:34:23 -07:00
) -> impl Service <
Arc < Block > ,
2020-08-15 23:20:01 -07:00
Response = block ::Hash ,
2020-07-20 18:34:23 -07:00
Error = Error ,
2020-08-15 23:20:01 -07:00
Future = impl Future < Output = Result < block ::Hash , Error > > ,
2020-07-20 18:34:23 -07:00
> + Send
+ Clone
+ 'static
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
BV ::Future : Send + 'static ,
2020-09-09 18:53:40 -07:00
S : Service < zs ::Request , Response = zs ::Response , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
S ::Future : Send + 'static ,
{
2020-07-29 03:51:26 -07:00
let max_checkpoint_height = checkpoint_list . clone ( ) . map ( | c | c . max_height ( ) ) ;
let initial_height = initial_tip . clone ( ) . map ( | b | b . coinbase_height ( ) ) . flatten ( ) ;
let initial_hash = initial_tip . clone ( ) . map ( | b | b . hash ( ) ) ;
tracing ::info! (
? network ,
2020-07-21 06:02:20 -07:00
? max_checkpoint_height ,
2020-07-29 03:51:26 -07:00
? initial_height ,
? initial_hash ,
" initialising ChainVerifier "
2020-07-21 06:02:20 -07:00
) ;
2020-07-29 03:51:26 -07:00
let sapling_activation = Sapling
. activation_height ( network )
. expect ( " Unexpected network upgrade info: Sapling must have an activation height " ) ;
2020-07-30 17:21:20 -07:00
let checkpoint = match ( initial_height , checkpoint_list , max_checkpoint_height ) {
2020-07-29 03:51:26 -07:00
// If we need to verify pre-Sapling blocks, make sure we have checkpoints for them.
( None , None , _ ) = > panic! ( " We have no checkpoints, and we have no cached blocks: Pre-Sapling blocks must be verified by checkpoints " ) ,
( Some ( initial_height ) , None , _ ) if ( initial_height < sapling_activation ) = > panic! ( " We have no checkpoints, and we don't have a cached Sapling activation block: Pre-Sapling blocks must be verified by checkpoints " ) ,
// If we're past the checkpoint range, don't create a checkpoint verifier.
2020-07-30 17:21:20 -07:00
( Some ( initial_height ) , _ , Some ( max_checkpoint_height ) ) if ( initial_height > max_checkpoint_height ) = > None ,
2020-07-29 03:51:26 -07:00
// No list, no checkpoint verifier
2020-07-30 17:21:20 -07:00
( _ , None , _ ) = > None ,
2020-07-29 03:51:26 -07:00
( _ , Some ( _ ) , None ) = > panic! ( " Missing max checkpoint height: height must be Some if verifier is Some " ) ,
2020-09-02 21:23:57 -07:00
2020-07-29 03:51:26 -07:00
// We've done all the checks we need to create a checkpoint verifier
2020-07-30 17:21:20 -07:00
( _ , Some ( list ) , Some ( max_height ) ) = > Some (
ChainCheckpointVerifier {
2020-09-02 21:23:57 -07:00
verifier : Buffer ::new ( CheckpointVerifier ::from_checkpoint_list ( list , initial_tip , state_service ) , 1 ) ,
2020-07-30 17:21:20 -07:00
max_height ,
} ) ,
2020-07-29 03:51:26 -07:00
} ;
2020-07-20 18:34:23 -07:00
Buffer ::new (
ChainVerifier {
block_verifier ,
2020-07-30 17:21:20 -07:00
checkpoint ,
2020-07-29 03:51:26 -07:00
last_block_height : initial_height ,
2020-07-20 18:34:23 -07:00
} ,
1 ,
)
}