2020-07-20 18:34:23 -07:00
//! Chain state updates for Zebra.
//!
//! Chain state updates occur in multiple stages:
//! - verify blocks (using `BlockVerifier` or `CheckpointVerifier`)
//! - update the list of verified blocks on disk
//! - create the chain state needed to verify child blocks
//! - choose the best tip from all the available chain tips
//! - update the mature chain state on disk
//! - prune orphaned side-chains
//!
//! Chain state updates are provided via a `tower::Service`, to support
//! backpressure and batch verification.
#[ cfg(test) ]
mod tests ;
2020-07-29 03:51:26 -07:00
use crate ::checkpoint ::{ CheckpointList , CheckpointVerifier } ;
2020-08-19 19:28:21 -07:00
use crate ::Config ;
2020-07-20 18:34:23 -07:00
use futures_util ::FutureExt ;
use std ::{
error ,
future ::Future ,
pin ::Pin ,
sync ::Arc ,
task ::{ Context , Poll } ,
} ;
use tower ::{ buffer ::Buffer , Service , ServiceExt } ;
2020-07-22 11:54:52 -07:00
use tracing_futures ::Instrument ;
2020-07-20 18:34:23 -07:00
2020-08-16 11:42:02 -07:00
use zebra_chain ::block ::{ self , Block } ;
2020-08-15 15:45:37 -07:00
use zebra_chain ::parameters ::{ Network , NetworkUpgrade ::Sapling } ;
2020-07-20 18:34:23 -07:00
2020-07-21 06:02:41 -07:00
/// The maximum expected gap between blocks.
///
/// Used to identify unexpected high blocks.
const MAX_EXPECTED_BLOCK_GAP : u32 = 100_000 ;
2020-07-30 17:21:20 -07:00
/// A wrapper type that holds the `ChainVerifier`'s `CheckpointVerifier`, and
/// its associated state.
#[ derive(Clone) ]
struct ChainCheckpointVerifier {
/// The underlying `CheckpointVerifier`, wrapped in a buffer, so we can
/// clone and share it with futures.
verifier : Buffer < CheckpointVerifier , Arc < Block > > ,
/// The maximum checkpoint height for `checkpoint_verifier`.
2020-08-16 11:42:02 -07:00
max_height : block ::Height ,
2020-07-30 17:21:20 -07:00
}
/// A service that verifies the chain, using its associated `CheckpointVerifier`
/// and `BlockVerifier`.
2020-07-29 03:51:26 -07:00
struct ChainVerifier < BV , S >
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-29 03:51:26 -07:00
BV ::Future : Send + 'static ,
S : Service < zebra_state ::Request , Response = zebra_state ::Response , Error = Error >
+ Send
+ Clone
+ 'static ,
S ::Future : Send + 'static ,
{
2020-07-20 18:34:23 -07:00
/// The underlying `BlockVerifier`, possibly wrapped in other services.
block_verifier : BV ,
2020-07-30 17:21:20 -07:00
/// The `ChainVerifier`'s underlying `CheckpointVerifier`, and its
/// associated state.
2020-07-29 03:51:26 -07:00
///
/// None if all the checkpoints have been verified.
2020-07-30 17:21:20 -07:00
checkpoint : Option < ChainCheckpointVerifier > ,
2020-07-20 18:34:23 -07:00
/// The underlying `ZebraState`, possibly wrapped in other services.
state_service : S ,
2020-07-21 06:02:41 -07:00
2020-07-29 03:51:26 -07:00
/// The most recent block height that was submitted to the verifier.
///
/// Used for debugging.
2020-07-21 06:02:41 -07:00
///
2020-07-29 03:51:26 -07:00
/// Updated before verification: the block at this height might not be valid.
2020-08-16 11:42:02 -07:00
last_block_height : Option < block ::Height > ,
2020-07-20 18:34:23 -07:00
}
/// The error type for the ChainVerifier Service.
// TODO(jlusby): Error = Report ?
type Error = Box < dyn error ::Error + Send + Sync + 'static > ;
/// The ChainVerifier service implementation.
///
/// After verification, blocks are added to the underlying state service.
impl < BV , S > Service < Arc < Block > > for ChainVerifier < BV , S >
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
BV ::Future : Send + 'static ,
S : Service < zebra_state ::Request , Response = zebra_state ::Response , Error = Error >
+ Send
+ Clone
+ 'static ,
S ::Future : Send + 'static ,
{
2020-08-15 23:20:01 -07:00
type Response = block ::Hash ;
2020-07-20 18:34:23 -07:00
type Error = Error ;
type Future =
Pin < Box < dyn Future < Output = Result < Self ::Response , Self ::Error > > + Send + 'static > > ;
fn poll_ready ( & mut self , _ : & mut Context < '_ > ) -> Poll < Result < ( ) , Self ::Error > > {
// We don't expect the state or verifiers to exert backpressure on our
// users, so we don't need to call `state_service.poll_ready()` here.
// (And we don't know which verifier to choose at this point, anyway.)
Poll ::Ready ( Ok ( ( ) ) )
}
fn call ( & mut self , block : Arc < Block > ) -> Self ::Future {
// TODO(jlusby): Error = Report, handle errors from state_service.
2020-09-03 15:06:21 -07:00
let height = block . coinbase_height ( ) ;
let hash = block . hash ( ) ;
let span = tracing ::debug_span! ( " block_verify " , ? height , ? hash , ) ;
2020-07-29 03:51:26 -07:00
let mut block_verifier = self . block_verifier . clone ( ) ;
let mut state_service = self . state_service . clone ( ) ;
2020-07-30 17:21:20 -07:00
let checkpoint_verifier = self . checkpoint . clone ( ) . map ( | c | c . verifier ) ;
let max_checkpoint_height = self . checkpoint . clone ( ) . map ( | c | c . max_height ) ;
2020-07-21 06:02:20 -07:00
2020-07-28 23:21:50 -07:00
// Log an info-level message on unexpected high blocks
2020-09-03 15:06:21 -07:00
let is_unexpected_high_block = match ( height , self . last_block_height ) {
( Some ( block ::Height ( height ) ) , Some ( block ::Height ( last_block_height ) ) )
if ( height > last_block_height + MAX_EXPECTED_BLOCK_GAP ) = >
2020-07-21 06:02:41 -07:00
{
2020-09-03 15:06:21 -07:00
self . last_block_height = Some ( block ::Height ( height ) ) ;
2020-07-21 06:02:41 -07:00
true
}
2020-09-03 15:06:21 -07:00
( Some ( height ) , _ ) = > {
self . last_block_height = Some ( height ) ;
2020-07-21 06:02:41 -07:00
false
}
2020-07-29 03:51:26 -07:00
// The other cases are covered by the verifiers
2020-07-21 06:02:41 -07:00
_ = > false ,
} ;
2020-07-20 18:34:23 -07:00
async move {
2020-07-28 23:21:50 -07:00
// TODO(teor): in the post-sapling checkpoint range, allow callers
2020-07-20 18:34:23 -07:00
// to use BlockVerifier, CheckpointVerifier, or both.
2020-07-21 06:02:20 -07:00
// Call a verifier based on the block height and checkpoints.
2020-09-03 15:06:21 -07:00
if is_higher_than_max_checkpoint ( height , max_checkpoint_height ) {
2020-08-05 22:26:26 -07:00
// Log a message on early high blocks.
2020-07-29 03:51:26 -07:00
// The sync service rejects most of these blocks, but we
// still want to know if a large number get through.
2020-08-05 22:26:26 -07:00
//
// This message can also happen if we keep getting unexpected
// low blocks. (We can't distinguish between these cases, until
// we've verified the blocks.)
2020-07-29 03:51:26 -07:00
if is_unexpected_high_block {
2020-09-03 15:06:21 -07:00
tracing ::debug! ( ? height , " unexpected high block, or recent unexpected low blocks " ) ;
2020-07-20 18:34:23 -07:00
}
2020-07-29 03:51:26 -07:00
block_verifier
. ready_and ( )
. await ?
. call ( block . clone ( ) )
. await ? ;
} else {
checkpoint_verifier
. expect ( " Missing checkpoint verifier: verifier must be Some if max checkpoint height is Some " )
. ready_and ( )
. await ?
. call ( block . clone ( ) )
. await ? ;
}
2020-07-20 18:34:23 -07:00
2020-09-03 15:06:21 -07:00
tracing ::trace! ( ? height , ? hash , " Verified block " ) ;
metrics ::gauge! (
" chain.verified.block.height " ,
height . expect ( " Valid blocks have a block height " ) . 0 as _
) ;
metrics ::counter! ( " chain.verified.block.count " , 1 ) ;
2020-07-20 18:34:23 -07:00
let add_block = state_service
. ready_and ( )
. await ?
. call ( zebra_state ::Request ::AddBlock { block } ) ;
match add_block . await ? {
zebra_state ::Response ::Added { hash } = > Ok ( hash ) ,
_ = > Err ( " adding block to zebra-state failed " . into ( ) ) ,
}
}
2020-07-22 11:54:52 -07:00
. instrument ( span )
2020-07-20 18:34:23 -07:00
. boxed ( )
}
}
2020-07-29 03:51:26 -07:00
/// Returns true if block_height is higher than the maximum checkpoint
/// height. Also returns true if there is no maximum checkpoint height.
///
/// Returns false if the block does not have a height.
fn is_higher_than_max_checkpoint (
2020-08-16 11:42:02 -07:00
block_height : Option < block ::Height > ,
max_checkpoint_height : Option < block ::Height > ,
2020-07-29 03:51:26 -07:00
) -> bool {
match ( block_height , max_checkpoint_height ) {
( Some ( block_height ) , Some ( max_checkpoint_height ) ) = > block_height > max_checkpoint_height ,
( _ , None ) = > true ,
( None , _ ) = > false ,
}
}
2020-08-19 19:28:21 -07:00
/// Return a chain verification service, using `config`, `network` and
/// `state_service`.
2020-07-23 18:47:48 -07:00
///
/// Gets the initial tip from the state service, and uses it to create a block
2020-07-29 03:51:26 -07:00
/// verifier and checkpoint verifier (if needed).
2020-07-20 19:24:58 -07:00
///
/// This function should only be called once for a particular state service. If
/// you need shared block or checkpoint verfiers, create them yourself, and pass
/// them to `init_from_verifiers`.
//
// TODO: revise this interface when we generate our own blocks, or validate
2020-07-20 21:09:19 -07:00
// mempool transactions. We might want to share the BlockVerifier, and we
// might not want to add generated blocks to the state.
2020-07-23 18:47:48 -07:00
pub async fn init < S > (
2020-08-19 19:28:21 -07:00
config : Config ,
2020-07-20 19:24:58 -07:00
network : Network ,
state_service : S ,
) -> impl Service <
Arc < Block > ,
2020-08-15 23:20:01 -07:00
Response = block ::Hash ,
2020-07-20 19:24:58 -07:00
Error = Error ,
2020-08-15 23:20:01 -07:00
Future = impl Future < Output = Result < block ::Hash , Error > > ,
2020-07-20 19:24:58 -07:00
> + Send
+ Clone
+ 'static
where
S : Service < zebra_state ::Request , Response = zebra_state ::Response , Error = Error >
+ Send
+ Clone
+ 'static ,
S ::Future : Send + 'static ,
{
2020-07-29 03:51:26 -07:00
let initial_tip = zebra_state ::current_tip ( state_service . clone ( ) )
2020-07-23 18:47:48 -07:00
. await
. expect ( " State service poll_ready is Ok " ) ;
2020-07-21 06:02:20 -07:00
2020-07-20 19:24:58 -07:00
let block_verifier = crate ::block ::init ( state_service . clone ( ) ) ;
2020-08-19 19:28:21 -07:00
let checkpoint_list = match config . checkpoint_sync {
true = > CheckpointList ::new ( network ) ,
false = > CheckpointList ::new_up_to ( network , Sapling ) ,
} ;
2020-07-20 19:24:58 -07:00
2020-07-29 03:51:26 -07:00
init_from_verifiers (
network ,
block_verifier ,
Some ( checkpoint_list ) ,
state_service ,
initial_tip ,
)
2020-07-20 19:24:58 -07:00
}
2020-07-29 03:51:26 -07:00
/// Return a chain verification service, using the provided block verifier,
/// checkpoint list, and state service.
2020-07-20 18:34:23 -07:00
///
/// The chain verifier holds a state service of type `S`, used as context for
/// block validation and to which newly verified blocks will be committed. This
/// state is pluggable to allow for testing or instrumentation.
///
/// The returned type is opaque to allow instrumentation or other wrappers, but
/// can be boxed for storage. It is also `Clone` to allow sharing of a
/// verification service.
///
/// This function should only be called once for a particular state service and
2020-07-29 03:51:26 -07:00
/// block verifier (and the result be shared, cloning if needed). Constructing
2020-07-20 18:34:23 -07:00
/// multiple services from the same underlying state might cause synchronisation
/// bugs.
2020-07-29 03:51:26 -07:00
///
/// Panics:
///
2020-08-19 19:28:21 -07:00
/// Panics if the `checkpoint_list` is None, and the `initial_tip_height` is
2020-07-29 03:51:26 -07:00
/// below the Sapling network upgrade for `network`. (The `block_verifier` can't
/// verify all the constraints on pre-Sapling blocks, so they require
/// checkpoints.)
pub ( crate ) fn init_from_verifiers < BV , S > (
network : Network ,
2020-07-20 18:34:23 -07:00
block_verifier : BV ,
2020-07-29 03:51:26 -07:00
checkpoint_list : Option < CheckpointList > ,
2020-07-20 18:34:23 -07:00
state_service : S ,
2020-07-29 03:51:26 -07:00
initial_tip : Option < Arc < Block > > ,
2020-07-20 18:34:23 -07:00
) -> impl Service <
Arc < Block > ,
2020-08-15 23:20:01 -07:00
Response = block ::Hash ,
2020-07-20 18:34:23 -07:00
Error = Error ,
2020-08-15 23:20:01 -07:00
Future = impl Future < Output = Result < block ::Hash , Error > > ,
2020-07-20 18:34:23 -07:00
> + Send
+ Clone
+ 'static
where
2020-08-15 23:20:01 -07:00
BV : Service < Arc < Block > , Response = block ::Hash , Error = Error > + Send + Clone + 'static ,
2020-07-20 18:34:23 -07:00
BV ::Future : Send + 'static ,
S : Service < zebra_state ::Request , Response = zebra_state ::Response , Error = Error >
+ Send
+ Clone
+ 'static ,
S ::Future : Send + 'static ,
{
2020-07-29 03:51:26 -07:00
let max_checkpoint_height = checkpoint_list . clone ( ) . map ( | c | c . max_height ( ) ) ;
let initial_height = initial_tip . clone ( ) . map ( | b | b . coinbase_height ( ) ) . flatten ( ) ;
let initial_hash = initial_tip . clone ( ) . map ( | b | b . hash ( ) ) ;
tracing ::info! (
? network ,
2020-07-21 06:02:20 -07:00
? max_checkpoint_height ,
2020-07-29 03:51:26 -07:00
? initial_height ,
? initial_hash ,
" initialising ChainVerifier "
2020-07-21 06:02:20 -07:00
) ;
2020-07-29 03:51:26 -07:00
let sapling_activation = Sapling
. activation_height ( network )
. expect ( " Unexpected network upgrade info: Sapling must have an activation height " ) ;
2020-07-30 17:21:20 -07:00
let checkpoint = match ( initial_height , checkpoint_list , max_checkpoint_height ) {
2020-07-29 03:51:26 -07:00
// If we need to verify pre-Sapling blocks, make sure we have checkpoints for them.
( None , None , _ ) = > panic! ( " We have no checkpoints, and we have no cached blocks: Pre-Sapling blocks must be verified by checkpoints " ) ,
( Some ( initial_height ) , None , _ ) if ( initial_height < sapling_activation ) = > panic! ( " We have no checkpoints, and we don't have a cached Sapling activation block: Pre-Sapling blocks must be verified by checkpoints " ) ,
// If we're past the checkpoint range, don't create a checkpoint verifier.
2020-07-30 17:21:20 -07:00
( Some ( initial_height ) , _ , Some ( max_checkpoint_height ) ) if ( initial_height > max_checkpoint_height ) = > None ,
2020-07-29 03:51:26 -07:00
// No list, no checkpoint verifier
2020-07-30 17:21:20 -07:00
( _ , None , _ ) = > None ,
2020-07-29 03:51:26 -07:00
( _ , Some ( _ ) , None ) = > panic! ( " Missing max checkpoint height: height must be Some if verifier is Some " ) ,
// We've done all the checks we need to create a checkpoint verifier
2020-07-30 17:21:20 -07:00
( _ , Some ( list ) , Some ( max_height ) ) = > Some (
ChainCheckpointVerifier {
verifier : Buffer ::new ( CheckpointVerifier ::from_checkpoint_list ( list , initial_tip ) , 1 ) ,
max_height ,
} ) ,
2020-07-29 03:51:26 -07:00
} ;
2020-07-20 18:34:23 -07:00
Buffer ::new (
ChainVerifier {
block_verifier ,
2020-07-30 17:21:20 -07:00
checkpoint ,
2020-07-20 18:34:23 -07:00
state_service ,
2020-07-29 03:51:26 -07:00
last_block_height : initial_height ,
2020-07-20 18:34:23 -07:00
} ,
1 ,
)
}