2021-12-21 18:07:52 -08:00
|
|
|
//! `copy-state` subcommand - copies state from one directory to another (debug only)
|
|
|
|
//!
|
|
|
|
//! Copying state helps Zebra developers modify and debug cached state formats.
|
|
|
|
//!
|
|
|
|
//! In order to test a new state format, blocks must be identical when they are:
|
|
|
|
//! - read from the old format,
|
|
|
|
//! - written to the new format, and
|
|
|
|
//! - read from the new format.
|
|
|
|
//!
|
|
|
|
//! The "old" and "new" states can also use the same format.
|
|
|
|
//! This tests the low-level state API's performance.
|
|
|
|
//!
|
|
|
|
//! ## Command Structure
|
|
|
|
//!
|
|
|
|
//! Copying cached state uses the following services and tasks:
|
|
|
|
//!
|
|
|
|
//! Tasks:
|
|
|
|
//! * Old to New Copy Task
|
|
|
|
//! * queries the source state for blocks,
|
|
|
|
//! copies those blocks to the target state, then
|
|
|
|
//! reads the copied blocks from the target state.
|
|
|
|
//!
|
|
|
|
//! Services:
|
|
|
|
//! * Source Old State Service
|
|
|
|
//! * fetches blocks from the best finalized chain from permanent storage,
|
|
|
|
//! in the old format
|
|
|
|
//! * Target New State Service
|
|
|
|
//! * writes best finalized chain blocks to permanent storage,
|
|
|
|
//! in the new format
|
|
|
|
//! * only performs essential contextual verification of blocks,
|
|
|
|
//! to make sure that block data hasn't been corrupted by
|
|
|
|
//! receiving blocks in the new format
|
|
|
|
//! * fetches blocks from the best finalized chain from permanent storage,
|
|
|
|
//! in the new format
|
|
|
|
|
|
|
|
use std::{cmp::min, path::PathBuf};
|
|
|
|
|
|
|
|
use abscissa_core::{config, Command, FrameworkError, Options, Runnable};
|
|
|
|
use color_eyre::eyre::{eyre, Report};
|
|
|
|
use tokio::time::Instant;
|
|
|
|
use tower::{Service, ServiceExt};
|
|
|
|
|
|
|
|
use zebra_chain::{block::Height, parameters::Network};
|
|
|
|
use zebra_state as old_zs;
|
|
|
|
use zebra_state as new_zs;
|
|
|
|
|
|
|
|
use crate::{
|
|
|
|
components::tokio::{RuntimeRun, TokioComponent},
|
|
|
|
config::ZebradConfig,
|
|
|
|
prelude::*,
|
|
|
|
BoxError,
|
|
|
|
};
|
|
|
|
|
|
|
|
/// How often we log info-level progress messages
|
|
|
|
const PROGRESS_HEIGHT_INTERVAL: u32 = 5_000;
|
|
|
|
|
|
|
|
/// `copy-state` subcommand
|
|
|
|
#[derive(Command, Debug, Options)]
|
|
|
|
pub struct CopyStateCmd {
|
|
|
|
/// Source height that the copy finishes at.
|
|
|
|
#[options(help = "stop copying at this source height")]
|
|
|
|
max_source_height: Option<u32>,
|
|
|
|
|
|
|
|
/// Path to a Zebra config.toml for the target state.
|
|
|
|
/// Uses an ephemeral config by default.
|
|
|
|
///
|
|
|
|
/// Zebra only uses the state options from this config.
|
|
|
|
/// All other options are ignored.
|
|
|
|
#[options(help = "config file path for the target state (default: ephemeral), \
|
|
|
|
the source state uses the main zebrad config")]
|
|
|
|
target_config_path: Option<PathBuf>,
|
|
|
|
|
|
|
|
/// Filter strings which override the config file and defaults
|
|
|
|
#[options(free, help = "tracing filters which override the zebrad.toml config")]
|
|
|
|
filters: Vec<String>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl CopyStateCmd {
|
|
|
|
/// Configure and launch the copy command
|
|
|
|
async fn start(&self) -> Result<(), Report> {
|
|
|
|
let base_config = app_config().clone();
|
|
|
|
let source_config = base_config.state.clone();
|
|
|
|
|
|
|
|
// The default load_config impl doesn't actually modify the app config.
|
|
|
|
let target_config = self
|
|
|
|
.target_config_path
|
|
|
|
.as_ref()
|
|
|
|
.map(|path| app_writer().load_config(path))
|
|
|
|
.transpose()?
|
|
|
|
.map(|app_config| app_config.state)
|
|
|
|
.unwrap_or_else(new_zs::Config::ephemeral);
|
|
|
|
|
|
|
|
info!(?base_config, "state copy base config");
|
|
|
|
|
|
|
|
self.copy(base_config.network.network, source_config, target_config)
|
|
|
|
.await
|
|
|
|
.map_err(|e| eyre!(e))
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Initialize the source and target states,
|
|
|
|
/// then copy from the source to the target state.
|
|
|
|
async fn copy(
|
|
|
|
&self,
|
|
|
|
network: Network,
|
|
|
|
source_config: old_zs::Config,
|
|
|
|
target_config: new_zs::Config,
|
|
|
|
) -> Result<(), BoxError> {
|
|
|
|
info!(
|
|
|
|
?source_config,
|
|
|
|
"initializing source state service (old format)"
|
|
|
|
);
|
|
|
|
|
|
|
|
let source_start_time = Instant::now();
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
|
|
|
|
// We're not verifying UTXOs here, so we don't need the maximum checkpoint height.
|
|
|
|
//
|
2022-03-11 05:58:22 -08:00
|
|
|
// TODO: use ReadStateService for the source?
|
|
|
|
let (
|
|
|
|
mut source_state,
|
|
|
|
_source_read_only_state_service,
|
|
|
|
_source_latest_chain_tip,
|
|
|
|
_source_chain_tip_change,
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
) = old_zs::spawn_init(source_config.clone(), network, Height::MAX, 0).await?;
|
2021-12-21 18:07:52 -08:00
|
|
|
|
|
|
|
let elapsed = source_start_time.elapsed();
|
|
|
|
info!(?elapsed, "finished initializing source state service");
|
|
|
|
|
|
|
|
info!(
|
|
|
|
?target_config, target_config_path = ?self.target_config_path,
|
|
|
|
"initializing target state service (new format)"
|
|
|
|
);
|
|
|
|
|
|
|
|
let target_start_time = Instant::now();
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
// We're not verifying UTXOs here, so we don't need the maximum checkpoint height.
|
|
|
|
//
|
2022-03-11 05:58:22 -08:00
|
|
|
// TODO: call Options::PrepareForBulkLoad()
|
|
|
|
// See "What's the fastest way to load data into RocksDB?" in
|
|
|
|
// https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ
|
|
|
|
let (
|
|
|
|
mut target_state,
|
|
|
|
_target_read_only_state_service,
|
|
|
|
_target_latest_chain_tip,
|
|
|
|
_target_chain_tip_change,
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
) = new_zs::spawn_init(target_config.clone(), network, Height::MAX, 0).await?;
|
2021-12-21 18:07:52 -08:00
|
|
|
|
|
|
|
let elapsed = target_start_time.elapsed();
|
|
|
|
info!(?elapsed, "finished initializing target state service");
|
|
|
|
|
|
|
|
info!("fetching source and target tip heights");
|
|
|
|
|
|
|
|
let source_tip = source_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(old_zs::Request::Tip)
|
|
|
|
.await?;
|
|
|
|
let source_tip = match source_tip {
|
|
|
|
old_zs::Response::Tip(Some(source_tip)) => source_tip,
|
|
|
|
old_zs::Response::Tip(None) => Err("empty source state: no blocks to copy")?,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Tip request: {:?}",
|
|
|
|
response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
let source_tip_height = source_tip.0 .0;
|
|
|
|
|
|
|
|
let initial_target_tip = target_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(new_zs::Request::Tip)
|
|
|
|
.await?;
|
|
|
|
let initial_target_tip = match initial_target_tip {
|
|
|
|
new_zs::Response::Tip(target_tip) => target_tip,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Tip request: {:?}",
|
|
|
|
response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
let min_target_height = initial_target_tip
|
|
|
|
.map(|target_tip| target_tip.0 .0 + 1)
|
|
|
|
.unwrap_or(0);
|
|
|
|
|
|
|
|
let max_copy_height = self
|
|
|
|
.max_source_height
|
|
|
|
.map(|max_source_height| min(source_tip_height, max_source_height))
|
|
|
|
.unwrap_or(source_tip_height);
|
|
|
|
|
|
|
|
if min_target_height >= max_copy_height {
|
|
|
|
info!(
|
|
|
|
?min_target_height,
|
|
|
|
?max_copy_height,
|
|
|
|
max_source_height = ?self.max_source_height,
|
|
|
|
?source_tip,
|
|
|
|
?initial_target_tip,
|
|
|
|
"target is already at or after max copy height"
|
|
|
|
);
|
|
|
|
|
|
|
|
return Ok(());
|
|
|
|
}
|
|
|
|
|
|
|
|
info!(
|
|
|
|
?min_target_height,
|
|
|
|
?max_copy_height,
|
|
|
|
max_source_height = ?self.max_source_height,
|
|
|
|
?source_tip,
|
|
|
|
?initial_target_tip,
|
|
|
|
"starting copy from source to target"
|
|
|
|
);
|
|
|
|
|
|
|
|
let copy_start_time = Instant::now();
|
|
|
|
for height in min_target_height..=max_copy_height {
|
|
|
|
// Read block from source
|
|
|
|
let source_block = source_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(old_zs::Request::Block(Height(height).into()))
|
|
|
|
.await?;
|
|
|
|
let source_block = match source_block {
|
|
|
|
old_zs::Response::Block(Some(source_block)) => {
|
|
|
|
trace!(?height, %source_block, "read source block");
|
|
|
|
source_block
|
|
|
|
}
|
|
|
|
old_zs::Response::Block(None) => Err(format!(
|
|
|
|
"unexpected missing source block, height: {}",
|
|
|
|
height,
|
|
|
|
))?,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Block request, height: {}, \n \
|
|
|
|
response: {:?}",
|
|
|
|
height, response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
let source_block_hash = source_block.hash();
|
|
|
|
|
|
|
|
// Write block to target
|
|
|
|
let target_block_commit_hash = target_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(new_zs::Request::CommitFinalizedBlock(
|
|
|
|
source_block.clone().into(),
|
|
|
|
))
|
|
|
|
.await?;
|
|
|
|
let target_block_commit_hash = match target_block_commit_hash {
|
|
|
|
new_zs::Response::Committed(target_block_commit_hash) => {
|
|
|
|
trace!(?target_block_commit_hash, "wrote target block");
|
|
|
|
target_block_commit_hash
|
|
|
|
}
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to CommitFinalizedBlock request, height: {}\n \
|
|
|
|
response: {:?}",
|
|
|
|
height, response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
|
|
|
|
// Read written block from target
|
|
|
|
let target_block = target_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(new_zs::Request::Block(Height(height).into()))
|
|
|
|
.await?;
|
|
|
|
let target_block = match target_block {
|
|
|
|
new_zs::Response::Block(Some(target_block)) => {
|
|
|
|
trace!(?height, %target_block, "read target block");
|
|
|
|
target_block
|
|
|
|
}
|
|
|
|
new_zs::Response::Block(None) => Err(format!(
|
|
|
|
"unexpected missing target block, height: {}",
|
|
|
|
height,
|
|
|
|
))?,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Block request, height: {},\n \
|
|
|
|
response: {:?}",
|
|
|
|
height, response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
let target_block_data_hash = target_block.hash();
|
|
|
|
|
|
|
|
// Check for data errors
|
|
|
|
//
|
|
|
|
// These checks make sure that Zebra doesn't corrupt the block data
|
|
|
|
// when serializing it in the new format.
|
|
|
|
// Zebra currently serializes `Block` structs into bytes while writing,
|
|
|
|
// then deserializes bytes into new `Block` structs when reading.
|
|
|
|
// So these checks are sufficient to detect block data corruption.
|
|
|
|
//
|
|
|
|
// If Zebra starts re-using cached `Block` structs after writing them,
|
|
|
|
// we'll also need to check `Block` structs created from the actual database bytes.
|
|
|
|
if source_block_hash != target_block_commit_hash
|
|
|
|
|| source_block_hash != target_block_data_hash
|
|
|
|
|| source_block != target_block
|
|
|
|
{
|
|
|
|
Err(format!(
|
|
|
|
"unexpected mismatch between source and target blocks,\n \
|
|
|
|
max copy height: {:?},\n \
|
|
|
|
source hash: {:?},\n \
|
|
|
|
target commit hash: {:?},\n \
|
|
|
|
target data hash: {:?},\n \
|
|
|
|
source block: {:?},\n \
|
|
|
|
target block: {:?}",
|
|
|
|
max_copy_height,
|
|
|
|
source_block_hash,
|
|
|
|
target_block_commit_hash,
|
|
|
|
target_block_data_hash,
|
|
|
|
source_block,
|
|
|
|
target_block,
|
|
|
|
))?;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Log progress
|
|
|
|
if height % PROGRESS_HEIGHT_INTERVAL == 0 {
|
|
|
|
let elapsed = copy_start_time.elapsed();
|
|
|
|
info!(
|
|
|
|
?height,
|
|
|
|
?max_copy_height,
|
|
|
|
?elapsed,
|
|
|
|
"copied block from source to target"
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
let elapsed = copy_start_time.elapsed();
|
|
|
|
info!(?max_copy_height, ?elapsed, "finished copying blocks");
|
|
|
|
|
|
|
|
info!(?max_copy_height, "fetching final target tip");
|
|
|
|
|
|
|
|
let final_target_tip = target_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(new_zs::Request::Tip)
|
|
|
|
.await?;
|
|
|
|
let final_target_tip = match final_target_tip {
|
|
|
|
new_zs::Response::Tip(Some(target_tip)) => target_tip,
|
|
|
|
new_zs::Response::Tip(None) => Err("empty target state: expected written blocks")?,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Tip request: {:?}",
|
|
|
|
response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
let final_target_tip_height = final_target_tip.0 .0;
|
|
|
|
let final_target_tip_hash = final_target_tip.1;
|
|
|
|
|
|
|
|
let target_tip_source_depth = source_state
|
|
|
|
.ready()
|
|
|
|
.await?
|
|
|
|
.call(old_zs::Request::Depth(final_target_tip_hash))
|
|
|
|
.await?;
|
|
|
|
let target_tip_source_depth = match target_tip_source_depth {
|
|
|
|
old_zs::Response::Depth(source_depth) => source_depth,
|
|
|
|
|
|
|
|
response => Err(format!(
|
|
|
|
"unexpected response to Depth request: {:?}",
|
|
|
|
response,
|
|
|
|
))?,
|
|
|
|
};
|
|
|
|
|
|
|
|
// Check the tips match
|
|
|
|
//
|
|
|
|
// This check works because Zebra doesn't cache tip structs.
|
|
|
|
// (See details above.)
|
|
|
|
if max_copy_height == source_tip_height {
|
|
|
|
let expected_target_depth = Some(0);
|
|
|
|
if source_tip != final_target_tip || target_tip_source_depth != expected_target_depth {
|
|
|
|
Err(format!(
|
|
|
|
"unexpected mismatch between source and target tips,\n \
|
|
|
|
max copy height: {:?},\n \
|
|
|
|
source tip: {:?},\n \
|
|
|
|
target tip: {:?},\n \
|
|
|
|
actual target tip depth in source: {:?},\n \
|
|
|
|
expect target tip depth in source: {:?}",
|
|
|
|
max_copy_height,
|
|
|
|
source_tip,
|
|
|
|
final_target_tip,
|
|
|
|
target_tip_source_depth,
|
|
|
|
expected_target_depth,
|
|
|
|
))?;
|
|
|
|
} else {
|
|
|
|
info!(
|
|
|
|
?max_copy_height,
|
|
|
|
?source_tip,
|
|
|
|
?final_target_tip,
|
|
|
|
?target_tip_source_depth,
|
|
|
|
"source and target states contain the same blocks"
|
|
|
|
);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
let expected_target_depth = source_tip_height.checked_sub(final_target_tip_height);
|
|
|
|
if target_tip_source_depth != expected_target_depth {
|
|
|
|
Err(format!(
|
|
|
|
"unexpected mismatch between source and target tips,\n \
|
|
|
|
max copy height: {:?},\n \
|
|
|
|
source tip: {:?},\n \
|
|
|
|
target tip: {:?},\n \
|
|
|
|
actual target tip depth in source: {:?},\n \
|
|
|
|
expect target tip depth in source: {:?}",
|
|
|
|
max_copy_height,
|
|
|
|
source_tip,
|
|
|
|
final_target_tip,
|
|
|
|
target_tip_source_depth,
|
|
|
|
expected_target_depth,
|
|
|
|
))?;
|
|
|
|
} else {
|
|
|
|
info!(
|
|
|
|
?max_copy_height,
|
|
|
|
?source_tip,
|
|
|
|
?final_target_tip,
|
|
|
|
?target_tip_source_depth,
|
|
|
|
"target state reached the max copy height"
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl Runnable for CopyStateCmd {
|
|
|
|
/// Start the application.
|
|
|
|
fn run(&self) {
|
|
|
|
info!(
|
|
|
|
max_source_height = ?self.max_source_height,
|
|
|
|
target_config_path = ?self.target_config_path,
|
|
|
|
"starting cached chain state copy"
|
|
|
|
);
|
|
|
|
let rt = app_writer()
|
|
|
|
.state_mut()
|
|
|
|
.components
|
|
|
|
.get_downcast_mut::<TokioComponent>()
|
|
|
|
.expect("TokioComponent should be available")
|
|
|
|
.rt
|
|
|
|
.take();
|
|
|
|
|
|
|
|
rt.expect("runtime should not already be taken")
|
|
|
|
.run(self.start());
|
|
|
|
|
|
|
|
info!("finished cached chain state copy");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl config::Override<ZebradConfig> for CopyStateCmd {
|
|
|
|
// Process the given command line options, overriding settings from
|
|
|
|
// a configuration file using explicit flags taken from command-line
|
|
|
|
// arguments.
|
|
|
|
fn override_config(&self, mut config: ZebradConfig) -> Result<ZebradConfig, FrameworkError> {
|
|
|
|
if !self.filters.is_empty() {
|
|
|
|
config.tracing.filter = Some(self.filters.join(","));
|
|
|
|
}
|
|
|
|
|
|
|
|
Ok(config)
|
|
|
|
}
|
|
|
|
}
|