2022-04-27 16:06:11 -07:00
|
|
|
//! Utility functions for tests that used cached Zebra state.
|
2022-05-11 07:06:58 -07:00
|
|
|
//!
|
|
|
|
//! Note: we allow dead code in this module, because it is mainly used by the gRPC tests,
|
|
|
|
//! which are optional.
|
|
|
|
|
|
|
|
#![allow(dead_code)]
|
2022-04-27 16:06:11 -07:00
|
|
|
|
2023-04-26 16:35:53 -07:00
|
|
|
use std::{
|
|
|
|
path::{Path, PathBuf},
|
|
|
|
time::Duration,
|
|
|
|
};
|
2022-11-09 19:40:21 -08:00
|
|
|
|
2022-04-27 16:06:11 -07:00
|
|
|
use color_eyre::eyre::{eyre, Result};
|
|
|
|
use tower::{util::BoxService, Service};
|
|
|
|
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
use zebra_chain::{
|
2023-04-26 16:35:53 -07:00
|
|
|
block::{self, Block, Height},
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
chain_tip::ChainTip,
|
|
|
|
parameters::Network,
|
2023-04-26 16:35:53 -07:00
|
|
|
serialization::ZcashDeserializeInto,
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
};
|
2023-04-26 16:35:53 -07:00
|
|
|
use zebra_node_services::rpc_client::RpcRequestClient;
|
|
|
|
use zebra_state::{ChainTipChange, LatestChainTip, MAX_BLOCK_REORG_HEIGHT};
|
2022-11-09 19:40:21 -08:00
|
|
|
|
|
|
|
use crate::common::{
|
|
|
|
launch::spawn_zebrad_for_rpc,
|
|
|
|
sync::{check_sync_logs_until, MempoolBehavior, SYNC_FINISHED_REGEX},
|
|
|
|
test_type::TestType,
|
|
|
|
};
|
|
|
|
|
2022-04-27 16:06:11 -07:00
|
|
|
/// Path to a directory containing a cached Zebra state.
|
2022-05-04 22:59:31 -07:00
|
|
|
pub const ZEBRA_CACHED_STATE_DIR: &str = "ZEBRA_CACHED_STATE_DIR";
|
2022-04-27 16:06:11 -07:00
|
|
|
|
|
|
|
/// Type alias for a boxed state service.
|
|
|
|
pub type BoxStateService =
|
|
|
|
BoxService<zebra_state::Request, zebra_state::Response, zebra_state::BoxError>;
|
|
|
|
|
|
|
|
/// Starts a state service using the provided `cache_dir` as the directory with the chain state.
|
2022-08-28 10:08:43 -07:00
|
|
|
#[tracing::instrument(skip(cache_dir))]
|
2022-04-27 16:06:11 -07:00
|
|
|
pub async fn start_state_service_with_cache_dir(
|
|
|
|
network: Network,
|
|
|
|
cache_dir: impl Into<PathBuf>,
|
|
|
|
) -> Result<(
|
|
|
|
BoxStateService,
|
|
|
|
impl Service<
|
|
|
|
zebra_state::ReadRequest,
|
|
|
|
Response = zebra_state::ReadResponse,
|
|
|
|
Error = zebra_state::BoxError,
|
|
|
|
>,
|
|
|
|
LatestChainTip,
|
|
|
|
ChainTipChange,
|
|
|
|
)> {
|
|
|
|
let config = zebra_state::Config {
|
|
|
|
cache_dir: cache_dir.into(),
|
|
|
|
..zebra_state::Config::default()
|
|
|
|
};
|
|
|
|
|
change(state): Write non-finalized blocks to the state in a separate thread, to avoid network and RPC hangs (#5257)
* Add a new block commit task and channels, that don't do anything yet
* Add last_block_hash_sent to the state service, to avoid database accesses
* Update last_block_hash_sent regardless of commit errors
* Rename a field to StateService.max_queued_finalized_height
* Commit finalized blocks to the state in a separate task
* Check for panics in the block write task
* Wait for the block commit task in tests, and check for errors
* Always run a proptest that sleeps once
* Add extra debugging to state shutdowns
* Work around a RocksDB shutdown bug
* Close the finalized block channel when we're finished with it
* Only reset state queue once per error
* Update some TODOs
* Add a module doc comment
* Drop channels and check for closed channels in the block commit task
* Close state channels and tasks on drop
* Remove some duplicate fields across StateService and ReadStateService
* Try tweaking the shutdown steps
* Update and clarify some comments
* Clarify another comment
* Don't try to cancel RocksDB background work on drop
* Fix up some comments
* Remove some duplicate code
* Remove redundant workarounds for shutdown issues
* Remode a redundant channel close in the block commit task
* Remove a mistaken `!force` shutdown condition
* Remove duplicate force-shutdown code and explain it better
* Improve RPC error logging
* Wait for chain tip updates in the RPC tests
* Wait 2 seconds for chain tip updates before skipping them
* Remove an unnecessary block_in_place()
* Fix some test error messages that were changed by earlier fixes
* Expand some comments, fix typos
Co-authored-by: Marek <mail@marek.onl>
* Actually drop children of failed blocks
* Explain why we drop descendants of failed blocks
* Clarify a comment
* Wait for chain tip updates in a failing test on macOS
* Clean duplicate finalized blocks when the non-finalized state activates
* Send an error when receiving a duplicate finalized block
* Update checkpoint block behaviour, document its consensus rule
* Wait for chain tip changes in inbound_block_height_lookahead_limit test
* Wait for the genesis block to commit in the fake peer set mempool tests
* Disable unreliable mempool verification check in the send transaction test
* Appease rustfmt
* Use clear_finalized_block_queue() everywhere that blocks are dropped
* Document how Finalized and NonFinalized clones are different
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* updates comments, renames send_process_queued, other minor cleanup
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* sends non-finalized blocks to the block write task
* passes ZebraDb to commit_new_chain, commit_block, and no_duplicates_in_finalized_chain instead of FinalizedState
* updates comments, renames send_process_queued, other minor cleanup
* Update zebra-state/src/service/write.rs
Co-authored-by: teor <teor@riseup.net>
* update assert_block_can_be_validated comment
* removes `mem` field from StateService
* removes `disk` field from StateService and updates block_iter to use `ZebraDb` instead of the finalized state
* updates tests that use the disk to use read_service.db instead
* moves best_tip to a read fn and returns finalized & non-finalized states from setup instead of the state service
* changes `contextual_validity` to get the network from the finalized_state instead of another param
* swaps out StateService with FinalizedState and NonFinalizedState in tests
* adds NotReadyToBeCommitted error and returns it from validate_and_commit when a blocks parent hash is not in any chain
* removes NonFinalizedWriteCmd and calls, moves update_latest_channels above rsp_tx.send
* makes parent_errors_map an indexmap
* clears non-finalized block queue when the receiver is dropped and when the StateService is being dropped
* removes duplicate field definitions on StateService that were a result of a bad merge
* update NotReadyToBeCommitted error message
* Appear rustfmt
* Fix doc links
* Rename a function to initial_contextual_validity()
* Do error tasks on Err, and success tasks on Ok
* Simplify parent_error_map truncation
* Rewrite best_tip() to use tip()
* Rename latest_mem() to latest_non_finalized_state()
```sh
fastmod latest_mem latest_non_finalized_state zebra*
cargo fmt --all
```
* Simplify latest_non_finalized_state() using a new WatchReceiver API
* Expand some error messages
* Send the result after updating the channels, and document why
* wait for chain_tip_update before cancelling download in mempool_cancel_mined
* adds `sent_non_finalized_block_hashes` field to StateService
* adds batched sent_hash insertions and checks sent hashes in queue_and_commit_non_finalized before adding a block to the queue
* check that the `curr_buf` in SentHashes is not empty before pushing it to the `sent_bufs`
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* Fix rustfmt
* Check for finalized block heights using zs_contains()
* adds known_utxos field to SentHashes
* updates comment on SentHashes.add method
* Apply suggestions from code review
Co-authored-by: teor <teor@riseup.net>
* return early when there's a duplicate hash in QueuedBlocks.queue instead of panicking
* Make finalized UTXOs near the final checkpoint available for full block verification
* Replace a checkpoint height literal with the actual config
* Update mainnet and testnet checkpoints - 7 October 2022
* Fix some state service init arguments
* Allow more lookahead in the downloader, but less lookahead in the syncer
* Add the latest config to the tests, and fix the latest config check
* Increase the number of finalized blocks checked for non-finalized block UTXO spends
* fix(log): reduce verbose logs for block commits (#5348)
* Remove some verbose block write channel logs
* Only warn about tracing endpoint if the address is actually set
* Use CloneError instead of formatting a non-cloneable error
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
* Increase block verify timeout
* Work around a known block timeout bug by using a shorter timeout
Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2022-10-11 12:25:45 -07:00
|
|
|
// These tests don't need UTXOs to be verified efficiently, because they use cached states.
|
|
|
|
Ok(zebra_state::init(config, network, Height::MAX, 0))
|
2022-04-27 16:06:11 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Loads the chain tip height from the state stored in a specified directory.
|
2022-08-28 10:08:43 -07:00
|
|
|
#[tracing::instrument]
|
2022-04-27 16:06:11 -07:00
|
|
|
pub async fn load_tip_height_from_state_directory(
|
|
|
|
network: Network,
|
|
|
|
state_path: &Path,
|
|
|
|
) -> Result<block::Height> {
|
|
|
|
let (_state_service, _read_state_service, latest_chain_tip, _chain_tip_change) =
|
|
|
|
start_state_service_with_cache_dir(network, state_path).await?;
|
|
|
|
|
|
|
|
let chain_tip_height = latest_chain_tip
|
|
|
|
.best_tip_height()
|
|
|
|
.ok_or_else(|| eyre!("State directory doesn't have a chain tip block"))?;
|
|
|
|
|
|
|
|
Ok(chain_tip_height)
|
|
|
|
}
|
|
|
|
|
2022-11-09 19:40:21 -08:00
|
|
|
/// Accepts a network, test_type, test_name, and num_blocks (how many blocks past the finalized tip to try getting)
|
|
|
|
///
|
|
|
|
/// Syncs zebra until the tip, gets some blocks near the tip, via getblock rpc calls,
|
|
|
|
/// shuts down zebra, and gets the finalized tip height of the updated cached state.
|
|
|
|
///
|
|
|
|
/// Returns retrieved and deserialized blocks that are above the finalized tip height of the cached state.
|
|
|
|
///
|
|
|
|
/// ## Panics
|
|
|
|
///
|
|
|
|
/// If the provided `test_type` doesn't need an rpc server and cached state, or if `max_num_blocks` is 0
|
|
|
|
pub async fn get_future_blocks(
|
|
|
|
network: Network,
|
|
|
|
test_type: TestType,
|
|
|
|
test_name: &str,
|
|
|
|
max_num_blocks: u32,
|
|
|
|
) -> Result<Vec<Block>> {
|
|
|
|
let blocks: Vec<Block> = get_raw_future_blocks(network, test_type, test_name, max_num_blocks)
|
|
|
|
.await?
|
|
|
|
.into_iter()
|
|
|
|
.map(hex::decode)
|
|
|
|
.map(|block_bytes| {
|
|
|
|
block_bytes
|
|
|
|
.expect("getblock rpc calls in get_raw_future_blocks should return valid hexdata")
|
|
|
|
.zcash_deserialize_into()
|
|
|
|
.expect("decoded hex data from getblock rpc calls should deserialize into blocks")
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
Ok(blocks)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Accepts a network, test_type, test_name, and num_blocks (how many blocks past the finalized tip to try getting)
|
|
|
|
///
|
|
|
|
/// Syncs zebra until the tip, gets some blocks near the tip, via getblock rpc calls,
|
|
|
|
/// shuts down zebra, and gets the finalized tip height of the updated cached state.
|
|
|
|
///
|
|
|
|
/// Returns hexdata of retrieved blocks that are above the finalized tip height of the cached state.
|
|
|
|
///
|
|
|
|
/// ## Panics
|
|
|
|
///
|
|
|
|
/// If the provided `test_type` doesn't need an rpc server and cached state, or if `max_num_blocks` is 0
|
|
|
|
pub async fn get_raw_future_blocks(
|
|
|
|
network: Network,
|
|
|
|
test_type: TestType,
|
|
|
|
test_name: &str,
|
|
|
|
max_num_blocks: u32,
|
|
|
|
) -> Result<Vec<String>> {
|
|
|
|
assert!(max_num_blocks > 0);
|
|
|
|
|
|
|
|
let max_num_blocks = max_num_blocks.min(MAX_BLOCK_REORG_HEIGHT);
|
|
|
|
let mut raw_blocks = Vec::with_capacity(max_num_blocks as usize);
|
|
|
|
|
|
|
|
assert!(
|
|
|
|
test_type.needs_zebra_cached_state() && test_type.needs_zebra_rpc_server(),
|
|
|
|
"get_raw_future_blocks needs zebra cached state and rpc server"
|
|
|
|
);
|
|
|
|
|
|
|
|
let should_sync = true;
|
|
|
|
let (zebrad, zebra_rpc_address) =
|
|
|
|
spawn_zebrad_for_rpc(network, test_name, test_type, should_sync)?
|
|
|
|
.ok_or_else(|| eyre!("get_raw_future_blocks requires a cached state"))?;
|
|
|
|
let rpc_address = zebra_rpc_address.expect("test type must have RPC port");
|
|
|
|
|
|
|
|
let mut zebrad = check_sync_logs_until(
|
|
|
|
zebrad,
|
|
|
|
network,
|
|
|
|
SYNC_FINISHED_REGEX,
|
|
|
|
MempoolBehavior::ShouldAutomaticallyActivate,
|
|
|
|
true,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
// Create an http client
|
2023-04-26 16:35:53 -07:00
|
|
|
let rpc_client = RpcRequestClient::new(rpc_address);
|
2022-11-09 19:40:21 -08:00
|
|
|
|
|
|
|
let blockchain_info: serde_json::Value = serde_json::from_str(
|
2022-11-16 13:23:29 -08:00
|
|
|
&rpc_client
|
|
|
|
.text_from_call("getblockchaininfo", "[]".to_string())
|
2022-11-09 19:40:21 -08:00
|
|
|
.await?,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
let tip_height: u32 = blockchain_info["result"]["blocks"]
|
|
|
|
.as_u64()
|
|
|
|
.expect("unexpected block height: doesn't fit in u64")
|
|
|
|
.try_into()
|
|
|
|
.expect("unexpected block height: doesn't fit in u32");
|
|
|
|
|
|
|
|
let estimated_finalized_tip_height = tip_height - MAX_BLOCK_REORG_HEIGHT;
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
?tip_height,
|
|
|
|
?estimated_finalized_tip_height,
|
|
|
|
"got tip height from blockchaininfo",
|
|
|
|
);
|
|
|
|
|
|
|
|
for block_height in (0..max_num_blocks).map(|idx| idx + estimated_finalized_tip_height) {
|
|
|
|
let raw_block: serde_json::Value = serde_json::from_str(
|
2022-11-16 13:23:29 -08:00
|
|
|
&rpc_client
|
|
|
|
.text_from_call("getblock", format!(r#"["{block_height}", 0]"#))
|
2022-11-09 19:40:21 -08:00
|
|
|
.await?,
|
|
|
|
)?;
|
|
|
|
|
|
|
|
raw_blocks.push((
|
|
|
|
block_height,
|
|
|
|
raw_block["result"]
|
|
|
|
.as_str()
|
|
|
|
.expect("unexpected getblock result: not a string")
|
|
|
|
.to_string(),
|
|
|
|
));
|
|
|
|
}
|
|
|
|
|
|
|
|
zebrad.kill(true)?;
|
|
|
|
|
|
|
|
// Sleep for a few seconds to make sure zebrad releases lock on cached state directory
|
|
|
|
std::thread::sleep(Duration::from_secs(3));
|
|
|
|
|
|
|
|
let zebrad_state_path = test_type
|
|
|
|
.zebrad_state_path(test_name)
|
|
|
|
.expect("already checked that there is a cached state path");
|
|
|
|
|
|
|
|
let Height(finalized_tip_height) =
|
|
|
|
load_tip_height_from_state_directory(network, zebrad_state_path.as_ref()).await?;
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
?finalized_tip_height,
|
|
|
|
non_finalized_tip_height = ?tip_height,
|
|
|
|
?estimated_finalized_tip_height,
|
|
|
|
"got finalized tip height from state directory"
|
|
|
|
);
|
|
|
|
|
|
|
|
let raw_future_blocks = raw_blocks
|
|
|
|
.into_iter()
|
|
|
|
.filter_map(|(height, raw_block)| height.gt(&finalized_tip_height).then_some(raw_block))
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
Ok(raw_future_blocks)
|
|
|
|
}
|