From ec43d63ed2384b7aca201e4dfd9c92caf9a5c72f Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 23 Feb 2023 10:10:11 +1000 Subject: [PATCH] change(log): Log a cute message for blocks that were mined by Zebra (off by default) (#6098) * Mark Zebra coinbase transactions with extra coinbase data * Log when we commit a block mined by Zebra to our state * Reduce logging instrumentation during block writes * Remove debug types in Zebra block log * Add network and commit to write task logs * Apply an allow-list before we log arbitrary user messages from blocks * Rate-limit Zebra mined block logging to once every 1000 blocks * Add mining configs for extra coinbase data and imitating zcashd, but don't use them yet * Check CoinbaseData size limit when building transparent transactions * Replace LIKE_ZCASHD constants with a config * Take extra coinbase data from the configured string * Update the zebrad configs in the tests with new config fields --- zebra-chain/src/transaction/builder.rs | 19 ++- zebra-chain/src/transparent.rs | 82 ++++++++--- zebra-chain/src/transparent/serialize.rs | 26 +++- zebra-rpc/src/methods.rs | 12 +- .../src/methods/get_block_template_rpcs.rs | 62 +++++++-- .../methods/get_block_template_rpcs/config.rs | 26 +++- .../get_block_template.rs | 17 ++- .../types/get_block_template.rs | 3 + .../methods/get_block_template_rpcs/zip317.rs | 21 ++- zebra-rpc/src/methods/tests/prop.rs | 14 ++ zebra-rpc/src/methods/tests/snapshot.rs | 1 + .../tests/snapshot/get_block_template_rpcs.rs | 2 + zebra-rpc/src/methods/tests/vectors.rs | 18 ++- zebra-rpc/src/server.rs | 6 +- zebra-state/src/service.rs | 21 +-- zebra-state/src/service/write.rs | 127 ++++++++++++++++-- .../configs/getblocktemplate-v1.0.0-rc.5.toml | 73 ++++++++++ 17 files changed, 459 insertions(+), 71 deletions(-) create mode 100644 zebrad/tests/common/configs/getblocktemplate-v1.0.0-rc.5.toml diff --git a/zebra-chain/src/transaction/builder.rs b/zebra-chain/src/transaction/builder.rs index 6b3dfb178..ab1eca500 100644 --- a/zebra-chain/src/transaction/builder.rs +++ b/zebra-chain/src/transaction/builder.rs @@ -15,6 +15,7 @@ impl Transaction { network: Network, height: Height, outputs: impl IntoIterator, transparent::Script)>, + extra_coinbase_data: Vec, ) -> Transaction { // # Consensus // @@ -36,13 +37,17 @@ impl Transaction { // // > A coinbase transaction script MUST have length in {2 .. 100} bytes. // - // Zebra does not add any extra coinbase data. + // Zebra adds extra coinbase data if configured to do so. // // Since we're not using a lock time, any sequence number is valid here. // See `Transaction::lock_time()` for the relevant consensus rules. // // - let inputs = vec![transparent::Input::new_coinbase(height, None, None)]; + let inputs = vec![transparent::Input::new_coinbase( + height, + Some(extra_coinbase_data), + None, + )]; // > The block subsidy is composed of a miner subsidy and a series of funding streams. // @@ -108,17 +113,23 @@ impl Transaction { height: Height, outputs: impl IntoIterator, transparent::Script)>, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> Transaction { - // `zcashd` includes an extra byte after the coinbase height in the coinbase data, - // and a sequence number of u32::MAX. let mut extra_data = None; let mut sequence = None; + // `zcashd` includes an extra byte after the coinbase height in the coinbase data, + // and a sequence number of u32::MAX. if like_zcashd { extra_data = Some(vec![0x00]); sequence = Some(u32::MAX); } + // Override like_zcashd if extra_coinbase_data was supplied + if !extra_coinbase_data.is_empty() { + extra_data = Some(extra_coinbase_data); + } + // # Consensus // // See the other consensus rules above in new_v5_coinbase(). diff --git a/zebra-chain/src/transparent.rs b/zebra-chain/src/transparent.rs index 4834afb5e..0dccca92b 100644 --- a/zebra-chain/src/transparent.rs +++ b/zebra-chain/src/transparent.rs @@ -1,5 +1,15 @@ //! Transparent-related (Bitcoin-inherited) functionality. +use std::{collections::HashMap, fmt, iter}; + +use crate::{ + amount::{Amount, NonNegative}, + block, + parameters::Network, + primitives::zcash_primitives, + transaction, +}; + mod address; mod keys; mod opcodes; @@ -9,7 +19,7 @@ mod utxo; pub use address::Address; pub use script::Script; -pub use serialize::GENESIS_COINBASE_DATA; +pub use serialize::{GENESIS_COINBASE_DATA, MAX_COINBASE_DATA_LEN, MAX_COINBASE_HEIGHT_DATA_LEN}; pub use utxo::{ new_ordered_outputs, new_outputs, outputs_from_utxos, utxos_from_ordered_utxos, CoinbaseSpendRestriction, OrderedUtxo, Utxo, @@ -20,24 +30,14 @@ pub use utxo::{ new_ordered_outputs_with_height, new_outputs_with_height, new_transaction_ordered_outputs, }; -#[cfg(any(test, feature = "proptest-impl"))] -use proptest_derive::Arbitrary; - #[cfg(any(test, feature = "proptest-impl"))] mod arbitrary; #[cfg(test)] mod tests; -use crate::{ - amount::{Amount, NonNegative}, - block, - parameters::Network, - primitives::zcash_primitives, - transaction, -}; - -use std::{collections::HashMap, fmt, iter}; +#[cfg(any(test, feature = "proptest-impl"))] +use proptest_derive::Arbitrary; /// The maturity threshold for transparent coinbase outputs. /// @@ -48,16 +48,28 @@ use std::{collections::HashMap, fmt, iter}; /// [7.1](https://zips.z.cash/protocol/nu5.pdf#txnencodingandconsensus) pub const MIN_TRANSPARENT_COINBASE_MATURITY: u32 = 100; +/// Extra coinbase data that identifies some coinbase transactions generated by Zebra. +/// +// +// # Note +// +// rust-analyzer will crash in some editors when moving over an actual Zebra emoji, +// so we encode it here. This is a known issue in emacs-lsp and other lsp implementations: +// - https://github.com/rust-lang/rust-analyzer/issues/9121 +// - https://github.com/emacs-lsp/lsp-mode/issues/2080 +// - https://github.com/rust-lang/rust-analyzer/issues/13709 +pub const EXTRA_ZEBRA_COINBASE_DATA: &str = "z\u{1F993}"; + /// Arbitrary data inserted by miners into a coinbase transaction. +// +// TODO: rename to ExtraCoinbaseData, because height is also part of the coinbase data? #[derive(Clone, Eq, PartialEq)] #[cfg_attr(any(test, feature = "proptest-impl"), derive(Serialize))] pub struct CoinbaseData( /// Invariant: this vec, together with the coinbase height, must be less than /// 100 bytes. We enforce this by only constructing CoinbaseData fields by - /// parsing blocks with 100-byte data fields. When we implement block - /// creation, we should provide a constructor for the coinbase data field - /// that restricts it to 95 = 100 -1 -4 bytes (safe for any block height up - /// to 500_000_000). + /// parsing blocks with 100-byte data fields, and checking newly created + /// CoinbaseData lengths in the transaction builder. pub(super) Vec, ); @@ -182,17 +194,39 @@ impl fmt::Display for Input { impl Input { /// Returns a new coinbase input for `height` with optional `data` and `sequence`. + /// + /// # Consensus + /// + /// The combined serialized size of `height` and `data` can be at most 100 bytes. + /// + /// > A coinbase transaction script MUST have length in {2 .. 100} bytes. + /// + /// + /// + /// # Panics + /// + /// If the coinbase data is greater than [`MAX_COINBASE_DATA_LEN`]. #[cfg(feature = "getblocktemplate-rpcs")] pub fn new_coinbase( height: block::Height, data: Option>, sequence: Option, ) -> Input { + // "No extra coinbase data" is the default. + let data = data.unwrap_or_default(); + let height_size = height.coinbase_zcash_serialized_size(); + + assert!( + data.len() + height_size <= MAX_COINBASE_DATA_LEN, + "invalid coinbase data: extra data {} bytes + height {height_size} bytes \ + must be {} or less", + data.len(), + MAX_COINBASE_DATA_LEN, + ); + Input::Coinbase { height, - - // "No extra coinbase data" is the default. - data: CoinbaseData(data.unwrap_or_default()), + data: CoinbaseData(data), // If the caller does not specify the sequence number, // use a sequence number that activates the LockTime. @@ -200,6 +234,14 @@ impl Input { } } + /// Returns the extra coinbase data in this input, if it is an [`Input::Coinbase`]. + pub fn extra_coinbase_data(&self) -> Option<&CoinbaseData> { + match self { + Input::PrevOut { .. } => None, + Input::Coinbase { data, .. } => Some(data), + } + } + /// Returns the input's sequence number. pub fn sequence(&self) -> u32 { match self { diff --git a/zebra-chain/src/transparent/serialize.rs b/zebra-chain/src/transparent/serialize.rs index 9512f3c65..1c769910c 100644 --- a/zebra-chain/src/transparent/serialize.rs +++ b/zebra-chain/src/transparent/serialize.rs @@ -5,9 +5,9 @@ use std::io; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crate::{ - block, + block::{self, Height}, serialization::{ - zcash_serialize_bytes, ReadZcashExt, SerializationError, ZcashDeserialize, + zcash_serialize_bytes, FakeWriter, ReadZcashExt, SerializationError, ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize, }, transaction, @@ -26,6 +26,16 @@ use super::{CoinbaseData, Input, OutPoint, Output, Script}; /// pub const MAX_COINBASE_DATA_LEN: usize = 100; +/// The maximum length of the encoded coinbase height. +/// +/// # Consensus +/// +/// > The length of heightBytes MUST be in the range {1 .. 5}. Then the encoding is the length +/// > of heightBytes encoded as one byte, followed by heightBytes itself. +/// +/// +pub const MAX_COINBASE_HEIGHT_DATA_LEN: usize = 6; + /// The minimum length of the coinbase data. /// /// Includes the encoded coinbase height, if any. @@ -100,7 +110,6 @@ impl ZcashDeserialize for OutPoint { pub(crate) fn parse_coinbase_height( mut data: Vec, ) -> Result<(block::Height, CoinbaseData), SerializationError> { - use block::Height; match (data.first(), data.len()) { // Blocks 1 through 16 inclusive encode block height with OP_N opcodes. (Some(op_n @ 0x51..=0x60), len) if len >= 1 => Ok(( @@ -226,6 +235,17 @@ pub(crate) fn write_coinbase_height( Ok(()) } +impl Height { + /// Get the size of `Height` when serialized into a coinbase input script. + pub fn coinbase_zcash_serialized_size(&self) -> usize { + let mut writer = FakeWriter(0); + let empty_data = CoinbaseData(Vec::new()); + + write_coinbase_height(*self, &empty_data, &mut writer).expect("writer should never fail"); + writer.0 + } +} + impl ZcashSerialize for Input { /// Serialize this transparent input. /// diff --git a/zebra-rpc/src/methods.rs b/zebra-rpc/src/methods.rs index c3e376f61..d80eef5e7 100644 --- a/zebra-rpc/src/methods.rs +++ b/zebra-rpc/src/methods.rs @@ -262,6 +262,10 @@ where /// no matter what the estimated height or local clock is. debug_force_finished_sync: bool, + /// Test-only option that makes RPC responses more like `zcashd`. + #[allow(dead_code)] + debug_like_zcashd: bool, + // Services // /// A handle to the mempool service. @@ -301,6 +305,7 @@ where app_version: Version, network: Network, debug_force_finished_sync: bool, + debug_like_zcashd: bool, mempool: Buffer, state: State, latest_chain_tip: Tip, @@ -323,6 +328,7 @@ where app_version, network, debug_force_finished_sync, + debug_like_zcashd, mempool: mempool.clone(), state: state.clone(), latest_chain_tip: latest_chain_tip.clone(), @@ -763,14 +769,14 @@ where use zebra_chain::block::MAX_BLOCK_BYTES; #[cfg(feature = "getblocktemplate-rpcs")] - /// Determines whether the output of this RPC is sorted like zcashd - const SHOULD_USE_ZCASHD_ORDER: bool = true; + // Determines whether the output of this RPC is sorted like zcashd + let should_use_zcashd_order = self.debug_like_zcashd; let mut mempool = self.mempool.clone(); async move { #[cfg(feature = "getblocktemplate-rpcs")] - let request = if SHOULD_USE_ZCASHD_ORDER { + let request = if should_use_zcashd_order { mempool::Request::FullTransactions } else { mempool::Request::TransactionIds diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index 1e2e0379a..87539cde9 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -17,7 +17,9 @@ use zebra_chain::{ parameters::Network, primitives, serialization::ZcashDeserializeInto, - transparent, + transparent::{ + self, EXTRA_ZEBRA_COINBASE_DATA, MAX_COINBASE_DATA_LEN, MAX_COINBASE_HEIGHT_DATA_LEN, + }, work::difficulty::{ExpandedDifficulty, U256}, }; use zebra_consensus::{ @@ -246,6 +248,14 @@ where /// Zebra currently only supports transparent addresses. miner_address: Option, + /// Extra data to include in coinbase transaction inputs. + /// Limited to around 95 bytes by the consensus rules. + extra_coinbase_data: Vec, + + /// Should Zebra's block templates try to imitate `zcashd`? + /// Developer-only config. + debug_like_zcashd: bool, + // Services // /// A handle to the mempool service. @@ -293,6 +303,10 @@ where AddressBook: AddressBookPeers + Clone + Send + Sync + 'static, { /// Create a new instance of the handler for getblocktemplate RPCs. + /// + /// # Panics + /// + /// If the `mining_config` is invalid. #[allow(clippy::too_many_arguments)] pub fn new( network: Network, @@ -304,9 +318,38 @@ where sync_status: SyncStatus, address_book: AddressBook, ) -> Self { + // A limit on the configured extra coinbase data, regardless of the current block height. + // This is different from the consensus rule, which limits the total height + data. + const EXTRA_COINBASE_DATA_LIMIT: usize = + MAX_COINBASE_DATA_LEN - MAX_COINBASE_HEIGHT_DATA_LEN; + + let debug_like_zcashd = mining_config.debug_like_zcashd; + + // Hex-decode to bytes if possible, otherwise UTF-8 encode to bytes. + let extra_coinbase_data = mining_config.extra_coinbase_data.unwrap_or_else(|| { + if debug_like_zcashd { + "" + } else { + EXTRA_ZEBRA_COINBASE_DATA + } + .to_string() + }); + let extra_coinbase_data = hex::decode(&extra_coinbase_data) + .unwrap_or_else(|_error| extra_coinbase_data.as_bytes().to_vec()); + + assert!( + extra_coinbase_data.len() <= EXTRA_COINBASE_DATA_LIMIT, + "extra coinbase data is {} bytes, but Zebra's limit is {}.\n\ + Configure mining.extra_coinbase_data with a shorter string", + extra_coinbase_data.len(), + EXTRA_COINBASE_DATA_LIMIT, + ); + Self { network, miner_address: mining_config.miner_address, + extra_coinbase_data, + debug_like_zcashd, mempool, state, latest_chain_tip, @@ -389,14 +432,11 @@ where &self, parameters: Option, ) -> BoxFuture> { - // Should we generate coinbase transactions that are exactly like zcashd's? - // - // This is useful for testing, but either way Zebra should obey the consensus rules. - const COINBASE_LIKE_ZCASHD: bool = true; - - // Clone Config + // Clone Configs let network = self.network; let miner_address = self.miner_address; + let debug_like_zcashd = self.debug_like_zcashd; + let extra_coinbase_data = self.extra_coinbase_data.clone(); // Clone Services let mempool = self.mempool.clone(); @@ -634,14 +674,13 @@ where ); // Randomly select some mempool transactions. - // - // TODO: sort these transactions to match zcashd's order, to make testing easier. let mempool_txs = zip317::select_mempool_transactions( network, next_block_height, miner_address, mempool_txs, - COINBASE_LIKE_ZCASHD, + debug_like_zcashd, + extra_coinbase_data.clone(), ) .await; @@ -662,7 +701,8 @@ where server_long_poll_id, mempool_txs, submit_old, - COINBASE_LIKE_ZCASHD, + debug_like_zcashd, + extra_coinbase_data, ); Ok(response.into()) diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/config.rs b/zebra-rpc/src/methods/get_block_template_rpcs/config.rs index 9e169c116..3747990b3 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/config.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/config.rs @@ -5,7 +5,7 @@ use serde::{Deserialize, Serialize}; use zebra_chain::transparent; /// Mining configuration section. -#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] pub struct Config { /// The address used for miner payouts. @@ -14,4 +14,28 @@ pub struct Config { /// Zebra sends mining fees and miner rewards to this address in the /// `getblocktemplate` RPC coinbase transaction. pub miner_address: Option, + + /// Extra data to include in coinbase transaction inputs. + /// Limited to around 95 bytes by the consensus rules. + /// + /// If this string is hex-encoded, it will be hex-decoded into bytes. + /// Otherwise, it will be UTF-8 encoded into bytes. + pub extra_coinbase_data: Option, + + /// Should Zebra's block templates try to imitate `zcashd`? + /// + /// This developer-only config is not supported for general use. + pub debug_like_zcashd: bool, +} + +impl Default for Config { + fn default() -> Self { + Self { + miner_address: None, + // For now, act like `zcashd` as much as possible. + // TODO: do we want to default to v5 transactions and Zebra coinbase data? + extra_coinbase_data: None, + debug_like_zcashd: true, + } + } } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs index 584e48524..16d4691d8 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/get_block_template.rs @@ -273,11 +273,18 @@ pub fn generate_coinbase_and_roots( mempool_txs: &[VerifiedUnminedTx], history_tree: Arc, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> (TransactionTemplate, DefaultRoots) { // Generate the coinbase transaction let miner_fee = calculate_miner_fee(mempool_txs); - let coinbase_txn = - generate_coinbase_transaction(network, height, miner_address, miner_fee, like_zcashd); + let coinbase_txn = generate_coinbase_transaction( + network, + height, + miner_address, + miner_fee, + like_zcashd, + extra_coinbase_data, + ); // Calculate block default roots // @@ -301,13 +308,15 @@ pub fn generate_coinbase_transaction( miner_address: transparent::Address, miner_fee: Amount, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> UnminedTx { let outputs = standard_coinbase_outputs(network, height, miner_address, miner_fee, like_zcashd); if like_zcashd { - Transaction::new_v4_coinbase(network, height, outputs, like_zcashd).into() + Transaction::new_v4_coinbase(network, height, outputs, like_zcashd, extra_coinbase_data) + .into() } else { - Transaction::new_v5_coinbase(network, height, outputs).into() + Transaction::new_v5_coinbase(network, height, outputs, extra_coinbase_data).into() } } diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 5a2237953..df85939e4 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -182,6 +182,7 @@ impl GetBlockTemplate { /// /// If `like_zcashd` is true, try to match the coinbase transactions generated by `zcashd` /// in the `getblocktemplate` RPC. + #[allow(clippy::too_many_arguments)] pub fn new( network: Network, miner_address: transparent::Address, @@ -190,6 +191,7 @@ impl GetBlockTemplate { mempool_txs: Vec, submit_old: Option, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> Self { // Calculate the next block height. let next_block_height = @@ -229,6 +231,7 @@ impl GetBlockTemplate { &mempool_txs, chain_tip_and_local_time.history_tree.clone(), like_zcashd, + extra_coinbase_data, ); // Convert difficulty diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs index adb4af608..2b0865226 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/zip317.rs @@ -46,11 +46,17 @@ pub async fn select_mempool_transactions( miner_address: transparent::Address, mempool_txs: Vec, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> Vec { // Use a fake coinbase transaction to break the dependency between transaction // selection, the miner fee, and the fee payment in the coinbase transaction. - let fake_coinbase_tx = - fake_coinbase_transaction(network, next_block_height, miner_address, like_zcashd); + let fake_coinbase_tx = fake_coinbase_transaction( + network, + next_block_height, + miner_address, + like_zcashd, + extra_coinbase_data, + ); // Setup the transaction lists. let (mut conventional_fee_txs, mut low_fee_txs): (Vec<_>, Vec<_>) = mempool_txs @@ -117,6 +123,7 @@ pub fn fake_coinbase_transaction( height: Height, miner_address: transparent::Address, like_zcashd: bool, + extra_coinbase_data: Vec, ) -> TransactionTemplate { // Block heights are encoded as variable-length (script) and `u32` (lock time, expiry height). // They can also change the `u32` consensus branch id. @@ -129,8 +136,14 @@ pub fn fake_coinbase_transaction( // https://developer.bitcoin.org/reference/transactions.html#txout-a-transaction-output let miner_fee = 1.try_into().expect("amount is valid and non-negative"); - let coinbase_tx = - generate_coinbase_transaction(network, height, miner_address, miner_fee, like_zcashd); + let coinbase_tx = generate_coinbase_transaction( + network, + height, + miner_address, + miner_fee, + like_zcashd, + extra_coinbase_data, + ); TransactionTemplate::from_coinbase(&coinbase_tx, miner_fee) } diff --git a/zebra-rpc/src/methods/tests/prop.rs b/zebra-rpc/src/methods/tests/prop.rs index 822a78200..8f780af08 100644 --- a/zebra-rpc/src/methods/tests/prop.rs +++ b/zebra-rpc/src/methods/tests/prop.rs @@ -43,6 +43,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -96,6 +97,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -154,6 +156,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -220,6 +223,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -275,6 +279,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -328,6 +333,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -424,6 +430,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -481,6 +488,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -527,6 +535,7 @@ proptest! { "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -576,6 +585,7 @@ proptest! { "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, @@ -661,6 +671,7 @@ proptest! { "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, @@ -723,6 +734,7 @@ proptest! { "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), chain_tip, @@ -773,6 +785,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -861,6 +874,7 @@ proptest! { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, diff --git a/zebra-rpc/src/methods/tests/snapshot.rs b/zebra-rpc/src/methods/tests/snapshot.rs index 474d026eb..8becc4bda 100644 --- a/zebra-rpc/src/methods/tests/snapshot.rs +++ b/zebra-rpc/src/methods/tests/snapshot.rs @@ -70,6 +70,7 @@ async fn test_rpc_response_data_for_network(network: Network) { "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index 3d0e718ae..48fcfd697 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -98,6 +98,8 @@ pub async fn test_responses( let mining_config = get_block_template_rpcs::config::Config { miner_address: Some(transparent::Address::from_script_hash(network, [0xad; 20])), + extra_coinbase_data: None, + debug_like_zcashd: true, }; // nu5 block height diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index c7cd54a17..50641ddc5 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -32,6 +32,7 @@ async fn rpc_getinfo() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -75,6 +76,7 @@ async fn rpc_getblock() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, @@ -225,6 +227,7 @@ async fn rpc_getblock_parse_error() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -266,6 +269,7 @@ async fn rpc_getblock_missing_error() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -333,6 +337,7 @@ async fn rpc_getbestblockhash() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, @@ -374,6 +379,7 @@ async fn rpc_getrawtransaction() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), read_state, latest_chain_tip, @@ -461,6 +467,7 @@ async fn rpc_getaddresstxids_invalid_arguments() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, @@ -603,6 +610,7 @@ async fn rpc_getaddresstxids_response_with( "RPC test", network, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, @@ -653,6 +661,7 @@ async fn rpc_getaddressutxos_invalid_arguments() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(state.clone(), 1), NoChainTip, @@ -700,6 +709,7 @@ async fn rpc_getaddressutxos_response() { "RPC test", Mainnet, false, + true, Buffer::new(mempool.clone(), 1), Buffer::new(read_state.clone(), 1), latest_chain_tip, @@ -1116,7 +1126,11 @@ async fn rpc_getblocktemplate_mining_address(use_p2pkh: bool) { true => Some(transparent::Address::from_pub_key_hash(Mainnet, [0x7e; 20])), }; - let mining_config = Config { miner_address }; + let mining_config = Config { + miner_address, + extra_coinbase_data: None, + debug_like_zcashd: true, + }; // nu5 block height let fake_tip_height = NetworkUpgrade::Nu5.activation_height(Mainnet).unwrap(); @@ -1510,6 +1524,8 @@ async fn rpc_getdifficulty() { let mining_config = Config { miner_address: None, + extra_coinbase_data: None, + debug_like_zcashd: true, }; // nu5 block height diff --git a/zebra-rpc/src/server.rs b/zebra-rpc/src/server.rs index 7745e5851..0a52a1eca 100644 --- a/zebra-rpc/src/server.rs +++ b/zebra-rpc/src/server.rs @@ -145,7 +145,7 @@ impl RpcServer { // Initialize the getblocktemplate rpc method handler let get_block_template_rpc_impl = GetBlockTemplateRpcImpl::new( network, - mining_config, + mining_config.clone(), mempool.clone(), state.clone(), latest_chain_tip.clone(), @@ -162,6 +162,10 @@ impl RpcServer { app_version.clone(), network, config.debug_force_finished_sync, + #[cfg(feature = "getblocktemplate-rpcs")] + mining_config.debug_like_zcashd, + #[cfg(not(feature = "getblocktemplate-rpcs"))] + true, mempool, state, latest_chain_tip, diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index d340c846c..00069478a 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -338,16 +338,19 @@ impl StateService { tokio::sync::mpsc::unbounded_channel(); let finalized_state_for_writing = finalized_state.clone(); + let span = Span::current(); let block_write_task = std::thread::spawn(move || { - write::write_blocks_from_channels( - finalized_block_write_receiver, - non_finalized_block_write_receiver, - finalized_state_for_writing, - non_finalized_state, - invalid_block_reset_sender, - chain_tip_sender, - non_finalized_state_sender, - ) + span.in_scope(move || { + write::write_blocks_from_channels( + finalized_block_write_receiver, + non_finalized_block_write_receiver, + finalized_state_for_writing, + non_finalized_state, + invalid_block_reset_sender, + chain_tip_sender, + non_finalized_state_sender, + ) + }) }); let block_write_task = Arc::new(block_write_task); diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 8f33af7b6..ad04dd075 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -6,7 +6,10 @@ use tokio::sync::{ watch, }; -use zebra_chain::block::{self, Height}; +use zebra_chain::{ + block::{self, Height}, + transparent::EXTRA_ZEBRA_COINBASE_DATA, +}; use crate::{ constants::MAX_BLOCK_REORG_HEIGHT, @@ -34,7 +37,15 @@ const PARENT_ERROR_MAP_LIMIT: usize = MAX_BLOCK_REORG_HEIGHT as usize * 2; /// Run contextual validation on the prepared block and add it to the /// non-finalized state if it is contextually valid. -#[tracing::instrument(level = "debug", skip(prepared), fields(height = ?prepared.height, hash = %prepared.hash))] +#[tracing::instrument( + level = "debug", + skip(finalized_state, non_finalized_state, prepared), + fields( + height = ?prepared.height, + hash = %prepared.hash, + chains = non_finalized_state.chain_set.len() + ) +)] pub(crate) fn validate_and_commit_non_finalized( finalized_state: &ZebraDb, non_finalized_state: &mut NonFinalizedState, @@ -56,16 +67,28 @@ pub(crate) fn validate_and_commit_non_finalized( /// channels with the latest non-finalized [`ChainTipBlock`] and /// [`Chain`]. /// +/// `last_zebra_mined_log_height` is used to rate-limit logging. +/// /// Returns the latest non-finalized chain tip height. /// /// # Panics /// /// If the `non_finalized_state` is empty. -#[instrument(level = "debug", skip(chain_tip_sender, non_finalized_state_sender))] +#[instrument( + level = "debug", + skip( + non_finalized_state, + chain_tip_sender, + non_finalized_state_sender, + last_zebra_mined_log_height + ), + fields(chains = non_finalized_state.chain_set.len()) +)] fn update_latest_chain_channels( non_finalized_state: &NonFinalizedState, chain_tip_sender: &mut ChainTipSender, non_finalized_state_sender: &watch::Sender, + last_zebra_mined_log_height: &mut Option, ) -> block::Height { let best_chain = non_finalized_state.best_chain().expect("unexpected empty non-finalized state: must commit at least one block before updating channels"); @@ -75,6 +98,8 @@ fn update_latest_chain_channels( .clone(); let tip_block = ChainTipBlock::from(tip_block); + log_if_mined_by_zebra(&tip_block, last_zebra_mined_log_height); + let tip_block_height = tip_block.height; // If the final receiver was just dropped, ignore the error. @@ -90,13 +115,21 @@ fn update_latest_chain_channels( /// `non_finalized_state_sender`. // TODO: make the task an object #[allow(clippy::too_many_arguments)] -#[instrument(skip( - finalized_block_write_receiver, - non_finalized_block_write_receiver, - invalid_block_reset_sender, - chain_tip_sender, - non_finalized_state_sender, -))] +#[instrument( + level = "debug", + skip( + finalized_block_write_receiver, + non_finalized_block_write_receiver, + finalized_state, + non_finalized_state, + invalid_block_reset_sender, + chain_tip_sender, + non_finalized_state_sender, + ), + fields( + network = %non_finalized_state.network + ) +)] pub fn write_blocks_from_channels( mut finalized_block_write_receiver: UnboundedReceiver, mut non_finalized_block_write_receiver: UnboundedReceiver, @@ -106,6 +139,8 @@ pub fn write_blocks_from_channels( mut chain_tip_sender: ChainTipSender, non_finalized_state_sender: watch::Sender, ) { + let mut last_zebra_mined_log_height = None; + // Write all the finalized blocks sent by the state, // until the state closes the finalized block channel's sender. while let Some(ordered_block) = finalized_block_write_receiver.blocking_recv() { @@ -147,6 +182,8 @@ pub fn write_blocks_from_channels( Ok(finalized) => { let tip_block = ChainTipBlock::from(finalized); + log_if_mined_by_zebra(&tip_block, &mut last_zebra_mined_log_height); + chain_tip_sender.set_finalized_tip(tip_block); } Err(error) => { @@ -243,6 +280,7 @@ pub fn write_blocks_from_channels( &non_finalized_state, &mut chain_tip_sender, &non_finalized_state_sender, + &mut last_zebra_mined_log_height, ); // Update the caller with the result. @@ -285,3 +323,72 @@ pub fn write_blocks_from_channels( finalized_state.db.shutdown(true); std::mem::drop(finalized_state); } + +/// Log a message if this block was mined by Zebra. +/// +/// Does not detect early Zebra blocks, and blocks with custom coinbase transactions. +/// Rate-limited to every 1000 blocks using `last_zebra_mined_log_height`. +fn log_if_mined_by_zebra( + tip_block: &ChainTipBlock, + last_zebra_mined_log_height: &mut Option, +) { + // This logs at most every 2-3 checkpoints, which seems fine. + const LOG_RATE_LIMIT: u32 = 1000; + + let height = tip_block.height.0; + + if let Some(last_height) = last_zebra_mined_log_height { + if height < last_height.0 + LOG_RATE_LIMIT { + // If we logged in the last 1000 blocks, don't log anything now. + return; + } + }; + + // This code is rate-limited, so we can do expensive transformations here. + let coinbase_data = tip_block.transactions[0].inputs()[0] + .extra_coinbase_data() + .expect("valid blocks must start with a coinbase input") + .clone(); + + if coinbase_data + .as_ref() + .starts_with(EXTRA_ZEBRA_COINBASE_DATA.as_bytes()) + { + let text = String::from_utf8_lossy(coinbase_data.as_ref()); + + *last_zebra_mined_log_height = Some(Height(height)); + + // No need for hex-encoded data if it's exactly what we expected. + if coinbase_data.as_ref() == EXTRA_ZEBRA_COINBASE_DATA.as_bytes() { + info!( + %text, + %height, + hash = %tip_block.hash, + "looks like this block was mined by Zebra!" + ); + } else { + // # Security + // + // Use the extra data as an allow-list, replacing unknown characters. + // This makes sure control characters and harmful messages don't get logged + // to the terminal. + let text = text.replace( + |c: char| { + !EXTRA_ZEBRA_COINBASE_DATA + .to_ascii_lowercase() + .contains(c.to_ascii_lowercase()) + }, + "?", + ); + let data = hex::encode(coinbase_data.as_ref()); + + info!( + %text, + %data, + %height, + hash = %tip_block.hash, + "looks like this block was mined by Zebra!" + ); + } + } +} diff --git a/zebrad/tests/common/configs/getblocktemplate-v1.0.0-rc.5.toml b/zebrad/tests/common/configs/getblocktemplate-v1.0.0-rc.5.toml new file mode 100644 index 000000000..87ed3dad8 --- /dev/null +++ b/zebrad/tests/common/configs/getblocktemplate-v1.0.0-rc.5.toml @@ -0,0 +1,73 @@ +# Default configuration for zebrad. +# +# This file can be used as a skeleton for custom configs. +# +# Unspecified fields use default values. Optional fields are Some(field) if the +# field is present and None if it is absent. +# +# This file is generated as an example using zebrad's current defaults. +# You should set only the config options you want to keep, and delete the rest. +# Only a subset of fields are present in the skeleton, since optional values +# whose default is None are omitted. +# +# The config format (including a complete list of sections and fields) is +# documented here: +# https://doc.zebra.zfnd.org/zebrad/config/struct.ZebradConfig.html +# +# zebrad attempts to load configs in the following order: +# +# 1. The -c flag on the command line, e.g., `zebrad -c myconfig.toml start`; +# 2. The file `zebrad.toml` in the users's preference directory (platform-dependent); +# 3. The default config. + +[consensus] +checkpoint_sync = true +debug_skip_parameter_preload = false + +[mempool] +eviction_memory_time = "1h" +tx_cost_limit = 80000000 + +[metrics] + +[mining] +debug_like_zcashd = true + +[network] +crawl_new_peer_interval = "1m 1s" +initial_mainnet_peers = [ + "dnsseed.z.cash:8233", + "dnsseed.str4d.xyz:8233", + "mainnet.seeder.zfnd.org:8233", + "mainnet.is.yolo.money:8233", +] +initial_testnet_peers = [ + "dnsseed.testnet.z.cash:18233", + "testnet.seeder.zfnd.org:18233", + "testnet.is.yolo.money:18233", +] +listen_addr = "0.0.0.0:8233" +network = "Mainnet" +peerset_initial_target_size = 25 + +[rpc] +debug_force_finished_sync = false +parallel_cpu_threads = 0 + +[state] +cache_dir = "cache_dir" +delete_old_database = true +ephemeral = false + +[sync] +checkpoint_verify_concurrency_limit = 1000 +download_concurrency_limit = 50 +full_verify_concurrency_limit = 20 +parallel_cpu_threads = 0 + +[tracing] +buffer_limit = 128000 +force_use_color = false +use_color = true +use_journald = false +