2020-11-18 18:05:06 -08:00
|
|
|
//! The primary implementation of the `zebra_state::Service` built upon rocksdb
|
2020-09-09 23:07:47 -07:00
|
|
|
|
2020-11-16 16:05:59 -08:00
|
|
|
mod disk_format;
|
2020-11-16 16:05:35 -08:00
|
|
|
|
2021-06-01 00:53:13 -07:00
|
|
|
#[cfg(test)]
|
|
|
|
mod tests;
|
|
|
|
|
2021-06-18 07:23:01 -07:00
|
|
|
use std::{collections::HashMap, convert::TryInto, path::Path, sync::Arc};
|
2020-09-09 23:07:47 -07:00
|
|
|
|
2020-06-15 14:41:26 -07:00
|
|
|
use zebra_chain::{
|
2020-09-10 10:19:45 -07:00
|
|
|
block::{self, Block},
|
2020-10-12 14:08:23 -07:00
|
|
|
parameters::{Network, GENESIS_PREVIOUS_BLOCK_HASH},
|
2021-07-14 05:06:43 -07:00
|
|
|
sprout,
|
2020-11-01 10:49:34 -08:00
|
|
|
transaction::{self, Transaction},
|
2021-07-14 05:06:43 -07:00
|
|
|
transparent,
|
2020-06-15 14:41:26 -07:00
|
|
|
};
|
|
|
|
|
2021-07-11 19:49:33 -07:00
|
|
|
use crate::{BoxError, Config, FinalizedBlock, HashOrHeight};
|
2020-10-23 14:54:59 -07:00
|
|
|
|
2020-11-16 16:05:59 -08:00
|
|
|
use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, TransactionLocation};
|
2020-10-23 14:54:59 -07:00
|
|
|
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
use super::QueuedFinalized;
|
2020-10-30 15:24:39 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
/// The finalized part of the chain state, stored in the db.
|
2020-09-24 15:46:04 -07:00
|
|
|
pub struct FinalizedState {
|
2020-09-09 23:07:47 -07:00
|
|
|
/// Queued blocks that arrived out of order, indexed by their parent block hash.
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
queued_by_prev_hash: HashMap<block::Hash, QueuedFinalized>,
|
2021-01-27 23:29:57 -08:00
|
|
|
/// A metric tracking the maximum height that's currently in `queued_by_prev_hash`
|
|
|
|
///
|
|
|
|
/// Set to `f64::NAN` if `queued_by_prev_hash` is empty, because grafana shows NaNs
|
|
|
|
/// as a break in the graph.
|
|
|
|
max_queued_height: f64,
|
2020-09-09 23:07:47 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
db: rocksdb::DB,
|
|
|
|
ephemeral: bool,
|
2020-10-27 02:25:29 -07:00
|
|
|
/// Commit blocks to the finalized state up to this height, then exit Zebra.
|
|
|
|
debug_stop_at_height: Option<block::Height>,
|
2020-06-15 14:41:26 -07:00
|
|
|
}
|
|
|
|
|
2020-09-24 15:46:04 -07:00
|
|
|
impl FinalizedState {
|
2020-09-09 21:15:08 -07:00
|
|
|
pub fn new(config: &Config, network: Network) -> Self {
|
2020-12-01 14:00:05 -08:00
|
|
|
let (path, db_options) = config.db_config(network);
|
|
|
|
let column_families = vec![
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("hash_by_height", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("height_by_hash", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("block_by_height", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("sprout_nullifiers", db_options.clone()),
|
|
|
|
rocksdb::ColumnFamilyDescriptor::new("sapling_nullifiers", db_options.clone()),
|
2021-06-01 00:53:13 -07:00
|
|
|
rocksdb::ColumnFamilyDescriptor::new("orchard_nullifiers", db_options.clone()),
|
2020-12-01 14:00:05 -08:00
|
|
|
];
|
2021-01-29 04:36:33 -08:00
|
|
|
let db_result = rocksdb::DB::open_cf_descriptors(&db_options, &path, column_families);
|
|
|
|
|
|
|
|
let db = match db_result {
|
|
|
|
Ok(d) => {
|
|
|
|
tracing::info!("Opened Zebra state cache at {}", path.display());
|
|
|
|
d
|
|
|
|
}
|
|
|
|
// TODO: provide a different hint if the disk is full, see #1623
|
|
|
|
Err(e) => panic!(
|
|
|
|
"Opening database {:?} failed: {:?}. \
|
|
|
|
Hint: Check if another zebrad process is running. \
|
|
|
|
Try changing the state cache_dir in the Zebra config.",
|
|
|
|
path, e,
|
|
|
|
),
|
|
|
|
};
|
2020-06-15 14:41:26 -07:00
|
|
|
|
2020-10-27 02:25:29 -07:00
|
|
|
let new_state = Self {
|
2020-09-09 23:07:47 -07:00
|
|
|
queued_by_prev_hash: HashMap::new(),
|
2021-01-27 23:29:57 -08:00
|
|
|
max_queued_height: f64::NAN,
|
2020-11-18 18:05:06 -08:00
|
|
|
db,
|
|
|
|
ephemeral: config.ephemeral,
|
2020-10-27 02:25:29 -07:00
|
|
|
debug_stop_at_height: config.debug_stop_at_height.map(block::Height),
|
|
|
|
};
|
|
|
|
|
|
|
|
if let Some(tip_height) = new_state.finalized_tip_height() {
|
2020-11-12 17:37:52 -08:00
|
|
|
if new_state.is_at_stop_height(tip_height) {
|
|
|
|
let debug_stop_at_height = new_state
|
|
|
|
.debug_stop_at_height
|
|
|
|
.expect("true from `is_at_stop_height` implies `debug_stop_at_height` is Some");
|
|
|
|
|
|
|
|
let tip_hash = new_state.finalized_tip_hash();
|
|
|
|
|
|
|
|
if tip_height > debug_stop_at_height {
|
|
|
|
tracing::error!(
|
|
|
|
?debug_stop_at_height,
|
|
|
|
?tip_height,
|
|
|
|
?tip_hash,
|
|
|
|
"previous state height is greater than the stop height",
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
tracing::info!(
|
|
|
|
?debug_stop_at_height,
|
|
|
|
?tip_height,
|
|
|
|
?tip_hash,
|
|
|
|
"state is already at the configured height"
|
|
|
|
);
|
|
|
|
|
2020-11-24 21:04:18 -08:00
|
|
|
// RocksDB can do a cleanup when column families are opened.
|
|
|
|
// So we want to drop it before we exit.
|
|
|
|
std::mem::drop(new_state);
|
2020-11-12 17:37:52 -08:00
|
|
|
std::process::exit(0);
|
|
|
|
}
|
2020-10-27 02:25:29 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
new_state
|
|
|
|
}
|
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
/// Stop the process if `block_height` is greater than or equal to the
|
|
|
|
/// configured stop height.
|
2020-11-12 17:37:52 -08:00
|
|
|
fn is_at_stop_height(&self, block_height: block::Height) -> bool {
|
2020-10-27 02:25:29 -07:00
|
|
|
let debug_stop_at_height = match self.debug_stop_at_height {
|
|
|
|
Some(debug_stop_at_height) => debug_stop_at_height,
|
2020-11-12 17:37:52 -08:00
|
|
|
None => return false,
|
2020-10-27 02:25:29 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
if block_height < debug_stop_at_height {
|
2020-11-12 17:37:52 -08:00
|
|
|
return false;
|
2020-10-27 02:25:29 -07:00
|
|
|
}
|
|
|
|
|
2020-11-12 17:37:52 -08:00
|
|
|
true
|
2020-06-15 14:41:26 -07:00
|
|
|
}
|
|
|
|
|
2020-09-09 23:07:47 -07:00
|
|
|
/// Queue a finalized block to be committed to the state.
|
2020-09-10 14:16:39 -07:00
|
|
|
///
|
2020-09-10 14:42:20 -07:00
|
|
|
/// After queueing a finalized block, this method checks whether the newly
|
|
|
|
/// queued block (and any of its descendants) can be committed to the state.
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
pub fn queue_and_commit_finalized(&mut self, queued: QueuedFinalized) {
|
|
|
|
let prev_hash = queued.0.block.header.previous_block_hash;
|
|
|
|
let height = queued.0.height;
|
|
|
|
self.queued_by_prev_hash.insert(prev_hash, queued);
|
2020-09-09 23:07:47 -07:00
|
|
|
|
2020-11-16 18:25:35 -08:00
|
|
|
while let Some(queued_block) = self.queued_by_prev_hash.remove(&self.finalized_tip_hash()) {
|
2020-10-09 18:49:44 -07:00
|
|
|
self.commit_finalized(queued_block);
|
2020-11-12 15:49:55 -08:00
|
|
|
metrics::counter!("state.finalized.committed.block.count", 1);
|
|
|
|
metrics::gauge!("state.finalized.committed.block.height", height.0 as _);
|
2020-09-09 23:07:47 -07:00
|
|
|
}
|
2020-10-09 18:49:44 -07:00
|
|
|
|
2020-11-12 15:49:55 -08:00
|
|
|
if self.queued_by_prev_hash.is_empty() {
|
2021-01-27 23:29:57 -08:00
|
|
|
self.max_queued_height = f64::NAN;
|
|
|
|
} else if self.max_queued_height.is_nan() || self.max_queued_height < height.0 as _ {
|
|
|
|
// if there are still blocks in the queue, then either:
|
|
|
|
// - the new block was lower than the old maximum, and there was a gap before it,
|
|
|
|
// so the maximum is still the same (and we skip this code), or
|
|
|
|
// - the new block is higher than the old maximum, and there is at least one gap
|
|
|
|
// between the finalized tip and the new maximum
|
|
|
|
self.max_queued_height = height.0 as _;
|
2020-11-12 15:49:55 -08:00
|
|
|
}
|
|
|
|
|
2021-01-27 23:29:57 -08:00
|
|
|
metrics::gauge!("state.finalized.queued.max.height", self.max_queued_height);
|
2020-10-09 18:49:44 -07:00
|
|
|
metrics::gauge!(
|
2020-11-12 15:49:55 -08:00
|
|
|
"state.finalized.queued.block.count",
|
2021-01-11 18:28:56 -08:00
|
|
|
self.queued_by_prev_hash.len() as f64
|
2020-10-09 18:49:44 -07:00
|
|
|
);
|
2020-09-09 23:07:47 -07:00
|
|
|
}
|
|
|
|
|
2020-10-07 20:07:32 -07:00
|
|
|
/// Returns the hash of the current finalized tip block.
|
|
|
|
pub fn finalized_tip_hash(&self) -> block::Hash {
|
2020-10-24 17:09:50 -07:00
|
|
|
self.tip()
|
2020-10-07 20:07:32 -07:00
|
|
|
.map(|(_, hash)| hash)
|
|
|
|
// if the state is empty, return the genesis previous block hash
|
2020-10-12 14:08:23 -07:00
|
|
|
.unwrap_or(GENESIS_PREVIOUS_BLOCK_HASH)
|
2020-10-07 20:07:32 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns the height of the current finalized tip block.
|
|
|
|
pub fn finalized_tip_height(&self) -> Option<block::Height> {
|
2020-11-01 10:49:34 -08:00
|
|
|
self.tip().map(|(height, _)| height)
|
2020-10-07 20:07:32 -07:00
|
|
|
}
|
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
fn is_empty(&self, cf: &rocksdb::ColumnFamily) -> bool {
|
|
|
|
// use iterator to check if it's empty
|
|
|
|
!self
|
|
|
|
.db
|
|
|
|
.iterator_cf(cf, rocksdb::IteratorMode::Start)
|
|
|
|
.valid()
|
|
|
|
}
|
|
|
|
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
/// Immediately commit `finalized` to the finalized state.
|
2021-07-09 05:47:09 -07:00
|
|
|
///
|
|
|
|
/// Use `source` as the source of the block in log messages.
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
pub fn commit_finalized_direct(
|
|
|
|
&mut self,
|
|
|
|
finalized: FinalizedBlock,
|
2021-07-09 05:47:09 -07:00
|
|
|
source: &str,
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
) -> Result<block::Hash, BoxError> {
|
|
|
|
block_precommit_metrics(&finalized);
|
2020-06-15 14:41:26 -07:00
|
|
|
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
let FinalizedBlock {
|
|
|
|
block,
|
|
|
|
hash,
|
|
|
|
height,
|
2020-11-23 12:02:57 -08:00
|
|
|
new_outputs,
|
2020-11-24 19:55:15 -08:00
|
|
|
transaction_hashes,
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
} = finalized;
|
2020-11-16 18:25:35 -08:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
|
|
|
|
let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
|
|
|
|
let block_by_height = self.db.cf_handle("block_by_height").unwrap();
|
|
|
|
let tx_by_hash = self.db.cf_handle("tx_by_hash").unwrap();
|
|
|
|
let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
|
|
|
|
let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap();
|
|
|
|
let sapling_nullifiers = self.db.cf_handle("sapling_nullifiers").unwrap();
|
2021-06-01 00:53:13 -07:00
|
|
|
let orchard_nullifiers = self.db.cf_handle("orchard_nullifiers").unwrap();
|
2020-11-18 18:05:06 -08:00
|
|
|
|
2020-11-16 18:25:35 -08:00
|
|
|
// Assert that callers (including unit tests) get the chain order correct
|
2020-11-18 18:05:06 -08:00
|
|
|
if self.is_empty(hash_by_height) {
|
2020-11-16 18:25:35 -08:00
|
|
|
assert_eq!(
|
2021-05-27 08:41:20 -07:00
|
|
|
GENESIS_PREVIOUS_BLOCK_HASH, block.header.previous_block_hash,
|
2021-07-09 05:47:09 -07:00
|
|
|
"the first block added to an empty state must be a genesis block, source: {}",
|
|
|
|
source,
|
2020-11-16 18:25:35 -08:00
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
block::Height(0),
|
|
|
|
height,
|
2021-07-09 05:47:09 -07:00
|
|
|
"cannot commit genesis: invalid height, source: {}",
|
|
|
|
source,
|
2020-11-16 18:25:35 -08:00
|
|
|
);
|
|
|
|
} else {
|
|
|
|
assert_eq!(
|
|
|
|
self.finalized_tip_height()
|
|
|
|
.expect("state must have a genesis block committed")
|
|
|
|
+ 1,
|
|
|
|
Some(height),
|
2021-07-09 05:47:09 -07:00
|
|
|
"committed block height must be 1 more than the finalized tip height, source: {}",
|
|
|
|
source,
|
2020-11-16 18:25:35 -08:00
|
|
|
);
|
|
|
|
|
|
|
|
assert_eq!(
|
|
|
|
self.finalized_tip_hash(),
|
|
|
|
block.header.previous_block_hash,
|
2021-07-09 05:47:09 -07:00
|
|
|
"committed block must be a child of the finalized tip, source: {}",
|
|
|
|
source,
|
2020-11-16 18:25:35 -08:00
|
|
|
);
|
|
|
|
}
|
2020-10-07 20:07:32 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
// We use a closure so we can use an early return for control flow in
|
|
|
|
// the genesis case
|
|
|
|
let prepare_commit = || -> rocksdb::WriteBatch {
|
|
|
|
let mut batch = rocksdb::WriteBatch::default();
|
2020-10-27 14:55:46 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
// Index the block
|
|
|
|
batch.zs_insert(hash_by_height, height, hash);
|
|
|
|
batch.zs_insert(height_by_hash, hash, height);
|
|
|
|
batch.zs_insert(block_by_height, height, &block);
|
2020-10-27 14:55:04 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
// TODO: sprout and sapling anchors (per block)
|
2020-10-23 15:29:52 -07:00
|
|
|
|
2021-07-12 13:11:33 -07:00
|
|
|
// "A transaction MUST NOT spend an output of the genesis block coinbase transaction.
|
|
|
|
// (There is one such zero-valued output, on each of Testnet and Mainnet .)"
|
|
|
|
// https://zips.z.cash/protocol/protocol.pdf#txnconsensus
|
|
|
|
if block.header.previous_block_hash == GENESIS_PREVIOUS_BLOCK_HASH {
|
2020-11-18 18:05:06 -08:00
|
|
|
return batch;
|
|
|
|
}
|
|
|
|
|
2020-11-23 12:02:57 -08:00
|
|
|
// Index all new transparent outputs
|
|
|
|
for (outpoint, utxo) in new_outputs.into_iter() {
|
|
|
|
batch.zs_insert(utxo_by_outpoint, outpoint, utxo);
|
|
|
|
}
|
|
|
|
|
|
|
|
// Index each transaction, spent inputs, nullifiers
|
2020-11-24 19:55:15 -08:00
|
|
|
for (transaction_index, (transaction, transaction_hash)) in block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.zip(transaction_hashes.into_iter())
|
|
|
|
.enumerate()
|
|
|
|
{
|
2020-11-18 18:05:06 -08:00
|
|
|
let transaction_location = TransactionLocation {
|
|
|
|
height,
|
|
|
|
index: transaction_index
|
|
|
|
.try_into()
|
|
|
|
.expect("no more than 4 billion transactions per block"),
|
|
|
|
};
|
|
|
|
batch.zs_insert(tx_by_hash, transaction_hash, transaction_location);
|
|
|
|
|
|
|
|
// Mark all transparent inputs as spent
|
|
|
|
for input in transaction.inputs() {
|
|
|
|
match input {
|
|
|
|
transparent::Input::PrevOut { outpoint, .. } => {
|
|
|
|
batch.delete_cf(utxo_by_outpoint, outpoint.as_bytes());
|
2020-10-23 15:29:52 -07:00
|
|
|
}
|
2020-11-18 18:05:06 -08:00
|
|
|
// Coinbase inputs represent new coins,
|
|
|
|
// so there are no UTXOs to mark as spent.
|
|
|
|
transparent::Input::Coinbase { .. } => {}
|
2020-10-14 14:06:32 -07:00
|
|
|
}
|
2020-11-18 18:05:06 -08:00
|
|
|
}
|
2020-10-14 14:06:32 -07:00
|
|
|
|
2021-06-01 00:53:13 -07:00
|
|
|
// Mark sprout, sapling and orchard nullifiers as spent
|
2020-11-18 18:05:06 -08:00
|
|
|
for sprout_nullifier in transaction.sprout_nullifiers() {
|
|
|
|
batch.zs_insert(sprout_nullifiers, sprout_nullifier, ());
|
|
|
|
}
|
|
|
|
for sapling_nullifier in transaction.sapling_nullifiers() {
|
|
|
|
batch.zs_insert(sapling_nullifiers, sapling_nullifier, ());
|
|
|
|
}
|
2021-06-01 00:53:13 -07:00
|
|
|
for orchard_nullifier in transaction.orchard_nullifiers() {
|
|
|
|
batch.zs_insert(orchard_nullifiers, orchard_nullifier, ());
|
|
|
|
}
|
2020-11-12 17:37:52 -08:00
|
|
|
}
|
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
batch
|
|
|
|
};
|
|
|
|
|
|
|
|
let batch = prepare_commit();
|
|
|
|
|
|
|
|
let result = self.db.write(batch).map(|()| hash);
|
|
|
|
|
2021-07-09 05:47:09 -07:00
|
|
|
tracing::trace!(?source, "committed block from");
|
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
if result.is_ok() && self.is_at_stop_height(height) {
|
2021-07-09 05:47:09 -07:00
|
|
|
tracing::info!(?source, "committed block from");
|
2020-11-12 17:37:52 -08:00
|
|
|
tracing::info!(?height, ?hash, "stopping at configured height");
|
2020-11-24 21:04:18 -08:00
|
|
|
// We'd like to drop the database here, because that closes the
|
|
|
|
// column families and the database. But Rust's ownership rules
|
|
|
|
// make that difficult, so we just flush instead.
|
|
|
|
self.db.flush().expect("flush is successful");
|
|
|
|
self.delete_ephemeral();
|
2020-11-12 17:37:52 -08:00
|
|
|
std::process::exit(0);
|
2020-10-27 02:25:29 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
result.map_err(Into::into)
|
2020-10-07 20:07:32 -07:00
|
|
|
}
|
2020-09-01 18:20:32 -07:00
|
|
|
|
2020-10-07 20:07:32 -07:00
|
|
|
/// Commit a finalized block to the state.
|
|
|
|
///
|
|
|
|
/// It's the caller's responsibility to ensure that blocks are committed in
|
|
|
|
/// order. This function is called by [`queue`], which ensures order.
|
|
|
|
/// It is intentionally not exposed as part of the public API of the
|
|
|
|
/// [`FinalizedState`].
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
fn commit_finalized(&mut self, queued_block: QueuedFinalized) {
|
|
|
|
let (finalized, rsp_tx) = queued_block;
|
2021-07-09 05:47:09 -07:00
|
|
|
let result = self.commit_finalized_direct(finalized, "CommitFinalized request");
|
2020-10-09 01:37:24 -07:00
|
|
|
let _ = rsp_tx.send(result.map_err(Into::into));
|
2020-06-15 14:41:26 -07:00
|
|
|
}
|
|
|
|
|
2020-11-01 10:49:34 -08:00
|
|
|
/// Returns the tip height and hash if there is one.
|
|
|
|
pub fn tip(&self) -> Option<(block::Height, block::Hash)> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
|
|
|
|
self.db
|
|
|
|
.iterator_cf(hash_by_height, rocksdb::IteratorMode::End)
|
2020-11-01 10:49:34 -08:00
|
|
|
.next()
|
|
|
|
.map(|(height_bytes, hash_bytes)| {
|
2020-11-18 18:05:06 -08:00
|
|
|
let height = block::Height::from_bytes(height_bytes);
|
|
|
|
let hash = block::Hash::from_bytes(hash_bytes);
|
2020-10-23 14:54:59 -07:00
|
|
|
|
2020-11-01 10:49:34 -08:00
|
|
|
(height, hash)
|
|
|
|
})
|
2020-06-15 14:41:26 -07:00
|
|
|
}
|
|
|
|
|
2020-11-01 10:49:34 -08:00
|
|
|
/// Returns the height of the given block if it exists.
|
|
|
|
pub fn height(&self, hash: block::Hash) -> Option<block::Height> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
|
2021-06-06 20:26:34 -07:00
|
|
|
self.db.zs_get(height_by_hash, &hash)
|
2020-07-30 16:21:54 -07:00
|
|
|
}
|
|
|
|
|
2020-11-01 10:49:34 -08:00
|
|
|
/// Returns the given block if it exists.
|
|
|
|
pub fn block(&self, hash_or_height: HashOrHeight) -> Option<Arc<Block>> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let height_by_hash = self.db.cf_handle("height_by_hash").unwrap();
|
|
|
|
let block_by_height = self.db.cf_handle("block_by_height").unwrap();
|
|
|
|
let height = hash_or_height.height_or_else(|hash| self.db.zs_get(height_by_hash, &hash))?;
|
2020-06-15 14:41:26 -07:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
self.db.zs_get(block_by_height, &height)
|
2020-09-10 10:19:45 -07:00
|
|
|
}
|
2020-10-14 14:06:32 -07:00
|
|
|
|
|
|
|
/// Returns the `transparent::Output` pointed to by the given
|
|
|
|
/// `transparent::OutPoint` if it is present.
|
2021-07-11 19:49:33 -07:00
|
|
|
pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Utxo> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
|
|
|
|
self.db.zs_get(utxo_by_outpoint, outpoint)
|
2020-10-14 14:06:32 -07:00
|
|
|
}
|
2020-10-26 13:54:19 -07:00
|
|
|
|
2021-07-14 05:06:43 -07:00
|
|
|
/// Returns `true` if the finalized state contains `sprout_nullifier`.
|
|
|
|
pub fn contains_sprout_nullifier(&self, sprout_nullifier: &sprout::Nullifier) -> bool {
|
|
|
|
let sprout_nullifiers = self.db.cf_handle("sprout_nullifiers").unwrap();
|
|
|
|
self.db.zs_contains(sprout_nullifiers, &sprout_nullifier)
|
|
|
|
}
|
|
|
|
|
2020-10-26 13:54:19 -07:00
|
|
|
/// Returns the finalized hash for a given `block::Height` if it is present.
|
2020-11-01 10:49:34 -08:00
|
|
|
pub fn hash(&self, height: block::Height) -> Option<block::Hash> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
|
|
|
|
self.db.zs_get(hash_by_height, &height)
|
2020-11-01 10:49:34 -08:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Returns the given transaction if it exists.
|
|
|
|
pub fn transaction(&self, hash: transaction::Hash) -> Option<Arc<Transaction>> {
|
2020-11-18 18:05:06 -08:00
|
|
|
let tx_by_hash = self.db.cf_handle("tx_by_hash").unwrap();
|
|
|
|
self.db
|
|
|
|
.zs_get(tx_by_hash, &hash)
|
2020-11-01 10:49:34 -08:00
|
|
|
.map(|TransactionLocation { index, height }| {
|
|
|
|
let block = self
|
|
|
|
.block(height.into())
|
|
|
|
.expect("block will exist if TransactionLocation does");
|
|
|
|
|
|
|
|
block.transactions[index as usize].clone()
|
|
|
|
})
|
2020-10-26 13:54:19 -07:00
|
|
|
}
|
2020-11-24 21:04:18 -08:00
|
|
|
|
|
|
|
/// If the database is `ephemeral`, delete it.
|
|
|
|
fn delete_ephemeral(&self) {
|
|
|
|
if self.ephemeral {
|
|
|
|
let path = self.db.path();
|
|
|
|
tracing::debug!("removing temporary database files {:?}", path);
|
|
|
|
// We'd like to use `rocksdb::Env::mem_env` for ephemeral databases,
|
|
|
|
// but the Zcash blockchain might not fit in memory. So we just
|
|
|
|
// delete the database files instead.
|
|
|
|
//
|
|
|
|
// We'd like to call `DB::destroy` here, but calling destroy on a
|
|
|
|
// live DB is undefined behaviour:
|
|
|
|
// https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ#basic-readwrite
|
|
|
|
//
|
|
|
|
// So we assume that all the database files are under `path`, and
|
|
|
|
// delete them using standard filesystem APIs. Deleting open files
|
|
|
|
// might cause errors on non-Unix platforms, so we ignore the result.
|
|
|
|
// (The OS will delete them eventually anyway.)
|
|
|
|
let _res = std::fs::remove_dir_all(path);
|
|
|
|
}
|
|
|
|
}
|
2021-06-18 07:23:01 -07:00
|
|
|
|
|
|
|
/// Returns the `Path` where the files used by this database are located.
|
2021-06-21 08:01:45 -07:00
|
|
|
#[allow(dead_code)]
|
2021-06-18 07:23:01 -07:00
|
|
|
pub fn path(&self) -> &Path {
|
|
|
|
self.db.path()
|
|
|
|
}
|
2020-07-30 16:21:54 -07:00
|
|
|
}
|
2020-11-12 20:34:49 -08:00
|
|
|
|
2020-11-18 18:05:06 -08:00
|
|
|
// Drop isn't guaranteed to run, such as when we panic, or if someone stored
|
|
|
|
// their FinalizedState in a static, but it should be fine if we don't clean
|
|
|
|
// this up since the files are placed in the os temp dir and should be cleaned
|
|
|
|
// up automatically eventually.
|
|
|
|
impl Drop for FinalizedState {
|
|
|
|
fn drop(&mut self) {
|
2020-11-24 21:04:18 -08:00
|
|
|
self.delete_ephemeral()
|
2020-11-18 18:05:06 -08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
state: introduce PreparedBlock, FinalizedBlock
This change introduces two new types:
- `PreparedBlock`, representing a block which has undergone semantic
validation and has been prepared for contextual validation;
- `FinalizedBlock`, representing a block which is ready to be finalized
immediately;
and changes the `Request::CommitBlock`,`Request::CommitFinalizedBlock`
variants to use these types instead of their previous fields.
This change solves the problem of passing data between semantic
validation and contextual validation, and cleans up the state code by
allowing it to pass around a bundle of data. Previously, the state code
just passed around an `Arc<Block>`, which forced it to needlessly
recompute block hashes and other data, and was incompatible with the
already-known but not-yet-implemented data transfer requirements, namely
passing in the Sprout and Sapling anchors computed during contextual
validation.
This commit propagates the `PreparedBlock` and `FinalizedBlock` types
through the state code but only uses their data opportunistically, e.g.,
changing .hash() computations to use the precomputed hash. In the
future, these structures can be extended to pass data through the
verification pipeline for reuse as appropriate. For instance, these
changes allow the sprout and sapling anchors to be propagated through
the state.
2020-11-21 01:16:14 -08:00
|
|
|
fn block_precommit_metrics(finalized: &FinalizedBlock) {
|
|
|
|
let (hash, height, block) = (finalized.hash, finalized.height, finalized.block.as_ref());
|
|
|
|
|
2020-11-12 20:34:49 -08:00
|
|
|
let transaction_count = block.transactions.len();
|
|
|
|
let transparent_prevout_count = block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|t| t.inputs().iter())
|
|
|
|
.count()
|
|
|
|
// Each block has a single coinbase input which is not a previous output.
|
|
|
|
- 1;
|
|
|
|
let transparent_newout_count = block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|t| t.outputs().iter())
|
|
|
|
.count();
|
|
|
|
|
|
|
|
let sprout_nullifier_count = block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|t| t.sprout_nullifiers())
|
|
|
|
.count();
|
|
|
|
|
|
|
|
let sapling_nullifier_count = block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|t| t.sapling_nullifiers())
|
|
|
|
.count();
|
|
|
|
|
2021-06-01 00:53:13 -07:00
|
|
|
let orchard_nullifier_count = block
|
|
|
|
.transactions
|
|
|
|
.iter()
|
|
|
|
.flat_map(|t| t.orchard_nullifiers())
|
|
|
|
.count();
|
|
|
|
|
2020-11-12 20:34:49 -08:00
|
|
|
tracing::debug!(
|
|
|
|
?hash,
|
|
|
|
?height,
|
|
|
|
transaction_count,
|
|
|
|
transparent_prevout_count,
|
|
|
|
transparent_newout_count,
|
|
|
|
sprout_nullifier_count,
|
|
|
|
sapling_nullifier_count,
|
2021-06-01 00:53:13 -07:00
|
|
|
orchard_nullifier_count,
|
2020-11-12 20:34:49 -08:00
|
|
|
"preparing to commit finalized block"
|
|
|
|
);
|
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.transactions",
|
|
|
|
transaction_count as u64
|
|
|
|
);
|
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.transparent_prevouts",
|
|
|
|
transparent_prevout_count as u64
|
|
|
|
);
|
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.transparent_newouts",
|
|
|
|
transparent_newout_count as u64
|
|
|
|
);
|
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.sprout_nullifiers",
|
|
|
|
sprout_nullifier_count as u64
|
|
|
|
);
|
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.sapling_nullifiers",
|
|
|
|
sapling_nullifier_count as u64
|
|
|
|
);
|
2021-06-01 00:53:13 -07:00
|
|
|
metrics::counter!(
|
|
|
|
"state.finalized.cumulative.orchard_nullifiers",
|
|
|
|
orchard_nullifier_count as u64
|
|
|
|
);
|
2020-11-12 20:34:49 -08:00
|
|
|
}
|