zecfoundation
/
zebra
mirror of https://github.com/ZcashFoundation/zebra.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

2. fix(state): index spending transaction IDs for each address (#4355) 

* Make jobs that use cached state wait for state rebuilds

* Run jobs that need cached state even if the rebuild was skipped

* Fix missing dependencies

And update a TODO

* Split writing transaction indexes into transparent and shielded

* Split writing transparent indexes into created and spent

* Correctly populate spending address transaction ID indexes

* Increment the database format to rebuild address tx ID indexes

* Update non-finalized docs to prevent similar bugs

* Fix a comment

* Make jobs that use cached state wait for state rebuilds

* Run jobs that need cached state even if the rebuild was skipped

* Fix missing dependencies

And update a TODO

* refactor(ci): look for available disks instead of files changed

This ensure that if the constants.rs file was changed, we search for disks available in the whole repository with the same state.

If there's no disk available a rebuild is triggered depending the missing disk. And if there's a disk available, tests are run with this one.

* fix(ci): lwd syncs needs to wait for zebra disk rebuild

* docs(ci): use better comments on integration tests

* fix(ci): we must authenticate to GCP to find disks

* fix(ci): add needed permissions for google auth

* fix(ci): the output needs to be echoed

* imp(ci): reduce diff with main

* fix(ci): remove redundant dependency

Co-authored-by: teor <teor@riseup.net>

* fix(ci): also add `false` to the JSON object output

* fix(ci): hasty copy/paste

* force a push event

* fix(ci): standardize comments

* fix(ci): run disk rebuilds if no disk was found

* fix(ci): do not restrict on push

* fix(ci): build on any event if a cached disk is not found

* fix(ci): sync .patch file with changes on the workflow

Co-authored-by: Gustavo Valverde <gustavo@iterativo.do>
This commit is contained in:
teor 2022-05-20 12:22:01 +10:00 committed by GitHub
parent 712ef40438
commit 2439bed3d2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 184 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
zebra-state/src/constants.rs
View File

@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
/// The database format version, incremented each time the database format changes.
pub const DATABASE_FORMAT_VERSION: u32 = 23;
pub const DATABASE_FORMAT_VERSION: u32 = 24;
/// The maximum number of blocks to check for NU5 transactions,
/// before we assume we are on a pre-NU5 legacy chain.

48
zebra-state/src/service/finalized_state/zebra_db/block.rs
View File

@ -379,14 +379,15 @@ impl DiskWriteBatch {
}
// Commit transaction indexes
self.prepare_transaction_index_batch(
self.prepare_transparent_transaction_batch(
db,
&finalized,
new_outputs_by_out_loc,
spent_utxos_by_out_loc,
&new_outputs_by_out_loc,
&spent_utxos_by_outpoint,
&spent_utxos_by_out_loc,
address_balances,
&mut note_commitment_trees,
)?;
self.prepare_shielded_transaction_batch(db, &finalized, &mut note_commitment_trees)?;
self.prepare_note_commitment_batch(db, &finalized, note_commitment_trees, history_tree)?;
@ -475,43 +476,4 @@ impl DiskWriteBatch {
false
}
// Write transaction methods
/// Prepare a database batch containing `finalized.block`'s transaction indexes,
/// and return it (without actually writing anything).
///
/// If this method returns an error, it will be propagated,
/// and the batch should not be written to the database.
///
/// # Errors
///
/// - Propagates any errors from updating note commitment trees
//
// TODO: move db, finalized, and maybe other arguments into DiskWriteBatch
pub fn prepare_transaction_index_batch(
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
utxos_spent_by_block: BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
note_commitment_trees: &mut NoteCommitmentTrees,
) -> Result<(), BoxError> {
let FinalizedBlock { block, .. } = finalized;
// Index each transaction's transparent and shielded data
for transaction in block.transactions.iter() {
self.prepare_nullifier_batch(db, transaction)?;
DiskWriteBatch::update_note_commitment_trees(transaction, note_commitment_trees)?;
}
self.prepare_transparent_outputs_batch(
db,
new_outputs_by_out_loc,
utxos_spent_by_block,
address_balances,
)
}
}

27
zebra-state/src/service/finalized_state/zebra_db/shielded.rs
View File

@ -166,6 +166,33 @@ impl ZebraDb {
}
impl DiskWriteBatch {
/// Prepare a database batch containing `finalized.block`'s shielded transaction indexes,
/// and return it (without actually writing anything).
///
/// If this method returns an error, it will be propagated,
/// and the batch should not be written to the database.
///
/// # Errors
///
/// - Propagates any errors from updating note commitment trees
pub fn prepare_shielded_transaction_batch(
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
note_commitment_trees: &mut NoteCommitmentTrees,
) -> Result<(), BoxError> {
let FinalizedBlock { block, .. } = finalized;
// Index each transaction's shielded data
for transaction in &block.transactions {
self.prepare_nullifier_batch(db, transaction)?;
DiskWriteBatch::update_note_commitment_trees(transaction, note_commitment_trees)?;
}
Ok(())
}
/// Prepare a database batch containing `finalized.block`'s nullifiers,
/// and return it (without actually writing anything).
///

175
zebra-state/src/service/finalized_state/zebra_db/transparent.rs
View File

@ -19,7 +19,8 @@ use std::{
use zebra_chain::{
amount::{self, Amount, NonNegative},
block::Height,
transaction, transparent,
transaction::{self, Transaction},
transparent::{self, Input},
};
use crate::{
@ -34,7 +35,7 @@ use crate::{
},
zebra_db::ZebraDb,
},
BoxError,
BoxError, FinalizedBlock,
};
impl ZebraDb {
@ -353,21 +354,72 @@ impl ZebraDb {
}
impl DiskWriteBatch {
/// Prepare a database batch containing `finalized.block`'s:
/// - transparent address balance changes,
/// - UTXO changes, and
/// - transparent address index changes,
/// Prepare a database batch containing `finalized.block`'s transparent transaction indexes,
/// and return it (without actually writing anything).
///
/// If this method returns an error, it will be propagated,
/// and the batch should not be written to the database.
///
/// # Errors
///
/// - Propagates any errors from updating note commitment trees
pub fn prepare_transparent_transaction_batch(
&mut self,
db: &DiskDb,
finalized: &FinalizedBlock,
new_outputs_by_out_loc: &BTreeMap<OutputLocation, transparent::Utxo>,
spent_utxos_by_outpoint: &HashMap<transparent::OutPoint, transparent::Utxo>,
spent_utxos_by_out_loc: &BTreeMap<OutputLocation, transparent::Utxo>,
mut address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> {
let FinalizedBlock { block, height, .. } = finalized;
// Update created and spent transparent outputs
self.prepare_new_transparent_outputs_batch(
db,
new_outputs_by_out_loc,
&mut address_balances,
)?;
self.prepare_spent_transparent_outputs_batch(
db,
spent_utxos_by_out_loc,
&mut address_balances,
)?;
// Index the transparent addresses that spent in each transaction
for (tx_index, transaction) in block.transactions.iter().enumerate() {
let spending_tx_location = TransactionLocation::from_usize(*height, tx_index);
self.prepare_spending_transparent_tx_ids_batch(
db,
spending_tx_location,
transaction,
spent_utxos_by_outpoint,
&address_balances,
)?;
}
self.prepare_transparent_balances_batch(db, address_balances)
}
/// Prepare a database batch for the new UTXOs in `new_outputs_by_out_loc`.
///
/// Adds the following changes to this batch:
/// - insert created UTXOs,
/// - insert transparent address UTXO index entries, and
/// - insert transparent address transaction entries,
/// without actually writing anything.
///
/// Also modifies the `address_balances` for these new UTXOs.
///
/// # Errors
///
/// - This method doesn't currently return any errors, but it might in future
pub fn prepare_transparent_outputs_batch(
pub fn prepare_new_transparent_outputs_batch(
&mut self,
db: &DiskDb,
new_outputs_by_out_loc: BTreeMap<OutputLocation, transparent::Utxo>,
utxos_spent_by_block: BTreeMap<OutputLocation, transparent::Utxo>,
mut address_balances: HashMap<transparent::Address, AddressBalanceLocation>,
new_outputs_by_out_loc: &BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: &mut HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> {
let utxo_by_out_loc = db.cf_handle("utxo_by_outpoint").unwrap();
let utxo_loc_by_transparent_addr_loc =
@ -375,9 +427,9 @@ impl DiskWriteBatch {
let tx_loc_by_transparent_addr_loc =
db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap();
// Index all new transparent outputs, before deleting any we've spent
// Index all new transparent outputs
for (new_output_location, utxo) in new_outputs_by_out_loc {
let unspent_output = utxo.output;
let unspent_output = &utxo.output;
let receiving_address = unspent_output.address(self.network());
// Update the address balance by adding this UTXO's value
@ -391,17 +443,17 @@ impl DiskWriteBatch {
// (the first location of the address in the chain).
let address_balance_location = address_balances
.entry(receiving_address)
.or_insert_with(|| AddressBalanceLocation::new(new_output_location));
.or_insert_with(|| AddressBalanceLocation::new(*new_output_location));
let receiving_address_location = address_balance_location.address_location();
// Update the balance for the address in memory.
address_balance_location
.receive_output(&unspent_output)
.receive_output(unspent_output)
.expect("balance overflow already checked");
// Create a link from the AddressLocation to the new OutputLocation in the database.
let address_unspent_output =
AddressUnspentOutput::new(receiving_address_location, new_output_location);
AddressUnspentOutput::new(receiving_address_location, *new_output_location);
self.zs_insert(
&utxo_loc_by_transparent_addr_loc,
address_unspent_output,
@ -423,11 +475,36 @@ impl DiskWriteBatch {
self.zs_insert(&utxo_by_out_loc, new_output_location, unspent_output);
}
Ok(())
}
/// Prepare a database batch for the spent outputs in `spent_utxos_by_out_loc`.
///
/// Adds the following changes to this batch:
/// - delete spent UTXOs, and
/// - delete transparent address UTXO index entries,
/// without actually writing anything.
///
/// Also modifies the `address_balances` for these new UTXOs.
///
/// # Errors
///
/// - This method doesn't currently return any errors, but it might in future
pub fn prepare_spent_transparent_outputs_batch(
&mut self,
db: &DiskDb,
spent_utxos_by_out_loc: &BTreeMap<OutputLocation, transparent::Utxo>,
address_balances: &mut HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> {
let utxo_by_out_loc = db.cf_handle("utxo_by_outpoint").unwrap();
let utxo_loc_by_transparent_addr_loc =
db.cf_handle("utxo_loc_by_transparent_addr_loc").unwrap();
// Mark all transparent inputs as spent.
//
// Coinbase inputs represent new coins, so there are no UTXOs to mark as spent.
for (spent_output_location, utxo) in utxos_spent_by_block {
let spent_output = utxo.output;
for (spent_output_location, utxo) in spent_utxos_by_out_loc {
let spent_output = &utxo.output;
let sending_address = spent_output.address(self.network());
// Fetch the balance, and the link from the address to the AddressLocation, from memory.
@ -438,31 +515,71 @@ impl DiskWriteBatch {
// Update the address balance by subtracting this UTXO's value, in memory.
address_balance_location
.spend_output(&spent_output)
.spend_output(spent_output)
.expect("balance underflow already checked");
let sending_address_location = address_balance_location.address_location();
// Delete the link from the AddressLocation to the spent OutputLocation in the database.
let address_spent_output = AddressUnspentOutput::new(
address_balance_location.address_location(),
spent_output_location,
*spent_output_location,
);
self.zs_delete(&utxo_loc_by_transparent_addr_loc, address_spent_output);
// Create a link from the AddressLocation to the spent TransactionLocation in the database.
// Unlike the OutputLocation link, this will never be deleted.
let address_transaction = AddressTransaction::new(
sending_address_location,
spent_output_location.transaction_location(),
);
self.zs_insert(&tx_loc_by_transparent_addr_loc, address_transaction, ());
}
// Delete the OutputLocation, and the copy of the spent Output in the database.
self.zs_delete(&utxo_by_out_loc, spent_output_location);
}
self.prepare_transparent_balances_batch(db, address_balances)?;
Ok(())
}
/// Prepare a database batch indexing the transparent addresses that spent in this transaction.
///
/// Adds the following changes to this batch:
/// - index spending transactions for each spent transparent output
/// (this is different from the transaction that created the output),
/// without actually writing anything.
///
/// # Errors
///
/// - This method doesn't currently return any errors, but it might in future
pub fn prepare_spending_transparent_tx_ids_batch(
&mut self,
db: &DiskDb,
spending_tx_location: TransactionLocation,
transaction: &Transaction,
spent_utxos_by_outpoint: &HashMap<transparent::OutPoint, transparent::Utxo>,
address_balances: &HashMap<transparent::Address, AddressBalanceLocation>,
) -> Result<(), BoxError> {
let tx_loc_by_transparent_addr_loc =
db.cf_handle("tx_loc_by_transparent_addr_loc").unwrap();
// Index the transparent addresses that spent in this transaction.
//
// Coinbase inputs represent new coins, so there are no UTXOs to mark as spent.
for spent_outpoint in transaction.inputs().iter().filter_map(Input::outpoint) {
let spent_utxo = spent_utxos_by_outpoint
.get(&spent_outpoint)
.expect("unexpected missing spent output");
let sending_address = spent_utxo.output.address(self.network());
// Fetch the balance, and the link from the address to the AddressLocation, from memory.
if let Some(sending_address) = sending_address {
let sending_address_location = address_balances
.get(&sending_address)
.expect("spent outputs must already have an address balance")
.address_location();
// Create a link from the AddressLocation to the spent TransactionLocation in the database.
// Unlike the OutputLocation link, this will never be deleted.
//
// The value is the location of this transaction,
// not the transaction the spent output is from.
let address_transaction =
AddressTransaction::new(sending_address_location, spending_tx_location);
self.zs_insert(&tx_loc_by_transparent_addr_loc, address_transaction, ());
}
}
Ok(())
}

7
zebra-state/src/service/non_finalized_state/chain.rs
View File

@ -791,9 +791,9 @@ impl UpdateWith<ContextuallyValidBlock> for Chain {
"transactions must be unique within a single chain"
);
// index the utxos this produced
// add the utxos this produced
self.update_chain_tip_with(&(outputs, &transaction_hash, new_outputs))?;
// index the utxos this consumed
// delete the utxos this consumed
self.update_chain_tip_with(&(inputs, &transaction_hash, spent_outputs))?;
// add the shielded data
@ -921,7 +921,7 @@ impl UpdateWith<ContextuallyValidBlock> for Chain {
// remove the utxos this produced
self.revert_chain_with(&(outputs, transaction_hash, new_outputs), position);
// remove the utxos this consumed
// reset the utxos this consumed
self.revert_chain_with(&(inputs, transaction_hash, spent_outputs), position);
// remove `transaction.hash` from `tx_by_hash`
@ -1081,6 +1081,7 @@ impl
// The inputs from a transaction in this block
&Vec<transparent::Input>,
// The hash of the transaction that the inputs are from
// (not the transaction the spent output was created by)
&transaction::Hash,
// The outputs for all inputs spent in this transaction (or block)
&HashMap<transparent::OutPoint, transparent::OrderedUtxo>,

1
zebra-state/src/service/non_finalized_state/chain/index.rs
View File

@ -129,6 +129,7 @@ impl
// The transparent input data
&transparent::Input,
// The hash of the transaction the input is from
// (not the transaction the spent output was created by)
&transaction::Hash,
// The output spent by the input
// Includes the location of the transaction that created the output